From pedronis at codespeak.net Tue Dec 1 09:23:12 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 1 Dec 2009 09:23:12 +0100 (CET) Subject: [pypy-svn] r69787 - pypy/build/bot2/pypybuildbot Message-ID: <20091201082312.5D80D16804C@codespeak.net> Author: pedronis Date: Tue Dec 1 09:23:10 2009 New Revision: 69787 Modified: pypy/build/bot2/pypybuildbot/master.py Log: oops, missing comma Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Tue Dec 1 09:23:10 2009 @@ -72,7 +72,7 @@ ) pypy_OjitTranslatedTestFactory = pypybuilds.Translated( - translationArgs=['-Ojit', '--gc=hybrid', '--no-translation-jit' + translationArgs=['-Ojit', '--gc=hybrid', '--no-translation-jit', '--gcrootfinder=asmgcc'], lib_python=True, app_tests=True From afa at codespeak.net Tue Dec 1 11:15:53 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 1 Dec 2009 11:15:53 +0100 (CET) Subject: [pypy-svn] r69789 - pypy/trunk/pypy Message-ID: <20091201101553.7D0DB168076@codespeak.net> Author: afa Date: Tue Dec 1 11:15:52 2009 New Revision: 69789 Modified: pypy/trunk/pypy/conftest.py Log: Add str_w to the TinyObjSpace, to "fix" app-level tests of the locale module (which are skipped on my machine anyway) Modified: pypy/trunk/pypy/conftest.py ============================================================================== --- pypy/trunk/pypy/conftest.py (original) +++ pypy/trunk/pypy/conftest.py Tue Dec 1 11:15:52 2009 @@ -138,6 +138,9 @@ def is_true(self, obj): return bool(obj) + def str_w(self, w_str): + return w_str + def newdict(self): return {} From arigo at codespeak.net Tue Dec 1 11:19:33 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 1 Dec 2009 11:19:33 +0100 (CET) Subject: [pypy-svn] r69790 - in pypy/branch/virtual-forcing/pypy/jit/metainterp: . test Message-ID: <20091201101933.D0085168076@codespeak.net> Author: arigo Date: Tue Dec 1 11:19:32 2009 New Revision: 69790 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Log: Test and fix for changing former virtuals. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py Tue Dec 1 11:19:32 2009 @@ -482,12 +482,13 @@ vinfo = virtuals[i] if vinfo is not None: self.virtuals[i] = vinfo.allocate(metainterp) + for i in range(len(virtuals)): + vinfo = virtuals[i] + if vinfo is not None: + vinfo.setfields(metainterp, self.virtuals[i], + self._decode_box) else: self.virtuals = v - for i in range(len(virtuals)): - vinfo = virtuals[i] - if vinfo is not None: - vinfo.setfields(metainterp, self.virtuals[i], self._decode_box) def consume_boxes(self): numb = self.cur_numb Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Tue Dec 1 11:19:32 2009 @@ -694,6 +694,54 @@ res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) assert res == f(123) + def test_external_read_sometimes_changing_virtuals(self): + jitdriver = JitDriver(greens = [], reds = ['frame'], + virtualizables = ['frame']) + + class Frame(object): + _virtualizable2_ = ['x', 'y'] + class Y: + pass + class SomewhereElse: + pass + somewhere_else = SomewhereElse() + + def g(): + somewhere_else.counter += 1 + if somewhere_else.counter == 70: + y = somewhere_else.top_frame.y # external read + debug_print(lltype.Void, '-+-+-+-+- external virtual write') + assert y.num == 123 + y.num += 2 + else: + y = None + return y + + def f(n): + frame = Frame() + frame.x = n + somewhere_else.counter = 0 + somewhere_else.top_frame = frame + while frame.x > 0: + jitdriver.can_enter_jit(frame=frame) + jitdriver.jit_merge_point(frame=frame) + frame.y = y = Y() + y.num = 123 + result = g() + if frame.y is not y: + return -660 + if result: + if result is not y: + return -661 + if y.num != 125: + return -662 + frame.y = None + frame.x -= 1 + return frame.x + + res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) + assert res == f(123) + def test_external_read_sometimes_with_exception(self): jitdriver = JitDriver(greens = [], reds = ['frame'], virtualizables = ['frame']) From arigo at codespeak.net Tue Dec 1 11:23:17 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 1 Dec 2009 11:23:17 +0100 (CET) Subject: [pypy-svn] r69791 - in pypy/branch/virtual-forcing/pypy/jit/metainterp: . test Message-ID: <20091201102317.53B69168076@codespeak.net> Author: arigo Date: Tue Dec 1 11:23:16 2009 New Revision: 69791 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_resume.py Log: Get rid of the "except AttributeError" hack. Instead, write the _already_allocated_resume_virtuals attribute explicitly in fake MetaInterp classes. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py Tue Dec 1 11:23:16 2009 @@ -472,10 +472,7 @@ def _prepare_virtuals(self, metainterp, virtuals): if virtuals: - try: - v = metainterp._already_allocated_resume_virtuals - except AttributeError: - v = None + v = metainterp._already_allocated_resume_virtuals if v is None: self.virtuals = [None] * len(virtuals) for i in range(len(virtuals)): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_resume.py Tue Dec 1 11:23:16 2009 @@ -41,6 +41,8 @@ assert not tagged_list_eq([tag(1, TAGBOX), tag(-2, TAGBOX)], [tag(1, TAGBOX)]) class MyMetaInterp: + _already_allocated_resume_virtuals = None + def __init__(self, cpu=None): if cpu is None: cpu = LLtypeMixin.cpu @@ -124,6 +126,7 @@ rd_numb = [] rd_consts = [] class FakeMetainterp(object): + _already_allocated_resume_virtuals = None cpu = None reader = ResumeDataReader(FakeStorage(), [], FakeMetainterp()) assert reader.virtuals == ["allocated", None] From arigo at codespeak.net Tue Dec 1 11:27:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 1 Dec 2009 11:27:29 +0100 (CET) Subject: [pypy-svn] r69792 - pypy/branch/virtual-forcing/pypy/jit/metainterp Message-ID: <20091201102729.64891168076@codespeak.net> Author: arigo Date: Tue Dec 1 11:27:28 2009 New Revision: 69792 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py Log: Diff-minimizing change. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py Tue Dec 1 11:27:28 2009 @@ -473,19 +473,19 @@ def _prepare_virtuals(self, metainterp, virtuals): if virtuals: v = metainterp._already_allocated_resume_virtuals - if v is None: - self.virtuals = [None] * len(virtuals) - for i in range(len(virtuals)): - vinfo = virtuals[i] - if vinfo is not None: - self.virtuals[i] = vinfo.allocate(metainterp) - for i in range(len(virtuals)): - vinfo = virtuals[i] - if vinfo is not None: - vinfo.setfields(metainterp, self.virtuals[i], - self._decode_box) - else: + if v is not None: self.virtuals = v + return + self.virtuals = [None] * len(virtuals) + for i in range(len(virtuals)): + vinfo = virtuals[i] + if vinfo is not None: + self.virtuals[i] = vinfo.allocate(metainterp) + for i in range(len(virtuals)): + vinfo = virtuals[i] + if vinfo is not None: + vinfo.setfields(metainterp, self.virtuals[i], + self._decode_box) def consume_boxes(self): numb = self.cur_numb From arigo at codespeak.net Tue Dec 1 11:56:37 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 1 Dec 2009 11:56:37 +0100 (CET) Subject: [pypy-svn] r69793 - in pypy/trunk/pypy: . annotation jit/backend jit/backend/cli/test jit/backend/llgraph jit/backend/llgraph/test jit/backend/llsupport jit/backend/llsupport/test jit/backend/test jit/backend/x86 jit/backend/x86/test jit/metainterp jit/metainterp/doc jit/metainterp/test rlib rpython rpython/lltypesystem rpython/lltypesystem/test rpython/memory/gctransform rpython/memory/gctransform/test rpython/ootypesystem rpython/test translator/backendopt translator/backendopt/test translator/stackless Message-ID: <20091201105637.9C2AA168071@codespeak.net> Author: arigo Date: Tue Dec 1 11:56:34 2009 New Revision: 69793 Added: pypy/trunk/pypy/jit/backend/cli/test/conftest.py - copied unchanged from r69788, pypy/branch/virtual-forcing/pypy/jit/backend/cli/test/conftest.py Removed: pypy/trunk/pypy/jit/metainterp/doc/linking.txt pypy/trunk/pypy/jit/metainterp/doc/matching_rules.txt pypy/trunk/pypy/jit/metainterp/doc/virtualizables.txt Modified: pypy/trunk/pypy/annotation/description.py pypy/trunk/pypy/jit/backend/llgraph/llimpl.py pypy/trunk/pypy/jit/backend/llgraph/runner.py pypy/trunk/pypy/jit/backend/llgraph/test/test_llgraph.py pypy/trunk/pypy/jit/backend/llsupport/regalloc.py pypy/trunk/pypy/jit/backend/llsupport/test/test_regalloc.py pypy/trunk/pypy/jit/backend/model.py pypy/trunk/pypy/jit/backend/test/runner_test.py pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/backend/x86/runner.py pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py pypy/trunk/pypy/jit/metainterp/codewriter.py pypy/trunk/pypy/jit/metainterp/compile.py pypy/trunk/pypy/jit/metainterp/doc/jitpl5.txt pypy/trunk/pypy/jit/metainterp/doc/loop.txt pypy/trunk/pypy/jit/metainterp/effectinfo.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/resoperation.py pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/test/test_basic.py pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py pypy/trunk/pypy/jit/metainterp/test/test_recursive.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py pypy/trunk/pypy/jit/metainterp/test/test_virtualizable.py pypy/trunk/pypy/jit/metainterp/typesystem.py pypy/trunk/pypy/jit/metainterp/virtualizable.py pypy/trunk/pypy/jit/metainterp/warmstate.py pypy/trunk/pypy/rlib/jit.py pypy/trunk/pypy/rpython/llinterp.py pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py pypy/trunk/pypy/rpython/lltypesystem/lloperation.py pypy/trunk/pypy/rpython/lltypesystem/opimpl.py pypy/trunk/pypy/rpython/lltypesystem/rffi.py pypy/trunk/pypy/rpython/lltypesystem/rvirtualizable2.py pypy/trunk/pypy/rpython/lltypesystem/test/test_rffi.py pypy/trunk/pypy/rpython/memory/gctransform/framework.py pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py pypy/trunk/pypy/rpython/ootypesystem/rvirtualizable2.py pypy/trunk/pypy/rpython/test/test_rvirtualizable2.py pypy/trunk/pypy/testrunner_cfg.py pypy/trunk/pypy/translator/backendopt/canraise.py pypy/trunk/pypy/translator/backendopt/graphanalyze.py pypy/trunk/pypy/translator/backendopt/test/test_canraise.py pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py pypy/trunk/pypy/translator/backendopt/writeanalyze.py pypy/trunk/pypy/translator/stackless/transform.py Log: Merge the branch/virtual-forcing so far, introducing proper forcing of virtualizables followed by a guard failure when we eventually return to assembler. This also changes the front-end so that it always aborts a trace after the virtualizable was forced. The changes are not very ootype-friendly. The CLI tests are disabled for now. Fix the write analyzer to give the correct answer even when calling external functions that may call back. svn merge -r69626:69792 svn+ssh://codespeak.net/svn/pypy/branch/virtual-forcing . Modified: pypy/trunk/pypy/annotation/description.py ============================================================================== --- pypy/trunk/pypy/annotation/description.py (original) +++ pypy/trunk/pypy/annotation/description.py Tue Dec 1 11:56:34 2009 @@ -202,6 +202,9 @@ graph.name = alt_name return graph + def getgraphs(self): + return self._cache.values() + def getuniquegraph(self): if len(self._cache) != 1: raise NoStandardGraph(self) Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/llimpl.py Tue Dec 1 11:56:34 2009 @@ -35,8 +35,13 @@ _TO_OPAQUE = {} def _to_opaque(value): - return lltype.opaqueptr(_TO_OPAQUE[value.__class__], 'opaque', - externalobj=value) + try: + return value._the_opaque_pointer + except AttributeError: + op = lltype.opaqueptr(_TO_OPAQUE[value.__class__], 'opaque', + externalobj=value) + value._the_opaque_pointer = op + return op def from_opaque_string(s): if isinstance(s, str): @@ -145,6 +150,9 @@ 'unicodesetitem' : (('ref', 'int', 'int'), 'int'), 'cast_ptr_to_int' : (('ref',), 'int'), 'debug_merge_point': (('ref',), None), + 'force_token' : ((), 'int'), + 'call_may_force' : (('int', 'varargs'), 'intorptr'), + 'guard_not_forced': ((), None) #'getitem' : (('void', 'ref', 'int'), 'int'), #'setitem' : (('void', 'ref', 'int', 'int'), None), #'newlist' : (('void', 'varargs'), 'ref'), @@ -384,6 +392,9 @@ def __init__(self, memocast): self.verbose = False self.memocast = memocast + self.opindex = 1 + self._forced = False + self._may_force = -1 def getenv(self, v): if isinstance(v, Constant): @@ -391,6 +402,19 @@ else: return self.env[v] + def _populate_fail_args(self, op, skip=None): + fail_args = [] + if op.fail_args: + for fail_arg in op.fail_args: + if fail_arg is None: + fail_args.append(None) + elif fail_arg is skip: + fail_args.append(fail_arg.concretetype._defl()) + else: + fail_args.append(self.getenv(fail_arg)) + self.fail_args = fail_args + self.fail_index = op.fail_index + def execute(self): """Execute all operations in a loop, possibly following to other loops as well. @@ -401,6 +425,7 @@ operations = self.loop.operations opindex = 0 while True: + self.opindex = opindex op = operations[opindex] args = [self.getenv(v) for v in op.args] if not op.is_final(): @@ -419,18 +444,11 @@ opindex = 0 continue else: - fail_args = [] - if op.fail_args: - for fail_arg in op.fail_args: - if fail_arg is None: - fail_args.append(None) - else: - fail_args.append(self.getenv(fail_arg)) + self._populate_fail_args(op) # a non-patched guard if self.verbose: log.trace('failed: %s' % ( ', '.join(map(str, fail_args)),)) - self.fail_args = fail_args return op.fail_index #verbose = self.verbose assert (result is None) == (op.result is None) @@ -453,7 +471,8 @@ if op.opnum == rop.JUMP: assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) - operations = op.jump_target.operations + self.loop = op.jump_target + operations = self.loop.operations opindex = 0 _stats.exec_jumps += 1 elif op.opnum == rop.FINISH: @@ -484,7 +503,7 @@ try: res = ophandler(self, descr, *values) finally: - if verbose: + if 0: # if verbose: argtypes, restype = TYPES[opname] if res is None: resdata = '' @@ -493,9 +512,9 @@ else: resdata = '-> ' + repr1(res, restype, self.memocast) # fish the types - #log.cpu('\t%s %s %s' % (opname, repr_list(values, argtypes, - # self.memocast), - # resdata)) + log.cpu('\t%s %s %s' % (opname, repr_list(values, argtypes, + self.memocast), + resdata)) return res def as_int(self, x): @@ -776,6 +795,24 @@ def op_uint_xor(self, descr, arg1, arg2): return arg1 ^ arg2 + def op_force_token(self, descr): + opaque_frame = _to_opaque(self) + return llmemory.cast_ptr_to_adr(opaque_frame) + + def op_call_may_force(self, calldescr, func, *args): + assert not self._forced + self._may_force = self.opindex + try: + return self.op_call(calldescr, func, *args) + finally: + self._may_force = -1 + + def op_guard_not_forced(self, descr): + forced = self._forced + self._forced = False + if forced: + raise GuardFailed + class OOFrame(Frame): @@ -1042,6 +1079,25 @@ return lltype.cast_opaque_ptr(llmemory.GCREF, _get_error(ZeroDivisionError).args[1]) +def force(opaque_frame): + frame = _from_opaque(opaque_frame) + assert not frame._forced + frame._forced = True + assert frame._may_force >= 0 + call_op = frame.loop.operations[frame._may_force] + guard_op = frame.loop.operations[frame._may_force+1] + assert call_op.opnum == rop.CALL_MAY_FORCE + frame._populate_fail_args(guard_op, skip=call_op.result) + return frame.fail_index + +def get_forced_token_frame(force_token): + opaque_frame = llmemory.cast_adr_to_ptr(force_token, + lltype.Ptr(_TO_OPAQUE[Frame])) + return opaque_frame + +def get_frame_forced_token(opaque_frame): + return llmemory.cast_ptr_to_adr(opaque_frame) + class MemoCast(object): def __init__(self): self.addresses = [llmemory.NULL] @@ -1411,6 +1467,9 @@ setannotation(get_overflow_error_value, annmodel.SomePtr(llmemory.GCREF)) setannotation(get_zero_division_error, annmodel.SomeAddress()) setannotation(get_zero_division_error_value, annmodel.SomePtr(llmemory.GCREF)) +setannotation(force, annmodel.SomeInteger()) +setannotation(get_forced_token_frame, s_Frame) +setannotation(get_frame_forced_token, annmodel.SomeAddress()) setannotation(new_memo_cast, s_MemoCast) setannotation(cast_adr_to_int, annmodel.SomeInteger()) Modified: pypy/trunk/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/runner.py Tue Dec 1 11:56:34 2009 @@ -4,6 +4,7 @@ import sys from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.objectmodel import we_are_translated from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.ootypesystem import ootype from pypy.rpython.llinterp import LLInterpreter @@ -20,31 +21,16 @@ class Descr(history.AbstractDescr): - name = None - ofs = -1 - typeinfo = '?' - - def __init__(self, ofs, typeinfo='?', extrainfo=None): + + def __init__(self, ofs, typeinfo, extrainfo=None, name=None): self.ofs = ofs self.typeinfo = typeinfo self.extrainfo = extrainfo + self.name = name def get_extra_info(self): return self.extrainfo - def __hash__(self): - return hash((self.ofs, self.typeinfo)) - - def __eq__(self, other): - if not isinstance(other, Descr): - return NotImplemented - return self.ofs == other.ofs and self.typeinfo == other.typeinfo - - def __ne__(self, other): - if not isinstance(other, Descr): - return NotImplemented - return self.ofs != other.ofs or self.typeinfo != other.typeinfo - def sort_key(self): return self.ofs @@ -75,9 +61,12 @@ raise TypeError("cannot use comparison on Descrs") def __repr__(self): + args = [repr(self.ofs), repr(self.typeinfo)] if self.name is not None: - return '' % (self.ofs, self.typeinfo, self.name) - return '' % (self.ofs, self.typeinfo) + args.append(repr(self.name)) + if self.extrainfo is not None: + args.append('E') + return '' % (', '.join(args),) history.TreeLoop._compiled_version = lltype.nullptr(llimpl.COMPILEDLOOP.TO) @@ -99,11 +88,21 @@ llimpl._stats = self.stats llimpl._llinterp = LLInterpreter(self.rtyper) self._future_values = [] + self._descrs = {} def _freeze_(self): assert self.translate_support_code return False + def getdescr(self, ofs, typeinfo='?', extrainfo=None, name=None): + key = (ofs, typeinfo, extrainfo, name) + try: + return self._descrs[key] + except KeyError: + descr = Descr(ofs, typeinfo, extrainfo, name) + self._descrs[key] = descr + return descr + def set_class_sizes(self, class_sizes): self.class_sizes = class_sizes for vtable, size in class_sizes.items(): @@ -233,6 +232,10 @@ def get_latest_value_float(self, index): return llimpl.frame_float_getvalue(self.latest_frame, index) + def get_latest_force_token(self): + token = llimpl.get_frame_forced_token(self.latest_frame) + return self.cast_adr_to_int(token) + # ---------- def get_exception(self): @@ -252,16 +255,9 @@ return (self.cast_adr_to_int(llimpl.get_zero_division_error()), llimpl.get_zero_division_error_value()) - @staticmethod - def sizeof(S): + def sizeof(self, S): assert not isinstance(S, lltype.Ptr) - return Descr(symbolic.get_size(S)) - - @staticmethod - def numof(S): - return 4 - - ##addresssuffix = '4' + return self.getdescr(symbolic.get_size(S)) def cast_adr_to_int(self, adr): return llimpl.cast_adr_to_int(self.memo_cast, adr) @@ -282,18 +278,14 @@ BaseCPU.__init__(self, *args, **kwds) self.fielddescrof_vtable = self.fielddescrof(rclass.OBJECT, 'typeptr') - @staticmethod - def fielddescrof(S, fieldname): + def fielddescrof(self, S, fieldname): ofs, size = symbolic.get_field_token(S, fieldname) token = history.getkind(getattr(S, fieldname)) - res = Descr(ofs, token[0]) - res.name = fieldname - return res + return self.getdescr(ofs, token[0], name=fieldname) - @staticmethod - def calldescrof(FUNC, ARGS, RESULT, extrainfo=None): + def calldescrof(self, FUNC, ARGS, RESULT, extrainfo=None): token = history.getkind(RESULT) - return Descr(0, token[0], extrainfo=extrainfo) + return self.getdescr(0, token[0], extrainfo=extrainfo) def get_exception(self): return self.cast_adr_to_int(llimpl.get_exception()) @@ -301,13 +293,12 @@ def get_exc_value(self): return llimpl.get_exc_value() - @staticmethod - def arraydescrof(A): + def arraydescrof(self, A): assert isinstance(A, lltype.GcArray) assert A.OF != lltype.Void size = symbolic.get_size(A) token = history.getkind(A.OF) - return Descr(size, token[0]) + return self.getdescr(size, token[0]) # ---------- the backend-dependent operations ---------- @@ -498,6 +489,14 @@ return history.BoxInt(llimpl.cast_to_int(ptrbox.getref_base(), self.memo_cast)) + def force(self, force_token): + token = self.cast_int_to_adr(force_token) + frame = llimpl.get_forced_token_frame(token) + fail_index = llimpl.force(frame) + self.latest_frame = frame + return self.get_fail_descr_from_number(fail_index) + + class OOtypeCPU(BaseCPU): is_oo = True ts = oohelper Modified: pypy/trunk/pypy/jit/backend/llgraph/test/test_llgraph.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/test/test_llgraph.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/test/test_llgraph.py Tue Dec 1 11:56:34 2009 @@ -10,6 +10,9 @@ from pypy.jit.backend.test.runner_test import LLtypeBackendTest class TestLLTypeLLGraph(LLtypeBackendTest): + # for individual tests see: + # ====> ../../test/runner_test.py + from pypy.jit.backend.llgraph.runner import LLtypeCPU as cpu_type def setup_method(self, _): Modified: pypy/trunk/pypy/jit/backend/llsupport/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/regalloc.py Tue Dec 1 11:56:34 2009 @@ -304,7 +304,7 @@ self.assembler.regalloc_mov(reg, to) # otherwise it's clean - def before_call(self, force_store=[]): + def before_call(self, force_store=[], save_all_regs=False): """ Spill registers before a call, as described by 'self.save_around_call_regs'. Registers are not spilled if they don't survive past the current operation, unless they @@ -316,8 +316,8 @@ del self.reg_bindings[v] self.free_regs.append(reg) continue - if reg not in self.save_around_call_regs: - # we don't need to + if not save_all_regs and reg not in self.save_around_call_regs: + # we don't have to continue self._sync_var(v) del self.reg_bindings[v] @@ -327,12 +327,12 @@ """ Adjust registers according to the result of the call, which is in variable v. """ - if v is not None: - self._check_type(v) - r = self.call_result_location(v) - self.reg_bindings[v] = r - self.free_regs = [fr for fr in self.free_regs if fr is not r] - + self._check_type(v) + r = self.call_result_location(v) + self.reg_bindings[v] = r + self.free_regs = [fr for fr in self.free_regs if fr is not r] + return r + # abstract methods, override def convert_to_imm(self, c): Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/test/test_regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/test/test_regalloc.py Tue Dec 1 11:56:34 2009 @@ -278,6 +278,30 @@ assert len(rm.reg_bindings) == 3 rm._check_invariants() + def test_call_support_save_all_regs(self): + class XRegisterManager(RegisterManager): + save_around_call_regs = [r1, r2] + + def call_result_location(self, v): + return r1 + + sm = TStackManager() + asm = MockAsm() + boxes, longevity = boxes_and_longevity(5) + rm = XRegisterManager(longevity, stack_manager=sm, + assembler=asm) + for b in boxes[:-1]: + rm.force_allocate_reg(b) + rm.before_call(save_all_regs=True) + assert len(rm.reg_bindings) == 0 + assert sm.stack_depth == 4 + assert len(asm.moves) == 4 + rm._check_invariants() + rm.after_call(boxes[-1]) + assert len(rm.reg_bindings) == 1 + rm._check_invariants() + + def test_different_stack_width(self): class XRegisterManager(RegisterManager): reg_width = 2 Modified: pypy/trunk/pypy/jit/backend/model.py ============================================================================== --- pypy/trunk/pypy/jit/backend/model.py (original) +++ pypy/trunk/pypy/jit/backend/model.py Tue Dec 1 11:56:34 2009 @@ -78,6 +78,11 @@ or from 'args' if it was a FINISH). Returns a ptr or an obj.""" raise NotImplementedError + def get_latest_force_token(self): + """After a GUARD_NOT_FORCED fails, this function returns the + same FORCE_TOKEN result as the one in the just-failed loop.""" + raise NotImplementedError + def get_exception(self): raise NotImplementedError @@ -205,6 +210,16 @@ def do_cast_ptr_to_int(self, ptrbox): raise NotImplementedError + def do_force_token(self): + # this should not be implemented at all by the backends + raise NotImplementedError + + def do_call_may_force(self, args, calldescr): + return self.do_call(args, calldescr) + + def force(self, force_token): + raise NotImplementedError + # ootype specific operations # -------------------------- Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Tue Dec 1 11:56:34 2009 @@ -1221,6 +1221,139 @@ else: assert record == [] + def test_force_operations_returning_void(self): + values = [] + def maybe_force(token, flag): + if flag: + descr = self.cpu.force(token) + values.append(descr) + values.append(self.cpu.get_latest_value_int(0)) + values.append(self.cpu.get_latest_value_int(1)) + + FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Void) + func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) + funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + cpu = self.cpu + i0 = BoxInt() + i1 = BoxInt() + tok = BoxInt() + faildescr = BasicFailDescr(1) + ops = [ + ResOperation(rop.FORCE_TOKEN, [], tok), + ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], None, + descr=calldescr), + ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), + ResOperation(rop.FINISH, [i0], None, descr=BasicFailDescr(0)) + ] + ops[2].fail_args = [i1, i0] + looptoken = LoopToken() + self.cpu.compile_loop([i0, i1], ops, looptoken) + self.cpu.set_future_value_int(0, 20) + self.cpu.set_future_value_int(1, 0) + fail = self.cpu.execute_token(looptoken) + assert fail.identifier == 0 + assert self.cpu.get_latest_value_int(0) == 20 + assert values == [] + + self.cpu.set_future_value_int(0, 10) + self.cpu.set_future_value_int(1, 1) + fail = self.cpu.execute_token(looptoken) + assert fail.identifier == 1 + assert self.cpu.get_latest_value_int(0) == 1 + assert self.cpu.get_latest_value_int(1) == 10 + assert values == [faildescr, 1, 10] + + def test_force_operations_returning_int(self): + values = [] + def maybe_force(token, flag): + if flag: + self.cpu.force(token) + values.append(self.cpu.get_latest_value_int(0)) + values.append(self.cpu.get_latest_value_int(2)) + return 42 + + FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) + func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) + funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + cpu = self.cpu + i0 = BoxInt() + i1 = BoxInt() + i2 = BoxInt() + tok = BoxInt() + faildescr = BasicFailDescr(1) + ops = [ + ResOperation(rop.FORCE_TOKEN, [], tok), + ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], i2, + descr=calldescr), + ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), + ResOperation(rop.FINISH, [i2], None, descr=BasicFailDescr(0)) + ] + ops[2].fail_args = [i1, i2, i0] + looptoken = LoopToken() + self.cpu.compile_loop([i0, i1], ops, looptoken) + self.cpu.set_future_value_int(0, 20) + self.cpu.set_future_value_int(1, 0) + fail = self.cpu.execute_token(looptoken) + assert fail.identifier == 0 + assert self.cpu.get_latest_value_int(0) == 42 + assert values == [] + + self.cpu.set_future_value_int(0, 10) + self.cpu.set_future_value_int(1, 1) + fail = self.cpu.execute_token(looptoken) + assert fail.identifier == 1 + assert self.cpu.get_latest_value_int(0) == 1 + assert self.cpu.get_latest_value_int(1) == 42 + assert self.cpu.get_latest_value_int(2) == 10 + assert values == [1, 10] + + def test_force_operations_returning_float(self): + values = [] + def maybe_force(token, flag): + if flag: + self.cpu.force(token) + values.append(self.cpu.get_latest_value_int(0)) + values.append(self.cpu.get_latest_value_int(2)) + return 42.5 + + FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Float) + func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) + funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + cpu = self.cpu + i0 = BoxInt() + i1 = BoxInt() + f2 = BoxFloat() + tok = BoxInt() + faildescr = BasicFailDescr(1) + ops = [ + ResOperation(rop.FORCE_TOKEN, [], tok), + ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], f2, + descr=calldescr), + ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), + ResOperation(rop.FINISH, [f2], None, descr=BasicFailDescr(0)) + ] + ops[2].fail_args = [i1, f2, i0] + looptoken = LoopToken() + self.cpu.compile_loop([i0, i1], ops, looptoken) + self.cpu.set_future_value_int(0, 20) + self.cpu.set_future_value_int(1, 0) + fail = self.cpu.execute_token(looptoken) + assert fail.identifier == 0 + assert self.cpu.get_latest_value_float(0) == 42.5 + assert values == [] + + self.cpu.set_future_value_int(0, 10) + self.cpu.set_future_value_int(1, 1) + fail = self.cpu.execute_token(looptoken) + assert fail.identifier == 1 + assert self.cpu.get_latest_value_int(0) == 1 + assert self.cpu.get_latest_value_float(1) == 42.5 + assert self.cpu.get_latest_value_int(2) == 10 + assert values == [1, 10] + # pure do_ / descr features def test_do_operations(self): Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Tue Dec 1 11:56:34 2009 @@ -9,7 +9,8 @@ from pypy.rpython.annlowlevel import llhelper from pypy.tool.uid import fixid from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, lower_byte,\ - X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs + X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs, FRAME_FIXED_SIZE,\ + FORCE_INDEX_OFS from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.jit.backend.x86 import codebuf from pypy.jit.backend.x86.ri386 import * @@ -22,8 +23,6 @@ # our calling convention - we pass first 6 args in registers # and the rest stays on the stack -RET_BP = 5 # ret ip + bp + bx + esi + edi = 5 words - if sys.platform == 'darwin': # darwin requires the stack to be 16 bytes aligned on calls CALL_ALIGN = 4 @@ -90,6 +89,7 @@ self.fail_boxes_int = NonmovableGrowableArraySigned() self.fail_boxes_ptr = NonmovableGrowableArrayGCREF() self.fail_boxes_float = NonmovableGrowableArrayFloat() + self.fail_ebp = 0 self.setup_failure_recovery() def leave_jitted_hook(self): @@ -200,7 +200,11 @@ # patch stack adjustment LEA # possibly align, e.g. for Mac OS X mc = codebuf.InMemoryCodeBuilder(adr_lea, adr_lea + 4) - mc.write(packimm32(-(stack_depth + RET_BP - 2) * WORD)) + # Compute the correct offset for the instruction LEA ESP, [EBP-4*words]. + # Given that [EBP] is where we saved EBP, i.e. in the last word + # of our fixed frame, then the 'words' value is: + words = (FRAME_FIXED_SIZE - 1) + stack_depth + mc.write(packimm32(-WORD * words)) mc.done() def _assemble_bootstrap_code(self, inputargs, arglocs): @@ -210,8 +214,8 @@ self.mc.PUSH(ebx) self.mc.PUSH(esi) self.mc.PUSH(edi) - # NB. exactly 4 pushes above; if this changes, fix stack_pos(). - # You must also keep _get_callshape() in sync. + # NB. the shape of the frame is hard-coded in get_basic_shape() too. + # Also, make sure this is consistent with FRAME_FIXED_SIZE. adr_stackadjust = self._patchable_stackadjust() tmp = X86RegisterManager.all_regs[0] xmmtmp = X86XMMRegisterManager.all_regs[0] @@ -266,9 +270,6 @@ regalloc_mov = mov # legacy interface - def regalloc_fstp(self, loc): - self.mc.FSTP(loc) - def regalloc_push(self, loc): if isinstance(loc, XMMREG): self.mc.SUB(esp, imm(2*WORD)) @@ -758,7 +759,8 @@ def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): exc = (guard_opnum == rop.GUARD_EXCEPTION or - guard_opnum == rop.GUARD_NO_EXCEPTION) + guard_opnum == rop.GUARD_NO_EXCEPTION or + guard_opnum == rop.GUARD_NOT_FORCED) return self.generate_quick_failure(faildescr, failargs, fail_locs, exc) def generate_quick_failure(self, faildescr, failargs, fail_locs, exc): @@ -876,75 +878,79 @@ arglocs.append(loc) return arglocs[:] + def grab_frame_values(self, bytecode, frame_addr, allregisters): + # no malloc allowed here!! + self.fail_ebp = allregisters[16 + ebp.op] + num = 0 + value_hi = 0 + while 1: + # decode the next instruction from the bytecode + code = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + if code >= 4*self.DESCR_FROMSTACK: + if code > 0x7F: + shift = 7 + code &= 0x7F + while True: + nextcode = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + code |= (nextcode & 0x7F) << shift + shift += 7 + if nextcode <= 0x7F: + break + # load the value from the stack + kind = code & 3 + code = (code >> 2) - self.DESCR_FROMSTACK + stackloc = frame_addr + get_ebp_ofs(code) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == self.DESCR_FLOAT: + value_hi = value + value = rffi.cast(rffi.LONGP, stackloc - 4)[0] + else: + # 'code' identifies a register: load its value + kind = code & 3 + if kind == self.DESCR_SPECIAL: + if code == self.DESCR_HOLE: + num += 1 + continue + assert code == self.DESCR_STOP + break + code >>= 2 + if kind == self.DESCR_FLOAT: + value = allregisters[2*code] + value_hi = allregisters[2*code + 1] + else: + value = allregisters[16 + code] + + # store the loaded value into fail_boxes_ + if kind == self.DESCR_INT: + tgt = self.fail_boxes_int.get_addr_for_num(num) + elif kind == self.DESCR_REF: + tgt = self.fail_boxes_ptr.get_addr_for_num(num) + elif kind == self.DESCR_FLOAT: + tgt = self.fail_boxes_float.get_addr_for_num(num) + rffi.cast(rffi.LONGP, tgt)[1] = value_hi + else: + assert 0, "bogus kind" + rffi.cast(rffi.LONGP, tgt)[0] = value + num += 1 + # + if not we_are_translated(): + assert bytecode[4] == 0xCC + fail_index = rffi.cast(rffi.LONGP, bytecode)[0] + return fail_index + def setup_failure_recovery(self): def failure_recovery_func(registers): - # no malloc allowed here!! # 'registers' is a pointer to a structure containing the # original value of the registers, optionally the original # value of XMM registers, and finally a reference to the # recovery bytecode. See _build_failure_recovery() for details. stack_at_ebp = registers[ebp.op] bytecode = rffi.cast(rffi.UCHARP, registers[8]) - num = 0 - value_hi = 0 - while 1: - # decode the next instruction from the bytecode - code = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - if code >= 4*self.DESCR_FROMSTACK: - if code > 0x7F: - shift = 7 - code &= 0x7F - while True: - nextcode = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - code |= (nextcode & 0x7F) << shift - shift += 7 - if nextcode <= 0x7F: - break - # load the value from the stack - kind = code & 3 - code = (code >> 2) - self.DESCR_FROMSTACK - stackloc = stack_at_ebp + get_ebp_ofs(code) - value = rffi.cast(rffi.LONGP, stackloc)[0] - if kind == self.DESCR_FLOAT: - value_hi = value - value = rffi.cast(rffi.LONGP, stackloc - 4)[0] - else: - # 'code' identifies a register: load its value - kind = code & 3 - if kind == self.DESCR_SPECIAL: - if code == self.DESCR_HOLE: - num += 1 - continue - assert code == self.DESCR_STOP - break - code >>= 2 - if kind == self.DESCR_FLOAT: - xmmregisters = rffi.ptradd(registers, -16) - value = xmmregisters[2*code] - value_hi = xmmregisters[2*code + 1] - else: - value = registers[code] - - # store the loaded value into fail_boxes_ - if kind == self.DESCR_INT: - tgt = self.fail_boxes_int.get_addr_for_num(num) - elif kind == self.DESCR_REF: - tgt = self.fail_boxes_ptr.get_addr_for_num(num) - elif kind == self.DESCR_FLOAT: - tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGP, tgt)[1] = value_hi - else: - assert 0, "bogus kind" - rffi.cast(rffi.LONGP, tgt)[0] = value - num += 1 - # - if not we_are_translated(): - assert bytecode[4] == 0xCC - fail_index = rffi.cast(rffi.LONGP, bytecode)[0] - return fail_index + allregisters = rffi.ptradd(registers, -16) + return self.grab_frame_values(bytecode, stack_at_ebp, allregisters) self.failure_recovery_func = failure_recovery_func self.failure_recovery_code = [0, 0, 0, 0] @@ -997,11 +1003,11 @@ # now we return from the complete frame, which starts from # _assemble_bootstrap_code(). The LEA below throws away most # of the frame, including all the PUSHes that we did just above. - mc.LEA(esp, addr_add(ebp, imm((-RET_BP + 2) * WORD))) - mc.POP(edi) - mc.POP(esi) - mc.POP(ebx) - mc.POP(ebp) + mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) + mc.POP(edi) # [ebp-12] + mc.POP(esi) # [ebp-8] + mc.POP(ebx) # [ebp-4] + mc.POP(ebp) # [ebp] mc.RET() self.mc2.done() self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr @@ -1042,14 +1048,14 @@ addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) mc.CALL(rel32(addr)) - # don't break the following code sequence! + # don't break the following code sequence! xxx no reason any more? mc = mc._mc - mc.LEA(esp, addr_add(ebp, imm((-RET_BP + 2) * WORD))) + mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) mc.MOV(eax, imm(fail_index)) - mc.POP(edi) - mc.POP(esi) - mc.POP(ebx) - mc.POP(ebp) + mc.POP(edi) # [ebp-12] + mc.POP(esi) # [ebp-8] + mc.POP(ebx) # [ebp-4] + mc.POP(ebp) # [ebp] mc.RET() @specialize.arg(2) @@ -1098,12 +1104,23 @@ self.mc.CALL(x) self.mark_gc_roots() self.mc.ADD(esp, imm(extra_on_stack)) - if size == 1: + if isinstance(resloc, MODRM64): + self.mc.FSTP(resloc) + elif size == 1: self.mc.AND(eax, imm(0xff)) elif size == 2: self.mc.AND(eax, imm(0xffff)) genop_call_pure = genop_call + + def genop_guard_call_may_force(self, op, guard_op, addr, + arglocs, result_loc): + faildescr = guard_op.descr + fail_index = self.cpu.get_fail_descr_number(faildescr) + self.mc.MOV(mem(ebp, FORCE_INDEX_OFS), imm(fail_index)) + self.genop_call(op, arglocs, result_loc) + self.mc.CMP(mem(ebp, FORCE_INDEX_OFS), imm(0)) + return self.implement_guard(addr, self.mc.JL) def genop_discard_cond_call_gc_wb(self, op, arglocs): # use 'mc._mc' directly instead of 'mc', to avoid @@ -1134,6 +1151,9 @@ assert 0 < offset <= 127 mc.overwrite(jz_location-1, [chr(offset)]) + def genop_force_token(self, op, arglocs, resloc): + self.mc.LEA(resloc, mem(ebp, FORCE_INDEX_OFS)) + def not_implemented_op_discard(self, op, arglocs): msg = "not implemented operation: %s" % op.getopname() print msg Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Tue Dec 1 11:56:34 2009 @@ -19,6 +19,8 @@ TempBox WORD = 4 +FRAME_FIXED_SIZE = 5 # ebp + ebx + esi + edi + force_index = 5 words +FORCE_INDEX_OFS = -4*WORD width_of_type = { INT : 1, @@ -98,10 +100,9 @@ def after_call(self, v): # the result is stored in st0, but we don't have this around, - # so we move it to some stack location - if v is not None: - loc = self.stack_manager.loc(v, 2) - self.assembler.regalloc_fstp(loc) + # so genop_call will move it to some stack location immediately + # after the call + return self.stack_manager.loc(v, 2) class X86StackManager(StackManager): @@ -287,7 +288,8 @@ self.assembler.regalloc_perform_with_guard(op, guard_op, faillocs, arglocs, result_loc, self.sm.stack_depth) - self.rm.possibly_free_var(op.result) + if op.result is not None: + self.possibly_free_var(op.result) self.possibly_free_vars(guard_op.fail_args) def perform_guard(self, guard_op, arglocs, result_loc): @@ -308,7 +310,10 @@ self.assembler.dump('%s(%s)' % (op, arglocs)) self.assembler.regalloc_perform_discard(op, arglocs) - def can_optimize_cmp_op(self, op, i, operations): + def can_merge_with_next_guard(self, op, i, operations): + if op.opnum == rop.CALL_MAY_FORCE: + assert operations[i + 1].opnum == rop.GUARD_NOT_FORCED + return True if not op.is_comparison(): return False if (operations[i + 1].opnum != rop.GUARD_TRUE and @@ -332,7 +337,7 @@ i += 1 self.possibly_free_vars(op.args) continue - if self.can_optimize_cmp_op(op, i, operations): + if self.can_merge_with_next_guard(op, i, operations): oplist[op.opnum](self, op, operations[i + 1]) i += 1 else: @@ -604,25 +609,38 @@ self.Perform(op, [loc0], loc1) self.rm.possibly_free_var(op.args[0]) - def _call(self, op, arglocs, force_store=[]): - self.rm.before_call(force_store) - self.xrm.before_call(force_store) - self.Perform(op, arglocs, eax) + def _call(self, op, arglocs, force_store=[], guard_not_forced_op=None): + save_all_regs = guard_not_forced_op is not None + self.rm.before_call(force_store, save_all_regs=save_all_regs) + self.xrm.before_call(force_store, save_all_regs=save_all_regs) if op.result is not None: if op.result.type == FLOAT: - self.xrm.after_call(op.result) + resloc = self.xrm.after_call(op.result) else: - self.rm.after_call(op.result) + resloc = self.rm.after_call(op.result) + else: + resloc = None + if guard_not_forced_op is not None: + self.perform_with_guard(op, guard_not_forced_op, arglocs, resloc) + else: + self.Perform(op, arglocs, resloc) - def consider_call(self, op, ignored): + def _consider_call(self, op, guard_not_forced_op=None): calldescr = op.descr assert isinstance(calldescr, BaseCallDescr) assert len(calldescr.arg_classes) == len(op.args) - 1 size = calldescr.get_result_size(self.translate_support_code) - self._call(op, [imm(size)] + [self.loc(arg) for arg in op.args]) + self._call(op, [imm(size)] + [self.loc(arg) for arg in op.args], + guard_not_forced_op=guard_not_forced_op) + def consider_call(self, op, ignored): + self._consider_call(op) consider_call_pure = consider_call + def consider_call_may_force(self, op, guard_op): + assert guard_op is not None + self._consider_call(op, guard_op) + def consider_cond_call_gc_wb(self, op, ignored): assert op.result is None arglocs = [self.loc(arg) for arg in op.args] @@ -927,6 +945,10 @@ assert reg is eax # ok to ignore this one return gcrootmap.compress_callshape(shape) + def consider_force_token(self, op, ignored): + loc = self.rm.force_allocate_reg(op.result) + self.Perform(op, [], loc) + def not_implemented_op(self, op, ignored): msg = "[regalloc] Not implemented operation: %s" % op.getopname() print msg @@ -942,10 +964,9 @@ def get_ebp_ofs(position): # Argument is a stack position (0, 1, 2...). - # Returns (ebp-16), (ebp-20), (ebp-24)... - # This depends on the fact that our function prologue contains - # exactly 4 PUSHes. - return -WORD * (4 + position) + # Returns (ebp-20), (ebp-24), (ebp-28)... + # i.e. the n'th word beyond the fixed frame size. + return -WORD * (FRAME_FIXED_SIZE + position) def lower_byte(reg): # argh, kill, use lowest8bits instead Modified: pypy/trunk/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/runner.py Tue Dec 1 11:56:34 2009 @@ -6,9 +6,9 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.jit.metainterp import history from pypy.jit.backend.x86.assembler import Assembler386 +from pypy.jit.backend.x86.regalloc import FORCE_INDEX_OFS from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU - class CPU386(AbstractLLCPU): debug = True supports_floats = True @@ -59,6 +59,9 @@ llmemory.GCREF.TO)) return ptrvalue + def get_latest_force_token(self): + return self.assembler.fail_ebp + FORCE_INDEX_OFS + def execute_token(self, executable_token): addr = executable_token._x86_bootstrap_code func = rffi.cast(lltype.Ptr(self.BOOTSTRAP_TP), addr) @@ -87,6 +90,27 @@ adr = llmemory.cast_ptr_to_adr(x) return CPU386.cast_adr_to_int(adr) + all_null_registers = lltype.malloc(rffi.LONGP.TO, 24, + flavor='raw', zero=True) + + def force(self, addr_of_force_index): + TP = rffi.CArrayPtr(lltype.Signed) + fail_index = rffi.cast(TP, addr_of_force_index)[0] + assert fail_index >= 0, "already forced!" + faildescr = self.get_fail_descr_from_number(fail_index) + rffi.cast(TP, addr_of_force_index)[0] = -1 + bytecode = rffi.cast(rffi.UCHARP, + faildescr._x86_failure_recovery_bytecode) + # start of "no gc operation!" block + fail_index_2 = self.assembler.grab_frame_values( + bytecode, + addr_of_force_index - FORCE_INDEX_OFS, + self.all_null_registers) + self.assembler.leave_jitted_hook() + # end of "no gc operation!" block + assert fail_index == fail_index_2 + return faildescr + class CPU386_NO_SSE2(CPU386): supports_floats = False Modified: pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py Tue Dec 1 11:56:34 2009 @@ -9,7 +9,7 @@ from pypy.jit.backend.llsupport.descr import GcCache from pypy.jit.backend.llsupport.gc import GcLLDescription from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD +from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, FRAME_FIXED_SIZE from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper @@ -83,7 +83,8 @@ # mark = regalloc.get_mark_gc_roots(cpu.gc_ll_descr.gcrootmap) assert mark[0] == 'compressed' - expected = ['ebx', 'esi', 'edi', -16, -20, -24] + base = -WORD * FRAME_FIXED_SIZE + expected = ['ebx', 'esi', 'edi', base, base-4, base-8] assert dict.fromkeys(mark[1:]) == dict.fromkeys(expected) class TestRegallocGcIntegration(BaseTestRegalloc): Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/codewriter.py Tue Dec 1 11:56:34 2009 @@ -14,6 +14,7 @@ from pypy.translator.backendopt.writeanalyze import WriteAnalyzer from pypy.jit.metainterp.typesystem import deref, arrayItem, fieldType from pypy.jit.metainterp.effectinfo import effectinfo_from_writeanalyze +from pypy.jit.metainterp.effectinfo import VirtualizableAnalyzer import py, sys from pypy.tool.ansi_print import ansi_log @@ -182,8 +183,10 @@ self.metainterp_sd = metainterp_sd self.cpu = metainterp_sd.cpu self.portal_runner_ptr = portal_runner_ptr - self.raise_analyzer = RaiseAnalyzer(self.rtyper.annotator.translator) - self.write_analyzer = WriteAnalyzer(self.rtyper.annotator.translator) + translator = self.rtyper.annotator.translator + self.raise_analyzer = RaiseAnalyzer(translator) + self.write_analyzer = WriteAnalyzer(translator) + self.virtualizable_analyzer = VirtualizableAnalyzer(translator) def make_portal_bytecode(self, graph): log.info("making JitCodes...") @@ -323,7 +326,9 @@ # ok if consider_effects_of is not None: effectinfo = effectinfo_from_writeanalyze( - self.write_analyzer.analyze(consider_effects_of), self.cpu) + self.write_analyzer.analyze(consider_effects_of), + self.cpu, + self.virtualizable_analyzer.analyze(consider_effects_of)) calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT, effectinfo) else: calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT) @@ -1203,12 +1208,19 @@ if op.opname == "direct_call": func = getattr(get_funcobj(op.args[0].value), '_callable', None) pure = getattr(func, "_pure_function_", False) + all_promoted_args = getattr(func, + "_pure_function_with_all_promoted_args_", False) + if pure and not all_promoted_args: + effectinfo = calldescr.get_extra_info() + assert (effectinfo is not None and + not effectinfo.promotes_virtualizables) try: canraise = self.codewriter.raise_analyzer.can_raise(op) except lltype.DelayedPointer: canraise = True # if we need to look into the delayed ptr that is # the portal, then it's certainly going to raise if pure: + # XXX check what to do about exceptions (also MemoryError?) self.emit('residual_call_pure') elif canraise: self.emit('residual_call') @@ -1236,9 +1248,8 @@ def handle_regular_indirect_call(self, op): self.codewriter.register_indirect_call_targets(op) args = op.args[1:-1] - calldescr, non_void_args = self.codewriter.getcalldescr(op.args[0], - args, - op.result) + calldescr, non_void_args = self.codewriter.getcalldescr( + op.args[0], args, op.result, consider_effects_of=op) self.minimize_variables() self.emit('indirect_call') self.emit(self.get_position(calldescr)) Modified: pypy/trunk/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/compile.py (original) +++ pypy/trunk/pypy/jit/metainterp/compile.py Tue Dec 1 11:56:34 2009 @@ -221,6 +221,7 @@ if box: fail_arg_types[i] = box.type self.fail_arg_types = fail_arg_types + # XXX ^^^ kill this attribute def handle_fail(self, metainterp_sd): from pypy.jit.metainterp.pyjitpl import MetaInterp @@ -236,6 +237,41 @@ send_bridge_to_backend(metainterp.staticdata, self, inputargs, new_loop.operations) + +class ResumeGuardForcedDescr(ResumeGuardDescr): + + def handle_fail(self, metainterp_sd): + from pypy.jit.metainterp.pyjitpl import MetaInterp + metainterp = MetaInterp(metainterp_sd) + token = metainterp_sd.cpu.get_latest_force_token() + metainterp._already_allocated_resume_virtuals = self.fetch_data(token) + self.counter = -2 # never compile + return metainterp.handle_guard_failure(self) + + def force_virtualizable(self, vinfo, virtualizable, force_token): + from pypy.jit.metainterp.pyjitpl import MetaInterp + from pypy.jit.metainterp.resume import force_from_resumedata + metainterp = MetaInterp(self.metainterp_sd) + metainterp.history = None # blackholing + liveboxes = metainterp.load_values_from_failure(self) + virtualizable_boxes, data = force_from_resumedata(metainterp, + liveboxes, self) + vinfo.write_boxes(virtualizable, virtualizable_boxes) + self.save_data(force_token, data) + + def save_data(self, key, value): + globaldata = self.metainterp_sd.globaldata + assert key not in globaldata.resume_virtuals + globaldata.resume_virtuals[key] = value + + def fetch_data(self, key): + globaldata = self.metainterp_sd.globaldata + assert key in globaldata.resume_virtuals + data = globaldata.resume_virtuals[key] + del globaldata.resume_virtuals[key] + return data + + class ResumeFromInterpDescr(ResumeDescr): def __init__(self, original_greenkey, redkey): ResumeDescr.__init__(self, original_greenkey) Modified: pypy/trunk/pypy/jit/metainterp/doc/jitpl5.txt ============================================================================== --- pypy/trunk/pypy/jit/metainterp/doc/jitpl5.txt (original) +++ pypy/trunk/pypy/jit/metainterp/doc/jitpl5.txt Tue Dec 1 11:56:34 2009 @@ -78,16 +78,11 @@ matches the real data -- but this is delicate because of the non-escaping flag. -Instead, this is done by doing tracing from the start of the loop again. -At the end, we don't do perfect specialization (for now), but simply -check that the already-computed specialization still applies, and then -jump to the already-compiled loop. (If it does not match, for now we -just cancel everything.) - -If the loop is not only executed but *entered* often enough, then after -this tracing, we generate a second copy of the loop (a "bridge") that -starts with all variables unspecialized, and ends with a jump to the -real loop. From this point on, we can just jump directly to the bridge +Instead, this is done by "entry bridges": we do tracing from +the start of the loop again, and at the end, we try to compile +the recorded trace as a "bridge" that comes from the +interpreter (i.e. with no virtuals at all) and goes to the old +loop. Later on, we can just jump directly to the entry bridge from the JUMP_ABSOLUTE bytecode. @@ -105,10 +100,7 @@ take the set of live values and put them back into boxes, and proceed with tracing for the rest of the loop. -For now, we just check at the end of the loop that it matches the -already-computed specialization. If not, we cancel creating the -compiled version of it (and mark the guard so that future failures -always fall back to interpretation). To do this, when we created the -original loop, at every guard, we needed to record the set of live -values (mostly in which register or stack location they are) as well as -an "escaped-so-far" flag for each pointer. +At the end of the loop, we check that it matches an already-computed +specialization. If not, we go on tracing. This might unroll the loop +once. (Note that there is a global limit on the length of the recorded +trace, to avoid tracing forever.) Modified: pypy/trunk/pypy/jit/metainterp/doc/loop.txt ============================================================================== --- pypy/trunk/pypy/jit/metainterp/doc/loop.txt (original) +++ pypy/trunk/pypy/jit/metainterp/doc/loop.txt Tue Dec 1 11:56:34 2009 @@ -36,32 +36,11 @@ . VirtualSpec(cls, name1=spec1, ...) | - VirtualizableSpec(cls, name1=spec1, ...) - | - FixedClassSpec(cls) - | NotSpec -For (a simplified) example, ``VirtualizableSpec(PyFrame, x = -VirtualSpec(W_IntObject, value = NotSpec))`` describes the virtualizable -frame for a loop in which the only used variable is ``x``, which is a -virtual ``W_IntObject``. - -The intersection rules are: - -* the intersection of two ``VirtualSpec`` of the same ``cls`` is a - further ``VirtualSpec``, and we proceed with the intersection of - each field. - -* the intersection of two ``VirtualizableSpec`` of the same ``cls`` is - like the previous case, except that some names may be omitted - completely from a given ``VirtualizableSpec``; in the case a name is - present in only one of the ``VirtualizableSpec``, we just keep it - unmodified in the intersection. - -* in other cases, the result is ``FixedClassSpec`` if the two specnodes - have the same class, or ``NotSpec`` if any is a ``NotSpec`` or if the - two classes differ. +For example, ``VirtualSpec(W_IntObject, value = NotSpec))`` describes a +variable which is a virtual ``W_IntObject``, containing a value that is +a real integer. Overall Approach Modified: pypy/trunk/pypy/jit/metainterp/effectinfo.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/effectinfo.py (original) +++ pypy/trunk/pypy/jit/metainterp/effectinfo.py Tue Dec 1 11:56:34 2009 @@ -2,21 +2,25 @@ from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype +from pypy.translator.backendopt.graphanalyze import BoolGraphAnalyzer class EffectInfo(object): _cache = {} - def __new__(cls, write_descrs_fields, write_descrs_arrays): - key = frozenset(write_descrs_fields), frozenset(write_descrs_arrays) + def __new__(cls, write_descrs_fields, write_descrs_arrays, + promotes_virtualizables=False): + key = (frozenset(write_descrs_fields), frozenset(write_descrs_arrays), + promotes_virtualizables) if key in cls._cache: return cls._cache[key] result = object.__new__(cls) result.write_descrs_fields = write_descrs_fields result.write_descrs_arrays = write_descrs_arrays + result.promotes_virtualizables = promotes_virtualizables cls._cache[key] = result return result -def effectinfo_from_writeanalyze(effects, cpu): +def effectinfo_from_writeanalyze(effects, cpu, promotes_virtualizables=False): from pypy.translator.backendopt.writeanalyze import top_set if effects is top_set: return None @@ -39,7 +43,8 @@ write_descrs_arrays.append(descr) else: assert 0 - return EffectInfo(write_descrs_fields, write_descrs_arrays) + return EffectInfo(write_descrs_fields, write_descrs_arrays, + promotes_virtualizables) def consider_struct(TYPE, fieldname): if fieldType(TYPE, fieldname) is lltype.Void: @@ -55,7 +60,6 @@ return False return True - def consider_array(ARRAY): if arrayItem(ARRAY) is lltype.Void: return False @@ -64,3 +68,9 @@ if not isinstance(ARRAY, lltype.GcArray): # can be a non-GC-array return False return True + +# ____________________________________________________________ + +class VirtualizableAnalyzer(BoolGraphAnalyzer): + def analyze_simple_operation(self, op): + return op.opname == 'promote_virtualizable' Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Tue Dec 1 11:56:34 2009 @@ -632,6 +632,7 @@ varargs = [jitcode.cfnptr] + varargs res = self.execute_varargs(rop.CALL, varargs, descr=jitcode.calldescr, exc=True) + self.metainterp.load_fields_from_virtualizable() else: # for oosends (ootype only): calldescr is a MethDescr res = self.execute_varargs(rop.OOSEND, varargs, @@ -651,7 +652,7 @@ @arguments("descr", "varargs") def opimpl_residual_call(self, calldescr, varargs): - return self.execute_varargs(rop.CALL, varargs, descr=calldescr, exc=True) + return self.do_residual_call(varargs, descr=calldescr, exc=True) @arguments("varargs") def opimpl_recursion_leave_prep(self, varargs): @@ -675,11 +676,11 @@ greenkey = varargs[1:num_green_args + 1] if warmrunnerstate.can_inline_callable(greenkey): return self.perform_call(portal_code, varargs[1:], greenkey) - return self.execute_varargs(rop.CALL, varargs, descr=calldescr, exc=True) + return self.do_residual_call(varargs, descr=calldescr, exc=True) @arguments("descr", "varargs") def opimpl_residual_call_noexception(self, calldescr, varargs): - self.execute_varargs(rop.CALL, varargs, descr=calldescr, exc=False) + self.do_residual_call(varargs, descr=calldescr, exc=False) @arguments("descr", "varargs") def opimpl_residual_call_pure(self, calldescr, varargs): @@ -696,8 +697,8 @@ return self.perform_call(jitcode, varargs) else: # but we should not follow calls to that graph - return self.execute_varargs(rop.CALL, [box] + varargs, - descr=calldescr, exc=True) + return self.do_residual_call([box] + varargs, + descr=calldescr, exc=True) @arguments("orgpc", "methdescr", "varargs") def opimpl_oosend(self, pc, methdescr, varargs): @@ -924,7 +925,6 @@ if isinstance(box, Const): # no need for a guard return metainterp = self.metainterp - metainterp_sd = metainterp.staticdata if metainterp.is_blackholing(): return saved_pc = self.pc @@ -933,8 +933,14 @@ moreargs = [box] + extraargs else: moreargs = list(extraargs) + metainterp_sd = metainterp.staticdata original_greenkey = metainterp.resumekey.original_greenkey - resumedescr = compile.ResumeGuardDescr(metainterp_sd, original_greenkey) + if opnum == rop.GUARD_NOT_FORCED: + resumedescr = compile.ResumeGuardForcedDescr(metainterp_sd, + original_greenkey) + else: + resumedescr = compile.ResumeGuardDescr(metainterp_sd, + original_greenkey) guard_op = metainterp.history.record(opnum, moreargs, None, descr=resumedescr) virtualizable_boxes = None @@ -980,6 +986,24 @@ return self.metainterp.handle_exception() return False + def do_residual_call(self, argboxes, descr, exc): + effectinfo = descr.get_extra_info() + if effectinfo is None or effectinfo.promotes_virtualizables: + # residual calls require attention to keep virtualizables in-sync + self.metainterp.vable_before_residual_call() + # xxx do something about code duplication + resbox = self.metainterp.execute_and_record_varargs( + rop.CALL_MAY_FORCE, argboxes, descr=descr) + self.metainterp.vable_after_residual_call() + if resbox is not None: + self.make_result_box(resbox) + self.generate_guard(self.pc, rop.GUARD_NOT_FORCED, None, []) + if exc: + return self.metainterp.handle_exception() + return False + else: + return self.execute_varargs(rop.CALL, argboxes, descr, exc) + # ____________________________________________________________ class MetaInterpStaticData(object): @@ -1142,6 +1166,7 @@ self.indirectcall_dict = None self.addr2name = None self.loopnumbering = 0 + self.resume_virtuals = {} # state = staticdata.state if state is not None: @@ -1168,6 +1193,8 @@ class MetaInterp(object): in_recursion = 0 + _already_allocated_resume_virtuals = None + def __init__(self, staticdata): self.staticdata = staticdata self.cpu = staticdata.cpu @@ -1298,7 +1325,8 @@ @specialize.arg(1) def execute_and_record(self, opnum, descr, *argboxes): history.check_descr(descr) - assert opnum != rop.CALL and opnum != rop.OOSEND + assert (opnum != rop.CALL and opnum != rop.CALL_MAY_FORCE + and opnum != rop.OOSEND) # execute the operation profiler = self.staticdata.profiler profiler.count_ops(opnum) @@ -1315,12 +1343,6 @@ @specialize.arg(1) def execute_and_record_varargs(self, opnum, argboxes, descr=None): history.check_descr(descr) - # residual calls require attention to keep virtualizables in-sync. - # CALL_PURE doesn't need it because so far 'promote_virtualizable' - # as an operation is enough to make the called function non-pure. - require_attention = (opnum == rop.CALL or opnum == rop.OOSEND) - if require_attention and not self.is_blackholing(): - self.before_residual_call() # execute the operation profiler = self.staticdata.profiler profiler.count_ops(opnum) @@ -1328,17 +1350,12 @@ if self.is_blackholing(): profiler.count_ops(opnum, BLACKHOLED_OPS) else: - if require_attention: - require_attention = self.after_residual_call() # check if the operation can be constant-folded away argboxes = list(argboxes) if rop._ALWAYS_PURE_FIRST <= opnum <= rop._ALWAYS_PURE_LAST: resbox = self._record_helper_pure_varargs(opnum, resbox, descr, argboxes) else: resbox = self._record_helper_nonpure_varargs(opnum, resbox, descr, argboxes) - # if we are blackholing require_attention has the initial meaning - if require_attention: - self.after_generate_residual_call() return resbox def _record_helper_pure(self, opnum, resbox, descr, *argboxes): @@ -1572,7 +1589,8 @@ self.framestack[-1].follow_jump() elif opnum == rop.GUARD_FALSE: # a goto_if_not that stops jumping self.framestack[-1].dont_follow_jump() - elif opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION: + elif (opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION + or opnum == rop.GUARD_NOT_FORCED): self.handle_exception() elif opnum == rop.GUARD_NO_OVERFLOW: # an overflow now detected self.raise_overflow_error() @@ -1722,36 +1740,39 @@ vinfo = self.staticdata.virtualizable_info virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) - vinfo.clear_vable_rti(virtualizable) + vinfo.clear_vable_token(virtualizable) - def before_residual_call(self): + def vable_before_residual_call(self): + if self.is_blackholing(): + return vinfo = self.staticdata.virtualizable_info if vinfo is not None: virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) vinfo.tracing_before_residual_call(virtualizable) + # + force_token_box = history.BoxInt() + self.history.record(rop.FORCE_TOKEN, [], force_token_box) + self.history.record(rop.SETFIELD_GC, [virtualizable_box, + force_token_box], + None, descr=vinfo.vable_token_descr) - def after_residual_call(self): - vinfo = self.staticdata.virtualizable_info - if vinfo is not None: - virtualizable_box = self.virtualizable_boxes[-1] - virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) - if vinfo.tracing_after_residual_call(virtualizable): - # This is after the residual call is done, but before it - # is actually generated. We first generate a store- - # everything-back, *without actually performing it now* - # as it contains the old values (before the call)! - self.gen_store_back_in_virtualizable_no_perform() - return True # must call after_generate_residual_call() - # otherwise, don't call after_generate_residual_call() - return False - - def after_generate_residual_call(self): - # Called after generating a residual call, and only if - # after_residual_call() returned True, i.e. if code in the residual - # call causes the virtualizable to escape. Reload the modified - # fields of the virtualizable. - self.gen_load_fields_from_virtualizable() + def vable_after_residual_call(self): + if self.is_blackholing(): + vable_escapes = True + else: + vable_escapes = False + vinfo = self.staticdata.virtualizable_info + if vinfo is not None: + virtualizable_box = self.virtualizable_boxes[-1] + virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) + if vinfo.tracing_after_residual_call(virtualizable): + # We just did the residual call, and it shows that the + # virtualizable escapes. + self.switch_to_blackhole() + vable_escapes = True + if vable_escapes: + self.load_fields_from_virtualizable() def handle_exception(self): etype = self.cpu.get_exception() @@ -1793,7 +1814,7 @@ # is and stays NULL. virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) - assert not virtualizable.vable_rti + assert not virtualizable.vable_token self.synchronize_virtualizable() def check_synchronized_virtualizable(self): @@ -1809,27 +1830,17 @@ virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) vinfo.write_boxes(virtualizable, self.virtualizable_boxes) - def gen_load_fields_from_virtualizable(self): + def load_fields_from_virtualizable(self): + # Force a reload of the virtualizable fields into the local + # boxes (called only in escaping cases) + assert self.is_blackholing() vinfo = self.staticdata.virtualizable_info if vinfo is not None: - vbox = self.virtualizable_boxes[-1] - for i in range(vinfo.num_static_extra_boxes): - descr = vinfo.static_field_descrs[i] - fieldbox = self.execute_and_record(rop.GETFIELD_GC, descr, - vbox) - self.virtualizable_boxes[i] = fieldbox - i = vinfo.num_static_extra_boxes - virtualizable = vinfo.unwrap_virtualizable_box(vbox) - for k in range(vinfo.num_arrays): - descr = vinfo.array_field_descrs[k] - abox = self.execute_and_record(rop.GETFIELD_GC, descr, vbox) - descr = vinfo.array_descrs[k] - for j in range(vinfo.get_array_length(virtualizable, k)): - itembox = self.execute_and_record(rop.GETARRAYITEM_GC, - descr, abox, ConstInt(j)) - self.virtualizable_boxes[i] = itembox - i += 1 - assert i + 1 == len(self.virtualizable_boxes) + virtualizable_box = self.virtualizable_boxes[-1] + virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) + self.virtualizable_boxes = vinfo.read_boxes(self.cpu, + virtualizable) + self.virtualizable_boxes.append(virtualizable_box) def gen_store_back_in_virtualizable(self): vinfo = self.staticdata.virtualizable_info @@ -1853,29 +1864,6 @@ abox, ConstInt(j), itembox) assert i + 1 == len(self.virtualizable_boxes) - def gen_store_back_in_virtualizable_no_perform(self): - vinfo = self.staticdata.virtualizable_info - # xxx only write back the fields really modified - vbox = self.virtualizable_boxes[-1] - for i in range(vinfo.num_static_extra_boxes): - fieldbox = self.virtualizable_boxes[i] - self.history.record(rop.SETFIELD_GC, [vbox, fieldbox], None, - descr=vinfo.static_field_descrs[i]) - i = vinfo.num_static_extra_boxes - virtualizable = vinfo.unwrap_virtualizable_box(vbox) - for k in range(vinfo.num_arrays): - abox = vinfo.BoxArray() - self.history.record(rop.GETFIELD_GC, [vbox], abox, - descr=vinfo.array_field_descrs[k]) - for j in range(vinfo.get_array_length(virtualizable, k)): - itembox = self.virtualizable_boxes[i] - i += 1 - self.history.record(rop.SETARRAYITEM_GC, - [abox, ConstInt(j), itembox], - None, - descr=vinfo.array_descrs[k]) - assert i + 1 == len(self.virtualizable_boxes) - def replace_box(self, oldbox, newbox): for frame in self.framestack: boxes = frame.env Modified: pypy/trunk/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resoperation.py (original) +++ pypy/trunk/pypy/jit/metainterp/resoperation.py Tue Dec 1 11:56:34 2009 @@ -125,6 +125,7 @@ 'GUARD_EXCEPTION', 'GUARD_NO_OVERFLOW', 'GUARD_OVERFLOW', + 'GUARD_NOT_FORCED', '_GUARD_LAST', # ----- end of guard operations ----- '_NOSIDEEFFECT_FIRST', # ----- start of no_side_effect operations ----- @@ -220,9 +221,11 @@ 'COND_CALL_GC_MALLOC', # [a, b, if_(a<=b)_result, if_(a>b)_call, args...] # => result (for mallocs) 'DEBUG_MERGE_POINT/1', # debugging only + 'FORCE_TOKEN/0', '_CANRAISE_FIRST', # ----- start of can_raise operations ----- 'CALL', + 'CALL_MAY_FORCE', 'OOSEND', # ootype operation '_CANRAISE_LAST', # ----- end of can_raise operations ----- Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Tue Dec 1 11:56:34 2009 @@ -455,6 +455,10 @@ metainterp.framestack.reverse() return virtualizable_boxes +def force_from_resumedata(metainterp, newboxes, storage): + resumereader = ResumeDataReader(storage, newboxes, metainterp) + return resumereader.consume_boxes(), resumereader.virtuals + class ResumeDataReader(object): virtuals = None @@ -468,6 +472,10 @@ def _prepare_virtuals(self, metainterp, virtuals): if virtuals: + v = metainterp._already_allocated_resume_virtuals + if v is not None: + self.virtuals = v + return self.virtuals = [None] * len(virtuals) for i in range(len(virtuals)): vinfo = virtuals[i] @@ -476,7 +484,8 @@ for i in range(len(virtuals)): vinfo = virtuals[i] if vinfo is not None: - vinfo.setfields(metainterp, self.virtuals[i], self._decode_box) + vinfo.setfields(metainterp, self.virtuals[i], + self._decode_box) def consume_boxes(self): numb = self.cur_numb Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Tue Dec 1 11:56:34 2009 @@ -50,8 +50,8 @@ assert get_stats().enter_count <= count def check_jumps(self, maxcount): assert get_stats().exec_jumps <= maxcount - def check_aborted_count(self, maxcount): - assert get_stats().aborted_count == maxcount + def check_aborted_count(self, count): + assert get_stats().aborted_count == count def meta_interp(self, *args, **kwds): kwds['CPUClass'] = self.CPUClass @@ -148,6 +148,9 @@ type_system = 'ootype' CPUClass = runner.OOtypeCPU + def setup_class(cls): + py.test.skip("ootype tests skipped for now") + @staticmethod def Ptr(T): return T Modified: pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py Tue Dec 1 11:56:34 2009 @@ -121,8 +121,8 @@ supports_floats = False def fielddescrof(self, STRUCT, fieldname): return ('fielddescr', STRUCT, fieldname) - def calldescrof(self, FUNC, NON_VOID_ARGS, RESULT, stuff=None): - return ('calldescr', FUNC, NON_VOID_ARGS, RESULT) + def calldescrof(self, FUNC, NON_VOID_ARGS, RESULT, effectinfo=None): + return ('calldescr', FUNC, NON_VOID_ARGS, RESULT, effectinfo) def typedescrof(self, CLASS): return ('typedescr', CLASS) def methdescrof(self, CLASS, methname): @@ -273,9 +273,17 @@ cw._start(self.metainterp_sd, None) jitcode = cw.make_one_bytecode((graphs[0], None), False) assert len(self.metainterp_sd.indirectcalls) == 1 - names = [jitcode.name for (fnaddress, jitcode) + names = [jitcode1.name for (fnaddress, jitcode1) in self.metainterp_sd.indirectcalls] assert dict.fromkeys(names) == {'g': None} + calldescrs = [calldescr for calldescr in jitcode.constants + if isinstance(calldescr, tuple) and + calldescr[0] == 'calldescr'] + assert len(calldescrs) == 1 + assert calldescrs[0][4] is not None + assert not calldescrs[0][4].write_descrs_fields + assert not calldescrs[0][4].write_descrs_arrays + assert not calldescrs[0][4].promotes_virtualizables def test_oosend_look_inside_only_one(self): class A: @@ -386,6 +394,47 @@ assert cw.list_of_addr2name[0][1].endswith('.A1') assert cw.list_of_addr2name[1][1] == 'A1.g' + def test_promote_virtualizable_effectinfo(self): + class Frame(object): + _virtualizable2_ = ['x'] + + def __init__(self, x, y): + self.x = x + self.y = y + + def g1(f): + f.x += 1 + + def g2(f): + return f.x + + def h(f): + f.y -= 1 + + def f(n): + f_inst = Frame(n+1, n+2) + g1(f_inst) + r = g2(f_inst) + h(f_inst) + return r + + graphs = self.make_graphs(f, [5]) + cw = CodeWriter(self.rtyper) + cw.candidate_graphs = [graphs[0]] + cw._start(self.metainterp_sd, None) + jitcode = cw.make_one_bytecode((graphs[0], None), False) + calldescrs = [calldescr for calldescr in jitcode.constants + if isinstance(calldescr, tuple) and + calldescr[0] == 'calldescr'] + assert len(calldescrs) == 4 # for __init__, g1, g2, h. + effectinfo_g1 = calldescrs[1][4] + effectinfo_g2 = calldescrs[2][4] + effectinfo_h = calldescrs[3][4] + assert effectinfo_g1.promotes_virtualizables + assert effectinfo_g2.promotes_virtualizables + assert not effectinfo_h.promotes_virtualizables + + class ImmutableFieldsTests: def test_fields(self): Modified: pypy/trunk/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_recursive.py Tue Dec 1 11:56:34 2009 @@ -144,10 +144,11 @@ f = self.get_interpreter(codes) assert self.meta_interp(f, [0, 0, 0], optimizer=OPTIMIZER_SIMPLE) == 42 - self.check_loops(int_add = 1, call = 1) + self.check_loops(int_add = 1, call_may_force = 1, call = 0) assert self.meta_interp(f, [0, 0, 0], optimizer=OPTIMIZER_SIMPLE, inline=True) == 42 - self.check_loops(int_add = 2, call = 0, guard_no_exception = 0) + self.check_loops(int_add = 2, call_may_force = 0, call = 0, + guard_no_exception = 0) def test_inline_jitdriver_check(self): code = "021" @@ -158,7 +159,7 @@ assert self.meta_interp(f, [0, 0, 0], optimizer=OPTIMIZER_SIMPLE, inline=True) == 42 - self.check_loops(call = 1) + self.check_loops(call_may_force = 1, call = 0) def test_inline_faulty_can_inline(self): code = "021" @@ -488,10 +489,10 @@ return loop(100) res = self.meta_interp(main, [0], optimizer=OPTIMIZER_SIMPLE, trace_limit=TRACE_LIMIT) - self.check_loops(call=1) + self.check_loops(call_may_force=1, call=0) res = self.meta_interp(main, [1], optimizer=OPTIMIZER_SIMPLE, trace_limit=TRACE_LIMIT) - self.check_loops(call=0) + self.check_loops(call_may_force=0, call=0) def test_leave_jit_hook(self): from pypy.rpython.annlowlevel import hlstr @@ -645,7 +646,7 @@ result += f('-c-----------l-', i+100) self.meta_interp(g, [10], backendopt=True) self.check_aborted_count(1) - self.check_history(call=1) + self.check_history(call_may_force=1, call=0) self.check_tree_loop_count(3) Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Tue Dec 1 11:56:34 2009 @@ -41,6 +41,8 @@ assert not tagged_list_eq([tag(1, TAGBOX), tag(-2, TAGBOX)], [tag(1, TAGBOX)]) class MyMetaInterp: + _already_allocated_resume_virtuals = None + def __init__(self, cpu=None): if cpu is None: cpu = LLtypeMixin.cpu @@ -124,6 +126,7 @@ rd_numb = [] rd_consts = [] class FakeMetainterp(object): + _already_allocated_resume_virtuals = None cpu = None reader = ResumeDataReader(FakeStorage(), [], FakeMetainterp()) assert reader.virtuals == ["allocated", None] Modified: pypy/trunk/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_virtualizable.py Tue Dec 1 11:56:34 2009 @@ -7,7 +7,6 @@ from pypy.rlib.jit import OPTIMIZER_SIMPLE, OPTIMIZER_FULL from pypy.rlib.rarithmetic import intmask from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin -from pypy.rpython.lltypesystem.rvirtualizable2 import VABLERTIPTR from pypy.rpython.rclass import FieldListAccessor from pypy.jit.metainterp.warmspot import get_stats, get_translator from pypy.jit.metainterp import history, heaptracker @@ -41,8 +40,7 @@ XY = lltype.GcStruct( 'XY', ('parent', rclass.OBJECT), - ('vable_base', llmemory.Address), - ('vable_rti', VABLERTIPTR), + ('vable_token', lltype.Signed), ('inst_x', lltype.Signed), ('inst_node', lltype.Ptr(LLtypeMixin.NODE)), hints = {'virtualizable2_accessor': FieldListAccessor()}) @@ -57,7 +55,7 @@ def setup(self): xy = lltype.malloc(self.XY) - xy.vable_rti = lltype.nullptr(VABLERTIPTR.TO) + xy.vable_token = 0 xy.parent.typeptr = self.xy_vtable return xy @@ -200,79 +198,12 @@ assert res == 134 self.check_loops(getfield_gc=1, setfield_gc=1) - def test_external_read_while_tracing(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'm', 'xy'], - virtualizables = ['xy']) - class Outer: - pass - outer = Outer() - def ext(): - xy = outer.xy - promote_virtualizable(xy, 'inst_x') - return xy.inst_x + 2 - def f(n): - xy = self.setup() - xy.inst_x = 10 - outer.xy = xy - m = 0 - while n > 0: - myjitdriver.can_enter_jit(xy=xy, n=n, m=m) - myjitdriver.jit_merge_point(xy=xy, n=n, m=m) - promote_virtualizable(xy, 'inst_x') - xy.inst_x = n + 9998 # virtualized away - m += ext() # 2x setfield_gc, 2x getfield_gc - promote_virtualizable(xy, 'inst_x') - xy.inst_x = 10 # virtualized away - n -= 1 - return m - assert f(20) == 10000*20 + (20*21)/2 - res = self.meta_interp(f, [20], policy=StopAtXPolicy(ext)) - assert res == 10000*20 + (20*21)/2 - # there are no getfields because the optimizer gets rid of them - self.check_loops(call=1, getfield_gc=0, setfield_gc=2) - # xxx for now a call that forces the virtualizable during tracing - # is supposed to always force it later too. - - def test_external_write_while_tracing(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'm', 'xy'], - virtualizables = ['xy']) - class Outer: - pass - outer = Outer() - def ext(): - xy = outer.xy - promote_virtualizable(xy, 'inst_x') - xy.inst_x += 2 - def f(n): - xy = self.setup() - xy.inst_x = 10 - outer.xy = xy - m = 0 - while n > 0: - myjitdriver.can_enter_jit(xy=xy, n=n, m=m) - myjitdriver.jit_merge_point(xy=xy, n=n, m=m) - promote_virtualizable(xy, 'inst_x') - xy.inst_x = n + 9998 # virtualized away - ext() # 2x setfield_gc, 2x getfield_gc - promote_virtualizable(xy, 'inst_x') - m += xy.inst_x # virtualized away - n -= 1 - return m - res = self.meta_interp(f, [20], policy=StopAtXPolicy(ext)) - assert res == f(20) - # the getfield_gc of inst_node is optimized away, because ext does not - # write to it - self.check_loops(call=1, getfield_gc=1, setfield_gc=2) - # xxx for now a call that forces the virtualizable during tracing - # is supposed to always force it later too. - # ------------------------------ XY2 = lltype.GcStruct( 'XY2', ('parent', rclass.OBJECT), - ('vable_base', llmemory.Address), - ('vable_rti', VABLERTIPTR), + ('vable_token', lltype.Signed), ('inst_x', lltype.Signed), ('inst_l1', lltype.Ptr(lltype.GcArray(lltype.Signed))), ('inst_l2', lltype.Ptr(lltype.GcArray(lltype.Signed))), @@ -285,7 +216,7 @@ def setup2(self): xy2 = lltype.malloc(self.XY2) - xy2.vable_rti = lltype.nullptr(VABLERTIPTR.TO) + xy2.vable_token = 0 xy2.parent.typeptr = self.xy2_vtable return xy2 @@ -458,7 +389,7 @@ def setup2sub(self): xy2 = lltype.malloc(self.XY2SUB) - xy2.parent.vable_rti = lltype.nullptr(VABLERTIPTR.TO) + xy2.parent.vable_token = 0 xy2.parent.parent.typeptr = self.xy2_vtable return xy2 @@ -649,7 +580,8 @@ res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) assert res == f(123) - + self.check_aborted_count(2) + self.check_tree_loop_count(0) def test_external_write(self): jitdriver = JitDriver(greens = [], reds = ['frame'], @@ -680,10 +612,10 @@ res = self.meta_interp(f, [240], policy=StopAtXPolicy(g)) assert res == f(240) + self.check_aborted_count(3) + self.check_tree_loop_count(0) def test_external_read_sometimes(self): - py.test.skip("known bug: access the frame in a residual call but" - " only sometimes, so that it's not seen during tracing") jitdriver = JitDriver(greens = [], reds = ['frame'], virtualizables = ['frame']) @@ -719,6 +651,226 @@ res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) assert res == f(123) + def test_external_read_sometimes_with_virtuals(self): + jitdriver = JitDriver(greens = [], reds = ['frame'], + virtualizables = ['frame']) + + class Frame(object): + _virtualizable2_ = ['x', 'y'] + class Y: + pass + class SomewhereElse: + pass + somewhere_else = SomewhereElse() + + def g(): + somewhere_else.counter += 1 + if somewhere_else.counter == 70: + y = somewhere_else.top_frame.y # external read + debug_print(lltype.Void, '-+-+-+-+- external read') + else: + y = None + return y + + def f(n): + frame = Frame() + frame.x = n + somewhere_else.counter = 0 + somewhere_else.top_frame = frame + while frame.x > 0: + jitdriver.can_enter_jit(frame=frame) + jitdriver.jit_merge_point(frame=frame) + frame.y = y = Y() + result = g() + if frame.y is not y: + return -660 + if result: + if result is not y: + return -661 + frame.y = None + frame.x -= 1 + return frame.x + + res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) + assert res == f(123) + + def test_external_read_sometimes_changing_virtuals(self): + jitdriver = JitDriver(greens = [], reds = ['frame'], + virtualizables = ['frame']) + + class Frame(object): + _virtualizable2_ = ['x', 'y'] + class Y: + pass + class SomewhereElse: + pass + somewhere_else = SomewhereElse() + + def g(): + somewhere_else.counter += 1 + if somewhere_else.counter == 70: + y = somewhere_else.top_frame.y # external read + debug_print(lltype.Void, '-+-+-+-+- external virtual write') + assert y.num == 123 + y.num += 2 + else: + y = None + return y + + def f(n): + frame = Frame() + frame.x = n + somewhere_else.counter = 0 + somewhere_else.top_frame = frame + while frame.x > 0: + jitdriver.can_enter_jit(frame=frame) + jitdriver.jit_merge_point(frame=frame) + frame.y = y = Y() + y.num = 123 + result = g() + if frame.y is not y: + return -660 + if result: + if result is not y: + return -661 + if y.num != 125: + return -662 + frame.y = None + frame.x -= 1 + return frame.x + + res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) + assert res == f(123) + + def test_external_read_sometimes_with_exception(self): + jitdriver = JitDriver(greens = [], reds = ['frame'], + virtualizables = ['frame']) + + class Frame(object): + _virtualizable2_ = ['x', 'y'] + class FooBarError(Exception): + pass + class SomewhereElse: + pass + somewhere_else = SomewhereElse() + + def g(): + somewhere_else.counter += 1 + if somewhere_else.counter == 70: + result = somewhere_else.top_frame.y # external read + debug_print(lltype.Void, '-+-+-+-+- external read:', result) + assert result == 79 + raise FooBarError + else: + result = 1 + return result + + def f(n): + frame = Frame() + frame.x = n + frame.y = 10 + somewhere_else.counter = 0 + somewhere_else.top_frame = frame + try: + while frame.x > 0: + jitdriver.can_enter_jit(frame=frame) + jitdriver.jit_merge_point(frame=frame) + frame.x -= g() + frame.y += 1 + except FooBarError: + pass + return frame.x + + res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) + assert res == f(123) + + def test_external_read_sometimes_dont_compile_guard(self): + jitdriver = JitDriver(greens = [], reds = ['frame'], + virtualizables = ['frame']) + + class Frame(object): + _virtualizable2_ = ['x', 'y'] + class SomewhereElse: + pass + somewhere_else = SomewhereElse() + + def g(): + somewhere_else.counter += 1 + if somewhere_else.counter == 70: + result = somewhere_else.top_frame.y # external read + debug_print(lltype.Void, '-+-+-+-+- external read:', result) + assert result == 79 + else: + result = 1 + return result + + def f(n): + frame = Frame() + frame.x = n + frame.y = 10 + somewhere_else.counter = 0 + somewhere_else.top_frame = frame + while frame.x > 0: + jitdriver.can_enter_jit(frame=frame) + jitdriver.jit_merge_point(frame=frame) + frame.x -= g() + frame.y += 1 + return frame.x + + res = self.meta_interp(f, [123], policy=StopAtXPolicy(g), repeat=7) + assert res == f(123) + + def test_external_read_sometimes_recursive(self): + jitdriver = JitDriver(greens = [], reds = ['frame', 'rec'], + virtualizables = ['frame']) + + class Frame(object): + _virtualizable2_ = ['x', 'y'] + class SomewhereElse: + pass + somewhere_else = SomewhereElse() + + def g(rec): + somewhere_else.counter += 1 + if somewhere_else.counter == 70: + frame = somewhere_else.top_frame + result1 = frame.y # external read + result2 = frame.back.y # external read + debug_print(lltype.Void, '-+-+-+-+- external read:', + result1, result2) + assert result1 == 13 + assert result2 == 1023 + result = 2 + elif rec: + res = f(4, False) + assert res == 0 or res == -1 + result = 1 + else: + result = 1 + return result + + def f(n, rec): + frame = Frame() + frame.x = n + frame.y = 10 + 1000 * rec + frame.back = somewhere_else.top_frame + somewhere_else.top_frame = frame + while frame.x > 0: + jitdriver.can_enter_jit(frame=frame, rec=rec) + jitdriver.jit_merge_point(frame=frame, rec=rec) + frame.x -= g(rec) + frame.y += 1 + somewhere_else.top_frame = frame.back + return frame.x + + def main(n): + somewhere_else.counter = 0 + somewhere_else.top_frame = None + return f(n, True) + + res = self.meta_interp(main, [123], policy=StopAtXPolicy(g)) + assert res == main(123) + def test_promote_index_in_virtualizable_list(self): jitdriver = JitDriver(greens = [], reds = ['frame', 'n'], virtualizables = ['frame']) @@ -829,26 +981,26 @@ assert res == 55 self.check_loops(new_with_vtable=0) - def test_check_for_nonstandardness_only_once(self): - myjitdriver = JitDriver(greens = [], reds = ['frame'], - virtualizables = ['frame']) - - class Frame(object): - _virtualizable2_ = ['x', 'y', 'z'] - - def __init__(self, x, y, z=1): - self = hint(self, access_directly=True) - self.x = x - self.y = y - self.z = z - - class SomewhereElse: - pass - somewhere_else = SomewhereElse() - - def f(n): - frame = Frame(n, 0) - somewhere_else.top_frame = frame # escapes + def test_check_for_nonstandardness_only_once(self): + myjitdriver = JitDriver(greens = [], reds = ['frame'], + virtualizables = ['frame']) + + class Frame(object): + _virtualizable2_ = ['x', 'y', 'z'] + + def __init__(self, x, y, z=1): + self = hint(self, access_directly=True) + self.x = x + self.y = y + self.z = z + + class SomewhereElse: + pass + somewhere_else = SomewhereElse() + + def f(n): + frame = Frame(n, 0) + somewhere_else.top_frame = frame # escapes frame = hint(frame, access_directly=True) while frame.x > 0: myjitdriver.can_enter_jit(frame=frame) Modified: pypy/trunk/pypy/jit/metainterp/typesystem.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/typesystem.py (original) +++ pypy/trunk/pypy/jit/metainterp/typesystem.py Tue Dec 1 11:56:34 2009 @@ -49,10 +49,6 @@ CONST_NULL = history.ConstPtr(history.ConstPtr.value) CVAL_NULLREF = None # patched by optimizeopt.py - def get_VABLERTI(self): - from pypy.rpython.lltypesystem.rvirtualizable2 import VABLERTIPTR - return VABLERTIPTR - def new_ConstRef(self, x): ptrval = lltype.cast_opaque_ptr(llmemory.GCREF, x) return history.ConstPtr(ptrval) @@ -159,10 +155,6 @@ loops_done_with_this_frame_ref = None # patched by compile.py CONST_NULL = history.ConstObj(history.ConstObj.value) CVAL_NULLREF = None # patched by optimizeopt.py - - def get_VABLERTI(self): - from pypy.rpython.ootypesystem.rvirtualizable2 import VABLERTI - return VABLERTI def new_ConstRef(self, x): obj = ootype.cast_to_object(x) Modified: pypy/trunk/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/trunk/pypy/jit/metainterp/virtualizable.py Tue Dec 1 11:56:34 2009 @@ -4,19 +4,24 @@ from pypy.rpython import rvirtualizable2 from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.nonconst import NonConstant from pypy.jit.metainterp.typesystem import deref, fieldType, arrayItem from pypy.jit.metainterp import history from pypy.jit.metainterp.warmstate import wrap, unwrap class VirtualizableInfo: + token_none = 0 + token_tracing = -1 + def __init__(self, warmrunnerdesc): self.warmrunnerdesc = warmrunnerdesc jitdriver = warmrunnerdesc.jitdriver cpu = warmrunnerdesc.cpu + if cpu.ts.name == 'ootype': + import py + py.test.skip("ootype: fix virtualizables") self.cpu = cpu - self.VABLERTI = cpu.ts.get_VABLERTI() - self.null_vable_rti = cpu.ts.nullptr(deref(self.VABLERTI)) self.BoxArray = cpu.ts.BoxRef # assert len(jitdriver.virtualizables) == 1 # for now @@ -29,6 +34,7 @@ self.VTYPEPTR = VTYPEPTR self.VTYPE = VTYPE = deref(VTYPEPTR) self.null_vable = cpu.ts.nullptr(VTYPE) + self.vable_token_descr = cpu.fielddescrof(VTYPE, 'vable_token') # accessor = VTYPE._hints['virtualizable2_accessor'] all_fields = accessor.fields @@ -148,7 +154,7 @@ def finish(self): # def force_if_necessary(virtualizable): - if virtualizable.vable_rti: + if virtualizable.vable_token: self.force_now(virtualizable) force_if_necessary._always_inline_ = True # @@ -169,72 +175,57 @@ def is_vtypeptr(self, TYPE): return rvirtualizable2.match_virtualizable_type(TYPE, self.VTYPEPTR) - def cast_instance_to_base_ptr(self, vable_rti): - if we_are_translated(): - return self.cpu.ts.cast_instance_to_base_ref(vable_rti) - else: - vable_rti._TYPE = self.VABLERTI # hack for non-translated mode - return vable_rti + def reset_vable_token(self, virtualizable): + virtualizable.vable_token = self.token_none - def clear_vable_rti(self, virtualizable): - if virtualizable.vable_rti: + def clear_vable_token(self, virtualizable): + if virtualizable.vable_token: self.force_now(virtualizable) - assert not virtualizable.vable_rti + assert not virtualizable.vable_token def tracing_before_residual_call(self, virtualizable): - assert not virtualizable.vable_rti - ptr = self.cast_instance_to_base_ptr(tracing_vable_rti) - virtualizable.vable_rti = ptr + assert not virtualizable.vable_token + virtualizable.vable_token = self.token_tracing def tracing_after_residual_call(self, virtualizable): - if virtualizable.vable_rti: + if virtualizable.vable_token: # not modified by the residual call; assert that it is still # set to 'tracing_vable_rti' and clear it. - ptr = self.cast_instance_to_base_ptr(tracing_vable_rti) - assert virtualizable.vable_rti == ptr - virtualizable.vable_rti = self.null_vable_rti + assert virtualizable.vable_token == self.token_tracing + virtualizable.vable_token = self.token_none return False else: # marker "modified during residual call" set. return True def force_now(self, virtualizable): - rti = virtualizable.vable_rti - virtualizable.vable_rti = self.null_vable_rti - if we_are_translated(): - rti = cast_base_ptr_to_instance(AbstractVableRti, rti) - rti.force_now(virtualizable) + token = virtualizable.vable_token + virtualizable.vable_token = self.token_none + if token == self.token_tracing: + # The values in the virtualizable are always correct during + # tracing. We only need to reset vable_token to token_none + # as a marker for the tracing, to tell it that this + # virtualizable escapes. + pass + else: + from pypy.jit.metainterp.compile import ResumeGuardForcedDescr + faildescr = self.cpu.force(token) + assert isinstance(faildescr, ResumeGuardForcedDescr) + faildescr.force_virtualizable(self, virtualizable, token) force_now._dont_inline_ = True # ____________________________________________________________ # -# The 'vable_rti' field of a virtualizable is either NULL or points -# to an instance of the following classes. It is: +# The 'vable_token' field of a virtualizable is either 0, -1, or points +# into the CPU stack to a particular field in the current frame. It is: # -# 1. NULL if not in the JIT at all, except as described below. +# 1. 0 (token_none) if not in the JIT at all, except as described below. # -# 2. always NULL when tracing is in progress. +# 2. equal to 0 when tracing is in progress; except: # -# 3. 'tracing_vable_rti' during tracing when we do a residual call, +# 3. equal to -1 (token_tracing) during tracing when we do a residual call, # calling random unknown other parts of the interpreter; it is -# reset to NULL as soon as something occurs to the virtualizable. +# reset to 0 as soon as something occurs to the virtualizable. # -# 4. NULL for now when running the machine code with a virtualizable; -# later it will be a RunningVableRti(). - - -class AbstractVableRti(object): - - def force_now(self, virtualizable): - raise NotImplementedError - - -class TracingVableRti(AbstractVableRti): - - def force_now(self, virtualizable): - # The values if the virtualizable are always correct during tracing. - # We only need to set a marker to tell that forcing occurred. - # As the caller resets vable_rti to NULL, it plays the role of marker. - pass - -tracing_vable_rti = TracingVableRti() +# 4. when running the machine code with a virtualizable, it is set +# to the address in the CPU stack by the FORCE_TOKEN operation. Modified: pypy/trunk/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmstate.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmstate.py Tue Dec 1 11:56:34 2009 @@ -213,6 +213,8 @@ virtualizable = vinfo.cast_to_vtype(virtualizable) assert virtualizable != globaldata.blackhole_virtualizable, ( "reentering same frame via blackhole") + else: + virtualizable = None # look for the cell corresponding to the current greenargs greenargs = args[:num_green_args] @@ -247,6 +249,8 @@ fail_descr = metainterp_sd.cpu.execute_token(loop_token) debug_stop("jit-running") metainterp_sd.profiler.end_running() + if vinfo is not None: + vinfo.reset_vable_token(virtualizable) loop_token = fail_descr.handle_fail(metainterp_sd) maybe_compile_and_run._dont_inline_ = True Modified: pypy/trunk/pypy/rlib/jit.py ============================================================================== --- pypy/trunk/pypy/rlib/jit.py (original) +++ pypy/trunk/pypy/rlib/jit.py Tue Dec 1 11:56:34 2009 @@ -22,6 +22,7 @@ def purefunction_promote(func): import inspect purefunction(func) + func._pure_function_with_all_promoted_args_ = True args, varargs, varkw, defaults = inspect.getargspec(func) args = ["v%s" % (i, ) for i in range(len(args))] assert varargs is None and varkw is None Modified: pypy/trunk/pypy/rpython/llinterp.py ============================================================================== --- pypy/trunk/pypy/rpython/llinterp.py (original) +++ pypy/trunk/pypy/rpython/llinterp.py Tue Dec 1 11:56:34 2009 @@ -807,9 +807,6 @@ def op_gc__collect(self, *gen): self.heap.collect(*gen) - def op_gc_assume_young_pointers(self, addr): - raise NotImplementedError - def op_gc_heap_stats(self): raise NotImplementedError Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Tue Dec 1 11:56:34 2009 @@ -448,6 +448,9 @@ self._storage._setitem(index, value, boundscheck=False) def getitems(self): + if self._TYPE.OF != lltype.Char: + raise Exception("cannot get all items of an unknown-length " + "array of %r" % self._TYPE.OF) _items = [] i = 0 while 1: Modified: pypy/trunk/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/lloperation.py Tue Dec 1 11:56:34 2009 @@ -460,7 +460,7 @@ # allocating non-GC structures only 'gc_thread_run' : LLOp(), 'gc_thread_die' : LLOp(), - 'gc_assume_young_pointers': LLOp(), + 'gc_assume_young_pointers': LLOp(canrun=True), 'gc_heap_stats' : LLOp(canunwindgc=True), # ------- JIT & GC interaction, only for some GCs ---------- Modified: pypy/trunk/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/opimpl.py Tue Dec 1 11:56:34 2009 @@ -486,6 +486,9 @@ def op_get_member_index(memberoffset): raise NotImplementedError +def op_gc_assume_young_pointers(addr): + pass + # ____________________________________________________________ def get_op_impl(opname): Modified: pypy/trunk/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rffi.py Tue Dec 1 11:56:34 2009 @@ -57,7 +57,7 @@ def llexternal(name, args, result, _callable=None, compilation_info=ExternalCompilationInfo(), sandboxsafe=False, threadsafe='auto', - canraise=False, _nowrapper=False, calling_conv='c', + _nowrapper=False, calling_conv='c', oo_primitive=None, pure_function=False): """Build an external function that will invoke the C function 'name' with the given 'args' types and 'result' type. @@ -68,6 +68,10 @@ pointing to a read-only null-terminated character of arrays, as usual for C. + The C function can have callbacks, but they must be specified explicitly + as constant RPython functions. We don't support yet C functions that + invoke callbacks passed otherwise (e.g. set by a previous C call). + threadsafe: whether it's ok to release the GIL around the call. Default is yes, unless sandboxsafe is set, in which case we consider that the function is really short-running and @@ -84,12 +88,22 @@ kwds = {} if oo_primitive: kwds['oo_primitive'] = oo_primitive + + has_callback = False + for ARG in args: + if _isfunctype(ARG): + has_callback = True + if has_callback: + kwds['_callbacks'] = callbackholder = CallbackHolder() + else: + callbackholder = None + funcptr = lltype.functionptr(ext_type, name, external='C', compilation_info=compilation_info, _callable=_callable, _safe_not_sandboxed=sandboxsafe, _debugexc=True, # on top of llinterp - canraise=canraise, + canraise=False, **kwds) if isinstance(_callable, ll2ctypes.LL2CtypesCallable): _callable.funcptr = funcptr @@ -170,9 +184,11 @@ # XXX pass additional arguments if invoke_around_handlers: arg = llhelper(TARGET, _make_wrapper_for(TARGET, arg, + callbackholder, aroundstate)) else: - arg = llhelper(TARGET, _make_wrapper_for(TARGET, arg)) + arg = llhelper(TARGET, _make_wrapper_for(TARGET, arg, + callbackholder)) else: SOURCE = lltype.typeOf(arg) if SOURCE != TARGET: @@ -202,7 +218,11 @@ return func_with_new_name(wrapper, name) -def _make_wrapper_for(TP, callable, aroundstate=None): +class CallbackHolder: + def __init__(self): + self.callbacks = {} + +def _make_wrapper_for(TP, callable, callbackholder, aroundstate=None): """ Function creating wrappers for callbacks. Note that this is cheating as we assume constant callbacks and we just memoize wrappers """ @@ -213,6 +233,7 @@ else: errorcode = TP.TO.RESULT._example() callable_name = getattr(callable, '__name__', '?') + callbackholder.callbacks[callable] = True args = ', '.join(['a%d' % i for i in range(len(TP.TO.ARGS))]) source = py.code.Source(r""" def wrapper(%s): # no *args - no GIL for mallocing the tuple Modified: pypy/trunk/pypy/rpython/lltypesystem/rvirtualizable2.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rvirtualizable2.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rvirtualizable2.py Tue Dec 1 11:56:34 2009 @@ -3,23 +3,21 @@ from pypy.rpython.lltypesystem.rclass import InstanceRepr, OBJECTPTR from pypy.rpython.rvirtualizable2 import AbstractVirtualizable2InstanceRepr -VABLERTIPTR = OBJECTPTR class Virtualizable2InstanceRepr(AbstractVirtualizable2InstanceRepr, InstanceRepr): def _setup_repr_llfields(self): llfields = [] if self.top_of_virtualizable_hierarchy: - llfields.append(('vable_base', llmemory.Address)) - llfields.append(('vable_rti', VABLERTIPTR)) + llfields.append(('vable_token', lltype.Signed)) return llfields def set_vable(self, llops, vinst, force_cast=False): if self.top_of_virtualizable_hierarchy: if force_cast: vinst = llops.genop('cast_pointer', [vinst], resulttype=self) - cname = inputconst(lltype.Void, 'vable_rti') - vvalue = inputconst(VABLERTIPTR, lltype.nullptr(VABLERTIPTR.TO)) - llops.genop('setfield', [vinst, cname, vvalue]) + cname = inputconst(lltype.Void, 'vable_token') + cvalue = inputconst(lltype.Signed, 0) + llops.genop('setfield', [vinst, cname, cvalue]) else: self.rbase.set_vable(llops, vinst, force_cast=True) Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/test/test_rffi.py Tue Dec 1 11:56:34 2009 @@ -387,6 +387,7 @@ fn = self.compile(f, []) assert fn() == 6 + assert eating_callback._ptr._obj._callbacks.callbacks == {g: True} def test_double_callback(self): eating_callback = self.eating_callback() @@ -406,6 +407,8 @@ fn = self.compile(f, [int]) assert fn(4) == 4 assert fn(1) == 3 + assert eating_callback._ptr._obj._callbacks.callbacks == {one: True, + two: True} def test_exception_callback(self): eating_callback = self.eating_callback() Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/framework.py Tue Dec 1 11:56:34 2009 @@ -29,12 +29,15 @@ def analyze_direct_call(self, graph, seen=None): try: func = graph.func + except AttributeError: + pass + else: if func is rstack.stack_check: return self.translator.config.translation.stackless - if func._gctransformer_hint_cannot_collect_: + if getattr(func, '_gctransformer_hint_cannot_collect_', False): return False - except AttributeError: - pass + if getattr(func, '_gctransformer_hint_close_stack_', False): + return True return graphanalyze.GraphAnalyzer.analyze_direct_call(self, graph, seen) Modified: pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py Tue Dec 1 11:56:34 2009 @@ -6,7 +6,7 @@ from pypy.rpython.memory.gctransform.transform import GcHighLevelOp from pypy.rpython.memory.gctransform.framework import FrameworkGCTransformer, \ CollectAnalyzer, find_initializing_stores, find_clean_setarrayitems -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, rffi from pypy.rpython.rtyper import LowLevelOpList from pypy.translator.c.gc import FrameworkGcPolicy from pypy.translator.translator import TranslationContext, graphof @@ -87,6 +87,33 @@ can_collect = CollectAnalyzer(t).analyze_direct_call(with_check_graph) assert can_collect +def test_cancollect_external(): + fext1 = rffi.llexternal('fext1', [], lltype.Void, threadsafe=False) + def g(): + fext1() + t = rtype(g, []) + gg = graphof(t, g) + assert not CollectAnalyzer(t).analyze_direct_call(gg) + + fext2 = rffi.llexternal('fext2', [], lltype.Void, threadsafe=True) + def g(): + fext2() + t = rtype(g, []) + gg = graphof(t, g) + assert CollectAnalyzer(t).analyze_direct_call(gg) + + S = lltype.GcStruct('S', ('x', lltype.Signed)) + FUNC = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Void)) + fext3 = rffi.llexternal('fext3', [FUNC], lltype.Void, threadsafe=False) + def h(x): + lltype.malloc(S, zero=True) + def g(): + fext3(h) + t = rtype(g, []) + gg = graphof(t, g) + assert CollectAnalyzer(t).analyze_direct_call(gg) + + class WriteBarrierTransformer(FrameworkGCTransformer): clean_sets = {} GC_PARAMS = {} Modified: pypy/trunk/pypy/rpython/ootypesystem/rvirtualizable2.py ============================================================================== --- pypy/trunk/pypy/rpython/ootypesystem/rvirtualizable2.py (original) +++ pypy/trunk/pypy/rpython/ootypesystem/rvirtualizable2.py Tue Dec 1 11:56:34 2009 @@ -11,7 +11,7 @@ def _setup_repr_llfields(self): llfields = [] if self.top_of_virtualizable_hierarchy: - llfields.append(('vable_rti', VABLERTI)) + llfields.append(('vable_token', VABLERTI)) return llfields def set_vable(self, llops, vinst, force_cast=False): Modified: pypy/trunk/pypy/rpython/test/test_rvirtualizable2.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rvirtualizable2.py (original) +++ pypy/trunk/pypy/rpython/test/test_rvirtualizable2.py Tue Dec 1 11:56:34 2009 @@ -361,7 +361,7 @@ assert res.item1 == 42 res = lltype.normalizeptr(res.item0) assert res.inst_v == 42 - assert not res.vable_rti + assert res.vable_token == 0 class TestOOtype(OORtypeMixin, BaseTest): prefix = 'o' Modified: pypy/trunk/pypy/testrunner_cfg.py ============================================================================== --- pypy/trunk/pypy/testrunner_cfg.py (original) +++ pypy/trunk/pypy/testrunner_cfg.py Tue Dec 1 11:56:34 2009 @@ -7,7 +7,8 @@ reldir.startswith('rlib/test') or reldir.startswith('rpython/memory/') or reldir.startswith('jit/backend/x86/') or - reldir.startswith('jit/backend/cli')): + #reldir.startswith('jit/backend/cli') or + 0): testdirs.extend(tests) else: testdirs.append(reldir) Modified: pypy/trunk/pypy/translator/backendopt/canraise.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/canraise.py (original) +++ pypy/trunk/pypy/translator/backendopt/canraise.py Tue Dec 1 11:56:34 2009 @@ -17,7 +17,7 @@ log.WARNING("Unknown operation: %s" % op.opname) return True - def analyze_external_call(self, op): + def analyze_external_call(self, op, seen=None): fnobj = get_funcobj(op.args[0].value) return getattr(fnobj, 'canraise', True) Modified: pypy/trunk/pypy/translator/backendopt/graphanalyze.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/graphanalyze.py (original) +++ pypy/trunk/pypy/translator/backendopt/graphanalyze.py Tue Dec 1 11:56:34 2009 @@ -1,4 +1,4 @@ -from pypy.translator.simplify import get_graph +from pypy.translator.simplify import get_graph, get_funcobj from pypy.rpython.lltypesystem.lloperation import llop, LL_OPERATIONS from pypy.rpython.lltypesystem import lltype @@ -38,8 +38,17 @@ def analyze_startblock(self, block, seen=None): return self.bottom_result() - def analyze_external_call(self, op): - return self.top_result() + def analyze_external_call(self, op, seen=None): + funcobj = get_funcobj(op.args[0].value) + result = self.bottom_result() + if hasattr(funcobj, '_callbacks'): + bk = self.translator.annotator.bookkeeper + for function in funcobj._callbacks.callbacks: + desc = bk.getdesc(function) + for graph in desc.getgraphs(): + result = self.join_two_results( + result, self.analyze_direct_call(graph, seen)) + return result def analyze_external_method(self, op, TYPE, meth): return self.top_result() @@ -59,7 +68,7 @@ if op.opname == "direct_call": graph = get_graph(op.args[0], self.translator) if graph is None: - return self.analyze_external_call(op) + return self.analyze_external_call(op, seen) return self.analyze_direct_call(graph, seen) elif op.opname == "indirect_call": if op.args[-1].value is None: Modified: pypy/trunk/pypy/translator/backendopt/test/test_canraise.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/test/test_canraise.py (original) +++ pypy/trunk/pypy/translator/backendopt/test/test_canraise.py Tue Dec 1 11:56:34 2009 @@ -189,7 +189,8 @@ result = ra.can_raise(fgraph.startblock.operations[0]) assert not result - z = llexternal('z', [lltype.Signed], lltype.Signed, canraise=True) + z = lltype.functionptr(lltype.FuncType([lltype.Signed], lltype.Signed), + 'foobar') def g(x): return z(x) t, ra = self.translate(g, [int]) Modified: pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py (original) +++ pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py Tue Dec 1 11:56:34 2009 @@ -178,6 +178,31 @@ assert name == "length" assert S1 is S2 + def test_llexternal_with_callback(self): + from pypy.rpython.lltypesystem.rffi import llexternal + from pypy.rpython.lltypesystem import lltype + + class Abc: + pass + abc = Abc() + + FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) + z = llexternal('z', [lltype.Ptr(FUNC)], lltype.Signed) + def g(n): + abc.foobar = n + return n + 1 + def f(x): + return z(g) + t, wa = self.translate(f, [int]) + fgraph = graphof(t, f) + backend_optimizations(t) + assert fgraph.startblock.operations[0].opname == 'direct_call' + + result = wa.analyze(fgraph.startblock.operations[0]) + assert len(result) == 1 + (struct, T, name), = result + assert struct == "struct" + assert name.endswith("foobar") class TestOOtype(BaseTestCanRaise): Modified: pypy/trunk/pypy/translator/backendopt/writeanalyze.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/writeanalyze.py (original) +++ pypy/trunk/pypy/translator/backendopt/writeanalyze.py Tue Dec 1 11:56:34 2009 @@ -1,6 +1,5 @@ from pypy.translator.backendopt import graphanalyze from pypy.rpython.ootypesystem import ootype -reload(graphanalyze) top_set = object() empty_set = frozenset() @@ -38,9 +37,6 @@ def _array_result(self, TYPE): return frozenset([("array", TYPE)]) - def analyze_external_call(self, op): - return self.bottom_result() # an external call cannot change anything - def analyze_external_method(self, op, TYPE, meth): if isinstance(TYPE, ootype.Array): methname = op.args[0].value Modified: pypy/trunk/pypy/translator/stackless/transform.py ============================================================================== --- pypy/trunk/pypy/translator/stackless/transform.py (original) +++ pypy/trunk/pypy/translator/stackless/transform.py Tue Dec 1 11:56:34 2009 @@ -260,7 +260,7 @@ return LL_OPERATIONS[op.opname].canunwindgc return False - def analyze_external_call(self, op): + def analyze_external_call(self, op, seen=None): # An external call cannot cause a stack unwind # Note that this is essential to get good performance in framework GCs # because there is a pseudo-external call to ROUND_UP_FOR_ALLOCATION From arigo at codespeak.net Tue Dec 1 11:59:15 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 1 Dec 2009 11:59:15 +0100 (CET) Subject: [pypy-svn] r69794 - pypy/branch/virtual-forcing Message-ID: <20091201105915.B6A6F168076@codespeak.net> Author: arigo Date: Tue Dec 1 11:59:15 2009 New Revision: 69794 Removed: pypy/branch/virtual-forcing/ Log: Kill the branch. From arigo at codespeak.net Tue Dec 1 11:59:46 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 1 Dec 2009 11:59:46 +0100 (CET) Subject: [pypy-svn] r69795 - pypy/branch/virtual-forcing Message-ID: <20091201105946.DBE37168076@codespeak.net> Author: arigo Date: Tue Dec 1 11:59:46 2009 New Revision: 69795 Added: pypy/branch/virtual-forcing/ - copied from r69793, pypy/trunk/ Log: Re-add the branch as a fresh copy of trunk. From pedronis at codespeak.net Tue Dec 1 12:02:23 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 1 Dec 2009 12:02:23 +0100 (CET) Subject: [pypy-svn] r69796 - pypy/branch/esp-params Message-ID: <20091201110223.A016C168076@codespeak.net> Author: pedronis Date: Tue Dec 1 12:02:23 2009 New Revision: 69796 Added: pypy/branch/esp-params/ - copied from r69795, pypy/trunk/ Log: branch to make the setup of call params in the x86 purely esp based and without esp adjusting From pedronis at codespeak.net Tue Dec 1 12:30:56 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 1 Dec 2009 12:30:56 +0100 (CET) Subject: [pypy-svn] r69797 - pypy/branch/esp-params/pypy/jit/backend/x86/test Message-ID: <20091201113056.95356168078@codespeak.net> Author: pedronis Date: Tue Dec 1 12:30:56 2009 New Revision: 69797 Modified: pypy/branch/esp-params/pypy/jit/backend/x86/test/test_regalloc.py Log: some tests about calls Modified: pypy/branch/esp-params/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/x86/test/test_regalloc.py Tue Dec 1 12:30:56 2009 @@ -85,6 +85,20 @@ fdescr2 = BasicFailDescr(2) fdescr3 = BasicFailDescr(3) + def f1(x): + return x+1 + + def f2(x, y): + return x*y + + F1PTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Signed)) + F2PTR = lltype.Ptr(lltype.FuncType([lltype.Signed]*2, lltype.Signed)) + f1ptr = llhelper(F1PTR, f1) + f2ptr = llhelper(F2PTR, f2) + + f1_calldescr = cpu.calldescrof(F1PTR.TO, F1PTR.TO.ARGS, F1PTR.TO.RESULT) + f2_calldescr = cpu.calldescrof(F2PTR.TO, F2PTR.TO.ARGS, F2PTR.TO.RESULT) + namespace = locals().copy() type_system = 'lltype' @@ -522,3 +536,64 @@ ''' loop = self.interpret(ops, [0.0, .1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1] + +class TestRegAllocCallAndStackDepth(BaseTestRegalloc): + + def test_one_call(self): + ops = ''' + [i0, i1] + i2 = call(ConstClass(f1ptr), i0, descr=f1_calldescr) + finish(i2) + ''' + loop = self.interpret(ops, [4, 7]) + assert self.getints(1) == [5] + + def test_two_calls(self): + ops = ''' + [i0, i1] + i2 = call(ConstClass(f1ptr), i0, descr=f1_calldescr) + i3 = call(ConstClass(f2ptr), i2, i1, descr=f2_calldescr) + finish(i3) + ''' + loop = self.interpret(ops, [4, 7]) + assert self.getints(1) == [5*7] + + def test_bridge_calls_1(self): + ops = ''' + [i0, i1] + i2 = call(ConstClass(f1ptr), i0, descr=f1_calldescr) + guard_value(i2, 0, descr=fdescr1) [i2, i1] + finish(i1) + ''' + loop = self.interpret(ops, [4, 7]) + assert self.getint(0) == 5 + ops = ''' + [i2, i1] + i3 = call(ConstClass(f2ptr), i2, i1, descr=f2_calldescr) + finish(i3, descr=fdescr2) + ''' + bridge = self.attach_bridge(ops, loop, -2) + self.cpu.set_future_value_int(0, 4) + self.cpu.set_future_value_int(1, 7) + self.run(loop) + assert self.getint(0) == 5*7 + + def test_bridge_calls_2(self): + ops = ''' + [i0, i1] + i2 = call(ConstClass(f2ptr), i0, i1, descr=f2_calldescr) + guard_value(i2, 0, descr=fdescr1) [i2] + finish(i1) + ''' + loop = self.interpret(ops, [4, 7]) + assert self.getint(0) == 4*7 + ops = ''' + [i2] + i3 = call(ConstClass(f1ptr), i2, descr=f1_calldescr) + finish(i3, descr=fdescr2) + ''' + bridge = self.attach_bridge(ops, loop, -2) + self.cpu.set_future_value_int(0, 4) + self.cpu.set_future_value_int(1, 7) + self.run(loop) + assert self.getint(0) == 29 From afa at codespeak.net Tue Dec 1 13:56:20 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 1 Dec 2009 13:56:20 +0100 (CET) Subject: [pypy-svn] r69798 - in pypy/trunk/pypy/module/_locale: . test Message-ID: <20091201125620.DF8251680AF@codespeak.net> Author: afa Date: Tue Dec 1 13:56:20 2009 New Revision: 69798 Modified: pypy/trunk/pypy/module/_locale/interp_locale.py pypy/trunk/pypy/module/_locale/test/test_locale.py Log: Fixes the tests for the _locale module on windows: - Remove utf-8 constants from the file: they are not reliable when geninterp'd, and windows XP removed support for utf-8 locale anyway. Use the (verbose) name of the characters instead, and the cp1257 code page to handle Polish letters. - remove some constants from the module interface, "LOCALE_SISO3166CTRYNAME" &co can't be used with the _locale module. Still remains: a segfault when pypy-c calls "_locale.setlocale(12345)" Modified: pypy/trunk/pypy/module/_locale/interp_locale.py ============================================================================== --- pypy/trunk/pypy/module/_locale/interp_locale.py (original) +++ pypy/trunk/pypy/module/_locale/interp_locale.py Tue Dec 1 13:56:20 2009 @@ -131,7 +131,7 @@ for name in langinfo_names: value = getattr(cConfig, name) - if value is not None: + if value is not None and sys.platform != 'win32': constants[name] = value locals().update(constants) Modified: pypy/trunk/pypy/module/_locale/test/test_locale.py ============================================================================== --- pypy/trunk/pypy/module/_locale/test/test_locale.py (original) +++ pypy/trunk/pypy/module/_locale/test/test_locale.py Tue Dec 1 13:56:20 2009 @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import py from pypy.conftest import gettestobjspace @@ -11,21 +10,31 @@ cls.w_language_en = cls.space.wrap("en_US") cls.w_language_utf8 = cls.space.wrap("en_US.UTF-8") cls.w_language_pl = cls.space.wrap("pl_PL.UTF-8") + cls.w_encoding_pl = cls.space.wrap("utf-8") else: cls.w_language_en = cls.space.wrap("English_US") cls.w_language_utf8 = cls.space.wrap("English_US.65001") - cls.w_language_pl = cls.space.wrap("Polish_Poland") + cls.w_language_pl = cls.space.wrap("Polish_Poland.1257") + cls.w_encoding_pl = cls.space.wrap("cp1257") import _locale # check whether used locales are installed, otherwise the tests will # fail current = _locale.setlocale(_locale.LC_ALL) try: try: - _locale.setlocale(_locale.LC_ALL, space.str_w(cls.w_language_en)) - _locale.setlocale(_locale.LC_ALL, space.str_w(cls.w_language_utf8)) - _locale.setlocale(_locale.LC_ALL, space.str_w(cls.w_language_pl)) + _locale.setlocale(_locale.LC_ALL, + space.str_w(cls.w_language_en)) + _locale.setlocale(_locale.LC_ALL, + space.str_w(cls.w_language_pl)) except _locale.Error: py.test.skip("necessary locales not installed") + + # Windows forbids the UTF-8 character set since Windows XP. + try: + _locale.setlocale(_locale.LC_ALL, + space.str_w(cls.w_language_utf8)) + except _locale.Error: + del cls.w_language_utf8 finally: _locale.setlocale(_locale.LC_ALL, current) @@ -76,13 +85,9 @@ for i in range(1, 13): _LANGINFO_NAMES.append("MON_%d" % i) _LANGINFO_NAMES.append("ABMON_%d" % i) - else: - _LANGINFO_NAMES = ('LOCALE_USER_DEFAULT LOCALE_SISO639LANGNAME ' - 'LOCALE_SISO3166CTRYNAME LOCALE_IDEFAULTLANGUAGE ' - '').split() - for constant in _LANGINFO_NAMES: - assert hasattr(_locale, constant) + for constant in _LANGINFO_NAMES: + assert hasattr(_locale, constant) def test_setlocale(self): import _locale @@ -95,6 +100,8 @@ assert _locale.setlocale(_locale.LC_ALL) def test_string_ulcase(self): + if not hasattr(self, 'language_utf8'): + skip("No utf8 locale on this platform") import _locale, string lcase = "abcdefghijklmnopqrstuvwxyz" @@ -143,9 +150,13 @@ _locale.setlocale(_locale.LC_ALL, self.language_pl) assert _locale.strcoll("a", "b") < 0 - assert _locale.strcoll("?", "b") < 0 - - assert _locale.strcoll("?", "b") > 0 + assert _locale.strcoll( + u"\N{LATIN SMALL LETTER A WITH OGONEK}".encode(self.encoding_pl), + "b") < 0 + + assert _locale.strcoll( + u"\N{LATIN SMALL LETTER C WITH ACUTE}".encode(self.encoding_pl), + "b") > 0 assert _locale.strcoll("c", "b") > 0 assert _locale.strcoll("b", "b") == 0 From cfbolz at codespeak.net Tue Dec 1 14:26:57 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 Dec 2009 14:26:57 +0100 (CET) Subject: [pypy-svn] r69799 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091201132657.465FC1680C1@codespeak.net> Author: cfbolz Date: Tue Dec 1 14:26:56 2009 New Revision: 69799 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/optimizeutil.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py Log: A different approach to sharing the descrlists of virtual info object: use a global dictionary. Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Tue Dec 1 14:26:56 2009 @@ -10,6 +10,7 @@ from pypy.jit.metainterp.specnode import VirtualArraySpecNode from pypy.jit.metainterp.specnode import VirtualStructSpecNode from pypy.jit.metainterp.optimizeutil import av_newdict2, _findall, sort_descrs +from pypy.jit.metainterp.optimizeutil import descrlist_dict from pypy.jit.metainterp.optimizeutil import InvalidLoop from pypy.jit.metainterp import resume, compile from pypy.jit.metainterp.typesystem import llhelper, oohelper @@ -172,6 +173,13 @@ def _make_virtual(self, modifier): raise NotImplementedError("abstract base") +def get_fielddescrlist_cache(cpu): + if not hasattr(cpu, '_optimizeopt_fielddescrlist_cache'): + result = descrlist_dict() + cpu._optimizeopt_fielddescrlist_cache = result + return result + return cpu._optimizeopt_fielddescrlist_cache +get_fielddescrlist_cache._annspecialcase_ = "specialize:memo" class AbstractVirtualStructValue(AbstractVirtualValue): _attrs_ = ('_fields', '_cached_sorted_fields') @@ -206,7 +214,6 @@ self._fields = None def _get_field_descr_list(self): - # this shares only per instance and not per type, but better than nothing _cached_sorted_fields = self._cached_sorted_fields if (_cached_sorted_fields is not None and len(self._fields) == len(_cached_sorted_fields)): @@ -214,6 +221,14 @@ else: lst = self._fields.keys() sort_descrs(lst) + cache = get_fielddescrlist_cache(self.optimizer.cpu) + result = cache.get(lst, None) + if result is None: + cache[lst] = lst + else: + lst = result + # store on self, to not have to repeatedly get it from the global + # cache, which involves sorting self._cached_sorted_fields = lst return lst Modified: pypy/trunk/pypy/jit/metainterp/optimizeutil.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeutil.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeutil.py Tue Dec 1 14:26:56 2009 @@ -54,3 +54,26 @@ quicksort(lst, 0, len(lst)-1) +def descrlist_hash(l): + res = 0x345678 + mult = 1000003 + z = len(l) + for descr in l: + y = descr.sort_key() + res = (res ^ y) * mult + z -= 1 + mult += 82520 + z + z + res += 97531 + return res + +def descrlist_eq(l1, l2): + if len(l1) != len(l2): + return False + for i in range(len(l1)): + if l1[i].sort_key() != l2[i].sort_key(): + return False + return True + +def descrlist_dict(): + return r_dict(descrlist_eq, descrlist_hash) + Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Tue Dec 1 14:26:56 2009 @@ -63,16 +63,27 @@ assert fdescr.rd_consts == [] def test_sharing_field_lists_of_virtual(): - virt1 = optimizeopt.AbstractVirtualStructValue(None, None) + class FakeOptimizer(object): + class cpu(object): + pass + opt = FakeOptimizer() + virt1 = optimizeopt.AbstractVirtualStructValue(opt, None) lst1 = virt1._get_field_descr_list() assert lst1 == [] lst2 = virt1._get_field_descr_list() assert lst1 is lst2 virt1.setfield(LLtypeMixin.valuedescr, optimizeopt.OptValue(None)) - lst1 = virt1._get_field_descr_list() - assert lst1 == [LLtypeMixin.valuedescr] - lst2 = virt1._get_field_descr_list() - assert lst1 is lst2 + lst3 = virt1._get_field_descr_list() + assert lst3 == [LLtypeMixin.valuedescr] + lst4 = virt1._get_field_descr_list() + assert lst3 is lst4 + + virt2 = optimizeopt.AbstractVirtualStructValue(opt, None) + lst5 = virt2._get_field_descr_list() + assert lst5 is lst1 + virt2.setfield(LLtypeMixin.valuedescr, optimizeopt.OptValue(None)) + lst6 = virt1._get_field_descr_list() + assert lst6 is lst3 def test_reuse_vinfo(): class FakeVInfo(object): @@ -88,7 +99,26 @@ assert vinfo3 is not vinfo2 vinfo4 = v1.make_virtual_info(None, [1, 2, 6]) assert vinfo3 is vinfo4 - + +def test_descrlist_dict(): + from pypy.jit.metainterp import optimizeutil + h1 = optimizeutil.descrlist_hash([]) + h2 = optimizeutil.descrlist_hash([LLtypeMixin.valuedescr]) + h3 = optimizeutil.descrlist_hash( + [LLtypeMixin.valuedescr, LLtypeMixin.nextdescr]) + assert h1 != h2 + assert h2 != h3 + assert optimizeutil.descrlist_eq([], []) + assert not optimizeutil.descrlist_eq([], [LLtypeMixin.valuedescr]) + assert optimizeutil.descrlist_eq([LLtypeMixin.valuedescr], + [LLtypeMixin.valuedescr]) + assert not optimizeutil.descrlist_eq([LLtypeMixin.valuedescr], + [LLtypeMixin.nextdescr]) + assert optimizeutil.descrlist_eq([LLtypeMixin.valuedescr, LLtypeMixin.nextdescr], + [LLtypeMixin.valuedescr, LLtypeMixin.nextdescr]) + assert not optimizeutil.descrlist_eq([LLtypeMixin.nextdescr, LLtypeMixin.valuedescr], + [LLtypeMixin.valuedescr, LLtypeMixin.nextdescr]) + # ____________________________________________________________ Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Tue Dec 1 14:26:56 2009 @@ -361,13 +361,18 @@ assert metainterp.framestack == fs2 +class FakeOptimizer_VirtualValue(object): + class cpu: + pass +fakeoptimizer = FakeOptimizer_VirtualValue() + def virtual_value(keybox, value, next): - vv = VirtualValue(None, ConstAddr(LLtypeMixin.node_vtable_adr, - LLtypeMixin.cpu), keybox) + vv = VirtualValue(fakeoptimizer, ConstAddr(LLtypeMixin.node_vtable_adr, + LLtypeMixin.cpu), keybox) if not isinstance(next, OptValue): next = OptValue(next) - vv.setfield(LLtypeMixin.nextdescr, next) vv.setfield(LLtypeMixin.valuedescr, OptValue(value)) + vv.setfield(LLtypeMixin.nextdescr, next) return vv def test_rebuild_from_resumedata_two_guards_w_virtuals(): @@ -800,18 +805,19 @@ modifier.liveboxes = {} modifier.vfieldboxes = {} - v2 = VirtualValue(None, ConstAddr(LLtypeMixin.node_vtable_adr, - LLtypeMixin.cpu), b2s) - v2._fields = {LLtypeMixin.nextdescr: b4s, - LLtypeMixin.valuedescr: c1s} - v2._cached_sorted_fields = [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr] - v4 = VirtualValue(None, ConstAddr(LLtypeMixin.node_vtable_adr2, + v4 = VirtualValue(fakeoptimizer, ConstAddr(LLtypeMixin.node_vtable_adr2, LLtypeMixin.cpu), b4s) - v4._fields = {LLtypeMixin.nextdescr: b2s, - LLtypeMixin.valuedescr: b3s, - LLtypeMixin.otherdescr: b5s} + v4.setfield(LLtypeMixin.nextdescr, OptValue(b2s)) + v4.setfield(LLtypeMixin.valuedescr, OptValue(b3s)) + v4.setfield(LLtypeMixin.otherdescr, OptValue(b5s)) v4._cached_sorted_fields = [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr, LLtypeMixin.otherdescr] + v2 = VirtualValue(fakeoptimizer, ConstAddr(LLtypeMixin.node_vtable_adr, + LLtypeMixin.cpu), b2s) + v2.setfield(LLtypeMixin.nextdescr, v4) + v2.setfield(LLtypeMixin.valuedescr, OptValue(c1s)) + v2._cached_sorted_fields = [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr] + modifier.register_virtual_fields(b2s, [b4s, c1s]) modifier.register_virtual_fields(b4s, [b2s, b3s, b5s]) values = {b2s: v2, b4s: v4} @@ -873,6 +879,8 @@ modifier.vfieldboxes = {} class FakeOptimizer(object): + class cpu: + pass def new_const_item(self, descr): return None v2 = VArrayValue(FakeOptimizer(), LLtypeMixin.arraydescr, 2, b2s) @@ -921,9 +929,9 @@ modifier.liveboxes_from_env = {} modifier.liveboxes = {} modifier.vfieldboxes = {} - v2 = VStructValue(None, LLtypeMixin.ssize, b2s) - v2._fields = {LLtypeMixin.adescr: c1s, LLtypeMixin.bdescr: b4s} - v2._cached_sorted_fields = [LLtypeMixin.adescr, LLtypeMixin.bdescr] + v2 = VStructValue(fakeoptimizer, LLtypeMixin.ssize, b2s) + v2.setfield(LLtypeMixin.adescr, OptValue(c1s)) + v2.setfield(LLtypeMixin.bdescr, OptValue(b4s)) modifier.register_virtual_fields(b2s, [c1s, b4s]) liveboxes = [] modifier._number_virtuals(liveboxes, {b2s: v2}, 0) From pedronis at codespeak.net Tue Dec 1 14:28:40 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 1 Dec 2009 14:28:40 +0100 (CET) Subject: [pypy-svn] r69800 - in pypy/build/testrunner: . test Message-ID: <20091201132840.C9ABA1680C1@codespeak.net> Author: pedronis Date: Tue Dec 1 14:28:40 2009 New Revision: 69800 Modified: pypy/build/testrunner/runner.py pypy/build/testrunner/test/test_runner.py Log: tentative fix Modified: pypy/build/testrunner/runner.py ============================================================================== --- pypy/build/testrunner/runner.py (original) +++ pypy/build/testrunner/runner.py Tue Dec 1 14:28:40 2009 @@ -88,12 +88,16 @@ return 'signal %d' % (n,) def execute_test(cwd, test, out, logfname, interp, test_driver, - do_dry_run=False, timeout=None): + do_dry_run=False, timeout=None, + _win32=(sys.platform=='win32')): args = interp + test_driver args += ['--resultlog=%s' % logfname, test] args = map(str, args) - args[0] = os.path.join(str(cwd), args[0]) + interp0 = args[0] + if (_win32 and not os.path.isabs(interp0) and + ('\\' in interp0 or '/' in interp0)): + args[0] = os.path.join(str(cwd), interp0) if do_dry_run: runfunc = dry_run Modified: pypy/build/testrunner/test/test_runner.py ============================================================================== --- pypy/build/testrunner/test/test_runner.py (original) +++ pypy/build/testrunner/test/test_runner.py Tue Dec 1 14:28:40 2009 @@ -72,14 +72,29 @@ test_driver=['driver', 'darg'], timeout='secs') - expected = ['/wd' + os.sep + 'INTERP', 'IARG', + expected = ['INTERP', 'IARG', + 'driver', 'darg', + '--resultlog=LOGFILE', + 'test_one'] + + assert self.called == (expected, '/wd', 'out', 'secs') + assert res == 0 + + def test_explicit_win32(self): + res = runner.execute_test('/wd', 'test_one', 'out', 'LOGFILE', + interp=['./INTERP', 'IARG'], + test_driver=['driver', 'darg'], + timeout='secs', + _win32=True + ) + + expected = ['/wd' + os.sep + './INTERP', 'IARG', 'driver', 'darg', '--resultlog=LOGFILE', 'test_one'] assert self.called == (expected, '/wd', 'out', 'secs') assert res == 0 - def test_error(self): self.exitcode[:] = [1] From pedronis at codespeak.net Tue Dec 1 14:35:49 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 1 Dec 2009 14:35:49 +0100 (CET) Subject: [pypy-svn] r69801 - in pypy/branch/esp-params/pypy/jit/backend: llsupport llsupport/test x86 x86/test Message-ID: <20091201133549.407AA1680C1@codespeak.net> Author: pedronis Date: Tue Dec 1 14:35:48 2009 New Revision: 69801 Modified: pypy/branch/esp-params/pypy/jit/backend/llsupport/regalloc.py pypy/branch/esp-params/pypy/jit/backend/llsupport/test/test_gc.py pypy/branch/esp-params/pypy/jit/backend/llsupport/test/test_regalloc.py pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py pypy/branch/esp-params/pypy/jit/backend/x86/jump.py pypy/branch/esp-params/pypy/jit/backend/x86/regalloc.py pypy/branch/esp-params/pypy/jit/backend/x86/test/test_assembler.py pypy/branch/esp-params/pypy/jit/backend/x86/test/test_gc_integration.py pypy/branch/esp-params/pypy/jit/backend/x86/test/test_jump.py pypy/branch/esp-params/pypy/jit/backend/x86/test/test_zrpy_gc.py Log: rename StackManager to FrameManager, and some stack_ to frame_. we are going to need both the concept of a frame_depth and of a param_depth Modified: pypy/branch/esp-params/pypy/jit/backend/llsupport/regalloc.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/llsupport/regalloc.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/llsupport/regalloc.py Tue Dec 1 14:35:48 2009 @@ -12,28 +12,28 @@ class NoVariableToSpill(Exception): pass -class StackManager(object): - """ Manage stack positions +class FrameManager(object): + """ Manage frame positions """ def __init__(self): - self.stack_bindings = {} - self.stack_depth = 0 + self.frame_bindings = {} + self.frame_depth = 0 def get(self, box): - return self.stack_bindings.get(box, None) + return self.frame_bindings.get(box, None) def loc(self, box, size): res = self.get(box) if res is not None: return res - newloc = self.stack_pos(self.stack_depth, size) - self.stack_bindings[box] = newloc - self.stack_depth += size + newloc = self.frame_pos(self.frame_depth, size) + self.frame_bindings[box] = newloc + self.frame_depth += size return newloc # abstract methods that need to be overwritten for specific assemblers @staticmethod - def stack_pos(loc, size): + def frame_pos(loc, size): raise NotImplementedError("Purely abstract") class RegisterManager(object): @@ -45,12 +45,12 @@ save_around_call_regs = [] reg_width = 1 # in terms of stack space eaten - def __init__(self, longevity, stack_manager=None, assembler=None): + def __init__(self, longevity, frame_manager=None, assembler=None): self.free_regs = self.all_regs[:] self.longevity = longevity self.reg_bindings = {} self.position = -1 - self.stack_manager = stack_manager + self.frame_manager = frame_manager self.assembler = assembler def stays_alive(self, v): @@ -147,8 +147,8 @@ selected_reg, need_lower_byte=need_lower_byte) loc = self.reg_bindings[v_to_spill] del self.reg_bindings[v_to_spill] - if self.stack_manager.get(v_to_spill) is None: - newloc = self.stack_manager.loc(v_to_spill, self.reg_width) + if self.frame_manager.get(v_to_spill) is None: + newloc = self.frame_manager.loc(v_to_spill, self.reg_width) self.assembler.regalloc_mov(loc, newloc) return loc @@ -204,7 +204,7 @@ try: return self.reg_bindings[box] except KeyError: - return self.stack_manager.loc(box, self.reg_width) + return self.frame_manager.loc(box, self.reg_width) def return_constant(self, v, forbidden_vars=[], selected_reg=None, imm_fine=True): @@ -260,7 +260,7 @@ self.reg_bindings[v] = loc self.assembler.regalloc_mov(prev_loc, loc) else: - loc = self.stack_manager.loc(v, self.reg_width) + loc = self.frame_manager.loc(v, self.reg_width) self.assembler.regalloc_mov(prev_loc, loc) def force_result_in_reg(self, result_v, v, forbidden_vars=[]): @@ -280,7 +280,7 @@ self.free_regs = [reg for reg in self.free_regs if reg is not loc] return loc if v not in self.reg_bindings: - prev_loc = self.stack_manager.loc(v, self.reg_width) + prev_loc = self.frame_manager.loc(v, self.reg_width) loc = self.force_allocate_reg(v, forbidden_vars) self.assembler.regalloc_mov(prev_loc, loc) assert v in self.reg_bindings @@ -289,7 +289,7 @@ # store result in the same place loc = self.reg_bindings[v] del self.reg_bindings[v] - if self.stack_manager.get(v) is None: + if self.frame_manager.get(v) is None: self._move_variable_away(v, loc) self.reg_bindings[result_v] = loc else: @@ -298,9 +298,9 @@ return loc def _sync_var(self, v): - if not self.stack_manager.get(v): + if not self.frame_manager.get(v): reg = self.reg_bindings[v] - to = self.stack_manager.loc(v, self.reg_width) + to = self.frame_manager.loc(v, self.reg_width) self.assembler.regalloc_mov(reg, to) # otherwise it's clean Modified: pypy/branch/esp-params/pypy/jit/backend/llsupport/test/test_gc.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/llsupport/test/test_gc.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/llsupport/test/test_gc.py Tue Dec 1 14:35:48 2009 @@ -61,11 +61,11 @@ assert addrs[i].address[0] == llmemory.cast_ptr_to_adr(allocs[i]) def test_GcRootMap_asmgcc(): - def stack_pos(n): + def frame_pos(n): return -4*(4+n) gcrootmap = GcRootMap_asmgcc() - num1 = stack_pos(1) - num2 = stack_pos(55) + num1 = frame_pos(1) + num2 = frame_pos(55) shape = gcrootmap.get_basic_shape() gcrootmap.add_ebp_offset(shape, num1) gcrootmap.add_ebp_offset(shape, num2) @@ -99,7 +99,7 @@ expected_shapeaddr = {} for i in range(1, 600): shape = gcrootmap.get_basic_shape() - gcrootmap.add_ebp_offset(shape, stack_pos(i)) + gcrootmap.add_ebp_offset(shape, frame_pos(i)) shapeaddr = gcrootmap.compress_callshape(shape) expected_shapeaddr[i] = shapeaddr retaddr = rffi.cast(llmemory.Address, 123456789 + i) Modified: pypy/branch/esp-params/pypy/jit/backend/llsupport/test/test_regalloc.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/llsupport/test/test_regalloc.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/llsupport/test/test_regalloc.py Tue Dec 1 14:35:48 2009 @@ -1,6 +1,6 @@ from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat -from pypy.jit.backend.llsupport.regalloc import StackManager +from pypy.jit.backend.llsupport.regalloc import FrameManager from pypy.jit.backend.llsupport.regalloc import RegisterManager as BaseRegMan def newboxes(*values): @@ -26,8 +26,8 @@ def convert_to_imm(self, v): return v -class TStackManager(StackManager): - def stack_pos(self, i, size): +class TFrameManager(FrameManager): + def frame_pos(self, i, size): return i class MockAsm(object): @@ -110,13 +110,13 @@ def test_force_allocate_reg(self): boxes, longevity = boxes_and_longevity(5) b0, b1, b2, b3, b4 = boxes - sm = TStackManager() + fm = TFrameManager() class XRegisterManager(RegisterManager): no_lower_byte_regs = [r2, r3] rm = XRegisterManager(longevity, - stack_manager=sm, + frame_manager=fm, assembler=MockAsm()) rm.next_instruction() loc = rm.force_allocate_reg(b0) @@ -140,13 +140,13 @@ def test_make_sure_var_in_reg(self): boxes, longevity = boxes_and_longevity(5) - sm = TStackManager() - rm = RegisterManager(longevity, stack_manager=sm, + fm = TFrameManager() + rm = RegisterManager(longevity, frame_manager=fm, assembler=MockAsm()) rm.next_instruction() # allocate a stack position b0, b1, b2, b3, b4 = boxes - sp = sm.loc(b0, 1) + sp = fm.loc(b0, 1) assert sp == 0 loc = rm.make_sure_var_in_reg(b0) assert isinstance(loc, FakeReg) @@ -155,9 +155,9 @@ def test_force_result_in_reg_1(self): b0, b1 = newboxes(0, 0) longevity = {b0: (0, 1), b1: (1, 3)} - sm = TStackManager() + fm = TFrameManager() asm = MockAsm() - rm = RegisterManager(longevity, stack_manager=sm, assembler=asm) + rm = RegisterManager(longevity, frame_manager=fm, assembler=asm) rm.next_instruction() # first path, var is already in reg and dies loc0 = rm.force_allocate_reg(b0) @@ -171,9 +171,9 @@ def test_force_result_in_reg_2(self): b0, b1 = newboxes(0, 0) longevity = {b0: (0, 2), b1: (1, 3)} - sm = TStackManager() + fm = TFrameManager() asm = MockAsm() - rm = RegisterManager(longevity, stack_manager=sm, assembler=asm) + rm = RegisterManager(longevity, frame_manager=fm, assembler=asm) rm.next_instruction() loc0 = rm.force_allocate_reg(b0) rm._check_invariants() @@ -187,9 +187,9 @@ def test_force_result_in_reg_3(self): b0, b1, b2, b3, b4 = newboxes(0, 0, 0, 0, 0) longevity = {b0: (0, 2), b1: (0, 2), b3: (0, 2), b2: (0, 2), b4: (1, 3)} - sm = TStackManager() + fm = TFrameManager() asm = MockAsm() - rm = RegisterManager(longevity, stack_manager=sm, assembler=asm) + rm = RegisterManager(longevity, frame_manager=fm, assembler=asm) rm.next_instruction() for b in b0, b1, b2, b3: rm.force_allocate_reg(b) @@ -203,11 +203,11 @@ def test_force_result_in_reg_4(self): b0, b1 = newboxes(0, 0) longevity = {b0: (0, 1), b1: (0, 1)} - sm = TStackManager() + fm = TFrameManager() asm = MockAsm() - rm = RegisterManager(longevity, stack_manager=sm, assembler=asm) + rm = RegisterManager(longevity, frame_manager=fm, assembler=asm) rm.next_instruction() - sm.loc(b0, 1) + fm.loc(b0, 1) rm.force_result_in_reg(b1, b0) rm._check_invariants() loc = rm.loc(b1) @@ -219,9 +219,9 @@ def test_return_constant(self): asm = MockAsm() boxes, longevity = boxes_and_longevity(5) - sm = TStackManager() + fm = TFrameManager() rm = RegisterManager(longevity, assembler=asm, - stack_manager=sm) + frame_manager=fm) rm.next_instruction() loc = rm.return_constant(ConstInt(0), imm_fine=False) assert isinstance(loc, FakeReg) @@ -241,9 +241,9 @@ def test_force_result_in_reg_const(self): boxes, longevity = boxes_and_longevity(2) - sm = TStackManager() + fm = TFrameManager() asm = MockAsm() - rm = RegisterManager(longevity, stack_manager=sm, + rm = RegisterManager(longevity, frame_manager=fm, assembler=asm) rm.next_instruction() c = ConstInt(0) @@ -262,16 +262,16 @@ def call_result_location(self, v): return r1 - sm = TStackManager() + fm = TFrameManager() asm = MockAsm() boxes, longevity = boxes_and_longevity(5) - rm = XRegisterManager(longevity, stack_manager=sm, + rm = XRegisterManager(longevity, frame_manager=fm, assembler=asm) for b in boxes[:-1]: rm.force_allocate_reg(b) rm.before_call() assert len(rm.reg_bindings) == 2 - assert sm.stack_depth == 2 + assert fm.frame_depth == 2 assert len(asm.moves) == 2 rm._check_invariants() rm.after_call(boxes[-1]) @@ -285,16 +285,16 @@ def call_result_location(self, v): return r1 - sm = TStackManager() + fm = TFrameManager() asm = MockAsm() boxes, longevity = boxes_and_longevity(5) - rm = XRegisterManager(longevity, stack_manager=sm, + rm = XRegisterManager(longevity, frame_manager=fm, assembler=asm) for b in boxes[:-1]: rm.force_allocate_reg(b) rm.before_call(save_all_regs=True) assert len(rm.reg_bindings) == 0 - assert sm.stack_depth == 4 + assert fm.frame_depth == 4 assert len(asm.moves) == 4 rm._check_invariants() rm.after_call(boxes[-1]) @@ -302,20 +302,20 @@ rm._check_invariants() - def test_different_stack_width(self): + def test_different_frame_width(self): class XRegisterManager(RegisterManager): reg_width = 2 - sm = TStackManager() + fm = TFrameManager() b0 = BoxInt() longevity = {b0: (0, 1)} asm = MockAsm() - rm = RegisterManager(longevity, stack_manager=sm, assembler=asm) + rm = RegisterManager(longevity, frame_manager=fm, assembler=asm) f0 = BoxFloat() longevity = {f0: (0, 1)} - xrm = XRegisterManager(longevity, stack_manager=sm, assembler=asm) + xrm = XRegisterManager(longevity, frame_manager=fm, assembler=asm) xrm.loc(f0) rm.loc(b0) - assert sm.stack_depth == 3 + assert fm.frame_depth == 3 Modified: pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py Tue Dec 1 14:35:48 2009 @@ -184,7 +184,7 @@ self.mc2.done() if we_are_translated() or self.cpu.dont_keepalive_stuff: self._regalloc = None # else keep it around for debugging - stack_depth = regalloc.sm.stack_depth + stack_depth = regalloc.fm.frame_depth jump_target_descr = regalloc.jump_target_descr if jump_target_descr is not None: target_stack_depth = jump_target_descr._x86_stack_depth @@ -837,7 +837,7 @@ self.fail_boxes_float.get_addr_for_num(i) def rebuild_faillocs_from_descr(self, bytecode): - from pypy.jit.backend.x86.regalloc import X86StackManager + from pypy.jit.backend.x86.regalloc import X86FrameManager bytecode = rffi.cast(rffi.UCHARP, bytecode) arglocs = [] while 1: @@ -862,7 +862,7 @@ size = 2 else: size = 1 - loc = X86StackManager.stack_pos(code, size) + loc = X86FrameManager.frame_pos(code, size) elif code == self.DESCR_STOP: break elif code == self.DESCR_HOLE: Modified: pypy/branch/esp-params/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/x86/jump.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/x86/jump.py Tue Dec 1 14:35:48 2009 @@ -5,7 +5,7 @@ class __extend__(OPERAND): __metaclass__ = extendabletype def _getregkey(self): - raise AssertionError("should only happen to registers and stack " + raise AssertionError("should only happen to registers and frame " "positions") class __extend__(REG): @@ -19,7 +19,7 @@ return self.position -def remap_stack_layout(assembler, src_locations, dst_locations, tmpreg): +def remap_frame_layout(assembler, src_locations, dst_locations, tmpreg): pending_dests = len(dst_locations) srccount = {} # maps dst_locations to how many times the same # location appears in src_locations Modified: pypy/branch/esp-params/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/x86/regalloc.py Tue Dec 1 14:35:48 2009 @@ -11,11 +11,11 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib import rgc from pypy.jit.backend.llsupport import symbolic -from pypy.jit.backend.x86.jump import remap_stack_layout +from pypy.jit.backend.x86.jump import remap_frame_layout from pypy.jit.metainterp.resoperation import rop from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr -from pypy.jit.backend.llsupport.regalloc import StackManager, RegisterManager,\ +from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\ TempBox WORD = 4 @@ -76,8 +76,8 @@ return lltype.malloc(rffi.CArray(lltype.Float), BASE_CONSTANT_SIZE, flavor='raw') - def __init__(self, longevity, stack_manager=None, assembler=None): - RegisterManager.__init__(self, longevity, stack_manager=stack_manager, + def __init__(self, longevity, frame_manager=None, assembler=None): + RegisterManager.__init__(self, longevity, frame_manager=frame_manager, assembler=assembler) self.constant_arrays = [self.new_const_array()] self.constant_arrays[-1][0] = NEG_ZERO @@ -100,14 +100,14 @@ def after_call(self, v): # the result is stored in st0, but we don't have this around, - # so genop_call will move it to some stack location immediately + # so genop_call will move it to some frame location immediately # after the call - return self.stack_manager.loc(v, 2) + return self.frame_manager.loc(v, 2) -class X86StackManager(StackManager): +class X86FrameManager(FrameManager): @staticmethod - def stack_pos(i, size): + def frame_pos(i, size): if size == 1: res = mem(ebp, get_ebp_ofs(i)) elif size == 2: @@ -130,16 +130,16 @@ self.jump_target_descr = None def _prepare(self, inputargs, operations): - self.sm = X86StackManager() + self.fm = X86FrameManager() cpu = self.assembler.cpu cpu.gc_ll_descr.rewrite_assembler(cpu, operations) # compute longevity of variables longevity = self._compute_vars_longevity(inputargs, operations) self.longevity = longevity self.rm = X86RegisterManager(longevity, - stack_manager = self.sm, + frame_manager = self.fm, assembler = self.assembler) - self.xrm = X86XMMRegisterManager(longevity, stack_manager = self.sm, + self.xrm = X86XMMRegisterManager(longevity, frame_manager = self.fm, assembler = self.assembler) def prepare_loop(self, inputargs, operations, looptoken): @@ -153,7 +153,7 @@ self._prepare(inputargs, operations) self.loop_consts = {} self._update_bindings(arglocs, inputargs) - self.sm.stack_depth = prev_stack_depth + self.fm.frame_depth = prev_stack_depth def _process_inputargs(self, inputargs): # XXX we can sort out here by longevity if we need something @@ -181,7 +181,7 @@ if reg: loc = reg else: - loc = self.sm.loc(arg, width_of_type[arg.type]) + loc = self.fm.loc(arg, width_of_type[arg.type]) if arg.type == FLOAT: floatlocs[i] = loc else: @@ -252,13 +252,13 @@ self.xrm.reg_bindings[arg] = loc used[loc] = None else: - self.sm.stack_bindings[arg] = loc + self.fm.frame_bindings[arg] = loc else: if isinstance(loc, REG): self.rm.reg_bindings[arg] = loc used[loc] = None else: - self.sm.stack_bindings[arg] = loc + self.fm.frame_bindings[arg] = loc self.rm.free_regs = [] for reg in X86RegisterManager.all_regs: if reg not in used: @@ -287,7 +287,7 @@ self.xrm.position += 1 self.assembler.regalloc_perform_with_guard(op, guard_op, faillocs, arglocs, result_loc, - self.sm.stack_depth) + self.fm.frame_depth) if op.result is not None: self.possibly_free_var(op.result) self.possibly_free_vars(guard_op.fail_args) @@ -302,7 +302,7 @@ self.assembler.dump('%s(%s)' % (guard_op, arglocs)) self.assembler.regalloc_perform_guard(guard_op, faillocs, arglocs, result_loc, - self.sm.stack_depth) + self.fm.frame_depth) self.possibly_free_vars(guard_op.fail_args) def PerformDiscard(self, op, arglocs): @@ -914,11 +914,11 @@ src_locations = [self.loc(arg) for arg in op.args if arg.type != FLOAT] assert tmploc not in nonfloatlocs dst_locations = [loc for loc in nonfloatlocs if loc is not None] - remap_stack_layout(assembler, src_locations, dst_locations, tmploc) + remap_frame_layout(assembler, src_locations, dst_locations, tmploc) # Part about floats src_locations = [self.loc(arg) for arg in op.args if arg.type == FLOAT] dst_locations = [loc for loc in floatlocs if loc is not None] - remap_stack_layout(assembler, src_locations, dst_locations, xmmtmp) + remap_frame_layout(assembler, src_locations, dst_locations, xmmtmp) self.rm.possibly_free_var(box) self.xrm.possibly_free_var(box1) self.possibly_free_vars(op.args) @@ -929,7 +929,7 @@ def get_mark_gc_roots(self, gcrootmap): shape = gcrootmap.get_basic_shape() - for v, val in self.sm.stack_bindings.items(): + for v, val in self.fm.frame_bindings.items(): if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): assert isinstance(val, MODRM) gcrootmap.add_ebp_offset(shape, get_ebp_ofs(val.position)) @@ -963,7 +963,7 @@ oplist[num] = value def get_ebp_ofs(position): - # Argument is a stack position (0, 1, 2...). + # Argument is a frame position (0, 1, 2...). # Returns (ebp-20), (ebp-24), (ebp-28)... # i.e. the n'th word beyond the fixed frame size. return -WORD * (FRAME_FIXED_SIZE + position) Modified: pypy/branch/esp-params/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/x86/test/test_assembler.py Tue Dec 1 14:35:48 2009 @@ -1,6 +1,6 @@ from pypy.jit.backend.x86.ri386 import * from pypy.jit.backend.x86.assembler import Assembler386 -from pypy.jit.backend.x86.regalloc import X86StackManager, get_ebp_ofs +from pypy.jit.backend.x86.regalloc import X86FrameManager, get_ebp_ofs from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat from pypy.rlib.rarithmetic import intmask from pypy.rpython.lltypesystem import lltype, llmemory, rffi @@ -23,12 +23,12 @@ failargs = [BoxInt(), BoxPtr(), BoxFloat()] * 3 failargs.insert(6, None) failargs.insert(7, None) - locs = [X86StackManager.stack_pos(0, 1), - X86StackManager.stack_pos(1, 1), - X86StackManager.stack_pos(10, 2), - X86StackManager.stack_pos(100, 1), - X86StackManager.stack_pos(101, 1), - X86StackManager.stack_pos(110, 2), + locs = [X86FrameManager.frame_pos(0, 1), + X86FrameManager.frame_pos(1, 1), + X86FrameManager.frame_pos(10, 2), + X86FrameManager.frame_pos(100, 1), + X86FrameManager.frame_pos(101, 1), + X86FrameManager.frame_pos(110, 2), None, None, ebx, Modified: pypy/branch/esp-params/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/x86/test/test_gc_integration.py Tue Dec 1 14:35:48 2009 @@ -19,7 +19,7 @@ from pypy.jit.backend.x86.test.test_regalloc import MockAssembler from pypy.jit.backend.x86.test.test_regalloc import BaseTestRegalloc -from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86StackManager,\ +from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86FrameManager,\ X86XMMRegisterManager from pypy.rpython.annlowlevel import llhelper @@ -64,10 +64,10 @@ longevity = {} for box in boxes: longevity[box] = (0, 1) - regalloc.sm = X86StackManager() - regalloc.rm = X86RegisterManager(longevity, regalloc.sm, + regalloc.fm = X86FrameManager() + regalloc.rm = X86RegisterManager(longevity, regalloc.fm, assembler=regalloc.assembler) - regalloc.xrm = X86XMMRegisterManager(longevity, regalloc.sm, + regalloc.xrm = X86XMMRegisterManager(longevity, regalloc.fm, assembler=regalloc.assembler) cpu = regalloc.assembler.cpu for box in boxes: Modified: pypy/branch/esp-params/pypy/jit/backend/x86/test/test_jump.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/x86/test/test_jump.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/x86/test/test_jump.py Tue Dec 1 14:35:48 2009 @@ -1,8 +1,8 @@ from pypy.jit.backend.x86.ri386 import * -from pypy.jit.backend.x86.regalloc import X86StackManager -from pypy.jit.backend.x86.jump import remap_stack_layout +from pypy.jit.backend.x86.regalloc import X86FrameManager +from pypy.jit.backend.x86.jump import remap_frame_layout -stack_pos = X86StackManager.stack_pos +frame_pos = X86FrameManager.frame_pos class MockAssembler: def __init__(self): @@ -36,33 +36,33 @@ def test_trivial(): assembler = MockAssembler() - remap_stack_layout(assembler, [], [], '?') + remap_frame_layout(assembler, [], [], '?') assert assembler.ops == [] - remap_stack_layout(assembler, [eax, ebx, ecx, edx, esi, edi], + remap_frame_layout(assembler, [eax, ebx, ecx, edx, esi, edi], [eax, ebx, ecx, edx, esi, edi], '?') assert assembler.ops == [] - s8 = stack_pos(1, 1) - s12 = stack_pos(31, 1) - s20 = stack_pos(6, 1) - remap_stack_layout(assembler, [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], + s8 = frame_pos(1, 1) + s12 = frame_pos(31, 1) + s20 = frame_pos(6, 1) + remap_frame_layout(assembler, [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], '?') assert assembler.ops == [] def test_simple_registers(): assembler = MockAssembler() - remap_stack_layout(assembler, [eax, ebx, ecx], [edx, esi, edi], '?') + remap_frame_layout(assembler, [eax, ebx, ecx], [edx, esi, edi], '?') assert assembler.ops == [('mov', eax, edx), ('mov', ebx, esi), ('mov', ecx, edi)] -def test_simple_stacklocs(): +def test_simple_framelocs(): assembler = MockAssembler() - s8 = stack_pos(0, 1) - s12 = stack_pos(13, 1) - s20 = stack_pos(20, 1) - s24 = stack_pos(221, 1) - remap_stack_layout(assembler, [s8, eax, s12], [s20, s24, edi], edx) + s8 = frame_pos(0, 1) + s12 = frame_pos(13, 1) + s20 = frame_pos(20, 1) + s24 = frame_pos(221, 1) + remap_frame_layout(assembler, [s8, eax, s12], [s20, s24, edi], edx) assert assembler.ops == [('mov', s8, edx), ('mov', edx, s20), ('mov', eax, s24), @@ -70,11 +70,11 @@ def test_reordering(): assembler = MockAssembler() - s8 = stack_pos(8, 1) - s12 = stack_pos(12, 1) - s20 = stack_pos(19, 1) - s24 = stack_pos(1, 1) - remap_stack_layout(assembler, [eax, s8, s20, ebx], + s8 = frame_pos(8, 1) + s12 = frame_pos(12, 1) + s20 = frame_pos(19, 1) + s24 = frame_pos(1, 1) + remap_frame_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, edi], '?') assert assembler.got([('mov', ebx, edi), ('mov', s8, ebx), @@ -83,11 +83,11 @@ def test_cycle(): assembler = MockAssembler() - s8 = stack_pos(8, 1) - s12 = stack_pos(12, 1) - s20 = stack_pos(19, 1) - s24 = stack_pos(1, 1) - remap_stack_layout(assembler, [eax, s8, s20, ebx], + s8 = frame_pos(8, 1) + s12 = frame_pos(12, 1) + s20 = frame_pos(19, 1) + s24 = frame_pos(1, 1) + remap_frame_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, s20], '?') assert assembler.got([('push', s8), ('mov', eax, s8), @@ -97,13 +97,13 @@ def test_cycle_2(): assembler = MockAssembler() - s8 = stack_pos(8, 1) - s12 = stack_pos(12, 1) - s20 = stack_pos(19, 1) - s24 = stack_pos(1, 1) - s2 = stack_pos(2, 1) - s3 = stack_pos(3, 1) - remap_stack_layout(assembler, + s8 = frame_pos(8, 1) + s12 = frame_pos(12, 1) + s20 = frame_pos(19, 1) + s24 = frame_pos(1, 1) + s2 = frame_pos(2, 1) + s3 = frame_pos(3, 1) + remap_frame_layout(assembler, [eax, s8, edi, s20, eax, s20, s24, esi, s2, s3], [s8, s20, edi, eax, edx, s24, ebx, s12, s3, s2], ecx) @@ -124,18 +124,18 @@ def test_constants(): assembler = MockAssembler() c3 = imm(3) - remap_stack_layout(assembler, [c3], [eax], '?') + remap_frame_layout(assembler, [c3], [eax], '?') assert assembler.ops == [('mov', c3, eax)] assembler = MockAssembler() - s12 = stack_pos(12, 1) - remap_stack_layout(assembler, [c3], [s12], '?') + s12 = frame_pos(12, 1) + remap_frame_layout(assembler, [c3], [s12], '?') assert assembler.ops == [('mov', c3, s12)] def test_constants_and_cycle(): assembler = MockAssembler() c3 = imm(3) - s12 = stack_pos(13, 1) - remap_stack_layout(assembler, [ebx, c3, s12], + s12 = frame_pos(13, 1) + remap_frame_layout(assembler, [ebx, c3, s12], [s12, eax, ebx], edi) assert assembler.ops == [('mov', c3, eax), ('push', s12), Modified: pypy/branch/esp-params/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/x86/test/test_zrpy_gc.py Tue Dec 1 14:35:48 2009 @@ -14,11 +14,8 @@ from pypy.rlib.jit import JitDriver, OPTIMIZER_SIMPLE, dont_look_inside from pypy.jit.backend.x86.runner import CPU386 from pypy.jit.backend.llsupport.gc import GcRefList, GcRootMap_asmgcc -from pypy.jit.backend.x86.regalloc import X86StackManager from pypy.tool.udir import udir -stack_pos = X86StackManager.stack_pos - class X(object): def __init__(self, x=0): self.x = x From afa at codespeak.net Tue Dec 1 14:53:08 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 1 Dec 2009 14:53:08 +0100 (CET) Subject: [pypy-svn] r69802 - in pypy/trunk/pypy/module/_locale: . test Message-ID: <20091201135308.872BC1680C1@codespeak.net> Author: afa Date: Tue Dec 1 14:53:07 2009 New Revision: 69802 Modified: pypy/trunk/pypy/module/_locale/interp_locale.py pypy/trunk/pypy/module/_locale/test/test_locale.py Log: Test and fix the _getdefaultlocale() implementation on win32. Modified: pypy/trunk/pypy/module/_locale/interp_locale.py ============================================================================== --- pypy/trunk/pypy/module/_locale/interp_locale.py (original) +++ pypy/trunk/pypy/module/_locale/interp_locale.py Tue Dec 1 14:53:07 2009 @@ -431,11 +431,11 @@ buf_country = lltype.malloc(rffi.CCHARP.TO, BUFSIZE, flavor='raw') try: - if (GetLocaleInfo(LOCALE_USER_DEFAULT, - LOCALE_SISO639LANGNAME, + if (GetLocaleInfo(cConfig.LOCALE_USER_DEFAULT, + cConfig.LOCALE_SISO639LANGNAME, buf_lang, BUFSIZE) and - GetLocaleInfo(LOCALE_USER_DEFAULT, - LOCALE_SISO3166CTRYNAME, + GetLocaleInfo(cConfig.LOCALE_USER_DEFAULT, + cConfig.LOCALE_SISO3166CTRYNAME, buf_country, BUFSIZE)): lang = rffi.charp2str(buf_lang) country = rffi.charp2str(buf_country) @@ -445,7 +445,8 @@ # If we end up here, this windows version didn't know about # ISO639/ISO3166 names (it's probably Windows 95). Return the # Windows language identifier instead (a hexadecimal number) - elif GetLocaleInfo(LOCALE_USER_DEFAULT, LOCALE_IDEFAULTLANGUAGE, + elif GetLocaleInfo(cConfig.LOCALE_USER_DEFAULT, + cConfig.LOCALE_IDEFAULTLANGUAGE, buf_lang, BUFSIZE): lang = rffi.charp2str(buf_lang) return space.newtuple([space.wrap("0x%s" % lang), Modified: pypy/trunk/pypy/module/_locale/test/test_locale.py ============================================================================== --- pypy/trunk/pypy/module/_locale/test/test_locale.py (original) +++ pypy/trunk/pypy/module/_locale/test/test_locale.py Tue Dec 1 14:53:07 2009 @@ -309,3 +309,14 @@ assert _locale.bind_textdomain_codeset('/', None) == 'UTF-8' assert _locale.bind_textdomain_codeset('', '') is None + + def test_getdefaultlocale(self): + import sys + if sys.platform != 'win32': + skip("No _getdefaultlocale() to test") + + import _locale + lang, encoding = _locale._getdefaultlocale() + assert lang is None or isinstance(lang, str) + assert encoding.startswith('cp') + From arigo at codespeak.net Tue Dec 1 15:20:17 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 1 Dec 2009 15:20:17 +0100 (CET) Subject: [pypy-svn] r69803 - in pypy/trunk/pypy/jit: backend/llgraph metainterp Message-ID: <20091201142017.864671680AB@codespeak.net> Author: arigo Date: Tue Dec 1 15:20:15 2009 New Revision: 69803 Modified: pypy/trunk/pypy/jit/backend/llgraph/runner.py pypy/trunk/pypy/jit/metainterp/optimizefindnode.py Log: Document sort_key(). Kill an unused method. Modified: pypy/trunk/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/runner.py Tue Dec 1 15:20:15 2009 @@ -32,6 +32,10 @@ return self.extrainfo def sort_key(self): + """Returns an integer that can be used as a key when sorting the + field descrs of a single structure. The property that this + number has is simply that two different field descrs of the same + structure give different numbers.""" return self.ofs def is_pointer_field(self): @@ -46,11 +50,6 @@ def is_array_of_floats(self): return self.typeinfo == FLOAT - def equals(self, other): - if not isinstance(other, Descr): - return False - return self.sort_key() == other.sort_key() - def __lt__(self, other): raise TypeError("cannot use comparison on Descrs") def __le__(self, other): Modified: pypy/trunk/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizefindnode.py Tue Dec 1 15:20:15 2009 @@ -371,6 +371,7 @@ d.setdefault(ofs, self.node_escaped) if d is not None: lst = d.keys() + # we always use the "standardized" order of fields sort_descrs(lst) for ofs in lst: try: From arigo at codespeak.net Tue Dec 1 15:38:11 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 1 Dec 2009 15:38:11 +0100 (CET) Subject: [pypy-svn] r69804 - in pypy/trunk/pypy/jit: backend metainterp Message-ID: <20091201143811.4F25D1680C1@codespeak.net> Author: arigo Date: Tue Dec 1 15:38:10 2009 New Revision: 69804 Modified: pypy/trunk/pypy/jit/backend/model.py pypy/trunk/pypy/jit/metainterp/specnode.py Log: We can write "o1 is o2" instead of playing the sort_key() given a warning big enough in fielddescrof(). Modified: pypy/trunk/pypy/jit/backend/model.py ============================================================================== --- pypy/trunk/pypy/jit/backend/model.py (original) +++ pypy/trunk/pypy/jit/backend/model.py Tue Dec 1 15:38:10 2009 @@ -108,6 +108,9 @@ @staticmethod def fielddescrof(S, fieldname): + """Return the Descr corresponding to field 'fieldname' on the + structure 'S'. It is important that this function (at least) + caches the results.""" raise NotImplementedError @staticmethod Modified: pypy/trunk/pypy/jit/metainterp/specnode.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/specnode.py (original) +++ pypy/trunk/pypy/jit/metainterp/specnode.py Tue Dec 1 15:38:10 2009 @@ -49,7 +49,7 @@ for i in range(len(self.fields)): o1, s1 = self.fields[i] o2, s2 = other.fields[i] - if not (o1.sort_key() == o2.sort_key() and s1.equals(s2, ge)): + if not (o1 is o2 and s1.equals(s2, ge)): return False return True From pedronis at codespeak.net Tue Dec 1 15:42:37 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 1 Dec 2009 15:42:37 +0100 (CET) Subject: [pypy-svn] r69805 - in pypy/branch/esp-params/pypy/jit/backend/x86: . test Message-ID: <20091201144237.290811680C1@codespeak.net> Author: pedronis Date: Tue Dec 1 15:42:36 2009 New Revision: 69805 Modified: pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py pypy/branch/esp-params/pypy/jit/backend/x86/regalloc.py pypy/branch/esp-params/pypy/jit/backend/x86/test/test_recompilation.py pypy/branch/esp-params/pypy/jit/backend/x86/test/test_regalloc.py Log: basic param frame space reservation for loops, renamings, bridges not done/tested yet Modified: pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py Tue Dec 1 15:42:36 2009 @@ -134,7 +134,8 @@ """adds the following attributes to looptoken: _x86_loop_code (an integer giving an address) _x86_bootstrap_code (an integer giving an address) - _x86_stack_depth + _x86_frame_depth + _x86_param_depth _x86_arglocs """ self.make_sure_mc_exists() @@ -144,10 +145,12 @@ looptoken._x86_bootstrap_code = self.mc.tell() adr_stackadjust = self._assemble_bootstrap_code(inputargs, arglocs) looptoken._x86_loop_code = self.mc.tell() - looptoken._x86_stack_depth = -1 # temporarily - stack_depth = self._assemble(regalloc, operations) - self._patch_stackadjust(adr_stackadjust, stack_depth) - looptoken._x86_stack_depth = stack_depth + looptoken._x86_frame_depth = -1 # temporarily + looptoken._x86_param_depth = -1 # temporarily + frame_depth, param_depth = self._assemble(regalloc, operations) + self._patch_stackadjust(adr_stackadjust, frame_depth+param_depth) + looptoken._x86_frame_depth = frame_depth + looptoken._x86_param_depth = param_depth def assemble_bridge(self, faildescr, inputargs, operations): self.make_sure_mc_exists() @@ -157,16 +160,18 @@ assert ([loc.assembler() for loc in arglocs] == [loc.assembler() for loc in faildescr._x86_debug_faillocs]) regalloc = RegAlloc(self, self.cpu.translate_support_code) - fail_stack_depth = faildescr._x86_current_stack_depth - regalloc.prepare_bridge(fail_stack_depth, inputargs, arglocs, - operations) + fail_frame_depth = faildescr._x86_current_frame_depth + fail_param_depth = faildescr._x86_current_param_depth + regalloc.prepare_bridge(fail_frame_depth, inputargs, arglocs, + operations) # xxx param_depth adr_bridge = self.mc.tell() adr_stackadjust = self._patchable_stackadjust() - stack_depth = self._assemble(regalloc, operations) - self._patch_stackadjust(adr_stackadjust, stack_depth) + frame_depth, param_depth = self._assemble(regalloc, operations) + self._patch_stackadjust(adr_stackadjust, frame_depth+param_depth) if not we_are_translated(): # for the benefit of tests - faildescr._x86_bridge_stack_depth = stack_depth + faildescr._x86_bridge_frame_depth = frame_depth + faildescr._x86_bridge_param_depth = param_depth # patch the jump from original guard self.patch_jump(faildescr, adr_bridge) @@ -184,26 +189,27 @@ self.mc2.done() if we_are_translated() or self.cpu.dont_keepalive_stuff: self._regalloc = None # else keep it around for debugging - stack_depth = regalloc.fm.frame_depth + frame_depth = regalloc.fm.frame_depth + param_depth = regalloc.param_depth jump_target_descr = regalloc.jump_target_descr if jump_target_descr is not None: - target_stack_depth = jump_target_descr._x86_stack_depth - stack_depth = max(stack_depth, target_stack_depth) - return stack_depth + target_frame_depth = jump_target_descr._x86_frame_depth + frame_depth = max(frame_depth, target_frame_depth) + return frame_depth, param_depth def _patchable_stackadjust(self): # stack adjustment LEA self.mc.LEA(esp, fixedsize_ebp_ofs(0)) return self.mc.tell() - 4 - def _patch_stackadjust(self, adr_lea, stack_depth): + def _patch_stackadjust(self, adr_lea, reserved_depth): # patch stack adjustment LEA # possibly align, e.g. for Mac OS X mc = codebuf.InMemoryCodeBuilder(adr_lea, adr_lea + 4) # Compute the correct offset for the instruction LEA ESP, [EBP-4*words]. # Given that [EBP] is where we saved EBP, i.e. in the last word # of our fixed frame, then the 'words' value is: - words = (FRAME_FIXED_SIZE - 1) + stack_depth + words = (FRAME_FIXED_SIZE - 1) + reserved_depth mc.write(packimm32(-WORD * words)) mc.done() @@ -299,10 +305,11 @@ genop_discard_list[op.opnum](self, op, arglocs) def regalloc_perform_with_guard(self, op, guard_op, faillocs, - arglocs, resloc, current_stack_depth): + arglocs, resloc, current_frame_depth): faildescr = guard_op.descr assert isinstance(faildescr, AbstractFailDescr) - faildescr._x86_current_stack_depth = current_stack_depth + faildescr._x86_current_frame_depth = current_frame_depth + faildescr._x86_current_param_depth = 0 # xxx failargs = guard_op.fail_args guard_opnum = guard_op.opnum failaddr = self.implement_guard_recovery(guard_opnum, @@ -319,9 +326,9 @@ faildescr._x86_adr_jump_offset = adr_jump_offset def regalloc_perform_guard(self, guard_op, faillocs, arglocs, resloc, - current_stack_depth): + current_frame_depth): self.regalloc_perform_with_guard(None, guard_op, faillocs, arglocs, - resloc, current_stack_depth) + resloc, current_frame_depth) def load_effective_addr(self, sizereg, baseofs, scale, result): self.mc.LEA(result, addr_add(imm(0), sizereg, baseofs, scale)) @@ -1071,6 +1078,9 @@ extra_on_stack = 0 for arg in range(2, nargs + 2): extra_on_stack += round_up_to_4(arglocs[arg].width) + + self._regalloc.reserve_param(extra_on_stack//4) + #extra_on_stack = self.align_stack_for_call(extra_on_stack) self.mc.SUB(esp, imm(extra_on_stack)) if isinstance(op.args[0], Const): Modified: pypy/branch/esp-params/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/x86/regalloc.py Tue Dec 1 15:42:36 2009 @@ -131,6 +131,7 @@ def _prepare(self, inputargs, operations): self.fm = X86FrameManager() + self.param_depth = 0 cpu = self.assembler.cpu cpu.gc_ll_descr.rewrite_assembler(cpu, operations) # compute longevity of variables @@ -149,11 +150,15 @@ self.loop_consts = loop_consts return self._process_inputargs(inputargs) - def prepare_bridge(self, prev_stack_depth, inputargs, arglocs, operations): + def prepare_bridge(self, prev_frame_depth, inputargs, arglocs, operations): self._prepare(inputargs, operations) self.loop_consts = {} self._update_bindings(arglocs, inputargs) - self.fm.frame_depth = prev_stack_depth + self.fm.frame_depth = prev_frame_depth + self.param_depth = 0 # xxx + + def reserve_param(self, n): + self.param_depth = max(self.param_depth, n) def _process_inputargs(self, inputargs): # XXX we can sort out here by longevity if we need something Modified: pypy/branch/esp-params/pypy/jit/backend/x86/test/test_recompilation.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/x86/test/test_recompilation.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/x86/test/test_recompilation.py Tue Dec 1 15:42:36 2009 @@ -33,7 +33,8 @@ jump(i1) ''' loop = self.interpret(ops, [0]) - previous = loop.token._x86_stack_depth + previous = loop.token._x86_frame_depth + assert loop.token._x86_param_depth == 0 assert self.getint(0) == 20 ops = ''' [i1] @@ -48,7 +49,8 @@ ''' bridge = self.attach_bridge(ops, loop, -2) descr = loop.operations[2].descr - new = descr._x86_bridge_stack_depth + new = descr._x86_bridge_frame_depth + assert descr._x86_bridge_param_depth == 0 assert new > previous self.cpu.set_future_value_int(0, 0) fail = self.run(loop) @@ -107,8 +109,10 @@ ''' bridge = self.attach_bridge(ops, loop, 5, looptoken=loop.token) guard_op = loop.operations[5] - loop_stack_depth = loop.token._x86_stack_depth - assert guard_op.descr._x86_bridge_stack_depth > loop_stack_depth + loop_frame_depth = loop.token._x86_frame_depth + assert loop.token._x86_param_depth == 0 + assert guard_op.descr._x86_bridge_frame_depth > loop_frame_depth + assert guard_op.descr._x86_bridge_param_depth == 0 self.cpu.set_future_value_int(0, 0) self.cpu.set_future_value_int(1, 0) self.cpu.set_future_value_int(2, 0) Modified: pypy/branch/esp-params/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/x86/test/test_regalloc.py Tue Dec 1 15:42:36 2009 @@ -547,6 +547,7 @@ ''' loop = self.interpret(ops, [4, 7]) assert self.getints(1) == [5] + assert loop.token._x86_param_depth == 1 def test_two_calls(self): ops = ''' @@ -557,7 +558,8 @@ ''' loop = self.interpret(ops, [4, 7]) assert self.getints(1) == [5*7] - + assert loop.token._x86_param_depth == 2 + def test_bridge_calls_1(self): ops = ''' [i0, i1] From cfbolz at codespeak.net Tue Dec 1 15:45:12 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 Dec 2009 15:45:12 +0100 (CET) Subject: [pypy-svn] r69806 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091201144512.A29C71680AB@codespeak.net> Author: cfbolz Date: Tue Dec 1 15:45:12 2009 New Revision: 69806 Modified: pypy/trunk/pypy/jit/metainterp/optimizeutil.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: don't compare via sort_key here either, but just use the identity of the descrs Modified: pypy/trunk/pypy/jit/metainterp/optimizeutil.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeutil.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeutil.py Tue Dec 1 15:45:12 2009 @@ -1,4 +1,4 @@ -from pypy.rlib.objectmodel import r_dict +from pypy.rlib.objectmodel import r_dict, compute_identity_hash from pypy.rlib.unroll import unrolling_iterable from pypy.jit.metainterp import resoperation @@ -59,7 +59,7 @@ mult = 1000003 z = len(l) for descr in l: - y = descr.sort_key() + y = compute_identity_hash(descr) res = (res ^ y) * mult z -= 1 mult += 82520 + z + z @@ -70,7 +70,7 @@ if len(l1) != len(l2): return False for i in range(len(l1)): - if l1[i].sort_key() != l2[i].sort_key(): + if l1[i] is not l2[i]: return False return True Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Tue Dec 1 15:45:12 2009 @@ -119,6 +119,14 @@ assert not optimizeutil.descrlist_eq([LLtypeMixin.nextdescr, LLtypeMixin.valuedescr], [LLtypeMixin.valuedescr, LLtypeMixin.nextdescr]) + # descrlist_eq should compare by identity of the descrs, not by the result + # of sort_key + class FakeDescr(object): + def sort_key(self): + return 1 + + assert not optimizeutil.descrlist_eq([FakeDescr()], [FakeDescr()]) + # ____________________________________________________________ From cfbolz at codespeak.net Tue Dec 1 15:55:09 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 Dec 2009 15:55:09 +0100 (CET) Subject: [pypy-svn] r69807 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091201145509.0CC161680AB@codespeak.net> Author: cfbolz Date: Tue Dec 1 15:55:09 2009 New Revision: 69807 Modified: pypy/trunk/pypy/jit/metainterp/optimizefindnode.py pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/optimizeutil.py Log: kill av_newdict and av_newdict2, normal dicts are just fine. Use simpler hash for descrlists. Modified: pypy/trunk/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizefindnode.py Tue Dec 1 15:55:09 2009 @@ -7,7 +7,7 @@ from pypy.jit.metainterp.history import AbstractValue, ConstInt, Const from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.executor import execute_nonspec -from pypy.jit.metainterp.optimizeutil import av_newdict, _findall, sort_descrs +from pypy.jit.metainterp.optimizeutil import _findall, sort_descrs from pypy.jit.metainterp.optimizeutil import InvalidLoop # ____________________________________________________________ @@ -225,7 +225,7 @@ field = op.descr assert isinstance(field, AbstractValue) if instnode.curfields is None: - instnode.curfields = av_newdict() + instnode.curfields = {} instnode.curfields[field] = fieldnode instnode.add_escape_dependency(fieldnode) @@ -243,7 +243,7 @@ fieldnode = InstanceNode(fromstart=True) instnode.add_escape_dependency(fieldnode) if instnode.origfields is None: - instnode.origfields = av_newdict() + instnode.origfields = {} instnode.origfields[field] = fieldnode else: return # nothing to be gained from tracking the field @@ -366,7 +366,7 @@ if d is not None: d = d.copy() else: - d = av_newdict() + d = {} for ofs in orig: d.setdefault(ofs, self.node_escaped) if d is not None: @@ -448,7 +448,7 @@ def make_instance_node(self): instnode = InstanceNode() instnode.knownclsbox = self.known_class - instnode.curfields = av_newdict() + instnode.curfields = {} for ofs, subspecnode in self.fields: instnode.curfields[ofs] = subspecnode.make_instance_node() return instnode Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Tue Dec 1 15:55:09 2009 @@ -9,7 +9,7 @@ from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.specnode import VirtualArraySpecNode from pypy.jit.metainterp.specnode import VirtualStructSpecNode -from pypy.jit.metainterp.optimizeutil import av_newdict2, _findall, sort_descrs +from pypy.jit.metainterp.optimizeutil import _findall, sort_descrs from pypy.jit.metainterp.optimizeutil import descrlist_dict from pypy.jit.metainterp.optimizeutil import InvalidLoop from pypy.jit.metainterp import resume, compile @@ -186,7 +186,7 @@ def __init__(self, optimizer, keybox, source_op=None): AbstractVirtualValue.__init__(self, optimizer, keybox, source_op) - self._fields = av_newdict2() + self._fields = {} self._cached_sorted_fields = None def getfield(self, ofs, default): @@ -840,9 +840,6 @@ def __init__(self, optimizer): self.optimizer = optimizer # cached OptValues for each field descr - # NOTE: it is important that this is not a av_newdict2 dict! - # we want more precision to prevent mixing up of unrelated fields, just - # because they are at the same offset (but in a different struct type) self.cached_fields = {} # cached OptValues for each field descr Modified: pypy/trunk/pypy/jit/metainterp/optimizeutil.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeutil.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeutil.py Tue Dec 1 15:55:09 2009 @@ -10,20 +10,6 @@ # ____________________________________________________________ # Misc. utilities -def av_eq(self, other): - return self.sort_key() == other.sort_key() - -def av_hash(self): - return self.sort_key() - -def av_newdict(): - return r_dict(av_eq, av_hash) - -def av_newdict2(): - # another implementation of av_newdict(), allowing different types for - # the values... - return r_dict(av_eq, av_hash) - def _findall(Class, name_prefix): result = [] for value, name in resoperation.opname.items(): @@ -56,14 +42,9 @@ def descrlist_hash(l): res = 0x345678 - mult = 1000003 - z = len(l) for descr in l: y = compute_identity_hash(descr) - res = (res ^ y) * mult - z -= 1 - mult += 82520 + z + z - res += 97531 + res = intmask((1000003 * res) ^ y) return res def descrlist_eq(l1, l2): From pedronis at codespeak.net Tue Dec 1 16:19:33 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 1 Dec 2009 16:19:33 +0100 (CET) Subject: [pypy-svn] r69808 - in pypy/branch/esp-params/pypy/jit/backend/x86: . test Message-ID: <20091201151933.263211680C1@codespeak.net> Author: pedronis Date: Tue Dec 1 16:19:32 2009 New Revision: 69808 Modified: pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py pypy/branch/esp-params/pypy/jit/backend/x86/test/test_regalloc.py Log: work-in-progress Modified: pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py Tue Dec 1 16:19:32 2009 @@ -194,7 +194,10 @@ jump_target_descr = regalloc.jump_target_descr if jump_target_descr is not None: target_frame_depth = jump_target_descr._x86_frame_depth + target_param_depth = jump_target_descr._x86_param_depth frame_depth = max(frame_depth, target_frame_depth) + # xxx tested? + param_depth = max(param_depth, target_param_depth) return frame_depth, param_depth def _patchable_stackadjust(self): Modified: pypy/branch/esp-params/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/x86/test/test_regalloc.py Tue Dec 1 16:19:32 2009 @@ -575,6 +575,9 @@ finish(i3, descr=fdescr2) ''' bridge = self.attach_bridge(ops, loop, -2) + + assert loop.operations[-2].descr._x86_bridge_param_depth == 2 + self.cpu.set_future_value_int(0, 4) self.cpu.set_future_value_int(1, 7) self.run(loop) @@ -595,7 +598,11 @@ finish(i3, descr=fdescr2) ''' bridge = self.attach_bridge(ops, loop, -2) + + assert loop.operations[-2].descr._x86_bridge_param_depth == 2 + self.cpu.set_future_value_int(0, 4) self.cpu.set_future_value_int(1, 7) self.run(loop) assert self.getint(0) == 29 + From cfbolz at codespeak.net Tue Dec 1 17:14:15 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 Dec 2009 17:14:15 +0100 (CET) Subject: [pypy-svn] r69809 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091201161415.1BB2A1680F1@codespeak.net> Author: cfbolz Date: Tue Dec 1 17:14:14 2009 New Revision: 69809 Modified: pypy/trunk/pypy/jit/metainterp/optimizeutil.py Log: oops, missing import Modified: pypy/trunk/pypy/jit/metainterp/optimizeutil.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeutil.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeutil.py Tue Dec 1 17:14:14 2009 @@ -1,4 +1,5 @@ from pypy.rlib.objectmodel import r_dict, compute_identity_hash +from pypy.rlib.rarithmetic import intmask from pypy.rlib.unroll import unrolling_iterable from pypy.jit.metainterp import resoperation From antocuni at codespeak.net Tue Dec 1 17:21:58 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 1 Dec 2009 17:21:58 +0100 (CET) Subject: [pypy-svn] r69810 - pypy/branch/cli-jit Message-ID: <20091201162158.B937E1680F1@codespeak.net> Author: antocuni Date: Tue Dec 1 17:21:58 2009 New Revision: 69810 Removed: pypy/branch/cli-jit/ Log: delete this branch, will be branched again from a newer version of trunk From antocuni at codespeak.net Tue Dec 1 17:24:06 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 1 Dec 2009 17:24:06 +0100 (CET) Subject: [pypy-svn] r69811 - pypy/branch/cli-jit Message-ID: <20091201162406.745E61680F1@codespeak.net> Author: antocuni Date: Tue Dec 1 17:24:05 2009 New Revision: 69811 Added: pypy/branch/cli-jit/ - copied from r69792, pypy/trunk/ Log: a branch where to keep the cli jit working, while it's being destroyed on trunk :-) From arigo at codespeak.net Tue Dec 1 17:51:15 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 1 Dec 2009 17:51:15 +0100 (CET) Subject: [pypy-svn] r69812 - in pypy/trunk/pypy/jit: backend backend/llgraph backend/test backend/x86 backend/x86/test metainterp metainterp/test Message-ID: <20091201165115.998A71680F5@codespeak.net> Author: arigo Date: Tue Dec 1 17:51:14 2009 New Revision: 69812 Modified: pypy/trunk/pypy/jit/backend/llgraph/runner.py pypy/trunk/pypy/jit/backend/model.py pypy/trunk/pypy/jit/backend/test/runner_test.py pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/runner.py pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py pypy/trunk/pypy/jit/metainterp/compile.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py Log: (cfbolz, arigo) Kill ResumeDescr.fail_arg_types, because it can be reconstructed from information that the backend needs to save anyway. Add the cpu.make_boxes_from_latest_value() interface to do that. Saves a byte per fail_arg on every guard. Small renamings in the corresponding logic in x86/assembler.py. Modified: pypy/trunk/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/runner.py Tue Dec 1 17:51:14 2009 @@ -166,6 +166,13 @@ if op.is_guard(): faildescr = op.descr assert isinstance(faildescr, history.AbstractFailDescr) + faildescr._fail_args_types = [] + for box in op.fail_args: + if box is None: + type = history.HOLE + else: + type = box.type + faildescr._fail_args_types.append(type) fail_index = self.get_fail_descr_number(faildescr) index = llimpl.compile_add_fail(c, fail_index) faildescr._compiled_fail = c, index @@ -235,6 +242,23 @@ token = llimpl.get_frame_forced_token(self.latest_frame) return self.cast_adr_to_int(token) + def make_boxes_from_latest_values(self, faildescr): + inputargs_and_holes = [] + for i in range(len(faildescr._fail_args_types)): + boxtype = faildescr._fail_args_types[i] + if boxtype == history.INT: + box = history.BoxInt(self.get_latest_value_int(i)) + elif boxtype == history.REF: + box = self.ts.BoxRef(self.get_latest_value_ref(i)) + elif boxtype == history.FLOAT: + box = history.BoxFloat(self.get_latest_value_float(i)) + elif boxtype == history.HOLE: + box = None + else: + assert False, "bad box type: num=%d" % ord(boxtype) + inputargs_and_holes.append(box) + return inputargs_and_holes + # ---------- def get_exception(self): Modified: pypy/trunk/pypy/jit/backend/model.py ============================================================================== --- pypy/trunk/pypy/jit/backend/model.py (original) +++ pypy/trunk/pypy/jit/backend/model.py Tue Dec 1 17:51:14 2009 @@ -83,6 +83,12 @@ same FORCE_TOKEN result as the one in the just-failed loop.""" raise NotImplementedError + def make_boxes_from_latest_value(self, faildescr): + """Build a list of Boxes (and None for holes) that contains + the current values, as would be returned by calls to + get_latest_value_xxx().""" + raise NotImplementedError + def get_exception(self): raise NotImplementedError Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Tue Dec 1 17:51:14 2009 @@ -238,6 +238,32 @@ res = self.cpu.get_latest_value_int(0) assert res == 20 + def test_make_boxes_from_latest_values(self): + i0 = BoxInt() + i1 = BoxInt() + i2 = BoxInt() + faildescr1 = BasicFailDescr(1) + looptoken = LoopToken() + operations = [ + ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1), + ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2), + ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1), + ResOperation(rop.JUMP, [i1], None, descr=looptoken), + ] + inputargs = [i0] + operations[2].fail_args = [None, i1, None] + self.cpu.compile_loop(inputargs, operations, looptoken) + + self.cpu.set_future_value_int(0, 2) + fail = self.cpu.execute_token(looptoken) + assert fail is faildescr1 + boxes = self.cpu.make_boxes_from_latest_values(faildescr1) + assert len(boxes) == 3 + assert boxes[0] is None + assert isinstance(boxes[1], BoxInt) + assert boxes[1].value == 10 + assert boxes[2] is None + def test_finish(self): i0 = BoxInt() class UntouchableFailDescr(AbstractFailDescr): Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Tue Dec 1 17:51:14 2009 @@ -1,8 +1,8 @@ import sys, os import ctypes from pypy.jit.backend.llsupport import symbolic -from pypy.jit.metainterp.history import Const, Box, BoxPtr, INT, REF, FLOAT -from pypy.jit.metainterp.history import AbstractFailDescr +from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat +from pypy.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT from pypy.rpython.lltypesystem import lltype, rffi, ll2ctypes, rstr, llmemory from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.lltypesystem.lloperation import llop @@ -798,9 +798,9 @@ DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 - DESCR_FROMSTACK = 8 - DESCR_STOP = 0 | DESCR_SPECIAL - DESCR_HOLE = 4 | DESCR_SPECIAL + CODE_FROMSTACK = 4*8 + CODE_STOP = 0 | DESCR_SPECIAL + CODE_HOLE = 4 | DESCR_SPECIAL def write_failure_recovery_description(self, mc, failargs, locs): for i in range(len(failargs)): @@ -816,7 +816,7 @@ raise AssertionError("bogus kind") loc = locs[i] if isinstance(loc, MODRM): - n = self.DESCR_FROMSTACK + loc.position + n = self.CODE_FROMSTACK//4 + loc.position else: assert isinstance(loc, REG) n = loc.op @@ -825,9 +825,9 @@ mc.writechr((n & 0x7F) | 0x80) n >>= 7 else: - n = self.DESCR_HOLE + n = self.CODE_HOLE mc.writechr(n) - mc.writechr(self.DESCR_STOP) + mc.writechr(self.CODE_STOP) # preallocate the fail_boxes i = len(failargs) - 1 if i >= 0: @@ -844,7 +844,7 @@ # decode the next instruction from the bytecode code = rffi.cast(lltype.Signed, bytecode[0]) bytecode = rffi.ptradd(bytecode, 1) - if code >= 4*self.DESCR_FROMSTACK: + if code >= self.CODE_FROMSTACK: # 'code' identifies a stack location if code > 0x7F: shift = 7 @@ -857,15 +857,15 @@ if nextcode <= 0x7F: break kind = code & 3 - code = (code >> 2) - self.DESCR_FROMSTACK + code = (code - self.CODE_FROMSTACK) >> 2 if kind == self.DESCR_FLOAT: size = 2 else: size = 1 loc = X86StackManager.stack_pos(code, size) - elif code == self.DESCR_STOP: + elif code == self.CODE_STOP: break - elif code == self.DESCR_HOLE: + elif code == self.CODE_HOLE: continue else: # 'code' identifies a register @@ -878,6 +878,39 @@ arglocs.append(loc) return arglocs[:] + def make_boxes_from_latest_values(self, bytecode): + bytecode = rffi.cast(rffi.UCHARP, bytecode) + boxes = [] + while 1: + # decode the next instruction from the bytecode + code = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + kind = code & 3 + while code > 0x7F: + code = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + index = len(boxes) + if kind == self.DESCR_INT: + box = BoxInt(self.fail_boxes_int.getitem(index)) + elif kind == self.DESCR_REF: + box = BoxPtr(self.fail_boxes_ptr.getitem(index)) + # clear after reading (xxx duplicates + # get_latest_value_ref()) + self.fail_boxes_ptr.setitem(index, lltype.nullptr( + llmemory.GCREF.TO)) + elif kind == self.DESCR_FLOAT: + box = BoxFloat(self.fail_boxes_float.getitem(index)) + else: + assert kind == self.DESCR_SPECIAL + if code == self.CODE_STOP: + break + elif code == self.CODE_HOLE: + box = None + else: + assert 0, "bad code" + boxes.append(box) + return boxes + def grab_frame_values(self, bytecode, frame_addr, allregisters): # no malloc allowed here!! self.fail_ebp = allregisters[16 + ebp.op] @@ -887,7 +920,7 @@ # decode the next instruction from the bytecode code = rffi.cast(lltype.Signed, bytecode[0]) bytecode = rffi.ptradd(bytecode, 1) - if code >= 4*self.DESCR_FROMSTACK: + if code >= self.CODE_FROMSTACK: if code > 0x7F: shift = 7 code &= 0x7F @@ -900,7 +933,7 @@ break # load the value from the stack kind = code & 3 - code = (code >> 2) - self.DESCR_FROMSTACK + code = (code - self.CODE_FROMSTACK) >> 2 stackloc = frame_addr + get_ebp_ofs(code) value = rffi.cast(rffi.LONGP, stackloc)[0] if kind == self.DESCR_FLOAT: @@ -910,10 +943,10 @@ # 'code' identifies a register: load its value kind = code & 3 if kind == self.DESCR_SPECIAL: - if code == self.DESCR_HOLE: + if code == self.CODE_HOLE: num += 1 continue - assert code == self.DESCR_STOP + assert code == self.CODE_STOP break code >>= 2 if kind == self.DESCR_FLOAT: Modified: pypy/trunk/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/runner.py Tue Dec 1 17:51:14 2009 @@ -62,6 +62,10 @@ def get_latest_force_token(self): return self.assembler.fail_ebp + FORCE_INDEX_OFS + def make_boxes_from_latest_values(self, faildescr): + return self.assembler.make_boxes_from_latest_values( + faildescr._x86_failure_recovery_bytecode) + def execute_token(self, executable_token): addr = executable_token._x86_bootstrap_code func = rffi.cast(lltype.Ptr(self.BOOTSTRAP_TP), addr) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py Tue Dec 1 17:51:14 2009 @@ -42,8 +42,8 @@ Assembler386.DESCR_INT + 4*(8+100), Assembler386.DESCR_REF + 4*(8+101), Assembler386.DESCR_FLOAT + 4*(8+110), - Assembler386.DESCR_HOLE, - Assembler386.DESCR_HOLE, + Assembler386.CODE_HOLE, + Assembler386.CODE_HOLE, Assembler386.DESCR_INT + 4*ebx.op, Assembler386.DESCR_REF + 4*esi.op, Assembler386.DESCR_FLOAT + 4*xmm2.op] @@ -52,7 +52,7 @@ double_byte_nums.append((num & 0x7F) | 0x80) double_byte_nums.append(num >> 7) assert mc.content == (nums[:3] + double_byte_nums + nums[6:] + - [assembler.DESCR_STOP]) + [assembler.CODE_STOP]) # also test rebuild_faillocs_from_descr(), which should not # reproduce the holes at all @@ -65,6 +65,35 @@ assert ([loc.assembler() for loc in newlocs] == [loc.assembler() for loc in locs if loc is not None]) + # finally, test make_boxes_from_latest_values(), which should + # reproduce the holes + expected_classes = [BoxInt, BoxPtr, BoxFloat, + BoxInt, BoxPtr, BoxFloat, + type(None), type(None), + BoxInt, BoxPtr, BoxFloat] + ptrvalues = {} + S = lltype.GcStruct('S') + for i, cls in enumerate(expected_classes): + if cls == BoxInt: + assembler.fail_boxes_int.setitem(i, 1000 + i) + elif cls == BoxPtr: + s = lltype.malloc(S) + s_ref = lltype.cast_opaque_ptr(llmemory.GCREF, s) + ptrvalues[i] = s_ref + assembler.fail_boxes_ptr.setitem(i, s_ref) + elif cls == BoxFloat: + assembler.fail_boxes_float.setitem(i, 42.5 + i) + boxes = assembler.make_boxes_from_latest_values(bytecode_addr) + assert len(boxes) == len(locs) == len(expected_classes) + for i, (box, expected_class) in enumerate(zip(boxes, expected_classes)): + assert type(box) is expected_class + if expected_class == BoxInt: + assert box.value == 1000 + i + elif expected_class == BoxPtr: + assert box.value == ptrvalues[i] + elif expected_class == BoxFloat: + assert box.value == 42.5 + i + # ____________________________________________________________ def test_failure_recovery_func_no_floats(): @@ -136,7 +165,7 @@ descr_bytecode = [] for i, (kind, loc) in enumerate(content): if kind == 'hole': - num = Assembler386.DESCR_HOLE + num = Assembler386.CODE_HOLE else: if kind == 'float': value, lo, hi = get_random_float() @@ -174,7 +203,7 @@ num >>= 7 descr_bytecode.append(num) - descr_bytecode.append(Assembler386.DESCR_STOP) + descr_bytecode.append(Assembler386.CODE_STOP) descr_bytecode.append(0xC3) # fail_index = 0x1C3 descr_bytecode.append(0x01) descr_bytecode.append(0x00) Modified: pypy/trunk/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/compile.py (original) +++ pypy/trunk/pypy/jit/metainterp/compile.py Tue Dec 1 17:51:14 2009 @@ -215,13 +215,6 @@ def store_final_boxes(self, guard_op, boxes): guard_op.fail_args = boxes self.guard_opnum = guard_op.opnum - fail_arg_types = [history.HOLE] * len(boxes) - for i in range(len(boxes)): - box = boxes[i] - if box: - fail_arg_types[i] = box.type - self.fail_arg_types = fail_arg_types - # XXX ^^^ kill this attribute def handle_fail(self, metainterp_sd): from pypy.jit.metainterp.pyjitpl import MetaInterp @@ -253,7 +246,7 @@ from pypy.jit.metainterp.resume import force_from_resumedata metainterp = MetaInterp(self.metainterp_sd) metainterp.history = None # blackholing - liveboxes = metainterp.load_values_from_failure(self) + liveboxes = metainterp.cpu.make_boxes_from_latest_values(self) virtualizable_boxes, data = force_from_resumedata(metainterp, liveboxes, self) vinfo.write_boxes(virtualizable, virtualizable_boxes) Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Tue Dec 1 17:51:14 2009 @@ -1692,7 +1692,7 @@ def initialize_state_from_guard_failure(self, resumedescr, must_compile): # guard failure: rebuild a complete MIFrame stack self.in_recursion = -1 # always one portal around - inputargs_and_holes = self.load_values_from_failure(resumedescr) + inputargs_and_holes = self.cpu.make_boxes_from_latest_values(resumedescr) if must_compile: self.history = history.History(self.cpu) self.history.inputargs = [box for box in inputargs_and_holes if box] @@ -1702,25 +1702,6 @@ self.history = None # this means that is_blackholing() is true self.rebuild_state_after_failure(resumedescr, inputargs_and_holes) - def load_values_from_failure(self, resumedescr): - cpu = self.cpu - fail_arg_types = resumedescr.fail_arg_types - inputargs_and_holes = [] - for i in range(len(fail_arg_types)): - boxtype = fail_arg_types[i] - if boxtype == history.INT: - box = history.BoxInt(cpu.get_latest_value_int(i)) - elif boxtype == history.REF: - box = cpu.ts.BoxRef(cpu.get_latest_value_ref(i)) - elif boxtype == history.FLOAT: - box = history.BoxFloat(cpu.get_latest_value_float(i)) - elif boxtype == history.HOLE: - box = None - else: - assert False, "bad box type: num=%d" % ord(boxtype) - inputargs_and_holes.append(box) - return inputargs_and_holes - def initialize_virtualizable(self, original_boxes): vinfo = self.staticdata.virtualizable_info if vinfo is not None: Modified: pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py Tue Dec 1 17:51:14 2009 @@ -134,8 +134,8 @@ class FakeCPU: ts = llhelper - def get_latest_value_int(self, index): - return index + def make_boxes_from_latest_values(self, faildescr): + return [BoxInt(0), None, BoxInt(2), None, BoxInt(4)] class FakeStaticData: cpu = FakeCPU() @@ -150,10 +150,6 @@ class FakeResumeDescr: pass resumedescr = FakeResumeDescr() - resumedescr.fail_arg_types = [history.INT, history.HOLE, - history.INT, history.HOLE, - history.INT] - metainterp.initialize_state_from_guard_failure(resumedescr, True) inp = metainterp.history.inputargs From antocuni at codespeak.net Tue Dec 1 17:55:01 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 1 Dec 2009 17:55:01 +0100 (CET) Subject: [pypy-svn] r69813 - in pypy/branch/cli-jit/pypy/translator/cli: . test Message-ID: <20091201165501.5DF721680F7@codespeak.net> Author: antocuni Date: Tue Dec 1 17:55:00 2009 New Revision: 69813 Modified: pypy/branch/cli-jit/pypy/translator/cli/support.py pypy/branch/cli-jit/pypy/translator/cli/test/test_pbc.py Log: aesthetic changes Modified: pypy/branch/cli-jit/pypy/translator/cli/support.py ============================================================================== --- pypy/branch/cli-jit/pypy/translator/cli/support.py (original) +++ pypy/branch/cli-jit/pypy/translator/cli/support.py Tue Dec 1 17:55:00 2009 @@ -63,7 +63,7 @@ keys.sort() for key in keys: label = ', '.join([str(item) for item in key]) - f.write('%s: %d\n' % (label, self.counters[key])) + f.write('%6d: %s\n' % (self.counters[key], label)) f.close() def getattr_ex(target, attr): Modified: pypy/branch/cli-jit/pypy/translator/cli/test/test_pbc.py ============================================================================== --- pypy/branch/cli-jit/pypy/translator/cli/test/test_pbc.py (original) +++ pypy/branch/cli-jit/pypy/translator/cli/test/test_pbc.py Tue Dec 1 17:55:00 2009 @@ -2,5 +2,7 @@ from pypy.translator.cli.test.runtest import CliTest from pypy.rpython.test.test_rpbc import BaseTestRPBC +# ====> ../../../rpython/test/test_rpbc.py + class TestCliPBC(CliTest, BaseTestRPBC): pass From afa at codespeak.net Tue Dec 1 18:11:25 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 1 Dec 2009 18:11:25 +0100 (CET) Subject: [pypy-svn] r69814 - pypy/trunk/pypy/module/_locale Message-ID: <20091201171125.2D3DA1680F7@codespeak.net> Author: afa Date: Tue Dec 1 18:11:24 2009 New Revision: 69814 Modified: pypy/trunk/pypy/module/_locale/interp_locale.py Log: Fix a segfault in test_locale, when run by "pypy-c test_all.py -A" (VS 8.0 aborts the program when invalid arguments are passed) CPython has the same issue, see http://bugs.python.org/issue7419 but it's not a reason not to fix it. Modified: pypy/trunk/pypy/module/_locale/interp_locale.py ============================================================================== --- pypy/trunk/pypy/module/_locale/interp_locale.py (original) +++ pypy/trunk/pypy/module/_locale/interp_locale.py Tue Dec 1 18:11:24 2009 @@ -86,6 +86,8 @@ 'LC_TELEPHONE', 'LC_MEASUREMENT', 'LC_IDENTIFICATION', + 'LC_MIN', + 'LC_MAX', # from limits.h 'CHAR_MAX', ) @@ -154,6 +156,10 @@ def setlocale(space, category, w_locale=None): "(integer,string=None) -> string. Activates/queries locale processing." + if cConfig.LC_MAX is not None: + if not cConfig.LC_MIN <= category <= cConfig.LC_MAX: + raise make_error(space, "invalid locale category") + if space.is_w(w_locale, space.w_None) or w_locale is None: result = _setlocale(rffi.cast(rffi.INT, category), None) if not result: From arigo at codespeak.net Tue Dec 1 18:25:09 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 1 Dec 2009 18:25:09 +0100 (CET) Subject: [pypy-svn] r69815 - pypy/trunk/pypy/translator/c Message-ID: <20091201172509.4B3591680F8@codespeak.net> Author: arigo Date: Tue Dec 1 18:25:08 2009 New Revision: 69815 Modified: pypy/trunk/pypy/translator/c/node.py Log: Argh. One ExternalCompilationInfo per function translated is a bit too much overhead. Modified: pypy/trunk/pypy/translator/c/node.py ============================================================================== --- pypy/trunk/pypy/translator/c/node.py (original) +++ pypy/trunk/pypy/translator/c/node.py Tue Dec 1 18:25:08 2009 @@ -731,8 +731,7 @@ else: self.name = (forcename or db.namespace.uniquename('g_' + self.basename())) - self.compilation_info = getattr(obj, 'compilation_info', - ExternalCompilationInfo()) + self.compilation_info = getattr(obj, 'compilation_info', None) self.make_funcgens() #self.dependencies = {} self.typename = db.gettype(T) #, who_asks=self) From antocuni at codespeak.net Tue Dec 1 18:53:26 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 1 Dec 2009 18:53:26 +0100 (CET) Subject: [pypy-svn] r69816 - in pypy/branch/cli-jit/pypy/translator: cli jvm oosupport Message-ID: <20091201175326.AE6431680FD@codespeak.net> Author: antocuni Date: Tue Dec 1 18:53:25 2009 New Revision: 69816 Modified: pypy/branch/cli-jit/pypy/translator/cli/constant.py pypy/branch/cli-jit/pypy/translator/jvm/constant.py pypy/branch/cli-jit/pypy/translator/oosupport/constant.py Log: not all the constants needs to be initialized (e.g., instances without fields, empty lists, etc.). This saves a bit of useless "ldsfld xxx; pop" sequences during the constant initialization phase Modified: pypy/branch/cli-jit/pypy/translator/cli/constant.py ============================================================================== --- pypy/branch/cli-jit/pypy/translator/cli/constant.py (original) +++ pypy/branch/cli-jit/pypy/translator/cli/constant.py Tue Dec 1 18:53:25 2009 @@ -293,9 +293,6 @@ gen.ilasm.opcode('ldftn', signature) gen.ilasm.new('instance void class %s::.ctor(object, native int)' % self.delegate_type) self.db.const_count.inc('StaticMethod') - - def initialize_data(self, constgen, gen): - return class CLIWeakRefConst(CLIBaseConstMixin, WeakRefConst): @@ -305,6 +302,9 @@ def get_type(self, include_class=True): return 'class ' + WEAKREF + + def needs_initialization(self): + return bool(self.value) def initialize_data(self, constgen, gen): if self.value is not None: Modified: pypy/branch/cli-jit/pypy/translator/jvm/constant.py ============================================================================== --- pypy/branch/cli-jit/pypy/translator/jvm/constant.py (original) +++ pypy/branch/cli-jit/pypy/translator/jvm/constant.py Tue Dec 1 18:53:25 2009 @@ -158,8 +158,6 @@ else: gen.push_null(jvm.jObject) - def initialize_data(self, constgen, gen): - return class JVMCustomDictConst(CustomDictConst): @@ -199,9 +197,8 @@ push_constant(self.db, self.value._TYPE, self.value, gen) gen.create_weakref(TYPE) - def initialize_data(self, constgen, gen): - gen.pop(ootype.ROOT) - return True + def needs_initialization(self): + return False Modified: pypy/branch/cli-jit/pypy/translator/oosupport/constant.py ============================================================================== --- pypy/branch/cli-jit/pypy/translator/oosupport/constant.py (original) +++ pypy/branch/cli-jit/pypy/translator/oosupport/constant.py Tue Dec 1 18:53:25 2009 @@ -282,6 +282,8 @@ """ Iterates through each constant, initializing its data. """ gen.add_section("Initialize Data Phase") for const in all_constants: + if not const.needs_initialization(): + continue self._consider_step(gen) gen.add_comment("Constant: %s" % const.name) self._push_constant_during_init(gen, const) @@ -426,6 +428,12 @@ assert not self.is_null() gen.new(self.value._TYPE) + def needs_initialization(self): + """ + if True, calls initialize_data + """ + return True + def initialize_data(self, constgen, gen): """ Initializes the internal data. Begins with a pointer to @@ -475,6 +483,9 @@ value = self.value._items[f_name] self._record_const_if_complex(FIELD_TYPE, value) + def needs_initialization(self): + return bool(self.value._TYPE._fields) + def initialize_data(self, constgen, gen): assert not self.is_null() SELFTYPE = self.value._TYPE @@ -543,13 +554,17 @@ return const_list + def needs_initialization(self): + self.const_list = self._sorted_const_list() + return bool(self.const_list) + def initialize_data(self, constgen, gen): assert not self.is_null() # Get a list of all the constants we'll need to initialize. # I am not clear on why this needs to be sorted, actually, # but we sort it. - const_list = self._sorted_const_list() + const_list = self.const_list # Push ourself on the stack, and cast to our actual type if it # is not the same as our static type @@ -585,8 +600,8 @@ INSTANCE = self.value._INSTANCE gen.getclassobject(INSTANCE) - def initialize_data(self, constgen, gen): - pass + def needs_initialization(self): + return False # ______________________________________________________________________ # List constants @@ -627,6 +642,9 @@ Void. """ return self.value._TYPE.ITEM is ootype.Void + def needs_initialization(self): + return bool(self.value._list) + def initialize_data(self, constgen, gen): assert not self.is_null() SELFTYPE = self.value._TYPE @@ -675,6 +693,9 @@ Void. """ return self.value._TYPE.ITEM is ootype.Void + def needs_initialization(self): + return bool(self.value._array) + def initialize_data(self, constgen, gen): assert not self.is_null() SELFTYPE = self.value._TYPE @@ -713,6 +734,9 @@ self._record_const_if_complex(self.value._TYPE._KEYTYPE, key) self._record_const_if_complex(self.value._TYPE._VALUETYPE, value) + def needs_initialization(self): + return bool(self.value._dict) + def initialize_data(self, constgen, gen): assert not self.is_null() SELFTYPE = self.value._TYPE @@ -752,8 +776,8 @@ self.db.pending_function(self.value.graph) self.delegate_type = self.db.record_delegate(self.value._TYPE) - def initialize_data(self, constgen, gen): - raise NotImplementedError + def needs_initialization(self): + return False # ______________________________________________________________________ # Weak Reference constants From arigo at codespeak.net Tue Dec 1 19:00:57 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 1 Dec 2009 19:00:57 +0100 (CET) Subject: [pypy-svn] r69817 - in pypy/trunk/pypy: jit/backend/x86 translator/c/src Message-ID: <20091201180057.7892B1680FC@codespeak.net> Author: arigo Date: Tue Dec 1 19:00:56 2009 New Revision: 69817 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/codebuf.py pypy/trunk/pypy/translator/c/src/support.h Log: On non-Windows platforms, when compiling the x86 JIT, ensure that the arguments -msse2 -mfpmath=sse are passed to gcc. This should be enough to fix one or two failures on the nightly run, caused by inconsistencies between the non-JITted world (using x87 instructions) and the JITted world (using sse2 instructions). Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Tue Dec 1 19:00:56 2009 @@ -129,6 +129,7 @@ if self.cpu.supports_floats: self._build_failure_recovery(False, withfloats=True) self._build_failure_recovery(True, withfloats=True) + codebuf.ensure_sse2_floats() def assemble_loop(self, inputargs, operations, looptoken): """adds the following attributes to looptoken: Modified: pypy/trunk/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/trunk/pypy/jit/backend/x86/codebuf.py Tue Dec 1 19:00:56 2009 @@ -1,6 +1,7 @@ -import os +import os, sys from pypy.rpython.lltypesystem import lltype, rffi +from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.jit.backend.x86.ri386 import I386CodeBuilder from pypy.rlib.rmmap import PTR, alloc, free from pypy.rlib.debug import make_sure_not_resized @@ -142,3 +143,14 @@ size = self._size assert size >= 0 free(self._data, size) + + +# ____________________________________________________________ + +if sys.platform == 'win32': + ensure_sse2_floats = lambda self: None +else: + _sse2_eci = ExternalCompilationInfo( + compile_extra = ['-msse2', '-mfpmath=sse']) + ensure_sse2_floats = rffi.llexternal('PYPY_NO_OP', [], lltype.Void, + compilation_info=_sse2_eci) Modified: pypy/trunk/pypy/translator/c/src/support.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/support.h (original) +++ pypy/trunk/pypy/translator/c/src/support.h Tue Dec 1 19:00:56 2009 @@ -19,6 +19,7 @@ #define FAIL_VAL(msg) FAIL_EXCEPTION(PyExc_ValueError, msg) #define FAIL_ZER(msg) FAIL_EXCEPTION(PyExc_ZeroDivisionError, msg) #define CFAIL() RPyConvertExceptionFromCPython() +#define PYPY_NO_OP() /* nothing */ /* the following macros are used by rpython/lltypesystem/rstr.py */ #define PyString_FromRPyString(rpystr) \ From fijal at codespeak.net Wed Dec 2 10:06:38 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 2 Dec 2009 10:06:38 +0100 (CET) Subject: [pypy-svn] r69818 - in pypy/trunk/pypy: jit/backend/x86 translator/c/src Message-ID: <20091202090638.5723C318139@codespeak.net> Author: fijal Date: Wed Dec 2 10:06:36 2009 New Revision: 69818 Modified: pypy/trunk/pypy/jit/backend/x86/codebuf.py pypy/trunk/pypy/translator/c/src/support.h Log: Cleanup up nonsense that broke tests. We have a way to find out whether eci works or not. Why is it a lambda btw? Modified: pypy/trunk/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/trunk/pypy/jit/backend/x86/codebuf.py Wed Dec 2 10:06:36 2009 @@ -1,6 +1,7 @@ import os, sys from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.tool import rffi_platform from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.jit.backend.x86.ri386 import I386CodeBuilder from pypy.rlib.rmmap import PTR, alloc, free @@ -148,9 +149,9 @@ # ____________________________________________________________ if sys.platform == 'win32': - ensure_sse2_floats = lambda self: None + ensure_sse2_floats = lambda : None else: _sse2_eci = ExternalCompilationInfo( compile_extra = ['-msse2', '-mfpmath=sse']) - ensure_sse2_floats = rffi.llexternal('PYPY_NO_OP', [], lltype.Void, - compilation_info=_sse2_eci) + ensure_sse2_floats = lambda : rffi_platform.verify_eci(_sse2_eci) + Modified: pypy/trunk/pypy/translator/c/src/support.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/support.h (original) +++ pypy/trunk/pypy/translator/c/src/support.h Wed Dec 2 10:06:36 2009 @@ -19,7 +19,6 @@ #define FAIL_VAL(msg) FAIL_EXCEPTION(PyExc_ValueError, msg) #define FAIL_ZER(msg) FAIL_EXCEPTION(PyExc_ZeroDivisionError, msg) #define CFAIL() RPyConvertExceptionFromCPython() -#define PYPY_NO_OP() /* nothing */ /* the following macros are used by rpython/lltypesystem/rstr.py */ #define PyString_FromRPyString(rpystr) \ From fijal at codespeak.net Wed Dec 2 10:16:01 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 2 Dec 2009 10:16:01 +0100 (CET) Subject: [pypy-svn] r69819 - pypy/trunk/pypy/jit/backend/x86 Message-ID: <20091202091601.77710318139@codespeak.net> Author: fijal Date: Wed Dec 2 10:16:00 2009 New Revision: 69819 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/codebuf.py Log: Clean up nonsense even more - there is no way platform can be invoked at runtime. Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Wed Dec 2 10:16:00 2009 @@ -129,7 +129,7 @@ if self.cpu.supports_floats: self._build_failure_recovery(False, withfloats=True) self._build_failure_recovery(True, withfloats=True) - codebuf.ensure_sse2_floats() + assert codebuf.sse2_floats def assemble_loop(self, inputargs, operations, looptoken): """adds the following attributes to looptoken: Modified: pypy/trunk/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/trunk/pypy/jit/backend/x86/codebuf.py Wed Dec 2 10:16:00 2009 @@ -6,7 +6,7 @@ from pypy.jit.backend.x86.ri386 import I386CodeBuilder from pypy.rlib.rmmap import PTR, alloc, free from pypy.rlib.debug import make_sure_not_resized - +from pypy.translator.platform import CompilationError class InMemoryCodeBuilder(I386CodeBuilder): _last_dump_start = 0 @@ -149,9 +149,13 @@ # ____________________________________________________________ if sys.platform == 'win32': - ensure_sse2_floats = lambda : None + sse2_floats = True else: - _sse2_eci = ExternalCompilationInfo( - compile_extra = ['-msse2', '-mfpmath=sse']) - ensure_sse2_floats = lambda : rffi_platform.verify_eci(_sse2_eci) + try: + _sse2_eci = ExternalCompilationInfo( + compile_extra = ['-msse2', '-mfpmath=sse']) + rffi_platform.verify_eci(_sse2_eci) + sse2_floats = True + except CompilationError: + sse2_floats = False From afa at codespeak.net Wed Dec 2 10:21:03 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 2 Dec 2009 10:21:03 +0100 (CET) Subject: [pypy-svn] r69820 - in pypy/trunk/pypy/module: select/test signal/test Message-ID: <20091202092103.21B9B49843D@codespeak.net> Author: afa Date: Wed Dec 2 10:21:02 2009 New Revision: 69820 Modified: pypy/trunk/pypy/module/select/test/test_select.py pypy/trunk/pypy/module/signal/test/test_signal.py Log: Skip tests that cant't possibly work on Windows. Modified: pypy/trunk/pypy/module/select/test/test_select.py ============================================================================== --- pypy/trunk/pypy/module/select/test/test_select.py (original) +++ pypy/trunk/pypy/module/select/test/test_select.py Wed Dec 2 10:21:02 2009 @@ -195,6 +195,8 @@ def test_select_bug(self): import select, os + if not hasattr(os, 'fork'): + skip("no fork() on this platform") read, write = os.pipe() pid = os.fork() if pid == 0: Modified: pypy/trunk/pypy/module/signal/test/test_signal.py ============================================================================== --- pypy/trunk/pypy/module/signal/test/test_signal.py (original) +++ pypy/trunk/pypy/module/signal/test/test_signal.py Wed Dec 2 10:21:02 2009 @@ -144,7 +144,10 @@ cls.space = space def test_alarm_raise(self): - from signal import alarm, signal, SIG_DFL, SIGALRM + try: + from signal import alarm, signal, SIG_DFL, SIGALRM + except ImportError: + skip("no SIGALRM on this platform") import _socket class Alarm(Exception): pass From fijal at codespeak.net Wed Dec 2 10:41:00 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 2 Dec 2009 10:41:00 +0100 (CET) Subject: [pypy-svn] r69821 - pypy/trunk/pypy/jit/backend/x86 Message-ID: <20091202094100.2138331813A@codespeak.net> Author: fijal Date: Wed Dec 2 10:40:59 2009 New Revision: 69821 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/codebuf.py Log: Hack differently. Now we can run it directly and options are correctly passed to resulting gcc. Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Wed Dec 2 10:40:59 2009 @@ -129,7 +129,7 @@ if self.cpu.supports_floats: self._build_failure_recovery(False, withfloats=True) self._build_failure_recovery(True, withfloats=True) - assert codebuf.sse2_floats + codebuf.ensure_sse2_floats() def assemble_loop(self, inputargs, operations, looptoken): """adds the following attributes to looptoken: Modified: pypy/trunk/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/trunk/pypy/jit/backend/x86/codebuf.py Wed Dec 2 10:40:59 2009 @@ -1,12 +1,11 @@ import os, sys from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rpython.tool import rffi_platform from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.jit.backend.x86.ri386 import I386CodeBuilder from pypy.rlib.rmmap import PTR, alloc, free from pypy.rlib.debug import make_sure_not_resized -from pypy.translator.platform import CompilationError + class InMemoryCodeBuilder(I386CodeBuilder): _last_dump_start = 0 @@ -149,13 +148,11 @@ # ____________________________________________________________ if sys.platform == 'win32': - sse2_floats = True + ensure_sse2_floats = lambda self: None else: - try: - _sse2_eci = ExternalCompilationInfo( - compile_extra = ['-msse2', '-mfpmath=sse']) - rffi_platform.verify_eci(_sse2_eci) - sse2_floats = True - except CompilationError: - sse2_floats = False - + _sse2_eci = ExternalCompilationInfo( + compile_extra = ['-msse2', '-mfpmath=sse'], + separate_module_sources = ['void PYPY_NO_OP() {}'], + ) + ensure_sse2_floats = rffi.llexternal('PYPY_NO_OP', [], lltype.Void, + compilation_info=_sse2_eci) From arigo at codespeak.net Wed Dec 2 11:32:26 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 2 Dec 2009 11:32:26 +0100 (CET) Subject: [pypy-svn] r69822 - in pypy/trunk/pypy/jit/backend: test x86 x86/test Message-ID: <20091202103226.23ACA1683D1@codespeak.net> Author: arigo Date: Wed Dec 2 11:32:25 2009 New Revision: 69822 Modified: pypy/trunk/pypy/jit/backend/test/support.py pypy/trunk/pypy/jit/backend/x86/codebuf.py pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py Log: Write a test for the fact that the options really are passed to gcc. Modified: pypy/trunk/pypy/jit/backend/test/support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/support.py (original) +++ pypy/trunk/pypy/jit/backend/test/support.py Wed Dec 2 11:32:25 2009 @@ -124,6 +124,7 @@ # XXX patch exceptions cbuilder = CBuilder(t, entry_point, config=t.config) cbuilder.generate_source() + self._check_cbuilder(cbuilder) exe_name = cbuilder.compile() debug_print('---------- Test starting ----------') stdout = cbuilder.cmdexec(" ".join([str(arg) for arg in args])) @@ -131,6 +132,9 @@ debug_print('---------- Test done (%d) ----------' % (res,)) return res + def _check_cbuilder(self, cbuilder): + pass + class CliCompiledMixin(BaseCompiledMixin): type_system = 'ootype' Modified: pypy/trunk/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/trunk/pypy/jit/backend/x86/codebuf.py Wed Dec 2 11:32:25 2009 @@ -148,11 +148,11 @@ # ____________________________________________________________ if sys.platform == 'win32': - ensure_sse2_floats = lambda self: None + ensure_sse2_floats = lambda : None else: _sse2_eci = ExternalCompilationInfo( compile_extra = ['-msse2', '-mfpmath=sse'], - separate_module_sources = ['void PYPY_NO_OP() {}'], + separate_module_sources = ['void PYPY_NO_OP(void) {}'], ) ensure_sse2_floats = rffi.llexternal('PYPY_NO_OP', [], lltype.Void, compilation_info=_sse2_eci) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py Wed Dec 2 11:32:25 2009 @@ -8,6 +8,12 @@ class TestTranslationX86(CCompiledMixin): CPUClass = CPU386 + def _check_cbuilder(self, cbuilder): + # We assume here that we have sse2. If not, the CPUClass + # needs to be changed to CPU386_NO_SSE2, but well. + assert '-msse2' in cbuilder.eci.compile_extra + assert '-mfpmath=sse' in cbuilder.eci.compile_extra + def test_stuff_translates(self): # this is a basic test that tries to hit a number of features and their # translation: From fijal at codespeak.net Wed Dec 2 11:44:33 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 2 Dec 2009 11:44:33 +0100 (CET) Subject: [pypy-svn] r69824 - pypy/trunk/pypy/translator/c/gcc Message-ID: <20091202104433.C06BF1683D1@codespeak.net> Author: fijal Date: Wed Dec 2 11:44:32 2009 New Revision: 69824 Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Log: list a bit more sse2 operations Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Wed Dec 2 11:44:32 2009 @@ -349,7 +349,8 @@ 'f', 'cvt', 'ucomi', 'subs', 'subp' , 'adds', 'addp', 'xorp', 'movap', 'movd', 'sqrtsd', - 'mins', 'minp', 'maxs', 'maxp', # sse2 + 'mins', 'minp', 'maxs', 'maxp', 'unpcklps', + 'unpcklpd', 'unpckhps', 'unpckhpd', # sse2 # arithmetic operations should not produce GC pointers 'inc', 'dec', 'not', 'neg', 'or', 'and', 'sbb', 'adc', 'shl', 'shr', 'sal', 'sar', 'rol', 'ror', 'mul', 'imul', 'div', 'idiv', From fijal at codespeak.net Wed Dec 2 11:46:10 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 2 Dec 2009 11:46:10 +0100 (CET) Subject: [pypy-svn] r69826 - pypy/trunk/pypy/translator/c/gcc Message-ID: <20091202104610.5DBDB1683D1@codespeak.net> Author: fijal Date: Wed Dec 2 11:46:09 2009 New Revision: 69826 Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Log: it can be said like this Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Wed Dec 2 11:46:09 2009 @@ -349,8 +349,7 @@ 'f', 'cvt', 'ucomi', 'subs', 'subp' , 'adds', 'addp', 'xorp', 'movap', 'movd', 'sqrtsd', - 'mins', 'minp', 'maxs', 'maxp', 'unpcklps', - 'unpcklpd', 'unpckhps', 'unpckhpd', # sse2 + 'mins', 'minp', 'maxs', 'maxp', 'unpck', # sse2 # arithmetic operations should not produce GC pointers 'inc', 'dec', 'not', 'neg', 'or', 'and', 'sbb', 'adc', 'shl', 'shr', 'sal', 'sar', 'rol', 'ror', 'mul', 'imul', 'div', 'idiv', From fijal at codespeak.net Wed Dec 2 11:58:34 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 2 Dec 2009 11:58:34 +0100 (CET) Subject: [pypy-svn] r69827 - pypy/trunk/pypy/translator/c/gcc Message-ID: <20091202105834.257301683D1@codespeak.net> Author: fijal Date: Wed Dec 2 11:58:33 2009 New Revision: 69827 Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Log: some more Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Wed Dec 2 11:58:33 2009 @@ -349,7 +349,7 @@ 'f', 'cvt', 'ucomi', 'subs', 'subp' , 'adds', 'addp', 'xorp', 'movap', 'movd', 'sqrtsd', - 'mins', 'minp', 'maxs', 'maxp', 'unpck', # sse2 + 'mins', 'minp', 'maxs', 'maxp', 'unpck', 'pxor', 'por', # sse2 # arithmetic operations should not produce GC pointers 'inc', 'dec', 'not', 'neg', 'or', 'and', 'sbb', 'adc', 'shl', 'shr', 'sal', 'sar', 'rol', 'ror', 'mul', 'imul', 'div', 'idiv', From cfbolz at codespeak.net Wed Dec 2 12:12:37 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 2 Dec 2009 12:12:37 +0100 (CET) Subject: [pypy-svn] r69830 - in pypy/trunk/pypy: module/_stackless module/_stackless/test rlib rlib/test Message-ID: <20091202111237.E13921683D1@codespeak.net> Author: cfbolz Date: Wed Dec 2 12:12:36 2009 New Revision: 69830 Added: pypy/trunk/pypy/module/_stackless/rcoroutine.py - copied unchanged from r66486, pypy/branch/io-lang/pypy/module/_stackless/rcoroutine.py pypy/trunk/pypy/rlib/test/test_rcoroutine.py - copied, changed from r66485, pypy/branch/io-lang/pypy/rlib/test/test_rcoroutine.py Removed: pypy/trunk/pypy/module/_stackless/test/test_interp_coroutine.py Modified: pypy/trunk/pypy/module/_stackless/interp_clonable.py pypy/trunk/pypy/module/_stackless/interp_coroutine.py pypy/trunk/pypy/module/_stackless/interp_greenlet.py pypy/trunk/pypy/rlib/rcoroutine.py Log: merge revisions 66485 and 66486 from io-lang branch: Make rlib.rcoroutine actually usable from anywhere other than the Python interpreter. The changes there are mostly a reindent. Modified: pypy/trunk/pypy/module/_stackless/interp_clonable.py ============================================================================== --- pypy/trunk/pypy/module/_stackless/interp_clonable.py (original) +++ pypy/trunk/pypy/module/_stackless/interp_clonable.py Wed Dec 2 12:12:36 2009 @@ -3,7 +3,7 @@ from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root from pypy.module._stackless.interp_coroutine import AppCoroutine, AppCoState from pypy.module._stackless.interp_coroutine import makeStaticMethod -from pypy.rlib.rcoroutine import AbstractThunk +from pypy.module._stackless.rcoroutine import AbstractThunk from pypy.module._stackless.rclonable import InterpClonableMixin Modified: pypy/trunk/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/trunk/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/trunk/pypy/module/_stackless/interp_coroutine.py Wed Dec 2 12:12:36 2009 @@ -24,7 +24,7 @@ from pypy.interpreter.function import StaticMethod from pypy.module._stackless.stackless_flags import StacklessFlags -from pypy.rlib.rcoroutine import Coroutine, BaseCoState, AbstractThunk +from pypy.module._stackless.rcoroutine import Coroutine, BaseCoState, AbstractThunk from pypy.rlib import rstack # for resume points from pypy.tool import stdlib_opcode as pythonopcode Modified: pypy/trunk/pypy/module/_stackless/interp_greenlet.py ============================================================================== --- pypy/trunk/pypy/module/_stackless/interp_greenlet.py (original) +++ pypy/trunk/pypy/module/_stackless/interp_greenlet.py Wed Dec 2 12:12:36 2009 @@ -4,8 +4,8 @@ from pypy.interpreter.gateway import NoneNotWrapped from pypy.interpreter.error import OperationError -from pypy.rlib.rcoroutine import Coroutine, BaseCoState -from pypy.rlib.rcoroutine import AbstractThunk, syncstate +from pypy.module._stackless.rcoroutine import Coroutine, BaseCoState +from pypy.module._stackless.rcoroutine import AbstractThunk, syncstate from pypy.module._stackless.interp_coroutine import makeStaticMethod Modified: pypy/trunk/pypy/rlib/rcoroutine.py ============================================================================== --- pypy/trunk/pypy/rlib/rcoroutine.py (original) +++ pypy/trunk/pypy/rlib/rcoroutine.py Wed Dec 2 12:12:36 2009 @@ -29,7 +29,6 @@ The type of a switch is determined by the target's costate. """ -from pypy.interpreter.baseobjspace import Wrappable from pypy.rlib.rstack import yield_current_frame_to_caller, resume_point from pypy.rlib.objectmodel import we_are_translated @@ -55,292 +54,294 @@ import sys, os +def make_coroutine_classes(baseclass): + class BaseCoState(object): + def __init__(self): + self.current = self.main = None + + def __repr__(self): + "NOT_RPYTHON" + # for debugging only + return '<%s current=%r>' % (self.__class__.__name__, self.current) + + def update(self, new): + syncstate.leaving = self.current + syncstate.entering = new + self.current = new + frame, new.frame = new.frame, None + return frame + + + class CoState(BaseCoState): + def __init__(self): + BaseCoState.__init__(self) + self.current = self.main = Coroutine(self) -class BaseCoState(object): - def __init__(self): - self.current = self.main = None - - def __repr__(self): - "NOT_RPYTHON" - # for debugging only - return '<%s current=%r>' % (self.__class__.__name__, self.current) - - def update(self, new): - syncstate.leaving = self.current - syncstate.entering = new - self.current = new - frame, new.frame = new.frame, None - return frame - - -class CoState(BaseCoState): - def __init__(self): - BaseCoState.__init__(self) - self.current = self.main = Coroutine(self) - -class CoroutineDamage(SystemError): - pass - - -class SyncState(object): - def __init__(self): - self.reset() - - def reset(self): - self.default_costate = None - self.leaving = None - self.entering = None - self.things_to_do = False - self.temp_exc = None - self.to_delete = [] - - def switched(self, incoming_frame): - left = syncstate.leaving - entered = syncstate.entering - syncstate.leaving = syncstate.entering = None - if left is not None: # mostly to work around an annotation problem; - # should not really be None - left.frame = incoming_frame - left.goodbye() - if entered is not None: - entered.hello() - if self.things_to_do: - self._do_things_to_do() - - def push_exception(self, exc): - self.things_to_do = True - self.temp_exc = exc - - def check_for_zombie(self, obj): - return co in self.to_delete - - def postpone_deletion(self, obj): - self.to_delete.append(obj) - self.things_to_do = True - - def _do_things_to_do(self): - if self.temp_exc is not None: - # somebody left an unhandled exception and switched to us. - # this both provides default exception handling and the - # way to inject an exception, like CoroutineExit. - e, self.temp_exc = self.temp_exc, None - self.things_to_do = bool(self.to_delete) - raise e - while self.to_delete: - delete, self.to_delete = self.to_delete, [] - for obj in delete: - obj.parent = obj.costate.current - obj._kill_finally() - else: - self.things_to_do = False - - def _freeze_(self): - self.reset() - return False - -syncstate = SyncState() + class CoroutineDamage(SystemError): + pass -class CoroutineExit(SystemExit): - # XXX SystemExit's __init__ creates problems in bookkeeper. - def __init__(self): - pass + class SyncState(object): + def __init__(self): + self.reset() + + def reset(self): + self.default_costate = None + self.leaving = None + self.entering = None + self.things_to_do = False + self.temp_exc = None + self.to_delete = [] -class AbstractThunk(object): - def call(self): - raise NotImplementedError("abstract base class") - - -class Coroutine(Wrappable): - def __init__(self, state=None): - self.frame = None - if state is None: - state = self._get_default_costate() - self.costate = state - self.parent = None - self.thunk = None - - def __repr__(self): - 'NOT_RPYTHON' - # just for debugging - if hasattr(self, '__name__'): - return '' % (self.__name__, self.frame, self.thunk is not None) - else: - return '' % (self.frame, self.thunk is not None) + def switched(self, incoming_frame): + left = syncstate.leaving + entered = syncstate.entering + syncstate.leaving = syncstate.entering = None + if left is not None: # mostly to work around an annotation problem; + # should not really be None + left.frame = incoming_frame + left.goodbye() + if entered is not None: + entered.hello() + if self.things_to_do: + self._do_things_to_do() + + def push_exception(self, exc): + self.things_to_do = True + self.temp_exc = exc + + def check_for_zombie(self, obj): + return co in self.to_delete + + def postpone_deletion(self, obj): + self.to_delete.append(obj) + self.things_to_do = True + + def _do_things_to_do(self): + if self.temp_exc is not None: + # somebody left an unhandled exception and switched to us. + # this both provides default exception handling and the + # way to inject an exception, like CoroutineExit. + e, self.temp_exc = self.temp_exc, None + self.things_to_do = bool(self.to_delete) + raise e + while self.to_delete: + delete, self.to_delete = self.to_delete, [] + for obj in delete: + obj.parent = obj.costate.current + obj._kill_finally() + else: + self.things_to_do = False + + def _freeze_(self): + self.reset() + return False + + syncstate = SyncState() + + + class CoroutineExit(SystemExit): + # XXX SystemExit's __init__ creates problems in bookkeeper. + def __init__(self): + pass - def _get_default_costate(): - state = syncstate.default_costate - if state is None: - state = syncstate.default_costate = CoState() - return state - _get_default_costate = staticmethod(_get_default_costate) - - def _get_default_parent(self): - return self.costate.current - - def bind(self, thunk): - assert isinstance(thunk, AbstractThunk) - if self.frame is not None: - raise CoroutineDamage - if self.parent is None: - self.parent = self._get_default_parent() - assert self.parent is not None - self.thunk = thunk - if we_are_translated(): - self.frame = self._bind() - else: - self.frame = self._greenlet_bind() + class AbstractThunk(object): + def call(self): + raise NotImplementedError("abstract base class") + + + class Coroutine(baseclass): + def __init__(self, state=None): + self.frame = None + if state is None: + state = self._get_default_costate() + self.costate = state + self.parent = None + self.thunk = None + + def __repr__(self): + 'NOT_RPYTHON' + # just for debugging + if hasattr(self, '__name__'): + return '' % (self.__name__, self.frame, self.thunk is not None) + else: + return '' % (self.frame, self.thunk is not None) + + def _get_default_costate(): + state = syncstate.default_costate + if state is None: + state = syncstate.default_costate = CoState() + return state + _get_default_costate = staticmethod(_get_default_costate) + + def _get_default_parent(self): + return self.costate.current + + def bind(self, thunk): + assert isinstance(thunk, AbstractThunk) + if self.frame is not None: + raise CoroutineDamage + if self.parent is None: + self.parent = self._get_default_parent() + assert self.parent is not None + self.thunk = thunk + if we_are_translated(): + self.frame = self._bind() + else: + self.frame = self._greenlet_bind() + + def _greenlet_bind(self): + weak = [self] + def _greenlet_execute(incoming_frame): + try: + chain2go2next = weak[0]._execute(incoming_frame) + except: + # no exception is supposed to get out of _execute() + # better report it directly into the main greenlet then, + # and hidden to prevent catching + main_greenlet.throw(AssertionError( + "unexpected exception out of Coroutine._execute()", + *sys.exc_info())) + assert 0 + del weak[0] + greenlet.getcurrent().parent = chain2go2next.greenlet + return None # as the result of the FrameChain.switch() + chain = FrameChain(_greenlet_execute) + return chain + + def _bind(self): + state = self.costate + incoming_frame = yield_current_frame_to_caller() + self = state.current + return self._execute(incoming_frame) - def _greenlet_bind(self): - weak = [self] - def _greenlet_execute(incoming_frame): - try: - chain2go2next = weak[0]._execute(incoming_frame) - except: - # no exception is supposed to get out of _execute() - # better report it directly into the main greenlet then, - # and hidden to prevent catching - main_greenlet.throw(AssertionError( - "unexpected exception out of Coroutine._execute()", - *sys.exc_info())) - assert 0 - del weak[0] - greenlet.getcurrent().parent = chain2go2next.greenlet - return None # as the result of the FrameChain.switch() - chain = FrameChain(_greenlet_execute) - return chain - - def _bind(self): - state = self.costate - incoming_frame = yield_current_frame_to_caller() - self = state.current - return self._execute(incoming_frame) - - def _execute(self, incoming_frame): - state = self.costate - try: + def _execute(self, incoming_frame): + state = self.costate try: try: - exc = None - thunk = self.thunk - self.thunk = None - syncstate.switched(incoming_frame) - thunk.call() - resume_point("coroutine__bind", state) - except Exception, e: - exc = e - raise - finally: - # warning! we must reload the 'self' from the costate, - # because after a clone() the 'self' of both copies - # point to the original! - self = state.current - self.finish(exc) - except CoroutineExit: - # ignore a shutdown exception - pass - except Exception, e: - # redirect all unhandled exceptions to the parent - syncstate.push_exception(e) - while self.parent is not None and self.parent.frame is None: - # greenlet behavior is fine - self.parent = self.parent.parent - return state.update(self.parent) + try: + exc = None + thunk = self.thunk + self.thunk = None + syncstate.switched(incoming_frame) + thunk.call() + resume_point("coroutine__bind", state) + except Exception, e: + exc = e + raise + finally: + # warning! we must reload the 'self' from the costate, + # because after a clone() the 'self' of both copies + # point to the original! + self = state.current + self.finish(exc) + except CoroutineExit: + # ignore a shutdown exception + pass + except Exception, e: + # redirect all unhandled exceptions to the parent + syncstate.push_exception(e) + while self.parent is not None and self.parent.frame is None: + # greenlet behavior is fine + self.parent = self.parent.parent + return state.update(self.parent) + + def switch(self): + if self.frame is None: + # considered a programming error. + # greenlets and tasklets have different ideas about this. + raise CoroutineDamage + state = self.costate + incoming_frame = state.update(self).switch() + resume_point("coroutine_switch", state, returns=incoming_frame) + syncstate.switched(incoming_frame) - def switch(self): - if self.frame is None: - # considered a programming error. - # greenlets and tasklets have different ideas about this. - raise CoroutineDamage - state = self.costate - incoming_frame = state.update(self).switch() - resume_point("coroutine_switch", state, returns=incoming_frame) - syncstate.switched(incoming_frame) - - def kill(self): - if self.frame is None: - return - state = self.costate - syncstate.push_exception(CoroutineExit()) - # careful here - if setting self.parent to state.current would - # create a loop, break it. The assumption is that 'self' - # will die, so that state.current's chain of parents can be - # modified to skip 'self' without too many people noticing. - p = state.current - if p is self or self.parent is None: - pass # killing the current of the main - don't change any parent - else: - while p.parent is not None: - if p.parent is self: - p.parent = self.parent - break - p = p.parent - self.parent = state.current - self.switch() - - def _kill_finally(self): - try: - self._userdel() - except Exception: - pass # maybe print a warning? - self.kill() + def kill(self): + if self.frame is None: + return + state = self.costate + syncstate.push_exception(CoroutineExit()) + # careful here - if setting self.parent to state.current would + # create a loop, break it. The assumption is that 'self' + # will die, so that state.current's chain of parents can be + # modified to skip 'self' without too many people noticing. + p = state.current + if p is self or self.parent is None: + pass # killing the current of the main - don't change any parent + else: + while p.parent is not None: + if p.parent is self: + p.parent = self.parent + break + p = p.parent + self.parent = state.current + self.switch() + + def _kill_finally(self): + try: + self._userdel() + except Exception: + pass # maybe print a warning? + self.kill() - __already_postponed = False + __already_postponed = False - def __del__(self): - # provide the necessary clean-up - # note that AppCoroutine has to take care about this - # as well, including a check for user-supplied __del__. - # Additionally note that in the context of __del__, we are - # not in the position to issue a switch. - # we defer it completely. - - # it is necessary to check whether syncstate is None because CPython - # sets it to None when it cleans up the modules, which will lead to - # very strange effects - - if not we_are_translated(): - # we need to make sure that we postpone each coroutine only once on - # top of CPython, because this resurrects the coroutine and CPython - # calls __del__ again, thus postponing and resurrecting the - # coroutine once more :-( - if self.__already_postponed: - return - self.__already_postponed = True - if syncstate is not None: - syncstate.postpone_deletion(self) - - # coroutines need complete control over their __del__ behaviour. In - # particular they need to care about calling space.userdel themselves - handle_del_manually = True + def __del__(self): + # provide the necessary clean-up + # note that AppCoroutine has to take care about this + # as well, including a check for user-supplied __del__. + # Additionally note that in the context of __del__, we are + # not in the position to issue a switch. + # we defer it completely. + + # it is necessary to check whether syncstate is None because CPython + # sets it to None when it cleans up the modules, which will lead to + # very strange effects + + if not we_are_translated(): + # we need to make sure that we postpone each coroutine only once on + # top of CPython, because this resurrects the coroutine and CPython + # calls __del__ again, thus postponing and resurrecting the + # coroutine once more :-( + if self.__already_postponed: + return + self.__already_postponed = True + if syncstate is not None: + syncstate.postpone_deletion(self) + + # coroutines need complete control over their __del__ behaviour. In + # particular they need to care about calling space.userdel themselves + handle_del_manually = True - def _userdel(self): - # override this for exposed coros - pass + def _userdel(self): + # override this for exposed coros + pass + + def is_alive(self): + return self.frame is not None or self is self.costate.current + + def is_zombie(self): + return self.frame is not None and syncstate.check_for_zombie(self) - def is_alive(self): - return self.frame is not None or self is self.costate.current + def getcurrent(): + costate = Coroutine._get_default_costate() + return costate.current + getcurrent = staticmethod(getcurrent) - def is_zombie(self): - return self.frame is not None and syncstate.check_for_zombie(self) + def getmain(): + costate = Coroutine._get_default_costate() + return costate.main + getmain = staticmethod(getmain) - def getcurrent(): - costate = Coroutine._get_default_costate() - return costate.current - getcurrent = staticmethod(getcurrent) - - def getmain(): - costate = Coroutine._get_default_costate() - return costate.main - getmain = staticmethod(getmain) + def hello(self): + "Called when execution is transferred into this coroutine." - def hello(self): - "Called when execution is transferred into this coroutine." + def goodbye(self): + "Called just after execution is transferred away from this coroutine." - def goodbye(self): - "Called just after execution is transferred away from this coroutine." + def finish(self, exc=None): + "stephan forgot me" - def finish(self, exc=None): - "stephan forgot me" + return locals() # _________________________________________________ Copied: pypy/trunk/pypy/rlib/test/test_rcoroutine.py (from r66485, pypy/branch/io-lang/pypy/rlib/test/test_rcoroutine.py) ============================================================================== --- pypy/branch/io-lang/pypy/rlib/test/test_rcoroutine.py (original) +++ pypy/trunk/pypy/rlib/test/test_rcoroutine.py Wed Dec 2 12:12:36 2009 @@ -304,6 +304,7 @@ costate.hello_goodbye = 0 def ep(): + syncstate.default_costate = costate costate.hello_goodbye = 0 c1 = C(4) c1.bind(T()) From cfbolz at codespeak.net Wed Dec 2 12:17:54 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 2 Dec 2009 12:17:54 +0100 (CET) Subject: [pypy-svn] r69831 - pypy/branch/io-lang Message-ID: <20091202111754.8654A1683D1@codespeak.net> Author: cfbolz Date: Wed Dec 2 12:17:53 2009 New Revision: 69831 Removed: pypy/branch/io-lang/ Log: remove branch, now that we got the interesting things out of it. From fijal at codespeak.net Wed Dec 2 13:11:19 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 2 Dec 2009 13:11:19 +0100 (CET) Subject: [pypy-svn] r69833 - in pypy: branch/listcopyop listcopyop Message-ID: <20091202121119.970101683D3@codespeak.net> Author: fijal Date: Wed Dec 2 13:11:17 2009 New Revision: 69833 Added: pypy/branch/listcopyop/ - copied from r69832, pypy/listcopyop/ Removed: pypy/listcopyop/ Log: oops oops, move it to branch From arigo at codespeak.net Wed Dec 2 13:45:33 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 2 Dec 2009 13:45:33 +0100 (CET) Subject: [pypy-svn] r69835 - pypy/extradoc/sprintinfo/leysin-winter-2010 Message-ID: <20091202124533.584521683DC@codespeak.net> Author: arigo Date: Wed Dec 2 13:45:32 2009 New Revision: 69835 Modified: pypy/extradoc/sprintinfo/leysin-winter-2010/announcement.txt Log: Update to the image I used for the blog post. Modified: pypy/extradoc/sprintinfo/leysin-winter-2010/announcement.txt ============================================================================== --- pypy/extradoc/sprintinfo/leysin-winter-2010/announcement.txt (original) +++ pypy/extradoc/sprintinfo/leysin-winter-2010/announcement.txt Wed Dec 2 13:45:32 2009 @@ -1,7 +1,7 @@ ===================================================================== PyPy Leysin Winter Sprint (23-30th January 2010) ===================================================================== -.. image:: http://www.ermina.ch/002.JPG +.. image:: http://www.ermina.ch/003.JPG The next PyPy sprint will be in Leysin, Switzerland, for the seventh time. This is a fully public sprint: newcomers and topics From cfbolz at codespeak.net Wed Dec 2 13:48:41 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 2 Dec 2009 13:48:41 +0100 (CET) Subject: [pypy-svn] r69837 - in pypy/trunk/pypy: bin lang/prolog translator/goal Message-ID: <20091202124841.EEF631683DE@codespeak.net> Author: cfbolz Date: Wed Dec 2 13:48:41 2009 New Revision: 69837 Removed: pypy/trunk/pypy/bin/pyrolog.py pypy/trunk/pypy/lang/prolog/ pypy/trunk/pypy/translator/goal/targetprologstandalone.py Log: kill prolog stuff here From cfbolz at codespeak.net Wed Dec 2 14:19:31 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 2 Dec 2009 14:19:31 +0100 (CET) Subject: [pypy-svn] r69840 - pypy/trunk/pypy/translator/goal Message-ID: <20091202131931.6A99D1683D9@codespeak.net> Author: cfbolz Date: Wed Dec 2 14:19:30 2009 New Revision: 69840 Removed: pypy/trunk/pypy/translator/goal/targetimageloadingsmalltalk.py pypy/trunk/pypy/translator/goal/targettinybenchsmalltalk.py Log: kill targets From cfbolz at codespeak.net Wed Dec 2 15:19:40 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 2 Dec 2009 15:19:40 +0100 (CET) Subject: [pypy-svn] r69845 - pypy/branch/spy-graphic Message-ID: <20091202141940.3775B31813A@codespeak.net> Author: cfbolz Date: Wed Dec 2 15:19:39 2009 New Revision: 69845 Removed: pypy/branch/spy-graphic/ Log: the content of this branch was moved to the new lang dir From cfbolz at codespeak.net Wed Dec 2 15:28:54 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 2 Dec 2009 15:28:54 +0100 (CET) Subject: [pypy-svn] r69846 - pypy/trunk/pypy/lang/smalltalk Message-ID: <20091202142854.B989A168006@codespeak.net> Author: cfbolz Date: Wed Dec 2 15:28:52 2009 New Revision: 69846 Removed: pypy/trunk/pypy/lang/smalltalk/ Log: this was moved to the lang dir From cfbolz at codespeak.net Wed Dec 2 15:45:29 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 2 Dec 2009 15:45:29 +0100 (CET) Subject: [pypy-svn] r69847 - pypy/trunk/pypy/lang/test Message-ID: <20091202144529.A44A1168006@codespeak.net> Author: cfbolz Date: Wed Dec 2 15:45:29 2009 New Revision: 69847 Modified: pypy/trunk/pypy/lang/test/test_translation.py Log: prolog moved away Modified: pypy/trunk/pypy/lang/test/test_translation.py ============================================================================== --- pypy/trunk/pypy/lang/test/test_translation.py (original) +++ pypy/trunk/pypy/lang/test/test_translation.py Wed Dec 2 15:45:29 2009 @@ -29,6 +29,3 @@ def test_js(self): self.translate('targetjsstandalone') - - def test_prolog(self): - self.translate('targetprologstandalone') From cfbolz at codespeak.net Wed Dec 2 17:33:11 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 2 Dec 2009 17:33:11 +0100 (CET) Subject: [pypy-svn] r69849 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091202163311.B3B0116800B@codespeak.net> Author: cfbolz Date: Wed Dec 2 17:33:10 2009 New Revision: 69849 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py Log: Compress the numbering lists. This seems to give a saving of about 10% and makes things slower. I will revert it afterwards, just want to have it checked in as not to lose it. Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Wed Dec 2 17:33:10 2009 @@ -160,13 +160,13 @@ self._really_force() return self.box - def make_virtual_info(self, modifier, fieldnums): + def make_virtual_info(self, modifier, fieldnums_compressed): vinfo = self._cached_vinfo - if vinfo is not None and resume.tagged_list_eq( - vinfo.fieldnums, fieldnums): + if (vinfo is not None and + vinfo.fieldnums_compressed == fieldnums_compressed): return vinfo vinfo = self._make_virtual(modifier) - vinfo.fieldnums = fieldnums + vinfo.fieldnums_compressed = fieldnums_compressed self._cached_vinfo = vinfo return vinfo Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Wed Dec 2 17:33:10 2009 @@ -60,48 +60,97 @@ storage.rd_snapshot = snapshot class Numbering(object): - __slots__ = ('prev', 'nums') + __slots__ = ('prev', 'nums_compressed') def __init__(self, prev, nums): self.prev = prev - self.nums = nums + self.nums_compressed = compress_tagged_list(nums) + + def nums(self): + return uncompress_tagged_list(self.nums_compressed) + +# _________________________________________________________ +# tagging helpers TAGMASK = 3 def tag(value, tagbits): if tagbits >> 2: raise ValueError - sx = value >> 13 - if sx != 0 and sx != -1: + if rarithmetic.intmask((value << 2)) >> 2 != value: raise ValueError - return rffi.r_short(value<<2|tagbits) + return value<<2|tagbits def untag(value): - value = rarithmetic.widen(value) - tagbits = value&TAGMASK + tagbits = value & TAGMASK return value>>2, tagbits -def tagged_eq(x, y): - # please rpython :( - return rarithmetic.widen(x) == rarithmetic.widen(y) - -def tagged_list_eq(tl1, tl2): - if len(tl1) != len(tl2): - return False - for i in range(len(tl1)): - if not tagged_eq(tl1[i], tl2[i]): - return False - return True - TAGCONST = 0 TAGINT = 1 TAGBOX = 2 TAGVIRTUAL = 3 -UNASSIGNED = tag(-2 ** 12 - 1, TAGBOX) -UNASSIGNEDVIRTUAL = tag(-2 ** 12 - 1, TAGVIRTUAL) +MINIMUM_VALUE = -2 ** 12 +UNASSIGNED = tag(MINIMUM_VALUE, TAGBOX) +UNASSIGNEDVIRTUAL = tag(MINIMUM_VALUE, TAGVIRTUAL) NULLREF = tag(-1, TAGCONST) +def compress_tagged_list(l): + res = ['\x00'] * (len(l) * 4) # maximum size + resindex = 0 + for tagged in l: + while True: + rest = tagged >> 7 + # tagged fits into 7 bits, if the remaining int is all zeroes or + # all ones + fits = (rest == 0) | (rest == -1) + # if the highest bit of tagged (which will corresponds to the sign + # bit on uncompressing) does not actually correspond with the sign + # of tagged, we need to output another byte + fits = fits & (bool(tagged & 0x40) == (tagged < 0)) + res[resindex] = chr((tagged & 0x7f) | 0x80 * (not fits)) + resindex += 1 + if fits: + break + tagged = rest + return "".join(res[:resindex]) + +def _decompress_next(s, i): + res = 0 + shift = 0 + while True: + byte = ord(s[i]) + i += 1 + more = bool(byte & 0x80) + byte &= 0x7f + res += byte << shift + if not more: + # sign-extend + if byte & 0x40: + res |= -1 << (shift + 7) + break + shift += 7 + return res, i + +def uncompress_tagged_list(s): + result = [-1] * len(s) # maximum size + i = 0 + resindex = 0 + while i < len(s): + res, i = _decompress_next(s, i) + result[resindex] = res + resindex += 1 + return result[:resindex] + +def compressed_length(s): + res = 0 + for char in s: + if ord(char) & 0x80 == 0: + res += 1 + return res + + +# ____________________________________________________________ class ResumeDataLoopMemo(object): @@ -130,7 +179,7 @@ except ValueError: pass tagged = self.large_ints.get(val, UNASSIGNED) - if not tagged_eq(tagged, UNASSIGNED): + if not tagged == UNASSIGNED: return tagged tagged = self._newconst(const) self.large_ints[val] = tagged @@ -140,7 +189,7 @@ if not val: return NULLREF tagged = self.refs.get(val, UNASSIGNED) - if not tagged_eq(tagged, UNASSIGNED): + if not tagged == UNASSIGNED: return tagged tagged = self._newconst(const) self.refs[val] = tagged @@ -310,13 +359,13 @@ i, tagbits = untag(tagged) if tagbits == TAGBOX: assert box not in self.liveboxes_from_env - assert tagged_eq(tagged, UNASSIGNED) + assert tagged == UNASSIGNED index = memo.assign_number_to_box(box, new_liveboxes) self.liveboxes[box] = tag(index, TAGBOX) count += 1 else: assert tagbits == TAGVIRTUAL - if tagged_eq(tagged, UNASSIGNEDVIRTUAL): + if tagged == UNASSIGNEDVIRTUAL: assert box not in self.liveboxes_from_env index = memo.assign_number_to_virtual(box) self.liveboxes[box] = tag(index, TAGVIRTUAL) @@ -337,10 +386,11 @@ value = values[virtualbox] fieldnums = [self._gettagged(box) for box in fieldboxes] - vinfo = value.make_virtual_info(self, fieldnums) - # if a new vinfo instance is made, we get the fieldnums list we + fieldnums_compressed = compress_tagged_list(fieldnums) + vinfo = value.make_virtual_info(self, fieldnums_compressed) + # if a new vinfo instance is made, we get the string we # pass in as an attribute. hackish. - if vinfo.fieldnums is not fieldnums: + if vinfo.fieldnums_compressed is not fieldnums_compressed: memo.nvreused += 1 virtuals[num] = vinfo @@ -370,25 +420,34 @@ def setfields(self, metainterp, box, fn_decode_box): raise NotImplementedError + def fieldnums(self): + return uncompress_tagged_list(self.fieldnums_compressed) + class AbstractVirtualStructInfo(AbstractVirtualInfo): def __init__(self, fielddescrs): self.fielddescrs = fielddescrs - #self.fieldnums = ... + #self.fieldnums_compressed = ... def setfields(self, metainterp, box, fn_decode_box): - for i in range(len(self.fielddescrs)): - fieldbox = fn_decode_box(self.fieldnums[i]) + fieldnums_compressed = self.fieldnums_compressed + i = 0 + j = 0 + while i < len(fieldnums_compressed): + tagged, i = _decompress_next(fieldnums_compressed, i) + fieldbox = fn_decode_box(tagged) metainterp.execute_and_record(rop.SETFIELD_GC, - self.fielddescrs[i], + self.fielddescrs[j], box, fieldbox) + j += 1 def debug_prints(self): - assert len(self.fielddescrs) == len(self.fieldnums) + fieldnums = self.fieldnums() + assert len(self.fielddescrs) == len(fieldnums) for i in range(len(self.fielddescrs)): debug_print("\t\t", str(self.fielddescrs[i]), - str(untag(self.fieldnums[i]))) + str(untag(fieldnums[i]))) class VirtualInfo(AbstractVirtualStructInfo): def __init__(self, known_class, fielddescrs): @@ -418,24 +477,30 @@ class VArrayInfo(AbstractVirtualInfo): def __init__(self, arraydescr): self.arraydescr = arraydescr - #self.fieldnums = ... + #self.fieldnums_compressed = ... def allocate(self, metainterp): - length = len(self.fieldnums) + length = compressed_length(self.fieldnums_compressed) return metainterp.execute_and_record(rop.NEW_ARRAY, self.arraydescr, ConstInt(length)) def setfields(self, metainterp, box, fn_decode_box): - for i in range(len(self.fieldnums)): - itembox = fn_decode_box(self.fieldnums[i]) + fieldnums_compressed = self.fieldnums_compressed + i = 0 + j = 0 + while i < len(fieldnums_compressed): + tagged, i = _decompress_next(fieldnums_compressed, i) + itembox = fn_decode_box(tagged) metainterp.execute_and_record(rop.SETARRAYITEM_GC, self.arraydescr, - box, ConstInt(i), itembox) + box, ConstInt(j), itembox) + j += 1 def debug_prints(self): debug_print("\tvarrayinfo", self.arraydescr) - for i in self.fieldnums: + fieldnums = self.fieldnums() + for i in fieldnums: debug_print("\t\t", str(untag(i))) @@ -490,18 +555,20 @@ def consume_boxes(self): numb = self.cur_numb assert numb is not None - nums = numb.nums - n = len(nums) + nums_compressed = numb.nums_compressed + n = compressed_length(nums_compressed) boxes = [None] * n + j = 0 for i in range(n): - boxes[i] = self._decode_box(nums[i]) + tagged, j = _decompress_next(nums_compressed, j) + boxes[i] = self._decode_box(tagged) self.cur_numb = numb.prev return boxes def _decode_box(self, tagged): num, tag = untag(tagged) if tag == TAGCONST: - if tagged_eq(tagged, NULLREF): + if tagged == NULLREF: return self.cpu.ts.CONST_NULL return self.consts[num] elif tag == TAGVIRTUAL: @@ -534,7 +601,7 @@ frameinfo = frameinfo.prev numb = storage.rd_numb while numb is not None: - debug_print('\tnumb', str([untag(i) for i in numb.nums]), + debug_print('\tnumb', str([untag(i) for i in numb.nums()]), 'at', compute_unique_id(numb)) numb = numb.prev for const in storage.rd_consts: Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Wed Dec 2 17:33:10 2009 @@ -53,12 +53,12 @@ # opt.store_final_boxes_in_guard(op) if op.fail_args == [b0, b1]: - assert fdescr.rd_numb.nums == [tag(1, TAGBOX)] - assert fdescr.rd_numb.prev.nums == [tag(0, TAGBOX)] + assert fdescr.rd_numb.nums() == [tag(1, TAGBOX)] + assert fdescr.rd_numb.prev.nums() == [tag(0, TAGBOX)] else: assert op.fail_args == [b1, b0] - assert fdescr.rd_numb.nums == [tag(0, TAGBOX)] - assert fdescr.rd_numb.prev.nums == [tag(1, TAGBOX)] + assert fdescr.rd_numb.nums() == [tag(0, TAGBOX)] + assert fdescr.rd_numb.prev.nums() == [tag(1, TAGBOX)] assert fdescr.rd_virtuals is None assert fdescr.rd_consts == [] Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Wed Dec 2 17:33:10 2009 @@ -1,4 +1,5 @@ import py +import sys from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.jit.metainterp.optimizeopt import VirtualValue, OptValue, VArrayValue from pypy.jit.metainterp.optimizeopt import VStructValue @@ -19,10 +20,10 @@ assert tag((1<<13)-1, 3) == rffi.r_short(((1<<15)-1)|3) assert tag(-1<<13, 3) == rffi.r_short((-1<<15)|3) py.test.raises(ValueError, tag, 3, 5) - py.test.raises(ValueError, tag, 1<<13, 0) - py.test.raises(ValueError, tag, (1<<13)+1, 0) - py.test.raises(ValueError, tag, (-1<<13)-1, 0) - py.test.raises(ValueError, tag, (-1<<13)-5, 0) + py.test.raises(ValueError, tag, sys.maxint, 0) + py.test.raises(ValueError, tag, sys.maxint//2, 0) + py.test.raises(ValueError, tag, -sys.maxint-1, 0) + py.test.raises(ValueError, tag, -sys.maxint//2, 0) def test_untag(): assert untag(tag(3, 1)) == (3, 1) @@ -30,15 +31,20 @@ assert untag(tag((1<<13)-1, 3)) == ((1<<13)-1, 3) assert untag(tag(-1<<13, 3)) == (-1<<13, 3) -def test_tagged_eq(): - assert tagged_eq(UNASSIGNED, UNASSIGNED) - assert not tagged_eq(tag(1, TAGBOX), UNASSIGNED) - -def test_tagged_list_eq(): - assert tagged_list_eq([UNASSIGNED, tag(1, TAGBOX), tag(-2, TAGVIRTUAL)], - [UNASSIGNED, tag(1, TAGBOX), tag(-2, TAGVIRTUAL)]) - assert not tagged_list_eq([tag(1, TAGBOX)], [tag(-2, TAGBOX)]) - assert not tagged_list_eq([tag(1, TAGBOX), tag(-2, TAGBOX)], [tag(1, TAGBOX)]) +def test_compress_tagged_list(): + assert compress_tagged_list([0, 1, 12, 1024]) == "\x00\x01\x0c\x80\x08" + assert compress_tagged_list([-1, 1, 12]) == "\x7f\x01\x0c" + for i in range(256): + assert uncompress_tagged_list(compress_tagged_list([i])) == [i] + l = range(-10000, 10000) + range(-100000000, 100000000, 10000) + compressed = compress_tagged_list(l) + assert uncompress_tagged_list(compressed) == l + assert compressed_length(compressed) == len(l) + for i in range(-64, 64): + assert len(compress_tagged_list([i])) == 1 + for i in range(-8192, -64) + range(64, 8192): + assert len(compress_tagged_list([i])) == 2 + class MyMetaInterp: _already_allocated_resume_virtuals = None @@ -190,12 +196,12 @@ l = [1, 2] numb = Numbering(None, l) assert numb.prev is None - assert numb.nums is l + assert numb.nums_compressed == "\x01\x02" - l1 = ['b3'] + l1 = [3] numb1 = Numbering(numb, l1) assert numb1.prev is numb - assert numb1.nums is l1 + assert numb1.nums_compressed == "\x03" def test_capture_resumedata(): b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] @@ -394,8 +400,8 @@ modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish(values) assert len(storage.rd_virtuals) == 1 - assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX), - tag(0, TAGCONST)] + assert storage.rd_virtuals[0].fieldnums() == [tag(-1, TAGBOX), + tag(0, TAGCONST)] b6 = BoxPtr() v6 = virtual_value(b6, c2, None) @@ -405,10 +411,10 @@ modifier = ResumeDataVirtualAdder(storage2, memo) liveboxes2 = modifier.finish(values) assert len(storage2.rd_virtuals) == 2 - assert storage2.rd_virtuals[0].fieldnums == [tag(len(liveboxes2)-1, TAGBOX), - tag(-1, TAGVIRTUAL)] - assert storage2.rd_virtuals[1].fieldnums == [tag(2, TAGINT), - tag(-1, TAGVIRTUAL)] + assert storage2.rd_virtuals[0].fieldnums() == [tag(len(liveboxes2)-1, TAGBOX), + tag(-1, TAGVIRTUAL)] + assert storage2.rd_virtuals[1].fieldnums() == [tag(2, TAGINT), + tag(-1, TAGVIRTUAL)] # now on to resuming metainterp = MyMetaInterp() @@ -450,8 +456,8 @@ modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish(values) assert len(storage.rd_virtuals) == 1 - assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX), - tag(0, TAGCONST)] + assert storage.rd_virtuals[0].fieldnums() == [tag(-1, TAGBOX), + tag(0, TAGCONST)] storage2 = Storage() fs = [FakeFrame("code0", 0, -1, b1, b4, b2)] @@ -460,7 +466,7 @@ modifier = ResumeDataVirtualAdder(storage2, memo) liveboxes = modifier.finish(values) assert len(storage2.rd_virtuals) == 2 - assert storage2.rd_virtuals[1].fieldnums == storage.rd_virtuals[0].fieldnums + assert storage2.rd_virtuals[1].fieldnums() == storage.rd_virtuals[0].fieldnums() assert storage2.rd_virtuals[1] is storage.rd_virtuals[0] @@ -479,10 +485,10 @@ liveboxes = modifier.finish(values) assert liveboxes == [b3] assert len(storage.rd_virtuals) == 2 - assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX), - tag(1, TAGVIRTUAL)] - assert storage.rd_virtuals[1].fieldnums == [tag(-1, TAGBOX), - tag(0, TAGVIRTUAL)] + assert storage.rd_virtuals[0].fieldnums() == [tag(-1, TAGBOX), + tag(1, TAGVIRTUAL)] + assert storage.rd_virtuals[1].fieldnums() == [tag(-1, TAGBOX), + tag(0, TAGVIRTUAL)] # ____________________________________________________________ @@ -494,12 +500,12 @@ assert untag(tagged) == (44, TAGINT) tagged = memo.getconst(ConstInt(-3)) assert untag(tagged) == (-3, TAGINT) - const = ConstInt(50000) + const = ConstInt(sys.maxint) tagged = memo.getconst(const) index, tagbits = untag(tagged) assert tagbits == TAGCONST assert memo.consts[index] is const - tagged = memo.getconst(ConstInt(50000)) + tagged = memo.getconst(ConstInt(sys.maxint)) index2, tagbits = untag(tagged) assert tagbits == TAGCONST assert index2 == index @@ -554,10 +560,10 @@ assert liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b3: tag(2, TAGBOX)} - assert numb.nums == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), - tag(1, TAGINT)] - assert numb.prev.nums == [tag(0, TAGBOX), tag(1, TAGINT), tag(1, TAGBOX), - tag(0, TAGBOX), tag(2, TAGINT)] + assert numb.nums() == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), + tag(1, TAGINT)] + assert numb.prev.nums() == [tag(0, TAGBOX), tag(1, TAGINT), tag(1, TAGBOX), + tag(0, TAGBOX), tag(2, TAGINT)] assert numb.prev.prev is None numb2, liveboxes2, v = memo.number({}, snap2) @@ -566,8 +572,8 @@ assert liveboxes2 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b3: tag(2, TAGBOX)} assert liveboxes2 is not liveboxes - assert numb2.nums == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), - tag(3, TAGINT)] + assert numb2.nums() == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), + tag(3, TAGINT)] assert numb2.prev is numb.prev env3 = [c3, b3, b1, c3] @@ -589,8 +595,8 @@ assert v == 0 assert liveboxes3 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX)} - assert numb3.nums == [tag(3, TAGINT), tag(4, TAGINT), tag(0, TAGBOX), - tag(3, TAGINT)] + assert numb3.nums() == [tag(3, TAGINT), tag(4, TAGINT), tag(0, TAGBOX), + tag(3, TAGINT)] assert numb3.prev is numb.prev # virtual @@ -602,8 +608,8 @@ assert liveboxes4 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b4: tag(0, TAGVIRTUAL)} - assert numb4.nums == [tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX), - tag(3, TAGINT)] + assert numb4.nums() == [tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX), + tag(3, TAGINT)] assert numb4.prev is numb.prev env5 = [b1, b4, b5] @@ -615,8 +621,8 @@ assert liveboxes5 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b4: tag(0, TAGVIRTUAL), b5: tag(1, TAGVIRTUAL)} - assert numb5.nums == [tag(0, TAGBOX), tag(0, TAGVIRTUAL), - tag(1, TAGVIRTUAL)] + assert numb5.nums() == [tag(0, TAGBOX), tag(0, TAGVIRTUAL), + tag(1, TAGVIRTUAL)] assert numb5.prev is numb4 def test_ResumeDataLoopMemo_number_boxes(): @@ -732,14 +738,14 @@ memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) modifier.finish({}) - assert len(memo.consts) == 2 + assert len(memo.consts) == 1 assert storage.rd_consts is memo.consts - b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**17), ConstInt(-65)] + b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(-sys.maxint), ConstInt(-65)] storage2 = make_storage(b1s, b2s, b3s) modifier2 = ResumeDataVirtualAdder(storage2, memo) modifier2.finish({}) - assert len(memo.consts) == 3 + assert len(memo.consts) == 2 assert storage2.rd_consts is memo.consts From cfbolz at codespeak.net Wed Dec 2 17:34:30 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 2 Dec 2009 17:34:30 +0100 (CET) Subject: [pypy-svn] r69850 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091202163430.7FE9116800B@codespeak.net> Author: cfbolz Date: Wed Dec 2 17:34:30 2009 New Revision: 69850 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py Log: revert r69849 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Wed Dec 2 17:34:30 2009 @@ -160,13 +160,13 @@ self._really_force() return self.box - def make_virtual_info(self, modifier, fieldnums_compressed): + def make_virtual_info(self, modifier, fieldnums): vinfo = self._cached_vinfo - if (vinfo is not None and - vinfo.fieldnums_compressed == fieldnums_compressed): + if vinfo is not None and resume.tagged_list_eq( + vinfo.fieldnums, fieldnums): return vinfo vinfo = self._make_virtual(modifier) - vinfo.fieldnums_compressed = fieldnums_compressed + vinfo.fieldnums = fieldnums self._cached_vinfo = vinfo return vinfo Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Wed Dec 2 17:34:30 2009 @@ -60,97 +60,48 @@ storage.rd_snapshot = snapshot class Numbering(object): - __slots__ = ('prev', 'nums_compressed') + __slots__ = ('prev', 'nums') def __init__(self, prev, nums): self.prev = prev - self.nums_compressed = compress_tagged_list(nums) - - def nums(self): - return uncompress_tagged_list(self.nums_compressed) - -# _________________________________________________________ -# tagging helpers + self.nums = nums TAGMASK = 3 def tag(value, tagbits): if tagbits >> 2: raise ValueError - if rarithmetic.intmask((value << 2)) >> 2 != value: + sx = value >> 13 + if sx != 0 and sx != -1: raise ValueError - return value<<2|tagbits + return rffi.r_short(value<<2|tagbits) def untag(value): - tagbits = value & TAGMASK + value = rarithmetic.widen(value) + tagbits = value&TAGMASK return value>>2, tagbits +def tagged_eq(x, y): + # please rpython :( + return rarithmetic.widen(x) == rarithmetic.widen(y) + +def tagged_list_eq(tl1, tl2): + if len(tl1) != len(tl2): + return False + for i in range(len(tl1)): + if not tagged_eq(tl1[i], tl2[i]): + return False + return True + TAGCONST = 0 TAGINT = 1 TAGBOX = 2 TAGVIRTUAL = 3 -MINIMUM_VALUE = -2 ** 12 -UNASSIGNED = tag(MINIMUM_VALUE, TAGBOX) -UNASSIGNEDVIRTUAL = tag(MINIMUM_VALUE, TAGVIRTUAL) +UNASSIGNED = tag(-2 ** 12 - 1, TAGBOX) +UNASSIGNEDVIRTUAL = tag(-2 ** 12 - 1, TAGVIRTUAL) NULLREF = tag(-1, TAGCONST) -def compress_tagged_list(l): - res = ['\x00'] * (len(l) * 4) # maximum size - resindex = 0 - for tagged in l: - while True: - rest = tagged >> 7 - # tagged fits into 7 bits, if the remaining int is all zeroes or - # all ones - fits = (rest == 0) | (rest == -1) - # if the highest bit of tagged (which will corresponds to the sign - # bit on uncompressing) does not actually correspond with the sign - # of tagged, we need to output another byte - fits = fits & (bool(tagged & 0x40) == (tagged < 0)) - res[resindex] = chr((tagged & 0x7f) | 0x80 * (not fits)) - resindex += 1 - if fits: - break - tagged = rest - return "".join(res[:resindex]) - -def _decompress_next(s, i): - res = 0 - shift = 0 - while True: - byte = ord(s[i]) - i += 1 - more = bool(byte & 0x80) - byte &= 0x7f - res += byte << shift - if not more: - # sign-extend - if byte & 0x40: - res |= -1 << (shift + 7) - break - shift += 7 - return res, i - -def uncompress_tagged_list(s): - result = [-1] * len(s) # maximum size - i = 0 - resindex = 0 - while i < len(s): - res, i = _decompress_next(s, i) - result[resindex] = res - resindex += 1 - return result[:resindex] - -def compressed_length(s): - res = 0 - for char in s: - if ord(char) & 0x80 == 0: - res += 1 - return res - - -# ____________________________________________________________ class ResumeDataLoopMemo(object): @@ -179,7 +130,7 @@ except ValueError: pass tagged = self.large_ints.get(val, UNASSIGNED) - if not tagged == UNASSIGNED: + if not tagged_eq(tagged, UNASSIGNED): return tagged tagged = self._newconst(const) self.large_ints[val] = tagged @@ -189,7 +140,7 @@ if not val: return NULLREF tagged = self.refs.get(val, UNASSIGNED) - if not tagged == UNASSIGNED: + if not tagged_eq(tagged, UNASSIGNED): return tagged tagged = self._newconst(const) self.refs[val] = tagged @@ -359,13 +310,13 @@ i, tagbits = untag(tagged) if tagbits == TAGBOX: assert box not in self.liveboxes_from_env - assert tagged == UNASSIGNED + assert tagged_eq(tagged, UNASSIGNED) index = memo.assign_number_to_box(box, new_liveboxes) self.liveboxes[box] = tag(index, TAGBOX) count += 1 else: assert tagbits == TAGVIRTUAL - if tagged == UNASSIGNEDVIRTUAL: + if tagged_eq(tagged, UNASSIGNEDVIRTUAL): assert box not in self.liveboxes_from_env index = memo.assign_number_to_virtual(box) self.liveboxes[box] = tag(index, TAGVIRTUAL) @@ -386,11 +337,10 @@ value = values[virtualbox] fieldnums = [self._gettagged(box) for box in fieldboxes] - fieldnums_compressed = compress_tagged_list(fieldnums) - vinfo = value.make_virtual_info(self, fieldnums_compressed) - # if a new vinfo instance is made, we get the string we + vinfo = value.make_virtual_info(self, fieldnums) + # if a new vinfo instance is made, we get the fieldnums list we # pass in as an attribute. hackish. - if vinfo.fieldnums_compressed is not fieldnums_compressed: + if vinfo.fieldnums is not fieldnums: memo.nvreused += 1 virtuals[num] = vinfo @@ -420,34 +370,25 @@ def setfields(self, metainterp, box, fn_decode_box): raise NotImplementedError - def fieldnums(self): - return uncompress_tagged_list(self.fieldnums_compressed) - class AbstractVirtualStructInfo(AbstractVirtualInfo): def __init__(self, fielddescrs): self.fielddescrs = fielddescrs - #self.fieldnums_compressed = ... + #self.fieldnums = ... def setfields(self, metainterp, box, fn_decode_box): - fieldnums_compressed = self.fieldnums_compressed - i = 0 - j = 0 - while i < len(fieldnums_compressed): - tagged, i = _decompress_next(fieldnums_compressed, i) - fieldbox = fn_decode_box(tagged) + for i in range(len(self.fielddescrs)): + fieldbox = fn_decode_box(self.fieldnums[i]) metainterp.execute_and_record(rop.SETFIELD_GC, - self.fielddescrs[j], + self.fielddescrs[i], box, fieldbox) - j += 1 def debug_prints(self): - fieldnums = self.fieldnums() - assert len(self.fielddescrs) == len(fieldnums) + assert len(self.fielddescrs) == len(self.fieldnums) for i in range(len(self.fielddescrs)): debug_print("\t\t", str(self.fielddescrs[i]), - str(untag(fieldnums[i]))) + str(untag(self.fieldnums[i]))) class VirtualInfo(AbstractVirtualStructInfo): def __init__(self, known_class, fielddescrs): @@ -477,30 +418,24 @@ class VArrayInfo(AbstractVirtualInfo): def __init__(self, arraydescr): self.arraydescr = arraydescr - #self.fieldnums_compressed = ... + #self.fieldnums = ... def allocate(self, metainterp): - length = compressed_length(self.fieldnums_compressed) + length = len(self.fieldnums) return metainterp.execute_and_record(rop.NEW_ARRAY, self.arraydescr, ConstInt(length)) def setfields(self, metainterp, box, fn_decode_box): - fieldnums_compressed = self.fieldnums_compressed - i = 0 - j = 0 - while i < len(fieldnums_compressed): - tagged, i = _decompress_next(fieldnums_compressed, i) - itembox = fn_decode_box(tagged) + for i in range(len(self.fieldnums)): + itembox = fn_decode_box(self.fieldnums[i]) metainterp.execute_and_record(rop.SETARRAYITEM_GC, self.arraydescr, - box, ConstInt(j), itembox) - j += 1 + box, ConstInt(i), itembox) def debug_prints(self): debug_print("\tvarrayinfo", self.arraydescr) - fieldnums = self.fieldnums() - for i in fieldnums: + for i in self.fieldnums: debug_print("\t\t", str(untag(i))) @@ -555,20 +490,18 @@ def consume_boxes(self): numb = self.cur_numb assert numb is not None - nums_compressed = numb.nums_compressed - n = compressed_length(nums_compressed) + nums = numb.nums + n = len(nums) boxes = [None] * n - j = 0 for i in range(n): - tagged, j = _decompress_next(nums_compressed, j) - boxes[i] = self._decode_box(tagged) + boxes[i] = self._decode_box(nums[i]) self.cur_numb = numb.prev return boxes def _decode_box(self, tagged): num, tag = untag(tagged) if tag == TAGCONST: - if tagged == NULLREF: + if tagged_eq(tagged, NULLREF): return self.cpu.ts.CONST_NULL return self.consts[num] elif tag == TAGVIRTUAL: @@ -601,7 +534,7 @@ frameinfo = frameinfo.prev numb = storage.rd_numb while numb is not None: - debug_print('\tnumb', str([untag(i) for i in numb.nums()]), + debug_print('\tnumb', str([untag(i) for i in numb.nums]), 'at', compute_unique_id(numb)) numb = numb.prev for const in storage.rd_consts: Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Wed Dec 2 17:34:30 2009 @@ -53,12 +53,12 @@ # opt.store_final_boxes_in_guard(op) if op.fail_args == [b0, b1]: - assert fdescr.rd_numb.nums() == [tag(1, TAGBOX)] - assert fdescr.rd_numb.prev.nums() == [tag(0, TAGBOX)] + assert fdescr.rd_numb.nums == [tag(1, TAGBOX)] + assert fdescr.rd_numb.prev.nums == [tag(0, TAGBOX)] else: assert op.fail_args == [b1, b0] - assert fdescr.rd_numb.nums() == [tag(0, TAGBOX)] - assert fdescr.rd_numb.prev.nums() == [tag(1, TAGBOX)] + assert fdescr.rd_numb.nums == [tag(0, TAGBOX)] + assert fdescr.rd_numb.prev.nums == [tag(1, TAGBOX)] assert fdescr.rd_virtuals is None assert fdescr.rd_consts == [] Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Wed Dec 2 17:34:30 2009 @@ -1,5 +1,4 @@ import py -import sys from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.jit.metainterp.optimizeopt import VirtualValue, OptValue, VArrayValue from pypy.jit.metainterp.optimizeopt import VStructValue @@ -20,10 +19,10 @@ assert tag((1<<13)-1, 3) == rffi.r_short(((1<<15)-1)|3) assert tag(-1<<13, 3) == rffi.r_short((-1<<15)|3) py.test.raises(ValueError, tag, 3, 5) - py.test.raises(ValueError, tag, sys.maxint, 0) - py.test.raises(ValueError, tag, sys.maxint//2, 0) - py.test.raises(ValueError, tag, -sys.maxint-1, 0) - py.test.raises(ValueError, tag, -sys.maxint//2, 0) + py.test.raises(ValueError, tag, 1<<13, 0) + py.test.raises(ValueError, tag, (1<<13)+1, 0) + py.test.raises(ValueError, tag, (-1<<13)-1, 0) + py.test.raises(ValueError, tag, (-1<<13)-5, 0) def test_untag(): assert untag(tag(3, 1)) == (3, 1) @@ -31,20 +30,15 @@ assert untag(tag((1<<13)-1, 3)) == ((1<<13)-1, 3) assert untag(tag(-1<<13, 3)) == (-1<<13, 3) -def test_compress_tagged_list(): - assert compress_tagged_list([0, 1, 12, 1024]) == "\x00\x01\x0c\x80\x08" - assert compress_tagged_list([-1, 1, 12]) == "\x7f\x01\x0c" - for i in range(256): - assert uncompress_tagged_list(compress_tagged_list([i])) == [i] - l = range(-10000, 10000) + range(-100000000, 100000000, 10000) - compressed = compress_tagged_list(l) - assert uncompress_tagged_list(compressed) == l - assert compressed_length(compressed) == len(l) - for i in range(-64, 64): - assert len(compress_tagged_list([i])) == 1 - for i in range(-8192, -64) + range(64, 8192): - assert len(compress_tagged_list([i])) == 2 - +def test_tagged_eq(): + assert tagged_eq(UNASSIGNED, UNASSIGNED) + assert not tagged_eq(tag(1, TAGBOX), UNASSIGNED) + +def test_tagged_list_eq(): + assert tagged_list_eq([UNASSIGNED, tag(1, TAGBOX), tag(-2, TAGVIRTUAL)], + [UNASSIGNED, tag(1, TAGBOX), tag(-2, TAGVIRTUAL)]) + assert not tagged_list_eq([tag(1, TAGBOX)], [tag(-2, TAGBOX)]) + assert not tagged_list_eq([tag(1, TAGBOX), tag(-2, TAGBOX)], [tag(1, TAGBOX)]) class MyMetaInterp: _already_allocated_resume_virtuals = None @@ -196,12 +190,12 @@ l = [1, 2] numb = Numbering(None, l) assert numb.prev is None - assert numb.nums_compressed == "\x01\x02" + assert numb.nums is l - l1 = [3] + l1 = ['b3'] numb1 = Numbering(numb, l1) assert numb1.prev is numb - assert numb1.nums_compressed == "\x03" + assert numb1.nums is l1 def test_capture_resumedata(): b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] @@ -400,8 +394,8 @@ modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish(values) assert len(storage.rd_virtuals) == 1 - assert storage.rd_virtuals[0].fieldnums() == [tag(-1, TAGBOX), - tag(0, TAGCONST)] + assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX), + tag(0, TAGCONST)] b6 = BoxPtr() v6 = virtual_value(b6, c2, None) @@ -411,10 +405,10 @@ modifier = ResumeDataVirtualAdder(storage2, memo) liveboxes2 = modifier.finish(values) assert len(storage2.rd_virtuals) == 2 - assert storage2.rd_virtuals[0].fieldnums() == [tag(len(liveboxes2)-1, TAGBOX), - tag(-1, TAGVIRTUAL)] - assert storage2.rd_virtuals[1].fieldnums() == [tag(2, TAGINT), - tag(-1, TAGVIRTUAL)] + assert storage2.rd_virtuals[0].fieldnums == [tag(len(liveboxes2)-1, TAGBOX), + tag(-1, TAGVIRTUAL)] + assert storage2.rd_virtuals[1].fieldnums == [tag(2, TAGINT), + tag(-1, TAGVIRTUAL)] # now on to resuming metainterp = MyMetaInterp() @@ -456,8 +450,8 @@ modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish(values) assert len(storage.rd_virtuals) == 1 - assert storage.rd_virtuals[0].fieldnums() == [tag(-1, TAGBOX), - tag(0, TAGCONST)] + assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX), + tag(0, TAGCONST)] storage2 = Storage() fs = [FakeFrame("code0", 0, -1, b1, b4, b2)] @@ -466,7 +460,7 @@ modifier = ResumeDataVirtualAdder(storage2, memo) liveboxes = modifier.finish(values) assert len(storage2.rd_virtuals) == 2 - assert storage2.rd_virtuals[1].fieldnums() == storage.rd_virtuals[0].fieldnums() + assert storage2.rd_virtuals[1].fieldnums == storage.rd_virtuals[0].fieldnums assert storage2.rd_virtuals[1] is storage.rd_virtuals[0] @@ -485,10 +479,10 @@ liveboxes = modifier.finish(values) assert liveboxes == [b3] assert len(storage.rd_virtuals) == 2 - assert storage.rd_virtuals[0].fieldnums() == [tag(-1, TAGBOX), - tag(1, TAGVIRTUAL)] - assert storage.rd_virtuals[1].fieldnums() == [tag(-1, TAGBOX), - tag(0, TAGVIRTUAL)] + assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX), + tag(1, TAGVIRTUAL)] + assert storage.rd_virtuals[1].fieldnums == [tag(-1, TAGBOX), + tag(0, TAGVIRTUAL)] # ____________________________________________________________ @@ -500,12 +494,12 @@ assert untag(tagged) == (44, TAGINT) tagged = memo.getconst(ConstInt(-3)) assert untag(tagged) == (-3, TAGINT) - const = ConstInt(sys.maxint) + const = ConstInt(50000) tagged = memo.getconst(const) index, tagbits = untag(tagged) assert tagbits == TAGCONST assert memo.consts[index] is const - tagged = memo.getconst(ConstInt(sys.maxint)) + tagged = memo.getconst(ConstInt(50000)) index2, tagbits = untag(tagged) assert tagbits == TAGCONST assert index2 == index @@ -560,10 +554,10 @@ assert liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b3: tag(2, TAGBOX)} - assert numb.nums() == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), - tag(1, TAGINT)] - assert numb.prev.nums() == [tag(0, TAGBOX), tag(1, TAGINT), tag(1, TAGBOX), - tag(0, TAGBOX), tag(2, TAGINT)] + assert numb.nums == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), + tag(1, TAGINT)] + assert numb.prev.nums == [tag(0, TAGBOX), tag(1, TAGINT), tag(1, TAGBOX), + tag(0, TAGBOX), tag(2, TAGINT)] assert numb.prev.prev is None numb2, liveboxes2, v = memo.number({}, snap2) @@ -572,8 +566,8 @@ assert liveboxes2 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b3: tag(2, TAGBOX)} assert liveboxes2 is not liveboxes - assert numb2.nums() == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), - tag(3, TAGINT)] + assert numb2.nums == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), + tag(3, TAGINT)] assert numb2.prev is numb.prev env3 = [c3, b3, b1, c3] @@ -595,8 +589,8 @@ assert v == 0 assert liveboxes3 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX)} - assert numb3.nums() == [tag(3, TAGINT), tag(4, TAGINT), tag(0, TAGBOX), - tag(3, TAGINT)] + assert numb3.nums == [tag(3, TAGINT), tag(4, TAGINT), tag(0, TAGBOX), + tag(3, TAGINT)] assert numb3.prev is numb.prev # virtual @@ -608,8 +602,8 @@ assert liveboxes4 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b4: tag(0, TAGVIRTUAL)} - assert numb4.nums() == [tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX), - tag(3, TAGINT)] + assert numb4.nums == [tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX), + tag(3, TAGINT)] assert numb4.prev is numb.prev env5 = [b1, b4, b5] @@ -621,8 +615,8 @@ assert liveboxes5 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b4: tag(0, TAGVIRTUAL), b5: tag(1, TAGVIRTUAL)} - assert numb5.nums() == [tag(0, TAGBOX), tag(0, TAGVIRTUAL), - tag(1, TAGVIRTUAL)] + assert numb5.nums == [tag(0, TAGBOX), tag(0, TAGVIRTUAL), + tag(1, TAGVIRTUAL)] assert numb5.prev is numb4 def test_ResumeDataLoopMemo_number_boxes(): @@ -738,14 +732,14 @@ memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) modifier.finish({}) - assert len(memo.consts) == 1 + assert len(memo.consts) == 2 assert storage.rd_consts is memo.consts - b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(-sys.maxint), ConstInt(-65)] + b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**17), ConstInt(-65)] storage2 = make_storage(b1s, b2s, b3s) modifier2 = ResumeDataVirtualAdder(storage2, memo) modifier2.finish({}) - assert len(memo.consts) == 2 + assert len(memo.consts) == 3 assert storage2.rd_consts is memo.consts From cfbolz at codespeak.net Wed Dec 2 17:41:39 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 2 Dec 2009 17:41:39 +0100 (CET) Subject: [pypy-svn] r69851 - pypy/extradoc/planning Message-ID: <20091202164139.9A64416800B@codespeak.net> Author: cfbolz Date: Wed Dec 2 17:41:38 2009 New Revision: 69851 Modified: pypy/extradoc/planning/jit.txt Log: this is done now Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Wed Dec 2 17:41:38 2009 @@ -35,8 +35,6 @@ - look into failing pypy-c-jit apptests, pypy-c-jit translate.py -- improve test running, compile only once - - we would like probably enabling sys.settrace() to leave the jit instead of just being ignored - list copy as ll operations, to avoid calling write barriers again and From fijal at codespeak.net Wed Dec 2 18:00:24 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 2 Dec 2009 18:00:24 +0100 (CET) Subject: [pypy-svn] r69852 - in pypy/branch/listcopyop/pypy/rpython: lltypesystem memory/test Message-ID: <20091202170024.6B23716800B@codespeak.net> Author: fijal Date: Wed Dec 2 18:00:23 2009 New Revision: 69852 Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py Log: Add a copy list operation. Right now it does not do anything special at all Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py Wed Dec 2 18:00:23 2009 @@ -358,6 +358,7 @@ 'resize_buffer': LLOp(canraise=(MemoryError,), canunwindgc=True), 'finish_building_buffer' : LLOp(canraise=(MemoryError,), canunwindgc=True), 'zero_gc_pointers_inside': LLOp(), + 'listcopy': LLOp(canrun=True), 'free': LLOp(), 'getfield': LLOp(sideeffects=False, canrun=True), 'getarrayitem': LLOp(sideeffects=False, canrun=True), Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py Wed Dec 2 18:00:23 2009 @@ -394,6 +394,10 @@ checkadr(addr2) return addr1 - addr2 +def op_listcopy(source, dest, source_start, dest_start, length): + for i in range(length): + dest[i + dest_start] = source[i + source_start] + def op_getfield(p, name): checkptr(p) TYPE = lltype.typeOf(p).TO Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py Wed Dec 2 18:00:23 2009 @@ -551,6 +551,19 @@ res = self.interpret(fn, [-1000], taggedpointers=True) assert res == 111 + def test_listcopy(self): + TP = lltype.GcArray(lltype.Signed) + def fn(): + l = lltype.malloc(TP, 100) + for i in range(100): + l[i] = 1 + l2 = lltype.malloc(TP, 50) + llop.listcopy(lltype.Void, l, l2, 50, 0, 50) + for i in range(50): + assert l2[i] == 1 + + self.interpret(fn, []) + from pypy.rlib.objectmodel import UnboxedValue From afa at codespeak.net Wed Dec 2 18:06:29 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 2 Dec 2009 18:06:29 +0100 (CET) Subject: [pypy-svn] r69853 - pypy/trunk/pypy/module/oracle Message-ID: <20091202170629.1D28416800B@codespeak.net> Author: afa Date: Wed Dec 2 18:06:28 2009 New Revision: 69853 Modified: pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_error.py Log: Don't import other library modules at compile-time, they may instanciate prebuilt constants that the translator won't be able to render. Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Wed Dec 2 18:06:28 2009 @@ -23,8 +23,6 @@ 'CLOB': 'interp_variable.VT_CLOB', 'OBJECT': 'interp_variable.VT_Object', 'Variable': 'interp_variable.W_Variable', - 'Timestamp': 'interp_error.get(space).w_DateTimeType', - 'Date': 'interp_error.get(space).w_DateType', 'SessionPool': 'interp_pool.W_SessionPool', } @@ -38,3 +36,17 @@ ProgrammingError Warning""".split(): appleveldefs[name] = "app_oracle.%s" % (name,) + def startup(self, space): + from pypy.module.oracle.interp_error import get + state = get(space) + (state.w_DecimalType, + state.w_DateTimeType, state.w_DateType, state.w_TimedeltaType, + ) = space.fixedview(space.appexec([], """(): + import decimal, datetime + return (decimal.Decimal, + datetime.datetime, datetime.date, datetime.timedelta) + """)) + space.setattr(space.wrap(self), + space.wrap("Timestamp"), state.w_DateTimeType) + space.setattr(space.wrap(self), + space.wrap("Date"), state.w_DateType) Modified: pypy/trunk/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_error.py (original) +++ pypy/trunk/pypy/module/oracle/interp_error.py Wed Dec 2 18:06:28 2009 @@ -24,16 +24,6 @@ self.w_Variable = get('Variable') self.w_Connection = get('Connection') - w_import = space.builtin.get('__import__') - w_decimal = space.call(w_import, space.newlist( - [space.wrap('decimal')])) - self.w_DecimalType = space.getattr(w_decimal, space.wrap("Decimal")) - w_datetime = space.call(w_import, space.newlist( - [space.wrap('datetime')])) - self.w_DateTimeType = space.getattr(w_datetime, space.wrap("datetime")) - self.w_DateType = space.getattr(w_datetime, space.wrap("date")) - self.w_TimedeltaType = space.getattr(w_datetime, space.wrap("timedelta")) - from pypy.module.oracle.interp_variable import all_variable_types self.variableTypeByPythonType = {} for varType in all_variable_types: From afa at codespeak.net Wed Dec 2 18:08:25 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 2 Dec 2009 18:08:25 +0100 (CET) Subject: [pypy-svn] r69854 - pypy/trunk/pypy/module/oracle Message-ID: <20091202170825.CCB7916800B@codespeak.net> Author: afa Date: Wed Dec 2 18:08:25 2009 New Revision: 69854 Modified: pypy/trunk/pypy/module/oracle/roci.py Log: Dance around the oci.h header which "#define boolean int" that broke other files which whant to use "boolean" as an identifier. Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Wed Dec 2 18:08:25 2009 @@ -14,19 +14,27 @@ "Please set ORACLE_HOME to the root of an Oracle client installation") if sys.platform == 'win32': - eci = ExternalCompilationInfo( - includes = ['oci.h'], - include_dirs = [str(ORACLE_HOME.join('OCI', 'include'))], - libraries = ['oci'], - library_dirs = [str(ORACLE_HOME.join('OCI', 'lib', 'MSVC'))], - ) + include_dirs = [str(ORACLE_HOME.join('OCI', 'include'))] + libraries = ['oci'] + library_dirs = [str(ORACLE_HOME.join('OCI', 'lib', 'MSVC'))] else: - eci = ExternalCompilationInfo( - includes = ['oci.h'], - include_dirs = [str(ORACLE_HOME.join('sdk', 'include'))], - libraries = ['clntsh'], - library_dirs = [str(ORACLE_HOME.join('lib'))], - ) + include_dirs = [str(ORACLE_HOME.join('sdk', 'include'))] + libraries = ['clntsh'] + library_dirs = [str(ORACLE_HOME.join('lib'))] + +eci = ExternalCompilationInfo( + post_include_bits = [ + # One single string, to be sure it will + # be rendered in this order + '#include \n' + + 'typedef boolean oci_boolean;\n' + + '#undef boolean' + ], + include_dirs = include_dirs, + libraries = libraries, + library_dirs = library_dirs, + ) + class CConfig: _compilation_info_ = eci @@ -39,7 +47,7 @@ sb4 = platform.SimpleType('sb4', rffi.INT) sword = platform.SimpleType('sword', rffi.INT) uword = platform.SimpleType('uword', rffi.UINT) - boolean = platform.SimpleType('boolean', rffi.UINT) + boolean = platform.SimpleType('oci_boolean', rffi.UINT) OCIDuration = platform.SimpleType('OCIDuration', rffi.UINT) OCIInd = platform.SimpleType('OCIInd', rffi.INT) OCIPinOpt = platform.SimpleType('OCIPinOpt', rffi.INT) From fijal at codespeak.net Wed Dec 2 20:47:22 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 2 Dec 2009 20:47:22 +0100 (CET) Subject: [pypy-svn] r69855 - in pypy/branch/listcopyop/pypy: rpython/lltypesystem rpython/memory/gctransform rpython/memory/test translator/c/test Message-ID: <20091202194722.32C0316800B@codespeak.net> Author: fijal Date: Wed Dec 2 20:47:20 2009 New Revision: 69855 Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py Log: Implement basic transformation by annotating stuff in opimpl. Probably not the smartest idea Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py Wed Dec 2 20:47:20 2009 @@ -395,8 +395,10 @@ return addr1 - addr2 def op_listcopy(source, dest, source_start, dest_start, length): - for i in range(length): + i = 0 + while i < length: dest[i + dest_start] = source[i + source_start] + i += 1 def op_getfield(p, name): checkptr(p) Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py Wed Dec 2 20:47:20 2009 @@ -12,7 +12,7 @@ from pypy.translator.backendopt.ssa import DataFlowFamilyBuilder from pypy.translator.backendopt.constfold import constant_fold_graph from pypy.annotation import model as annmodel -from pypy.rpython import rmodel, annlowlevel +from pypy.rpython import rmodel from pypy.rpython.memory import gc from pypy.rpython.memory.gctransform.support import var_ispyobj from pypy.rpython.annlowlevel import MixLevelHelperAnnotator @@ -380,6 +380,16 @@ def gct_zero_gc_pointers_inside(self, hop): pass + def gct_listcopy(self, hop): + # by default, this becomes a raw memcopy + op = hop.spaceop + from pypy.rpython.lltypesystem import opimpl + llptr = self.annotate_helper(opimpl.op_listcopy, + [arg.concretetype for arg in op.args], + op.result.concretetype, inline=True) + c_func = rmodel.inputconst(lltype.Void, llptr) + hop.genop('direct_call', [c_func] + op.args) + def gct_gc_identityhash(self, hop): # must be implemented in the various GCs raise NotImplementedError Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Wed Dec 2 20:47:20 2009 @@ -843,6 +843,25 @@ assert (totsize - 26 * size_of_int) % 4 == 0 # ^^^ a crude assumption that totsize - varsize would be dividable by 4 # (and give fixedsize) + + + def define_listcopy(cls): + TP = lltype.GcArray(lltype.Signed) + def fn(): + l = lltype.malloc(TP, 100) + for i in range(100): + l[i] = 1 + l2 = lltype.malloc(TP, 50) + llop.listcopy(lltype.Void, l, l2, 50, 0, 50) + for i in range(50): + assert l2[i] == 1 + return 0 + + return fn + + def test_listcopy(self): + run = self.runner("listcopy") + run([]) # ________________________________________________________________ Modified: pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py Wed Dec 2 20:47:20 2009 @@ -1027,6 +1027,24 @@ res = self.run("tagged") assert res == expected + + def define_listcopy(cls): + TP = lltype.GcArray(lltype.Signed) + def fn(): + l = lltype.malloc(TP, 100) + for i in range(100): + l[i] = 1 + l2 = lltype.malloc(TP, 50) + llop.listcopy(lltype.Void, l, l2, 50, 0, 50) + for i in range(50): + assert l2[i] == 1 + return 0 + + return fn + + def test_listcopy(self): + self.run("listcopy") + from pypy.rlib.objectmodel import UnboxedValue class TaggedBase(object): From afa at codespeak.net Thu Dec 3 01:59:08 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 3 Dec 2009 01:59:08 +0100 (CET) Subject: [pypy-svn] r69856 - pypy/trunk/pypy/doc/discussion Message-ID: <20091203005908.51709168006@codespeak.net> Author: afa Date: Thu Dec 3 01:59:07 2009 New Revision: 69856 Added: pypy/trunk/pypy/doc/discussion/improve-rpython.txt Log: save a first draft of random ideas about rpython Added: pypy/trunk/pypy/doc/discussion/improve-rpython.txt ============================================================================== --- (empty file) +++ pypy/trunk/pypy/doc/discussion/improve-rpython.txt Thu Dec 3 01:59:07 2009 @@ -0,0 +1,93 @@ +Possible improvements of the rpython language +============================================= + +Improve the interpreter API +--------------------------- + +- Rationalize the modules, and the names, of the differents functions needed to + implement a pypy module. A typical rpython file is likely to contain many + `import` statements:: + + from pypy.interpreter.baseobjspace import Wrappable + from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped + from pypy.interpreter.argument import Arguments + from pypy.interpreter.typedef import TypeDef, GetSetProperty + from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w + from pypy.interpreter.gateway import interp2app + from pypy.interpreter.error import OperationError + from pypy.rpython.lltypesystem import rffi, lltype + +- A more direct declarative way to write Typedef:: + + class W_Socket(Wrappable): + _typedef_name_ = 'socket' + _typedef_base_ = W_EventualBaseClass + + @interp2app_method("connect", ['self', ObjSpace, W_Root]) + def connect_w(self, space, w_addr): + ... + +- Support for metaclasses written in rpython. For a sample, see the skipped test + `pypy.objspace.std.test.TestTypeObject.test_metaclass_typedef` + +RPython language +---------------- + +- Arithmetic with unsigned integer, and between integer of different signedness, + when this is not ambiguous. At least, comparison and assignement with + constants should be allowed. + +- Allocate variables on the stack, and pass their address ("by reference") to + llexternal functions. For a typical usage, see + `pypy.rlib.rsocket.RSocket.getsockopt_int`. + +- Support context managers and the `with` statement. This could be a workaround + before the previous point is available. + +Extensible type system for llexternal +------------------------------------- + +llexternal allows the description of a C function, and conveys the same +information about the arguments as a C header. But this is often not enough. +For example, a parameter of type `int*` is converted to +`rffi.CArrayPtr(rffi.INT)`, but this information is not enough to use the +function. The parameter could be an array of int, a reference to a single value, +for input or output... + +A "type system" could hold this additional information, and automatically +generate some conversion code to ease the usage of the function from +rpython. For example:: + + # double frexp(double x, int *exp); + frexp = llexternal("frexp", [rffi.DOUBLE, OutPtr(rffi.int)], rffi.DOUBLE) + +`OutPtr` indicates that the parameter is output-only, which need not to be +initialized, and which *value* is returned to the caller. In rpython the call +becomes:: + + fraction, exponent = frexp(value) + +Also, we could imagine that one item in the llexternal argument list corresponds +to two parameters in C. Here, OutCharBufferN indicates that the caller will pass +a rpython string; the framework will pass buffer and length to the function:: + + # ssize_t write(int fd, const void *buf, size_t count); + write = llexternal("write", [rffi.INT, CharBufferAndSize], rffi.SSIZE_T) + +The rpython code that calls this function is very simple:: + + written = write(fd, data) + +compared with the present:: + + count = len(data) + buf = rffi.get_nonmovingbuffer(data) + try: + written = rffi.cast(lltype.Signed, os_write( + rffi.cast(rffi.INT, fd), + buf, rffi.cast(rffi.SIZE_T, count))) + finally: + rffi.free_nonmovingbuffer(data, buf) + +Typemaps are very useful for large APIs where the same conversions are needed in +many places. XXX example From fijal at codespeak.net Thu Dec 3 11:12:55 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 3 Dec 2009 11:12:55 +0100 (CET) Subject: [pypy-svn] r69857 - in pypy/branch/listcopyop/pypy/rpython/memory: gc gctransform test Message-ID: <20091203101255.7458216800D@codespeak.net> Author: fijal Date: Thu Dec 3 11:12:53 2009 New Revision: 69857 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/support.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Log: intermediate checkin * Specialize listcopy over lltype * Support having listcopy on gc, so one can overwrite it * Write a test, but it does not crash yet because of lack of write barrier, need to improve Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Thu Dec 3 11:12:53 2009 @@ -5,7 +5,7 @@ from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE -from pypy.rlib.objectmodel import free_non_gc_object +from pypy.rlib.objectmodel import free_non_gc_object, specialize from pypy.rlib.debug import ll_assert from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.lltypesystem.lloperation import llop @@ -475,6 +475,20 @@ objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS self.last_generation_root_objects.append(addr_struct) + @specialize.arglltype(0) + def listcopy(source, dest, source_start, dest_start, length): + TP = lltype.typeOf(source).TO + if isinstance(TP.OF, lltype.Ptr): + pass # do write barrier + source_addr = (llmemory.cast_ptr_to_adr(source) + + llmemory.itemoffsetof(TP, 0) + + llmemory.sizeof(TP.OF) * source_start) + dest_addr = (llmemory.cast_ptr_to_adr(dest) + + llmemory.itemoffsetof(TP, 0) + + llmemory.sizeof(TP.OF) * dest_start) + llmemory.raw_memcopy(source_addr, dest_addr, + llmemory.sizeof(TP.OF) * length) + def write_into_last_generation_obj(self, addr_struct, addr): objhdr = self.header(addr_struct) if objhdr.tid & GCFLAG_NO_HEAP_PTRS: Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py Thu Dec 3 11:12:53 2009 @@ -776,6 +776,16 @@ TYPE = v_ob.concretetype.TO gen_zero_gc_pointers(TYPE, v_ob, hop.llops) + def gct_listcopy(self, hop): + if not hasattr(self.GCClass, 'listcopy'): + return GCTransformer.gct_listcopy(self, hop) + op = hop.spaceop + ARGS = [arg.concretetype for arg in op.args] + llptr = self.annotate_helper(self.GCClass.listcopy.im_func, ARGS, + op.result.concretetype, inline=True) + c_func = rmodel.inputconst(lltype.Void, llptr) + hop.genop('direct_call', [c_func] + op.args) + def gct_weakref_create(self, hop): op = hop.spaceop Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/support.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/support.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/support.py Thu Dec 3 11:12:53 2009 @@ -1,6 +1,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.extregistry import ExtRegistryEntry from pypy.annotation import model as annmodel +from pypy.rlib.objectmodel import specialize import os def var_ispyobj(var): @@ -106,3 +107,10 @@ os.write(2, "a destructor raised an exception, ignoring it\n") except: pass + + at specialize.arglltype(0) +def ll_listcopy(source, dest, source_start, dest_start, length): + i = 0 + while i < length: + dest[i + dest_start] = source[i + source_start] + i += 1 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py Thu Dec 3 11:12:53 2009 @@ -5,7 +5,7 @@ from pypy.translator.unsimplify import insert_empty_block from pypy.translator.unsimplify import insert_empty_startblock from pypy.translator.unsimplify import starts_with_empty_block -from pypy.translator.backendopt.support import var_needsgc +from pypy.translator.backendopt.support import var_needsgc, ll_listcopy from pypy.translator.backendopt import inline from pypy.translator.backendopt import graphanalyze from pypy.translator.backendopt.canraise import RaiseAnalyzer @@ -383,8 +383,7 @@ def gct_listcopy(self, hop): # by default, this becomes a raw memcopy op = hop.spaceop - from pypy.rpython.lltypesystem import opimpl - llptr = self.annotate_helper(opimpl.op_listcopy, + llptr = self.annotate_helper(ll_listcopy, [arg.concretetype for arg in op.args], op.result.concretetype, inline=True) c_func = rmodel.inputconst(lltype.Void, llptr) Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Thu Dec 3 11:12:53 2009 @@ -862,7 +862,27 @@ def test_listcopy(self): run = self.runner("listcopy") run([]) - + + def define_listcopy_ptr(cls): + S = lltype.GcStruct('S') + TP = lltype.GcArray(lltype.Ptr(S)) + def fn(): + l = lltype.malloc(TP, 100) + for i in range(100): + l[i] = lltype.malloc(S) + l2 = lltype.malloc(TP, 50) + llop.listcopy(lltype.Void, l, l2, 50, 0, 50) + llop.gc__collect(lltype.Void) + for i in range(50): + assert l2[i] is l[i + 50] + return 0 + + return fn + + def test_listcopy_ptr(self): + run = self.runner("listcopy_ptr") + run([]) + # ________________________________________________________________ class TestMarkSweepGC(GenericGCTests): From fijal at codespeak.net Thu Dec 3 11:23:45 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 3 Dec 2009 11:23:45 +0100 (CET) Subject: [pypy-svn] r69858 - pypy/branch/listcopyop/pypy/rpython/memory/gctransform Message-ID: <20091203102345.C386D16800D@codespeak.net> Author: fijal Date: Thu Dec 3 11:23:44 2009 New Revision: 69858 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py Log: oops, fix import Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py Thu Dec 3 11:23:44 2009 @@ -5,7 +5,7 @@ from pypy.translator.unsimplify import insert_empty_block from pypy.translator.unsimplify import insert_empty_startblock from pypy.translator.unsimplify import starts_with_empty_block -from pypy.translator.backendopt.support import var_needsgc, ll_listcopy +from pypy.translator.backendopt.support import var_needsgc from pypy.translator.backendopt import inline from pypy.translator.backendopt import graphanalyze from pypy.translator.backendopt.canraise import RaiseAnalyzer @@ -14,7 +14,7 @@ from pypy.annotation import model as annmodel from pypy.rpython import rmodel from pypy.rpython.memory import gc -from pypy.rpython.memory.gctransform.support import var_ispyobj +from pypy.rpython.memory.gctransform.support import var_ispyobj, ll_listcopy from pypy.rpython.annlowlevel import MixLevelHelperAnnotator from pypy.rpython.rtyper import LowLevelOpList from pypy.rpython.rbuiltin import gen_cast From fijal at codespeak.net Thu Dec 3 11:45:03 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 3 Dec 2009 11:45:03 +0100 (CET) Subject: [pypy-svn] r69859 - in pypy/branch/listcopyop/pypy/rpython: . lltypesystem memory memory/test Message-ID: <20091203104503.3938E16800D@codespeak.net> Author: fijal Date: Thu Dec 3 11:45:02 2009 New Revision: 69859 Modified: pypy/branch/listcopyop/pypy/rpython/llinterp.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Log: shuffle stuff a bit, so we can now test on top of test_gc. Make test explode in case we don't have write barrier Modified: pypy/branch/listcopyop/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/llinterp.py (original) +++ pypy/branch/listcopyop/pypy/rpython/llinterp.py Thu Dec 3 11:45:02 2009 @@ -754,6 +754,9 @@ def op_zero_gc_pointers_inside(self, obj): raise NotImplementedError("zero_gc_pointers_inside") + def op_listcopy(self, source, dest, source_start, dest_start, length): + self.heap.listcopy(source, dest, source_start, dest_start, length) + def op_getfield(self, obj, field): checkptr(obj) # check the difference between op_getfield and op_getsubstruct: Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py Thu Dec 3 11:45:02 2009 @@ -394,12 +394,6 @@ checkadr(addr2) return addr1 - addr2 -def op_listcopy(source, dest, source_start, dest_start, length): - i = 0 - while i < length: - dest[i + dest_start] = source[i + source_start] - i += 1 - def op_getfield(p, name): checkptr(p) TYPE = lltype.typeOf(p).TO Modified: pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py Thu Dec 3 11:45:02 2009 @@ -128,6 +128,14 @@ ptr = lltype.cast_opaque_ptr(llmemory.GCREF, ptr) return self.gc.id(ptr) + def listcopy(self, source, dest, source_start, dest_start, length): + if hasattr(self.gc, 'listcopy'): + return self.gc.listcopy.im_func(source, dest, source_start, + dest_start, length) + i = 0 + while i < length: + dest[dest_start + i] = source[source_start + i] + i += 1 # ____________________________________________________________ Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py Thu Dec 3 11:45:02 2009 @@ -564,6 +564,26 @@ self.interpret(fn, []) + def test_listcopy_ptr(self): + S = lltype.GcStruct('S') + TP = lltype.GcArray(lltype.Ptr(S)) + def fn(): + l = lltype.malloc(TP, 100) + l2 = lltype.malloc(TP, 100) + for i in range(100): + l[i] = lltype.malloc(S) + llop.listcopy(lltype.Void, l, l2, 50, 0, 50) + x = [] + # force minor collect + t = (1, lltype.malloc(S)) + for i in range(20): + x.append(t) + for i in range(50): + assert l2[i] + return 0 + + self.interpret(fn, []) + from pypy.rlib.objectmodel import UnboxedValue Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Thu Dec 3 11:45:02 2009 @@ -868,13 +868,12 @@ TP = lltype.GcArray(lltype.Ptr(S)) def fn(): l = lltype.malloc(TP, 100) + l2 = lltype.malloc(TP, 100) for i in range(100): l[i] = lltype.malloc(S) - l2 = lltype.malloc(TP, 50) llop.listcopy(lltype.Void, l, l2, 50, 0, 50) - llop.gc__collect(lltype.Void) for i in range(50): - assert l2[i] is l[i + 50] + assert l2[i] return 0 return fn From fijal at codespeak.net Thu Dec 3 11:50:34 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 3 Dec 2009 11:50:34 +0100 (CET) Subject: [pypy-svn] r69860 - in pypy/branch/listcopyop/pypy/rpython/memory: . gc gctransform test Message-ID: <20091203105034.7652E16800D@codespeak.net> Author: fijal Date: Thu Dec 3 11:50:33 2009 New Revision: 69860 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Log: Shuffle things around, so we can actually pass a gc pointer as argument to listcopy (we'll need it for write barrier) Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Thu Dec 3 11:50:33 2009 @@ -475,8 +475,8 @@ objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS self.last_generation_root_objects.append(addr_struct) - @specialize.arglltype(0) - def listcopy(source, dest, source_start, dest_start, length): + @specialize.arglltype(1) + def listcopy(self, source, dest, source_start, dest_start, length): TP = lltype.typeOf(source).TO if isinstance(TP.OF, lltype.Ptr): pass # do write barrier Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py Thu Dec 3 11:50:33 2009 @@ -780,11 +780,12 @@ if not hasattr(self.GCClass, 'listcopy'): return GCTransformer.gct_listcopy(self, hop) op = hop.spaceop - ARGS = [arg.concretetype for arg in op.args] - llptr = self.annotate_helper(self.GCClass.listcopy.im_func, ARGS, + ARGS = ([self.c_const_gc.concretetype] + + [arg.concretetype for arg in op.args]) + llptr = self.annotate_helper(self.GCClass.listcopy, ARGS, op.result.concretetype, inline=True) c_func = rmodel.inputconst(lltype.Void, llptr) - hop.genop('direct_call', [c_func] + op.args) + hop.genop('direct_call', [c_func, self.c_const_gc] + op.args) def gct_weakref_create(self, hop): op = hop.spaceop Modified: pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py Thu Dec 3 11:50:33 2009 @@ -130,8 +130,8 @@ def listcopy(self, source, dest, source_start, dest_start, length): if hasattr(self.gc, 'listcopy'): - return self.gc.listcopy.im_func(source, dest, source_start, - dest_start, length) + return self.gc.listcopy(source, dest, source_start, + dest_start, length) i = 0 while i < length: dest[dest_start + i] = source[source_start + i] Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Thu Dec 3 11:50:33 2009 @@ -872,6 +872,10 @@ for i in range(100): l[i] = lltype.malloc(S) llop.listcopy(lltype.Void, l, l2, 50, 0, 50) + # force nursery collect + x = [] + for i in range(20): + x.append((1, lltype.malloc(S))) for i in range(50): assert l2[i] return 0 From arigo at codespeak.net Thu Dec 3 11:56:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 3 Dec 2009 11:56:40 +0100 (CET) Subject: [pypy-svn] r69861 - pypy/branch/virtual-forcing Message-ID: <20091203105640.919C616800D@codespeak.net> Author: arigo Date: Thu Dec 3 11:56:40 2009 New Revision: 69861 Added: pypy/branch/virtual-forcing/ - copied from r69860, pypy/trunk/ Log: Update the branch to the current state of trunk. From fijal at codespeak.net Thu Dec 3 12:07:13 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 3 Dec 2009 12:07:13 +0100 (CET) Subject: [pypy-svn] r69862 - pypy/branch/listcopyop/pypy/rpython/memory/gc Message-ID: <20091203110713.E8C6F16800D@codespeak.net> Author: fijal Date: Thu Dec 3 12:07:13 2009 New Revision: 69862 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Log: implement write barrier here Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Thu Dec 3 12:07:13 2009 @@ -478,15 +478,24 @@ @specialize.arglltype(1) def listcopy(self, source, dest, source_start, dest_start, length): TP = lltype.typeOf(source).TO + source_addr = llmemory.cast_ptr_to_adr(source) + dest_addr = llmemory.cast_ptr_to_adr(dest) if isinstance(TP.OF, lltype.Ptr): - pass # do write barrier - source_addr = (llmemory.cast_ptr_to_adr(source) + - llmemory.itemoffsetof(TP, 0) + - llmemory.sizeof(TP.OF) * source_start) - dest_addr = (llmemory.cast_ptr_to_adr(dest) - + llmemory.itemoffsetof(TP, 0) + - llmemory.sizeof(TP.OF) * dest_start) - llmemory.raw_memcopy(source_addr, dest_addr, + if self.header(source_addr).tid & GCFLAG_NO_YOUNG_PTRS: + # if GCFLAG_NO_YOUNG_PTRS is set, we don't need to update + pass + elif self.header(dest_addr).tid & GCFLAG_NO_YOUNG_PTRS == 0: + pass # dest either in nursery or has the flag already zeroed, + # ignore + else: + # this means source contains young ptrs and dest is not in + # a nursery and dest is not yet in list + self.assume_young_pointers(dest_addr) + cp_source_addr = (source_addr + llmemory.itemoffsetof(TP, 0) + + llmemory.sizeof(TP.OF) * source_start) + cp_dest_addr = (dest_addr + llmemory.itemoffsetof(TP, 0) + + llmemory.sizeof(TP.OF) * dest_start) + llmemory.raw_memcopy(cp_source_addr, cp_dest_addr, llmemory.sizeof(TP.OF) * length) def write_into_last_generation_obj(self, addr_struct, addr): From arigo at codespeak.net Thu Dec 3 12:37:49 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 3 Dec 2009 12:37:49 +0100 (CET) Subject: [pypy-svn] r69863 - in pypy/trunk/pypy/rpython: lltypesystem test Message-ID: <20091203113749.0A03216800D@codespeak.net> Author: arigo Date: Thu Dec 3 12:37:48 2009 New Revision: 69863 Modified: pypy/trunk/pypy/rpython/lltypesystem/rclass.py pypy/trunk/pypy/rpython/test/test_rvirtualizable2.py Log: Test and fix. Bah. Modified: pypy/trunk/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rclass.py Thu Dec 3 12:37:48 2009 @@ -499,7 +499,7 @@ cvalue = inputconst(r.lowleveltype, r.convert_desc_or_const(value)) self.setfield(vptr, fldname, cvalue, llops, - {'access_directly': True}) + flags={'access_directly': True}) return vptr def rtype_type(self, hop): Modified: pypy/trunk/pypy/rpython/test/test_rvirtualizable2.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rvirtualizable2.py (original) +++ pypy/trunk/pypy/rpython/test/test_rvirtualizable2.py Thu Dec 3 12:37:48 2009 @@ -4,10 +4,14 @@ from pypy.rpython.rvirtualizable2 import replace_promote_virtualizable_with_call from pypy.rlib.jit import hint from pypy.objspace.flow.model import summary +from pypy.rpython.llinterp import LLInterpreter +from pypy import conftest class V(object): _virtualizable2_ = ['v'] + v = -12 + w = -62 def __init__(self, v): self.v = v @@ -137,7 +141,8 @@ break def mycall(vinst_ll): - pass + if vinst_ll.vable_token: + raise ValueError annhelper = MixLevelHelperAnnotator(rtyper) if self.type_system == 'lltype': s_vinst = annmodel.SomePtr(v_inst_ll_type) @@ -158,12 +163,18 @@ op_getfield = block.operations[-1] assert op_getfield.opname in ('getfield', 'oogetfield') funcptr = self.replace_promote_virtualizable(rtyper, [graph]) + if getattr(conftest.option, 'view', False): + graph.show() op_promote = block.operations[-2] op_getfield = block.operations[-1] assert op_getfield.opname in ('getfield', 'oogetfield') assert op_promote.opname == 'direct_call' assert op_promote.args[0].value == funcptr assert op_promote.args[1] == op_getfield.args[0] + # + interp = LLInterpreter(rtyper) + res = interp.eval_graph(graph, [61]) + assert res == 61 def test_access_directly(self): def g(b): From fijal at codespeak.net Thu Dec 3 13:28:36 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 3 Dec 2009 13:28:36 +0100 (CET) Subject: [pypy-svn] r69864 - pypy/branch/listcopyop/pypy/rpython/memory/gc Message-ID: <20091203122836.A6EEE16800D@codespeak.net> Author: fijal Date: Thu Dec 3 13:28:35 2009 New Revision: 69864 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Log: Fix write barrier Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Thu Dec 3 13:28:35 2009 @@ -481,15 +481,12 @@ source_addr = llmemory.cast_ptr_to_adr(source) dest_addr = llmemory.cast_ptr_to_adr(dest) if isinstance(TP.OF, lltype.Ptr): - if self.header(source_addr).tid & GCFLAG_NO_YOUNG_PTRS: - # if GCFLAG_NO_YOUNG_PTRS is set, we don't need to update - pass - elif self.header(dest_addr).tid & GCFLAG_NO_YOUNG_PTRS == 0: - pass # dest either in nursery or has the flag already zeroed, - # ignore - else: - # this means source contains young ptrs and dest is not in - # a nursery and dest is not yet in list + need_set = False + if self.header(source_addr).tid & GCFLAG_NO_YOUNG_PTRS == 0: + need_set = True + if self.header(source_addr).tid & GCFLAG_NO_HEAP_PTRS == 0: + need_set = True + if need_set: self.assume_young_pointers(dest_addr) cp_source_addr = (source_addr + llmemory.itemoffsetof(TP, 0) + llmemory.sizeof(TP.OF) * source_start) From arigo at codespeak.net Thu Dec 3 13:43:48 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 3 Dec 2009 13:43:48 +0100 (CET) Subject: [pypy-svn] r69865 - in pypy/trunk/pypy/jit: backend/x86/test metainterp/test Message-ID: <20091203124348.DE59916800D@codespeak.net> Author: arigo Date: Thu Dec 3 13:43:48 2009 New Revision: 69865 Added: pypy/trunk/pypy/jit/backend/x86/test/test_float.py (contents, props changed) Modified: pypy/trunk/pypy/jit/metainterp/test/test_float.py Log: More tests for abs(). Added: pypy/trunk/pypy/jit/backend/x86/test/test_float.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/backend/x86/test/test_float.py Thu Dec 3 13:43:48 2009 @@ -0,0 +1,9 @@ + +import py +from pypy.jit.backend.x86.test.test_basic import Jit386Mixin +from pypy.jit.metainterp.test.test_float import FloatTests + +class TestFloat(Jit386Mixin, FloatTests): + # for the individual tests see + # ====> ../../../metainterp/test/test_float.py + pass Modified: pypy/trunk/pypy/jit/metainterp/test/test_float.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_float.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_float.py Thu Dec 3 13:43:48 2009 @@ -1,3 +1,4 @@ +import math from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin @@ -17,6 +18,15 @@ res = self.interp_operations(f, [12.0]) assert res == 1.0 + def test_abs(self): + def f(a): + return abs(a) + res = self.interp_operations(f, [-5.25]) + assert res == 5.25 + x = 281474976710656.31 + res = self.interp_operations(f, [x]) + assert res == x + class TestOOtype(FloatTests, OOJitMixin): pass From fijal at codespeak.net Thu Dec 3 13:57:49 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 3 Dec 2009 13:57:49 +0100 (CET) Subject: [pypy-svn] r69866 - pypy/trunk/pypy/rpython/memory/test Message-ID: <20091203125749.896FE16800D@codespeak.net> Author: fijal Date: Thu Dec 3 13:57:48 2009 New Revision: 69866 Modified: pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py Log: Some failing tests Modified: pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py Thu Dec 3 13:57:48 2009 @@ -843,7 +843,50 @@ assert (totsize - 26 * size_of_int) % 4 == 0 # ^^^ a crude assumption that totsize - varsize would be dividable by 4 # (and give fixedsize) + + def define_listcopy(cls): + TP = lltype.GcArray(lltype.Signed) + def fn(): + l = lltype.malloc(TP, 100) + for i in range(100): + l[i] = 1 + l2 = lltype.malloc(TP, 50) + #llop.listcopy(lltype.Void, l, l2, 50, 0, 50) + #for i in range(50): + # assert l2[i] == 1 + return 0 + + return fn + + def test_listcopy(self): + run = self.runner("listcopy") + run([]) + + def define_listcopy_ptr(cls): + S = lltype.GcStruct('S') + TP = lltype.GcArray(lltype.Ptr(S)) + def fn(): + l = lltype.malloc(TP, 100) + l2 = lltype.malloc(TP, 100) + for i in range(100): + l[i] = lltype.malloc(S) + #llop.listcopy(lltype.Void, l, l2, 50, 0, 50) + # force nursery collect + x = [] + for i in range(20): + x.append((1, lltype.malloc(S))) + #for i in range(50): + # assert l2[i] + return 0 + + return fn + + def test_listcopy_ptr(self): + py.test.skip("explodes") + run = self.runner("listcopy_ptr") + run([]) + # ________________________________________________________________ class TestMarkSweepGC(GenericGCTests): From fijal at codespeak.net Thu Dec 3 15:54:00 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 3 Dec 2009 15:54:00 +0100 (CET) Subject: [pypy-svn] r69868 - pypy/branch/listcopyop/pypy/rpython/lltypesystem Message-ID: <20091203145400.E78C016800D@codespeak.net> Author: fijal Date: Thu Dec 3 15:53:59 2009 New Revision: 69868 Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py Log: Actually use this interface for list resizing Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py Thu Dec 3 15:53:59 2009 @@ -214,18 +214,7 @@ p = before_len - 1 else: p = new_allocated - 1 - ITEM = typeOf(l).TO.ITEM - if isinstance(ITEM, Ptr): - while p >= 0: - newitems[p] = items[p] - items[p] = nullptr(ITEM.TO) - p -= 1 - else: - source = cast_ptr_to_adr(items) + itemoffsetof(typeOf(l.items).TO, 0) - dest = cast_ptr_to_adr(newitems) + itemoffsetof(typeOf(l.items).TO, 0) - s = p + 1 - raw_memcopy(source, dest, sizeof(ITEM) * s) - keepalive_until_here(items) + llop.listcopy(items, newitems, 0, 0, p) l.length = newsize l.items = newitems _ll_list_resize_really._annenforceargs_ = (None, int) From fijal at codespeak.net Thu Dec 3 16:05:36 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 3 Dec 2009 16:05:36 +0100 (CET) Subject: [pypy-svn] r69869 - in pypy/branch/listcopyop/pypy/rpython: lltypesystem memory/test Message-ID: <20091203150536.199E716800D@codespeak.net> Author: fijal Date: Thu Dec 3 16:05:36 2009 New Revision: 69869 Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/llheap.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Log: make various kinds of tests happier. fix imports Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/llheap.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/llheap.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/llheap.py Thu Dec 3 16:05:36 2009 @@ -35,6 +35,12 @@ ll_str.chars[i] = buf.chars[i] return ll_str +def listcopy(source, dest, source_start, dest_start, length): + i = 0 + while i < length: + dest[i + dest_start] = source[i + source_start] + i += 1 + def thread_prepare(): pass Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py Thu Dec 3 16:05:36 2009 @@ -394,6 +394,12 @@ checkadr(addr2) return addr1 - addr2 +def op_listcopy(source, dest, source_start, dest_start, length): + i = 0 + while i < length: + dest[i + dest_start] = source[i + source_start] + i += 1 + def op_getfield(p, name): checkptr(p) TYPE = lltype.typeOf(p).TO Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py Thu Dec 3 16:05:36 2009 @@ -15,10 +15,9 @@ from pypy.rpython import robject from pypy.rlib.debug import ll_assert from pypy.rlib.rarithmetic import ovfcheck -from pypy.rpython.lltypesystem.llmemory import cast_ptr_to_adr, raw_memclear,\ - raw_memcopy, sizeof, itemoffsetof, offsetof from pypy.rpython.lltypesystem import rffi from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rpython.lltypesystem.lloperation import llop # ____________________________________________________________ # @@ -211,10 +210,10 @@ newitems = malloc(typeOf(l).TO.items.TO, new_allocated) before_len = l.length if before_len < new_allocated: - p = before_len - 1 + p = before_len else: - p = new_allocated - 1 - llop.listcopy(items, newitems, 0, 0, p) + p = new_allocated + llop.listcopy(Void, items, newitems, 0, 0, p) l.length = newsize l.items = newitems _ll_list_resize_really._annenforceargs_ = (None, int) Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Thu Dec 3 16:05:36 2009 @@ -834,7 +834,7 @@ return d * 1000 + c * 100 + b * 10 + a return f - def test_gc_heap_stats(self): + def xxxtest_gc_heap_stats(self): run = self.runner("gc_heap_stats") res = run([]) assert res % 10000 == 2611 @@ -894,7 +894,7 @@ class transformerclass(framework.FrameworkGCTransformer): GC_PARAMS = {'start_heap_size': 4096 } root_stack_depth = 200 - + from pypy.rpython.memory.gc.marksweep import MarkSweepGC as GCClass def define_cloning(cls): B = lltype.GcStruct('B', ('x', lltype.Signed)) From arigo at codespeak.net Thu Dec 3 16:05:50 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 3 Dec 2009 16:05:50 +0100 (CET) Subject: [pypy-svn] r69870 - in pypy/trunk/pypy/jit/backend/x86: . test Message-ID: <20091203150550.910FC168008@codespeak.net> Author: arigo Date: Thu Dec 3 16:05:49 2009 New Revision: 69870 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/backend/x86/ri386setup.py pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py Log: Kill the hacks to get the constants for neg and abs. Enhance a test to shows why such hacks are not working when translated. Replace them with a proper 16-bytes-aligned buffer, only in assembler.py, not in regalloc.py. Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Thu Dec 3 16:05:49 2009 @@ -90,6 +90,8 @@ self.fail_boxes_ptr = NonmovableGrowableArrayGCREF() self.fail_boxes_float = NonmovableGrowableArrayFloat() self.fail_ebp = 0 + self.loc_float_const_neg = None + self.loc_float_const_abs = None self.setup_failure_recovery() def leave_jitted_hook(self): @@ -130,6 +132,26 @@ self._build_failure_recovery(False, withfloats=True) self._build_failure_recovery(True, withfloats=True) codebuf.ensure_sse2_floats() + self._build_float_constants() + + def _build_float_constants(self): + # 11 words: 8 words for the data, and up to 3 words for alignment + addr = lltype.malloc(rffi.CArray(lltype.Signed), 11, flavor='raw') + if not we_are_translated(): + self._keepalive_malloced_float_consts = addr + float_constants = rffi.cast(lltype.Signed, addr) + float_constants = (float_constants + 15) & ~15 # align to 16 bytes + addr = rffi.cast(rffi.CArrayPtr(lltype.Signed), float_constants) + addr[0] = 0 # \ + addr[1] = -2147483648 # / for neg + addr[2] = 0 # + addr[3] = 0 # + addr[4] = -1 # \ + addr[5] = 2147483647 # / for abs + addr[6] = 0 # + addr[7] = 0 # + self.loc_float_const_neg = heap64(float_constants) + self.loc_float_const_abs = heap64(float_constants + 16) def assemble_loop(self, inputargs, operations, looptoken): """adds the following attributes to looptoken: @@ -445,15 +467,13 @@ genop_guard_uint_le = _cmpop_guard("BE", "AE", "A", "B") genop_guard_uint_ge = _cmpop_guard("AE", "BE", "B", "A") - # for now all chars are being considered ints, although we should make - # a difference at some point - xxx_genop_char_eq = genop_int_eq - def genop_float_neg(self, op, arglocs, resloc): - self.mc.XORPD(arglocs[0], arglocs[1]) + # Following what gcc does: res = x ^ 0x8000000000000000 + self.mc.XORPD(arglocs[0], self.loc_float_const_neg) def genop_float_abs(self, op, arglocs, resloc): - self.mc.ANDPD(arglocs[0], arglocs[1]) + # Following what gcc does: res = x & 0x7FFFFFFFFFFFFFFF + self.mc.ANDPD(arglocs[0], self.loc_float_const_abs) def genop_float_is_true(self, op, arglocs, resloc): loc0, loc1 = arglocs Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Thu Dec 3 16:05:49 2009 @@ -54,15 +54,6 @@ BASE_CONSTANT_SIZE = 1000 -# cheat cheat cheat.... -# why not -0.0? People tell me it's platform-dependent -# nan is not portable -import struct -NEG_ZERO, = struct.unpack('d', struct.pack('ll', 0, -2147483648)) -NAN, = struct.unpack('d', struct.pack('ll', -1, 2147483647)) -# XXX These are actually masks for float_neg and float_abs. -# They should not be converted to 'double' and given -# names that reflect their float value. class X86XMMRegisterManager(RegisterManager): @@ -80,9 +71,7 @@ RegisterManager.__init__(self, longevity, stack_manager=stack_manager, assembler=assembler) self.constant_arrays = [self.new_const_array()] - self.constant_arrays[-1][0] = NEG_ZERO - self.constant_arrays[-1][1] = NAN - self.constant_array_counter = 2 + self.constant_array_counter = 0 def convert_to_imm(self, c): if self.constant_array_counter >= BASE_CONSTANT_SIZE: @@ -564,28 +553,13 @@ consider_float_ge = _consider_float_cmp def consider_float_neg(self, op, ignored): - # Following what gcc does... - # XXX we can ignore having constant in a reg, but we need - # to be careful with 128-bit alignment loc0 = self.xrm.force_result_in_reg(op.result, op.args[0]) - constloc = self.xrm.get_addr_of_const_float(0, 0) - tmpbox = TempBox() - loc1 = self.xrm.force_allocate_reg(tmpbox, op.args) - self.assembler.regalloc_mov(constloc, loc1) - self.Perform(op, [loc0, loc1], loc0) - self.xrm.possibly_free_var(tmpbox) + self.Perform(op, [loc0], loc0) self.xrm.possibly_free_var(op.args[0]) def consider_float_abs(self, op, ignored): - # XXX we can ignore having constant in a reg, but we need - # to be careful with 128-bit alignment loc0 = self.xrm.force_result_in_reg(op.result, op.args[0]) - constloc = self.xrm.get_addr_of_const_float(0, 1) - tmpbox = TempBox() - loc1 = self.xrm.force_allocate_reg(tmpbox, op.args) - self.assembler.regalloc_mov(constloc, loc1) - self.Perform(op, [loc0, loc1], loc0) - self.xrm.possibly_free_var(tmpbox) + self.Perform(op, [loc0], loc0) self.xrm.possibly_free_var(op.args[0]) def consider_float_is_true(self, op, ignored): Modified: pypy/trunk/pypy/jit/backend/x86/ri386setup.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/ri386setup.py (original) +++ pypy/trunk/pypy/jit/backend/x86/ri386setup.py Thu Dec 3 16:05:49 2009 @@ -541,13 +541,11 @@ UCOMISD = Instruction() UCOMISD.mode2(XMMREG, MODRM64, ['\x66\x0F\x2E', register(1, 8), modrm(2)]) -XORPD = Instruction() -XORPD.mode2(XMMREG, XMMREG, ['\x66\x0f\x57', register(1, 8), register(2), - '\xC0']) +XORPD = Instruction() # warning: a memory argument must be aligned to 16 bytes +XORPD.mode2(XMMREG, MODRM64, ['\x66\x0f\x57', register(1, 8), modrm(2)]) -ANDPD = Instruction() -ANDPD.mode2(XMMREG, XMMREG, ['\x66\x0F\x54', register(1, 8), register(2), - '\xC0']) +ANDPD = Instruction() # warning: a memory argument must be aligned to 16 bytes +ANDPD.mode2(XMMREG, MODRM64, ['\x66\x0F\x54', register(1, 8), modrm(2)]) CVTTSD2SI = Instruction() CVTTSD2SI.mode2(REG, XMMREG, ['\xF2\x0F\x2C', register(1, 8), register(2), Modified: pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py Thu Dec 3 16:05:49 2009 @@ -22,6 +22,7 @@ # - set_param interface # - profiler # - full optimizer + # - floats neg and abs class Frame(object): _virtualizable2_ = ['i'] @@ -29,9 +30,14 @@ def __init__(self, i): self.i = i - jitdriver = JitDriver(greens = [], reds = ['frame', 'total'], + @dont_look_inside + def myabs(x): + return abs(x) + + jitdriver = JitDriver(greens = [], + reds = ['frame', 'total', 'j'], virtualizables = ['frame']) - def f(i): + def f(i, j): for param in unroll_parameters: defl = PARAMETERS[param] jitdriver.set_param(param, defl) @@ -40,15 +46,20 @@ total = 0 frame = Frame(i) while frame.i > 3: - jitdriver.can_enter_jit(frame=frame, total=total) - jitdriver.jit_merge_point(frame=frame, total=total) + jitdriver.can_enter_jit(frame=frame, total=total, j=j) + jitdriver.jit_merge_point(frame=frame, total=total, j=j) total += frame.i if frame.i >= 20: frame.i -= 2 frame.i -= 1 + j *= -0.712 + if j + (-j): raise ValueError + k = myabs(j) + if k - abs(j): raise ValueError + if k - abs(-j): raise ValueError return total * 10 - res = self.meta_interp(f, [40]) - assert res == f(40) + res = self.meta_interp(f, [40, -49]) + assert res == f(40, -49) def test_external_exception_handling_translates(self): jitdriver = JitDriver(greens = [], reds = ['n', 'total']) From arigo at codespeak.net Thu Dec 3 18:01:47 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 3 Dec 2009 18:01:47 +0100 (CET) Subject: [pypy-svn] r69871 - in pypy/branch/virtual-forcing/pypy: jit/metainterp/test rlib Message-ID: <20091203170147.C336916800B@codespeak.net> Author: arigo Date: Thu Dec 3 18:01:46 2009 New Revision: 69871 Added: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py (contents, props changed) Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py Log: Adding the first test. Added: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py ============================================================================== --- (empty file) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py Thu Dec 3 18:01:46 2009 @@ -0,0 +1,34 @@ +from pypy.rlib.jit import JitDriver, dont_look_inside, virtual_ref +from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin + + +class VRefTests: + + def test_simple_no_access(self): + myjitdriver = JitDriver(greens = [], reds = ['n']) + # + class XY: + pass + class ExCtx: + pass + exctx = ExCtx() + # + @dont_look_inside + def externalfn(): + return 1 + # + def f(n): + while n > 0: + myjitdriver.can_enter_jit(n=n) + myjitdriver.jit_merge_point(n=n) + xy = XY() + exctx.topframeref = virtual_ref(xy) + n -= externalfn() + exctx.topframeref = None + # + self.meta_interp(f, [15]) + self.check_loops(new_with_vtable=0) + + +class TestLLtype(VRefTests, LLJitMixin): + pass Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/jit.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/jit.py Thu Dec 3 18:01:46 2009 @@ -98,6 +98,9 @@ hop.exception_cannot_occur() return hop.inputconst(lltype.Signed, _we_are_jitted) +def virtual_ref(x): + return None # XXX! + # ____________________________________________________________ # User interface for the hotpath JIT policy From arigo at codespeak.net Thu Dec 3 18:04:10 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 3 Dec 2009 18:04:10 +0100 (CET) Subject: [pypy-svn] r69872 - pypy/branch/virtual-forcing/pypy/jit/metainterp/test Message-ID: <20091203170410.35E0716800B@codespeak.net> Author: arigo Date: Thu Dec 3 18:04:09 2009 New Revision: 69872 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py Log: Adding the second test, skipped. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py Thu Dec 3 18:04:09 2009 @@ -1,3 +1,4 @@ +import py from pypy.rlib.jit import JitDriver, dont_look_inside, virtual_ref from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin @@ -29,6 +30,35 @@ self.meta_interp(f, [15]) self.check_loops(new_with_vtable=0) + def test_simple_force_always(self): + py.test.skip("in-progress") + myjitdriver = JitDriver(greens = [], reds = ['n']) + # + class XY: + pass + class ExCtx: + pass + exctx = ExCtx() + # + @dont_look_inside + def externalfn(n): + m = exctx.topframeref().n + assert m == n + return 1 + # + def f(n): + while n > 0: + myjitdriver.can_enter_jit(n=n) + myjitdriver.jit_merge_point(n=n) + xy = XY() + xy.n = n + exctx.topframeref = virtual_ref(xy) + n -= externalfn(n) + exctx.topframeref = None + # + self.meta_interp(f, [15]) + self.check_loops(new_with_vtable=0) + class TestLLtype(VRefTests, LLJitMixin): pass From arigo at codespeak.net Thu Dec 3 18:36:31 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 3 Dec 2009 18:36:31 +0100 (CET) Subject: [pypy-svn] r69873 - in pypy/branch/virtual-forcing/pypy/rlib: . test Message-ID: <20091203173631.60940168015@codespeak.net> Author: arigo Date: Thu Dec 3 18:36:30 2009 New Revision: 69873 Added: pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py (contents, props changed) pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py (contents, props changed) Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py Log: Write tests and implement them: annotation, easy case. Added: pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py ============================================================================== --- (empty file) +++ pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py Thu Dec 3 18:36:30 2009 @@ -0,0 +1,10 @@ +from pypy.annotation import model as annmodel + + +class SomeVRef(annmodel.SomeObject): + + def __init__(self, s_instance): + self.s_instance = s_instance + + def simple_call(self): + return self.s_instance Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/jit.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/jit.py Thu Dec 3 18:36:30 2009 @@ -98,8 +98,30 @@ hop.exception_cannot_occur() return hop.inputconst(lltype.Signed, _we_are_jitted) +# ____________________________________________________________ +# VRefs + def virtual_ref(x): - return None # XXX! + return DirectVRef(x) + +class DirectVRef(object): + def __init__(self, x): + self._x = x + def __call__(self): + return self._x + def _freeze_(self): + raise Exception("should not see a prebuilt pypy.rlib.jit.virtual_ref") + +class Entry(ExtRegistryEntry): + _about_ = virtual_ref + + def compute_result_annotation(self, s_obj): + from pypy.rlib import _jit_vref + return _jit_vref.SomeVRef(s_obj) + + def specialize_call(self, hop): + from pypy.rlib import _jit_vref + return _jit_vref.specialize_call(self, hop) # ____________________________________________________________ # User interface for the hotpath JIT policy Added: pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py ============================================================================== --- (empty file) +++ pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py Thu Dec 3 18:36:30 2009 @@ -0,0 +1,32 @@ +from pypy.rlib.jit import virtual_ref +from pypy.rlib._jit_vref import SomeVRef +from pypy.annotation import model as annmodel +from pypy.annotation.annrpython import RPythonAnnotator + + +class X(object): + pass + + +def test_direct(): + x1 = X() + vref = virtual_ref(x1) + assert vref() is x1 + +def test_annotate_1(): + def f(): + return virtual_ref(X()) + a = RPythonAnnotator() + s = a.build_types(f, []) + assert isinstance(s, SomeVRef) + assert isinstance(s.s_instance, annmodel.SomeInstance) + assert s.s_instance.classdef == a.bookkeeper.getuniqueclassdef(X) + +def test_annotate_2(): + def f(): + vref = virtual_ref(X()) + return vref() + a = RPythonAnnotator() + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInstance) + assert s.classdef == a.bookkeeper.getuniqueclassdef(X) From arigo at codespeak.net Thu Dec 3 18:56:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 3 Dec 2009 18:56:29 +0100 (CET) Subject: [pypy-svn] r69874 - in pypy/branch/virtual-forcing/pypy/rlib: . test Message-ID: <20091203175629.ADA61168015@codespeak.net> Author: arigo Date: Thu Dec 3 18:56:29 2009 New Revision: 69874 Modified: pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py pypy/branch/virtual-forcing/pypy/rlib/jit.py pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py Log: Rtyping. Still no real work. Modified: pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py Thu Dec 3 18:56:29 2009 @@ -1,4 +1,6 @@ from pypy.annotation import model as annmodel +from pypy.rpython.rclass import getinstancerepr +from pypy.rpython.rmodel import Repr class SomeVRef(annmodel.SomeObject): @@ -8,3 +10,28 @@ def simple_call(self): return self.s_instance + + def rtyper_makerepr(self, rtyper): + return get_vref(rtyper) + + def rtyper_makekey(self): + return self.__class__, + + +def specialize_call(hop): + [v] = hop.inputargs(getinstancerepr(hop.rtyper, None)) + return v + +def get_vref(rtyper): + if not hasattr(rtyper, '_vrefrepr'): + rtyper._vrefrepr = VRefRepr(rtyper) + return rtyper._vrefrepr + + +class VRefRepr(Repr): + def __init__(self, rtyper): + self.lowleveltype = getinstancerepr(rtyper, None).lowleveltype + + def rtype_simple_call(self, hop): + [v] = hop.inputargs(self) + return hop.genop('cast_pointer', [v], resulttype = hop.r_result) Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/jit.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/jit.py Thu Dec 3 18:56:29 2009 @@ -121,7 +121,7 @@ def specialize_call(self, hop): from pypy.rlib import _jit_vref - return _jit_vref.specialize_call(self, hop) + return _jit_vref.specialize_call(hop) # ____________________________________________________________ # User interface for the hotpath JIT policy Modified: pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py Thu Dec 3 18:56:29 2009 @@ -2,6 +2,9 @@ from pypy.rlib._jit_vref import SomeVRef from pypy.annotation import model as annmodel from pypy.annotation.annrpython import RPythonAnnotator +from pypy.rpython.test.test_llinterp import interpret +from pypy.rpython.lltypesystem.rclass import OBJECTPTR +from pypy.rpython.lltypesystem import lltype class X(object): @@ -30,3 +33,16 @@ s = a.build_types(f, []) assert isinstance(s, annmodel.SomeInstance) assert s.classdef == a.bookkeeper.getuniqueclassdef(X) + +def test_rtype_1(): + def f(): + return virtual_ref(X()) + x = interpret(f, []) + assert lltype.typeOf(x) == OBJECTPTR + +def test_rtype_2(): + def f(): + vref = virtual_ref(X()) + return vref() + x = interpret(f, []) + assert lltype.castable(OBJECTPTR, lltype.typeOf(x)) > 0 From fijal at codespeak.net Thu Dec 3 19:15:42 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 3 Dec 2009 19:15:42 +0100 (CET) Subject: [pypy-svn] r69875 - in pypy/branch/listcopyop/pypy: rlib rpython rpython/lltypesystem rpython/memory/gctransform rpython/memory/test Message-ID: <20091203181542.8E29E16800D@codespeak.net> Author: fijal Date: Thu Dec 3 19:15:41 2009 New Revision: 69875 Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py pypy/branch/listcopyop/pypy/rpython/llinterp.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/llheap.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Log: Start moving listcopy into rgc helper, revert most of the stuff done, so far framework gc does not do anything. Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rlib/rgc.py (original) +++ pypy/branch/listcopyop/pypy/rlib/rgc.py Thu Dec 3 19:15:41 2009 @@ -328,3 +328,16 @@ hop.exception_cannot_occur() return hop.genop('finish_building_buffer', vlist, resulttype=hop.r_result.lowleveltype) + +def listcopy(source, dest, source_start, dest_start, length): + from pypy.rpython.lltypesystem.lloperation import llop + from pypy.rpython.lltypesystem import lltype + + if llop.gc_listcopy(lltype.Void, source, dest, source_start, dest_start, + length): + return # gc supports nicely copying lists + i = 0 + while i < length: + dest[i + dest_start] = source[i + source_start] + i += 1 + Modified: pypy/branch/listcopyop/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/llinterp.py (original) +++ pypy/branch/listcopyop/pypy/rpython/llinterp.py Thu Dec 3 19:15:41 2009 @@ -755,7 +755,10 @@ raise NotImplementedError("zero_gc_pointers_inside") def op_listcopy(self, source, dest, source_start, dest_start, length): - self.heap.listcopy(source, dest, source_start, dest_start, length) + if hasattr(self.heap, 'listcopy'): + self.heap.listcopy(source, dest, source_start, dest_start, length) + return True + return False def op_getfield(self, obj, field): checkptr(obj) Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/llheap.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/llheap.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/llheap.py Thu Dec 3 19:15:41 2009 @@ -35,12 +35,6 @@ ll_str.chars[i] = buf.chars[i] return ll_str -def listcopy(source, dest, source_start, dest_start, length): - i = 0 - while i < length: - dest[i + dest_start] = source[i + source_start] - i += 1 - def thread_prepare(): pass Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py Thu Dec 3 19:15:41 2009 @@ -358,7 +358,7 @@ 'resize_buffer': LLOp(canraise=(MemoryError,), canunwindgc=True), 'finish_building_buffer' : LLOp(canraise=(MemoryError,), canunwindgc=True), 'zero_gc_pointers_inside': LLOp(), - 'listcopy': LLOp(canrun=True), + 'gc_listcopy': LLOp(canrun=True), 'free': LLOp(), 'getfield': LLOp(sideeffects=False, canrun=True), 'getarrayitem': LLOp(sideeffects=False, canrun=True), Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py Thu Dec 3 19:15:41 2009 @@ -394,11 +394,8 @@ checkadr(addr2) return addr1 - addr2 -def op_listcopy(source, dest, source_start, dest_start, length): - i = 0 - while i < length: - dest[i + dest_start] = source[i + source_start] - i += 1 +def op_gc_listcopy(source, dest, source_start, dest_start, length): + return False # no special support def op_getfield(p, name): checkptr(p) Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py Thu Dec 3 19:15:41 2009 @@ -18,6 +18,7 @@ from pypy.rpython.lltypesystem import rffi from pypy.rlib.objectmodel import keepalive_until_here from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rlib import rgc # ____________________________________________________________ # @@ -213,7 +214,7 @@ p = before_len else: p = new_allocated - llop.listcopy(Void, items, newitems, 0, 0, p) + rgc.listcopy(items, newitems, 0, 0, p) l.length = newsize l.items = newitems _ll_list_resize_really._annenforceargs_ = (None, int) Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py Thu Dec 3 19:15:41 2009 @@ -776,16 +776,16 @@ TYPE = v_ob.concretetype.TO gen_zero_gc_pointers(TYPE, v_ob, hop.llops) - def gct_listcopy(self, hop): - if not hasattr(self.GCClass, 'listcopy'): - return GCTransformer.gct_listcopy(self, hop) - op = hop.spaceop - ARGS = ([self.c_const_gc.concretetype] + - [arg.concretetype for arg in op.args]) - llptr = self.annotate_helper(self.GCClass.listcopy, ARGS, - op.result.concretetype, inline=True) - c_func = rmodel.inputconst(lltype.Void, llptr) - hop.genop('direct_call', [c_func, self.c_const_gc] + op.args) + # def gct_listcopy(self, hop): + # if not hasattr(self.GCClass, 'listcopy'): + # return GCTransformer.gct_listcopy(self, hop) + # op = hop.spaceop + # ARGS = ([self.c_const_gc.concretetype] + + # [arg.concretetype for arg in op.args]) + # llptr = self.annotate_helper(self.GCClass.listcopy, ARGS, + # op.result.concretetype, inline=True) + # c_func = rmodel.inputconst(lltype.Void, llptr) + # hop.genop('direct_call', [c_func, self.c_const_gc] + op.args) def gct_weakref_create(self, hop): op = hop.spaceop Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py Thu Dec 3 19:15:41 2009 @@ -380,14 +380,8 @@ def gct_zero_gc_pointers_inside(self, hop): pass - def gct_listcopy(self, hop): - # by default, this becomes a raw memcopy - op = hop.spaceop - llptr = self.annotate_helper(ll_listcopy, - [arg.concretetype for arg in op.args], - op.result.concretetype, inline=True) - c_func = rmodel.inputconst(lltype.Void, llptr) - hop.genop('direct_call', [c_func] + op.args) + def gct_gc_listcopy(self, hop): + return rmodel.inputconst(lltype.Bool, False) def gct_gc_identityhash(self, hop): # must be implemented in the various GCs Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Thu Dec 3 19:15:41 2009 @@ -852,9 +852,9 @@ for i in range(100): l[i] = 1 l2 = lltype.malloc(TP, 50) - llop.listcopy(lltype.Void, l, l2, 50, 0, 50) - for i in range(50): - assert l2[i] == 1 + if llop.gc_listcopy(lltype.Void, l, l2, 50, 0, 50): + for i in range(50): + assert l2[i] == 1 return 0 return fn @@ -871,13 +871,13 @@ l2 = lltype.malloc(TP, 100) for i in range(100): l[i] = lltype.malloc(S) - llop.listcopy(lltype.Void, l, l2, 50, 0, 50) - # force nursery collect - x = [] - for i in range(20): - x.append((1, lltype.malloc(S))) - for i in range(50): - assert l2[i] + if llop.gc_listcopy(lltype.Void, l, l2, 50, 0, 50): + # force nursery collect + x = [] + for i in range(20): + x.append((1, lltype.malloc(S))) + for i in range(50): + assert l2[i] return 0 return fn From fijal at codespeak.net Thu Dec 3 19:18:53 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 3 Dec 2009 19:18:53 +0100 (CET) Subject: [pypy-svn] r69876 - pypy/branch/listcopyop/pypy/rpython/memory/gctransform Message-ID: <20091203181853.652BC16800D@codespeak.net> Author: fijal Date: Thu Dec 3 19:18:52 2009 New Revision: 69876 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/support.py Log: Kill unused helper Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/support.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/support.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/support.py Thu Dec 3 19:18:52 2009 @@ -107,10 +107,3 @@ os.write(2, "a destructor raised an exception, ignoring it\n") except: pass - - at specialize.arglltype(0) -def ll_listcopy(source, dest, source_start, dest_start, length): - i = 0 - while i < length: - dest[i + dest_start] = source[i + source_start] - i += 1 From arigo at codespeak.net Thu Dec 3 20:03:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 3 Dec 2009 20:03:42 +0100 (CET) Subject: [pypy-svn] r69877 - in pypy/branch/virtual-forcing/pypy: rlib rlib/test rpython/lltypesystem Message-ID: <20091203190342.D1C4F16800D@codespeak.net> Author: arigo Date: Thu Dec 3 20:03:41 2009 New Revision: 69877 Modified: pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py pypy/branch/virtual-forcing/pypy/rlib/jit.py pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/lloperation.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/opimpl.py Log: Finish the part in rlib. Missing tests for some cases, which I think I really need to write in metainterp/test. Modified: pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py Thu Dec 3 20:03:41 2009 @@ -1,6 +1,9 @@ from pypy.annotation import model as annmodel +from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rpython.rclass import getinstancerepr from pypy.rpython.rmodel import Repr +from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rpython.lltypesystem.rclass import OBJECTPTR class SomeVRef(annmodel.SomeObject): @@ -12,26 +15,39 @@ return self.s_instance def rtyper_makerepr(self, rtyper): - return get_vref(rtyper) + if not hasattr(rtyper, '_vrefrepr'): + rtyper._vrefrepr = VRefRepr(rtyper) + return rtyper._vrefrepr def rtyper_makekey(self): return self.__class__, -def specialize_call(hop): - [v] = hop.inputargs(getinstancerepr(hop.rtyper, None)) - return v - -def get_vref(rtyper): - if not hasattr(rtyper, '_vrefrepr'): - rtyper._vrefrepr = VRefRepr(rtyper) - return rtyper._vrefrepr - - class VRefRepr(Repr): def __init__(self, rtyper): self.lowleveltype = getinstancerepr(rtyper, None).lowleveltype + def specialize_call(self, hop): + [v] = hop.inputargs(getinstancerepr(hop.rtyper, None)) + return v + def rtype_simple_call(self, hop): [v] = hop.inputargs(self) + v = hop.genop('jit_virtual_force', [v], resulttype = OBJECTPTR) return hop.genop('cast_pointer', [v], resulttype = hop.r_result) + +# ____________________________________________________________ + + +def jit_virtual_ref(x): + raise Exception("should not be reached") + +class Entry(ExtRegistryEntry): + _about_ = jit_virtual_ref + + def compute_result_annotation(self, s_obj): + return SomeVRef(s_obj) + + def specialize_call(self, hop): + [v] = hop.inputargs(getinstancerepr(hop.rtyper, None)) + return hop.genop('jit_virtual_ref', [v], resulttype = hop.r_result) Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/jit.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/jit.py Thu Dec 3 20:03:41 2009 @@ -102,7 +102,11 @@ # VRefs def virtual_ref(x): - return DirectVRef(x) + if we_are_jitted(): + from pypy.rlib import _jit_vref + return _jit_vref.jit_virtual_ref(x) + else: + return DirectVRef(x) class DirectVRef(object): def __init__(self, x): @@ -110,18 +114,17 @@ def __call__(self): return self._x def _freeze_(self): - raise Exception("should not see a prebuilt pypy.rlib.jit.virtual_ref") + raise Exception("should not see a prebuilt virtual_ref") class Entry(ExtRegistryEntry): - _about_ = virtual_ref + _about_ = DirectVRef def compute_result_annotation(self, s_obj): from pypy.rlib import _jit_vref return _jit_vref.SomeVRef(s_obj) def specialize_call(self, hop): - from pypy.rlib import _jit_vref - return _jit_vref.specialize_call(hop) + return hop.r_result.specialize_call(hop) # ____________________________________________________________ # User interface for the hotpath JIT policy Modified: pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py Thu Dec 3 20:03:41 2009 @@ -10,6 +10,12 @@ class X(object): pass +class Y(X): + pass + +class Z(X): + pass + def test_direct(): x1 = X() @@ -34,6 +40,18 @@ assert isinstance(s, annmodel.SomeInstance) assert s.classdef == a.bookkeeper.getuniqueclassdef(X) +def test_annotate_3(): + def f(n): + if n > 0: + return virtual_ref(Y()) + else: + return virtual_ref(Z()) + a = RPythonAnnotator() + s = a.build_types(f, [int]) + assert isinstance(s, SomeVRef) + assert isinstance(s.s_instance, annmodel.SomeInstance) + assert s.s_instance.classdef == a.bookkeeper.getuniqueclassdef(X) + def test_rtype_1(): def f(): return virtual_ref(X()) @@ -46,3 +64,14 @@ return vref() x = interpret(f, []) assert lltype.castable(OBJECTPTR, lltype.typeOf(x)) > 0 + +def test_rtype_3(): + def f(n): + if n > 0: + return virtual_ref(Y()) + else: + return virtual_ref(Z()) + x = interpret(f, [-5]) + assert lltype.typeOf(x) == OBJECTPTR + +# the path "we_are_jitted()" is tested in jit/metainterp/test/test_codewriter. Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/lloperation.py Thu Dec 3 20:03:41 2009 @@ -434,6 +434,8 @@ 'do_malloc_varsize_clear': LLOp(canunwindgc=True), 'get_write_barrier_failing_case': LLOp(sideeffects=False), 'gc_get_type_info_group': LLOp(sideeffects=False), + 'jit_virtual_ref': LLOp(), + 'jit_virtual_force': LLOp(canfold=True), # __________ GC operations __________ Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/opimpl.py Thu Dec 3 20:03:41 2009 @@ -489,6 +489,9 @@ def op_gc_assume_young_pointers(addr): pass +def op_jit_virtual_force(x): + return x + # ____________________________________________________________ def get_op_impl(opname): From arigo at codespeak.net Thu Dec 3 21:25:06 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 3 Dec 2009 21:25:06 +0100 (CET) Subject: [pypy-svn] r69878 - in pypy/branch/virtual-forcing/pypy: jit/metainterp jit/metainterp/test rlib rpython rpython/lltypesystem rpython/test translator/c translator/cli translator/jvm Message-ID: <20091203202506.BC9D5168012@codespeak.net> Author: arigo Date: Thu Dec 3 21:25:04 2009 New Revision: 69878 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py pypy/branch/virtual-forcing/pypy/rlib/jit.py pypy/branch/virtual-forcing/pypy/rpython/llinterp.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/lloperation.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/opimpl.py pypy/branch/virtual-forcing/pypy/rpython/rvirtualizable2.py pypy/branch/virtual-forcing/pypy/rpython/test/test_rvirtualizable2.py pypy/branch/virtual-forcing/pypy/translator/c/funcgen.py pypy/branch/virtual-forcing/pypy/translator/cli/opcodes.py pypy/branch/virtual-forcing/pypy/translator/jvm/opcodes.py Log: * rename 'promote_virtualizable' in 'jit_force_virtualizable', which is closer to the current meaning. * add 'jit_force_virtual', not implemented yet. * slowly move 'virtual_ref' closer to pyjitpl.py. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py Thu Dec 3 21:25:04 2009 @@ -1213,7 +1213,7 @@ if pure and not all_promoted_args: effectinfo = calldescr.get_extra_info() assert (effectinfo is not None and - not effectinfo.promotes_virtualizables) + not effectinfo.forces_virtual_or_virtualizable) try: canraise = self.codewriter.raise_analyzer.can_raise(op) except lltype.DelayedPointer: @@ -1279,6 +1279,9 @@ return self._do_builtin_call(op, oopspec_name, args) def _do_builtin_call(self, op, oopspec_name, args): + if oopspec_name == 'virtual_ref': + self.handle_virtual_ref_call(op, args) + return argtypes = [v.concretetype for v in args] resulttype = op.result.concretetype c_func, TP = support.builtin_func_for_spec(self.codewriter.rtyper, @@ -1299,6 +1302,11 @@ self.emit_varargs([c_func] + non_void_args) self.register_var(op.result) + def handle_virtual_ref_call(self, op, args): + self.emit('virtual_ref') + self.emit(self.var_position(args[0])) + self.register_var(op.result) + def _array_of_voids(self, ARRAY): if isinstance(ARRAY, ootype.Array): return ARRAY.ITEM == ootype.Void @@ -1557,7 +1565,7 @@ log.WARNING("found debug_assert in %r; should have be removed" % (self.graph,)) - def serialize_op_promote_virtualizable(self, op): + def serialize_op_jit_force_virtualizable(self, op): vinfo = self.codewriter.metainterp_sd.virtualizable_info assert vinfo is not None assert vinfo.is_vtypeptr(op.args[0].concretetype) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py Thu Dec 3 21:25:04 2009 @@ -8,19 +8,20 @@ _cache = {} def __new__(cls, write_descrs_fields, write_descrs_arrays, - promotes_virtualizables=False): + forces_virtual_or_virtualizable=False): key = (frozenset(write_descrs_fields), frozenset(write_descrs_arrays), - promotes_virtualizables) + forces_virtual_or_virtualizable) if key in cls._cache: return cls._cache[key] result = object.__new__(cls) result.write_descrs_fields = write_descrs_fields result.write_descrs_arrays = write_descrs_arrays - result.promotes_virtualizables = promotes_virtualizables + result.forces_virtual_or_virtualizable= forces_virtual_or_virtualizable cls._cache[key] = result return result -def effectinfo_from_writeanalyze(effects, cpu, promotes_virtualizables=False): +def effectinfo_from_writeanalyze(effects, cpu, + forces_virtual_or_virtualizable=False): from pypy.translator.backendopt.writeanalyze import top_set if effects is top_set: return None @@ -44,7 +45,7 @@ else: assert 0 return EffectInfo(write_descrs_fields, write_descrs_arrays, - promotes_virtualizables) + forces_virtual_or_virtualizable) def consider_struct(TYPE, fieldname): if fieldType(TYPE, fieldname) is lltype.Void: @@ -73,4 +74,5 @@ class VirtualizableAnalyzer(BoolGraphAnalyzer): def analyze_simple_operation(self, op): - return op.opname == 'promote_virtualizable' + return op.opname in ('jit_force_virtualizable', + 'jit_force_virtual') Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Thu Dec 3 21:25:04 2009 @@ -988,7 +988,7 @@ def do_residual_call(self, argboxes, descr, exc): effectinfo = descr.get_extra_info() - if effectinfo is None or effectinfo.promotes_virtualizables: + if effectinfo is None or effectinfo.forces_virtual_or_virtualizable: # residual calls require attention to keep virtualizables in-sync self.metainterp.vable_before_residual_call() # xxx do something about code duplication Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py Thu Dec 3 21:25:04 2009 @@ -205,6 +205,7 @@ 'NEW/0d', 'NEW_WITH_VTABLE/1', 'NEW_ARRAY/1d', + 'FORCE_TOKEN/0', '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- 'SETARRAYITEM_GC/3d', @@ -221,7 +222,6 @@ 'COND_CALL_GC_MALLOC', # [a, b, if_(a<=b)_result, if_(a>b)_call, args...] # => result (for mallocs) 'DEBUG_MERGE_POINT/1', # debugging only - 'FORCE_TOKEN/0', '_CANRAISE_FIRST', # ----- start of can_raise operations ----- 'CALL', Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py Thu Dec 3 21:25:04 2009 @@ -283,7 +283,7 @@ assert calldescrs[0][4] is not None assert not calldescrs[0][4].write_descrs_fields assert not calldescrs[0][4].write_descrs_arrays - assert not calldescrs[0][4].promotes_virtualizables + assert not calldescrs[0][4].forces_virtual_or_virtualizable def test_oosend_look_inside_only_one(self): class A: @@ -394,7 +394,7 @@ assert cw.list_of_addr2name[0][1].endswith('.A1') assert cw.list_of_addr2name[1][1] == 'A1.g' - def test_promote_virtualizable_effectinfo(self): + def test_jit_force_virtualizable_effectinfo(self): class Frame(object): _virtualizable2_ = ['x'] @@ -430,9 +430,23 @@ effectinfo_g1 = calldescrs[1][4] effectinfo_g2 = calldescrs[2][4] effectinfo_h = calldescrs[3][4] - assert effectinfo_g1.promotes_virtualizables - assert effectinfo_g2.promotes_virtualizables - assert not effectinfo_h.promotes_virtualizables + assert effectinfo_g1.forces_virtual_or_virtualizable + assert effectinfo_g2.forces_virtual_or_virtualizable + assert not effectinfo_h.forces_virtual_or_virtualizable + + def test_vref_simple(self): + class X: + pass + def f(): + return jit.virtual_ref(X()) + graphs = self.make_graphs(f, []) + assert graphs[0].func is f + assert graphs[1].func is jit.virtual_ref + cw = CodeWriter(self.rtyper) + cw.candidate_graphs = [graphs[0]] + cw._start(self.metainterp_sd, None) + jitcode = cw.make_one_bytecode((graphs[0], None), False) + assert 'virtual_ref' in jitcode._source class ImmutableFieldsTests: Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Thu Dec 3 21:25:04 2009 @@ -28,7 +28,7 @@ hop.inputconst(lltype.Void, hop.args_v[1].value), hop.inputconst(lltype.Void, {})] hop.exception_cannot_occur() - return hop.genop('promote_virtualizable', + return hop.genop('jit_force_virtualizable', args_v, resulttype=lltype.Void) debug_print = lloperation.llop.debug_print Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py Thu Dec 3 21:25:04 2009 @@ -162,7 +162,7 @@ ts = self.warmrunnerdesc.cpu.ts (_, FUNCPTR) = ts.get_FuncType([self.VTYPEPTR], lltype.Void) funcptr = self.warmrunnerdesc.helper_func(FUNCPTR, force_if_necessary) - rvirtualizable2.replace_promote_virtualizable_with_call( + rvirtualizable2.replace_force_virtualizable_with_call( all_graphs, self.VTYPEPTR, funcptr) def unwrap_virtualizable_box(self, virtualizable_box): Modified: pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py Thu Dec 3 21:25:04 2009 @@ -35,19 +35,3 @@ [v] = hop.inputargs(self) v = hop.genop('jit_virtual_force', [v], resulttype = OBJECTPTR) return hop.genop('cast_pointer', [v], resulttype = hop.r_result) - -# ____________________________________________________________ - - -def jit_virtual_ref(x): - raise Exception("should not be reached") - -class Entry(ExtRegistryEntry): - _about_ = jit_virtual_ref - - def compute_result_annotation(self, s_obj): - return SomeVRef(s_obj) - - def specialize_call(self, hop): - [v] = hop.inputargs(getinstancerepr(hop.rtyper, None)) - return hop.genop('jit_virtual_ref', [v], resulttype = hop.r_result) Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/jit.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/jit.py Thu Dec 3 21:25:04 2009 @@ -102,11 +102,8 @@ # VRefs def virtual_ref(x): - if we_are_jitted(): - from pypy.rlib import _jit_vref - return _jit_vref.jit_virtual_ref(x) - else: - return DirectVRef(x) + return DirectVRef(x) +virtual_ref.oopspec = 'virtual_ref(x)' class DirectVRef(object): def __init__(self, x): Modified: pypy/branch/virtual-forcing/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/llinterp.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/llinterp.py Thu Dec 3 21:25:04 2009 @@ -546,9 +546,6 @@ def op_jit_marker(self, *args): pass - def op_promote_virtualizable(self, *args): - pass - def op_get_exception_addr(self, *args): raise NotImplementedError Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/lloperation.py Thu Dec 3 21:25:04 2009 @@ -427,15 +427,14 @@ # __________ used by the JIT ________ 'jit_marker': LLOp(), - 'promote_virtualizable':LLOp(canrun=True), + 'jit_force_virtualizable':LLOp(canrun=True), + 'jit_force_virtual': LLOp(canrun=True), 'get_exception_addr': LLOp(), 'get_exc_value_addr': LLOp(), 'do_malloc_fixedsize_clear': LLOp(canunwindgc=True), 'do_malloc_varsize_clear': LLOp(canunwindgc=True), 'get_write_barrier_failing_case': LLOp(sideeffects=False), 'gc_get_type_info_group': LLOp(sideeffects=False), - 'jit_virtual_ref': LLOp(), - 'jit_virtual_force': LLOp(canfold=True), # __________ GC operations __________ Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/opimpl.py Thu Dec 3 21:25:04 2009 @@ -436,8 +436,11 @@ def op_gc_stack_bottom(): pass # marker for trackgcroot.py -def op_promote_virtualizable(object, fieldname, flags): - pass # XXX should do something +def op_jit_force_virtualizable(*args): + pass + +def op_jit_force_virtual(x): + return x def op_get_group_member(TYPE, grpptr, memberoffset): from pypy.rpython.lltypesystem import llgroup @@ -489,9 +492,6 @@ def op_gc_assume_young_pointers(addr): pass -def op_jit_virtual_force(x): - return x - # ____________________________________________________________ def get_op_impl(opname): Modified: pypy/branch/virtual-forcing/pypy/rpython/rvirtualizable2.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/rvirtualizable2.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/rvirtualizable2.py Thu Dec 3 21:25:04 2009 @@ -52,10 +52,10 @@ #if not flags.get('access_directly'): if cname.value in self.my_redirected_fields: cflags = inputconst(lltype.Void, flags) - llops.genop('promote_virtualizable', [vinst, cname, cflags]) + llops.genop('jit_force_virtualizable', [vinst, cname, cflags]) -def replace_promote_virtualizable_with_call(graphs, VTYPEPTR, funcptr): +def replace_force_virtualizable_with_call(graphs, VTYPEPTR, funcptr): # funcptr should be an ll or oo function pointer with a VTYPEPTR argument c_funcptr = inputconst(lltype.typeOf(funcptr), funcptr) count = 0 @@ -65,7 +65,7 @@ continue newoplist = [] for i, op in enumerate(block.operations): - if (op.opname == 'promote_virtualizable' and + if (op.opname == 'jit_force_virtualizable' and match_virtualizable_type(op.args[0].concretetype, VTYPEPTR)): if op.args[-1].value.get('access_directly', False): @@ -75,7 +75,7 @@ count += 1 newoplist.append(op) block.operations = newoplist - log("replaced %d 'promote_virtualizable' with %r" % (count, funcptr)) + log("replaced %d 'jit_force_virtualizable' with %r" % (count, funcptr)) def match_virtualizable_type(TYPE, VTYPEPTR): if isinstance(TYPE, ootype.Instance): Modified: pypy/branch/virtual-forcing/pypy/rpython/test/test_rvirtualizable2.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/test/test_rvirtualizable2.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/test/test_rvirtualizable2.py Thu Dec 3 21:25:04 2009 @@ -1,7 +1,7 @@ import py from pypy.rpython.lltypesystem import lltype from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin -from pypy.rpython.rvirtualizable2 import replace_promote_virtualizable_with_call +from pypy.rpython.rvirtualizable2 import replace_force_virtualizable_with_call from pypy.rlib.jit import hint from pypy.objspace.flow.model import summary @@ -29,15 +29,15 @@ def __init__(self, v0): self.v0 = v0 -def get_promote_virtualizable_flags(graph): +def get_force_virtualizable_flags(graph): res = [] for block, op in graph.iterblockops(): - if op.opname == 'promote_virtualizable': + if op.opname == 'jit_force_virtualizable': res.append(op.args[-1].value) return res class BaseTest(BaseRtypingTest): - def test_generate_promote_virtualizable(self): + def test_generate_force_virtualizable(self): def fn(n): vinst = V(n) return vinst.v @@ -47,11 +47,11 @@ op_getfield = block.operations[-1] assert op_getfield.opname in ('getfield', 'oogetfield') v_inst = op_getfield.args[0] - assert op_promote.opname == 'promote_virtualizable' + assert op_promote.opname == 'jit_force_virtualizable' assert op_promote.args[0] is v_inst assert op_promote.args[-1].value == {} - def test_generate_promote_virtualizable_subclass(self): + def test_generate_force_virtualizable_subclass(self): def fn(n): V(n) # to attach v to V vinst = SubclassV(n) @@ -62,11 +62,11 @@ op_getfield = block.operations[-1] assert op_getfield.opname in ('getfield', 'oogetfield') v_inst = op_getfield.args[0] - assert op_promote.opname == 'promote_virtualizable' + assert op_promote.opname == 'jit_force_virtualizable' assert op_promote.args[0] is v_inst assert op_promote.args[-1].value == {} - def test_no_promote_virtualizable_for_other_fields(self): + def test_no_force_virtualizable_for_other_fields(self): def fn(n): vinst = V(n) return vinst.w @@ -77,7 +77,7 @@ assert op_getfield.opname in ('getfield', 'oogetfield') assert op_call.opname == 'direct_call' # to V.__init__ - def test_generate_promote_virtualizable_array(self): + def test_generate_force_virtualizable_array(self): def fn(n): vinst = VArray([n, n+1]) return vinst.lst[1] @@ -89,7 +89,7 @@ assert op_getarrayitem.opname == 'direct_call' # to ll_getitem_xxx assert op_getfield.opname in ('getfield', 'oogetfield') v_inst = op_getfield.args[0] - assert op_promote.opname == 'promote_virtualizable' + assert op_promote.opname == 'jit_force_virtualizable' assert op_promote.args[0] is v_inst assert op_promote.args[-1].value == {} @@ -126,13 +126,13 @@ TYPE = self.gettype(w_inst) assert 'virtualizable2_accessor' not in TYPE._hints - def replace_promote_virtualizable(self, rtyper, graphs): + def replace_force_virtualizable(self, rtyper, graphs): from pypy.annotation import model as annmodel from pypy.rpython.annlowlevel import MixLevelHelperAnnotator graph = graphs[0] for block, op in graph.iterblockops(): - if op.opname == 'promote_virtualizable': + if op.opname == 'jit_force_virtualizable': v_inst_ll_type = op.args[0].concretetype break @@ -145,11 +145,10 @@ s_vinst = annmodel.SomeOOInstance(v_inst_ll_type) funcptr = annhelper.delayedfunction(mycall, [s_vinst], annmodel.s_None) annhelper.finish() - replace_promote_virtualizable_with_call(graphs, v_inst_ll_type, - funcptr) + replace_force_virtualizable_with_call(graphs, v_inst_ll_type, funcptr) return funcptr - def test_replace_promote_virtualizable_with_call(self): + def test_replace_force_virtualizable_with_call(self): def fn(n): vinst = V(n) return vinst.v @@ -157,7 +156,7 @@ block = graph.startblock op_getfield = block.operations[-1] assert op_getfield.opname in ('getfield', 'oogetfield') - funcptr = self.replace_promote_virtualizable(rtyper, [graph]) + funcptr = self.replace_force_virtualizable(rtyper, [graph]) op_promote = block.operations[-2] op_getfield = block.operations[-1] assert op_getfield.opname in ('getfield', 'oogetfield') @@ -179,9 +178,9 @@ g_graph = t._graphof(g) expected = [{'access_directly': True}] * 3 - assert get_promote_virtualizable_flags(g_graph) == expected + assert get_force_virtualizable_flags(g_graph) == expected - self.replace_promote_virtualizable(typer, [g_graph]) + self.replace_force_virtualizable(typer, [g_graph]) assert summary(g_graph) == {self.GETFIELD: 2, self.SETFIELD: 1, 'int_add': 1} res = self.interpret(f, [23]) @@ -202,7 +201,7 @@ f_graph = t._graphof(f) g_graph = t._graphof(g) - self.replace_promote_virtualizable(typer, [f_graph, g_graph]) + self.replace_force_virtualizable(typer, [f_graph, g_graph]) t.checkgraphs() res = self.interpret(f, [23]) @@ -225,12 +224,12 @@ g_graphs.sort() assert g_graphs[0][0] is None - assert get_promote_virtualizable_flags(g_graphs[0][1]) == [{}] + assert get_force_virtualizable_flags(g_graphs[0][1]) == [{}] expected = [{'access_directly': True}] - assert get_promote_virtualizable_flags(g_graphs[1][1]) == expected + assert get_force_virtualizable_flags(g_graphs[1][1]) == expected - self.replace_promote_virtualizable(typer, [g_graphs[0][1], - g_graphs[1][1]]) + self.replace_force_virtualizable(typer, [g_graphs[0][1], + g_graphs[1][1]]) assert summary(g_graphs[0][1]) == {'direct_call': 1, self.GETFIELD: 1} assert summary(g_graphs[1][1]) == {self.GETFIELD: 1} @@ -265,8 +264,9 @@ assert summary(g_graphs[1][1]) == {self.SETFIELD: 1} h_graph = t._graphof(h) - assert summary(h_graph) == {'promote_virtualizable': 1, self.GETFIELD: 1} - assert get_promote_virtualizable_flags(h_graph) == [{}] + assert summary(h_graph) == {'jit_force_virtualizable': 1, + self.GETFIELD: 1} + assert get_force_virtualizable_flags(h_graph) == [{}] res = self.interpret(f, [23]) assert res == 23 @@ -292,7 +292,7 @@ t, typer, graph = self.gengraph(f, [int]) g_graph = t._graphof(A.g.im_func) - self.replace_promote_virtualizable(typer, [g_graph]) + self.replace_force_virtualizable(typer, [g_graph]) assert summary(g_graph) == {self.GETFIELD: 1, 'int_mul': 1} Modified: pypy/branch/virtual-forcing/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/c/funcgen.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/c/funcgen.py Thu Dec 3 21:25:04 2009 @@ -793,8 +793,12 @@ def OP_JIT_MARKER(self, op): return '/* JIT_MARKER %s */' % op - def OP_PROMOTE_VIRTUALIZABLE(self, op): - return '/* PROMOTE_VIRTUALIZABLE %s */' % op + def OP_JIT_FORCE_VIRTUALIZABLE(self, op): + return '/* JIT_FORCE_VIRTUALIZABLE %s */' % op + + def OP_JIT_FORCE_VIRTUAL(self, op): + return '%s = %s; /* JIT_FORCE_VIRTUAL */' % (self.expr(op.result), + self.expr(op.args[0]))) def OP_GET_GROUP_MEMBER(self, op): typename = self.db.gettype(op.result.concretetype) Modified: pypy/branch/virtual-forcing/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/cli/opcodes.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/cli/opcodes.py Thu Dec 3 21:25:04 2009 @@ -84,7 +84,8 @@ 'debug_fatalerror': [PushAllArgs, 'call void [pypylib]pypy.runtime.Debug::DEBUG_FATALERROR(string)'], 'keepalive': Ignore, 'jit_marker': Ignore, - 'promote_virtualizable': Ignore, + 'jit_force_virtualizable': Ignore, + 'jit_force_virtual': DoNothing, } # __________ numeric operations __________ Modified: pypy/branch/virtual-forcing/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/jvm/opcodes.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/jvm/opcodes.py Thu Dec 3 21:25:04 2009 @@ -97,7 +97,8 @@ 'gc_set_max_heap_size': Ignore, 'resume_point': Ignore, 'jit_marker': Ignore, - 'promote_virtualizable': Ignore, + 'jit_force_virtualizable': Ignore, + 'jit_force_virtual': DoNothing, 'debug_assert': [], # TODO: implement? From arigo at codespeak.net Thu Dec 3 21:57:11 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 3 Dec 2009 21:57:11 +0100 (CET) Subject: [pypy-svn] r69879 - in pypy/branch/virtual-forcing/pypy: annotation rlib rlib/test Message-ID: <20091203205711.25E75168012@codespeak.net> Author: arigo Date: Thu Dec 3 21:57:09 2009 New Revision: 69879 Modified: pypy/branch/virtual-forcing/pypy/annotation/binaryop.py pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py Log: Mixing with None is fine. Modified: pypy/branch/virtual-forcing/pypy/annotation/binaryop.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/annotation/binaryop.py (original) +++ pypy/branch/virtual-forcing/pypy/annotation/binaryop.py Thu Dec 3 21:57:09 2009 @@ -785,6 +785,8 @@ glob = globals() loc = locals() source = py.code.Source(""" + from pypy.tool.pairtype import pairtype + from pypy.annotation.model import SomePBC, SomeObject class __extend__(pairtype(%(classname)s, SomePBC)): def union((obj, pbc)): if pbc.isNone(): Modified: pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py Thu Dec 3 21:57:09 2009 @@ -1,4 +1,5 @@ from pypy.annotation import model as annmodel +from pypy.annotation.binaryop import _make_none_union from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rpython.rclass import getinstancerepr from pypy.rpython.rmodel import Repr @@ -11,6 +12,9 @@ def __init__(self, s_instance): self.s_instance = s_instance + def can_be_none(self): + return True + def simple_call(self): return self.s_instance @@ -22,6 +26,8 @@ def rtyper_makekey(self): return self.__class__, +_make_none_union('SomeVRef', 'obj.s_instance', globals()) + class VRefRepr(Repr): def __init__(self, rtyper): @@ -33,5 +39,5 @@ def rtype_simple_call(self, hop): [v] = hop.inputargs(self) - v = hop.genop('jit_virtual_force', [v], resulttype = OBJECTPTR) + v = hop.genop('jit_force_virtual', [v], resulttype = OBJECTPTR) return hop.genop('cast_pointer', [v], resulttype = hop.r_result) Modified: pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py Thu Dec 3 21:57:09 2009 @@ -52,6 +52,18 @@ assert isinstance(s.s_instance, annmodel.SomeInstance) assert s.s_instance.classdef == a.bookkeeper.getuniqueclassdef(X) +def test_annotate_4(): + def f(n): + if n > 0: + return virtual_ref(X()) + else: + return None + a = RPythonAnnotator() + s = a.build_types(f, [int]) + assert isinstance(s, SomeVRef) + assert isinstance(s.s_instance, annmodel.SomeInstance) + assert s.s_instance.classdef == a.bookkeeper.getuniqueclassdef(X) + def test_rtype_1(): def f(): return virtual_ref(X()) From arigo at codespeak.net Thu Dec 3 22:02:38 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 3 Dec 2009 22:02:38 +0100 (CET) Subject: [pypy-svn] r69880 - in pypy/branch/virtual-forcing/pypy/rlib: . test Message-ID: <20091203210238.31AE3168011@codespeak.net> Author: arigo Date: Thu Dec 3 22:02:37 2009 New Revision: 69880 Modified: pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py Log: Mixing with None, more of it. Modified: pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py Thu Dec 3 22:02:37 2009 @@ -5,6 +5,8 @@ from pypy.rpython.rmodel import Repr from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.lltypesystem.rclass import OBJECTPTR +from pypy.rpython.lltypesystem import lltype +from pypy.rpython.error import TyperError class SomeVRef(annmodel.SomeObject): @@ -19,9 +21,7 @@ return self.s_instance def rtyper_makerepr(self, rtyper): - if not hasattr(rtyper, '_vrefrepr'): - rtyper._vrefrepr = VRefRepr(rtyper) - return rtyper._vrefrepr + return vrefrepr def rtyper_makekey(self): return self.__class__, @@ -30,8 +30,7 @@ class VRefRepr(Repr): - def __init__(self, rtyper): - self.lowleveltype = getinstancerepr(rtyper, None).lowleveltype + lowleveltype = OBJECTPTR def specialize_call(self, hop): [v] = hop.inputargs(getinstancerepr(hop.rtyper, None)) @@ -41,3 +40,10 @@ [v] = hop.inputargs(self) v = hop.genop('jit_force_virtual', [v], resulttype = OBJECTPTR) return hop.genop('cast_pointer', [v], resulttype = hop.r_result) + + def convert_const(self, value): + if value: + raise TyperError("only supports None as a prebuilt virtual_ref") + return lltype.nullptr(OBJECTPTR.TO) + +vrefrepr = VRefRepr() Modified: pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py Thu Dec 3 22:02:37 2009 @@ -86,4 +86,14 @@ x = interpret(f, [-5]) assert lltype.typeOf(x) == OBJECTPTR +def test_rtype_4(): + def f(n): + if n > 0: + return virtual_ref(X()) + else: + return None + x = interpret(f, [-5]) + assert lltype.typeOf(x) == OBJECTPTR + assert not x + # the path "we_are_jitted()" is tested in jit/metainterp/test/test_codewriter. From arigo at codespeak.net Thu Dec 3 22:16:26 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 3 Dec 2009 22:16:26 +0100 (CET) Subject: [pypy-svn] r69881 - in pypy/branch/virtual-forcing/pypy/jit: backend/test backend/x86 metainterp metainterp/test Message-ID: <20091203211626.E8208168012@codespeak.net> Author: arigo Date: Thu Dec 3 22:16:25 2009 New Revision: 69881 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py Log: Step 1: implement rop.VIRTUAL_REF as a SAME_AS operation everywhere. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Thu Dec 3 22:16:25 2009 @@ -779,6 +779,12 @@ r = self.execute_operation(rop.SAME_AS, [BoxFloat(5.5)], 'float') assert r.value == 5.5 + def test_virtual_ref(self): + # if VIRTUAL_REF reaches the backend, it should just be a SAME_AS + u_box = self.alloc_unicode(u"hello\u1234") + r = self.execute_operation(rop.VIRTUAL_REF, [u_box], 'ref') + assert r.value == u_box.value + def test_jump(self): # this test generates small loops where the JUMP passes many # arguments of various types, shuffling them around. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py Thu Dec 3 22:16:25 2009 @@ -506,6 +506,7 @@ def genop_same_as(self, op, arglocs, resloc): self.mov(arglocs[0], resloc) genop_cast_ptr_to_int = genop_same_as + genop_virtual_ref = genop_same_as def genop_int_mod(self, op, arglocs, resloc): self.mc.CDQ() Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py Thu Dec 3 22:16:25 2009 @@ -869,6 +869,7 @@ resloc = self.force_allocate_reg(op.result) self.Perform(op, [argloc], resloc) consider_cast_ptr_to_int = consider_same_as + consider_virtual_ref = consider_same_as def consider_strlen(self, op, ignored): base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py Thu Dec 3 22:16:25 2009 @@ -1571,6 +1571,10 @@ assert vinfo.is_vtypeptr(op.args[0].concretetype) self.vable_flags[op.args[0]] = op.args[2].value + def serialize_op_jit_force_virtual(self, op): + raise ForcingVirtualRef("forcing a virtual_ref, i.e. calling it, " + "should not be seen by the JIT") + serialize_op_oostring = handle_builtin_call serialize_op_oounicode = handle_builtin_call serialize_op_gc_identityhash = handle_builtin_call @@ -1696,3 +1700,6 @@ def __str__(self): return "using virtualizable array in illegal way in %r" % ( self.args[0],) + +class ForcingVirtualRef(Exception): + pass Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py Thu Dec 3 22:16:25 2009 @@ -103,6 +103,9 @@ def do_same_as(cpu, box1): return box1 +def do_virtual_ref(cpu, box1): + return box1.clonebox() + def do_oois(cpu, box1, box2): tp = box1.type assert tp == box2.type Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Thu Dec 3 22:16:25 2009 @@ -238,7 +238,7 @@ for _opimpl in ['int_is_true', 'int_neg', 'int_invert', 'bool_not', 'cast_ptr_to_int', 'cast_float_to_int', 'cast_int_to_float', 'float_neg', 'float_abs', - 'float_is_true', + 'float_is_true', 'virtual_ref', ]: exec py.code.Source(''' @arguments("box") Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py Thu Dec 3 22:16:25 2009 @@ -206,6 +206,7 @@ 'NEW_WITH_VTABLE/1', 'NEW_ARRAY/1d', 'FORCE_TOKEN/0', + 'VIRTUAL_REF/1', '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- 'SETARRAYITEM_GC/3d', Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py Thu Dec 3 22:16:25 2009 @@ -2,7 +2,7 @@ from pypy.rlib import jit from pypy.jit.metainterp import support, typesystem from pypy.jit.metainterp.policy import JitPolicy -from pypy.jit.metainterp.codewriter import CodeWriter +from pypy.jit.metainterp.codewriter import CodeWriter, ForcingVirtualRef from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.translator.translator import graphof from pypy.rpython.lltypesystem.rbuiltin import ll_instantiate @@ -448,6 +448,21 @@ jitcode = cw.make_one_bytecode((graphs[0], None), False) assert 'virtual_ref' in jitcode._source + def test_vref_forced(self): + class X: + pass + def f(): + vref = jit.virtual_ref(X()) + return vref() + graphs = self.make_graphs(f, []) + assert graphs[0].func is f + assert graphs[1].func is jit.virtual_ref + cw = CodeWriter(self.rtyper) + cw.candidate_graphs = [graphs[0]] + cw._start(self.metainterp_sd, None) + py.test.raises(ForcingVirtualRef, cw.make_one_bytecode, + (graphs[0], None), False) # assert it does not work + class ImmutableFieldsTests: Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py Thu Dec 3 22:16:25 2009 @@ -5,6 +5,21 @@ class VRefTests: + def test_make_vref_simple(self): + class X: + pass + class ExCtx: + pass + exctx = ExCtx() + # + def f(): + exctx.topframeref = virtual_ref(X()) + exctx.topframeref = None + return 1 + # + self.interp_operations(f, []) + self.check_operations_history(virtual_ref=1) + def test_simple_no_access(self): myjitdriver = JitDriver(greens = [], reds = ['n']) # From arigo at codespeak.net Thu Dec 3 23:47:44 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 3 Dec 2009 23:47:44 +0100 (CET) Subject: [pypy-svn] r69882 - pypy/branch/virtual-forcing/pypy/rpython Message-ID: <20091203224744.00540168012@codespeak.net> Author: arigo Date: Thu Dec 3 23:47:43 2009 New Revision: 69882 Modified: pypy/branch/virtual-forcing/pypy/rpython/rvirtualizable2.py Log: Simplify this function. No changes in functionality. Modified: pypy/branch/virtual-forcing/pypy/rpython/rvirtualizable2.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/rvirtualizable2.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/rvirtualizable2.py Thu Dec 3 23:47:43 2009 @@ -61,10 +61,7 @@ count = 0 for graph in graphs: for block in graph.iterblocks(): - if not block.operations: - continue - newoplist = [] - for i, op in enumerate(block.operations): + for op in block.operations: if (op.opname == 'jit_force_virtualizable' and match_virtualizable_type(op.args[0].concretetype, VTYPEPTR)): @@ -73,8 +70,6 @@ op.opname = 'direct_call' op.args = [c_funcptr, op.args[0]] count += 1 - newoplist.append(op) - block.operations = newoplist log("replaced %d 'jit_force_virtualizable' with %r" % (count, funcptr)) def match_virtualizable_type(TYPE, VTYPEPTR): From arigo at codespeak.net Thu Dec 3 23:52:57 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 3 Dec 2009 23:52:57 +0100 (CET) Subject: [pypy-svn] r69883 - pypy/branch/virtual-forcing/pypy/rpython Message-ID: <20091203225257.4629A168012@codespeak.net> Author: arigo Date: Thu Dec 3 23:52:56 2009 New Revision: 69883 Modified: pypy/branch/virtual-forcing/pypy/rpython/rvirtualizable2.py Log: Sorry, revert broken version. Modified: pypy/branch/virtual-forcing/pypy/rpython/rvirtualizable2.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/rvirtualizable2.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/rvirtualizable2.py Thu Dec 3 23:52:56 2009 @@ -61,7 +61,10 @@ count = 0 for graph in graphs: for block in graph.iterblocks(): - for op in block.operations: + if not block.operations: + continue + newoplist = [] + for i, op in enumerate(block.operations): if (op.opname == 'jit_force_virtualizable' and match_virtualizable_type(op.args[0].concretetype, VTYPEPTR)): @@ -70,6 +73,8 @@ op.opname = 'direct_call' op.args = [c_funcptr, op.args[0]] count += 1 + newoplist.append(op) + block.operations = newoplist log("replaced %d 'jit_force_virtualizable' with %r" % (count, funcptr)) def match_virtualizable_type(TYPE, VTYPEPTR): From arigo at codespeak.net Thu Dec 3 23:55:10 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 3 Dec 2009 23:55:10 +0100 (CET) Subject: [pypy-svn] r69884 - in pypy/branch/virtual-forcing/pypy/jit/metainterp: . test Message-ID: <20091203225510.C4018168012@codespeak.net> Author: arigo Date: Thu Dec 3 23:55:09 2009 New Revision: 69884 Added: pypy/branch/virtual-forcing/pypy/jit/metainterp/vref.py (contents, props changed) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py Log: Support for virtual_ref(), step 2: implement a logic that seems a bit more correct now. It allows us to at least detect during tracing when some place forces a virtual, and give up tracing. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Thu Dec 3 23:55:09 2009 @@ -5,7 +5,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_start, debug_stop, debug_print -from pypy.jit.metainterp import history, compile, resume +from pypy.jit.metainterp import history, compile, resume, vref from pypy.jit.metainterp.history import Const, ConstInt, Box from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp import codewriter, executor @@ -238,7 +238,7 @@ for _opimpl in ['int_is_true', 'int_neg', 'int_invert', 'bool_not', 'cast_ptr_to_int', 'cast_float_to_int', 'cast_int_to_float', 'float_neg', 'float_abs', - 'float_is_true', 'virtual_ref', + 'float_is_true', ]: exec py.code.Source(''' @arguments("box") @@ -886,6 +886,14 @@ return self.metainterp.finishframe_exception(self.exception_box, self.exc_value_box) + @arguments("box") + def opimpl_virtual_ref(self, box): + obj = box.getref_base() + res = vref.virtual_ref_during_tracing(self.metainterp, obj) + resbox = history.BoxPtr(res) + self.metainterp.history.record(rop.VIRTUAL_REF, [box], resbox) + self.make_result_box(resbox) + # ------------------------------ def setup_call(self, argboxes): @@ -994,6 +1002,7 @@ # xxx do something about code duplication resbox = self.metainterp.execute_and_record_varargs( rop.CALL_MAY_FORCE, argboxes, descr=descr) + self.metainterp.virtual_after_residual_call() self.metainterp.vable_after_residual_call() if resbox is not None: self.make_result_box(resbox) @@ -1194,6 +1203,7 @@ class MetaInterp(object): in_recursion = 0 _already_allocated_resume_virtuals = None + _force_token_mem = None def __init__(self, staticdata): self.staticdata = staticdata @@ -1755,6 +1765,15 @@ if vable_escapes: self.load_fields_from_virtualizable() + def virtual_after_residual_call(self): + if self.is_blackholing() or not vref.was_forced(self): + return + # during tracing, more precisely during the CALL_MAY_FORCE, at least + # one of the vrefs was read. If we continue to trace and make + # assembler from there, we will get assembler that probably always + # forces a vref. So we just cancel now. + self.switch_to_blackhole() + def handle_exception(self): etype = self.cpu.get_exception() evalue = self.cpu.get_exc_value() Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Thu Dec 3 23:55:09 2009 @@ -944,9 +944,11 @@ for block, op in graph.iterblockops() if op.opname == 'direct_call'] - assert direct_calls(f_graph) == ['__init__', 'force_if_necessary', 'll_portal_runner'] - assert direct_calls(portal_graph) == ['force_if_necessary', 'maybe_enter_jit'] - + assert direct_calls(f_graph) == ['__init__', + 'force_virtualizable_if_necessary', + 'll_portal_runner'] + assert direct_calls(portal_graph)==['force_virtualizable_if_necessary', + 'maybe_enter_jit'] assert direct_calls(init_graph) == [] def test_virtual_child_frame(self): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py Thu Dec 3 23:55:09 2009 @@ -20,7 +20,36 @@ self.interp_operations(f, []) self.check_operations_history(virtual_ref=1) + def test_make_vref_and_force(self): + jitdriver = JitDriver(greens = [], reds = ['total', 'n']) + # + class X: + pass + class ExCtx: + pass + exctx = ExCtx() + # + @dont_look_inside + def force_me(): + return exctx.topframeref().n + # + def f(n): + total = 0 + while total < 300: + jitdriver.can_enter_jit(total=total, n=n) + jitdriver.jit_merge_point(total=total, n=n) + x = X() + x.n = n + 123 + exctx.topframeref = virtual_ref(x) + total += force_me() - 100 + return total + # + res = self.meta_interp(f, [-4]) + assert res == 16 * 19 + self.check_loops({}) # because we aborted tracing + def test_simple_no_access(self): + py.test.skip("in-progress") myjitdriver = JitDriver(greens = [], reds = ['n']) # class XY: Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py Thu Dec 3 23:55:09 2009 @@ -153,15 +153,16 @@ def finish(self): # - def force_if_necessary(virtualizable): + def force_virtualizable_if_necessary(virtualizable): if virtualizable.vable_token: self.force_now(virtualizable) - force_if_necessary._always_inline_ = True + force_virtualizable_if_necessary._always_inline_ = True # all_graphs = self.warmrunnerdesc.translator.graphs ts = self.warmrunnerdesc.cpu.ts (_, FUNCPTR) = ts.get_FuncType([self.VTYPEPTR], lltype.Void) - funcptr = self.warmrunnerdesc.helper_func(FUNCPTR, force_if_necessary) + funcptr = self.warmrunnerdesc.helper_func( + FUNCPTR, force_virtualizable_if_necessary) rvirtualizable2.replace_force_virtualizable_with_call( all_graphs, self.VTYPEPTR, funcptr) Added: pypy/branch/virtual-forcing/pypy/jit/metainterp/vref.py ============================================================================== --- (empty file) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/vref.py Thu Dec 3 23:55:09 2009 @@ -0,0 +1,74 @@ +from pypy.rpython.rmodel import inputconst, log +from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass + + +def replace_force_virtual_with_call(graphs, funcptr): + # similar to rvirtualizable2.replace_force_virtualizable_with_call(). + # funcptr should be an ll function pointer with a signature + # OBJECTPTR -> OBJECTPTR. + c_funcptr = inputconst(lltype.typeOf(funcptr), funcptr) + count = 0 + for graph in graphs: + for block in graph.iterblocks(): + for op in block.operations: + if op.opname == 'jit_force_virtual': + op.opname = 'direct_call' + op.args = [c_funcptr, op.args[0]] + count += 1 + log("replaced %d 'jit_force_virtual' with %r" % (count, funcptr)) + +# ____________________________________________________________ + + +# we make the low-level type of an RPython class directly +JIT_VIRTUAL_REF = lltype.GcStruct('JitVirtualRef', + ('super', rclass.OBJECT), + ('force_token', llmemory.Address), + ('forced', rclass.OBJECTPTR)) +jit_virtual_ref_vtable = lltype.malloc(rclass.OBJECT_VTABLE, zero=True, + flavor='raw') + +class ForceTokenMem(object): + def __init__(self): + self.allocated = lltype.malloc(rffi.CArray(lltype.Signed), 1, + flavor='raw') + self.allocated[0] = 0 + + def __del__(self): + lltype.free(self.allocated, flavor='raw') + +def get_force_token(metainterp): + if not metainterp._force_token_mem: + metainterp._force_token_mem = ForceTokenMem() + return llmemory.cast_ptr_to_adr(metainterp._force_token_mem.allocated) + +def was_forced(metainterp): + if not metainterp._force_token_mem: + return False + return metainterp._force_token_mem.allocated[0] == -1 + +def virtual_ref_during_tracing(metainterp, real_object): + vref = lltype.malloc(JIT_VIRTUAL_REF) + p = lltype.cast_pointer(rclass.OBJECTPTR, vref) + p.typeptr = jit_virtual_ref_vtable + vref.force_token = get_force_token(metainterp) + vref.forced = lltype.cast_opaque_ptr(rclass.OBJECTPTR, real_object) + assert vref.forced + return lltype.cast_opaque_ptr(llmemory.GCREF, vref) + +# ____________________________________________________________ + +def force_virtual_if_necessary(inst): + if not inst or inst.typeptr != jit_virtual_ref_vtable: + return inst # common, fast case + return force_virtual(inst) + +def force_virtual(inst): + vref = lltype.cast_pointer(lltype.Ptr(JIT_VIRTUAL_REF), inst) + if vref.force_token: + if not vref.forced: + xxxx + vref.force_token.signed[0] = -1 + vref.force_token = llmemory.NULL + return vref.forced +force_virtual._dont_inline_ = True Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py Thu Dec 3 23:55:09 2009 @@ -1,4 +1,4 @@ -import sys +import sys, py from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr from pypy.rpython.ootypesystem import ootype from pypy.rpython.annlowlevel import llhelper, MixLevelHelperAnnotator,\ @@ -17,7 +17,7 @@ from pypy.translator.unsimplify import call_final_function from pypy.jit.metainterp import codewriter -from pypy.jit.metainterp import support, history, pyjitpl, gc +from pypy.jit.metainterp import support, history, pyjitpl, gc, vref from pypy.jit.metainterp.pyjitpl import MetaInterpStaticData, MetaInterp from pypy.jit.metainterp.policy import JitPolicy from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper @@ -177,6 +177,7 @@ ) self.rewrite_can_enter_jit() self.rewrite_set_param() + self.rewrite_force_virtual() self.add_profiler_finish() self.metainterp_sd.finish_setup(optimizer=optimizer) @@ -599,6 +600,15 @@ op.opname = 'direct_call' op.args[:3] = [closures[funcname]] + def rewrite_force_virtual(self): + if self.cpu.ts.name != 'lltype': + py.test.skip("rewrite_force_virtual: port it to ootype") + FUNC = lltype.FuncType([rclass.OBJECTPTR], rclass.OBJECTPTR) + funcptr = self.helper_func(lltype.Ptr(FUNC), + vref.force_virtual_if_necessary) + all_graphs = self.translator.graphs + vref.replace_force_virtual_with_call(all_graphs, funcptr) + def decode_hp_hint_args(op): # Returns (list-of-green-vars, list-of-red-vars) without Voids. From fijal at codespeak.net Fri Dec 4 11:23:36 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 4 Dec 2009 11:23:36 +0100 (CET) Subject: [pypy-svn] r69886 - in pypy/branch/listcopyop/pypy/rpython: . lltypesystem memory memory/gc memory/gctransform memory/test Message-ID: <20091204102336.72C6C168011@codespeak.net> Author: fijal Date: Fri Dec 4 11:23:35 2009 New Revision: 69886 Modified: pypy/branch/listcopyop/pypy/rpython/llinterp.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Log: * Rename listcopy to arraycopy * Improve tests * Implement framework transform * Helper on GC can only accept addresses which point to GcArray of gc pointers Modified: pypy/branch/listcopyop/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/llinterp.py (original) +++ pypy/branch/listcopyop/pypy/rpython/llinterp.py Fri Dec 4 11:23:35 2009 @@ -754,9 +754,9 @@ def op_zero_gc_pointers_inside(self, obj): raise NotImplementedError("zero_gc_pointers_inside") - def op_listcopy(self, source, dest, source_start, dest_start, length): - if hasattr(self.heap, 'listcopy'): - self.heap.listcopy(source, dest, source_start, dest_start, length) + def op_gc_arraycopy(self, source, dest, source_start, dest_start, length): + if hasattr(self.heap, 'arraycopy'): + self.heap.arraycopy(source, dest, source_start, dest_start, length) return True return False Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py Fri Dec 4 11:23:35 2009 @@ -358,7 +358,7 @@ 'resize_buffer': LLOp(canraise=(MemoryError,), canunwindgc=True), 'finish_building_buffer' : LLOp(canraise=(MemoryError,), canunwindgc=True), 'zero_gc_pointers_inside': LLOp(), - 'gc_listcopy': LLOp(canrun=True), + 'gc_arraycopy': LLOp(canrun=True), 'free': LLOp(), 'getfield': LLOp(sideeffects=False, canrun=True), 'getarrayitem': LLOp(sideeffects=False, canrun=True), Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py Fri Dec 4 11:23:35 2009 @@ -394,7 +394,7 @@ checkadr(addr2) return addr1 - addr2 -def op_gc_listcopy(source, dest, source_start, dest_start, length): +def op_gc_arraycopy(source, dest, source_start, dest_start, length): return False # no special support def op_getfield(p, name): Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py Fri Dec 4 11:23:35 2009 @@ -214,7 +214,7 @@ p = before_len else: p = new_allocated - rgc.listcopy(items, newitems, 0, 0, p) + rgc.ll_arraycopy(items, newitems, 0, 0, p) l.length = newsize l.items = newitems _ll_list_resize_really._annenforceargs_ = (None, int) Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Fri Dec 4 11:23:35 2009 @@ -475,25 +475,25 @@ objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS self.last_generation_root_objects.append(addr_struct) - @specialize.arglltype(1) - def listcopy(self, source, dest, source_start, dest_start, length): - TP = lltype.typeOf(source).TO - source_addr = llmemory.cast_ptr_to_adr(source) - dest_addr = llmemory.cast_ptr_to_adr(dest) - if isinstance(TP.OF, lltype.Ptr): - need_set = False - if self.header(source_addr).tid & GCFLAG_NO_YOUNG_PTRS == 0: - need_set = True - if self.header(source_addr).tid & GCFLAG_NO_HEAP_PTRS == 0: - need_set = True - if need_set: - self.assume_young_pointers(dest_addr) - cp_source_addr = (source_addr + llmemory.itemoffsetof(TP, 0) + - llmemory.sizeof(TP.OF) * source_start) - cp_dest_addr = (dest_addr + llmemory.itemoffsetof(TP, 0) + - llmemory.sizeof(TP.OF) * dest_start) + def arraycopy(self, source_addr, dest_addr, source_start, + dest_start, length): + typeid = self.get_type_id(source_addr) + assert self.is_gcarrayofgcptr(typeid) + need_set = False + if self.header(source_addr).tid & GCFLAG_NO_YOUNG_PTRS == 0: + need_set = True + if self.header(source_addr).tid & GCFLAG_NO_HEAP_PTRS == 0: + need_set = True + if need_set: + self.assume_young_pointers(dest_addr) + itemsize = llmemory.gcarrayofptr_singleitemoffset + cp_source_addr = (source_addr + llmemory.gcarrayofptr_itemsoffset + + itemsize * source_start) + cp_dest_addr = (dest_addr + llmemory.gcarrayofptr_itemsoffset + + itemsize * dest_start) llmemory.raw_memcopy(cp_source_addr, cp_dest_addr, - llmemory.sizeof(TP.OF) * length) + itemsize * length) + return True def write_into_last_generation_obj(self, addr_struct, addr): objhdr = self.header(addr_struct) Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py Fri Dec 4 11:23:35 2009 @@ -292,6 +292,11 @@ [s_gc, annmodel.SomeInteger(knowntype=rffi.r_ushort)], annmodel.SomeInteger()) + if hasattr(GCClass, 'arraycopy'): + self.arraycopy_ptr = getfn(GCClass.arraycopy.im_func, + [s_gc] + [annmodel.SomeAddress()] * 2 + + [annmodel.SomeInteger()] * 3, annmodel.SomeBool()) + # in some GCs we can inline the common case of # malloc_fixedsize(typeid, size, True, False, False) if getattr(GCClass, 'inline_simple_malloc', False): @@ -776,16 +781,17 @@ TYPE = v_ob.concretetype.TO gen_zero_gc_pointers(TYPE, v_ob, hop.llops) - # def gct_listcopy(self, hop): - # if not hasattr(self.GCClass, 'listcopy'): - # return GCTransformer.gct_listcopy(self, hop) - # op = hop.spaceop - # ARGS = ([self.c_const_gc.concretetype] + - # [arg.concretetype for arg in op.args]) - # llptr = self.annotate_helper(self.GCClass.listcopy, ARGS, - # op.result.concretetype, inline=True) - # c_func = rmodel.inputconst(lltype.Void, llptr) - # hop.genop('direct_call', [c_func, self.c_const_gc] + op.args) + def gct_gc_arraycopy(self, hop): + if not hasattr(self, 'arraycopy_ptr'): + return GCTransformer.gct_gc_arraycopy(self, hop) + op = hop.spaceop + source_addr = hop.genop('cast_ptr_to_adr', [op.args[0]], + resulttype=llmemory.Address) + dest_addr = hop.genop('cast_ptr_to_adr', [op.args[1]], + resulttype=llmemory.Address) + hop.genop('direct_call', [self.arraycopy_ptr, self.c_const_gc, + source_addr, dest_addr] + op.args[2:], + resultvar=op.result) def gct_weakref_create(self, hop): op = hop.spaceop Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py Fri Dec 4 11:23:35 2009 @@ -14,7 +14,7 @@ from pypy.annotation import model as annmodel from pypy.rpython import rmodel from pypy.rpython.memory import gc -from pypy.rpython.memory.gctransform.support import var_ispyobj, ll_listcopy +from pypy.rpython.memory.gctransform.support import var_ispyobj from pypy.rpython.annlowlevel import MixLevelHelperAnnotator from pypy.rpython.rtyper import LowLevelOpList from pypy.rpython.rbuiltin import gen_cast @@ -380,7 +380,7 @@ def gct_zero_gc_pointers_inside(self, hop): pass - def gct_gc_listcopy(self, hop): + def gct_gc_arraycopy(self, hop): return rmodel.inputconst(lltype.Bool, False) def gct_gc_identityhash(self, hop): Modified: pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py Fri Dec 4 11:23:35 2009 @@ -128,10 +128,12 @@ ptr = lltype.cast_opaque_ptr(llmemory.GCREF, ptr) return self.gc.id(ptr) - def listcopy(self, source, dest, source_start, dest_start, length): - if hasattr(self.gc, 'listcopy'): - return self.gc.listcopy(source, dest, source_start, - dest_start, length) + def arraycopy(self, source, dest, source_start, dest_start, length): + if hasattr(self.gc, 'arraycopy'): + source_addr = llmemory.cast_ptr_to_adr(source) + dest_addr = llmemory.cast_ptr_to_adr(dest) + return self.gc.arraycopy(source_addr, dest_addr, source_start, + dest_start, length) i = 0 while i < length: dest[dest_start + i] = source[source_start + i] Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py Fri Dec 4 11:23:35 2009 @@ -551,20 +551,7 @@ res = self.interpret(fn, [-1000], taggedpointers=True) assert res == 111 - def test_listcopy(self): - TP = lltype.GcArray(lltype.Signed) - def fn(): - l = lltype.malloc(TP, 100) - for i in range(100): - l[i] = 1 - l2 = lltype.malloc(TP, 50) - llop.listcopy(lltype.Void, l, l2, 50, 0, 50) - for i in range(50): - assert l2[i] == 1 - - self.interpret(fn, []) - - def test_listcopy_ptr(self): + def test_arraycopy(self): S = lltype.GcStruct('S') TP = lltype.GcArray(lltype.Ptr(S)) def fn(): @@ -572,14 +559,14 @@ l2 = lltype.malloc(TP, 100) for i in range(100): l[i] = lltype.malloc(S) - llop.listcopy(lltype.Void, l, l2, 50, 0, 50) - x = [] - # force minor collect - t = (1, lltype.malloc(S)) - for i in range(20): - x.append(t) - for i in range(50): - assert l2[i] + if llop.gc_arraycopy(lltype.Void, l, l2, 50, 0, 50): + x = [] + # force minor collect + t = (1, lltype.malloc(S)) + for i in range(20): + x.append(t) + for i in range(50): + assert l2[i] return 0 self.interpret(fn, []) Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Fri Dec 4 11:23:35 2009 @@ -844,26 +844,7 @@ # ^^^ a crude assumption that totsize - varsize would be dividable by 4 # (and give fixedsize) - - def define_listcopy(cls): - TP = lltype.GcArray(lltype.Signed) - def fn(): - l = lltype.malloc(TP, 100) - for i in range(100): - l[i] = 1 - l2 = lltype.malloc(TP, 50) - if llop.gc_listcopy(lltype.Void, l, l2, 50, 0, 50): - for i in range(50): - assert l2[i] == 1 - return 0 - - return fn - - def test_listcopy(self): - run = self.runner("listcopy") - run([]) - - def define_listcopy_ptr(cls): + def define_arraycopy(cls): S = lltype.GcStruct('S') TP = lltype.GcArray(lltype.Ptr(S)) def fn(): @@ -871,7 +852,7 @@ l2 = lltype.malloc(TP, 100) for i in range(100): l[i] = lltype.malloc(S) - if llop.gc_listcopy(lltype.Void, l, l2, 50, 0, 50): + if llop.gc_arraycopy(lltype.Void, l, l2, 50, 0, 50): # force nursery collect x = [] for i in range(20): @@ -882,8 +863,8 @@ return fn - def test_listcopy_ptr(self): - run = self.runner("listcopy_ptr") + def test_arraycopy(self): + run = self.runner("arraycopy") run([]) # ________________________________________________________________ From fijal at codespeak.net Fri Dec 4 11:50:05 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 4 Dec 2009 11:50:05 +0100 (CET) Subject: [pypy-svn] r69887 - pypy/branch/listcopyop/pypy/rlib Message-ID: <20091204105005.ADF21168016@codespeak.net> Author: fijal Date: Fri Dec 4 11:50:04 2009 New Revision: 69887 Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py Log: Improve arraycopy, missing checkin Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rlib/rgc.py (original) +++ pypy/branch/listcopyop/pypy/rlib/rgc.py Fri Dec 4 11:50:04 2009 @@ -329,15 +329,26 @@ return hop.genop('finish_building_buffer', vlist, resulttype=hop.r_result.lowleveltype) -def listcopy(source, dest, source_start, dest_start, length): +def ll_arraycopy(source, dest, source_start, dest_start, length): from pypy.rpython.lltypesystem.lloperation import llop - from pypy.rpython.lltypesystem import lltype - - if llop.gc_listcopy(lltype.Void, source, dest, source_start, dest_start, - length): - return # gc supports nicely copying lists - i = 0 - while i < length: - dest[i + dest_start] = source[i + source_start] - i += 1 - + from pypy.rpython.lltypesystem import lltype, llmemory + + TP = lltype.typeOf(source).TO + if isinstance(TP.OF, lltype.Ptr): + if llop.gc_arraycopy(lltype.Void, source, dest, source_start, dest_start, + length): + return # gc supports nicely copying lists + i = 0 + while i < length: + dest[i + dest_start] = source[i + source_start] + i += 1 + # it's safe to do memcpy + source_addr = llmemory.cast_ptr_to_adr(source) + dest_addr = llmemory.cast_ptr_to_adr(dest) + cp_source_addr = (source_addr + llmemory.itemoffsetof(TP, 0) + + llmemory.sizeof(TP.OF) * source_start) + cp_dest_addr = (dest_addr + llmemory.itemoffsetof(TP, 0) + + llmemory.sizeof(TP.OF) * dest_start) + llmemory.raw_memcopy(cp_source_addr, cp_dest_addr, + llmemory.sizeof(TP.OF) * length) + From fijal at codespeak.net Fri Dec 4 12:07:31 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 4 Dec 2009 12:07:31 +0100 (CET) Subject: [pypy-svn] r69888 - in pypy/branch/listcopyop/pypy: rlib rlib/test rpython/lltypesystem Message-ID: <20091204110731.19D7216801B@codespeak.net> Author: fijal Date: Fri Dec 4 12:07:30 2009 New Revision: 69888 Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py pypy/branch/listcopyop/pypy/rlib/test/test_rgc.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/llmemory.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py Log: (arigo, fijal) * fix llmemory to handle raw_memcopy on arrays of any GC pointers. * write tests for rgc.ll_arraycopy(). Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rlib/rgc.py (original) +++ pypy/branch/listcopyop/pypy/rlib/rgc.py Fri Dec 4 12:07:30 2009 @@ -333,8 +333,9 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.lltypesystem import lltype, llmemory + assert source != dest TP = lltype.typeOf(source).TO - if isinstance(TP.OF, lltype.Ptr): + if isinstance(TP.OF, lltype.Ptr) and TP.OF.TO._gckind == 'gc': if llop.gc_arraycopy(lltype.Void, source, dest, source_start, dest_start, length): return # gc supports nicely copying lists @@ -342,6 +343,7 @@ while i < length: dest[i + dest_start] = source[i + source_start] i += 1 + return # it's safe to do memcpy source_addr = llmemory.cast_ptr_to_adr(source) dest_addr = llmemory.cast_ptr_to_adr(dest) Modified: pypy/branch/listcopyop/pypy/rlib/test/test_rgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rlib/test/test_rgc.py (original) +++ pypy/branch/listcopyop/pypy/rlib/test/test_rgc.py Fri Dec 4 12:07:30 2009 @@ -68,3 +68,61 @@ return hlstr(rgc.finish_building_buffer(ptr, 2)) assert f() == 'ab' + +def test_ll_arraycopy_1(): + TYPE = lltype.GcArray(lltype.Signed) + a1 = lltype.malloc(TYPE, 10) + a2 = lltype.malloc(TYPE, 6) + for i in range(10): a1[i] = 100 + i + for i in range(6): a2[i] = 200 + i + rgc.ll_arraycopy(a1, a2, 4, 2, 3) + for i in range(10): + assert a1[i] == 100 + i + for i in range(6): + if 2 <= i < 5: + assert a2[i] == a1[i+2] + else: + assert a2[i] == 200 + i + +def test_ll_arraycopy_2(): + TYPE = lltype.GcArray(lltype.Void) + a1 = lltype.malloc(TYPE, 10) + a2 = lltype.malloc(TYPE, 6) + rgc.ll_arraycopy(a1, a2, 4, 2, 3) + # nothing to assert here, should not crash... + +def test_ll_arraycopy_3(): + S = lltype.Struct('S') # non-gc + TYPE = lltype.GcArray(lltype.Ptr(S)) + a1 = lltype.malloc(TYPE, 10) + a2 = lltype.malloc(TYPE, 6) + org1 = [None] * 10 + org2 = [None] * 6 + for i in range(10): a1[i] = org1[i] = lltype.malloc(S, immortal=True) + for i in range(6): a2[i] = org2[i] = lltype.malloc(S, immortal=True) + rgc.ll_arraycopy(a1, a2, 4, 2, 3) + for i in range(10): + assert a1[i] == org1[i] + for i in range(6): + if 2 <= i < 5: + assert a2[i] == a1[i+2] + else: + assert a2[i] == org2[i] + +def test_ll_arraycopy_4(): + S = lltype.GcStruct('S') + TYPE = lltype.GcArray(lltype.Ptr(S)) + a1 = lltype.malloc(TYPE, 10) + a2 = lltype.malloc(TYPE, 6) + org1 = [None] * 10 + org2 = [None] * 6 + for i in range(10): a1[i] = org1[i] = lltype.malloc(S) + for i in range(6): a2[i] = org2[i] = lltype.malloc(S) + rgc.ll_arraycopy(a1, a2, 4, 2, 3) + for i in range(10): + assert a1[i] == org1[i] + for i in range(6): + if 2 <= i < 5: + assert a2[i] == a1[i+2] + else: + assert a2[i] == org2[i] Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/llmemory.py Fri Dec 4 12:07:30 2009 @@ -118,6 +118,9 @@ return if isinstance(self.TYPE, lltype.ContainerType): PTR = lltype.Ptr(self.TYPE) + elif self.TYPE == GCREF: + self._raw_memcopy_gcrefs(srcadr, dstadr) + return else: PTR = lltype.Ptr(lltype.FixedSizeArray(self.TYPE, 1)) while True: @@ -130,6 +133,18 @@ srcadr += ItemOffset(self.TYPE) dstadr += ItemOffset(self.TYPE) + def _raw_memcopy_gcrefs(self, srcadr, dstadr): + # special case to handle arrays of any GC pointers + repeat = self.repeat + while True: + data = srcadr.address[0] + dstadr.address[0] = data + repeat -= 1 + if repeat <= 0: + break + srcadr += ItemOffset(self.TYPE) + dstadr += ItemOffset(self.TYPE) + _end_markers = weakref.WeakKeyDictionary() # -> _endmarker class _endmarker_struct(lltype._struct): __slots__ = () Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py Fri Dec 4 12:07:30 2009 @@ -395,6 +395,11 @@ return addr1 - addr2 def op_gc_arraycopy(source, dest, source_start, dest_start, length): + A = lltype.typeOf(source) + assert A == lltype.typeOf(dest) + assert isinstance(A.TO, lltype.GcArray) + assert isinstance(A.TO.OF, lltype.Ptr) + assert A.TO.OF.TO._gckind == 'gc' return False # no special support def op_getfield(p, name): From pedronis at codespeak.net Fri Dec 4 14:01:58 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 4 Dec 2009 14:01:58 +0100 (CET) Subject: [pypy-svn] r69890 - in pypy/branch/esp-params/pypy/jit/backend/x86: . test Message-ID: <20091204130158.EF5DD168020@codespeak.net> Author: pedronis Date: Fri Dec 4 14:01:56 2009 New Revision: 69890 Modified: pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py pypy/branch/esp-params/pypy/jit/backend/x86/regalloc.py pypy/branch/esp-params/pypy/jit/backend/x86/test/test_regalloc.py Log: progress: basic logic in place, use sentinels local value in tests next: factor out/cleanup up the logic in genop_call and reuse it consistently Modified: pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py Fri Dec 4 14:01:56 2009 @@ -160,10 +160,9 @@ assert ([loc.assembler() for loc in arglocs] == [loc.assembler() for loc in faildescr._x86_debug_faillocs]) regalloc = RegAlloc(self, self.cpu.translate_support_code) - fail_frame_depth = faildescr._x86_current_frame_depth - fail_param_depth = faildescr._x86_current_param_depth - regalloc.prepare_bridge(fail_frame_depth, inputargs, arglocs, - operations) # xxx param_depth + fail_depths = faildescr._x86_current_depths + regalloc.prepare_bridge(fail_depths, inputargs, arglocs, + operations) adr_bridge = self.mc.tell() adr_stackadjust = self._patchable_stackadjust() frame_depth, param_depth = self._assemble(regalloc, operations) @@ -196,7 +195,6 @@ target_frame_depth = jump_target_descr._x86_frame_depth target_param_depth = jump_target_descr._x86_param_depth frame_depth = max(frame_depth, target_frame_depth) - # xxx tested? param_depth = max(param_depth, target_param_depth) return frame_depth, param_depth @@ -308,11 +306,10 @@ genop_discard_list[op.opnum](self, op, arglocs) def regalloc_perform_with_guard(self, op, guard_op, faillocs, - arglocs, resloc, current_frame_depth): + arglocs, resloc, current_depths): faildescr = guard_op.descr assert isinstance(faildescr, AbstractFailDescr) - faildescr._x86_current_frame_depth = current_frame_depth - faildescr._x86_current_param_depth = 0 # xxx + faildescr._x86_current_depths = current_depths failargs = guard_op.fail_args guard_opnum = guard_op.opnum failaddr = self.implement_guard_recovery(guard_opnum, @@ -329,9 +326,9 @@ faildescr._x86_adr_jump_offset = adr_jump_offset def regalloc_perform_guard(self, guard_op, faillocs, arglocs, resloc, - current_frame_depth): + current_depths): self.regalloc_perform_with_guard(None, guard_op, faillocs, arglocs, - resloc, current_frame_depth) + resloc, current_depths) def load_effective_addr(self, sizereg, baseofs, scale, result): self.mc.LEA(result, addr_add(imm(0), sizereg, baseofs, scale)) @@ -1083,9 +1080,8 @@ extra_on_stack += round_up_to_4(arglocs[arg].width) self._regalloc.reserve_param(extra_on_stack//4) - + #extra_on_stack = self.align_stack_for_call(extra_on_stack) - self.mc.SUB(esp, imm(extra_on_stack)) if isinstance(op.args[0], Const): x = rel32(op.args[0].getint()) else: @@ -1116,7 +1112,6 @@ p += round_up_to_4(loc.width) self.mc.CALL(x) self.mark_gc_roots() - self.mc.ADD(esp, imm(extra_on_stack)) if isinstance(resloc, MODRM64): self.mc.FSTP(resloc) elif size == 1: Modified: pypy/branch/esp-params/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/x86/regalloc.py Fri Dec 4 14:01:56 2009 @@ -150,12 +150,12 @@ self.loop_consts = loop_consts return self._process_inputargs(inputargs) - def prepare_bridge(self, prev_frame_depth, inputargs, arglocs, operations): + def prepare_bridge(self, prev_depths, inputargs, arglocs, operations): self._prepare(inputargs, operations) self.loop_consts = {} self._update_bindings(arglocs, inputargs) - self.fm.frame_depth = prev_frame_depth - self.param_depth = 0 # xxx + self.fm.frame_depth = prev_depths[0] + self.param_depth = prev_depths[1] def reserve_param(self, n): self.param_depth = max(self.param_depth, n) @@ -290,9 +290,10 @@ faillocs = self.locs_for_fail(guard_op) self.rm.position += 1 self.xrm.position += 1 + current_depths = (self.fm.frame_depth, self.param_depth) self.assembler.regalloc_perform_with_guard(op, guard_op, faillocs, arglocs, result_loc, - self.fm.frame_depth) + current_depths) if op.result is not None: self.possibly_free_var(op.result) self.possibly_free_vars(guard_op.fail_args) @@ -305,9 +306,10 @@ arglocs)) else: self.assembler.dump('%s(%s)' % (guard_op, arglocs)) + current_depths = (self.fm.frame_depth, self.param_depth) self.assembler.regalloc_perform_guard(guard_op, faillocs, arglocs, result_loc, - self.fm.frame_depth) + current_depths) self.possibly_free_vars(guard_op.fail_args) def PerformDiscard(self, op, arglocs): Modified: pypy/branch/esp-params/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/x86/test/test_regalloc.py Fri Dec 4 14:01:56 2009 @@ -541,23 +541,23 @@ def test_one_call(self): ops = ''' - [i0, i1] - i2 = call(ConstClass(f1ptr), i0, descr=f1_calldescr) - finish(i2) + [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + i10 = call(ConstClass(f1ptr), i0, descr=f1_calldescr) + finish(i10, i1, i2, i3, i4, i5, i6, i7, i8, i9) ''' - loop = self.interpret(ops, [4, 7]) - assert self.getints(1) == [5] + loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9, 9]) + assert self.getints(11) == [5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9] assert loop.token._x86_param_depth == 1 def test_two_calls(self): ops = ''' - [i0, i1] - i2 = call(ConstClass(f1ptr), i0, descr=f1_calldescr) - i3 = call(ConstClass(f2ptr), i2, i1, descr=f2_calldescr) - finish(i3) + [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + i10 = call(ConstClass(f1ptr), i0, descr=f1_calldescr) + i11 = call(ConstClass(f2ptr), i10, i1, descr=f2_calldescr) + finish(i11, i1, i2, i3, i4, i5, i6, i7, i8, i9) ''' - loop = self.interpret(ops, [4, 7]) - assert self.getints(1) == [5*7] + loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9, 9]) + assert self.getints(11) == [5*7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9] assert loop.token._x86_param_depth == 2 def test_bridge_calls_1(self): From fijal at codespeak.net Fri Dec 4 14:05:31 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 4 Dec 2009 14:05:31 +0100 (CET) Subject: [pypy-svn] r69891 - in pypy/branch/listcopyop/pypy/rpython/memory: . gc Message-ID: <20091204130531.82785168021@codespeak.net> Author: fijal Date: Fri Dec 4 14:05:30 2009 New Revision: 69891 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/base.py pypy/branch/listcopyop/pypy/rpython/memory/gctypelayout.py Log: Performance hack (an attempt at least) - store info whether object can have gc pointers at all, otherwise don't trace it Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gc/base.py Fri Dec 4 14:05:30 2009 @@ -46,6 +46,7 @@ DEBUG = False def set_query_functions(self, is_varsize, has_gcptr_in_varsize, + has_gcptr, is_gcarrayofgcptr, getfinalizer, offsets_to_gc_pointers, @@ -58,6 +59,7 @@ self.getfinalizer = getfinalizer self.is_varsize = is_varsize self.has_gcptr_in_varsize = has_gcptr_in_varsize + self.has_gcptr = has_gcptr self.is_gcarrayofgcptr = is_gcarrayofgcptr self.offsets_to_gc_pointers = offsets_to_gc_pointers self.fixed_size = fixed_size @@ -160,6 +162,8 @@ Typically, 'callback' is a bound method and 'arg' can be None. """ typeid = self.get_type_id(obj) + if not self.has_gcptr(typeid): + return if self.is_gcarrayofgcptr(typeid): # a performance shortcut for GcArray(gcptr) length = (obj + llmemory.gcarrayofptr_lengthoffset).signed[0] Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctypelayout.py Fri Dec 4 14:05:30 2009 @@ -62,6 +62,10 @@ infobits = self.get(typeid).infobits return (infobits & T_HAS_GCPTR_IN_VARSIZE) != 0 + def q_has_gcptrs(self, typeid): + infobits = self.get(typeid).infobits + return (infobits & T_HAS_GCPTR) != 0 + def q_is_gcarrayofgcptr(self, typeid): infobits = self.get(typeid).infobits return (infobits & T_IS_GCARRAY_OF_GCPTR) != 0 @@ -101,6 +105,7 @@ gc.set_query_functions( self.q_is_varsize, self.q_has_gcptr_in_varsize, + self.q_has_gcptrs, self.q_is_gcarrayofgcptr, self.q_finalizer, self.q_offsets_to_gc_pointers, @@ -117,8 +122,9 @@ T_MEMBER_INDEX = 0xffff T_IS_VARSIZE = 0x10000 T_HAS_GCPTR_IN_VARSIZE = 0x20000 -T_IS_GCARRAY_OF_GCPTR = 0x40000 -T_IS_WEAKREF = 0x80000 +T_HAS_GCPTR = 0x40000 +T_IS_GCARRAY_OF_GCPTR = 0x80000 +T_IS_WEAKREF = 0x100000 def _check_typeid(typeid): ll_assert(llop.is_group_member_nonzero(lltype.Bool, typeid), @@ -131,6 +137,7 @@ infobits = index info.ofstoptrs = builder.offsets2table(offsets, TYPE) info.finalizer = builder.make_finalizer_funcptr_for_type(TYPE) + arroffsets = None if not TYPE._is_varsize(): info.fixedsize = llarena.round_up_for_allocation( llmemory.sizeof(TYPE), builder.GCClass.object_minimal_size) @@ -157,15 +164,18 @@ varinfo.ofstovar = llmemory.itemoffsetof(TYPE, 0) assert isinstance(ARRAY, lltype.Array) if ARRAY.OF != lltype.Void: - offsets = offsets_to_gc_pointers(ARRAY.OF) + arroffsets = offsets_to_gc_pointers(ARRAY.OF) else: - offsets = () - if len(offsets) > 0: + arroffsets = () + if len(arroffsets) > 0: infobits |= T_HAS_GCPTR_IN_VARSIZE - varinfo.varofstoptrs = builder.offsets2table(offsets, ARRAY.OF) + varinfo.varofstoptrs = builder.offsets2table(arroffsets, ARRAY.OF) varinfo.varitemsize = llmemory.sizeof(ARRAY.OF) if TYPE == WEAKREF: infobits |= T_IS_WEAKREF + if offsets or arroffsets: + infobits |= T_HAS_GCPTR + info.infobits = infobits # ____________________________________________________________ From arigo at codespeak.net Fri Dec 4 15:18:14 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 4 Dec 2009 15:18:14 +0100 (CET) Subject: [pypy-svn] r69892 - pypy/branch/listcopyop/py/impl/path Message-ID: <20091204141814.13FAB16802D@codespeak.net> Author: arigo Date: Fri Dec 4 15:18:13 2009 New Revision: 69892 Modified: pypy/branch/listcopyop/py/impl/path/svnwc.py Log: Backport a fix from the hg trunk, needed for fixeol to work. Modified: pypy/branch/listcopyop/py/impl/path/svnwc.py ============================================================================== --- pypy/branch/listcopyop/py/impl/path/svnwc.py (original) +++ pypy/branch/listcopyop/py/impl/path/svnwc.py Fri Dec 4 15:18:13 2009 @@ -808,9 +808,11 @@ def notsvn(path): return path.basename != '.svn' - paths = [self.__class__(p, auth=self.auth) - for p in self.localpath.listdir() - if notsvn(p) and (not fil or fil(p))] + paths = [] + for localpath in self.localpath.listdir(notsvn): + p = self.__class__(localpath, auth=self.auth) + if notsvn(p) and (not fil or fil(p)): + paths.append(p) self._sortlist(paths, sort) return paths From arigo at codespeak.net Fri Dec 4 15:19:59 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 4 Dec 2009 15:19:59 +0100 (CET) Subject: [pypy-svn] r69893 - pypy/trunk/py/impl/path Message-ID: <20091204141959.7D36C168027@codespeak.net> Author: arigo Date: Fri Dec 4 15:19:58 2009 New Revision: 69893 Modified: pypy/trunk/py/impl/path/svnwc.py Log: Backport a fix from the hg trunk, needed for fixeol to work. Modified: pypy/trunk/py/impl/path/svnwc.py ============================================================================== --- pypy/trunk/py/impl/path/svnwc.py (original) +++ pypy/trunk/py/impl/path/svnwc.py Fri Dec 4 15:19:58 2009 @@ -808,9 +808,11 @@ def notsvn(path): return path.basename != '.svn' - paths = [self.__class__(p, auth=self.auth) - for p in self.localpath.listdir() - if notsvn(p) and (not fil or fil(p))] + paths = [] + for localpath in self.localpath.listdir(notsvn): + p = self.__class__(localpath, auth=self.auth) + if notsvn(p) and (not fil or fil(p)): + paths.append(p) self._sortlist(paths, sort) return paths From hpk at codespeak.net Fri Dec 4 15:25:36 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 4 Dec 2009 15:25:36 +0100 (CET) Subject: [pypy-svn] r69894 - in pypy/trunk/py: . impl/cmdline impl/compat impl/log impl/path impl/test impl/test/dist impl/test/looponfail plugin Message-ID: <20091204142536.B6D9416803B@codespeak.net> Author: hpk Date: Fri Dec 4 15:25:35 2009 New Revision: 69894 Modified: pypy/trunk/py/__init__.py pypy/trunk/py/impl/cmdline/pytest.py pypy/trunk/py/impl/compat/dep_doctest.py pypy/trunk/py/impl/compat/dep_optparse.py pypy/trunk/py/impl/compat/dep_subprocess.py pypy/trunk/py/impl/compat/dep_textwrap.py pypy/trunk/py/impl/log/log.py pypy/trunk/py/impl/log/warning.py pypy/trunk/py/impl/path/svnwc.py pypy/trunk/py/impl/test/config.py pypy/trunk/py/impl/test/dist/gwmanage.py pypy/trunk/py/impl/test/dist/nodemanage.py pypy/trunk/py/impl/test/looponfail/remote.py pypy/trunk/py/impl/test/pluginmanager.py pypy/trunk/py/plugin/pytest_default.py pypy/trunk/py/plugin/pytest_pytester.py pypy/trunk/py/plugin/pytest_skipping.py pypy/trunk/py/plugin/pytest_terminal.py pypy/trunk/py/plugin/pytest_tmpdir.py Log: updating from py-1.1.0 to py-1.1.1 to fix fixeol/svn-related bugs Modified: pypy/trunk/py/__init__.py ============================================================================== --- pypy/trunk/py/__init__.py (original) +++ pypy/trunk/py/__init__.py Fri Dec 4 15:25:35 2009 @@ -9,7 +9,7 @@ (c) Holger Krekel and others, 2009 """ -version = "1.1.0" +version = "1.1.1" __version__ = version = version or "1.1.x" import py.apipkg @@ -68,6 +68,9 @@ 'Function' : '.impl.test.pycollect:Function', '_fillfuncargs' : '.impl.test.funcargs:fillfuncargs', }, + 'cmdline': { + 'main' : '.impl.test.cmdline:main', # backward compat + }, }, # hook into the top-level standard library Modified: pypy/trunk/py/impl/cmdline/pytest.py ============================================================================== --- pypy/trunk/py/impl/cmdline/pytest.py (original) +++ pypy/trunk/py/impl/cmdline/pytest.py Fri Dec 4 15:25:35 2009 @@ -1,5 +1,5 @@ #!/usr/bin/env python import py -def main(): - py.test.cmdline.main() +def main(args): + py.test.cmdline.main(args) Modified: pypy/trunk/py/impl/compat/dep_doctest.py ============================================================================== --- pypy/trunk/py/impl/compat/dep_doctest.py (original) +++ pypy/trunk/py/impl/compat/dep_doctest.py Fri Dec 4 15:25:35 2009 @@ -1,4 +1,5 @@ import py -py.log._apiwarn("1.1", "py.compat.doctest deprecated, use standard library version.", stacklevel="initpkg") +py.log._apiwarn("1.1", "py.compat.doctest deprecated, use standard library version.", +stacklevel="apipkg") doctest = py.std.doctest Modified: pypy/trunk/py/impl/compat/dep_optparse.py ============================================================================== --- pypy/trunk/py/impl/compat/dep_optparse.py (original) +++ pypy/trunk/py/impl/compat/dep_optparse.py Fri Dec 4 15:25:35 2009 @@ -1,4 +1,4 @@ import py -py.log._apiwarn("1.1", "py.compat.optparse deprecated, use standard library version.", stacklevel="initpkg") +py.log._apiwarn("1.1", "py.compat.optparse deprecated, use standard library version.", stacklevel="apipkg") optparse = py.std.optparse Modified: pypy/trunk/py/impl/compat/dep_subprocess.py ============================================================================== --- pypy/trunk/py/impl/compat/dep_subprocess.py (original) +++ pypy/trunk/py/impl/compat/dep_subprocess.py Fri Dec 4 15:25:35 2009 @@ -1,4 +1,5 @@ import py -py.log._apiwarn("1.1", "py.compat.subprocess deprecated, use standard library version.", stacklevel="initpkg") +py.log._apiwarn("1.1", "py.compat.subprocess deprecated, use standard library version.", +stacklevel="apipkg") subprocess = py.std.subprocess Modified: pypy/trunk/py/impl/compat/dep_textwrap.py ============================================================================== --- pypy/trunk/py/impl/compat/dep_textwrap.py (original) +++ pypy/trunk/py/impl/compat/dep_textwrap.py Fri Dec 4 15:25:35 2009 @@ -1,4 +1,5 @@ import py -py.log._apiwarn("1.1", "py.compat.textwrap deprecated, use standard library version.", stacklevel="initpkg") +py.log._apiwarn("1.1", "py.compat.textwrap deprecated, use standard library version.", + stacklevel="apipkg") textwrap = py.std.textwrap Modified: pypy/trunk/py/impl/log/log.py ============================================================================== --- pypy/trunk/py/impl/log/log.py (original) +++ pypy/trunk/py/impl/log/log.py Fri Dec 4 15:25:35 2009 @@ -132,6 +132,8 @@ def __call__(self, msg): """ write a message to the log """ self._file.write(str(msg) + "\n") + if hasattr(self._file, 'flush'): + self._file.flush() class Path(object): """ log consumer that opens and writes to a Path """ Modified: pypy/trunk/py/impl/log/warning.py ============================================================================== --- pypy/trunk/py/impl/log/warning.py (original) +++ pypy/trunk/py/impl/log/warning.py Fri Dec 4 15:25:35 2009 @@ -13,14 +13,18 @@ def _apiwarn(startversion, msg, stacklevel=2, function=None): # below is mostly COPIED from python2.4/warnings.py's def warn() # Get context information - if stacklevel == "initpkg": - frame = sys._getframe(stacklevel == "initpkg" and 1 or stacklevel) - level = 2 + if isinstance(stacklevel, str): + frame = sys._getframe(1) + level = 1 + found = frame.f_code.co_filename.find(stacklevel) != -1 while frame: co = frame.f_code - if co.co_name == "__getattr__" and co.co_filename.find("initpkg") !=-1: - stacklevel = level - break + if co.co_filename.find(stacklevel) == -1: + if found: + stacklevel = level + break + else: + found = True level += 1 frame = frame.f_back else: Modified: pypy/trunk/py/impl/path/svnwc.py ============================================================================== --- pypy/trunk/py/impl/path/svnwc.py (original) +++ pypy/trunk/py/impl/path/svnwc.py Fri Dec 4 15:25:35 2009 @@ -77,7 +77,7 @@ # svn support code -ALLOWED_CHARS = "_ -/\\=$.~+" #add characters as necessary when tested +ALLOWED_CHARS = "_ -/\\=$.~+%" #add characters as necessary when tested if sys.platform == "win32": ALLOWED_CHARS += ":" ALLOWED_CHARS_HOST = ALLOWED_CHARS + '@:' @@ -521,9 +521,12 @@ args.append(url) self._authsvn('co', args) - def update(self, rev='HEAD'): + def update(self, rev='HEAD', interactive=True): """ update working copy item to given revision. (None -> HEAD). """ - self._authsvn('up', ['-r', rev, "--non-interactive"],) + opts = ['-r', rev] + if not interactive: + opts.append("--non-interactive") + self._authsvn('up', opts) def write(self, content, mode='w'): """ write content into local filesystem wc. """ Modified: pypy/trunk/py/impl/test/config.py ============================================================================== --- pypy/trunk/py/impl/test/config.py (original) +++ pypy/trunk/py/impl/test/config.py Fri Dec 4 15:25:35 2009 @@ -74,6 +74,7 @@ def _preparse(self, args): self._conftest.setinitial(args) + self.pluginmanager.consider_setuptools_entrypoints() self.pluginmanager.consider_preparse(args) self.pluginmanager.consider_env() self.pluginmanager.do_addoption(self._parser) Modified: pypy/trunk/py/impl/test/dist/gwmanage.py ============================================================================== --- pypy/trunk/py/impl/test/dist/gwmanage.py (original) +++ pypy/trunk/py/impl/test/dist/gwmanage.py Fri Dec 4 15:25:35 2009 @@ -10,9 +10,9 @@ class GatewayManager: RemoteError = RemoteError def __init__(self, specs, hook, defaultchdir="pyexecnetcache"): - self.gateways = [] self.specs = [] self.hook = hook + self.group = execnet.Group() for spec in specs: if not isinstance(spec, execnet.XSpec): spec = execnet.XSpec(spec) @@ -21,48 +21,19 @@ self.specs.append(spec) def makegateways(self): - assert not self.gateways + assert not list(self.group) for spec in self.specs: - gw = execnet.makegateway(spec) - self.gateways.append(gw) - gw.id = "[%s]" % len(self.gateways) + gw = self.group.makegateway(spec) self.hook.pytest_gwmanage_newgateway( gateway=gw, platinfo=gw._rinfo()) - def getgateways(self, remote=True, inplacelocal=True): - if not self.gateways and self.specs: - self.makegateways() - l = [] - for gw in self.gateways: - if gw.spec._samefilesystem(): - if inplacelocal: - l.append(gw) - else: - if remote: - l.append(gw) - return execnet.MultiGateway(gateways=l) - - def multi_exec(self, source, inplacelocal=True): - """ remote execute code on all gateways. - @param inplacelocal=False: don't send code to inplacelocal hosts. - """ - multigw = self.getgateways(inplacelocal=inplacelocal) - return multigw.remote_exec(source) - - def multi_chdir(self, basename, inplacelocal=True): - """ perform a remote chdir to the given path, may be relative. - @param inplacelocal=False: don't send code to inplacelocal hosts. - """ - self.multi_exec("import os ; os.chdir(%r)" % basename, - inplacelocal=inplacelocal).waitclose() - def rsync(self, source, notify=None, verbose=False, ignores=None): """ perform rsync to all remote hosts. """ rsync = HostRSync(source, verbose=verbose, ignores=ignores) seen = py.builtin.set() gateways = [] - for gateway in self.gateways: + for gateway in self.group: spec = gateway.spec if not spec._samefilesystem(): if spec not in seen: @@ -84,9 +55,7 @@ ) def exit(self): - while self.gateways: - gw = self.gateways.pop() - gw.exit() + self.group.terminate() class HostRSync(execnet.RSync): """ RSyncer that filters out common files Modified: pypy/trunk/py/impl/test/dist/nodemanage.py ============================================================================== --- pypy/trunk/py/impl/test/dist/nodemanage.py (original) +++ pypy/trunk/py/impl/test/dist/nodemanage.py Fri Dec 4 15:25:35 2009 @@ -57,7 +57,7 @@ def setup_nodes(self, putevent): self.rsync_roots() self.trace("setting up nodes") - for gateway in self.gwmanager.gateways: + for gateway in self.gwmanager.group: node = TXNode(gateway, self.config, putevent, slaveready=self._slaveready) gateway.node = node # to keep node alive self.trace("started node %r" % node) @@ -67,7 +67,7 @@ #assert node.gateway.node == node self.nodes.append(node) self.trace("%s slave node ready %r" % (node.gateway.id, node)) - if len(self.nodes) == len(self.gwmanager.gateways): + if len(self.nodes) == len(list(self.gwmanager.group)): self._nodesready.set() def wait_nodesready(self, timeout=None): Modified: pypy/trunk/py/impl/test/looponfail/remote.py ============================================================================== --- pypy/trunk/py/impl/test/looponfail/remote.py (original) +++ pypy/trunk/py/impl/test/looponfail/remote.py Fri Dec 4 15:25:35 2009 @@ -54,7 +54,7 @@ py.builtin.print_("RemoteControl:", msg) def initgateway(self): - return execnet.PopenGateway() + return execnet.makegateway("popen") def setup(self, out=None): if out is None: Modified: pypy/trunk/py/impl/test/pluginmanager.py ============================================================================== --- pypy/trunk/py/impl/test/pluginmanager.py (original) +++ pypy/trunk/py/impl/test/pluginmanager.py Fri Dec 4 15:25:35 2009 @@ -77,6 +77,17 @@ for spec in self._envlist("PYTEST_PLUGINS"): self.import_plugin(spec) + def consider_setuptools_entrypoints(self): + try: + from pkg_resources import iter_entry_points + except ImportError: + return # XXX issue a warning + for ep in iter_entry_points('pytest11'): + if ep.name in self._name2plugin: + continue + plugin = ep.load() + self.register(plugin, name=ep.name) + def consider_preparse(self, args): for opt1,opt2 in zip(args, args[1:]): if opt1 == "-p": Modified: pypy/trunk/py/plugin/pytest_default.py ============================================================================== --- pypy/trunk/py/plugin/pytest_default.py (original) +++ pypy/trunk/py/plugin/pytest_default.py Fri Dec 4 15:25:35 2009 @@ -7,6 +7,9 @@ import execnet except ImportError: execnet = None +else: + if not hasattr(execnet, 'Group'): + execnet = None def pytest_pyfunc_call(__multicall__, pyfuncitem): if not __multicall__.execute(): @@ -70,7 +73,7 @@ add_dist_options(parser) else: parser.epilog = ( - "'execnet' package required for --looponfailing / distributed testing.") + "'execnet>=1.0.0b4' package required for --looponfailing / distributed testing.") def add_dist_options(parser): # see http://pytest.org/help/dist") Modified: pypy/trunk/py/plugin/pytest_pytester.py ============================================================================== --- pypy/trunk/py/plugin/pytest_pytester.py (original) +++ pypy/trunk/py/plugin/pytest_pytester.py Fri Dec 4 15:25:35 2009 @@ -319,7 +319,7 @@ return self.runpybin("py.test", *args) def spawn_pytest(self, string, expect_timeout=10.0): - pexpect = py.test.importorskip("pexpect", "2.3") + pexpect = py.test.importorskip("pexpect", "2.4") basetemp = self.tmpdir.mkdir("pexpect") invoke = "%s %s" % self._getpybinargs("py.test") cmd = "%s --basetemp=%s %s" % (invoke, basetemp, string) Modified: pypy/trunk/py/plugin/pytest_skipping.py ============================================================================== --- pypy/trunk/py/plugin/pytest_skipping.py (original) +++ pypy/trunk/py/plugin/pytest_skipping.py Fri Dec 4 15:25:35 2009 @@ -162,10 +162,9 @@ xfailed = tr.stats.get("xfailed") if xfailed: if not tr.hasopt('xfailed'): - if tr.config.getvalue("verbose"): - tr.write_line( - "%d expected failures, use --report=xfailed for more info" % - len(xfailed)) + tr.write_line( + "%d expected failures, use --report=xfailed for more info" % + len(xfailed)) return tr.write_sep("_", "expected failures") for rep in xfailed: @@ -220,10 +219,9 @@ skipped = tr.stats.get('skipped', []) if skipped: if not tr.hasopt('skipped'): - if tr.config.getvalue("verbose"): - tr.write_line( - "%d skipped tests, use --report=skipped for more info" % - len(skipped)) + tr.write_line( + "%d skipped tests, use --report=skipped for more info" % + len(skipped)) return fskips = folded_skips(skipped) if fskips: Modified: pypy/trunk/py/plugin/pytest_terminal.py ============================================================================== --- pypy/trunk/py/plugin/pytest_terminal.py (original) +++ pypy/trunk/py/plugin/pytest_terminal.py Fri Dec 4 15:25:35 2009 @@ -15,7 +15,7 @@ help="show locals in tracebacks (disabled by default).") group.addoption('--report', action="store", dest="report", default=None, metavar="opts", - help="comma separated reporting options") + help="comma separated options, valid: skipped,xfailed") group._addoption('--tb', metavar="style", action="store", dest="tbstyle", default='long', type="choice", choices=['long', 'short', 'no'], @@ -150,7 +150,7 @@ else: d['extra'] = "" d['cwd'] = platinfo.cwd - infoline = ("%(id)s %(spec)s -- platform %(platform)s, " + infoline = ("[%(id)s] %(spec)s -- platform %(platform)s, " "Python %(version)s " "cwd: %(cwd)s" "%(extra)s" % d) @@ -158,14 +158,14 @@ self.gateway2info[gateway] = infoline def pytest_gwmanage_rsyncstart(self, source, gateways): - targets = ", ".join([gw.id for gw in gateways]) + targets = ", ".join(["[%s]" % gw.id for gw in gateways]) msg = "rsyncstart: %s -> %s" %(source, targets) if not self.config.option.verbose: msg += " # use --verbose to see rsync progress" self.write_line(msg) def pytest_gwmanage_rsyncfinish(self, source, gateways): - targets = ", ".join([gw.id for gw in gateways]) + targets = ", ".join(["[%s]" % gw.id for gw in gateways]) self.write_line("rsyncfinish: %s -> %s" %(source, targets)) def pytest_plugin_registered(self, plugin): @@ -177,11 +177,11 @@ self.write_line(msg) def pytest_testnodeready(self, node): - self.write_line("%s txnode ready to receive tests" %(node.gateway.id,)) + self.write_line("[%s] txnode ready to receive tests" %(node.gateway.id,)) def pytest_testnodedown(self, node, error): if error: - self.write_line("%s node down, error: %s" %(node.gateway.id, error)) + self.write_line("[%s] node down, error: %s" %(node.gateway.id, error)) def pytest_trace(self, category, msg): if self.config.option.debug or \ @@ -203,7 +203,7 @@ line = self._reportinfoline(item) extra = "" if node: - extra = "-> " + str(node.gateway.id) + extra = "-> [%s]" % node.gateway.id self.write_ensure_prefix(line, extra) else: if self.config.option.verbose: @@ -238,7 +238,7 @@ else: self.ensure_newline() if hasattr(rep, 'node'): - self._tw.write("%s " % rep.node.gateway.id) + self._tw.write("[%s] " % rep.node.gateway.id) self._tw.write(word, **markup) self._tw.write(" " + line) self.currentfspath = -2 Modified: pypy/trunk/py/plugin/pytest_tmpdir.py ============================================================================== --- pypy/trunk/py/plugin/pytest_tmpdir.py (original) +++ pypy/trunk/py/plugin/pytest_tmpdir.py Fri Dec 4 15:25:35 2009 @@ -18,4 +18,5 @@ path object. """ name = request.function.__name__ - return request.config.mktemp(name, numbered=True) + x = request.config.mktemp(name, numbered=True) + return x.realpath() From afa at codespeak.net Fri Dec 4 15:36:49 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 4 Dec 2009 15:36:49 +0100 (CET) Subject: [pypy-svn] r69895 - pypy/trunk/pypy/translator/c/gcc Message-ID: <20091204143649.B229416803B@codespeak.net> Author: afa Date: Fri Dec 4 15:36:47 2009 New Revision: 69895 Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Log: Fixes for trackgcroot on Windows: The oracle module makes more use of 2-bytes and 1-byte numbers. Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Fri Dec 4 15:36:47 2009 @@ -910,7 +910,9 @@ stop = True elif self.r_gcroot_marker_var.search(line): stop = True - elif line.startswith("\tmov\t%s," % (reg,)): + elif (line.startswith("\tmov\t%s," % (reg,)) or + line.startswith("\tmovsx\t%s," % (reg,)) or + line.startswith("\tmovzx\t%s," % (reg,))): # mov reg, stop = True elif line.startswith("\txor\t%s, %s" % (reg, reg)): @@ -920,7 +922,8 @@ imul = self.r_binaryinsn.match(line) imul_arg1 = imul.group("target") imul_arg2 = imul.group("source") - break + if imul_arg1 == reg or imul_arg2 == reg: + break # the register may not appear in other instructions elif reg in line: assert False, (line, lineno) From arigo at codespeak.net Fri Dec 4 15:57:19 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 4 Dec 2009 15:57:19 +0100 (CET) Subject: [pypy-svn] r69896 - in pypy/branch/virtual-forcing/pypy/rlib: . test Message-ID: <20091204145719.7F9B8168025@codespeak.net> Author: arigo Date: Fri Dec 4 15:57:18 2009 New Revision: 69896 Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py Log: Using this function may help during debugging. Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/jit.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/jit.py Fri Dec 4 15:57:18 2009 @@ -1,7 +1,7 @@ import py import sys from pypy.rpython.extregistry import ExtRegistryEntry -from pypy.rlib.objectmodel import CDefinedIntSymbolic +from pypy.rlib.objectmodel import CDefinedIntSymbolic, we_are_translated from pypy.rlib.unroll import unrolling_iterable def purefunction(func): @@ -105,10 +105,17 @@ return DirectVRef(x) virtual_ref.oopspec = 'virtual_ref(x)' +def virtual_ref_finish(x): + if not we_are_translated(): + x._forced = x._forced or -1 + class DirectVRef(object): + _forced = 0 def __init__(self, x): self._x = x def __call__(self): + assert self._forced >= 0, "too late to force the virtual_ref!" + self._forced = 1 return self._x def _freeze_(self): raise Exception("should not see a prebuilt virtual_ref") Modified: pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py Fri Dec 4 15:57:18 2009 @@ -1,4 +1,5 @@ -from pypy.rlib.jit import virtual_ref +import py +from pypy.rlib.jit import virtual_ref, virtual_ref_finish from pypy.rlib._jit_vref import SomeVRef from pypy.annotation import model as annmodel from pypy.annotation.annrpython import RPythonAnnotator @@ -22,6 +23,17 @@ vref = virtual_ref(x1) assert vref() is x1 +def test_finish(): + vref = virtual_ref(X()) + virtual_ref_finish(vref) + py.test.raises(AssertionError, "vref()") + # + x1 = X() + vref = virtual_ref(x1) + assert vref() is x1 + virtual_ref_finish(vref) + assert vref() is x1 + def test_annotate_1(): def f(): return virtual_ref(X()) From arigo at codespeak.net Fri Dec 4 15:58:02 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 4 Dec 2009 15:58:02 +0100 (CET) Subject: [pypy-svn] r69897 - in pypy/branch/virtual-forcing/pypy/jit/metainterp: . test Message-ID: <20091204145802.21A1D16802D@codespeak.net> Author: arigo Date: Fri Dec 4 15:58:01 2009 New Revision: 69897 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py pypy/branch/virtual-forcing/pypy/jit/metainterp/vref.py pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py Log: Small progress, clean-ups, unification of some concepts with virtualizable.py. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Fri Dec 4 15:58:01 2009 @@ -889,9 +889,16 @@ @arguments("box") def opimpl_virtual_ref(self, box): obj = box.getref_base() - res = vref.virtual_ref_during_tracing(self.metainterp, obj) + res = vref.virtual_ref_during_tracing(obj) + self.metainterp.all_virtual_refs.append(res) resbox = history.BoxPtr(res) self.metainterp.history.record(rop.VIRTUAL_REF, [box], resbox) + # Note: we create a JIT_VIRTUAL_REF here, in order to detect when + # the virtual escapes during tracing already. We record it as a + # VIRTUAL_REF operation, although the backend sees this operation + # as a no-op. The point is that the backend should not really see + # it in practice, as optimizeopt.py should either kill it or + # replace it with a NEW_WITH_VTABLE followed by SETFIELD_GCs. self.make_result_box(resbox) # ------------------------------ @@ -1203,13 +1210,13 @@ class MetaInterp(object): in_recursion = 0 _already_allocated_resume_virtuals = None - _force_token_mem = None def __init__(self, staticdata): self.staticdata = staticdata self.cpu = staticdata.cpu self.portal_trace_positions = [] self.greenkey_of_huge_function = None + self.all_virtual_refs = [] def is_blackholing(self): return self.history is None @@ -1766,7 +1773,12 @@ self.load_fields_from_virtualizable() def virtual_after_residual_call(self): - if self.is_blackholing() or not vref.was_forced(self): + if self.is_blackholing(): + return + for gcref in self.all_virtual_refs: + if vref.was_forced(gcref): + break + else: return # during tracing, more precisely during the CALL_MAY_FORCE, at least # one of the vrefs was read. If we continue to trace and make Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py Fri Dec 4 15:58:01 2009 @@ -1,5 +1,6 @@ import py from pypy.rlib.jit import JitDriver, dont_look_inside, virtual_ref +from pypy.rlib.objectmodel import compute_unique_id from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin @@ -59,7 +60,9 @@ exctx = ExCtx() # @dont_look_inside - def externalfn(): + def externalfn(n): + if n > 1000: + return compute_unique_id(exctx.topframeref()) return 1 # def f(n): @@ -67,12 +70,14 @@ myjitdriver.can_enter_jit(n=n) myjitdriver.jit_merge_point(n=n) xy = XY() + xy.next1 = XY() + xy.next2 = XY() exctx.topframeref = virtual_ref(xy) - n -= externalfn() + n -= externalfn(n) exctx.topframeref = None # self.meta_interp(f, [15]) - self.check_loops(new_with_vtable=0) + self.check_loops(new_with_vtable=1) # the vref, not the XYs def test_simple_force_always(self): py.test.skip("in-progress") Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py Fri Dec 4 15:58:01 2009 @@ -11,8 +11,8 @@ class VirtualizableInfo: - token_none = 0 - token_tracing = -1 + TOKEN_NONE = 0 + TOKEN_TRACING = -1 def __init__(self, warmrunnerdesc): self.warmrunnerdesc = warmrunnerdesc @@ -177,7 +177,7 @@ return rvirtualizable2.match_virtualizable_type(TYPE, self.VTYPEPTR) def reset_vable_token(self, virtualizable): - virtualizable.vable_token = self.token_none + virtualizable.vable_token = self.TOKEN_NONE def clear_vable_token(self, virtualizable): if virtualizable.vable_token: @@ -186,14 +186,14 @@ def tracing_before_residual_call(self, virtualizable): assert not virtualizable.vable_token - virtualizable.vable_token = self.token_tracing + virtualizable.vable_token = self.TOKEN_TRACING def tracing_after_residual_call(self, virtualizable): if virtualizable.vable_token: # not modified by the residual call; assert that it is still # set to 'tracing_vable_rti' and clear it. - assert virtualizable.vable_token == self.token_tracing - virtualizable.vable_token = self.token_none + assert virtualizable.vable_token == self.TOKEN_TRACING + virtualizable.vable_token = self.TOKEN_NONE return False else: # marker "modified during residual call" set. @@ -201,10 +201,10 @@ def force_now(self, virtualizable): token = virtualizable.vable_token - virtualizable.vable_token = self.token_none - if token == self.token_tracing: + virtualizable.vable_token = self.TOKEN_NONE + if token == self.TOKEN_TRACING: # The values in the virtualizable are always correct during - # tracing. We only need to reset vable_token to token_none + # tracing. We only need to reset vable_token to TOKEN_NONE # as a marker for the tracing, to tell it that this # virtualizable escapes. pass @@ -220,11 +220,11 @@ # The 'vable_token' field of a virtualizable is either 0, -1, or points # into the CPU stack to a particular field in the current frame. It is: # -# 1. 0 (token_none) if not in the JIT at all, except as described below. +# 1. 0 (TOKEN_NONE) if not in the JIT at all, except as described below. # # 2. equal to 0 when tracing is in progress; except: # -# 3. equal to -1 (token_tracing) during tracing when we do a residual call, +# 3. equal to -1 (TOKEN_TRACING) during tracing when we do a residual call, # calling random unknown other parts of the interpreter; it is # reset to 0 as soon as something occurs to the virtualizable. # Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/vref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/vref.py Fri Dec 4 15:58:01 2009 @@ -2,20 +2,29 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass -def replace_force_virtual_with_call(graphs, funcptr): +def replace_force_virtual_with_call(make_helper_func, graphs): # similar to rvirtualizable2.replace_force_virtualizable_with_call(). - # funcptr should be an ll function pointer with a signature - # OBJECTPTR -> OBJECTPTR. - c_funcptr = inputconst(lltype.typeOf(funcptr), funcptr) + c_funcptr = None count = 0 for graph in graphs: for block in graph.iterblocks(): for op in block.operations: if op.opname == 'jit_force_virtual': + # first compute c_funcptr, but only if there is any + # 'jit_force_virtual' around + if c_funcptr is None: + FUNC = lltype.FuncType([rclass.OBJECTPTR], + rclass.OBJECTPTR) + funcptr = make_helper_func( + lltype.Ptr(FUNC), + force_virtual_if_necessary) + c_funcptr = inputconst(lltype.typeOf(funcptr), funcptr) + # op.opname = 'direct_call' op.args = [c_funcptr, op.args[0]] count += 1 - log("replaced %d 'jit_force_virtual' with %r" % (count, funcptr)) + if c_funcptr is not None: + log("replaced %d 'jit_force_virtual' with %r" % (count, funcptr)) # ____________________________________________________________ @@ -23,39 +32,32 @@ # we make the low-level type of an RPython class directly JIT_VIRTUAL_REF = lltype.GcStruct('JitVirtualRef', ('super', rclass.OBJECT), - ('force_token', llmemory.Address), + ('virtual_token', lltype.Signed), ('forced', rclass.OBJECTPTR)) jit_virtual_ref_vtable = lltype.malloc(rclass.OBJECT_VTABLE, zero=True, flavor='raw') -class ForceTokenMem(object): - def __init__(self): - self.allocated = lltype.malloc(rffi.CArray(lltype.Signed), 1, - flavor='raw') - self.allocated[0] = 0 - - def __del__(self): - lltype.free(self.allocated, flavor='raw') +# The 'virtual_token' field has the same meaning as the 'vable_token' field +# of a virtualizable. It is equal to: +# * -1 (TOKEN_TRACING) when tracing; +# * addr in the CPU stack (set by FORCE_TOKEN) when running the assembler; +# * 0 (TOKEN_NONE) after the virtual is forced, if it is forced at all. +TOKEN_NONE = 0 +TOKEN_TRACING = -1 -def get_force_token(metainterp): - if not metainterp._force_token_mem: - metainterp._force_token_mem = ForceTokenMem() - return llmemory.cast_ptr_to_adr(metainterp._force_token_mem.allocated) - -def was_forced(metainterp): - if not metainterp._force_token_mem: - return False - return metainterp._force_token_mem.allocated[0] == -1 - -def virtual_ref_during_tracing(metainterp, real_object): +def virtual_ref_during_tracing(real_object): + assert real_object vref = lltype.malloc(JIT_VIRTUAL_REF) p = lltype.cast_pointer(rclass.OBJECTPTR, vref) p.typeptr = jit_virtual_ref_vtable - vref.force_token = get_force_token(metainterp) + vref.virtual_token = TOKEN_TRACING vref.forced = lltype.cast_opaque_ptr(rclass.OBJECTPTR, real_object) - assert vref.forced return lltype.cast_opaque_ptr(llmemory.GCREF, vref) +def was_forced(gcref): + vref = lltype.cast_opaque_ptr(lltype.Ptr(JIT_VIRTUAL_REF), gcref) + return vref.virtual_token != TOKEN_TRACING + # ____________________________________________________________ def force_virtual_if_necessary(inst): @@ -65,10 +67,8 @@ def force_virtual(inst): vref = lltype.cast_pointer(lltype.Ptr(JIT_VIRTUAL_REF), inst) - if vref.force_token: - if not vref.forced: - xxxx - vref.force_token.signed[0] = -1 - vref.force_token = llmemory.NULL + if not vref.forced: + xxxx + vref.virtual_token = TOKEN_NONE return vref.forced force_virtual._dont_inline_ = True Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py Fri Dec 4 15:58:01 2009 @@ -603,11 +603,8 @@ def rewrite_force_virtual(self): if self.cpu.ts.name != 'lltype': py.test.skip("rewrite_force_virtual: port it to ootype") - FUNC = lltype.FuncType([rclass.OBJECTPTR], rclass.OBJECTPTR) - funcptr = self.helper_func(lltype.Ptr(FUNC), - vref.force_virtual_if_necessary) all_graphs = self.translator.graphs - vref.replace_force_virtual_with_call(all_graphs, funcptr) + vref.replace_force_virtual_with_call(self.helper_func, all_graphs) def decode_hp_hint_args(op): From fijal at codespeak.net Fri Dec 4 16:07:46 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 4 Dec 2009 16:07:46 +0100 (CET) Subject: [pypy-svn] r69898 - pypy/branch/listcopyop/pypy/rpython/memory/gc Message-ID: <20091204150746.EB1BA168038@codespeak.net> Author: fijal Date: Fri Dec 4 16:07:46 2009 New Revision: 69898 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/base.py Log: A temporary move of function, to increase granularity of valgrind data Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gc/base.py Fri Dec 4 16:07:46 2009 @@ -155,6 +155,18 @@ def x_clone(self, clonedata): raise RuntimeError("no support for x_clone in the GC") + def trace_gcarray(self, obj, callback, arg): + # a performance shortcut for GcArray(gcptr) + length = (obj + llmemory.gcarrayofptr_lengthoffset).signed[0] + item = obj + llmemory.gcarrayofptr_itemsoffset + while length > 0: + if self.points_to_valid_gc_object(item): + callback(item, arg) + item += llmemory.gcarrayofptr_singleitemoffset + length -= 1 + trace_gcarray._dont_inline_ = True + trace_gcarray._annspecialcase_ = 'specialize:arg(2)' + def trace(self, obj, callback, arg): """Enumerate the locations inside the given obj that can contain GC pointers. For each such location, callback(pointer, arg) is @@ -165,14 +177,7 @@ if not self.has_gcptr(typeid): return if self.is_gcarrayofgcptr(typeid): - # a performance shortcut for GcArray(gcptr) - length = (obj + llmemory.gcarrayofptr_lengthoffset).signed[0] - item = obj + llmemory.gcarrayofptr_itemsoffset - while length > 0: - if self.points_to_valid_gc_object(item): - callback(item, arg) - item += llmemory.gcarrayofptr_singleitemoffset - length -= 1 + self.trace_gcarray(obj, callback, arg) return offsets = self.offsets_to_gc_pointers(typeid) i = 0 From arigo at codespeak.net Fri Dec 4 16:28:45 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 4 Dec 2009 16:28:45 +0100 (CET) Subject: [pypy-svn] r69899 - in pypy/branch/virtual-forcing/pypy: jit/metainterp jit/metainterp/test rpython/lltypesystem Message-ID: <20091204152845.7CD9B168038@codespeak.net> Author: arigo Date: Fri Dec 4 16:28:44 2009 New Revision: 69899 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/virtual-forcing/pypy/jit/metainterp/vref.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rclass.py Log: Move initialization of 'vtable.name' to a helper, and reuse it from jit tests. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py Fri Dec 4 16:28:44 2009 @@ -1,6 +1,6 @@ import py, random -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE @@ -38,12 +38,13 @@ type_system = 'lltype' def get_class_of_box(self, box): - from pypy.rpython.lltypesystem import rclass return box.getref(rclass.OBJECTPTR).typeptr node_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True) + node_vtable.name = rclass.alloc_array_name('node') node_vtable_adr = llmemory.cast_ptr_to_adr(node_vtable) node_vtable2 = lltype.malloc(OBJECT_VTABLE, immortal=True) + node_vtable2.name = rclass.alloc_array_name('node2') node_vtable_adr2 = llmemory.cast_ptr_to_adr(node_vtable2) cpu = runner.LLtypeCPU(None) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/vref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/vref.py Fri Dec 4 16:28:44 2009 @@ -36,6 +36,7 @@ ('forced', rclass.OBJECTPTR)) jit_virtual_ref_vtable = lltype.malloc(rclass.OBJECT_VTABLE, zero=True, flavor='raw') +jit_virtual_ref_vtable.name = rclass.alloc_array_name('jit_virtual_ref') # The 'virtual_token' field has the same meaning as the 'vable_token' field # of a virtualizable. It is equal to: Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rclass.py Fri Dec 4 16:28:44 2009 @@ -86,6 +86,13 @@ vtable = vtable.super return vtable +def alloc_array_name(name): + p = malloc(Array(Char), len(name)+1, immortal=True) + for i in range(len(name)): + p[i] = name[i] + p[len(name)] = '\x00' + return p + class ClassRepr(AbstractClassRepr): def __init__(self, rtyper, classdef): @@ -192,10 +199,7 @@ name = 'object' else: name = rsubcls.classdef.shortname - vtable.name = malloc(Array(Char), len(name)+1, immortal=True) - for i in range(len(name)): - vtable.name[i] = name[i] - vtable.name[len(name)] = '\x00' + vtable.name = alloc_array_name(name) if hasattr(rsubcls.classdef, 'my_instantiate_graph'): graph = rsubcls.classdef.my_instantiate_graph vtable.instantiate = self.rtyper.getcallable(graph) From afa at codespeak.net Fri Dec 4 16:52:27 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 4 Dec 2009 16:52:27 +0100 (CET) Subject: [pypy-svn] r69900 - in pypy/trunk/pypy: interpreter module/__builtin__ module/_demo module/_demo/test module/oracle Message-ID: <20091204155227.6154D16803C@codespeak.net> Author: afa Date: Fri Dec 4 16:52:26 2009 New Revision: 69900 Added: pypy/trunk/pypy/module/_demo/test/ (props changed) pypy/trunk/pypy/module/_demo/test/test_import.py (contents, props changed) Modified: pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/interpreter/module.py pypy/trunk/pypy/module/__builtin__/importing.py pypy/trunk/pypy/module/_demo/__init__.py pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_error.py Log: Change the way built-in modules are initialized and imported: - All built-in modules are inserted into space.builtin_modules when they are discovered, but not in sys.modules. - The end of space initialization calls setup_builtin_modules(), which calls mod.setup_after_space_initialization() for every module. At this stage, sys.modules contains the "sys" module, and other modules imported by mod.setup_after_space_initialization(). - The space and modules are frozen in this state, and translated. - When pypy-c starts, it calls mod.startup() on built-in modules already in sys.modules. - when importing another builtin module, pypy fetches it from space.builtin_modules, adds it to sys.modules, and calls its setup() method. space.getbuiltinmodule(name) is still the main interface to import-or-get a built-in module. This speeds up pypy startup, but more importantly, solves bootstraping issues where for example a higher-level module would "import posix" before it is fully initialized; when this happens, os.environ is empty! Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Fri Dec 4 16:52:26 2009 @@ -239,6 +239,8 @@ config = get_pypy_config(translating=False) self.config = config + self.builtin_modules = {} + # import extra modules for side-effects import pypy.interpreter.nestedscope # register *_DEREF bytecodes @@ -266,28 +268,34 @@ def startup(self): # To be called before using the space - # Initialize all builtin modules + # Initialize already imported builtin modules from pypy.interpreter.module import Module + w_modules = self.sys.get('modules') for w_modname in self.unpackiterable( self.sys.get('builtin_module_names')): + try: + w_mod = self.getitem(w_modules, w_modname) + except OperationError, e: + if e.match(self, self.w_KeyError): + continue + raise modname = self.str_w(w_modname) - mod = self.interpclass_w(self.getbuiltinmodule(modname)) - if isinstance(mod, Module): - import time + mod = self.interpclass_w(w_mod) + if isinstance(mod, Module) and not mod.startup_called: self.timer.start("startup " + modname) mod.startup(self) + mod.startup_called = True self.timer.stop("startup " + modname) + def finish(self): w_exitfunc = self.sys.getdictvalue(self, 'exitfunc') if w_exitfunc is not None: self.call_function(w_exitfunc) from pypy.interpreter.module import Module - for w_modname in self.unpackiterable( - self.sys.get('builtin_module_names')): - modname = self.str_w(w_modname) - mod = self.interpclass_w(self.getbuiltinmodule(modname)) - if isinstance(mod, Module): + for w_mod in self.builtin_modules.values(): + mod = self.interpclass_w(w_mod) + if isinstance(mod, Module) and mod.startup_called: mod.shutdown(self) if self.config.objspace.std.withdictmeasurement: from pypy.objspace.std.dictmultiobject import report @@ -339,14 +347,37 @@ w_name = self.wrap(name) w_mod = self.wrap(Module(self, w_name)) - w_modules = self.sys.get('modules') - self.setitem(w_modules, w_name, w_mod) + self.builtin_modules[name] = w_mod return name def getbuiltinmodule(self, name): w_name = self.wrap(name) w_modules = self.sys.get('modules') - return self.getitem(w_modules, w_name) + try: + return self.getitem(w_modules, w_name) + except OperationError, e: + if not e.match(self, self.w_KeyError): + raise + + # If the module is a builtin but not yet imported, + # retrieve it and initialize it + try: + w_mod = self.builtin_modules[name] + except KeyError: + raise e + else: + # Add the module to sys.modules + self.setitem(w_modules, w_name, w_mod) + + # And initialize it + from pypy.interpreter.module import Module + mod = self.interpclass_w(w_mod) + if isinstance(mod, Module) and not mod.startup_called: + self.timer.start("startup " + name) + mod.startup(self) + mod.startup_called = True + self.timer.stop("startup " + name) + return w_mod def get_builtinmodule_to_install(self): """NOT_RPYTHON""" @@ -397,16 +428,15 @@ w_name = self.wrap('sys') self.sys = Module(self, w_name) w_modules = self.sys.get('modules') - self.setitem(w_modules, w_name, self.wrap(self.sys)) + self.builtin_modules['sys'] = self.wrap(self.sys) - self.setitem(w_modules, w_name_exceptions, - self.wrap(self.exceptions_module)) + self.builtin_modules['exceptions'] = self.wrap(self.exceptions_module) from pypy.module.__builtin__ import Module w_name = self.wrap('__builtin__') self.builtin = Module(self, w_name) w_builtin = self.wrap(self.builtin) - self.setitem(w_modules, w_name, w_builtin) + self.builtin_modules['__builtin__'] = self.wrap(w_builtin) self.setitem(self.builtin.w_dict, self.wrap('__builtins__'), w_builtin) bootstrap_modules = ['sys', '__builtin__', 'exceptions'] @@ -480,12 +510,9 @@ def setup_builtin_modules(self): "NOT_RPYTHON: only for initializing the space." - from pypy.interpreter.module import Module - for w_modname in self.unpackiterable(self.sys.get('builtin_module_names')): - modname = self.unwrap(w_modname) - mod = self.getbuiltinmodule(modname) - if isinstance(mod, Module): - mod.setup_after_space_initialization() + self.getbuiltinmodule('sys') + for mod in self.builtin_modules.values(): + mod.setup_after_space_initialization() def initialize(self): """NOT_RPYTHON: Abstract method that should put some minimal Modified: pypy/trunk/pypy/interpreter/module.py ============================================================================== --- pypy/trunk/pypy/interpreter/module.py (original) +++ pypy/trunk/pypy/interpreter/module.py Fri Dec 4 16:52:26 2009 @@ -16,21 +16,22 @@ self.w_name = w_name if w_name is not None: space.setitem(w_dict, space.new_interned_str('__name__'), w_name) + self.startup_called = False def setup_after_space_initialization(self): """NOT_RPYTHON: to allow built-in modules to do some more setup after the space is fully initialized.""" def startup(self, space): - """This is called at runtime before the space gets uses to allow - the module to do initialization at runtime. + """This is called at runtime on import to allow the module to + do initialization when it is imported for the first time. """ def shutdown(self, space): """This is called when the space is shut down, just after - sys.exitfunc(). + sys.exitfunc(), if the module has been imported. """ - + def getdict(self): return self.w_dict Modified: pypy/trunk/pypy/module/__builtin__/importing.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/importing.py (original) +++ pypy/trunk/pypy/module/__builtin__/importing.py Fri Dec 4 16:52:26 2009 @@ -291,6 +291,9 @@ return w_mod + # check the builtin modules + if partname in space.builtin_modules: + return space.getbuiltinmodule(partname) if w_path is not None: for path in space.unpackiterable(w_path): Modified: pypy/trunk/pypy/module/_demo/__init__.py ============================================================================== --- pypy/trunk/pypy/module/_demo/__init__.py (original) +++ pypy/trunk/pypy/module/_demo/__init__.py Fri Dec 4 16:52:26 2009 @@ -12,3 +12,13 @@ appleveldefs = { 'DemoError' : 'app_demo.DemoError', } + + # Used in tests + demo_events = [] + def setup_after_space_initialization(self): + Module.demo_events.append('setup') + def startup(self, space): + Module.demo_events.append('startup') + def shutdown(self, space): + Module.demo_events.append('shutdown') + Added: pypy/trunk/pypy/module/_demo/test/test_import.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/_demo/test/test_import.py Fri Dec 4 16:52:26 2009 @@ -0,0 +1,28 @@ +from pypy.conftest import gettestobjspace +from pypy.module import _demo + +class TestImport: + + def setup_method(self, func): + _demo.Module.demo_events = [] + + def test_startup(self): + space = gettestobjspace(usemodules=('_demo',)) + w_modules = space.sys.get('modules') + + assert _demo.Module.demo_events == ['setup'] + assert not space.is_true(space.contains(w_modules, + space.wrap('_demo'))) + + # first import + w_import = space.builtin.get('__import__') + w_demo = space.call(w_import, + space.newlist([space.wrap('_demo')])) + assert _demo.Module.demo_events == ['setup', 'startup'] + + # reload the module, this should not call startup again + space.delitem(w_modules, + space.wrap('_demo')) + w_demo = space.call(w_import, + space.newlist([space.wrap('_demo')])) + assert _demo.Module.demo_events == ['setup', 'startup'] Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Fri Dec 4 16:52:26 2009 @@ -39,6 +39,7 @@ def startup(self, space): from pypy.module.oracle.interp_error import get state = get(space) + state.startup(space) (state.w_DecimalType, state.w_DateTimeType, state.w_DateType, state.w_TimedeltaType, ) = space.fixedview(space.appexec([], """(): Modified: pypy/trunk/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_error.py (original) +++ pypy/trunk/pypy/module/oracle/interp_error.py Fri Dec 4 16:52:26 2009 @@ -4,32 +4,47 @@ from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError + from pypy.module.oracle import roci, config +from pypy.rlib.unroll import unrolling_iterable + +exported_names = unrolling_iterable(""" + DatabaseError OperationalError InterfaceError ProgrammingError + NotSupportedError IntegrityError InternalError DataError + Variable Connection""".split()) class State: # XXX move to another file + def __init__(self, space): - w_module = space.getbuiltinmodule('cx_Oracle') - def get(name): - return space.getattr(w_module, space.wrap(name)) + "NOT_RPYTHON" + self.variableTypeByPythonType = {} + self.w_DecimalType = None + self.w_DateTimeType = None + self.w_DateType = None + self.w_TimedeltaType = None - self.w_DatabaseError = get('DatabaseError') - self.w_OperationalError = get('OperationalError') - self.w_InterfaceError = get('InterfaceError') - self.w_ProgrammingError = get('ProgrammingError') - self.w_NotSupportedError = get('NotSupportedError') - self.w_IntegrityError = get('IntegrityError') - self.w_InternalError = get('InternalError') - self.w_DataError = get('DataError') - self.w_Variable = get('Variable') - self.w_Connection = get('Connection') + for name in exported_names: + setattr(self, 'w_' + name, None) + + def startup(self, space): + w_module = space.getbuiltinmodule('cx_Oracle') + for name in exported_names: + setattr(self, 'w_' + name, space.getattr(w_module, space.wrap(name))) from pypy.module.oracle.interp_variable import all_variable_types - self.variableTypeByPythonType = {} for varType in all_variable_types: w_type = space.gettypeobject(varType.typedef) self.variableTypeByPythonType[w_type] = varType + (self.w_DecimalType, + self.w_DateTimeType, self.w_DateType, self.w_TimedeltaType, + ) = space.fixedview(space.appexec([], """(): + import decimal, datetime + return (decimal.Decimal, + datetime.datetime, datetime.date, datetime.timedelta) + """)) + def get(space): return space.fromcache(State) From afa at codespeak.net Fri Dec 4 18:39:43 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 4 Dec 2009 18:39:43 +0100 (CET) Subject: [pypy-svn] r69901 - pypy/trunk/pypy/module/_locale Message-ID: <20091204173943.80CDA16804C@codespeak.net> Author: afa Date: Fri Dec 4 18:39:41 2009 New Revision: 69901 Modified: pypy/trunk/pypy/module/_locale/__init__.py pypy/trunk/pypy/module/_locale/interp_locale.py Log: darwin is a Unix platform, unlike the older 'apple'. no need for a specific getdefaultlocale() Modified: pypy/trunk/pypy/module/_locale/__init__.py ============================================================================== --- pypy/trunk/pypy/module/_locale/__init__.py (original) +++ pypy/trunk/pypy/module/_locale/__init__.py Fri Dec 4 18:39:41 2009 @@ -12,7 +12,7 @@ 'strxfrm': 'interp_locale.strxfrm', } - if sys.platform in ('win32', 'darwin'): + if sys.platform == 'win32': interpleveldefs.update({ '_getdefaultlocale': 'interp_locale.getdefaultlocale', }) Modified: pypy/trunk/pypy/module/_locale/interp_locale.py ============================================================================== --- pypy/trunk/pypy/module/_locale/interp_locale.py (original) +++ pypy/trunk/pypy/module/_locale/interp_locale.py Fri Dec 4 18:39:41 2009 @@ -462,5 +462,3 @@ finally: lltype.free(buf_lang, flavor='raw') lltype.free(buf_country, flavor='raw') -elif sys.platform == 'darwin': - raise NotImplementedError() From afa at codespeak.net Fri Dec 4 18:54:18 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 4 Dec 2009 18:54:18 +0100 (CET) Subject: [pypy-svn] r69902 - in pypy/trunk/pypy: interpreter module/__builtin__ module/_demo module/_demo/test module/oracle Message-ID: <20091204175418.15DC316804D@codespeak.net> Author: afa Date: Fri Dec 4 18:54:17 2009 New Revision: 69902 Removed: pypy/trunk/pypy/module/_demo/test/ Modified: pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/interpreter/module.py pypy/trunk/pypy/module/__builtin__/importing.py pypy/trunk/pypy/module/_demo/__init__.py pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_error.py Log: Bah. revert r69900, which broke imports like from email.Parser import Parser Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Fri Dec 4 18:54:17 2009 @@ -239,8 +239,6 @@ config = get_pypy_config(translating=False) self.config = config - self.builtin_modules = {} - # import extra modules for side-effects import pypy.interpreter.nestedscope # register *_DEREF bytecodes @@ -268,34 +266,28 @@ def startup(self): # To be called before using the space - # Initialize already imported builtin modules + # Initialize all builtin modules from pypy.interpreter.module import Module - w_modules = self.sys.get('modules') for w_modname in self.unpackiterable( self.sys.get('builtin_module_names')): - try: - w_mod = self.getitem(w_modules, w_modname) - except OperationError, e: - if e.match(self, self.w_KeyError): - continue - raise modname = self.str_w(w_modname) - mod = self.interpclass_w(w_mod) - if isinstance(mod, Module) and not mod.startup_called: + mod = self.interpclass_w(self.getbuiltinmodule(modname)) + if isinstance(mod, Module): + import time self.timer.start("startup " + modname) mod.startup(self) - mod.startup_called = True self.timer.stop("startup " + modname) - def finish(self): w_exitfunc = self.sys.getdictvalue(self, 'exitfunc') if w_exitfunc is not None: self.call_function(w_exitfunc) from pypy.interpreter.module import Module - for w_mod in self.builtin_modules.values(): - mod = self.interpclass_w(w_mod) - if isinstance(mod, Module) and mod.startup_called: + for w_modname in self.unpackiterable( + self.sys.get('builtin_module_names')): + modname = self.str_w(w_modname) + mod = self.interpclass_w(self.getbuiltinmodule(modname)) + if isinstance(mod, Module): mod.shutdown(self) if self.config.objspace.std.withdictmeasurement: from pypy.objspace.std.dictmultiobject import report @@ -347,37 +339,14 @@ w_name = self.wrap(name) w_mod = self.wrap(Module(self, w_name)) - self.builtin_modules[name] = w_mod + w_modules = self.sys.get('modules') + self.setitem(w_modules, w_name, w_mod) return name def getbuiltinmodule(self, name): w_name = self.wrap(name) w_modules = self.sys.get('modules') - try: - return self.getitem(w_modules, w_name) - except OperationError, e: - if not e.match(self, self.w_KeyError): - raise - - # If the module is a builtin but not yet imported, - # retrieve it and initialize it - try: - w_mod = self.builtin_modules[name] - except KeyError: - raise e - else: - # Add the module to sys.modules - self.setitem(w_modules, w_name, w_mod) - - # And initialize it - from pypy.interpreter.module import Module - mod = self.interpclass_w(w_mod) - if isinstance(mod, Module) and not mod.startup_called: - self.timer.start("startup " + name) - mod.startup(self) - mod.startup_called = True - self.timer.stop("startup " + name) - return w_mod + return self.getitem(w_modules, w_name) def get_builtinmodule_to_install(self): """NOT_RPYTHON""" @@ -428,15 +397,16 @@ w_name = self.wrap('sys') self.sys = Module(self, w_name) w_modules = self.sys.get('modules') - self.builtin_modules['sys'] = self.wrap(self.sys) + self.setitem(w_modules, w_name, self.wrap(self.sys)) - self.builtin_modules['exceptions'] = self.wrap(self.exceptions_module) + self.setitem(w_modules, w_name_exceptions, + self.wrap(self.exceptions_module)) from pypy.module.__builtin__ import Module w_name = self.wrap('__builtin__') self.builtin = Module(self, w_name) w_builtin = self.wrap(self.builtin) - self.builtin_modules['__builtin__'] = self.wrap(w_builtin) + self.setitem(w_modules, w_name, w_builtin) self.setitem(self.builtin.w_dict, self.wrap('__builtins__'), w_builtin) bootstrap_modules = ['sys', '__builtin__', 'exceptions'] @@ -510,9 +480,12 @@ def setup_builtin_modules(self): "NOT_RPYTHON: only for initializing the space." - self.getbuiltinmodule('sys') - for mod in self.builtin_modules.values(): - mod.setup_after_space_initialization() + from pypy.interpreter.module import Module + for w_modname in self.unpackiterable(self.sys.get('builtin_module_names')): + modname = self.unwrap(w_modname) + mod = self.getbuiltinmodule(modname) + if isinstance(mod, Module): + mod.setup_after_space_initialization() def initialize(self): """NOT_RPYTHON: Abstract method that should put some minimal Modified: pypy/trunk/pypy/interpreter/module.py ============================================================================== --- pypy/trunk/pypy/interpreter/module.py (original) +++ pypy/trunk/pypy/interpreter/module.py Fri Dec 4 18:54:17 2009 @@ -16,22 +16,21 @@ self.w_name = w_name if w_name is not None: space.setitem(w_dict, space.new_interned_str('__name__'), w_name) - self.startup_called = False def setup_after_space_initialization(self): """NOT_RPYTHON: to allow built-in modules to do some more setup after the space is fully initialized.""" def startup(self, space): - """This is called at runtime on import to allow the module to - do initialization when it is imported for the first time. + """This is called at runtime before the space gets uses to allow + the module to do initialization at runtime. """ def shutdown(self, space): """This is called when the space is shut down, just after - sys.exitfunc(), if the module has been imported. + sys.exitfunc(). """ - + def getdict(self): return self.w_dict Modified: pypy/trunk/pypy/module/__builtin__/importing.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/importing.py (original) +++ pypy/trunk/pypy/module/__builtin__/importing.py Fri Dec 4 18:54:17 2009 @@ -291,9 +291,6 @@ return w_mod - # check the builtin modules - if partname in space.builtin_modules: - return space.getbuiltinmodule(partname) if w_path is not None: for path in space.unpackiterable(w_path): Modified: pypy/trunk/pypy/module/_demo/__init__.py ============================================================================== --- pypy/trunk/pypy/module/_demo/__init__.py (original) +++ pypy/trunk/pypy/module/_demo/__init__.py Fri Dec 4 18:54:17 2009 @@ -12,13 +12,3 @@ appleveldefs = { 'DemoError' : 'app_demo.DemoError', } - - # Used in tests - demo_events = [] - def setup_after_space_initialization(self): - Module.demo_events.append('setup') - def startup(self, space): - Module.demo_events.append('startup') - def shutdown(self, space): - Module.demo_events.append('shutdown') - Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Fri Dec 4 18:54:17 2009 @@ -39,7 +39,6 @@ def startup(self, space): from pypy.module.oracle.interp_error import get state = get(space) - state.startup(space) (state.w_DecimalType, state.w_DateTimeType, state.w_DateType, state.w_TimedeltaType, ) = space.fixedview(space.appexec([], """(): Modified: pypy/trunk/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_error.py (original) +++ pypy/trunk/pypy/module/oracle/interp_error.py Fri Dec 4 18:54:17 2009 @@ -4,47 +4,32 @@ from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError - from pypy.module.oracle import roci, config -from pypy.rlib.unroll import unrolling_iterable - -exported_names = unrolling_iterable(""" - DatabaseError OperationalError InterfaceError ProgrammingError - NotSupportedError IntegrityError InternalError DataError - Variable Connection""".split()) class State: # XXX move to another file - def __init__(self, space): - "NOT_RPYTHON" - self.variableTypeByPythonType = {} - self.w_DecimalType = None - self.w_DateTimeType = None - self.w_DateType = None - self.w_TimedeltaType = None - - for name in exported_names: - setattr(self, 'w_' + name, None) - - def startup(self, space): w_module = space.getbuiltinmodule('cx_Oracle') - for name in exported_names: - setattr(self, 'w_' + name, space.getattr(w_module, space.wrap(name))) + def get(name): + return space.getattr(w_module, space.wrap(name)) + + self.w_DatabaseError = get('DatabaseError') + self.w_OperationalError = get('OperationalError') + self.w_InterfaceError = get('InterfaceError') + self.w_ProgrammingError = get('ProgrammingError') + self.w_NotSupportedError = get('NotSupportedError') + self.w_IntegrityError = get('IntegrityError') + self.w_InternalError = get('InternalError') + self.w_DataError = get('DataError') + self.w_Variable = get('Variable') + self.w_Connection = get('Connection') from pypy.module.oracle.interp_variable import all_variable_types + self.variableTypeByPythonType = {} for varType in all_variable_types: w_type = space.gettypeobject(varType.typedef) self.variableTypeByPythonType[w_type] = varType - (self.w_DecimalType, - self.w_DateTimeType, self.w_DateType, self.w_TimedeltaType, - ) = space.fixedview(space.appexec([], """(): - import decimal, datetime - return (decimal.Decimal, - datetime.datetime, datetime.date, datetime.timedelta) - """)) - def get(space): return space.fromcache(State) From arigo at codespeak.net Fri Dec 4 20:06:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 4 Dec 2009 20:06:40 +0100 (CET) Subject: [pypy-svn] r69903 - pypy/branch/virtual-forcing/pypy/jit/metainterp/test Message-ID: <20091204190640.29AB2168051@codespeak.net> Author: arigo Date: Fri Dec 4 20:06:37 2009 New Revision: 69903 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Log: Add a failing test about virtualizables. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Fri Dec 4 20:06:37 2009 @@ -871,6 +871,43 @@ res = self.meta_interp(main, [123], policy=StopAtXPolicy(g)) assert res == main(123) + def test_external_write_sometimes(self): + jitdriver = JitDriver(greens = [], reds = ['frame'], + virtualizables = ['frame']) + + class Frame(object): + _virtualizable2_ = ['x', 'y'] + class SomewhereElse: + pass + somewhere_else = SomewhereElse() + + def g(): + somewhere_else.counter += 1 + if somewhere_else.counter == 70: + debug_print(lltype.Void, '-+-+-+-+- external write: 7000') + somewhere_else.top_frame.y = 7000 + result = 2 + else: + result = 1 + return result + + def f(n): + frame = Frame() + frame.x = n + frame.y = 10 + somewhere_else.counter = 0 + somewhere_else.top_frame = frame + while frame.x > 0: + jitdriver.can_enter_jit(frame=frame) + jitdriver.jit_merge_point(frame=frame) + frame.x -= g() + frame.y += 1 + return frame.y + + res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) + py.test.skip("in-progress") + assert res == f(123) + def test_promote_index_in_virtualizable_list(self): jitdriver = JitDriver(greens = [], reds = ['frame', 'n'], virtualizables = ['frame']) From arigo at codespeak.net Fri Dec 4 20:53:08 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 4 Dec 2009 20:53:08 +0100 (CET) Subject: [pypy-svn] r69904 - in pypy/branch/virtual-forcing/pypy/jit/metainterp: . test Message-ID: <20091204195308.F1E40168071@codespeak.net> Author: arigo Date: Fri Dec 4 20:53:07 2009 New Revision: 69904 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Log: Fix. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py Fri Dec 4 20:53:07 2009 @@ -237,7 +237,10 @@ from pypy.jit.metainterp.pyjitpl import MetaInterp metainterp = MetaInterp(metainterp_sd) token = metainterp_sd.cpu.get_latest_force_token() - metainterp._already_allocated_resume_virtuals = self.fetch_data(token) + data = self.fetch_data(token) + if data is None: + data = [] + metainterp._already_allocated_resume_virtuals = data self.counter = -2 # never compile return metainterp.handle_guard_failure(self) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Fri Dec 4 20:53:07 2009 @@ -1821,6 +1821,11 @@ virtualizable_boxes = resume.rebuild_from_resumedata(self, newboxes, resumedescr, expect_virtualizable) if expect_virtualizable: self.virtualizable_boxes = virtualizable_boxes + if self._already_allocated_resume_virtuals is not None: + # resuming from a ResumeGuardForcedDescr: load the new values + # currently stored on the virtualizable fields + self.load_fields_from_virtualizable() + return # just jumped away from assembler (case 4 in the comment in # virtualizable.py) into tracing (case 2); check that vable_rti # is and stays NULL. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Fri Dec 4 20:53:07 2009 @@ -905,7 +905,6 @@ return frame.y res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) - py.test.skip("in-progress") assert res == f(123) def test_promote_index_in_virtualizable_list(self): From arigo at codespeak.net Fri Dec 4 20:54:50 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 4 Dec 2009 20:54:50 +0100 (CET) Subject: [pypy-svn] r69905 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20091204195450.E8678168073@codespeak.net> Author: arigo Date: Fri Dec 4 20:54:50 2009 New Revision: 69905 Modified: pypy/trunk/pypy/jit/metainterp/test/test_float.py Log: For completion, a test about float_neg in addition to float_abs. Modified: pypy/trunk/pypy/jit/metainterp/test/test_float.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_float.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_float.py Fri Dec 4 20:54:50 2009 @@ -27,6 +27,15 @@ res = self.interp_operations(f, [x]) assert res == x + def test_neg(self): + def f(a): + return -a + res = self.interp_operations(f, [-5.25]) + assert res == 5.25 + x = 281474976710656.31 + res = self.interp_operations(f, [x]) + assert res == -x + class TestOOtype(FloatTests, OOJitMixin): pass From arigo at codespeak.net Fri Dec 4 20:55:19 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 4 Dec 2009 20:55:19 +0100 (CET) Subject: [pypy-svn] r69906 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091204195519.89C83168075@codespeak.net> Author: arigo Date: Fri Dec 4 20:55:18 2009 New Revision: 69906 Modified: pypy/trunk/pypy/jit/metainterp/compile.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_virtualizable.py Log: Merge r69903 and r69904 from branch/virtual-forcing: a test and fix for virtualizables. Modified: pypy/trunk/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/compile.py (original) +++ pypy/trunk/pypy/jit/metainterp/compile.py Fri Dec 4 20:55:18 2009 @@ -237,7 +237,10 @@ from pypy.jit.metainterp.pyjitpl import MetaInterp metainterp = MetaInterp(metainterp_sd) token = metainterp_sd.cpu.get_latest_force_token() - metainterp._already_allocated_resume_virtuals = self.fetch_data(token) + data = self.fetch_data(token) + if data is None: + data = [] + metainterp._already_allocated_resume_virtuals = data self.counter = -2 # never compile return metainterp.handle_guard_failure(self) Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Fri Dec 4 20:55:18 2009 @@ -1790,6 +1790,11 @@ virtualizable_boxes = resume.rebuild_from_resumedata(self, newboxes, resumedescr, expect_virtualizable) if expect_virtualizable: self.virtualizable_boxes = virtualizable_boxes + if self._already_allocated_resume_virtuals is not None: + # resuming from a ResumeGuardForcedDescr: load the new values + # currently stored on the virtualizable fields + self.load_fields_from_virtualizable() + return # just jumped away from assembler (case 4 in the comment in # virtualizable.py) into tracing (case 2); check that vable_rti # is and stays NULL. Modified: pypy/trunk/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_virtualizable.py Fri Dec 4 20:55:18 2009 @@ -871,6 +871,42 @@ res = self.meta_interp(main, [123], policy=StopAtXPolicy(g)) assert res == main(123) + def test_external_write_sometimes(self): + jitdriver = JitDriver(greens = [], reds = ['frame'], + virtualizables = ['frame']) + + class Frame(object): + _virtualizable2_ = ['x', 'y'] + class SomewhereElse: + pass + somewhere_else = SomewhereElse() + + def g(): + somewhere_else.counter += 1 + if somewhere_else.counter == 70: + debug_print(lltype.Void, '-+-+-+-+- external write: 7000') + somewhere_else.top_frame.y = 7000 + result = 2 + else: + result = 1 + return result + + def f(n): + frame = Frame() + frame.x = n + frame.y = 10 + somewhere_else.counter = 0 + somewhere_else.top_frame = frame + while frame.x > 0: + jitdriver.can_enter_jit(frame=frame) + jitdriver.jit_merge_point(frame=frame) + frame.x -= g() + frame.y += 1 + return frame.y + + res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) + assert res == f(123) + def test_promote_index_in_virtualizable_list(self): jitdriver = JitDriver(greens = [], reds = ['frame', 'n'], virtualizables = ['frame']) From arigo at codespeak.net Fri Dec 4 21:58:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 4 Dec 2009 21:58:42 +0100 (CET) Subject: [pypy-svn] r69908 - in pypy/branch/virtual-forcing/pypy/jit: backend metainterp metainterp/test Message-ID: <20091204205842.B6919168076@codespeak.net> Author: arigo Date: Fri Dec 4 21:58:40 2009 New Revision: 69908 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/model.py pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/vref.py Log: Use the phase of optimizeopt.py to replace the VIRTUAL_REF operation with a real structure of type vref.JIT_VIRTUAL_REF. Small other fixes left and right. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/model.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/model.py Fri Dec 4 21:58:40 2009 @@ -219,10 +219,6 @@ def do_cast_ptr_to_int(self, ptrbox): raise NotImplementedError - def do_force_token(self): - # this should not be implemented at all by the backends - raise NotImplementedError - def do_call_may_force(self, args, calldescr): return self.do_call(args, calldescr) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py Fri Dec 4 21:58:40 2009 @@ -103,9 +103,6 @@ def do_same_as(cpu, box1): return box1 -def do_virtual_ref(cpu, box1): - return box1.clonebox() - def do_oois(cpu, box1, box2): tp = box1.type assert tp == box2.type @@ -223,6 +220,12 @@ # ____________________________________________________________ +def do_force_token(cpu): + raise NotImplementedError + +def do_virtual_ref(cpu, box1): + raise NotImplementedError + def do_debug_merge_point(cpu, box1): from pypy.jit.metainterp.warmspot import get_stats loc = box1._get_str() Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py Fri Dec 4 21:58:40 2009 @@ -736,6 +736,26 @@ def optimize_OOIS(self, op): self._optimize_oois_ooisnot(op, False) + def optimize_VIRTUAL_REF(self, op): + value = self.getvalue(op.args[0]) + if not value.is_virtual(): # virtual_ref(non-virtual) gives bad + raise compile.GiveUp # results, so don't bother compiling it + # + # get some constants (these calls are all 'memo') + from pypy.jit.metainterp import vref + c_cls = vref.get_jit_virtual_ref_const_class(self.cpu) + descr_virtual_token = vref.get_descr_virtual_token(self.cpu) + descr_forced = vref.get_descr_forced(self.cpu) + # + # Replace the VIRTUAL_REF operation with a virtual structure of type + # 'vref.JIT_VIRTUAL_REF'. The virtual structure may be forced soon, + # but the point is that doing so does not force the original structure. + op = ResOperation(rop.NEW_WITH_VTABLE, [c_cls], op.result) + vrefvalue = self.make_virtual(c_cls, op.result, op) + tokenbox = BoxInt() + self.emit_operation(ResOperation(rop.FORCE_TOKEN, [], tokenbox)) + vrefvalue.setfield(descr_virtual_token, self.getvalue(tokenbox)) + def optimize_GETFIELD_GC(self, op): value = self.getvalue(op.args[0]) if value.is_virtual(): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Fri Dec 4 21:58:40 2009 @@ -1775,8 +1775,8 @@ def virtual_after_residual_call(self): if self.is_blackholing(): return - for gcref in self.all_virtual_refs: - if vref.was_forced(gcref): + for vr in self.all_virtual_refs: + if vref.was_forced(vr): break else: return Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py Fri Dec 4 21:58:40 2009 @@ -99,6 +99,12 @@ nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([], [])) writeadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([adescr], [])) writearraydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([adescr], [arraydescr])) + mayforcevirtdescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo([], [], forces_virtual_or_virtualizable=True)) + + from pypy.jit.metainterp.vref import jit_virtual_ref_vtable + from pypy.jit.metainterp.vref import JIT_VIRTUAL_REF + virtualtokendescr = cpu.fielddescrof(JIT_VIRTUAL_REF, 'virtual_token') cpu.class_sizes = {cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), cpu.cast_adr_to_int(node_vtable_adr2): cpu.sizeof(NODE2), Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py Fri Dec 4 21:58:40 2009 @@ -2031,6 +2031,43 @@ """ self.optimize_loop(ops, 'Not, Not, Not', expected) + def test_vref_nonvirtual(self): + ops = """ + [p1] + p2 = virtual_ref(p1) + jump(p1) + """ + py.test.raises(compile.GiveUp, self.optimize_loop, ops, 'Not', ops) + + def test_vref_virtual_1(self): + ops = """ + [p0, i1] + # + p1 = new_with_vtable(ConstClass(node_vtable)) + p1b = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1b, 252, descr=valuedescr) + setfield_gc(p1, p1b, descr=nextdescr) + # + p2 = virtual_ref(p1) + setfield_gc(p0, p2, descr=nextdescr) + call_may_force(i1, descr=mayforcevirtdescr) + guard_not_forced() [i1] + setfield_gc(p0, NULL, descr=nextdescr) + jump(p0, i1) + """ + expected = """ + [p0, i1] + i3 = force_token() + p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p0, p2, descr=nextdescr) + call_may_force(i1, descr=mayforcevirtdescr) + guard_not_forced() [i1] + setfield_gc(p0, NULL, descr=nextdescr) + jump(p0, i1) + """ + self.optimize_loop(ops, 'Not, Not', expected) + class TestOOtype(BaseTestOptimizeOpt, OOtypeMixin): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_pyjitpl.py Fri Dec 4 21:58:40 2009 @@ -33,7 +33,8 @@ def test_simple_opimpl_exist(): rop = resoperation.rop for opnum, opname in resoperation.opname.items(): - if opnum in (rop.SAME_AS, rop.CALL_PURE, rop.OOSEND_PURE): + if opnum in (rop.SAME_AS, rop.CALL_PURE, rop.OOSEND_PURE, + rop.FORCE_TOKEN): continue if rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST: assert hasattr(pyjitpl.MIFrame, 'opimpl_' + opname.lower()), opname Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/vref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/vref.py Fri Dec 4 21:58:40 2009 @@ -1,5 +1,7 @@ from pypy.rpython.rmodel import inputconst, log from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass +from pypy.rlib.objectmodel import specialize +from pypy.jit.metainterp import history def replace_force_virtual_with_call(make_helper_func, graphs): @@ -46,6 +48,19 @@ TOKEN_NONE = 0 TOKEN_TRACING = -1 + at specialize.memo() +def get_jit_virtual_ref_const_class(cpu): + adr = llmemory.cast_ptr_to_adr(jit_virtual_ref_vtable) + return history.ConstAddr(adr, cpu) + + at specialize.memo() +def get_descr_virtual_token(cpu): + return cpu.fielddescrof(JIT_VIRTUAL_REF, 'virtual_token') + + at specialize.memo() +def get_descr_forced(cpu): + return cpu.fielddescrof(JIT_VIRTUAL_REF, 'forced') + def virtual_ref_during_tracing(real_object): assert real_object vref = lltype.malloc(JIT_VIRTUAL_REF) From arigo at codespeak.net Fri Dec 4 22:57:07 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 4 Dec 2009 22:57:07 +0100 (CET) Subject: [pypy-svn] r69909 - in pypy/branch/virtual-forcing/pypy/jit/metainterp: . test Message-ID: <20091204215707.EA8D4168078@codespeak.net> Author: arigo Date: Fri Dec 4 22:57:05 2009 New Revision: 69909 Added: pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py - copied unchanged from r69908, pypy/branch/virtual-forcing/pypy/jit/metainterp/vref.py Removed: pypy/branch/virtual-forcing/pypy/jit/metainterp/vref.py Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py Log: * rename vref.py to virtualref.py, to avoid confusion with local variables that are also called vref. * support virtual_ref_finish in the jit; its exact use is kept purposefully obscure to confuse readers, but see next checkin. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py Fri Dec 4 22:57:05 2009 @@ -1279,8 +1279,8 @@ return self._do_builtin_call(op, oopspec_name, args) def _do_builtin_call(self, op, oopspec_name, args): - if oopspec_name == 'virtual_ref': - self.handle_virtual_ref_call(op, args) + if oopspec_name.startswith('virtual_ref'): + self.handle_virtual_ref_call(op, oopspec_name, args) return argtypes = [v.concretetype for v in args] resulttype = op.result.concretetype @@ -1302,8 +1302,8 @@ self.emit_varargs([c_func] + non_void_args) self.register_var(op.result) - def handle_virtual_ref_call(self, op, args): - self.emit('virtual_ref') + def handle_virtual_ref_call(self, op, oopspec_name, args): + self.emit(oopspec_name) # 'virtual_ref' or 'virtual_ref_finish' self.emit(self.var_position(args[0])) self.register_var(op.result) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py Fri Dec 4 22:57:05 2009 @@ -742,10 +742,10 @@ raise compile.GiveUp # results, so don't bother compiling it # # get some constants (these calls are all 'memo') - from pypy.jit.metainterp import vref - c_cls = vref.get_jit_virtual_ref_const_class(self.cpu) - descr_virtual_token = vref.get_descr_virtual_token(self.cpu) - descr_forced = vref.get_descr_forced(self.cpu) + from pypy.jit.metainterp import virtualref + c_cls = virtualref.get_jit_virtual_ref_const_class(self.cpu) + descr_virtual_token = virtualref.get_descr_virtual_token(self.cpu) + descr_forced = virtualref.get_descr_forced(self.cpu) # # Replace the VIRTUAL_REF operation with a virtual structure of type # 'vref.JIT_VIRTUAL_REF'. The virtual structure may be forced soon, Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Fri Dec 4 22:57:05 2009 @@ -5,7 +5,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_start, debug_stop, debug_print -from pypy.jit.metainterp import history, compile, resume, vref +from pypy.jit.metainterp import history, compile, resume, virtualref from pypy.jit.metainterp.history import Const, ConstInt, Box from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp import codewriter, executor @@ -889,11 +889,16 @@ @arguments("box") def opimpl_virtual_ref(self, box): obj = box.getref_base() - res = vref.virtual_ref_during_tracing(obj) - self.metainterp.all_virtual_refs.append(res) - resbox = history.BoxPtr(res) + vref = virtualref.virtual_ref_during_tracing(obj) + # + if self.metainterp.all_virtual_refs is None: + self.metainterp.all_virtual_refs = [] + self.metainterp.all_virtual_refs.append(vref) + # + resbox = history.BoxPtr(vref) self.metainterp.history.record(rop.VIRTUAL_REF, [box], resbox) - # Note: we create a JIT_VIRTUAL_REF here, in order to detect when + # Note: we allocate a JIT_VIRTUAL_REF here + # (in virtual_ref_during_tracing()), in order to detect when # the virtual escapes during tracing already. We record it as a # VIRTUAL_REF operation, although the backend sees this operation # as a no-op. The point is that the backend should not really see @@ -901,6 +906,14 @@ # replace it with a NEW_WITH_VTABLE followed by SETFIELD_GCs. self.make_result_box(resbox) + @arguments("box") + def opimpl_virtual_ref_finish(self, box): + # virtual_ref_finish() assumes that we have a stack-like, last-in + # first-out order. + if not self.metainterp.is_blackholing(): + lastitem = self.metainterp.all_virtual_refs.pop() + assert box.getref_base() == lastitem + # ------------------------------ def setup_call(self, argboxes): @@ -1209,6 +1222,7 @@ class MetaInterp(object): in_recursion = 0 + all_virtual_refs = None _already_allocated_resume_virtuals = None def __init__(self, staticdata): @@ -1216,7 +1230,6 @@ self.cpu = staticdata.cpu self.portal_trace_positions = [] self.greenkey_of_huge_function = None - self.all_virtual_refs = [] def is_blackholing(self): return self.history is None @@ -1773,10 +1786,10 @@ self.load_fields_from_virtualizable() def virtual_after_residual_call(self): - if self.is_blackholing(): + if self.is_blackholing() or self.all_virtual_refs is None: return for vr in self.all_virtual_refs: - if vref.was_forced(vr): + if virtualref.was_forced(vr): break else: return Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py Fri Dec 4 22:57:05 2009 @@ -102,8 +102,8 @@ mayforcevirtdescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([], [], forces_virtual_or_virtualizable=True)) - from pypy.jit.metainterp.vref import jit_virtual_ref_vtable - from pypy.jit.metainterp.vref import JIT_VIRTUAL_REF + from pypy.jit.metainterp.virtualref import jit_virtual_ref_vtable + from pypy.jit.metainterp.virtualref import JIT_VIRTUAL_REF virtualtokendescr = cpu.fielddescrof(JIT_VIRTUAL_REF, 'virtual_token') cpu.class_sizes = {cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py Fri Dec 4 22:57:05 2009 @@ -2068,6 +2068,42 @@ """ self.optimize_loop(ops, 'Not, Not', expected) + def test_vref_virtual_2(self): + self.make_fail_descr() + ops = """ + [p0, i1] + # + p1 = new_with_vtable(ConstClass(node_vtable)) + p1b = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1b, i1, descr=valuedescr) + setfield_gc(p1, p1b, descr=nextdescr) + # + p2 = virtual_ref(p1) + setfield_gc(p0, p2, descr=nextdescr) + call_may_force(i1, descr=mayforcevirtdescr) + guard_not_forced(descr=fdescr) [p1] + setfield_gc(p0, NULL, descr=nextdescr) + jump(p0, i1) + """ + expected = """ + [p0, i1] + i3 = force_token() + p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p0, p2, descr=nextdescr) + call_may_force(i1, descr=mayforcevirtdescr) + guard_not_forced(descr=fdescr) [i1] + setfield_gc(p0, NULL, descr=nextdescr) + jump(p0, i1) + """ + # the point of this test is that 'i1' should show up in the fail_args + # of 'guard_not_forced', because it was stored in the virtual 'p1b'. + self.optimize_loop(ops, 'Not, Not', expected) + self.check_expanded_fail_descr('''p1 + where p1 is a node_vtable, nextdescr=p1b + where p1b is a node_vtable, valuedescr=i1 + ''') + class TestOOtype(BaseTestOptimizeOpt, OOtypeMixin): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py Fri Dec 4 22:57:05 2009 @@ -1,5 +1,6 @@ import py -from pypy.rlib.jit import JitDriver, dont_look_inside, virtual_ref +from pypy.rlib.jit import JitDriver, dont_look_inside +from pypy.rlib.jit import virtual_ref, virtual_ref_finish from pypy.rlib.objectmodel import compute_unique_id from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin @@ -14,8 +15,9 @@ exctx = ExCtx() # def f(): - exctx.topframeref = virtual_ref(X()) + exctx.topframeref = vref = virtual_ref(X()) exctx.topframeref = None + virtual_ref_finish(vref) return 1 # self.interp_operations(f, []) @@ -43,6 +45,8 @@ x.n = n + 123 exctx.topframeref = virtual_ref(x) total += force_me() - 100 + exctx.topframeref = None + virtual_ref_finish(x) return total # res = self.meta_interp(f, [-4]) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py Fri Dec 4 22:57:05 2009 @@ -17,7 +17,7 @@ from pypy.translator.unsimplify import call_final_function from pypy.jit.metainterp import codewriter -from pypy.jit.metainterp import support, history, pyjitpl, gc, vref +from pypy.jit.metainterp import support, history, pyjitpl, gc, virtualref from pypy.jit.metainterp.pyjitpl import MetaInterpStaticData, MetaInterp from pypy.jit.metainterp.policy import JitPolicy from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper @@ -604,7 +604,8 @@ if self.cpu.ts.name != 'lltype': py.test.skip("rewrite_force_virtual: port it to ootype") all_graphs = self.translator.graphs - vref.replace_force_virtual_with_call(self.helper_func, all_graphs) + virtualref.replace_force_virtual_with_call(self.helper_func, + all_graphs) def decode_hp_hint_args(op): From fijal at codespeak.net Sat Dec 5 12:58:24 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 5 Dec 2009 12:58:24 +0100 (CET) Subject: [pypy-svn] r69912 - in pypy/branch/listcopyop/pypy/rpython/memory: . gc Message-ID: <20091205115824.E8D5C168011@codespeak.net> Author: fijal Date: Sat Dec 5 12:58:24 2009 New Revision: 69912 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/base.py pypy/branch/listcopyop/pypy/rpython/memory/gctypelayout.py Log: Revert up to 69890 - these were two irrelevant changes Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gc/base.py Sat Dec 5 12:58:24 2009 @@ -46,7 +46,6 @@ DEBUG = False def set_query_functions(self, is_varsize, has_gcptr_in_varsize, - has_gcptr, is_gcarrayofgcptr, getfinalizer, offsets_to_gc_pointers, @@ -59,7 +58,6 @@ self.getfinalizer = getfinalizer self.is_varsize = is_varsize self.has_gcptr_in_varsize = has_gcptr_in_varsize - self.has_gcptr = has_gcptr self.is_gcarrayofgcptr = is_gcarrayofgcptr self.offsets_to_gc_pointers = offsets_to_gc_pointers self.fixed_size = fixed_size @@ -155,18 +153,6 @@ def x_clone(self, clonedata): raise RuntimeError("no support for x_clone in the GC") - def trace_gcarray(self, obj, callback, arg): - # a performance shortcut for GcArray(gcptr) - length = (obj + llmemory.gcarrayofptr_lengthoffset).signed[0] - item = obj + llmemory.gcarrayofptr_itemsoffset - while length > 0: - if self.points_to_valid_gc_object(item): - callback(item, arg) - item += llmemory.gcarrayofptr_singleitemoffset - length -= 1 - trace_gcarray._dont_inline_ = True - trace_gcarray._annspecialcase_ = 'specialize:arg(2)' - def trace(self, obj, callback, arg): """Enumerate the locations inside the given obj that can contain GC pointers. For each such location, callback(pointer, arg) is @@ -174,10 +160,15 @@ Typically, 'callback' is a bound method and 'arg' can be None. """ typeid = self.get_type_id(obj) - if not self.has_gcptr(typeid): - return if self.is_gcarrayofgcptr(typeid): - self.trace_gcarray(obj, callback, arg) + # a performance shortcut for GcArray(gcptr) + length = (obj + llmemory.gcarrayofptr_lengthoffset).signed[0] + item = obj + llmemory.gcarrayofptr_itemsoffset + while length > 0: + if self.points_to_valid_gc_object(item): + callback(item, arg) + item += llmemory.gcarrayofptr_singleitemoffset + length -= 1 return offsets = self.offsets_to_gc_pointers(typeid) i = 0 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctypelayout.py Sat Dec 5 12:58:24 2009 @@ -62,10 +62,6 @@ infobits = self.get(typeid).infobits return (infobits & T_HAS_GCPTR_IN_VARSIZE) != 0 - def q_has_gcptrs(self, typeid): - infobits = self.get(typeid).infobits - return (infobits & T_HAS_GCPTR) != 0 - def q_is_gcarrayofgcptr(self, typeid): infobits = self.get(typeid).infobits return (infobits & T_IS_GCARRAY_OF_GCPTR) != 0 @@ -105,7 +101,6 @@ gc.set_query_functions( self.q_is_varsize, self.q_has_gcptr_in_varsize, - self.q_has_gcptrs, self.q_is_gcarrayofgcptr, self.q_finalizer, self.q_offsets_to_gc_pointers, @@ -122,9 +117,8 @@ T_MEMBER_INDEX = 0xffff T_IS_VARSIZE = 0x10000 T_HAS_GCPTR_IN_VARSIZE = 0x20000 -T_HAS_GCPTR = 0x40000 -T_IS_GCARRAY_OF_GCPTR = 0x80000 -T_IS_WEAKREF = 0x100000 +T_IS_GCARRAY_OF_GCPTR = 0x40000 +T_IS_WEAKREF = 0x80000 def _check_typeid(typeid): ll_assert(llop.is_group_member_nonzero(lltype.Bool, typeid), @@ -137,7 +131,6 @@ infobits = index info.ofstoptrs = builder.offsets2table(offsets, TYPE) info.finalizer = builder.make_finalizer_funcptr_for_type(TYPE) - arroffsets = None if not TYPE._is_varsize(): info.fixedsize = llarena.round_up_for_allocation( llmemory.sizeof(TYPE), builder.GCClass.object_minimal_size) @@ -164,18 +157,15 @@ varinfo.ofstovar = llmemory.itemoffsetof(TYPE, 0) assert isinstance(ARRAY, lltype.Array) if ARRAY.OF != lltype.Void: - arroffsets = offsets_to_gc_pointers(ARRAY.OF) + offsets = offsets_to_gc_pointers(ARRAY.OF) else: - arroffsets = () - if len(arroffsets) > 0: + offsets = () + if len(offsets) > 0: infobits |= T_HAS_GCPTR_IN_VARSIZE - varinfo.varofstoptrs = builder.offsets2table(arroffsets, ARRAY.OF) + varinfo.varofstoptrs = builder.offsets2table(offsets, ARRAY.OF) varinfo.varitemsize = llmemory.sizeof(ARRAY.OF) if TYPE == WEAKREF: infobits |= T_IS_WEAKREF - if offsets or arroffsets: - infobits |= T_HAS_GCPTR - info.infobits = infobits # ____________________________________________________________ From pedronis at codespeak.net Sun Dec 6 02:01:38 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 6 Dec 2009 02:01:38 +0100 (CET) Subject: [pypy-svn] r69916 - pypy/branch/esp-params/pypy/jit/backend/x86 Message-ID: <20091206010138.0F56C168025@codespeak.net> Author: pedronis Date: Sun Dec 6 02:01:36 2009 New Revision: 69916 Modified: pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py Log: factor out the common logic in a helper, no esp adjusting for calls anymore Modified: pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/esp-params/pypy/jit/backend/x86/assembler.py Sun Dec 6 02:01:36 2009 @@ -391,14 +391,34 @@ ## self.mc.PUSH(imm(0)) --- or just use a single SUB(esp, imm) ## return extra_on_stack - def call(self, addr, args, res): - nargs = len(args) - extra_on_stack = nargs #self.align_stack_for_call(nargs) - for i in range(nargs-1, -1, -1): - self.mc.PUSH(args[i]) - self.mc.CALL(rel32(addr)) + def _emit_call(self, x, arglocs, start=0, tmp=eax): + p = 0 + n = len(arglocs) + for i in range(start, n): + loc = arglocs[i] + if isinstance(loc, REG): + if isinstance(loc, XMMREG): + self.mc.MOVSD(mem64(esp, p), loc) + else: + self.mc.MOV(mem(esp, p), loc) + p += round_up_to_4(loc.width) + p = 0 + for i in range(start, n): + loc = arglocs[i] + if not isinstance(loc, REG): + if isinstance(loc, MODRM64): + self.mc.MOVSD(xmm0, loc) + self.mc.MOVSD(mem64(esp, p), xmm0) + else: + self.mc.MOV(tmp, loc) + self.mc.MOV(mem(esp, p), tmp) + p += round_up_to_4(loc.width) + self._regalloc.reserve_param(p//WORD) + self.mc.CALL(x) self.mark_gc_roots() - self.mc.ADD(esp, imm(extra_on_stack * WORD)) + + def call(self, addr, args, res): + self._emit_call(rel32(addr), args) assert res is eax genop_int_neg = _unaryop("NEG") @@ -1074,14 +1094,7 @@ sizeloc = arglocs[0] assert isinstance(sizeloc, IMM32) size = sizeloc.value - nargs = len(op.args)-1 - extra_on_stack = 0 - for arg in range(2, nargs + 2): - extra_on_stack += round_up_to_4(arglocs[arg].width) - self._regalloc.reserve_param(extra_on_stack//4) - - #extra_on_stack = self.align_stack_for_call(extra_on_stack) if isinstance(op.args[0], Const): x = rel32(op.args[0].getint()) else: @@ -1090,28 +1103,9 @@ tmp = ecx else: tmp = eax - p = 0 - for i in range(2, nargs + 2): - loc = arglocs[i] - if isinstance(loc, REG): - if isinstance(loc, XMMREG): - self.mc.MOVSD(mem64(esp, p), loc) - else: - self.mc.MOV(mem(esp, p), loc) - p += round_up_to_4(loc.width) - p = 0 - for i in range(2, nargs + 2): - loc = arglocs[i] - if not isinstance(loc, REG): - if isinstance(loc, MODRM64): - self.mc.MOVSD(xmm0, loc) - self.mc.MOVSD(mem64(esp, p), xmm0) - else: - self.mc.MOV(tmp, loc) - self.mc.MOV(mem(esp, p), tmp) - p += round_up_to_4(loc.width) - self.mc.CALL(x) - self.mark_gc_roots() + + self._emit_call(x, arglocs, 2, tmp=tmp) + if isinstance(resloc, MODRM64): self.mc.FSTP(resloc) elif size == 1: @@ -1199,14 +1193,13 @@ mc.CMP(edx, heap(nursery_top_adr)) mc.write(constlistofchars('\x76\x00')) # JNA after the block jmp_adr = mc.get_relative_pos() - mc.PUSH(imm(size)) - mc.CALL(rel32(slowpath_addr)) - self.mark_gc_roots() + self._emit_call(rel32(slowpath_addr), [imm(size)]) + # note that slowpath_addr returns a "long long", or more precisely # two results, which end up in eax and edx. # eax should contain the result of allocation, edx new value # of nursery_free_adr - mc.ADD(esp, imm(4)) + offset = mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 mc.overwrite(jmp_adr-1, [chr(offset)]) From arigo at codespeak.net Sun Dec 6 17:13:49 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 17:13:49 +0100 (CET) Subject: [pypy-svn] r69921 - in pypy/branch/virtual-forcing/pypy/jit/metainterp: . test Message-ID: <20091206161349.46AD1168016@codespeak.net> Author: arigo Date: Sun Dec 6 17:13:47 2009 New Revision: 69921 Added: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py - copied, changed from r69909, pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py Removed: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_resume.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Log: Replace 'all_virtual_refs' with a regular list of boxes, 'virtualref_boxes', that is saved and restored by resume.py. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Sun Dec 6 17:13:47 2009 @@ -890,11 +890,6 @@ def opimpl_virtual_ref(self, box): obj = box.getref_base() vref = virtualref.virtual_ref_during_tracing(obj) - # - if self.metainterp.all_virtual_refs is None: - self.metainterp.all_virtual_refs = [] - self.metainterp.all_virtual_refs.append(vref) - # resbox = history.BoxPtr(vref) self.metainterp.history.record(rop.VIRTUAL_REF, [box], resbox) # Note: we allocate a JIT_VIRTUAL_REF here @@ -904,15 +899,17 @@ # as a no-op. The point is that the backend should not really see # it in practice, as optimizeopt.py should either kill it or # replace it with a NEW_WITH_VTABLE followed by SETFIELD_GCs. + self.metainterp.virtualref_boxes.append(box) + self.metainterp.virtualref_boxes.append(resbox) self.make_result_box(resbox) @arguments("box") - def opimpl_virtual_ref_finish(self, box): + def opimpl_virtual_ref_finish(self, vrefbox): # virtual_ref_finish() assumes that we have a stack-like, last-in # first-out order. - if not self.metainterp.is_blackholing(): - lastitem = self.metainterp.all_virtual_refs.pop() - assert box.getref_base() == lastitem + lastvrefbox = self.metainterp.virtualref_boxes.pop() + assert vrefbox.getref_base() == lastvrefbox.getref_base() + self.metainterp.virtualref_boxes.pop() # ------------------------------ @@ -975,7 +972,7 @@ if metainterp.staticdata.virtualizable_info is not None: virtualizable_boxes = metainterp.virtualizable_boxes resume.capture_resumedata(metainterp.framestack, virtualizable_boxes, - resumedescr) + metainterp.virtualref_boxes, resumedescr) self.metainterp.staticdata.profiler.count_ops(opnum, GUARDS) # count metainterp.attach_debug_info(guard_op) @@ -1222,7 +1219,6 @@ class MetaInterp(object): in_recursion = 0 - all_virtual_refs = None _already_allocated_resume_virtuals = None def __init__(self, staticdata): @@ -1230,6 +1226,7 @@ self.cpu = staticdata.cpu self.portal_trace_positions = [] self.greenkey_of_huge_function = None + self.virtualref_boxes = [] def is_blackholing(self): return self.history is None @@ -1786,10 +1783,11 @@ self.load_fields_from_virtualizable() def virtual_after_residual_call(self): - if self.is_blackholing() or self.all_virtual_refs is None: + if self.is_blackholing(): return - for vr in self.all_virtual_refs: - if virtualref.was_forced(vr): + for i in range(1, len(self.virtualref_boxes), 2): + vrefbox = self.virtualref_boxes[i] + if virtualref.was_forced(vrefbox.getref_base()): break else: return @@ -1831,7 +1829,9 @@ vinfo = self.staticdata.virtualizable_info self.framestack = [] expect_virtualizable = vinfo is not None - virtualizable_boxes = resume.rebuild_from_resumedata(self, newboxes, resumedescr, expect_virtualizable) + virtualizable_boxes, virtualref_boxes = resume.rebuild_from_resumedata( + self, newboxes, resumedescr, expect_virtualizable) + self.virtualref_boxes = virtualref_boxes if expect_virtualizable: self.virtualizable_boxes = virtualizable_boxes if self._already_allocated_resume_virtuals is not None: @@ -1840,8 +1840,8 @@ self.load_fields_from_virtualizable() return # just jumped away from assembler (case 4 in the comment in - # virtualizable.py) into tracing (case 2); check that vable_rti - # is and stays NULL. + # virtualizable.py) into tracing (case 2); check that vable_token + # is and stays 0. virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) assert not virtualizable.vable_token Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py Sun Dec 6 17:13:47 2009 @@ -47,7 +47,8 @@ back.parent_resumedata_snapshot, back.env[:]) -def capture_resumedata(framestack, virtualizable_boxes, storage): +def capture_resumedata(framestack, virtualizable_boxes, virtualref_boxes, + storage): n = len(framestack)-1 top = framestack[n] _ensure_parent_resumedata(framestack, n) @@ -55,6 +56,7 @@ top) storage.rd_frame_info_list = frame_info_list snapshot = Snapshot(top.parent_resumedata_snapshot, top.env[:]) + snapshot = Snapshot(snapshot, virtualref_boxes[:]) # xxx for now if virtualizable_boxes is not None: snapshot = Snapshot(snapshot, virtualizable_boxes[:]) # xxx for now storage.rd_snapshot = snapshot @@ -444,6 +446,7 @@ virtualizable_boxes = None if expects_virtualizables: virtualizable_boxes = resumereader.consume_boxes() + virtualref_boxes = resumereader.consume_boxes() frameinfo = storage.rd_frame_info_list while True: env = resumereader.consume_boxes() @@ -453,7 +456,7 @@ if frameinfo is None: break metainterp.framestack.reverse() - return virtualizable_boxes + return virtualizable_boxes, virtualref_boxes def force_from_resumedata(metainterp, newboxes, storage): resumereader = ResumeDataReader(storage, newboxes, metainterp) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_resume.py Sun Dec 6 17:13:47 2009 @@ -203,29 +203,32 @@ fs = [FakeFrame("code0", 0, -1, b1, c1, b2)] storage = Storage() - capture_resumedata(fs, None, storage) + capture_resumedata(fs, None, [], storage) assert fs[0].parent_resumedata_snapshot is None assert fs[0].parent_resumedata_frame_info_list is None assert storage.rd_frame_info_list.prev is None assert storage.rd_frame_info_list.jitcode == 'code0' - assert storage.rd_snapshot.prev is None - assert storage.rd_snapshot.boxes == fs[0].env - assert storage.rd_snapshot.boxes is not fs[0].env + assert storage.rd_snapshot.boxes == [] # for virtualrefs + snapshot = storage.rd_snapshot.prev + assert snapshot.prev is None + assert snapshot.boxes == fs[0].env + assert snapshot.boxes is not fs[0].env storage = Storage() fs = [FakeFrame("code0", 0, -1, b1, c1, b2), FakeFrame("code1", 3, 7, b3, c2, b1), FakeFrame("code2", 9, -1, c3, b2)] - capture_resumedata(fs, None, storage) + capture_resumedata(fs, None, [], storage) frame_info_list = storage.rd_frame_info_list assert frame_info_list.prev is fs[2].parent_resumedata_frame_info_list assert frame_info_list.jitcode == 'code2' assert frame_info_list.pc == 9 - snapshot = storage.rd_snapshot + assert storage.rd_snapshot.boxes == [] # for virtualrefs + snapshot = storage.rd_snapshot.prev assert snapshot.prev is fs[2].parent_resumedata_snapshot assert snapshot.boxes == fs[2].env assert snapshot.boxes is not fs[2].env @@ -249,8 +252,9 @@ fs[2].env = [b2, b3] fs[2].pc = 15 vbs = [b1, b2] - capture_resumedata(fs, vbs, storage) - + vrs = [b3] + capture_resumedata(fs, vbs, vrs, storage) + frame_info_list = storage.rd_frame_info_list assert frame_info_list.prev is fs[2].parent_resumedata_frame_info_list assert frame_info_list.jitcode == 'code2' @@ -261,6 +265,10 @@ assert snapshot.boxes is not vbs snapshot = snapshot.prev + assert snapshot.boxes == vrs + assert snapshot.boxes is not vrs + + snapshot = snapshot.prev assert snapshot.prev is fs[2].parent_resumedata_snapshot assert snapshot.boxes == fs[2].env @@ -278,7 +286,7 @@ fs = [FakeFrame("code0", 0, -1, b1, c1, b2), FakeFrame("code1", 3, 7, b3, c2, b1), FakeFrame("code2", 9, -1, c3, b2)] - capture_resumedata(fs, None, storage) + capture_resumedata(fs, None, [], storage) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish({}) @@ -289,7 +297,7 @@ result = rebuild_from_resumedata(metainterp, newboxes, storage, False) - assert result is None + assert result == (None, []) fs2 = [FakeFrame("code0", 0, -1, b1t, c1, b2t), FakeFrame("code1", 3, 7, b3t, c2, b1t), FakeFrame("code2", 9, -1, c3, b2t)] @@ -302,7 +310,7 @@ fs = [FakeFrame("code0", 0, -1, b1, c1, b2), FakeFrame("code1", 3, 7, b3, c2, b1), FakeFrame("code2", 9, -1, c3, b2)] - capture_resumedata(fs, [b4], storage) + capture_resumedata(fs, [b4], [], storage) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish({}) @@ -313,7 +321,7 @@ result = rebuild_from_resumedata(metainterp, newboxes, storage, True) - assert result == [b4t] + assert result == ([b4t], []) fs2 = [FakeFrame("code0", 0, -1, b1t, c1, b2t), FakeFrame("code1", 3, 7, b3t, c2, b1t), FakeFrame("code2", 9, -1, c3, b2t)] @@ -326,10 +334,10 @@ fs = [FakeFrame("code0", 0, -1, b1, c1, b2), FakeFrame("code1", 3, 7, b3, c2, b1), FakeFrame("code2", 9, -1, c3, b2)] - capture_resumedata(fs, None, storage) + capture_resumedata(fs, None, [], storage) storage2 = Storage() fs = fs[:-1] + [FakeFrame("code2", 10, -1, c3, b2, b4)] - capture_resumedata(fs, None, storage2) + capture_resumedata(fs, None, [], storage2) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) @@ -345,7 +353,7 @@ result = rebuild_from_resumedata(metainterp, newboxes, storage, False) - assert result is None + assert result == (None, []) fs2 = [FakeFrame("code0", 0, -1, b1t, c1, b2t), FakeFrame("code1", 3, 7, b3t, c2, b1t), FakeFrame("code2", 9, -1, c3, b2t)] @@ -356,7 +364,7 @@ metainterp.framestack = [] result = rebuild_from_resumedata(metainterp, newboxes, storage2, False) - assert result is None + assert result == (None, []) fs2 = fs2[:-1] + [FakeFrame("code2", 10, -1, c3, b2t, b4t)] assert metainterp.framestack == fs2 @@ -384,10 +392,10 @@ fs = [FakeFrame("code0", 0, -1, b1, c1, b2), FakeFrame("code1", 3, 7, b3, c2, b1), FakeFrame("code2", 9, -1, c3, b2)] - capture_resumedata(fs, None, storage) + capture_resumedata(fs, None, [], storage) storage2 = Storage() fs = fs[:-1] + [FakeFrame("code2", 10, -1, c3, b2, b4)] - capture_resumedata(fs, None, storage2) + capture_resumedata(fs, None, [], storage2) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) values = {b2: virtual_value(b2, b5, c4)} @@ -443,7 +451,7 @@ LLtypeMixin.nodebox.constbox()] storage = Storage() fs = [FakeFrame("code0", 0, -1, c1, b2, b3)] - capture_resumedata(fs, None, storage) + capture_resumedata(fs, None, [], storage) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) values = {b2: virtual_value(b2, b5, c4)} @@ -455,7 +463,7 @@ storage2 = Storage() fs = [FakeFrame("code0", 0, -1, b1, b4, b2)] - capture_resumedata(fs, None, storage2) + capture_resumedata(fs, None, [], storage2) values[b4] = virtual_value(b4, b6, c4) modifier = ResumeDataVirtualAdder(storage2, memo) liveboxes = modifier.finish(values) @@ -468,7 +476,7 @@ b1, b2, b3 = [BoxPtr(), BoxPtr(), BoxInt()] storage = Storage() fs = [FakeFrame("code0", 0, -1, b1, b2)] - capture_resumedata(fs, None, storage) + capture_resumedata(fs, None, [], storage) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) v1 = virtual_value(b1, b3, None) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Sun Dec 6 17:13:47 2009 @@ -1160,8 +1160,7 @@ self.check_loops(getfield_gc=0, setfield_gc=0) def test_blackhole_should_not_reenter(self): - from pypy.jit.backend.test.support import BaseCompiledMixin - if isinstance(self, BaseCompiledMixin): + if not self.basic: py.test.skip("purely frontend test") myjitdriver = JitDriver(greens = [], reds = ['frame', 'fail'], Copied: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py (from r69909, pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py) ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Sun Dec 6 17:13:47 2009 @@ -1,8 +1,12 @@ import py +from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rlib.jit import JitDriver, dont_look_inside from pypy.rlib.jit import virtual_ref, virtual_ref_finish from pypy.rlib.objectmodel import compute_unique_id from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin +from pypy.jit.metainterp.resoperation import rop +from pypy.jit.metainterp import history +from pypy.jit.metainterp.virtualref import JIT_VIRTUAL_REF class VRefTests: @@ -23,6 +27,56 @@ self.interp_operations(f, []) self.check_operations_history(virtual_ref=1) + def test_make_vref_guard(self): + if not self.basic: + py.test.skip("purely frontend test") + # + class X: + def __init__(self, n): + self.n = n + class ExCtx: + pass + exctx = ExCtx() + # + @dont_look_inside + def external(n): + if n > 100: + return exctx.topframeref().n + return n + def enter(n): + exctx.topframeref = virtual_ref(X(n + 10)) + def leave(): + virtual_ref_finish(exctx.topframeref) + exctx.topframeref = None + def f(n): + enter(n) + n = external(n) + # ^^^ the point is that X() should be kept alive here + leave() + return n + # + res = self.interp_operations(f, [5]) + assert res == 5 + self.check_operations_history(virtual_ref=1, guard_not_forced=1) + # + [guard_op] = [op for op in self.metainterp.history.operations + if op.opnum == rop.GUARD_NOT_FORCED] + bxs1 = [box for box in guard_op.fail_args + if isinstance(box, history.BoxPtr) and + lltype.typeOf(box.value._obj.container)._name.endswith('.X')] + assert len(bxs1) == 1 + bxs2 = [box for box in guard_op.fail_args + if isinstance(box, history.BoxPtr) and + lltype.typeOf(box.value._obj.container) == JIT_VIRTUAL_REF] + assert len(bxs2) == 1 + # + self.metainterp.rebuild_state_after_failure(guard_op.descr, + guard_op.fail_args[:]) + assert len(self.metainterp.framestack) == 1 + assert len(self.metainterp.virtualref_boxes) == 2 + assert self.metainterp.virtualref_boxes[0].value == bxs1[0].value + assert self.metainterp.virtualref_boxes[1].value == bxs2[0].value + def test_make_vref_and_force(self): jitdriver = JitDriver(greens = [], reds = ['total', 'n']) # @@ -45,8 +99,8 @@ x.n = n + 123 exctx.topframeref = virtual_ref(x) total += force_me() - 100 + virtual_ref_finish(exctx.topframeref) exctx.topframeref = None - virtual_ref_finish(x) return total # res = self.meta_interp(f, [-4]) From arigo at codespeak.net Sun Dec 6 17:16:06 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 17:16:06 +0100 (CET) Subject: [pypy-svn] r69922 - in pypy/branch/virtual-forcing/pypy/jit/metainterp: . test Message-ID: <20091206161606.2E593168016@codespeak.net> Author: arigo Date: Sun Dec 6 17:16:05 2009 New Revision: 69922 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Log: Fix for the next test. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py Sun Dec 6 17:16:05 2009 @@ -1306,6 +1306,11 @@ self.emit(oopspec_name) # 'virtual_ref' or 'virtual_ref_finish' self.emit(self.var_position(args[0])) self.register_var(op.result) + # + from pypy.jit.metainterp.virtualref import jit_virtual_ref_vtable + from pypy.jit.metainterp.virtualref import JIT_VIRTUAL_REF + self.codewriter.register_known_gctype(jit_virtual_ref_vtable, + JIT_VIRTUAL_REF) def _array_of_voids(self, ARRAY): if isinstance(ARRAY, ootype.Array): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Sun Dec 6 17:16:05 2009 @@ -108,7 +108,7 @@ self.check_loops({}) # because we aborted tracing def test_simple_no_access(self): - py.test.skip("in-progress") + #py.test.skip("in-progress") myjitdriver = JitDriver(greens = [], reds = ['n']) # class XY: From arigo at codespeak.net Sun Dec 6 17:21:08 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 17:21:08 +0100 (CET) Subject: [pypy-svn] r69923 - in pypy/branch/virtual-forcing/pypy/jit/metainterp: . test Message-ID: <20091206162108.4713E168016@codespeak.net> Author: arigo Date: Sun Dec 6 17:21:07 2009 New Revision: 69923 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Log: Small fixes. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Sun Dec 6 17:21:07 2009 @@ -1558,6 +1558,7 @@ len(self.virtualizable_boxes)-1, duplicates) live_arg_boxes += self.virtualizable_boxes[:-1] + assert len(self.virtualref_boxes) == 0, "missing virtual_ref_finish()?" # Called whenever we reach the 'can_enter_jit' hint. # First, attempt to make a bridge: # - if self.resumekey is a ResumeGuardDescr, it starts from a guard @@ -1900,6 +1901,10 @@ for i in range(len(boxes)): if boxes[i] is oldbox: boxes[i] = newbox + boxes = self.virtualref_boxes + for i in range(len(boxes)): + if boxes[i] is oldbox: + boxes[i] = newbox if self.staticdata.virtualizable_info is not None: boxes = self.virtualizable_boxes for i in range(len(boxes)): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Sun Dec 6 17:21:07 2009 @@ -108,7 +108,6 @@ self.check_loops({}) # because we aborted tracing def test_simple_no_access(self): - #py.test.skip("in-progress") myjitdriver = JitDriver(greens = [], reds = ['n']) # class XY: @@ -132,13 +131,13 @@ xy.next2 = XY() exctx.topframeref = virtual_ref(xy) n -= externalfn(n) + virtual_ref_finish(exctx.topframeref) exctx.topframeref = None # self.meta_interp(f, [15]) self.check_loops(new_with_vtable=1) # the vref, not the XYs def test_simple_force_always(self): - py.test.skip("in-progress") myjitdriver = JitDriver(greens = [], reds = ['n']) # class XY: @@ -161,6 +160,7 @@ xy.n = n exctx.topframeref = virtual_ref(xy) n -= externalfn(n) + virtual_ref_finish(exctx.topframeref) exctx.topframeref = None # self.meta_interp(f, [15]) From arigo at codespeak.net Sun Dec 6 18:08:41 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 18:08:41 +0100 (CET) Subject: [pypy-svn] r69924 - in pypy/branch/virtual-forcing/pypy/jit: backend/test metainterp metainterp/test Message-ID: <20091206170841.F1A41168012@codespeak.net> Author: arigo Date: Sun Dec 6 18:08:41 2009 New Revision: 69924 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py Log: Done! Now all that's left is many small bugs here and there... Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Sun Dec 6 18:08:41 2009 @@ -782,7 +782,8 @@ def test_virtual_ref(self): # if VIRTUAL_REF reaches the backend, it should just be a SAME_AS u_box = self.alloc_unicode(u"hello\u1234") - r = self.execute_operation(rop.VIRTUAL_REF, [u_box], 'ref') + r = self.execute_operation(rop.VIRTUAL_REF, [u_box, ConstInt(2)], + 'ref') assert r.value == u_box.value def test_jump(self): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py Sun Dec 6 18:08:41 2009 @@ -237,23 +237,50 @@ from pypy.jit.metainterp.pyjitpl import MetaInterp metainterp = MetaInterp(metainterp_sd) token = metainterp_sd.cpu.get_latest_force_token() - data = self.fetch_data(token) - if data is None: - data = [] - metainterp._already_allocated_resume_virtuals = data + all_virtuals = self.fetch_data(token) + if all_virtuals is None: + all_virtuals = [] + metainterp._already_allocated_resume_virtuals = all_virtuals self.counter = -2 # never compile return metainterp.handle_guard_failure(self) - def force_virtualizable(self, vinfo, virtualizable, force_token): + @staticmethod + def force_now(cpu, token): + # Called during a residual call from the assembler, if the code + # actually needs to force one of the virtualrefs or the virtualizable. + # Implemented by forcing *all* virtualrefs and the virtualizable. + faildescr = cpu.force(token) + assert isinstance(faildescr, ResumeGuardForcedDescr) + faildescr.handle_async_forcing(token) + + def handle_async_forcing(self, force_token): from pypy.jit.metainterp.pyjitpl import MetaInterp from pypy.jit.metainterp.resume import force_from_resumedata - metainterp = MetaInterp(self.metainterp_sd) + from pypy.jit.metainterp.virtualref import forced_single_vref + # To handle the forcing itself, we create a temporary MetaInterp + # as a convenience to move the various data to its proper place. + metainterp_sd = self.metainterp_sd + metainterp = MetaInterp(metainterp_sd) metainterp.history = None # blackholing - liveboxes = metainterp.cpu.make_boxes_from_latest_values(self) - virtualizable_boxes, data = force_from_resumedata(metainterp, - liveboxes, self) - vinfo.write_boxes(virtualizable, virtualizable_boxes) - self.save_data(force_token, data) + liveboxes = metainterp_sd.cpu.make_boxes_from_latest_values(self) + # + expect_virtualizable = metainterp_sd.virtualizable_info is not None + forced_data = force_from_resumedata(metainterp, liveboxes, self, + expect_virtualizable) + virtualizable_boxes, virtualref_boxes, all_virtuals = forced_data + # + # Handle virtualref_boxes: mark each JIT_VIRTUAL_REF as forced + for i in range(0, len(virtualref_boxes), 2): + virtualbox = virtualref_boxes[i] + vrefbox = virtualref_boxes[i+1] + forced_single_vref(vrefbox.getref_base(), virtualbox.getref_base()) + # Handle virtualizable_boxes: store them on the real virtualizable now + if expect_virtualizable: + metainterp.virtualizable_boxes = virtualizable_boxes + metainterp.synchronize_virtualizable() + # Handle all_virtuals: keep them for later blackholing from the + # future failure of the GUARD_NOT_FORCED + self.save_data(force_token, all_virtuals) def save_data(self, key, value): globaldata = self.metainterp_sd.globaldata Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py Sun Dec 6 18:08:41 2009 @@ -740,12 +740,14 @@ value = self.getvalue(op.args[0]) if not value.is_virtual(): # virtual_ref(non-virtual) gives bad raise compile.GiveUp # results, so don't bother compiling it + indexbox = op.args[1] # # get some constants (these calls are all 'memo') from pypy.jit.metainterp import virtualref - c_cls = virtualref.get_jit_virtual_ref_const_class(self.cpu) - descr_virtual_token = virtualref.get_descr_virtual_token(self.cpu) - descr_forced = virtualref.get_descr_forced(self.cpu) + cpu = self.cpu + c_cls = virtualref.get_jit_virtual_ref_const_class(cpu) + descr_virtual_token = virtualref.get_descr_virtual_token(cpu) + descr_virtualref_index = virtualref.get_descr_virtualref_index(cpu) # # Replace the VIRTUAL_REF operation with a virtual structure of type # 'vref.JIT_VIRTUAL_REF'. The virtual structure may be forced soon, @@ -755,6 +757,7 @@ tokenbox = BoxInt() self.emit_operation(ResOperation(rop.FORCE_TOKEN, [], tokenbox)) vrefvalue.setfield(descr_virtual_token, self.getvalue(tokenbox)) + vrefvalue.setfield(descr_virtualref_index, self.getvalue(indexbox)) def optimize_GETFIELD_GC(self, op): value = self.getvalue(op.args[0]) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Sun Dec 6 18:08:41 2009 @@ -891,7 +891,8 @@ obj = box.getref_base() vref = virtualref.virtual_ref_during_tracing(obj) resbox = history.BoxPtr(vref) - self.metainterp.history.record(rop.VIRTUAL_REF, [box], resbox) + cindex = history.ConstInt(len(self.metainterp.virtualref_boxes) // 2) + self.metainterp.history.record(rop.VIRTUAL_REF, [box, cindex], resbox) # Note: we allocate a JIT_VIRTUAL_REF here # (in virtual_ref_during_tracing()), in order to detect when # the virtual escapes during tracing already. We record it as a Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py Sun Dec 6 18:08:41 2009 @@ -206,7 +206,7 @@ 'NEW_WITH_VTABLE/1', 'NEW_ARRAY/1d', 'FORCE_TOKEN/0', - 'VIRTUAL_REF/1', + 'VIRTUAL_REF/2', '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- 'SETARRAYITEM_GC/3d', Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py Sun Dec 6 18:08:41 2009 @@ -441,7 +441,8 @@ debug_print("\t\t", str(untag(i))) -def rebuild_from_resumedata(metainterp, newboxes, storage, expects_virtualizables): +def rebuild_from_resumedata(metainterp, newboxes, storage, + expects_virtualizables): resumereader = ResumeDataReader(storage, newboxes, metainterp) virtualizable_boxes = None if expects_virtualizables: @@ -458,9 +459,14 @@ metainterp.framestack.reverse() return virtualizable_boxes, virtualref_boxes -def force_from_resumedata(metainterp, newboxes, storage): +def force_from_resumedata(metainterp, newboxes, storage, + expects_virtualizables): resumereader = ResumeDataReader(storage, newboxes, metainterp) - return resumereader.consume_boxes(), resumereader.virtuals + virtualizable_boxes = None + if expects_virtualizables: + virtualizable_boxes = resumereader.consume_boxes() + virtualref_boxes = resumereader.consume_boxes() + return virtualizable_boxes, virtualref_boxes, resumereader.virtuals class ResumeDataReader(object): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Sun Dec 6 18:08:41 2009 @@ -164,7 +164,37 @@ exctx.topframeref = None # self.meta_interp(f, [15]) - self.check_loops(new_with_vtable=0) + self.check_loops({}) # because we aborted tracing + + def test_simple_force_sometimes(self): + myjitdriver = JitDriver(greens = [], reds = ['n']) + # + class XY: + pass + class ExCtx: + pass + exctx = ExCtx() + # + @dont_look_inside + def externalfn(n): + if n == 13: + exctx.m = exctx.topframeref().n + return 1 + # + def f(n): + while n > 0: + myjitdriver.can_enter_jit(n=n) + myjitdriver.jit_merge_point(n=n) + xy = XY() + xy.n = n + exctx.topframeref = virtual_ref(xy) + n -= externalfn(n) + virtual_ref_finish(exctx.topframeref) + exctx.topframeref = None + return exctx.m + # + res = self.meta_interp(f, [30]) + assert res == 13 class TestLLtype(VRefTests, LLJitMixin): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py Sun Dec 6 18:08:41 2009 @@ -200,19 +200,16 @@ return True def force_now(self, virtualizable): - token = virtualizable.vable_token - virtualizable.vable_token = self.TOKEN_NONE - if token == self.TOKEN_TRACING: + if virtualizable.vable_token == self.TOKEN_TRACING: # The values in the virtualizable are always correct during # tracing. We only need to reset vable_token to TOKEN_NONE # as a marker for the tracing, to tell it that this # virtualizable escapes. - pass + virtualizable.vable_token = self.TOKEN_NONE else: from pypy.jit.metainterp.compile import ResumeGuardForcedDescr - faildescr = self.cpu.force(token) - assert isinstance(faildescr, ResumeGuardForcedDescr) - faildescr.force_virtualizable(self, virtualizable, token) + ResumeGuardForcedDescr.force_now(self.cpu, token) + assert virtualizable.vable_token == self.TOKEN_NONE force_now._dont_inline_ = True # ____________________________________________________________ Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py Sun Dec 6 18:08:41 2009 @@ -4,7 +4,7 @@ from pypy.jit.metainterp import history -def replace_force_virtual_with_call(make_helper_func, graphs): +def replace_force_virtual_with_call(warmrunnerdesc, graphs): # similar to rvirtualizable2.replace_force_virtualizable_with_call(). c_funcptr = None count = 0 @@ -15,18 +15,14 @@ # first compute c_funcptr, but only if there is any # 'jit_force_virtual' around if c_funcptr is None: - FUNC = lltype.FuncType([rclass.OBJECTPTR], - rclass.OBJECTPTR) - funcptr = make_helper_func( - lltype.Ptr(FUNC), - force_virtual_if_necessary) - c_funcptr = inputconst(lltype.typeOf(funcptr), funcptr) + c_funcptr = get_force_virtual_fnptr(warmrunnerdesc) # op.opname = 'direct_call' op.args = [c_funcptr, op.args[0]] count += 1 if c_funcptr is not None: - log("replaced %d 'jit_force_virtual' with %r" % (count, funcptr)) + log("replaced %d 'jit_force_virtual' with %r" % (count, + c_funcptr.value)) # ____________________________________________________________ @@ -35,6 +31,7 @@ JIT_VIRTUAL_REF = lltype.GcStruct('JitVirtualRef', ('super', rclass.OBJECT), ('virtual_token', lltype.Signed), + ('virtualref_index', lltype.Signed), ('forced', rclass.OBJECTPTR)) jit_virtual_ref_vtable = lltype.malloc(rclass.OBJECT_VTABLE, zero=True, flavor='raw') @@ -58,6 +55,10 @@ return cpu.fielddescrof(JIT_VIRTUAL_REF, 'virtual_token') @specialize.memo() +def get_descr_virtualref_index(cpu): + return cpu.fielddescrof(JIT_VIRTUAL_REF, 'virtualref_index') + + at specialize.memo() def get_descr_forced(cpu): return cpu.fielddescrof(JIT_VIRTUAL_REF, 'forced') @@ -74,17 +75,45 @@ vref = lltype.cast_opaque_ptr(lltype.Ptr(JIT_VIRTUAL_REF), gcref) return vref.virtual_token != TOKEN_TRACING +def forced_single_vref(gcref, real_object): + assert real_object + vref = lltype.cast_opaque_ptr(lltype.Ptr(JIT_VIRTUAL_REF), gcref) + assert (vref.virtual_token != TOKEN_NONE and + vref.virtual_token != TOKEN_TRACING) + vref.forced = lltype.cast_opaque_ptr(rclass.OBJECTPTR, real_object) + vref.virtual_token = TOKEN_NONE + # ____________________________________________________________ -def force_virtual_if_necessary(inst): - if not inst or inst.typeptr != jit_virtual_ref_vtable: - return inst # common, fast case - return force_virtual(inst) +def get_force_virtual_fnptr(warmrunnerdesc): + cpu = warmrunnerdesc.cpu + # + def force_virtual_if_necessary(inst): + if not inst or inst.typeptr != jit_virtual_ref_vtable: + return inst # common, fast case + return force_virtual(cpu, inst) + # + FUNC = lltype.FuncType([rclass.OBJECTPTR], rclass.OBJECTPTR) + funcptr = warmrunnerdesc.helper_func( + lltype.Ptr(FUNC), + force_virtual_if_necessary) + return inputconst(lltype.typeOf(funcptr), funcptr) -def force_virtual(inst): +def force_virtual(cpu, inst): vref = lltype.cast_pointer(lltype.Ptr(JIT_VIRTUAL_REF), inst) - if not vref.forced: - xxxx - vref.virtual_token = TOKEN_NONE + token = vref.virtual_token + if token != TOKEN_NONE: + if token == TOKEN_TRACING: + # The "virtual" is not a virtual at all during tracing. + # We only need to reset virtual_token to TOKEN_NONE + # as a marker for the tracing, to tell it that this + # "virtual" escapes. + vref.virtual_token = TOKEN_NONE + else: + assert not vref.forced + from pypy.jit.metainterp.compile import ResumeGuardForcedDescr + ResumeGuardForcedDescr.force_now(cpu, token) + assert vref.virtual_token == TOKEN_NONE + assert vref.forced return vref.forced force_virtual._dont_inline_ = True Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py Sun Dec 6 18:08:41 2009 @@ -604,8 +604,7 @@ if self.cpu.ts.name != 'lltype': py.test.skip("rewrite_force_virtual: port it to ootype") all_graphs = self.translator.graphs - virtualref.replace_force_virtual_with_call(self.helper_func, - all_graphs) + virtualref.replace_force_virtual_with_call(self, all_graphs) def decode_hp_hint_args(op): From arigo at codespeak.net Sun Dec 6 18:12:20 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 18:12:20 +0100 (CET) Subject: [pypy-svn] r69925 - pypy/branch/virtual-forcing/pypy/jit/metainterp Message-ID: <20091206171220.44478168016@codespeak.net> Author: arigo Date: Sun Dec 6 18:12:19 2009 New Revision: 69925 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py Log: Fix the handling of virtualizables. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py Sun Dec 6 18:12:19 2009 @@ -276,8 +276,7 @@ forced_single_vref(vrefbox.getref_base(), virtualbox.getref_base()) # Handle virtualizable_boxes: store them on the real virtualizable now if expect_virtualizable: - metainterp.virtualizable_boxes = virtualizable_boxes - metainterp.synchronize_virtualizable() + metainterp_sd.virtualizable_info.forced_vable(virtualizable_boxes) # Handle all_virtuals: keep them for later blackholing from the # future failure of the GUARD_NOT_FORCED self.save_data(force_token, all_virtuals) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py Sun Dec 6 18:12:19 2009 @@ -200,7 +200,8 @@ return True def force_now(self, virtualizable): - if virtualizable.vable_token == self.TOKEN_TRACING: + token = virtualizable.vable_token + if token == self.TOKEN_TRACING: # The values in the virtualizable are always correct during # tracing. We only need to reset vable_token to TOKEN_NONE # as a marker for the tracing, to tell it that this @@ -212,6 +213,12 @@ assert virtualizable.vable_token == self.TOKEN_NONE force_now._dont_inline_ = True + def forced_vable(self, virtualizable_boxes): + virtualizable_box = virtualizable_boxes[-1] + virtualizable = self.unwrap_virtualizable_box(virtualizable_box) + self.write_boxes(virtualizable, virtualizable_boxes) + virtualizable.vable_token = self.TOKEN_NONE + # ____________________________________________________________ # # The 'vable_token' field of a virtualizable is either 0, -1, or points From arigo at codespeak.net Sun Dec 6 18:14:47 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 18:14:47 +0100 (CET) Subject: [pypy-svn] r69926 - pypy/branch/virtual-forcing/pypy/jit/metainterp/test Message-ID: <20091206171447.259CE168016@codespeak.net> Author: arigo Date: Sun Dec 6 18:14:46 2009 New Revision: 69926 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py Log: Fix the tests for optimizeopt. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py Sun Dec 6 18:14:46 2009 @@ -105,6 +105,7 @@ from pypy.jit.metainterp.virtualref import jit_virtual_ref_vtable from pypy.jit.metainterp.virtualref import JIT_VIRTUAL_REF virtualtokendescr = cpu.fielddescrof(JIT_VIRTUAL_REF, 'virtual_token') + virtualrefindexdescr = cpu.fielddescrof(JIT_VIRTUAL_REF,'virtualref_index') cpu.class_sizes = {cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), cpu.cast_adr_to_int(node_vtable_adr2): cpu.sizeof(NODE2), Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py Sun Dec 6 18:14:46 2009 @@ -2034,7 +2034,7 @@ def test_vref_nonvirtual(self): ops = """ [p1] - p2 = virtual_ref(p1) + p2 = virtual_ref(p1, 5) jump(p1) """ py.test.raises(compile.GiveUp, self.optimize_loop, ops, 'Not', ops) @@ -2048,7 +2048,7 @@ setfield_gc(p1b, 252, descr=valuedescr) setfield_gc(p1, p1b, descr=nextdescr) # - p2 = virtual_ref(p1) + p2 = virtual_ref(p1, 3) setfield_gc(p0, p2, descr=nextdescr) call_may_force(i1, descr=mayforcevirtdescr) guard_not_forced() [i1] @@ -2060,6 +2060,7 @@ i3 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, 3, descr=virtualrefindexdescr) setfield_gc(p0, p2, descr=nextdescr) call_may_force(i1, descr=mayforcevirtdescr) guard_not_forced() [i1] @@ -2078,7 +2079,7 @@ setfield_gc(p1b, i1, descr=valuedescr) setfield_gc(p1, p1b, descr=nextdescr) # - p2 = virtual_ref(p1) + p2 = virtual_ref(p1, 2) setfield_gc(p0, p2, descr=nextdescr) call_may_force(i1, descr=mayforcevirtdescr) guard_not_forced(descr=fdescr) [p1] @@ -2090,6 +2091,7 @@ i3 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, 2, descr=virtualrefindexdescr) setfield_gc(p0, p2, descr=nextdescr) call_may_force(i1, descr=mayforcevirtdescr) guard_not_forced(descr=fdescr) [i1] From arigo at codespeak.net Sun Dec 6 18:24:58 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 18:24:58 +0100 (CET) Subject: [pypy-svn] r69927 - pypy/branch/virtual-forcing/pypy/rlib Message-ID: <20091206172458.EFD38168016@codespeak.net> Author: arigo Date: Sun Dec 6 18:24:58 2009 New Revision: 69927 Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py Log: Forgot to check this in. Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/jit.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/jit.py Sun Dec 6 18:24:58 2009 @@ -2,6 +2,7 @@ import sys from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rlib.objectmodel import CDefinedIntSymbolic, we_are_translated +from pypy.rlib.objectmodel import keepalive_until_here from pypy.rlib.unroll import unrolling_iterable def purefunction(func): @@ -108,6 +109,8 @@ def virtual_ref_finish(x): if not we_are_translated(): x._forced = x._forced or -1 + keepalive_until_here(x) # otherwise the whole function call is removed +virtual_ref_finish.oopspec = 'virtual_ref_finish(x)' class DirectVRef(object): _forced = 0 From arigo at codespeak.net Sun Dec 6 18:34:54 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 18:34:54 +0100 (CET) Subject: [pypy-svn] r69928 - pypy/branch/virtual-forcing/pypy/jit/metainterp Message-ID: <20091206173454.E4261168016@codespeak.net> Author: arigo Date: Sun Dec 6 18:34:54 2009 New Revision: 69928 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py Log: Fix. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py Sun Dec 6 18:34:54 2009 @@ -223,7 +223,7 @@ def do_force_token(cpu): raise NotImplementedError -def do_virtual_ref(cpu, box1): +def do_virtual_ref(cpu, box1, box2): raise NotImplementedError def do_debug_merge_point(cpu, box1): From arigo at codespeak.net Sun Dec 6 18:43:06 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 18:43:06 +0100 (CET) Subject: [pypy-svn] r69929 - pypy/branch/virtual-forcing/pypy/translator/c Message-ID: <20091206174306.25B9F168016@codespeak.net> Author: arigo Date: Sun Dec 6 18:43:05 2009 New Revision: 69929 Modified: pypy/branch/virtual-forcing/pypy/translator/c/funcgen.py Log: Uh? Sorry. Modified: pypy/branch/virtual-forcing/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/c/funcgen.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/c/funcgen.py Sun Dec 6 18:43:05 2009 @@ -798,7 +798,7 @@ def OP_JIT_FORCE_VIRTUAL(self, op): return '%s = %s; /* JIT_FORCE_VIRTUAL */' % (self.expr(op.result), - self.expr(op.args[0]))) + self.expr(op.args[0])) def OP_GET_GROUP_MEMBER(self, op): typename = self.db.gettype(op.result.concretetype) From arigo at codespeak.net Sun Dec 6 18:54:57 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 18:54:57 +0100 (CET) Subject: [pypy-svn] r69930 - in pypy/branch/virtual-forcing/pypy/rpython/lltypesystem: . test Message-ID: <20091206175457.412AD168016@codespeak.net> Author: arigo Date: Sun Dec 6 18:54:56 2009 New Revision: 69930 Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: Remove this hack from ll2ctypes. It gets in the way of running virtualref tests from the x86 backend, and it was apparently only used by the old jit. Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py Sun Dec 6 18:54:56 2009 @@ -22,7 +22,6 @@ from pypy.annotation import model as annmodel from pypy.rpython.llinterp import LLInterpreter, LLException from pypy.rpython.lltypesystem.rclass import OBJECT -from pypy.rpython.annlowlevel import base_ptr_lltype from pypy.rpython import raddress from pypy.translator.platform import platform @@ -500,25 +499,6 @@ _callback2obj = {} _callback_exc_info = None -# this is just another hack that passes around references to applevel types -# disguised as base_ptr_lltype -class Dummy(object): - pass - -_opaque_cache = {Dummy():0} -_opaque_list = [Dummy()] - -def new_opaque_object(llobj): - try: - return _opaque_cache[llobj] - except KeyError: - assert len(_opaque_cache) == len(_opaque_list) - ctypes_type = get_ctypes_type(base_ptr_lltype()) - val = ctypes.cast(len(_opaque_cache), ctypes_type) - _opaque_list.append(llobj) - _opaque_cache[llobj] = val - return val - def get_rtyper(): llinterp = LLInterpreter.current_interpreter if llinterp is not None: @@ -542,8 +522,6 @@ return ctypes.c_void_p(0) return get_ctypes_type(T)() - if T is base_ptr_lltype(): - return new_opaque_object(llobj) if T == llmemory.GCREF: if isinstance(llobj._obj, _llgcopaque): return ctypes.c_void_p(llobj._obj.intval) @@ -694,8 +672,6 @@ if isinstance(T, lltype.Ptr): if not cobj: # NULL pointer return lltype.nullptr(T.TO) - if T is base_ptr_lltype(): - return _opaque_list[ctypes.cast(cobj, ctypes.c_void_p).value] if isinstance(T.TO, lltype.Struct): if T.TO._arrayfld is not None: carray = getattr(cobj.contents, T.TO._arrayfld) Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sun Dec 6 18:54:56 2009 @@ -928,34 +928,6 @@ assert op.args[1].value == pypy.rpython.lltypesystem.rstr.LLHelpers assert op.args[3].value == -2 - def test_pass_around_t_object(self): - from pypy.rpython.annlowlevel import base_ptr_lltype - T = base_ptr_lltype() - - class X(object): - _TYPE = T - x = 10 - - def callback(x): - return x.x - - c_source = py.code.Source(""" - long eating_callback(void *arg, long(*call)(void*)) - { - return call(arg); - } - """) - - eci = ExternalCompilationInfo(separate_module_sources=[c_source], - export_symbols=['eating_callback']) - - args = [T, rffi.CCallback([T], rffi.LONG)] - eating_callback = rffi.llexternal('eating_callback', args, rffi.LONG, - compilation_info=eci) - - res = eating_callback(X(), callback) - assert res == 10 - def test_recursive_struct_more(self): NODE = lltype.ForwardReference() NODE.become(lltype.Struct('NODE', ('value', lltype.Signed), From arigo at codespeak.net Sun Dec 6 19:21:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 19:21:29 +0100 (CET) Subject: [pypy-svn] r69931 - in pypy/branch/virtual-forcing/pypy/jit: backend/x86/test metainterp/test Message-ID: <20091206182129.87B6D168016@codespeak.net> Author: arigo Date: Sun Dec 6 19:21:28 2009 New Revision: 69931 Added: pypy/branch/virtual-forcing/pypy/jit/backend/x86/test/test_virtualref.py (contents, props changed) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Log: Add test_virtualref in the x86 backend. Fails. Fun ahead fighting ll2ctypes. Added: pypy/branch/virtual-forcing/pypy/jit/backend/x86/test/test_virtualref.py ============================================================================== --- (empty file) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/test/test_virtualref.py Sun Dec 6 19:21:28 2009 @@ -0,0 +1,8 @@ + +from pypy.jit.metainterp.test.test_virtualref import VRefTests +from pypy.jit.backend.x86.test.test_basic import Jit386Mixin + +class TestVRef(Jit386Mixin, VRefTests): + # for the individual tests see + # ====> ../../../metainterp/test/test_virtualref.py + pass Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Sun Dec 6 19:21:28 2009 @@ -5,7 +5,6 @@ from pypy.rlib.objectmodel import compute_unique_id from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.jit.metainterp.resoperation import rop -from pypy.jit.metainterp import history from pypy.jit.metainterp.virtualref import JIT_VIRTUAL_REF @@ -28,7 +27,7 @@ self.check_operations_history(virtual_ref=1) def test_make_vref_guard(self): - if not self.basic: + if not isinstance(self, TestLLtype): py.test.skip("purely frontend test") # class X: @@ -62,12 +61,10 @@ [guard_op] = [op for op in self.metainterp.history.operations if op.opnum == rop.GUARD_NOT_FORCED] bxs1 = [box for box in guard_op.fail_args - if isinstance(box, history.BoxPtr) and - lltype.typeOf(box.value._obj.container)._name.endswith('.X')] + if str(box._getrepr_()).endswith('.X')] assert len(bxs1) == 1 bxs2 = [box for box in guard_op.fail_args - if isinstance(box, history.BoxPtr) and - lltype.typeOf(box.value._obj.container) == JIT_VIRTUAL_REF] + if str(box._getrepr_()).endswith('JitVirtualRef')] assert len(bxs2) == 1 # self.metainterp.rebuild_state_after_failure(guard_op.descr, From arigo at codespeak.net Sun Dec 6 19:28:24 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 19:28:24 +0100 (CET) Subject: [pypy-svn] r69932 - pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test Message-ID: <20091206182824.70535168016@codespeak.net> Author: arigo Date: Sun Dec 6 19:28:23 2009 New Revision: 69932 Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: Write a test, which passes. Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sun Dec 6 19:28:23 2009 @@ -1106,7 +1106,24 @@ #import pdb; pdb.set_trace() assert adr1_2 == adr1 assert adr1 == adr1_2 - + + def test_object_subclass(self): + from pypy.rpython.lltypesystem import rclass + SCLASS = lltype.GcStruct('SCLASS', + ('parent', rclass.OBJECT), + ('x', lltype.Signed)) + def f(n): + s = lltype.malloc(SCLASS) + s.x = n + gcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) + as_num = rffi.cast(lltype.Signed, gcref) + gcref2 = rffi.cast(llmemory.GCREF, as_num) + t = lltype.cast_opaque_ptr(rclass.OBJECTPTR, gcref2) + u = lltype.cast_pointer(lltype.Ptr(SCLASS), t) + return u.x + res = interpret(f, [123]) + assert res == 123 + class TestPlatform(object): def test_lib_on_libpaths(self): from pypy.translator.platform import platform From arigo at codespeak.net Sun Dec 6 20:04:30 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 20:04:30 +0100 (CET) Subject: [pypy-svn] r69933 - in pypy/branch/virtual-forcing/pypy/rpython: . lltypesystem lltypesystem/test Message-ID: <20091206190430.17B4E168012@codespeak.net> Author: arigo Date: Sun Dec 6 20:04:30 2009 New Revision: 69933 Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py pypy/branch/virtual-forcing/pypy/rpython/rtyper.py Log: Tentative test and fix: remove ll2ctypes._parent_cache. Instead, recontruct all the information about the real type of an instance of OBJECT, purely in ctypes2lltype(). This is possible to do, as a small extension of code that is already there. Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py Sun Dec 6 20:04:30 2009 @@ -30,7 +30,6 @@ _ctypes_cache = {} _eci_cache = {} -_parent_cache = {} def _setup_ctypes_cache(): from pypy.rpython.lltypesystem import rffi @@ -254,7 +253,6 @@ convert_struct(field_value, csubstruct) subcontainer = getattr(container, field_name) substorage = subcontainer._storage - update_parent_cache(substorage, subcontainer) elif field_name == STRUCT._arrayfld: # inlined var-sized part csubarray = getattr(cstruct, field_name) convert_array(field_value, csubarray) @@ -313,7 +311,6 @@ struct_storage = getattr(ctypes_storage, field_name) struct_use_ctypes_storage(struct_container, struct_storage) struct_container._setparentstructure(container, field_name) - update_parent_cache(ctypes_storage, struct_container) elif isinstance(FIELDTYPE, lltype.Array): assert FIELDTYPE._hints.get('nolength', False) == False arraycontainer = _array_of_known_length(FIELDTYPE) @@ -633,8 +630,6 @@ raise NotImplementedError(T) container._ctypes_storage_was_allocated() storage = container._storage - if lltype.parentlink(container)[0] is not None: - update_parent_cache(storage, container) p = ctypes.pointer(storage) if index: p = ctypes.cast(p, ctypes.c_void_p) @@ -673,26 +668,25 @@ if not cobj: # NULL pointer return lltype.nullptr(T.TO) if isinstance(T.TO, lltype.Struct): + REAL_TYPE = T.TO if T.TO._arrayfld is not None: carray = getattr(cobj.contents, T.TO._arrayfld) container = lltype._struct(T.TO, carray.length) else: # special treatment of 'OBJECT' subclasses - if get_rtyper() and lltype._castdepth(T.TO, OBJECT) > 0: - ctypes_object = get_ctypes_type(lltype.Ptr(OBJECT)) - as_obj = ctypes2lltype(lltype.Ptr(OBJECT), - ctypes.cast(cobj, ctypes_object)) - TObj = get_rtyper().get_type_for_typeptr(as_obj.typeptr) - if TObj != T.TO: - ctypes_instance = get_ctypes_type(lltype.Ptr(TObj)) - return lltype.cast_pointer(T, - ctypes2lltype(lltype.Ptr(TObj), - ctypes.cast(cobj, ctypes_instance))) - container = lltype._struct(T.TO) + if get_rtyper() and lltype._castdepth(REAL_TYPE, OBJECT) >= 0: + # figure out the real type of the object + containerheader = lltype._struct(OBJECT) + struct_use_ctypes_storage(containerheader, cobj.contents) + REAL_TYPE = get_rtyper().get_type_for_typeptr( + containerheader.typeptr) + REAL_T = lltype.Ptr(REAL_TYPE) + cobj = ctypes.cast(cobj, get_ctypes_type(REAL_T)) + container = lltype._struct(REAL_TYPE) struct_use_ctypes_storage(container, cobj.contents) - addr = ctypes.addressof(cobj.contents) - if addr in _parent_cache: - setparentstructure(container, _parent_cache[addr]) + if REAL_TYPE != T.TO: + p = container._as_ptr() + container = lltype.cast_pointer(T, p)._as_obj() elif isinstance(T.TO, lltype.Array): if T.TO._hints.get('nolength', False): container = _array_of_unknown_length(T.TO) @@ -1139,46 +1133,6 @@ return hop.genop('cast_adr_to_int', [adr], resulttype = lltype.Signed) -# ------------------------------------------------------------ - -def parentchain(container): - current = container - links = [] - while True: - link = lltype.parentlink(current) - if link[0] is None: - try: - addr = ctypes.addressof(container._storage) - actual = _parent_cache[addr] - if len(links) < len(actual): - return actual - except KeyError: - pass - return links - links.append(link) - current = link[0] - -def update_parent_cache(storage, container): - chain = parentchain(container) - addr = ctypes.addressof(storage) - try: - current = _parent_cache[addr] - if len(chain) > len(current): - _parent_cache[addr] = chain - except KeyError: - _parent_cache[addr] = chain - -def setparentstructure(container, chain): - TP = lltype.typeOf(container) - current = container - for i, elem in enumerate(chain): - if lltype.typeOf(elem[0]) == TP: - chain = chain[i + 1:] - break - for elem in chain: - current._setparentstructure(*elem) - current = elem[0] - # ____________________________________________________________ # errno Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sun Dec 6 20:04:30 2009 @@ -1109,17 +1109,18 @@ def test_object_subclass(self): from pypy.rpython.lltypesystem import rclass - SCLASS = lltype.GcStruct('SCLASS', - ('parent', rclass.OBJECT), - ('x', lltype.Signed)) + from pypy.rpython.annlowlevel import cast_instance_to_base_ptr + from pypy.rpython.annlowlevel import cast_base_ptr_to_instance + class S: + pass def f(n): - s = lltype.malloc(SCLASS) + s = S() s.x = n - gcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - as_num = rffi.cast(lltype.Signed, gcref) - gcref2 = rffi.cast(llmemory.GCREF, as_num) - t = lltype.cast_opaque_ptr(rclass.OBJECTPTR, gcref2) - u = lltype.cast_pointer(lltype.Ptr(SCLASS), t) + ls = cast_instance_to_base_ptr(s) + as_num = rffi.cast(lltype.Signed, ls) + # --- around this point, only 'as_num' is passed + t = rffi.cast(rclass.OBJECTPTR, as_num) + u = cast_base_ptr_to_instance(S, t) return u.x res = interpret(f, [123]) assert res == 123 Modified: pypy/branch/virtual-forcing/pypy/rpython/rtyper.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/rtyper.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/rtyper.py Sun Dec 6 20:04:30 2009 @@ -133,15 +133,22 @@ return result def get_type_for_typeptr(self, typeptr): + search = typeptr._obj try: - return self.type_for_typeptr[typeptr._obj] + return self.type_for_typeptr[search] except KeyError: - # rehash the dictionary + # rehash the dictionary, and perform a non-dictionary scan + # for the case of ll2ctypes typeptr + found = None type_for_typeptr = {} for key, value in self.type_for_typeptr.items(): type_for_typeptr[key] = value + if key == search: + found = value self.type_for_typeptr = type_for_typeptr - return self.type_for_typeptr[typeptr._obj] + if found is None: + raise KeyError(search) + return found def makekey(self, s_obj): return pair(self.type_system, s_obj).rtyper_makekey(self) From arigo at codespeak.net Sun Dec 6 20:34:27 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 20:34:27 +0100 (CET) Subject: [pypy-svn] r69934 - in pypy/branch/virtual-forcing/pypy/rpython: . lltypesystem lltypesystem/test Message-ID: <20091206193427.8C7D1168020@codespeak.net> Author: arigo Date: Sun Dec 6 20:34:26 2009 New Revision: 69934 Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rclass.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py pypy/branch/virtual-forcing/pypy/rpython/rtyper.py Log: Found a bug. Writing a test. Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py Sun Dec 6 20:34:26 2009 @@ -677,7 +677,10 @@ if get_rtyper() and lltype._castdepth(REAL_TYPE, OBJECT) >= 0: # figure out the real type of the object containerheader = lltype._struct(OBJECT) - struct_use_ctypes_storage(containerheader, cobj.contents) + cobjheader = ctypes.cast(cobj, + get_ctypes_type(lltype.Ptr(OBJECT))) + struct_use_ctypes_storage(containerheader, + cobjheader.contents) REAL_TYPE = get_rtyper().get_type_for_typeptr( containerheader.typeptr) REAL_T = lltype.Ptr(REAL_TYPE) Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rclass.py Sun Dec 6 20:34:26 2009 @@ -383,7 +383,7 @@ ll_runtime_type_info, OBJECT, destrptr) vtable = self.rclass.getvtable() - self.rtyper.type_for_typeptr[vtable._obj] = self.lowleveltype.TO + self.rtyper.set_type_for_typeptr(vtable, self.lowleveltype.TO) self.rtyper.lltype2vtable[self.lowleveltype.TO] = vtable def common_repr(self): # -> object or nongcobject reprs @@ -693,3 +693,23 @@ break raise AttributeError("%s has no field %s" % (lltype.typeOf(widest), name)) + +def declare_type_for_typeptr(vtable, TYPE): + """Hack for custom low-level-only 'subclasses' of OBJECT: + call this somewhere annotated, in order to declare that it is + of the given TYPE and has got the corresponding vtable.""" + +class Entry(ExtRegistryEntry): + _about_ = declare_type_for_typeptr + def compute_result_annotation(self, s_vtable, s_TYPE): + assert s_vtable.is_constant() + assert s_TYPE.is_constant() + return annmodel.s_None + def specialize_call(self, hop): + vtable = hop.args_v[0].value + TYPE = hop.args_v[1].value + assert lltype.typeOf(vtable) == CLASSTYPE + assert isinstance(TYPE, GcStruct) + assert lltype._castdepth(TYPE, OBJECT) > 0 + hop.rtyper.set_type_for_typeptr(vtable, TYPE) + return hop.inputconst(lltype.Void, None) Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sun Dec 6 20:34:26 2009 @@ -1125,6 +1125,26 @@ res = interpret(f, [123]) assert res == 123 + def test_object_subclass_2(self): + from pypy.rpython.lltypesystem import rclass + SCLASS = lltype.GcStruct('SCLASS', + ('parent', rclass.OBJECT), + ('n', lltype.Signed)) + sclass_vtable = lltype.malloc(rclass.OBJECT_VTABLE, zero=True, + immortal=True) + sclass_vtable.name = rclass.alloc_array_name('SClass') + def f(n): + rclass.declare_type_for_typeptr(sclass_vtable, SCLASS) + s = lltype.malloc(SCLASS) + s.parent.typeptr = sclass_vtable + s.n = n + as_num = rffi.cast(lltype.Signed, s) + # --- around this point, only 'as_num' is passed + t = rffi.cast(lltype.Ptr(SCLASS), as_num) + return t.n + res = interpret(f, [123]) + assert res == 123 + class TestPlatform(object): def test_lib_on_libpaths(self): from pypy.translator.platform import platform Modified: pypy/branch/virtual-forcing/pypy/rpython/rtyper.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/rtyper.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/rtyper.py Sun Dec 6 20:34:26 2009 @@ -137,7 +137,7 @@ try: return self.type_for_typeptr[search] except KeyError: - # rehash the dictionary, and perform a non-dictionary scan + # rehash the dictionary, and perform a linear scan # for the case of ll2ctypes typeptr found = None type_for_typeptr = {} @@ -150,6 +150,9 @@ raise KeyError(search) return found + def set_type_for_typeptr(self, typeptr, TYPE): + self.type_for_typeptr[typeptr._obj] = TYPE + def makekey(self, s_obj): return pair(self.type_system, s_obj).rtyper_makekey(self) From arigo at codespeak.net Sun Dec 6 20:44:15 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 20:44:15 +0100 (CET) Subject: [pypy-svn] r69935 - pypy/branch/virtual-forcing/pypy/jit/metainterp Message-ID: <20091206194415.446A6168020@codespeak.net> Author: arigo Date: Sun Dec 6 20:44:14 2009 New Revision: 69935 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py Log: We must call set_type_for_typeptr() explicitly. That's enough to make the x86 test_virtual_ref passes. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py Sun Dec 6 20:44:14 2009 @@ -23,6 +23,10 @@ if c_funcptr is not None: log("replaced %d 'jit_force_virtual' with %r" % (count, c_funcptr.value)) + # + # record the type JIT_VIRTUAL_REF explicitly in the rtyper, too + warmrunnerdesc.rtyper.set_type_for_typeptr(jit_virtual_ref_vtable, + JIT_VIRTUAL_REF) # ____________________________________________________________ From arigo at codespeak.net Sun Dec 6 21:08:22 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 21:08:22 +0100 (CET) Subject: [pypy-svn] r69936 - pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test Message-ID: <20091206200822.9E3E616801B@codespeak.net> Author: arigo Date: Sun Dec 6 21:08:19 2009 New Revision: 69936 Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: Two extra passing tests. Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sun Dec 6 21:08:19 2009 @@ -1145,6 +1145,46 @@ res = interpret(f, [123]) assert res == 123 + def test_object_subclass_3(self): + from pypy.rpython.lltypesystem import rclass + from pypy.rpython.annlowlevel import cast_instance_to_base_ptr + from pypy.rpython.annlowlevel import cast_base_ptr_to_instance + class S: + pass + def f(n): + s = S() + s.x = n + ls = cast_instance_to_base_ptr(s) + as_num = rffi.cast(lltype.Signed, ls) + # --- around this point, only 'as_num' is passed + r = rffi.cast(llmemory.GCREF, as_num) + t = lltype.cast_opaque_ptr(rclass.OBJECTPTR, r) + u = cast_base_ptr_to_instance(S, t) + return u.x + res = interpret(f, [123]) + assert res == 123 + + def test_object_subclass_4(self): + from pypy.rpython.lltypesystem import rclass + SCLASS = lltype.GcStruct('SCLASS', + ('parent', rclass.OBJECT), + ('n', lltype.Signed)) + sclass_vtable = lltype.malloc(rclass.OBJECT_VTABLE, zero=True, + immortal=True) + sclass_vtable.name = rclass.alloc_array_name('SClass') + def f(n): + rclass.declare_type_for_typeptr(sclass_vtable, SCLASS) + s = lltype.malloc(SCLASS) + s.parent.typeptr = sclass_vtable + s.n = n + as_num = rffi.cast(lltype.Signed, s) + # --- around this point, only 'as_num' is passed + r = rffi.cast(llmemory.GCREF, as_num) + t = lltype.cast_opaque_ptr(lltype.Ptr(SCLASS), r) + return t.n + res = interpret(f, [123]) + assert res == 123 + class TestPlatform(object): def test_lib_on_libpaths(self): from pypy.translator.platform import platform From arigo at codespeak.net Sun Dec 6 21:27:48 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 21:27:48 +0100 (CET) Subject: [pypy-svn] r69937 - in pypy/branch/virtual-forcing/pypy/rpython: . lltypesystem lltypesystem/test Message-ID: <20091206202748.5C6E316801B@codespeak.net> Author: arigo Date: Sun Dec 6 21:27:45 2009 New Revision: 69937 Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py pypy/branch/virtual-forcing/pypy/rpython/rtyper.py Log: Special treatment of vtables, in addition to special treatment of objects. Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py Sun Dec 6 21:27:45 2009 @@ -21,7 +21,7 @@ from pypy.rlib.rarithmetic import r_uint, r_singlefloat, intmask from pypy.annotation import model as annmodel from pypy.rpython.llinterp import LLInterpreter, LLException -from pypy.rpython.lltypesystem.rclass import OBJECT +from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE from pypy.rpython import raddress from pypy.translator.platform import platform @@ -690,6 +690,14 @@ if REAL_TYPE != T.TO: p = container._as_ptr() container = lltype.cast_pointer(T, p)._as_obj() + # special treatment of 'OBJECT_VTABLE' subclasses + if get_rtyper() and lltype._castdepth(REAL_TYPE, + OBJECT_VTABLE) >= 0: + # figure out the real object that this vtable points to, + # and just return that + p = get_rtyper().get_real_typeptr_for_typeptr( + container._as_ptr()) + container = lltype.cast_pointer(T, p)._as_obj() elif isinstance(T.TO, lltype.Array): if T.TO._hints.get('nolength', False): container = _array_of_unknown_length(T.TO) Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sun Dec 6 21:27:45 2009 @@ -1185,6 +1185,25 @@ res = interpret(f, [123]) assert res == 123 + def test_object_subclass_5(self): + from pypy.rpython.lltypesystem import rclass + from pypy.rpython.annlowlevel import cast_instance_to_base_ptr + from pypy.rpython.annlowlevel import cast_base_ptr_to_instance + class S: + x = 5 # entry in the vtable + class T(S): + x = 6 + def f(): + s = T() + ls = cast_instance_to_base_ptr(s) + as_num = rffi.cast(lltype.Signed, ls) + # --- around this point, only 'as_num' is passed + t = rffi.cast(rclass.OBJECTPTR, as_num) + u = cast_base_ptr_to_instance(S, t) + return u.x + res = interpret(f, []) + assert res == 6 + class TestPlatform(object): def test_lib_on_libpaths(self): from pypy.translator.platform import platform Modified: pypy/branch/virtual-forcing/pypy/rpython/rtyper.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/rtyper.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/rtyper.py Sun Dec 6 21:27:45 2009 @@ -153,6 +153,14 @@ def set_type_for_typeptr(self, typeptr, TYPE): self.type_for_typeptr[typeptr._obj] = TYPE + def get_real_typeptr_for_typeptr(self, typeptr): + # perform a linear scan for the case of ll2ctypes typeptr + search = typeptr._obj + for key, value in self.type_for_typeptr.items(): + if key == search: + return key._as_ptr() + raise KeyError(search) + def makekey(self, s_obj): return pair(self.type_system, s_obj).rtyper_makekey(self) From arigo at codespeak.net Sun Dec 6 22:32:31 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 22:32:31 +0100 (CET) Subject: [pypy-svn] r69938 - in pypy/branch/virtual-forcing/pypy/jit/metainterp: . test Message-ID: <20091206213231.3B26C168011@codespeak.net> Author: arigo Date: Sun Dec 6 22:32:29 2009 New Revision: 69938 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Log: Test when forcing occurs on a bridge. Works for virtualizables, but not yet for virtualrefs. Changes in the code are no-ops. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Sun Dec 6 22:32:29 2009 @@ -1836,18 +1836,20 @@ self.virtualref_boxes = virtualref_boxes if expect_virtualizable: self.virtualizable_boxes = virtualizable_boxes - if self._already_allocated_resume_virtuals is not None: - # resuming from a ResumeGuardForcedDescr: load the new values - # currently stored on the virtualizable fields - self.load_fields_from_virtualizable() - return # just jumped away from assembler (case 4 in the comment in # virtualizable.py) into tracing (case 2); check that vable_token - # is and stays 0. + # is and stays 0. Note the call to reset_vable_token() in + # warmstate.py. virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) assert not virtualizable.vable_token - self.synchronize_virtualizable() + if self._already_allocated_resume_virtuals is not None: + # resuming from a ResumeGuardForcedDescr: load the new values + # currently stored on the virtualizable fields + self.load_fields_from_virtualizable() + else: + # normal case: fill the virtualizable with the local boxes + self.synchronize_virtualizable() def check_synchronized_virtualizable(self): if not we_are_translated(): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Sun Dec 6 22:32:29 2009 @@ -907,6 +907,39 @@ res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) assert res == f(123) + def test_bridge_forces(self): + jitdriver = JitDriver(greens = [], reds = ['frame'], + virtualizables = ['frame']) + + class Frame(object): + _virtualizable2_ = ['x', 'y'] + class SomewhereElse: + pass + somewhere_else = SomewhereElse() + + def g(): + n = somewhere_else.top_frame.y + 700 + debug_print(lltype.Void, '-+-+-+-+- external write:', n) + somewhere_else.top_frame.y = n + + def f(n): + frame = Frame() + frame.x = n + frame.y = 10 + somewhere_else.counter = 0 + somewhere_else.top_frame = frame + while frame.x > 0: + jitdriver.can_enter_jit(frame=frame) + jitdriver.jit_merge_point(frame=frame) + if frame.y > 17: + g() + frame.x -= 5 + frame.y += 1 + return frame.y + + res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) + assert res == f(123) + def test_promote_index_in_virtualizable_list(self): jitdriver = JitDriver(greens = [], reds = ['frame', 'n'], virtualizables = ['frame']) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Sun Dec 6 22:32:29 2009 @@ -193,6 +193,38 @@ res = self.meta_interp(f, [30]) assert res == 13 + def test_bridge_forces(self): + py.test.skip("in-progress") + myjitdriver = JitDriver(greens = [], reds = ['n']) + # + class XY: + pass + class ExCtx: + pass + exctx = ExCtx() + # + @dont_look_inside + def externalfn(n): + exctx.m = exctx.topframeref().n + return 1 + # + def f(n): + while n > 0: + myjitdriver.can_enter_jit(n=n) + myjitdriver.jit_merge_point(n=n) + xy = XY() + xy.n = n + exctx.topframeref = virtual_ref(xy) + if n == 13: + externalfn(n) + n -= 1 + virtual_ref_finish(exctx.topframeref) + exctx.topframeref = None + return exctx.m + # + res = self.meta_interp(f, [30]) + assert res == 13 + class TestLLtype(VRefTests, LLJitMixin): pass From arigo at codespeak.net Sun Dec 6 22:43:35 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 22:43:35 +0100 (CET) Subject: [pypy-svn] r69939 - in pypy/branch/virtual-forcing/pypy/jit/metainterp: . test Message-ID: <20091206214335.EAB15168011@codespeak.net> Author: arigo Date: Sun Dec 6 22:43:35 2009 New Revision: 69939 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py Log: Fix test_bridge_forces. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Sun Dec 6 22:43:35 2009 @@ -1227,7 +1227,6 @@ self.cpu = staticdata.cpu self.portal_trace_positions = [] self.greenkey_of_huge_function = None - self.virtualref_boxes = [] def is_blackholing(self): return self.history is None @@ -1715,6 +1714,7 @@ f = self.newframe(self.staticdata.portal_code) f.pc = 0 f.env = original_boxes[:] + self.virtualref_boxes = [] self.initialize_virtualizable(original_boxes) return original_boxes @@ -1833,7 +1833,17 @@ expect_virtualizable = vinfo is not None virtualizable_boxes, virtualref_boxes = resume.rebuild_from_resumedata( self, newboxes, resumedescr, expect_virtualizable) + # + # virtual refs: make the vrefs point to the freshly allocated virtuals self.virtualref_boxes = virtualref_boxes + for i in range(0, len(virtualref_boxes), 2): + virtualbox = virtualref_boxes[i] + vrefbox = virtualref_boxes[i+1] + virtualref.continue_tracing(vrefbox.getref_base(), + virtualbox.getref_base()) + # + # virtualizable: synchronize the real virtualizable and the local + # boxes, in whichever direction is appropriate if expect_virtualizable: self.virtualizable_boxes = virtualizable_boxes # just jumped away from assembler (case 4 in the comment in Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Sun Dec 6 22:43:35 2009 @@ -66,6 +66,7 @@ bxs2 = [box for box in guard_op.fail_args if str(box._getrepr_()).endswith('JitVirtualRef')] assert len(bxs2) == 1 + bxs2[0].getref(lltype.Ptr(JIT_VIRTUAL_REF)).virtual_token = 1234567 # self.metainterp.rebuild_state_after_failure(guard_op.descr, guard_op.fail_args[:]) @@ -194,7 +195,6 @@ assert res == 13 def test_bridge_forces(self): - py.test.skip("in-progress") myjitdriver = JitDriver(greens = [], reds = ['n']) # class XY: Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py Sun Dec 6 22:43:35 2009 @@ -87,6 +87,12 @@ vref.forced = lltype.cast_opaque_ptr(rclass.OBJECTPTR, real_object) vref.virtual_token = TOKEN_NONE +def continue_tracing(gcref, real_object): + vref = lltype.cast_opaque_ptr(lltype.Ptr(JIT_VIRTUAL_REF), gcref) + assert vref.virtual_token != TOKEN_TRACING + vref.forced = lltype.cast_opaque_ptr(rclass.OBJECTPTR, real_object) + vref.virtual_token = TOKEN_TRACING + # ____________________________________________________________ def get_force_virtual_fnptr(warmrunnerdesc): From arigo at codespeak.net Sun Dec 6 22:48:51 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 22:48:51 +0100 (CET) Subject: [pypy-svn] r69940 - pypy/branch/virtual-forcing/pypy/jit/backend/llgraph Message-ID: <20091206214851.97C01168011@codespeak.net> Author: arigo Date: Sun Dec 6 22:48:50 2009 New Revision: 69940 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py Log: Fix. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py Sun Dec 6 22:48:50 2009 @@ -152,7 +152,8 @@ 'debug_merge_point': (('ref',), None), 'force_token' : ((), 'int'), 'call_may_force' : (('int', 'varargs'), 'intorptr'), - 'guard_not_forced': ((), None) + 'guard_not_forced': ((), None), + 'virtual_ref' : (('ref', 'int'), 'ref'), #'getitem' : (('void', 'ref', 'int'), 'int'), #'setitem' : (('void', 'ref', 'int', 'int'), None), #'newlist' : (('void', 'varargs'), 'ref'), @@ -813,6 +814,9 @@ if forced: raise GuardFailed + def op_virtual_ref(self, _, virtual, index): + return virtual + class OOFrame(Frame): From arigo at codespeak.net Sun Dec 6 22:54:20 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 6 Dec 2009 22:54:20 +0100 (CET) Subject: [pypy-svn] r69941 - pypy/branch/virtual-forcing/pypy/jit/backend/x86/test Message-ID: <20091206215420.EA7BC168011@codespeak.net> Author: arigo Date: Sun Dec 6 22:54:20 2009 New Revision: 69941 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/test/test_tlc.py Log: At least the test inherited from metainterp passes. I did not investigate why this file adds new tests, and why they fail. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/test/test_tlc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/test/test_tlc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/test/test_tlc.py Sun Dec 6 22:54:20 2009 @@ -1,6 +1,5 @@ import py -py.test.skip("Widening to trash error") from pypy.jit.backend.x86.test.test_basic import Jit386Mixin from pypy.jit.metainterp.test.test_tlc import TLCTests from pypy.jit.tl import tlc @@ -10,6 +9,7 @@ # ====> ../../test/test_tlc.py def test_accumulator(self): + py.test.skip("investigate, maybe") path = py.path.local(tlc.__file__).dirpath('accumulator.tlc.src') code = path.read() res = self.exec_code(code, 20) @@ -18,6 +18,7 @@ assert res == 10 def test_fib(self): + py.test.skip("investigate, maybe") path = py.path.local(tlc.__file__).dirpath('fibo.tlc.src') code = path.read() res = self.exec_code(code, 7) From cfbolz at codespeak.net Mon Dec 7 15:36:24 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 7 Dec 2009 15:36:24 +0100 (CET) Subject: [pypy-svn] r69946 - pypy/trunk/pypy/objspace/std Message-ID: <20091207143624.21826168020@codespeak.net> Author: cfbolz Date: Mon Dec 7 15:36:22 2009 New Revision: 69946 Modified: pypy/trunk/pypy/objspace/std/inlinedict.py Log: make comment sound less dangerous Modified: pypy/trunk/pypy/objspace/std/inlinedict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/inlinedict.py (original) +++ pypy/trunk/pypy/objspace/std/inlinedict.py Mon Dec 7 15:36:22 2009 @@ -96,7 +96,9 @@ def setdictvalue(self, space, attr, w_value, shadows_type=True): if self._inlined_dict_valid(): - # XXX don't ignore shadows_type + # XXX so far we ignore shadows_type, which is a small + # performance-degradation if the JIT is not used (i.e. shadow + # tracking does not work). Maybe we don't care. self.impl_setitem_str(attr, w_value) return True w_dict = self.getdict() From afa at codespeak.net Mon Dec 7 15:46:29 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 7 Dec 2009 15:46:29 +0100 (CET) Subject: [pypy-svn] r69947 - in pypy/trunk/pypy: interpreter module/__builtin__ module/__builtin__/test module/_demo module/_demo/test module/oracle Message-ID: <20091207144629.8CE28168020@codespeak.net> Author: afa Date: Mon Dec 7 15:46:29 2009 New Revision: 69947 Added: pypy/trunk/pypy/module/_demo/test/ (props changed) - copied from r69901, pypy/trunk/pypy/module/_demo/test/ pypy/trunk/pypy/module/_demo/test/test_import.py - copied unchanged from r69901, pypy/trunk/pypy/module/_demo/test/test_import.py Modified: pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/interpreter/mixedmodule.py pypy/trunk/pypy/interpreter/module.py pypy/trunk/pypy/module/__builtin__/importing.py pypy/trunk/pypy/module/__builtin__/test/test_import.py pypy/trunk/pypy/module/_demo/__init__.py pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_error.py Log: Change the way built-in modules are initialized and imported: (2nd attempt) - All built-in modules are inserted into space.builtin_modules when they are discovered, but not in sys.modules. - The end of space initialization calls setup_builtin_modules(), which calls mod.setup_after_space_initialization() for every module. At this stage, sys.modules contains the "sys" module, and other modules imported by mod.setup_after_space_initialization(). - The space and modules are frozen in this state, and translated. - When pypy-c starts, it calls mod.startup() on built-in modules already in sys.modules. - when importing another builtin module, pypy fetches it from space.builtin_modules, adds it to sys.modules, and calls its setup() method. space.getbuiltinmodule(name) is still the main interface to import-or-get a built-in module. This speeds up pypy startup, but more importantly, solves bootstraping issues where for example a higher-level module would "import posix" before it is fully initialized; when this happens, os.environ is empty! Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Mon Dec 7 15:46:29 2009 @@ -239,6 +239,8 @@ config = get_pypy_config(translating=False) self.config = config + self.builtin_modules = {} + # import extra modules for side-effects import pypy.interpreter.nestedscope # register *_DEREF bytecodes @@ -266,28 +268,34 @@ def startup(self): # To be called before using the space - # Initialize all builtin modules + # Initialize already imported builtin modules from pypy.interpreter.module import Module + w_modules = self.sys.get('modules') for w_modname in self.unpackiterable( self.sys.get('builtin_module_names')): + try: + w_mod = self.getitem(w_modules, w_modname) + except OperationError, e: + if e.match(self, self.w_KeyError): + continue + raise modname = self.str_w(w_modname) - mod = self.interpclass_w(self.getbuiltinmodule(modname)) - if isinstance(mod, Module): - import time + mod = self.interpclass_w(w_mod) + if isinstance(mod, Module) and not mod.startup_called: self.timer.start("startup " + modname) mod.startup(self) + mod.startup_called = True self.timer.stop("startup " + modname) + def finish(self): w_exitfunc = self.sys.getdictvalue(self, 'exitfunc') if w_exitfunc is not None: self.call_function(w_exitfunc) from pypy.interpreter.module import Module - for w_modname in self.unpackiterable( - self.sys.get('builtin_module_names')): - modname = self.str_w(w_modname) - mod = self.interpclass_w(self.getbuiltinmodule(modname)) - if isinstance(mod, Module): + for w_mod in self.builtin_modules.values(): + mod = self.interpclass_w(w_mod) + if isinstance(mod, Module) and mod.startup_called: mod.shutdown(self) if self.config.objspace.std.withdictmeasurement: from pypy.objspace.std.dictmultiobject import report @@ -339,14 +347,37 @@ w_name = self.wrap(name) w_mod = self.wrap(Module(self, w_name)) - w_modules = self.sys.get('modules') - self.setitem(w_modules, w_name, w_mod) + self.builtin_modules[name] = w_mod return name def getbuiltinmodule(self, name): w_name = self.wrap(name) w_modules = self.sys.get('modules') - return self.getitem(w_modules, w_name) + try: + return self.getitem(w_modules, w_name) + except OperationError, e: + if not e.match(self, self.w_KeyError): + raise + + # If the module is a builtin but not yet imported, + # retrieve it and initialize it + try: + w_mod = self.builtin_modules[name] + except KeyError: + raise e + else: + # Add the module to sys.modules + self.setitem(w_modules, w_name, w_mod) + + # And initialize it + from pypy.interpreter.module import Module + mod = self.interpclass_w(w_mod) + if isinstance(mod, Module) and not mod.startup_called: + self.timer.start("startup " + name) + mod.startup(self) + mod.startup_called = True + self.timer.stop("startup " + name) + return w_mod def get_builtinmodule_to_install(self): """NOT_RPYTHON""" @@ -397,16 +428,15 @@ w_name = self.wrap('sys') self.sys = Module(self, w_name) w_modules = self.sys.get('modules') - self.setitem(w_modules, w_name, self.wrap(self.sys)) + self.builtin_modules['sys'] = self.wrap(self.sys) - self.setitem(w_modules, w_name_exceptions, - self.wrap(self.exceptions_module)) + self.builtin_modules['exceptions'] = self.wrap(self.exceptions_module) from pypy.module.__builtin__ import Module w_name = self.wrap('__builtin__') self.builtin = Module(self, w_name) w_builtin = self.wrap(self.builtin) - self.setitem(w_modules, w_name, w_builtin) + self.builtin_modules['__builtin__'] = self.wrap(w_builtin) self.setitem(self.builtin.w_dict, self.wrap('__builtins__'), w_builtin) bootstrap_modules = ['sys', '__builtin__', 'exceptions'] @@ -480,12 +510,9 @@ def setup_builtin_modules(self): "NOT_RPYTHON: only for initializing the space." - from pypy.interpreter.module import Module - for w_modname in self.unpackiterable(self.sys.get('builtin_module_names')): - modname = self.unwrap(w_modname) - mod = self.getbuiltinmodule(modname) - if isinstance(mod, Module): - mod.setup_after_space_initialization() + self.getbuiltinmodule('sys') + for mod in self.builtin_modules.values(): + mod.setup_after_space_initialization() def initialize(self): """NOT_RPYTHON: Abstract method that should put some minimal Modified: pypy/trunk/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/trunk/pypy/interpreter/mixedmodule.py (original) +++ pypy/trunk/pypy/interpreter/mixedmodule.py Mon Dec 7 15:46:29 2009 @@ -87,6 +87,7 @@ def _freeze_(self): self.getdict() + self.startup_called = False # hint for the annotator: Modules can hold state, so they are # not constant return False Modified: pypy/trunk/pypy/interpreter/module.py ============================================================================== --- pypy/trunk/pypy/interpreter/module.py (original) +++ pypy/trunk/pypy/interpreter/module.py Mon Dec 7 15:46:29 2009 @@ -16,21 +16,22 @@ self.w_name = w_name if w_name is not None: space.setitem(w_dict, space.new_interned_str('__name__'), w_name) + self.startup_called = False def setup_after_space_initialization(self): """NOT_RPYTHON: to allow built-in modules to do some more setup after the space is fully initialized.""" def startup(self, space): - """This is called at runtime before the space gets uses to allow - the module to do initialization at runtime. + """This is called at runtime on import to allow the module to + do initialization when it is imported for the first time. """ def shutdown(self, space): """This is called when the space is shut down, just after - sys.exitfunc(). + sys.exitfunc(), if the module has been imported. """ - + def getdict(self): return self.w_dict Modified: pypy/trunk/pypy/module/__builtin__/importing.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/importing.py (original) +++ pypy/trunk/pypy/module/__builtin__/importing.py Mon Dec 7 15:46:29 2009 @@ -228,7 +228,11 @@ def _absolute_import(space, modulename, baselevel, w_fromlist, tentative): w = space.wrap - + + # check the builtin modules + if modulename in space.builtin_modules: + return space.getbuiltinmodule(modulename) + w_mod = None parts = modulename.split('.') prefix = [] Modified: pypy/trunk/pypy/module/__builtin__/test/test_import.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/test_import.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/test_import.py Mon Dec 7 15:46:29 2009 @@ -41,7 +41,9 @@ relative_a = "import a", abs_b = "import b", abs_x_y = "import x.y", + abs_sys = "import sys", string = "inpackage = 1", + errno = "", absolute = "from __future__ import absolute_import\nimport string", relative_b = "from __future__ import absolute_import\nfrom . import string", relative_c = "from __future__ import absolute_import\nfrom .string import inpackage", @@ -251,6 +253,17 @@ import pkg_r.inpkg raises(ImportError,imp) + def test_import_builtin_inpackage(self): + def imp(): + import pkg.sys + raises(ImportError,imp) + + import sys, pkg.abs_sys + assert pkg.abs_sys.sys is sys + + import errno, pkg.errno + assert pkg.errno is not errno + def test_import_Globals_Are_None(self): import sys m = __import__('sys') Modified: pypy/trunk/pypy/module/_demo/__init__.py ============================================================================== --- pypy/trunk/pypy/module/_demo/__init__.py (original) +++ pypy/trunk/pypy/module/_demo/__init__.py Mon Dec 7 15:46:29 2009 @@ -12,3 +12,13 @@ appleveldefs = { 'DemoError' : 'app_demo.DemoError', } + + # Used in tests + demo_events = [] + def setup_after_space_initialization(self): + Module.demo_events.append('setup') + def startup(self, space): + Module.demo_events.append('startup') + def shutdown(self, space): + Module.demo_events.append('shutdown') + Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Mon Dec 7 15:46:29 2009 @@ -39,6 +39,7 @@ def startup(self, space): from pypy.module.oracle.interp_error import get state = get(space) + state.startup(space) (state.w_DecimalType, state.w_DateTimeType, state.w_DateType, state.w_TimedeltaType, ) = space.fixedview(space.appexec([], """(): Modified: pypy/trunk/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_error.py (original) +++ pypy/trunk/pypy/module/oracle/interp_error.py Mon Dec 7 15:46:29 2009 @@ -4,32 +4,47 @@ from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError + from pypy.module.oracle import roci, config +from pypy.rlib.unroll import unrolling_iterable + +exported_names = unrolling_iterable(""" + DatabaseError OperationalError InterfaceError ProgrammingError + NotSupportedError IntegrityError InternalError DataError + Variable Connection""".split()) class State: # XXX move to another file + def __init__(self, space): - w_module = space.getbuiltinmodule('cx_Oracle') - def get(name): - return space.getattr(w_module, space.wrap(name)) + "NOT_RPYTHON" + self.variableTypeByPythonType = {} + self.w_DecimalType = None + self.w_DateTimeType = None + self.w_DateType = None + self.w_TimedeltaType = None - self.w_DatabaseError = get('DatabaseError') - self.w_OperationalError = get('OperationalError') - self.w_InterfaceError = get('InterfaceError') - self.w_ProgrammingError = get('ProgrammingError') - self.w_NotSupportedError = get('NotSupportedError') - self.w_IntegrityError = get('IntegrityError') - self.w_InternalError = get('InternalError') - self.w_DataError = get('DataError') - self.w_Variable = get('Variable') - self.w_Connection = get('Connection') + for name in exported_names: + setattr(self, 'w_' + name, None) + + def startup(self, space): + w_module = space.getbuiltinmodule('cx_Oracle') + for name in exported_names: + setattr(self, 'w_' + name, space.getattr(w_module, space.wrap(name))) from pypy.module.oracle.interp_variable import all_variable_types - self.variableTypeByPythonType = {} for varType in all_variable_types: w_type = space.gettypeobject(varType.typedef) self.variableTypeByPythonType[w_type] = varType + (self.w_DecimalType, + self.w_DateTimeType, self.w_DateType, self.w_TimedeltaType, + ) = space.fixedview(space.appexec([], """(): + import decimal, datetime + return (decimal.Decimal, + datetime.datetime, datetime.date, datetime.timedelta) + """)) + def get(space): return space.fromcache(State) From fijal at codespeak.net Mon Dec 7 16:16:45 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 7 Dec 2009 16:16:45 +0100 (CET) Subject: [pypy-svn] r69949 - in pypy: lang/js trunk/pypy/translator/goal Message-ID: <20091207151645.4DED9168027@codespeak.net> Author: fijal Date: Mon Dec 7 16:16:44 2009 New Revision: 69949 Added: pypy/lang/js/targetjsstandalone.py - copied unchanged from r69948, pypy/trunk/pypy/translator/goal/targetjsstandalone.py Removed: pypy/trunk/pypy/translator/goal/targetjsstandalone.py Log: Move targetjs into lang/js From fijal at codespeak.net Mon Dec 7 16:17:34 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 7 Dec 2009 16:17:34 +0100 (CET) Subject: [pypy-svn] r69950 - pypy/trunk/pypy/lang/js Message-ID: <20091207151734.6439B168027@codespeak.net> Author: fijal Date: Mon Dec 7 16:17:33 2009 New Revision: 69950 Removed: pypy/trunk/pypy/lang/js/ Log: Remove js directory from here From arigo at codespeak.net Mon Dec 7 16:22:58 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 7 Dec 2009 16:22:58 +0100 (CET) Subject: [pypy-svn] r69951 - in pypy/branch/virtual-forcing/pypy/rlib: . test Message-ID: <20091207152258.BCF34168027@codespeak.net> Author: arigo Date: Mon Dec 7 16:22:58 2009 New Revision: 69951 Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py Log: Write docstrings for the functions virtual_ref and virtual_ref_finish. Fix the tests according to the docstrings. Note that this behavior would be more relaxed than what 'metainterp' currently implements, but I think it's possible and a good idea for the pypy interpreter. Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/jit.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/jit.py Mon Dec 7 16:22:58 2009 @@ -1,7 +1,7 @@ import py import sys from pypy.rpython.extregistry import ExtRegistryEntry -from pypy.rlib.objectmodel import CDefinedIntSymbolic, we_are_translated +from pypy.rlib.objectmodel import CDefinedIntSymbolic from pypy.rlib.objectmodel import keepalive_until_here from pypy.rlib.unroll import unrolling_iterable @@ -103,22 +103,30 @@ # VRefs def virtual_ref(x): + """Creates a 'vref' object that contains a reference to 'x'. Calls + to virtual_ref/virtual_ref_finish must be properly nested. The idea + is that the object 'x' is supposed to be JITted as a virtual (the + JIT will abort if it is not), at least between the calls to + virtual_ref and virtual_ref_finish. The point is that the 'vref' + returned by virtual_ref may escape early. If at runtime it is + dereferenced (by calling it, as in 'vref()') before the + virtual_ref_finish, then we get out of the assembler. If it is not + dereferenced at all, or only after the virtual_ref_finish, then + nothing special occurs. + """ return DirectVRef(x) virtual_ref.oopspec = 'virtual_ref(x)' def virtual_ref_finish(x): - if not we_are_translated(): - x._forced = x._forced or -1 + """See docstring in virtual_ref(x). Note that virtual_ref_finish + takes as argument the real object, not the vref.""" keepalive_until_here(x) # otherwise the whole function call is removed virtual_ref_finish.oopspec = 'virtual_ref_finish(x)' class DirectVRef(object): - _forced = 0 def __init__(self, x): self._x = x def __call__(self): - assert self._forced >= 0, "too late to force the virtual_ref!" - self._forced = 1 return self._x def _freeze_(self): raise Exception("should not see a prebuilt virtual_ref") Modified: pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py Mon Dec 7 16:22:58 2009 @@ -18,20 +18,17 @@ pass -def test_direct(): +def test_direct_1(): x1 = X() vref = virtual_ref(x1) assert vref() is x1 + virtual_ref_finish(x1) + assert vref() is x1 -def test_finish(): - vref = virtual_ref(X()) - virtual_ref_finish(vref) - py.test.raises(AssertionError, "vref()") - # +def test_direct_2(): x1 = X() vref = virtual_ref(x1) - assert vref() is x1 - virtual_ref_finish(vref) + virtual_ref_finish(x1) assert vref() is x1 def test_annotate_1(): @@ -45,8 +42,11 @@ def test_annotate_2(): def f(): - vref = virtual_ref(X()) - return vref() + x1 = X() + vref = virtual_ref(x1) + x2 = vref() + virtual_ref_finish(x1) + return x2 a = RPythonAnnotator() s = a.build_types(f, []) assert isinstance(s, annmodel.SomeInstance) @@ -84,8 +84,11 @@ def test_rtype_2(): def f(): - vref = virtual_ref(X()) - return vref() + x1 = X() + vref = virtual_ref(x1) + x2 = vref() + virtual_ref_finish(x2) + return x2 x = interpret(f, []) assert lltype.castable(OBJECTPTR, lltype.typeOf(x)) > 0 @@ -107,5 +110,3 @@ x = interpret(f, [-5]) assert lltype.typeOf(x) == OBJECTPTR assert not x - -# the path "we_are_jitted()" is tested in jit/metainterp/test/test_codewriter. From cfbolz at codespeak.net Mon Dec 7 17:23:54 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 7 Dec 2009 17:23:54 +0100 (CET) Subject: [pypy-svn] r69952 - pypy/trunk/pypy/doc Message-ID: <20091207162354.DC081168030@codespeak.net> Author: cfbolz Date: Mon Dec 7 17:23:53 2009 New Revision: 69952 Modified: pypy/trunk/pypy/doc/project-ideas.txt Log: kill unused link Modified: pypy/trunk/pypy/doc/project-ideas.txt ============================================================================== --- pypy/trunk/pypy/doc/project-ideas.txt (original) +++ pypy/trunk/pypy/doc/project-ideas.txt Mon Dec 7 17:23:53 2009 @@ -83,7 +83,6 @@ .. _`efficient propagators for specialized finite domains`: http://codespeak.net/svn/pypy/extradoc/soc-2006/constraints.txt .. _`py.test`: http://codespeak.net/py/current/doc/test.html .. _`py.execnet`: http://codespeak.net/py/current/doc/execnet.html -.. _`Prolog interpreter`: http://codespeak.net/svn/user/cfbolz/hack/prolog/interpreter .. _`JavaScript interpreter`: ../../pypy/lang/js .. _`object spaces`: objspace.html .. _`code templating solution`: http://codespeak.net/svn/pypy/extradoc/soc-2006/code-templating.txt From cfbolz at codespeak.net Mon Dec 7 17:25:22 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 7 Dec 2009 17:25:22 +0100 (CET) Subject: [pypy-svn] r69953 - in pypy: lang/prolog/trunk/prolog trunk/pypy/doc Message-ID: <20091207162522.503C616804D@codespeak.net> Author: cfbolz Date: Mon Dec 7 17:25:21 2009 New Revision: 69953 Added: pypy/lang/prolog/trunk/prolog/doc.txt - copied unchanged from r69952, pypy/trunk/pypy/doc/prolog-interpreter.txt Removed: pypy/trunk/pypy/doc/prolog-interpreter.txt Log: move prolog docs too (didn't even remember we had them) From cfbolz at codespeak.net Mon Dec 7 17:42:26 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 7 Dec 2009 17:42:26 +0100 (CET) Subject: [pypy-svn] r69954 - pypy/trunk/pypy/doc Message-ID: <20091207164226.E3C32168030@codespeak.net> Author: cfbolz Date: Mon Dec 7 17:42:26 2009 New Revision: 69954 Modified: pypy/trunk/pypy/doc/_ref.txt pypy/trunk/pypy/doc/docindex.txt pypy/trunk/pypy/doc/faq.txt pypy/trunk/pypy/doc/rlib.txt Log: kill some references to the prolog interpreter Modified: pypy/trunk/pypy/doc/_ref.txt ============================================================================== --- pypy/trunk/pypy/doc/_ref.txt (original) +++ pypy/trunk/pypy/doc/_ref.txt Mon Dec 7 17:42:26 2009 @@ -47,11 +47,6 @@ .. _`pypy/interpreter/typedef.py`: ../../pypy/interpreter/typedef.py .. _`lang/`: ../../pypy/lang .. _`lang/js/`: ../../pypy/lang/js -.. _`lang/prolog/`: ../../pypy/lang/prolog -.. _`pypy/lang/prolog/builtin/`: ../../pypy/lang/prolog/builtin -.. _`pypy/lang/prolog/builtin/control.py`: ../../pypy/lang/prolog/builtin/control.py -.. _`pypy/lang/prolog/interpreter/engine.py`: ../../pypy/lang/prolog/interpreter/engine.py -.. _`pypy/lang/prolog/interpreter/term.py`: ../../pypy/lang/prolog/interpreter/term.py .. _`lib/`: .. _`pypy/lib/`: ../../pypy/lib .. _`lib/app_test/`: ../../pypy/lib/app_test @@ -109,7 +104,6 @@ .. _`translator/cli/`: ../../pypy/translator/cli .. _`translator/goal/`: ../../pypy/translator/goal .. _`pypy/translator/goal/targetnopstandalone.py`: ../../pypy/translator/goal/targetnopstandalone.py -.. _`pypy/translator/goal/targetprologstandalone.py`: ../../pypy/translator/goal/targetprologstandalone.py .. _`translator/jvm/`: ../../pypy/translator/jvm .. _`translator/stackless/`: ../../pypy/translator/stackless .. _`translator/tool/`: ../../pypy/translator/tool Modified: pypy/trunk/pypy/doc/docindex.txt ============================================================================== --- pypy/trunk/pypy/doc/docindex.txt (original) +++ pypy/trunk/pypy/doc/docindex.txt Mon Dec 7 17:42:26 2009 @@ -27,9 +27,6 @@ * `JIT Generation in PyPy`_ * `Sandboxing Python code`_ -`PyPy Prolog Interpreter`_ describes an implementation of -Prolog that makes use of our Translation Tool chain. - Status_ of the project. @@ -210,12 +207,6 @@ `interpreter/astcompiler/`_ interpreter-level bytecode compiler, via an AST representation -`lang/`_ interpreters for non-Python languages, written in RPython_ - -`lang/js/`_ a JavaScript interpreter (in-progress) - -`lang/prolog/`_ a `Prolog interpreter`_ - `lib/`_ PyPy's wholesale reimplementations of CPython modules_ and experimental new application-level modules @@ -319,8 +310,6 @@ .. _`object-oriented type system`: rtyper.html#oo-type .. _`garbage collector`: garbage_collection.html .. _`Stackless Transform`: translation.html#the-stackless-transform -.. _`PyPy Prolog Interpreter`: prolog-interpreter.html -.. _`Prolog Interpreter`: prolog-interpreter.html .. _`main PyPy-translation scripts`: getting-started-python.html#translating-the-pypy-python-interpreter .. _`.NET`: http://www.microsoft.com/net/ .. _Mono: http://www.mono-project.com/ Modified: pypy/trunk/pypy/doc/faq.txt ============================================================================== --- pypy/trunk/pypy/doc/faq.txt (original) +++ pypy/trunk/pypy/doc/faq.txt Mon Dec 7 17:42:26 2009 @@ -163,9 +163,9 @@ Currently, we have preliminary versions of a JavaScript interpreter (Leonardo Santagada as his Summer of PyPy project), a `Prolog interpreter`_ (Carl Friedrich Bolz as his Bachelor thesis), and a `SmallTalk interpreter`_ -(produced during a sprint). All of them are unfinised at the moment. +(produced during a sprint). All of them are unfinished at the moment. -.. _`Prolog interpreter`: prolog-interpreter.html +.. _`Prolog interpreter`: http://codespeak.net/svn/pypy/lang/prolog/ .. _`SmallTalk interpreter`: http://dx.doi.org/10.1007/978-3-540-89275-5_7 Development @@ -409,10 +409,7 @@ You can have a look at intermediate C source code, which is (at the moment) put in ``/tmp/usession-*/testing_1/testing_1.c``. Of course, all the functions and stuff used directly and indirectly by your -``entry_point()`` function has to be RPython_. Another example you may -want to look at is `pypy/translator/goal/targetprologstandalone.py`_, -the target for the in-progress Prolog implementation; this target for -example enables a stackless build programmatically. +``entry_point()`` function has to be RPython_. .. _`RPython`: coding-guide.html#rpython Modified: pypy/trunk/pypy/doc/rlib.txt ============================================================================== --- pypy/trunk/pypy/doc/rlib.txt (original) +++ pypy/trunk/pypy/doc/rlib.txt Mon Dec 7 17:42:26 2009 @@ -530,6 +530,6 @@ -.. _`Prolog interpreter`: ../../pypy/lang/prolog/ +.. _`Prolog interpreter`: http://codespeak.net/svn/pypy/lang/prolog/ .. _parsing: ../../pypy/rlib/parsing/ .. _`json format`: http://www.json.org From afa at codespeak.net Mon Dec 7 18:47:50 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 7 Dec 2009 18:47:50 +0100 (CET) Subject: [pypy-svn] r69956 - pypy/trunk/pypy/interpreter Message-ID: <20091207174750.7BCA416800B@codespeak.net> Author: afa Date: Mon Dec 7 18:47:49 2009 New Revision: 69956 Modified: pypy/trunk/pypy/interpreter/baseobjspace.py Log: Force __builtin__ in the initial list of sys.modules. This fixes tests in module/sys and module/zipimport Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Mon Dec 7 18:47:49 2009 @@ -511,6 +511,7 @@ def setup_builtin_modules(self): "NOT_RPYTHON: only for initializing the space." self.getbuiltinmodule('sys') + self.getbuiltinmodule('__builtin__') for mod in self.builtin_modules.values(): mod.setup_after_space_initialization() From arigo at codespeak.net Mon Dec 7 21:22:54 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 7 Dec 2009 21:22:54 +0100 (CET) Subject: [pypy-svn] r69957 - in pypy/branch/virtual-forcing/pypy/jit: backend/llgraph metainterp metainterp/test Message-ID: <20091207202254.4E92916803C@codespeak.net> Author: arigo Date: Mon Dec 7 21:22:52 2009 New Revision: 69957 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py Log: More tweaks: * make the code in pyjitpl.py look even more like it is for virtualizables; * implement the more general policy documented in the docstring of virtual_ref(). Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py Mon Dec 7 21:22:52 2009 @@ -154,6 +154,7 @@ 'call_may_force' : (('int', 'varargs'), 'intorptr'), 'guard_not_forced': ((), None), 'virtual_ref' : (('ref', 'int'), 'ref'), + 'virtual_ref_finish':(('ref', 'ref'), None), #'getitem' : (('void', 'ref', 'int'), 'int'), #'setitem' : (('void', 'ref', 'int', 'int'), None), #'newlist' : (('void', 'varargs'), 'ref'), @@ -817,6 +818,9 @@ def op_virtual_ref(self, _, virtual, index): return virtual + def op_virtual_ref_finish(self, _, vref, virtual): + pass + class OOFrame(Frame): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py Mon Dec 7 21:22:52 2009 @@ -6,7 +6,7 @@ from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask -from pypy.jit.metainterp.history import BoxInt, ConstInt, check_descr +from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt, check_descr from pypy.jit.metainterp.history import INT, REF, ConstFloat from pypy.jit.metainterp import resoperation from pypy.jit.metainterp.resoperation import rop @@ -226,6 +226,9 @@ def do_virtual_ref(cpu, box1, box2): raise NotImplementedError +def do_virtual_ref_finish(cpu, box1, box2): + raise NotImplementedError + def do_debug_merge_point(cpu, box1): from pypy.jit.metainterp.warmspot import get_stats loc = box1._get_str() Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py Mon Dec 7 21:22:52 2009 @@ -759,6 +759,27 @@ vrefvalue.setfield(descr_virtual_token, self.getvalue(tokenbox)) vrefvalue.setfield(descr_virtualref_index, self.getvalue(indexbox)) + def optimize_VIRTUAL_REF_FINISH(self, op): + value = self.getvalue(op.args[1]) + if not value.is_virtual(): # virtual_ref(non-virtual) gives bad + raise compile.GiveUp # results, so don't bother compiling it + # + # Set the 'forced' field of the virtual_ref. + # In good cases, this is all virtual, so has no effect. + # Otherwise, this forces the real object -- but only now, as + # opposed to much earlier. This is important because the object is + # typically a PyPy PyFrame, and now is the end of its execution, so + # forcing it now does not have catastrophic effects. + from pypy.jit.metainterp import virtualref + # - set 'forced' to point to the real object + op1 = ResOperation(rop.SETFIELD_GC, op.args, None, + descr = virtualref.get_descr_forced(self.cpu)) + self.optimize_SETFIELD_GC(op1) + # - set 'virtual_token' to TOKEN_NONE + op1 = ResOperation(rop.SETFIELD_GC, [op.args[0], ConstInt(0)], None, + descr = virtualref.get_descr_virtual_token(self.cpu)) + self.optimize_SETFIELD_GC(op1) + def optimize_GETFIELD_GC(self, op): value = self.getvalue(op.args[0]) if value.is_virtual(): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Mon Dec 7 21:22:52 2009 @@ -888,29 +888,37 @@ @arguments("box") def opimpl_virtual_ref(self, box): - obj = box.getref_base() - vref = virtualref.virtual_ref_during_tracing(obj) - resbox = history.BoxPtr(vref) - cindex = history.ConstInt(len(self.metainterp.virtualref_boxes) // 2) - self.metainterp.history.record(rop.VIRTUAL_REF, [box, cindex], resbox) - # Note: we allocate a JIT_VIRTUAL_REF here - # (in virtual_ref_during_tracing()), in order to detect when - # the virtual escapes during tracing already. We record it as a - # VIRTUAL_REF operation, although the backend sees this operation - # as a no-op. The point is that the backend should not really see - # it in practice, as optimizeopt.py should either kill it or - # replace it with a NEW_WITH_VTABLE followed by SETFIELD_GCs. - self.metainterp.virtualref_boxes.append(box) - self.metainterp.virtualref_boxes.append(resbox) + metainterp = self.metainterp + if metainterp.is_blackholing(): + resbox = box # good enough when blackholing + else: + obj = box.getref_base() + vref = virtualref.virtual_ref_during_tracing(obj) + resbox = history.BoxPtr(vref) + cindex = history.ConstInt(len(metainterp.virtualref_boxes) // 2) + metainterp.history.record(rop.VIRTUAL_REF, [box, cindex], resbox) + # Note: we allocate a JIT_VIRTUAL_REF here + # (in virtual_ref_during_tracing()), in order to detect when + # the virtual escapes during tracing already. We record it as a + # VIRTUAL_REF operation, although the backend sees this operation + # as a no-op. The point is that the backend should not really see + # it in practice, as optimizeopt.py should either kill it or + # replace it with a NEW_WITH_VTABLE followed by SETFIELD_GCs. + metainterp.virtualref_boxes.append(box) + metainterp.virtualref_boxes.append(resbox) self.make_result_box(resbox) @arguments("box") - def opimpl_virtual_ref_finish(self, vrefbox): + def opimpl_virtual_ref_finish(self, box): # virtual_ref_finish() assumes that we have a stack-like, last-in # first-out order. - lastvrefbox = self.metainterp.virtualref_boxes.pop() - assert vrefbox.getref_base() == lastvrefbox.getref_base() - self.metainterp.virtualref_boxes.pop() + metainterp = self.metainterp + vrefbox = metainterp.virtualref_boxes.pop() + lastbox = metainterp.virtualref_boxes.pop() + assert box.getref_base() == lastbox.getref_base() + if not metainterp.is_blackholing(): + metainterp.history.record(rop.VIRTUAL_REF_FINISH, + [vrefbox, lastbox], None) # ------------------------------ @@ -1016,12 +1024,11 @@ effectinfo = descr.get_extra_info() if effectinfo is None or effectinfo.forces_virtual_or_virtualizable: # residual calls require attention to keep virtualizables in-sync - self.metainterp.vable_before_residual_call() + self.metainterp.vable_and_vrefs_before_residual_call() # xxx do something about code duplication resbox = self.metainterp.execute_and_record_varargs( rop.CALL_MAY_FORCE, argboxes, descr=descr) - self.metainterp.virtual_after_residual_call() - self.metainterp.vable_after_residual_call() + self.metainterp.vable_and_vrefs_after_residual_call() if resbox is not None: self.make_result_box(resbox) self.generate_guard(self.pc, rop.GUARD_NOT_FORCED, None, []) @@ -1752,9 +1759,17 @@ virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) vinfo.clear_vable_token(virtualizable) - def vable_before_residual_call(self): + def vable_and_vrefs_before_residual_call(self): if self.is_blackholing(): return + # + for i in range(1, len(self.virtualref_boxes), 2): + vrefbox = self.virtualref_boxes[i] + vref = vrefbox.getref_base() + virtualref.tracing_before_residual_call(vref) + # the FORCE_TOKEN is already set at runtime in each vrefs when + # they are created, by optimizeopt.py. + # vinfo = self.staticdata.virtualizable_info if vinfo is not None: virtualizable_box = self.virtualizable_boxes[-1] @@ -1767,38 +1782,33 @@ force_token_box], None, descr=vinfo.vable_token_descr) - def vable_after_residual_call(self): + def vable_and_vrefs_after_residual_call(self): if self.is_blackholing(): - vable_escapes = True + escapes = True else: - vable_escapes = False + escapes = False + # + for i in range(1, len(self.virtualref_boxes), 2): + vrefbox = self.virtualref_boxes[i] + vref = vrefbox.getref_base() + if virtualref.tracing_after_residual_call(vref): + # this vref escaped during CALL_MAY_FORCE. + escapes = True + # vinfo = self.staticdata.virtualizable_info if vinfo is not None: virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) if vinfo.tracing_after_residual_call(virtualizable): - # We just did the residual call, and it shows that the - # virtualizable escapes. - self.switch_to_blackhole() - vable_escapes = True - if vable_escapes: + # the virtualizable escaped during CALL_MAY_FORCE. + escapes = True + # + if escapes: + self.switch_to_blackhole() + # + if escapes: self.load_fields_from_virtualizable() - def virtual_after_residual_call(self): - if self.is_blackholing(): - return - for i in range(1, len(self.virtualref_boxes), 2): - vrefbox = self.virtualref_boxes[i] - if virtualref.was_forced(vrefbox.getref_base()): - break - else: - return - # during tracing, more precisely during the CALL_MAY_FORCE, at least - # one of the vrefs was read. If we continue to trace and make - # assembler from there, we will get assembler that probably always - # forces a vref. So we just cancel now. - self.switch_to_blackhole() - def handle_exception(self): etype = self.cpu.get_exception() evalue = self.cpu.get_exc_value() Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py Mon Dec 7 21:22:52 2009 @@ -223,6 +223,7 @@ 'COND_CALL_GC_MALLOC', # [a, b, if_(a<=b)_result, if_(a>b)_call, args...] # => result (for mallocs) 'DEBUG_MERGE_POINT/1', # debugging only + 'VIRTUAL_REF_FINISH/2', '_CANRAISE_FIRST', # ----- start of can_raise operations ----- 'CALL', Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Mon Dec 7 21:22:52 2009 @@ -1,5 +1,5 @@ import py -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, lloperation from pypy.rlib.jit import JitDriver, dont_look_inside from pypy.rlib.jit import virtual_ref, virtual_ref_finish from pypy.rlib.objectmodel import compute_unique_id @@ -7,6 +7,8 @@ from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.virtualref import JIT_VIRTUAL_REF +debug_print = lloperation.llop.debug_print + class VRefTests: @@ -18,13 +20,16 @@ exctx = ExCtx() # def f(): - exctx.topframeref = vref = virtual_ref(X()) + x = X() + exctx.topframeref = virtual_ref(x) exctx.topframeref = None - virtual_ref_finish(vref) + virtual_ref_finish(x) return 1 # self.interp_operations(f, []) - self.check_operations_history(virtual_ref=1) + self.check_operations_history(new_with_vtable=1, # X() + virtual_ref=1, + virtual_ref_finish=1) def test_make_vref_guard(self): if not isinstance(self, TestLLtype): @@ -43,14 +48,16 @@ return exctx.topframeref().n return n def enter(n): - exctx.topframeref = virtual_ref(X(n + 10)) + x = X(n + 10) + exctx._frame = x + exctx.topframeref = virtual_ref(x) def leave(): - virtual_ref_finish(exctx.topframeref) exctx.topframeref = None + virtual_ref_finish(exctx._frame) def f(n): enter(n) n = external(n) - # ^^^ the point is that X() should be kept alive here + # ^^^ the point is that X() and the vref should be kept alive here leave() return n # @@ -75,6 +82,39 @@ assert self.metainterp.virtualref_boxes[0].value == bxs1[0].value assert self.metainterp.virtualref_boxes[1].value == bxs2[0].value + def test_make_vref_escape_after_finish(self): + jitdriver = JitDriver(greens = [], reds = ['n']) + # + class X: + pass + class ExCtx: + pass + exctx = ExCtx() + # + @dont_look_inside + def g(vref): + debug_print(lltype.Void, '-+-+-+-+- external read:', vref().n) + # + def f(n): + while n > 0: + jitdriver.can_enter_jit(n=n) + jitdriver.jit_merge_point(n=n) + x = X() + x.n = n + exctx.topframeref = vref = virtual_ref(x) + # here, 'x' should be virtual. (This is ensured because + # we call virtual_ref(x).) + exctx.topframeref = None + virtual_ref_finish(x) + # 'vref' is allowed to escape, and even be forced, even after + # the call to finish(). + g(vref) + n -= 1 + return 1 + # + self.meta_interp(f, [10]) + self.check_loops(new_with_vtable=2) # the vref, and later the X + def test_make_vref_and_force(self): jitdriver = JitDriver(greens = [], reds = ['total', 'n']) # @@ -97,7 +137,7 @@ x.n = n + 123 exctx.topframeref = virtual_ref(x) total += force_me() - 100 - virtual_ref_finish(exctx.topframeref) + virtual_ref_finish(x) exctx.topframeref = None return total # @@ -127,13 +167,18 @@ xy = XY() xy.next1 = XY() xy.next2 = XY() + xy.next3 = XY() exctx.topframeref = virtual_ref(xy) n -= externalfn(n) - virtual_ref_finish(exctx.topframeref) exctx.topframeref = None + xy.next1 = None + xy.next2 = None + xy.next3 = None + virtual_ref_finish(xy) # self.meta_interp(f, [15]) - self.check_loops(new_with_vtable=1) # the vref, not the XYs + self.check_loops(new_with_vtable=2) # the vref, and xy so far, + # but not xy.next1/2/3 def test_simple_force_always(self): myjitdriver = JitDriver(greens = [], reds = ['n']) @@ -158,7 +203,7 @@ xy.n = n exctx.topframeref = virtual_ref(xy) n -= externalfn(n) - virtual_ref_finish(exctx.topframeref) + virtual_ref_finish(xy) exctx.topframeref = None # self.meta_interp(f, [15]) @@ -187,14 +232,15 @@ xy.n = n exctx.topframeref = virtual_ref(xy) n -= externalfn(n) - virtual_ref_finish(exctx.topframeref) + virtual_ref_finish(xy) exctx.topframeref = None return exctx.m # res = self.meta_interp(f, [30]) assert res == 13 + self.check_loop_count(1) - def test_bridge_forces(self): + def test_blackhole_forces(self): myjitdriver = JitDriver(greens = [], reds = ['n']) # class XY: @@ -218,12 +264,75 @@ if n == 13: externalfn(n) n -= 1 - virtual_ref_finish(exctx.topframeref) exctx.topframeref = None + virtual_ref_finish(xy) return exctx.m # res = self.meta_interp(f, [30]) assert res == 13 + self.check_loop_count(1) + + def test_bridge_forces(self): + myjitdriver = JitDriver(greens = [], reds = ['n']) + # + class XY: + pass + class ExCtx: + pass + exctx = ExCtx() + # + @dont_look_inside + def externalfn(n): + exctx.m = exctx.topframeref().n + return 1 + # + def f(n): + while n > 0: + myjitdriver.can_enter_jit(n=n) + myjitdriver.jit_merge_point(n=n) + xy = XY() + xy.n = n + exctx.topframeref = virtual_ref(xy) + if n % 6 == 0: + externalfn(n) + n -= 1 + exctx.topframeref = None + virtual_ref_finish(xy) + return exctx.m + # + res = self.meta_interp(f, [72]) + assert res == 6 + self.check_loop_count(1) # the bridge should not be compiled + + def test_access_vref_later(self): + myjitdriver = JitDriver(greens = [], reds = ['n']) + # + class XY: + pass + class ExCtx: + pass + exctx = ExCtx() + # + @dont_look_inside + def g(): + return exctx.later().n + # + def f(n): + later = None + while n > 0: + myjitdriver.can_enter_jit(n=n) + myjitdriver.jit_merge_point(n=n) + xy = XY() + xy.n = n + exctx.topframeref = virtual_ref(xy) + exctx.later = exctx.topframeref + n -= 1 + exctx.topframeref = None + virtual_ref_finish(xy) + return g() + # + res = self.meta_interp(f, [15]) + assert res == 1 class TestLLtype(VRefTests, LLJitMixin): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py Mon Dec 7 21:22:52 2009 @@ -11,8 +11,8 @@ class VirtualizableInfo: - TOKEN_NONE = 0 - TOKEN_TRACING = -1 + TOKEN_NONE = 0 + TOKEN_TRACING_RESCALL = -1 def __init__(self, warmrunnerdesc): self.warmrunnerdesc = warmrunnerdesc @@ -186,13 +186,13 @@ def tracing_before_residual_call(self, virtualizable): assert not virtualizable.vable_token - virtualizable.vable_token = self.TOKEN_TRACING + virtualizable.vable_token = self.TOKEN_TRACING_RESCALL def tracing_after_residual_call(self, virtualizable): if virtualizable.vable_token: # not modified by the residual call; assert that it is still - # set to 'tracing_vable_rti' and clear it. - assert virtualizable.vable_token == self.TOKEN_TRACING + # set to TOKEN_TRACING_RESCALL and clear it. + assert virtualizable.vable_token == self.TOKEN_TRACING_RESCALL virtualizable.vable_token = self.TOKEN_NONE return False else: @@ -201,7 +201,7 @@ def force_now(self, virtualizable): token = virtualizable.vable_token - if token == self.TOKEN_TRACING: + if token == self.TOKEN_TRACING_RESCALL: # The values in the virtualizable are always correct during # tracing. We only need to reset vable_token to TOKEN_NONE # as a marker for the tracing, to tell it that this @@ -228,9 +228,9 @@ # # 2. equal to 0 when tracing is in progress; except: # -# 3. equal to -1 (TOKEN_TRACING) during tracing when we do a residual call, -# calling random unknown other parts of the interpreter; it is -# reset to 0 as soon as something occurs to the virtualizable. +# 3. equal to -1 (TOKEN_TRACING_RESCALL) during tracing when we do a +# residual call, calling random unknown other parts of the interpreter; +# it is reset to 0 as soon as something occurs to the virtualizable. # # 4. when running the machine code with a virtualizable, it is set # to the address in the CPU stack by the FORCE_TOKEN operation. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py Mon Dec 7 21:22:52 2009 @@ -43,11 +43,12 @@ # The 'virtual_token' field has the same meaning as the 'vable_token' field # of a virtualizable. It is equal to: -# * -1 (TOKEN_TRACING) when tracing; +# * 0 (TOKEN_NONE) when tracing, except as described below; +# * -1 (TOKEN_TRACING_RESCALL) during tracing when we do a residual call; # * addr in the CPU stack (set by FORCE_TOKEN) when running the assembler; # * 0 (TOKEN_NONE) after the virtual is forced, if it is forced at all. -TOKEN_NONE = 0 -TOKEN_TRACING = -1 +TOKEN_NONE = 0 +TOKEN_TRACING_RESCALL = -1 @specialize.memo() def get_jit_virtual_ref_const_class(cpu): @@ -71,27 +72,40 @@ vref = lltype.malloc(JIT_VIRTUAL_REF) p = lltype.cast_pointer(rclass.OBJECTPTR, vref) p.typeptr = jit_virtual_ref_vtable - vref.virtual_token = TOKEN_TRACING + vref.virtual_token = TOKEN_NONE vref.forced = lltype.cast_opaque_ptr(rclass.OBJECTPTR, real_object) return lltype.cast_opaque_ptr(llmemory.GCREF, vref) -def was_forced(gcref): +def tracing_before_residual_call(gcref): + vref = lltype.cast_opaque_ptr(lltype.Ptr(JIT_VIRTUAL_REF), gcref) + assert not vref.virtual_token + vref.virtual_token = TOKEN_TRACING_RESCALL + +def tracing_after_residual_call(gcref): vref = lltype.cast_opaque_ptr(lltype.Ptr(JIT_VIRTUAL_REF), gcref) - return vref.virtual_token != TOKEN_TRACING + if vref.virtual_token: + # not modified by the residual call; assert that it is still + # set to TOKEN_TRACING_RESCALL and clear it. + assert vref.virtual_token == TOKEN_TRACING_RESCALL + vref.virtual_token = TOKEN_NONE + return False + else: + # marker "modified during residual call" set. + return True def forced_single_vref(gcref, real_object): assert real_object vref = lltype.cast_opaque_ptr(lltype.Ptr(JIT_VIRTUAL_REF), gcref) assert (vref.virtual_token != TOKEN_NONE and - vref.virtual_token != TOKEN_TRACING) - vref.forced = lltype.cast_opaque_ptr(rclass.OBJECTPTR, real_object) + vref.virtual_token != TOKEN_TRACING_RESCALL) vref.virtual_token = TOKEN_NONE + vref.forced = lltype.cast_opaque_ptr(rclass.OBJECTPTR, real_object) def continue_tracing(gcref, real_object): vref = lltype.cast_opaque_ptr(lltype.Ptr(JIT_VIRTUAL_REF), gcref) - assert vref.virtual_token != TOKEN_TRACING + assert vref.virtual_token != TOKEN_TRACING_RESCALL + vref.virtual_token = TOKEN_NONE vref.forced = lltype.cast_opaque_ptr(rclass.OBJECTPTR, real_object) - vref.virtual_token = TOKEN_TRACING # ____________________________________________________________ @@ -113,7 +127,7 @@ vref = lltype.cast_pointer(lltype.Ptr(JIT_VIRTUAL_REF), inst) token = vref.virtual_token if token != TOKEN_NONE: - if token == TOKEN_TRACING: + if token == TOKEN_TRACING_RESCALL: # The "virtual" is not a virtual at all during tracing. # We only need to reset virtual_token to TOKEN_NONE # as a marker for the tracing, to tell it that this From arigo at codespeak.net Mon Dec 7 23:25:00 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 7 Dec 2009 23:25:00 +0100 (CET) Subject: [pypy-svn] r69958 - in pypy/branch/virtual-forcing/pypy/rlib: . test Message-ID: <20091207222500.3165D168023@codespeak.net> Author: arigo Date: Mon Dec 7 23:24:59 2009 New Revision: 69958 Modified: pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py pypy/branch/virtual-forcing/pypy/rlib/jit.py pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py Log: Change the way 'None' are accepted: now we can make a non_virtual_ref() too, that can hide either None or a normal frame. This is without JIT support at all: it's just an annotation-level hack to hide and reveal them as virtual_refs. Modified: pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py Mon Dec 7 23:24:59 2009 @@ -1,5 +1,5 @@ from pypy.annotation import model as annmodel -from pypy.annotation.binaryop import _make_none_union +from pypy.tool.pairtype import pairtype from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rpython.rclass import getinstancerepr from pypy.rpython.rmodel import Repr @@ -11,11 +11,14 @@ class SomeVRef(annmodel.SomeObject): - def __init__(self, s_instance): + def __init__(self, s_instance=annmodel.s_None): + assert (isinstance(s_instance, annmodel.SomeInstance) or + annmodel.s_None.contains(s_instance)) self.s_instance = s_instance def can_be_none(self): - return True + return False # but it can contain s_None, which is only accessible + # via simple_call() anyway def simple_call(self): return self.s_instance @@ -26,14 +29,18 @@ def rtyper_makekey(self): return self.__class__, -_make_none_union('SomeVRef', 'obj.s_instance', globals()) +class __extend__(pairtype(SomeVRef, SomeVRef)): + + def union((vref1, vref2)): + return SomeVRef(annmodel.unionof(vref1.s_instance, vref2.s_instance)) class VRefRepr(Repr): lowleveltype = OBJECTPTR def specialize_call(self, hop): - [v] = hop.inputargs(getinstancerepr(hop.rtyper, None)) + r_generic_object = getinstancerepr(hop.rtyper, None) + [v] = hop.inputargs(r_generic_object) # might generate a cast_pointer return v def rtype_simple_call(self, hop): @@ -42,8 +49,9 @@ return hop.genop('cast_pointer', [v], resulttype = hop.r_result) def convert_const(self, value): - if value: - raise TyperError("only supports None as a prebuilt virtual_ref") + if value() is not None: + raise TyperError("only supports virtual_ref_None as a" + " prebuilt virtual_ref") return lltype.nullptr(OBJECTPTR.TO) vrefrepr = VRefRepr() Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/jit.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/jit.py Mon Dec 7 23:24:59 2009 @@ -109,12 +109,12 @@ JIT will abort if it is not), at least between the calls to virtual_ref and virtual_ref_finish. The point is that the 'vref' returned by virtual_ref may escape early. If at runtime it is - dereferenced (by calling it, as in 'vref()') before the + dereferenced (by the syntax 'vref()' or by virtual_ref_deref()) before the virtual_ref_finish, then we get out of the assembler. If it is not dereferenced at all, or only after the virtual_ref_finish, then nothing special occurs. """ - return DirectVRef(x) + return DirectJitVRef(x) virtual_ref.oopspec = 'virtual_ref(x)' def virtual_ref_finish(x): @@ -123,16 +123,26 @@ keepalive_until_here(x) # otherwise the whole function call is removed virtual_ref_finish.oopspec = 'virtual_ref_finish(x)' +def non_virtual_ref(x): + """Creates a 'vref' that just returns x when called; nothing more special. + Used for None or for frames outside JIT scope.""" + return DirectVRef(x) + +# ---------- implementation-specific ---------- + class DirectVRef(object): def __init__(self, x): self._x = x def __call__(self): return self._x - def _freeze_(self): - raise Exception("should not see a prebuilt virtual_ref") + +class DirectJitVRef(DirectVRef): + def __init__(self, x): + assert x is not None, "virtual_ref(None) is not allowed" + DirectVRef.__init__(self, x) class Entry(ExtRegistryEntry): - _about_ = DirectVRef + _about_ = (non_virtual_ref, DirectJitVRef) def compute_result_annotation(self, s_obj): from pypy.rlib import _jit_vref @@ -141,6 +151,17 @@ def specialize_call(self, hop): return hop.r_result.specialize_call(hop) +class Entry(ExtRegistryEntry): + _type_ = DirectVRef + + def compute_annotation(self): + from pypy.rlib import _jit_vref + assert isinstance(self.instance, DirectVRef) + s_obj = self.bookkeeper.immutablevalue(self.instance()) + return _jit_vref.SomeVRef(s_obj) + +vref_None = non_virtual_ref(None) + # ____________________________________________________________ # User interface for the hotpath JIT policy Modified: pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py Mon Dec 7 23:24:59 2009 @@ -1,5 +1,6 @@ import py from pypy.rlib.jit import virtual_ref, virtual_ref_finish +from pypy.rlib.jit import vref_None, non_virtual_ref from pypy.rlib._jit_vref import SomeVRef from pypy.annotation import model as annmodel from pypy.annotation.annrpython import RPythonAnnotator @@ -57,11 +58,12 @@ if n > 0: return virtual_ref(Y()) else: - return virtual_ref(Z()) + return non_virtual_ref(Z()) a = RPythonAnnotator() s = a.build_types(f, [int]) assert isinstance(s, SomeVRef) assert isinstance(s.s_instance, annmodel.SomeInstance) + assert not s.s_instance.can_be_None assert s.s_instance.classdef == a.bookkeeper.getuniqueclassdef(X) def test_annotate_4(): @@ -69,11 +71,12 @@ if n > 0: return virtual_ref(X()) else: - return None + return vref_None a = RPythonAnnotator() s = a.build_types(f, [int]) assert isinstance(s, SomeVRef) assert isinstance(s.s_instance, annmodel.SomeInstance) + assert s.s_instance.can_be_None assert s.s_instance.classdef == a.bookkeeper.getuniqueclassdef(X) def test_rtype_1(): @@ -97,7 +100,7 @@ if n > 0: return virtual_ref(Y()) else: - return virtual_ref(Z()) + return non_virtual_ref(Z()) x = interpret(f, [-5]) assert lltype.typeOf(x) == OBJECTPTR @@ -106,7 +109,7 @@ if n > 0: return virtual_ref(X()) else: - return None + return vref_None x = interpret(f, [-5]) assert lltype.typeOf(x) == OBJECTPTR assert not x From arigo at codespeak.net Mon Dec 7 23:34:48 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 7 Dec 2009 23:34:48 +0100 (CET) Subject: [pypy-svn] r69959 - in pypy/branch/virtual-forcing/pypy: interpreter interpreter/test module/_stackless module/pypyjit module/sys objspace/flow Message-ID: <20091207223448.E8DAE168023@codespeak.net> Author: arigo Date: Mon Dec 7 23:34:47 2009 New Revision: 69959 Modified: pypy/branch/virtual-forcing/pypy/interpreter/executioncontext.py pypy/branch/virtual-forcing/pypy/interpreter/generator.py pypy/branch/virtual-forcing/pypy/interpreter/pyframe.py pypy/branch/virtual-forcing/pypy/interpreter/pytraceback.py pypy/branch/virtual-forcing/pypy/interpreter/test/test_executioncontext.py pypy/branch/virtual-forcing/pypy/module/_stackless/interp_coroutine.py pypy/branch/virtual-forcing/pypy/module/pypyjit/interp_jit.py pypy/branch/virtual-forcing/pypy/module/sys/vm.py pypy/branch/virtual-forcing/pypy/objspace/flow/flowcontext.py Log: Revert the 'f_forward' hack everywhere. Replace it with virtual_refs: the 'f_back' field is now called 'f_backref', and 'topframe' is called 'topframeref' to make it clear. In-progress. Modified: pypy/branch/virtual-forcing/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/interpreter/executioncontext.py (original) +++ pypy/branch/virtual-forcing/pypy/interpreter/executioncontext.py Mon Dec 7 23:34:47 2009 @@ -17,7 +17,8 @@ def __init__(self, space): self.space = space - self._init_frame_chain() + self.topframeref = jit.vref_None + self.framestackdepth = 0 # tracing: space.frame_trace_action.fire() must be called to ensure # that tracing occurs whenever self.w_tracefunc or self.is_tracing # is modified. @@ -27,169 +28,43 @@ self.profilefunc = None self.w_profilefuncarg = None + def gettopframe(self): + return self.topframeref() + def gettopframe_nohidden(self): - frame = self.gettopframe() - # I guess this should just use getnextframe_nohidden XXX + frame = self.topframeref() while frame and frame.hide(): - frame = frame.f_back() + frame = frame.f_backref() return frame @staticmethod def getnextframe_nohidden(frame): - frame = frame.f_back() + frame = frame.f_backref() while frame and frame.hide(): - frame = frame.f_back() + frame = frame.f_backref() return frame def enter(self, frame): if self.framestackdepth > self.space.sys.recursionlimit: raise OperationError(self.space.w_RuntimeError, self.space.wrap("maximum recursion depth exceeded")) - self._chain(frame) + self.framestackdepth += 1 + frame.f_backref = self.topframeref + self.topframeref = jit.virtual_ref(frame) def leave(self, frame): - if self.profilefunc: - self._trace(frame, 'leaveframe', self.space.w_None) + try: + if self.profilefunc: + self._trace(frame, 'leaveframe', self.space.w_None) + finally: + self.topframeref = frame.f_backref + self.framestackdepth -= 1 + jit.virtual_ref_finish(frame) - self._unchain(frame) - if self.w_tracefunc is not None and not frame.hide(): self.space.frame_trace_action.fire() # ________________________________________________________________ - # the methods below are used for chaining frames in JIT-friendly way - # part of that stuff is obscure - - @jit.unroll_safe - def gettopframe(self): - frame = self.some_frame - if frame is not None: - while frame.f_forward is not None: - frame = frame.f_forward - return frame - - def _init_frame_chain(self): - # 'some_frame' points to any frame from this thread's frame stack - # (although in general it should point to the top one). - # XXX not true: some_frame must point to a frame from which we can - # reach the top frame by following the chain of f_forward - self.some_frame = None - self.framestackdepth = 0 - - @staticmethod - def _init_chaining_attributes(frame): - """ - explanation of the f_back handling: - ----------------------------------- - - in the non-JIT case, the frames simply form a doubly linked list via the - attributes f_back_some and f_forward. - - When the JIT is used, things become more complex, as functions can be - inlined into each other. In this case a frame chain can look like this: - - +---------------+ - | real_frame | - +---------------+ - | - | f_back_some - | - | - | +--------------+ - | | virtual frame| - | +--------------+ - | ^ - | | f_forward - | +--------------+ - | | virtual frame| - | +--------------+ - | ^ - | | - v | f_forward - +---------------+ - | real_frame | - +---------------+ - | - | - v - ... - - This ensures that the virtual frames don't escape via the f_back of the - real frames. For the same reason, the executioncontext's some_frame - attribute should only point to real frames. - - All places where a frame can become accessed from applevel-code (like - sys._getframe and traceback catching) need to call force_f_back to ensure - that the intermediate virtual frames are forced to be real ones. - - """ - frame.f_back_some = None - frame.f_forward = None - frame.f_back_forced = False - - def _chain(self, frame): - self.framestackdepth += 1 - # - frame.f_back_some = self.some_frame - if self._we_are_jitted(): - curtopframe = self.gettopframe() - assert curtopframe is not None - curtopframe.f_forward = frame - else: - self.some_frame = frame - - def _unchain(self, frame): - #assert frame is self.gettopframe() --- slowish - if self.some_frame is frame: - self.some_frame = frame.f_back_some - else: - f_back = frame.f_back() - if f_back is not None: - f_back.f_forward = None - - self.framestackdepth -= 1 - - @staticmethod - def _jit_rechain_frame(ec, frame): - # this method is called after the jit has seen enter (and thus _chain) - # of a frame, but then does not actually inline it. This method thus - # needs to make sure that the state is as if the _chain method had been - # executed outside of the jit. Note that this makes it important that - # _unchain does not call we_are_jitted - frame.f_back().f_forward = None - ec.some_frame = frame - - @staticmethod - @jit.unroll_safe - def _extract_back_from_frame(frame): - back_some = frame.f_back_some - if frame.f_back_forced: - # don't check back_some.f_forward in this case - return back_some - if back_some is None: - return None - while True: - f_forward = back_some.f_forward - if f_forward is frame or f_forward is None: - return back_some - back_some = f_forward - - @staticmethod - def _force_back_of_frame(frame): - orig_frame = frame - while frame is not None and not frame.f_back_forced: - frame.f_back_some = f_back = ExecutionContext._extract_back_from_frame(frame) - frame.f_back_forced = True - # now that we force the whole chain, we also have to set the - # forward links to None - frame.f_forward = None - frame = f_back - return orig_frame.f_back_some - - _we_are_jitted = staticmethod(we_are_jitted) # indirection for testing - - # the methods above are used for chaining frames in JIT-friendly way - # ________________________________________________________________ class Subcontext(object): @@ -204,7 +79,7 @@ self.is_tracing = 0 def enter(self, ec): - ec.some_frame = self.topframe + ec.topframeref = jit.non_virtual_ref(self.topframe) ec.framestackdepth = self.framestackdepth ec.w_tracefunc = self.w_tracefunc ec.profilefunc = self.profilefunc @@ -247,7 +122,7 @@ while index > 0: index -= 1 lst[index] = f - f = f.f_back() + f = f.f_backref() assert f is None return lst # coroutine: I think this is all, folks! Modified: pypy/branch/virtual-forcing/pypy/interpreter/generator.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/interpreter/generator.py (original) +++ pypy/branch/virtual-forcing/pypy/interpreter/generator.py Mon Dec 7 23:34:47 2009 @@ -2,6 +2,7 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import NoneNotWrapped from pypy.rlib.rarithmetic import intmask +from pypy.rlib import jit from pypy.interpreter.pyopcode import LoopBlock @@ -64,7 +65,7 @@ else: return w_result # YIELDed finally: - self.frame.f_back_some = None + self.frame.f_backref = jit.vref_None self.running = False def descr_throw(self, w_type, w_val=None, w_tb=None): Modified: pypy/branch/virtual-forcing/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/interpreter/pyframe.py (original) +++ pypy/branch/virtual-forcing/pypy/interpreter/pyframe.py Mon Dec 7 23:34:47 2009 @@ -34,13 +34,13 @@ * 'builtin' is the attached built-in module * 'valuestack_w', 'blockstack', control the interpretation """ - __metaclass__ = extendabletype frame_finished_execution = False last_instr = -1 last_exception = None + f_backref = jit.vref_None w_f_trace = None # For tracing instr_lb = 0 @@ -48,6 +48,9 @@ instr_prev = -1 is_being_profiled = False + # XXX temporary, kill me + f_back = property(lambda: None, lambda: None, lambda: None) # f_backref! + def __init__(self, space, code, w_globals, closure): self = hint(self, access_directly=True, fresh_virtualizable=True) assert isinstance(code, pycode.PyCode) @@ -64,7 +67,6 @@ self.fastlocals_w = [None]*self.numlocals make_sure_not_resized(self.fastlocals_w) self.f_lineno = code.co_firstlineno - ExecutionContext._init_chaining_attributes(self) def append_block(self, block): block.previous = self.lastblock @@ -308,7 +310,7 @@ w_tb = w(self.last_exception.application_traceback) tup_state = [ - w(self.f_back()), + w(self.f_backref()), w(self.get_builtin()), w(self.pycode), w_valuestack, @@ -360,8 +362,8 @@ # do not use the instance's __init__ but the base's, because we set # everything like cells from here PyFrame.__init__(self, space, pycode, w_globals, closure) - new_frame.f_back_some = space.interp_w(PyFrame, w_f_back, can_be_None=True) - new_frame.f_back_forced = True + f_back = space.interp_w(PyFrame, w_f_back, can_be_None=True) + new_frame.f_backref = jit.non_virtual_ref(f_back) new_frame.builtin = space.interp_w(Module, w_builtin) new_frame.set_blocklist([unpickle_block(space, w_blk) @@ -431,12 +433,6 @@ def _setcellvars(self, cellvars): pass - def f_back(self): - return ExecutionContext._extract_back_from_frame(self) - - def force_f_back(self): - return ExecutionContext._force_back_of_frame(self) - ### line numbers ### # for f*_f_* unwrapping through unwrap_spec in typedef.py @@ -584,7 +580,7 @@ return self.get_builtin().getdict() def fget_f_back(space, self): - return self.space.wrap(self.f_back()) + return self.space.wrap(self.f_backref()) def fget_f_lasti(space, self): return self.space.wrap(self.last_instr) @@ -605,27 +601,27 @@ def fget_f_exc_type(space, self): if self.last_exception is not None: - f = self.f_back() + f = self.f_backref() while f is not None and f.last_exception is None: - f = f.f_back() + f = f.f_backref() if f is not None: return f.last_exception.w_type return space.w_None def fget_f_exc_value(space, self): if self.last_exception is not None: - f = self.f_back() + f = self.f_backref() while f is not None and f.last_exception is None: - f = f.f_back() + f = f.f_backref() if f is not None: return f.last_exception.w_value return space.w_None def fget_f_exc_traceback(space, self): if self.last_exception is not None: - f = self.f_back() + f = self.f_backref() while f is not None and f.last_exception is None: - f = f.f_back() + f = f.f_backref() if f is not None: return space.wrap(f.last_exception.application_traceback) return space.w_None Modified: pypy/branch/virtual-forcing/pypy/interpreter/pytraceback.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/interpreter/pytraceback.py (original) +++ pypy/branch/virtual-forcing/pypy/interpreter/pytraceback.py Mon Dec 7 23:34:47 2009 @@ -49,7 +49,6 @@ self.next = space.interp_w(PyTraceback, w_next, can_be_None=True) def record_application_traceback(space, operror, frame, last_instruction): - frame.force_f_back() if frame.pycode.hidden_applevel: return tb = operror.application_traceback Modified: pypy/branch/virtual-forcing/pypy/interpreter/test/test_executioncontext.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/interpreter/test/test_executioncontext.py (original) +++ pypy/branch/virtual-forcing/pypy/interpreter/test/test_executioncontext.py Mon Dec 7 23:34:47 2009 @@ -1,6 +1,5 @@ import py from pypy.interpreter import executioncontext -from pypy.interpreter.executioncontext import ExecutionContext from pypy.conftest import gettestobjspace class Finished(Exception): @@ -220,542 +219,3 @@ """) events = space.unwrap(w_events) assert [i[0] for i in events] == ['c_call', 'c_return', 'c_call'] - - - -class TestFrameChaining(object): - class EC(ExecutionContext): - _some_frame = None - def __init__(self, jitted=False): - self.jitted = jitted - self.virtualizable = None - self.framestackdepth = 0 - self._init_frame_chain() - - def _we_are_jitted(self): - return self.jitted - - def _get_some_frame(self): - if self._some_frame: - self._some_frame.look_at() - return self._some_frame - def _set_some_frame(self, frame): - if frame is not None: - frame.force() - self._some_frame = frame - some_frame = property(_get_some_frame, _set_some_frame) - - class Frame(object): - _f_back_some = None - _f_forward = None - - def __init__(self, ec, virtual_with_base_frame=None): - self.ec = ec - self.virtual_with_base_frame = virtual_with_base_frame - self.escaped = not virtual_with_base_frame - ExecutionContext._init_chaining_attributes(self) - - def f_back(self): - return ExecutionContext._extract_back_from_frame(self) - - def force_f_back(self): - return ExecutionContext._force_back_of_frame(self) - - def force(self): - if not self.escaped: - self.virtual_with_base_frame = None - self.escaped = True - if self._f_back_some: - self._f_back_some.force() - if self._f_forward: - self._f_back_some.force() - - def look_at(self): - if (not self.ec.jitted or - self.ec.virtualizable is not self.virtual_with_base_frame): - self.force() - - def store_ref_to(self, other): - if (other.virtual_with_base_frame is not self and - other.virtual_with_base_frame is not self.virtual_with_base_frame): - other.force() - - def _get_f_back_some(self): - self.look_at() - return self._f_back_some - def _set_f_back_some(self, frame): - self.look_at() - if frame: - frame.look_at() - self.store_ref_to(frame) - self._f_back_some = frame - f_back_some = property(_get_f_back_some, _set_f_back_some) - - def _get_f_forward(self): - self.look_at() - return self._f_forward - def _set_f_forward(self, frame): - self.look_at() - if frame: - frame.look_at() - self.store_ref_to(frame) - self._f_forward = frame - f_forward = property(_get_f_forward, _set_f_forward) - - def test_f_back_no_jit(self): - ec = self.EC() - frame = self.Frame(ec) - frame2 = self.Frame(ec) - frame2.f_back_some = frame - - frame3 = self.Frame(ec) - frame3.f_back_some = frame2 - - assert frame3.f_back() is frame2 - assert frame2.f_back() is frame - assert frame.f_back() is None - - def test_f_back_jit(self): - ec = self.EC() - frame = self.Frame(ec) # real frame - frame2 = self.Frame(ec) # virtual frame - frame2.f_back_some = frame - frame.f_forward = frame2 - - frame3 = self.Frame(ec) # virtual frame - frame3.f_back_some = frame - frame2.f_forward = frame3 - - assert frame3.f_back() is frame2 - assert frame2.f_back() is frame - assert frame.f_back() is None - - frame4 = self.Frame(ec) # real frame again - frame4.f_back_some = frame - assert frame4.f_back() is frame3 - - def test_gettopframe_no_jit(self): - ec = self.EC() - frame = self.Frame(ec) - ec.some_frame = frame - assert ec.gettopframe() is frame - - def test_gettopframe_jit(self): - ec = self.EC() - frame = self.Frame(ec) # real frame - ec.some_frame = frame - assert ec.gettopframe() is frame - - frame2 = self.Frame(ec) # virtual frame - frame2.f_back_some = frame - frame.f_forward = frame2 - assert ec.gettopframe() is frame2 - - frame3 = self.Frame(ec) # virtual frame - frame3.f_back_some = frame - frame2.f_forward = frame3 - assert ec.gettopframe() is frame3 - - frame4 = self.Frame(ec) # real frame again - frame4.f_back_some = frame - ec.some_frame = frame4 - assert ec.gettopframe() is frame4 - - def test_frame_chain(self): - - ec = self.EC() - - assert ec.some_frame is None - assert ec.framestackdepth == 0 - - frame = self.Frame(ec) - ec._chain(frame) - assert ec.some_frame is frame - assert ec.framestackdepth == 1 - assert frame.f_back_some is None - assert frame.f_forward is None - assert ec.gettopframe() is frame - assert ec._extract_back_from_frame(frame) is None - - frame2 = self.Frame(ec) - ec._chain(frame2) - assert ec.some_frame is frame2 - assert ec.framestackdepth == 2 - assert frame2.f_back_some is frame - assert frame.f_forward is None - assert frame2.f_forward is None - assert ec.gettopframe() is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - frame3 = self.Frame(ec) - ec._chain(frame3) - assert ec.some_frame is frame3 - assert frame3.f_back_some is frame2 - assert frame2.f_forward is None - assert ec.gettopframe() is frame3 - assert ec._extract_back_from_frame(frame3) is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - assert frame3.f_back() is frame2 - ec._unchain(frame3) - assert ec.some_frame is frame2 - assert ec.framestackdepth == 2 - assert frame2.f_forward is None - assert frame3.f_back_some is frame2 - assert ec.gettopframe() is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - assert frame2.f_back() is frame - ec._unchain(frame2) - assert ec.some_frame is frame - assert ec.framestackdepth == 1 - assert frame.f_forward is None - assert frame2.f_back_some is frame - assert ec.gettopframe() is frame - assert ec._extract_back_from_frame(frame) is None - - assert frame.f_back() is None - ec._unchain(frame) - assert ec.some_frame is None - assert ec.framestackdepth == 0 - assert frame.f_back_some is None - assert ec.gettopframe() is None - - def test_frame_chain_forced(self): - - ec = self.EC() - - frame = self.Frame(ec) - ec._chain(frame) - assert ec.gettopframe() is frame - assert ec._extract_back_from_frame(frame) is None - - frame2 = self.Frame(ec) - ec._chain(frame2) - assert ec.some_frame is frame2 - assert ec.framestackdepth == 2 - assert frame2.f_back_some is frame - assert frame.f_forward is None - assert frame2.f_forward is None - res = frame2.force_f_back() - assert res is frame - assert frame.f_back_forced - assert ec.gettopframe() is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - frame3 = self.Frame(ec) - ec._chain(frame3) - assert ec.gettopframe() is frame3 - assert ec._extract_back_from_frame(frame3) is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - assert frame3.f_back() is frame2 - ec._unchain(frame3) - assert ec.some_frame is frame2 - assert frame3.f_back_some is frame2 - assert ec.gettopframe() is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - assert frame2.f_back() is frame - ec._unchain(frame2) - assert frame2.f_back_some is frame - assert ec.gettopframe() is frame - assert ec._extract_back_from_frame(frame) is None - - assert frame.f_back() is None - ec._unchain(frame) - assert ec.some_frame is None - assert frame.f_back_some is None - - assert frame2.f_back() is frame - assert frame.f_back() is None - assert ec.gettopframe() is None - - - def test_frame_chain_jitted(self): - - ec = self.EC() - - assert ec.some_frame is None - assert ec.framestackdepth == 0 - assert ec.gettopframe() is None - - frame = self.Frame(ec) - ec._chain(frame) - assert ec.some_frame is frame - assert ec.framestackdepth == 1 - assert frame.f_back_some is None - assert frame.f_forward is None - assert ec.gettopframe() is frame - assert ec._extract_back_from_frame(frame) is None - - ec.jitted = True - ec.virtualizable = frame - frame2 = self.Frame(ec, frame) - ec._chain(frame2) - assert ec.some_frame is frame - assert ec.framestackdepth == 2 - assert frame2.f_back_some is frame - assert frame.f_forward is frame2 - assert frame2.f_forward is None - assert ec.gettopframe() is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - # recursive enter/leave seen by the jit - frame3 = self.Frame(ec, frame) - ec._chain(frame3) - assert ec.some_frame is frame - assert frame3.f_back_some is frame - assert frame2.f_forward is frame3 - assert ec.gettopframe() is frame3 - assert ec._extract_back_from_frame(frame3) is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - assert frame3.f_back() is frame2 - ec._unchain(frame3) - assert ec.some_frame is frame - assert ec.framestackdepth == 2 - assert frame2.f_forward is None - assert frame3.f_back_some is frame - assert not frame3.escaped - assert not frame2.escaped - assert ec.gettopframe() is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - # recursive enter/leave not seen by the jit - ec.jitted = False - ec.virtualizable = None - ec._chain(frame3) - assert not frame2.escaped - assert ec.some_frame is frame3 - assert frame3.f_back_some is frame - assert frame2.f_forward is None - assert frame3.escaped - assert ec.gettopframe() is frame3 - assert ec._extract_back_from_frame(frame3) is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - assert frame3.f_back() is frame2 - ec._unchain(frame3) - assert ec.some_frame is frame - assert ec.framestackdepth == 2 - assert frame2.f_forward is None - assert frame3.f_back_some is frame - assert ec.gettopframe() is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - ec.jitted = True - ec.virtualizable = frame - - assert frame2.f_back() is frame - ec._unchain(frame2) - assert ec.some_frame is frame - assert ec.framestackdepth == 1 - assert frame.f_forward is None - assert frame2.f_back_some is frame - assert ec.gettopframe() is frame - assert ec._extract_back_from_frame(frame) is None - - ec.jitted = False - assert frame.f_back() is None - ec._unchain(frame) - assert ec.some_frame is None - assert ec.framestackdepth == 0 - assert frame.f_back_some is None - assert ec.gettopframe() is None - - @py.test.mark.xfail - def test_frame_chain_jitted_forced(self): - - ec = self.EC() - - assert ec.some_frame is None - assert ec.framestackdepth == 0 - - frame = self.Frame(ec) - ec._chain(frame) - - ec.jitted = True - frame2 = self.Frame(ec) - ec._chain(frame2) - - # recursive enter/leave seen by the jit - frame3 = self.Frame(ec) - ec._chain(frame3) - assert ec.gettopframe() is frame3 - res = frame3.force_f_back() - assert res is frame2 - assert ec.gettopframe() is frame3 - - assert frame3.f_back() is frame2 - ec._unchain(frame3) - - assert frame2.f_back() is frame - ec._unchain(frame2) - ec.jitted = False - assert frame.f_back() is None - ec._unchain(frame) - - assert frame3.f_back() is frame2 - assert frame2.f_back() is frame - assert frame.f_back() is None - - def enter_two_jitted_levels(self): - ec = self.EC() - - assert ec.some_frame is None - assert ec.framestackdepth == 0 - - frame = self.Frame(ec) - ec._chain(frame) - - ec.jitted = True - ec.virtualizable = frame - frame2 = self.Frame(ec, frame) - ec._chain(frame2) - assert not frame2.escaped - return ec, frame, frame2 - - def leave_two_jitted_levels(self, ec, frame, frame2): - assert frame2.f_back() is frame - ec._unchain(frame2) - ec.jitted = False - assert frame.f_back() is None - ec._unchain(frame) - - - def test_check_escaping_all_inlined(self): - ec, frame, frame2 = self.enter_two_jitted_levels() - - # recursive enter/leave seen by the jit - frame3 = self.Frame(ec, frame) - ec._chain(frame3) - assert not frame2.escaped - assert not frame3.escaped - - assert frame3.f_back() is frame2 - ec._unchain(frame3) - assert not frame2.escaped - self.leave_two_jitted_levels(ec, frame, frame2) - - - def test_check_escaping_not_all_inlined_enter_leave_not_seen(self): - ec, frame, frame2 = self.enter_two_jitted_levels() - - ec.jitted = False - # recursive enter/leave not seen by the jit - frame3 = self.Frame(ec) - ec._chain(frame3) - - assert not frame2.escaped - assert frame3.escaped - - ec._unchain(frame3) - ec.jitted = True - assert not frame2.escaped - - self.leave_two_jitted_levels(ec, frame, frame2) - - def test_check_escaping_not_all_inlined_enter_leave_seen(self): - ec, frame, frame2 = self.enter_two_jitted_levels() - - # recursive enter/leave seen by the jit - frame3 = self.Frame(ec, frame) - ec._chain(frame3) - ExecutionContext._jit_rechain_frame(ec, frame3) - ec.jitted = False - frame3.look_at() - assert not frame2.escaped - assert frame3.escaped - - ec.jitted = True - assert frame3.f_back() is frame2 - ec._unchain(frame3) - assert not frame2.escaped - - self.leave_two_jitted_levels(ec, frame, frame2) - - - def test_check_escaping_multi_non_jitted_levels(self): - ec, frame, frame2 = self.enter_two_jitted_levels() - - # recursive enter/leave seen by the jit - frame3 = self.Frame(ec, frame) - ec._chain(frame3) - ExecutionContext._jit_rechain_frame(ec, frame3) - ec.jitted = False - - assert frame3.escaped - assert not frame2.escaped - assert frame3.escaped - - frame4 = self.Frame(ec) - ec._chain(frame4) - assert ec.framestackdepth == 4 - - ec._unchain(frame4) - assert frame3.escaped - assert not frame2.escaped - - ec.jitted = True - assert frame3.f_back() is frame2 - ec._unchain(frame3) - assert not frame2.escaped - - self.leave_two_jitted_levels(ec, frame, frame2) - - def test_check_escaping_jitted_with_two_different_virtualizables(self): - ec, frame, frame2 = self.enter_two_jitted_levels() - - frame3 = self.Frame(ec, frame) - ec._chain(frame3) - # frame3 is not inlined, but contains a loop itself, for which code has - # been generated - ExecutionContext._jit_rechain_frame(ec, frame3) - ec.virtualizable = frame3 - - frame3.look_at() - assert not frame2.escaped - assert frame3.escaped - - frame4 = self.Frame(ec, frame3) - ec._chain(frame4) - assert ec.framestackdepth == 4 - assert not frame4.escaped - - ec._unchain(frame4) - assert frame3.escaped - assert not frame2.escaped - - ec.virtualizable = frame - - ec._unchain(frame3) - assert not frame2.escaped - - def test_frame_top_is_virtualizable(self): - ec, frame, frame2 = self.enter_two_jitted_levels() - frame3 = self.Frame(ec, frame2) - ec.jitted = False - ec._chain(frame3) - ec.gettopframe() - frame3.force_f_back() - ec._unchain(frame3) - assert not frame2.f_forward - assert ec.gettopframe() is frame2 - ec.jitted = True - ec._unchain(frame2) - assert not frame.f_forward - assert ec.gettopframe() is frame - ec._unchain(frame) - assert ec.gettopframe() is None Modified: pypy/branch/virtual-forcing/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/branch/virtual-forcing/pypy/module/_stackless/interp_coroutine.py Mon Dec 7 23:34:47 2009 @@ -275,11 +275,10 @@ return space.newtuple([]) items = [None] * index f = self.subctx.topframe - f.force_f_back() while index > 0: index -= 1 items[index] = space.wrap(f) - f = f.f_back() + f = f.f_backref() assert f is None return space.newtuple(items) Modified: pypy/branch/virtual-forcing/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/virtual-forcing/pypy/module/pypyjit/interp_jit.py Mon Dec 7 23:34:47 2009 @@ -21,7 +21,7 @@ PyFrame._virtualizable2_ = ['last_instr', 'pycode', 'valuestackdepth', 'valuestack_w[*]', - 'fastlocals_w[*]', 'f_forward', + 'fastlocals_w[*]', 'last_exception', ] Modified: pypy/branch/virtual-forcing/pypy/module/sys/vm.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/sys/vm.py (original) +++ pypy/branch/virtual-forcing/pypy/module/sys/vm.py Mon Dec 7 23:34:47 2009 @@ -31,7 +31,6 @@ space.wrap("frame index must not be negative")) ec = space.getexecutioncontext() f = ec.gettopframe_nohidden() - f.force_f_back() while True: if f is None: raise OperationError(space.w_ValueError, Modified: pypy/branch/virtual-forcing/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/objspace/flow/flowcontext.py (original) +++ pypy/branch/virtual-forcing/pypy/objspace/flow/flowcontext.py Mon Dec 7 23:34:47 2009 @@ -5,6 +5,7 @@ from pypy.interpreter.argument import ArgumentsForTranslation from pypy.objspace.flow.model import * from pypy.objspace.flow.framestate import FrameState +from pypy.rlib import jit class OperationThatShouldNotBePropagatedError(OperationError): @@ -260,8 +261,8 @@ except StopFlowing: continue # restarting a dead SpamBlock try: - old_frame = self.some_frame - self.some_frame = frame + old_frameref = self.topframeref + self.topframeref = jit.non_virtual_ref(frame) self.crnt_frame = frame try: w_result = frame.dispatch(frame.pycode, @@ -269,7 +270,7 @@ self) finally: self.crnt_frame = None - self.some_frame = old_frame + self.topframeref = old_frameref except OperationThatShouldNotBePropagatedError, e: raise Exception( From arigo at codespeak.net Mon Dec 7 23:36:23 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 7 Dec 2009 23:36:23 +0100 (CET) Subject: [pypy-svn] r69960 - pypy/branch/virtual-forcing/pypy/interpreter/test Message-ID: <20091207223623.502A9168023@codespeak.net> Author: arigo Date: Mon Dec 7 23:36:22 2009 New Revision: 69960 Modified: pypy/branch/virtual-forcing/pypy/interpreter/test/test_zzpickle_and_slow.py Log: Fix this test too. Modified: pypy/branch/virtual-forcing/pypy/interpreter/test/test_zzpickle_and_slow.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/interpreter/test/test_zzpickle_and_slow.py (original) +++ pypy/branch/virtual-forcing/pypy/interpreter/test/test_zzpickle_and_slow.py Mon Dec 7 23:36:22 2009 @@ -2,6 +2,7 @@ from pypy import conftest from pypy.conftest import gettestobjspace from pypy.interpreter import gateway +from pypy.rlib.jit import non_virtual_ref, vref_None class AppTestSlow: def setup_class(cls): @@ -30,21 +31,18 @@ from pypy.interpreter import pytraceback def hide_top_frame(space, w_frame): w_last = None - while w_frame.f_back(): - # should have been forced by traceback capturing - assert w_frame.f_back_forced + while w_frame.f_backref(): w_last = w_frame - w_frame = w_frame.f_back() + w_frame = w_frame.f_backref() assert w_last - w_saved = w_last.f_back() - w_last.f_back_some = None + w_saved = w_last.f_backref() + w_last.f_backref = vref_None return w_saved def restore_top_frame(space, w_frame, w_saved): - while w_frame.f_back(): - assert w_frame.f_back_forced - w_frame = w_frame.f_back() - w_frame.f_back_some = w_saved + while w_frame.f_backref(): + w_frame = w_frame.f_backref() + w_frame.f_backref = non_virtual_ref(w_saved) def read_exc_type(space, w_frame): if w_frame.last_exception is None: From arigo at codespeak.net Tue Dec 8 09:31:23 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 8 Dec 2009 09:31:23 +0100 (CET) Subject: [pypy-svn] r69964 - pypy/branch/virtual-forcing/pypy/module/pypyjit Message-ID: <20091208083123.8255B1680FD@codespeak.net> Author: arigo Date: Tue Dec 8 09:31:22 2009 New Revision: 69964 Modified: pypy/branch/virtual-forcing/pypy/module/pypyjit/interp_jit.py Log: Kill this, no longer needed (or present at all). Modified: pypy/branch/virtual-forcing/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/virtual-forcing/pypy/module/pypyjit/interp_jit.py Tue Dec 8 09:31:22 2009 @@ -35,12 +35,6 @@ name = opcode_method_names[ord(bytecode.co_code[next_instr])] return '%s #%d %s' % (bytecode.get_repr(), next_instr, name) -def leave(next_instr, pycode, frame, ec): - from pypy.interpreter.executioncontext import ExecutionContext - # can't use a method here, since this function is seen later than the main - # annotation XXX no longer true, could be fixed - ExecutionContext._jit_rechain_frame(ec, frame) - def get_jitcell_at(next_instr, bytecode): return bytecode.jit_cells.get(next_instr, None) @@ -63,7 +57,6 @@ pypyjitdriver = PyPyJitDriver(can_inline = can_inline, get_printable_location = get_printable_location, - leave = leave, get_jitcell_at = get_jitcell_at, set_jitcell_at = set_jitcell_at) From fijal at codespeak.net Tue Dec 8 10:05:55 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 10:05:55 +0100 (CET) Subject: [pypy-svn] r69965 - pypy/trunk/pypy/lang/test Message-ID: <20091208090555.812E0168105@codespeak.net> Author: fijal Date: Tue Dec 8 10:05:54 2009 New Revision: 69965 Modified: pypy/trunk/pypy/lang/test/test_translation.py Log: Delete this test. Modified: pypy/trunk/pypy/lang/test/test_translation.py ============================================================================== --- pypy/trunk/pypy/lang/test/test_translation.py (original) +++ pypy/trunk/pypy/lang/test/test_translation.py Tue Dec 8 10:05:54 2009 @@ -26,6 +26,3 @@ def test_scheme(self): self.translate('targetscheme') - - def test_js(self): - self.translate('targetjsstandalone') From fijal at codespeak.net Tue Dec 8 10:07:20 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 10:07:20 +0100 (CET) Subject: [pypy-svn] r69966 - pypy/trunk/pypy/doc Message-ID: <20091208090720.3119E168105@codespeak.net> Author: fijal Date: Tue Dec 8 10:07:19 2009 New Revision: 69966 Modified: pypy/trunk/pypy/doc/_ref.txt Log: remove unused ref Modified: pypy/trunk/pypy/doc/_ref.txt ============================================================================== --- pypy/trunk/pypy/doc/_ref.txt (original) +++ pypy/trunk/pypy/doc/_ref.txt Tue Dec 8 10:07:19 2009 @@ -46,7 +46,6 @@ .. _`pypy/interpreter/astcompiler/ast.py`: ../../pypy/interpreter/astcompiler/ast.py .. _`pypy/interpreter/typedef.py`: ../../pypy/interpreter/typedef.py .. _`lang/`: ../../pypy/lang -.. _`lang/js/`: ../../pypy/lang/js .. _`lib/`: .. _`pypy/lib/`: ../../pypy/lib .. _`lib/app_test/`: ../../pypy/lib/app_test From fijal at codespeak.net Tue Dec 8 10:09:03 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 10:09:03 +0100 (CET) Subject: [pypy-svn] r69967 - pypy/trunk/pypy/doc Message-ID: <20091208090903.D545E168105@codespeak.net> Author: fijal Date: Tue Dec 8 10:09:03 2009 New Revision: 69967 Modified: pypy/trunk/pypy/doc/project-ideas.txt Log: Remove reference here. Add note that this is outdated, I would go for removing it. Modified: pypy/trunk/pypy/doc/project-ideas.txt ============================================================================== --- pypy/trunk/pypy/doc/project-ideas.txt (original) +++ pypy/trunk/pypy/doc/project-ideas.txt Tue Dec 8 10:09:03 2009 @@ -1,6 +1,8 @@ Independent project ideas relating to PyPy ========================================== +*NOTE: This is an extremely outdated list, don't consider it seriously* + PyPy allows experimentation in many directions -- indeed facilitating experimentation in language implementation was one of the main motivations for the project. This page is meant to collect some ideas @@ -83,7 +85,6 @@ .. _`efficient propagators for specialized finite domains`: http://codespeak.net/svn/pypy/extradoc/soc-2006/constraints.txt .. _`py.test`: http://codespeak.net/py/current/doc/test.html .. _`py.execnet`: http://codespeak.net/py/current/doc/execnet.html -.. _`JavaScript interpreter`: ../../pypy/lang/js .. _`object spaces`: objspace.html .. _`code templating solution`: http://codespeak.net/svn/pypy/extradoc/soc-2006/code-templating.txt From arigo at codespeak.net Tue Dec 8 10:09:09 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 8 Dec 2009 10:09:09 +0100 (CET) Subject: [pypy-svn] r69968 - in pypy/trunk/pypy/rlib: . test Message-ID: <20091208090909.579CE168108@codespeak.net> Author: arigo Date: Tue Dec 8 10:09:08 2009 New Revision: 69968 Modified: pypy/trunk/pypy/rlib/jit.py pypy/trunk/pypy/rlib/test/test_jit.py Log: Detect type errors in the arguments passed to various jit_merge_point/can_enter_jit. Modified: pypy/trunk/pypy/rlib/jit.py ============================================================================== --- pypy/trunk/pypy/rlib/jit.py (original) +++ pypy/trunk/pypy/rlib/jit.py Tue Dec 8 10:09:08 2009 @@ -237,6 +237,23 @@ "arguments: %s" % (self.instance, expected)) + try: + cache = self.bookkeeper._jit_annotation_cache[driver] + except AttributeError: + cache = {} + self.bookkeeper._jit_annotation_cache = {driver: cache} + except KeyError: + cache = {} + self.bookkeeper._jit_annotation_cache[driver] = cache + for key, s_value in kwds_s.items(): + s_previous = cache.get(key, annmodel.s_ImpossibleValue) + s_value = annmodel.unionof(s_previous, s_value) + if annmodel.isdegenerated(s_value): + raise JitHintError("mixing incompatible types in argument %s" + " of jit_merge_point/can_enter_jit" % + key[2:]) + cache[key] = s_value + if self.instance.__name__ == 'jit_merge_point': self.annotate_hooks(**kwds_s) Modified: pypy/trunk/pypy/rlib/test/test_jit.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_jit.py (original) +++ pypy/trunk/pypy/rlib/test/test_jit.py Tue Dec 8 10:09:08 2009 @@ -1,5 +1,6 @@ import py from pypy.rlib.jit import hint, we_are_jitted, JitDriver, purefunction_promote +from pypy.rlib.jit import JitHintError from pypy.translator.translator import TranslationContext, graphof from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin from pypy.rpython.lltypesystem import lltype @@ -65,6 +66,28 @@ get_printable_location_args = getargs(get_printable_location) assert can_inline_args == get_printable_location_args == [lltype.Float] + def test_annotate_argumenterror(self): + myjitdriver = JitDriver(greens=['m'], reds=['n']) + def fn(n): + while n > 0: + myjitdriver.can_enter_jit(m=42.5, n=n) + myjitdriver.jit_merge_point(n=n) + n -= 1 + return n + py.test.raises(JitHintError, self.gengraph, fn, [int]) + + def test_annotate_typeerror(self): + myjitdriver = JitDriver(greens=['m'], reds=['n']) + class A(object): pass + class B(object): pass + def fn(n): + while n > 0: + myjitdriver.can_enter_jit(m=A(), n=n) + myjitdriver.jit_merge_point(m=B(), n=n) + n -= 1 + return n + py.test.raises(JitHintError, self.gengraph, fn, [int]) + class TestJITLLtype(BaseTestJIT, LLRtypeMixin): pass From afa at codespeak.net Tue Dec 8 10:14:44 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 8 Dec 2009 10:14:44 +0100 (CET) Subject: [pypy-svn] r69969 - in pypy/trunk/pypy/module/__builtin__: . test Message-ID: <20091208091444.50C53168105@codespeak.net> Author: afa Date: Tue Dec 8 10:14:43 2009 New Revision: 69969 Modified: pypy/trunk/pypy/module/__builtin__/importing.py pypy/trunk/pypy/module/__builtin__/test/test_import.py Log: Be sure to check the import hooks before the list of available builtin modules. This fixes lib-python.test_importhooks Modified: pypy/trunk/pypy/module/__builtin__/importing.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/importing.py (original) +++ pypy/trunk/pypy/module/__builtin__/importing.py Tue Dec 8 10:14:43 2009 @@ -229,10 +229,6 @@ def _absolute_import(space, modulename, baselevel, w_fromlist, tentative): w = space.wrap - # check the builtin modules - if modulename in space.builtin_modules: - return space.getbuiltinmodule(modulename) - w_mod = None parts = modulename.split('.') prefix = [] @@ -295,6 +291,9 @@ return w_mod + # check the builtin modules + if modulename in space.builtin_modules: + return space.getbuiltinmodule(modulename) if w_path is not None: for path in space.unpackiterable(w_path): Modified: pypy/trunk/pypy/module/__builtin__/test/test_import.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/test_import.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/test_import.py Tue Dec 8 10:14:43 2009 @@ -704,14 +704,39 @@ def find_module(self, fullname, path=None): tried_imports.append((fullname, path)) - import sys + import sys, math + sys.meta_path.append(Importer()) try: - sys.meta_path.append(Importer()) import datetime assert len(tried_imports) == 1 - tried_imports[0][0] == "datetime" + assert tried_imports[0][0] == "datetime" + finally: + sys.meta_path.pop() + + def test_meta_path_block(self): + class ImportBlocker(object): + "Specified modules can't be imported, even if they are built-in" + def __init__(self, *namestoblock): + self.namestoblock = dict.fromkeys(namestoblock) + def find_module(self, fullname, path=None): + if fullname in self.namestoblock: + return self + def load_module(self, fullname): + raise ImportError, "blocked" + + import sys + modname = "errno" # an arbitrary harmless builtin module + mod = None + if modname in sys.modules: + mod = sys.modules + del sys.modules[modname] + sys.meta_path.append(ImportBlocker(modname)) + try: + raises(ImportError, __import__, modname) finally: sys.meta_path.pop() + if mod: + sys.modules[modname] = mod def test_path_hooks_leaking(self): class Importer(object): From cfbolz at codespeak.net Tue Dec 8 10:24:25 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 8 Dec 2009 10:24:25 +0100 (CET) Subject: [pypy-svn] r69970 - pypy/trunk/pypy/translator/stackless/test Message-ID: <20091208092425.02CD249843E@codespeak.net> Author: cfbolz Date: Tue Dec 8 10:24:25 2009 New Revision: 69970 Modified: pypy/trunk/pypy/translator/stackless/test/test_coroutine_reconstruction.py Log: I broke this test via the merge from the io branch. Fix it. Modified: pypy/trunk/pypy/translator/stackless/test/test_coroutine_reconstruction.py ============================================================================== --- pypy/trunk/pypy/translator/stackless/test/test_coroutine_reconstruction.py (original) +++ pypy/trunk/pypy/translator/stackless/test/test_coroutine_reconstruction.py Tue Dec 8 10:24:25 2009 @@ -5,10 +5,15 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.lltypesystem import lltype +namespace = rcoroutine.make_coroutine_classes(object) +syncstate = namespace['syncstate'] +AbstractThunk = namespace['AbstractThunk'] +Coroutine = namespace['Coroutine'] + class TestCoroutineReconstruction: def setup_meth(self): - rcoroutine.syncstate.reset() + syncstate.reset() def test_simple_ish(self): @@ -23,7 +28,7 @@ rstack.resume_point("f_1", coro, n, x) output.append(x) - class T(rcoroutine.AbstractThunk): + class T(AbstractThunk): def __init__(self, arg_coro, arg_n, arg_x): self.arg_coro = arg_coro self.arg_n = arg_n @@ -32,17 +37,17 @@ f(self.arg_coro, self.arg_n, self.arg_x) def example(): - main_coro = rcoroutine.Coroutine.getcurrent() - sub_coro = rcoroutine.Coroutine() + main_coro = Coroutine.getcurrent() + sub_coro = Coroutine() thunk_f = T(main_coro, 5, 1) sub_coro.bind(thunk_f) sub_coro.switch() - new_coro = rcoroutine.Coroutine() + new_coro = Coroutine() new_thunk_f = T(main_coro, 5, 1) new_coro.bind(new_thunk_f) - costate = rcoroutine.Coroutine._get_default_costate() + costate = Coroutine._get_default_costate() bottom = resume_state_create(None, "yield_current_frame_to_caller_1") _bind_frame = resume_state_create(bottom, "coroutine__bind", costate) f_frame_1 = resume_state_create(_bind_frame, "f_1", main_coro, 5, 1) From arigo at codespeak.net Tue Dec 8 11:03:11 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 8 Dec 2009 11:03:11 +0100 (CET) Subject: [pypy-svn] r69971 - in pypy/branch/virtual-forcing/pypy: jit/metainterp/test rlib Message-ID: <20091208100311.20E3649843E@codespeak.net> Author: arigo Date: Tue Dec 8 11:03:10 2009 New Revision: 69971 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py pypy/branch/virtual-forcing/pypy/rlib/jit.py Log: Typo and fix the tests. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Tue Dec 8 11:03:10 2009 @@ -1,7 +1,7 @@ import py from pypy.rpython.lltypesystem import lltype, llmemory, lloperation from pypy.rlib.jit import JitDriver, dont_look_inside -from pypy.rlib.jit import virtual_ref, virtual_ref_finish +from pypy.rlib.jit import virtual_ref, virtual_ref_finish, vref_None from pypy.rlib.objectmodel import compute_unique_id from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.jit.metainterp.resoperation import rop @@ -22,7 +22,7 @@ def f(): x = X() exctx.topframeref = virtual_ref(x) - exctx.topframeref = None + exctx.topframeref = vref_None virtual_ref_finish(x) return 1 # @@ -52,7 +52,7 @@ exctx._frame = x exctx.topframeref = virtual_ref(x) def leave(): - exctx.topframeref = None + exctx.topframeref = vref_None virtual_ref_finish(exctx._frame) def f(n): enter(n) @@ -104,7 +104,7 @@ exctx.topframeref = vref = virtual_ref(x) # here, 'x' should be virtual. (This is ensured because # we call virtual_ref(x).) - exctx.topframeref = None + exctx.topframeref = vref_None virtual_ref_finish(x) # 'vref' is allowed to escape, and even be forced, even after # the call to finish(). @@ -138,7 +138,7 @@ exctx.topframeref = virtual_ref(x) total += force_me() - 100 virtual_ref_finish(x) - exctx.topframeref = None + exctx.topframeref = vref_None return total # res = self.meta_interp(f, [-4]) @@ -170,7 +170,7 @@ xy.next3 = XY() exctx.topframeref = virtual_ref(xy) n -= externalfn(n) - exctx.topframeref = None + exctx.topframeref = vref_None xy.next1 = None xy.next2 = None xy.next3 = None @@ -204,7 +204,7 @@ exctx.topframeref = virtual_ref(xy) n -= externalfn(n) virtual_ref_finish(xy) - exctx.topframeref = None + exctx.topframeref = vref_None # self.meta_interp(f, [15]) self.check_loops({}) # because we aborted tracing @@ -233,7 +233,7 @@ exctx.topframeref = virtual_ref(xy) n -= externalfn(n) virtual_ref_finish(xy) - exctx.topframeref = None + exctx.topframeref = vref_None return exctx.m # res = self.meta_interp(f, [30]) @@ -264,7 +264,7 @@ if n == 13: externalfn(n) n -= 1 - exctx.topframeref = None + exctx.topframeref = vref_None virtual_ref_finish(xy) return exctx.m # @@ -296,7 +296,7 @@ if n % 6 == 0: externalfn(n) n -= 1 - exctx.topframeref = None + exctx.topframeref = vref_None virtual_ref_finish(xy) return exctx.m # @@ -327,7 +327,7 @@ exctx.topframeref = virtual_ref(xy) exctx.later = exctx.topframeref n -= 1 - exctx.topframeref = None + exctx.topframeref = vref_None virtual_ref_finish(xy) return g() # Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/jit.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/jit.py Tue Dec 8 11:03:10 2009 @@ -109,7 +109,7 @@ JIT will abort if it is not), at least between the calls to virtual_ref and virtual_ref_finish. The point is that the 'vref' returned by virtual_ref may escape early. If at runtime it is - dereferenced (by the syntax 'vref()' or by virtual_ref_deref()) before the + dereferenced (by the call syntax 'vref()') before the virtual_ref_finish, then we get out of the assembler. If it is not dereferenced at all, or only after the virtual_ref_finish, then nothing special occurs. From arigo at codespeak.net Tue Dec 8 11:12:00 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 8 Dec 2009 11:12:00 +0100 (CET) Subject: [pypy-svn] r69972 - in pypy/branch/virtual-forcing/pypy/jit/metainterp: . test Message-ID: <20091208101200.3354516802D@codespeak.net> Author: arigo Date: Tue Dec 8 11:12:00 2009 New Revision: 69972 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py pypy/branch/virtual-forcing/pypy/jit/metainterp/support.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Log: Implement 'jit_force_virtual' in JITted code too. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py Tue Dec 8 11:12:00 2009 @@ -1577,8 +1577,7 @@ self.vable_flags[op.args[0]] = op.args[2].value def serialize_op_jit_force_virtual(self, op): - raise ForcingVirtualRef("forcing a virtual_ref, i.e. calling it, " - "should not be seen by the JIT") + self._do_builtin_call(op, 'jit_force_virtual', op.args) serialize_op_oostring = handle_builtin_call serialize_op_oounicode = handle_builtin_call Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/support.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/support.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/support.py Tue Dec 8 11:12:00 2009 @@ -3,6 +3,7 @@ from pypy.rpython import rlist from pypy.rpython.lltypesystem import rstr as ll_rstr, rdict as ll_rdict from pypy.rpython.lltypesystem import rlist as lltypesystem_rlist +from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.ootypesystem import rdict as oo_rdict from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.extregistry import ExtRegistryEntry @@ -136,6 +137,9 @@ def _ll_1_gc_identityhash(x): return lltype.identityhash(x) +def _ll_1_jit_force_virtual(inst): + return llop.jit_force_virtual(lltype.typeOf(inst), inst) + class LLtypeHelpers: Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Tue Dec 8 11:12:00 2009 @@ -334,6 +334,32 @@ res = self.meta_interp(f, [15]) assert res == 1 + def test_jit_force_virtual_seen(self): + myjitdriver = JitDriver(greens = [], reds = ['n']) + # + class XY: + pass + class ExCtx: + pass + exctx = ExCtx() + # + def f(n): + later = None + while n > 0: + myjitdriver.can_enter_jit(n=n) + myjitdriver.jit_merge_point(n=n) + xy = XY() + xy.n = n + exctx.topframeref = virtual_ref(xy) + n = exctx.topframeref().n - 1 + exctx.topframeref = vref_None + virtual_ref_finish(xy) + return 1 + # + res = self.meta_interp(f, [15]) + assert res == 1 + self.check_loops({}) # because we aborted tracing + class TestLLtype(VRefTests, LLJitMixin): pass From fijal at codespeak.net Tue Dec 8 11:30:10 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 11:30:10 +0100 (CET) Subject: [pypy-svn] r69973 - in pypy/branch/listcopyop/pypy: rlib rpython rpython/lltypesystem rpython/memory rpython/memory/gc rpython/memory/gctransform rpython/memory/test Message-ID: <20091208103010.77099168047@codespeak.net> Author: fijal Date: Tue Dec 8 11:30:09 2009 New Revision: 69973 Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py pypy/branch/listcopyop/pypy/rpython/llinterp.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Log: Kill a lot of code :-) Change arraycopy into arraycopy_writebarrier that only does the gc part (after which it's safe to do raw_memcopy, provided there is no collection in between). Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rlib/rgc.py (original) +++ pypy/branch/listcopyop/pypy/rlib/rgc.py Tue Dec 8 11:30:09 2009 @@ -336,15 +336,9 @@ assert source != dest TP = lltype.typeOf(source).TO if isinstance(TP.OF, lltype.Ptr) and TP.OF.TO._gckind == 'gc': - if llop.gc_arraycopy(lltype.Void, source, dest, source_start, dest_start, - length): - return # gc supports nicely copying lists - i = 0 - while i < length: - dest[i + dest_start] = source[i + source_start] - i += 1 - return - # it's safe to do memcpy + # perform a write barrier that copies necessary flags from + # source to dest + llop.gc_arraycopy_writebarrier(lltype.Void, source, dest) source_addr = llmemory.cast_ptr_to_adr(source) dest_addr = llmemory.cast_ptr_to_adr(dest) cp_source_addr = (source_addr + llmemory.itemoffsetof(TP, 0) + Modified: pypy/branch/listcopyop/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/llinterp.py (original) +++ pypy/branch/listcopyop/pypy/rpython/llinterp.py Tue Dec 8 11:30:09 2009 @@ -754,11 +754,9 @@ def op_zero_gc_pointers_inside(self, obj): raise NotImplementedError("zero_gc_pointers_inside") - def op_gc_arraycopy(self, source, dest, source_start, dest_start, length): - if hasattr(self.heap, 'arraycopy'): - self.heap.arraycopy(source, dest, source_start, dest_start, length) - return True - return False + def op_gc_arraycopy_writebarrier(self, source, dest): + if hasattr(self.heap, 'arraycopy_writebarrier'): + self.heap.arraycopy_writebarrier(source, dest) def op_getfield(self, obj, field): checkptr(obj) Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py Tue Dec 8 11:30:09 2009 @@ -358,7 +358,6 @@ 'resize_buffer': LLOp(canraise=(MemoryError,), canunwindgc=True), 'finish_building_buffer' : LLOp(canraise=(MemoryError,), canunwindgc=True), 'zero_gc_pointers_inside': LLOp(), - 'gc_arraycopy': LLOp(canrun=True), 'free': LLOp(), 'getfield': LLOp(sideeffects=False, canrun=True), 'getarrayitem': LLOp(sideeffects=False, canrun=True), @@ -462,6 +461,7 @@ 'gc_thread_run' : LLOp(), 'gc_thread_die' : LLOp(), 'gc_assume_young_pointers': LLOp(canrun=True), + 'gc_arraycopy_writebarrier': LLOp(canrun=True), 'gc_heap_stats' : LLOp(canunwindgc=True), # ------- JIT & GC interaction, only for some GCs ---------- Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py Tue Dec 8 11:30:09 2009 @@ -394,13 +394,12 @@ checkadr(addr2) return addr1 - addr2 -def op_gc_arraycopy(source, dest, source_start, dest_start, length): +def op_gc_arraycopy_writebarrier(source, dest): A = lltype.typeOf(source) assert A == lltype.typeOf(dest) assert isinstance(A.TO, lltype.GcArray) assert isinstance(A.TO.OF, lltype.Ptr) assert A.TO.OF.TO._gckind == 'gc' - return False # no special support def op_getfield(p, name): checkptr(p) Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Tue Dec 8 11:30:09 2009 @@ -475,8 +475,7 @@ objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS self.last_generation_root_objects.append(addr_struct) - def arraycopy(self, source_addr, dest_addr, source_start, - dest_start, length): + def arraycopy_writebarrier(self, source_addr, dest_addr): typeid = self.get_type_id(source_addr) assert self.is_gcarrayofgcptr(typeid) need_set = False @@ -486,14 +485,6 @@ need_set = True if need_set: self.assume_young_pointers(dest_addr) - itemsize = llmemory.gcarrayofptr_singleitemoffset - cp_source_addr = (source_addr + llmemory.gcarrayofptr_itemsoffset + - itemsize * source_start) - cp_dest_addr = (dest_addr + llmemory.gcarrayofptr_itemsoffset + - itemsize * dest_start) - llmemory.raw_memcopy(cp_source_addr, cp_dest_addr, - itemsize * length) - return True def write_into_last_generation_obj(self, addr_struct, addr): objhdr = self.header(addr_struct) Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py Tue Dec 8 11:30:09 2009 @@ -292,8 +292,8 @@ [s_gc, annmodel.SomeInteger(knowntype=rffi.r_ushort)], annmodel.SomeInteger()) - if hasattr(GCClass, 'arraycopy'): - self.arraycopy_ptr = getfn(GCClass.arraycopy.im_func, + if hasattr(GCClass, 'arraycopy_writebarrier'): + self.arraycopy_wb_p = getfn(GCClass.arraycopy_writebarrier.im_func, [s_gc] + [annmodel.SomeAddress()] * 2 + [annmodel.SomeInteger()] * 3, annmodel.SomeBool()) @@ -781,17 +781,17 @@ TYPE = v_ob.concretetype.TO gen_zero_gc_pointers(TYPE, v_ob, hop.llops) - def gct_gc_arraycopy(self, hop): - if not hasattr(self, 'arraycopy_ptr'): - return GCTransformer.gct_gc_arraycopy(self, hop) + def gct_gc_arraycopy_writebarrier(self, hop): + if not hasattr(self, 'arraycopy_wb_p'): + # should do nothing + return GCTransformer.gct_gc_arraycopy_writebarrier(self, hop) op = hop.spaceop source_addr = hop.genop('cast_ptr_to_adr', [op.args[0]], resulttype=llmemory.Address) dest_addr = hop.genop('cast_ptr_to_adr', [op.args[1]], resulttype=llmemory.Address) hop.genop('direct_call', [self.arraycopy_ptr, self.c_const_gc, - source_addr, dest_addr] + op.args[2:], - resultvar=op.result) + source_addr, dest_addr]) def gct_weakref_create(self, hop): op = hop.spaceop Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py Tue Dec 8 11:30:09 2009 @@ -380,8 +380,8 @@ def gct_zero_gc_pointers_inside(self, hop): pass - def gct_gc_arraycopy(self, hop): - return rmodel.inputconst(lltype.Bool, False) + def gct_gc_arraycopy_writebarrier(self, hop): + pass def gct_gc_identityhash(self, hop): # must be implemented in the various GCs Modified: pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py Tue Dec 8 11:30:09 2009 @@ -128,16 +128,11 @@ ptr = lltype.cast_opaque_ptr(llmemory.GCREF, ptr) return self.gc.id(ptr) - def arraycopy(self, source, dest, source_start, dest_start, length): - if hasattr(self.gc, 'arraycopy'): + def arraycopy_writebarrier(self, source, dest): + if hasattr(self.gc, 'arraycopy_writebarrier'): source_addr = llmemory.cast_ptr_to_adr(source) dest_addr = llmemory.cast_ptr_to_adr(dest) - return self.gc.arraycopy(source_addr, dest_addr, source_start, - dest_start, length) - i = 0 - while i < length: - dest[dest_start + i] = source[source_start + i] - i += 1 + return self.gc.arraycopy_writebarrier(source_addr, dest_addr) # ____________________________________________________________ Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py Tue Dec 8 11:30:09 2009 @@ -551,7 +551,10 @@ res = self.interpret(fn, [-1000], taggedpointers=True) assert res == 111 - def test_arraycopy(self): + def test_arraycopy_writebarrier(self): + import new + ll_arraycopy = new.function(rgc.ll_arraycopy.func_code, {}) + S = lltype.GcStruct('S') TP = lltype.GcArray(lltype.Ptr(S)) def fn(): @@ -559,14 +562,14 @@ l2 = lltype.malloc(TP, 100) for i in range(100): l[i] = lltype.malloc(S) - if llop.gc_arraycopy(lltype.Void, l, l2, 50, 0, 50): - x = [] - # force minor collect - t = (1, lltype.malloc(S)) - for i in range(20): - x.append(t) - for i in range(50): - assert l2[i] + ll_arraycopy(l, l2, 50, 0, 50) + x = [] + # force minor collect + t = (1, lltype.malloc(S)) + for i in range(20): + x.append(t) + for i in range(50): + assert l2[i] return 0 self.interpret(fn, []) Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Tue Dec 8 11:30:09 2009 @@ -834,7 +834,9 @@ return d * 1000 + c * 100 + b * 10 + a return f - def xxxtest_gc_heap_stats(self): + @py.test.mark.xfail("Fails if followed by any test") + def test_gc_heap_stats(self): + XXX # this test makes next test to crash run = self.runner("gc_heap_stats") res = run([]) assert res % 10000 == 2611 @@ -845,6 +847,9 @@ # (and give fixedsize) def define_arraycopy(cls): + import new + ll_arraycopy = new.function(rgc.ll_arraycopy.func_code, {}) + S = lltype.GcStruct('S') TP = lltype.GcArray(lltype.Ptr(S)) def fn(): @@ -852,13 +857,13 @@ l2 = lltype.malloc(TP, 100) for i in range(100): l[i] = lltype.malloc(S) - if llop.gc_arraycopy(lltype.Void, l, l2, 50, 0, 50): - # force nursery collect - x = [] - for i in range(20): - x.append((1, lltype.malloc(S))) - for i in range(50): - assert l2[i] + ll_arraycopy(l, l2, 50, 0, 50) + # force nursery collect + x = [] + for i in range(20): + x.append((1, lltype.malloc(S))) + for i in range(50): + assert l2[i] return 0 return fn From fijal at codespeak.net Tue Dec 8 11:36:05 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 11:36:05 +0100 (CET) Subject: [pypy-svn] r69974 - pypy/branch/listcopyop/pypy/rpython/memory/test Message-ID: <20091208103605.38E59168047@codespeak.net> Author: fijal Date: Tue Dec 8 11:36:04 2009 New Revision: 69974 Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Log: Give up on using xfail. Exits py.test if run with -v -x Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Tue Dec 8 11:36:04 2009 @@ -834,8 +834,8 @@ return d * 1000 + c * 100 + b * 10 + a return f - @py.test.mark.xfail("Fails if followed by any test") def test_gc_heap_stats(self): + py.test.skip("XFail fails, so skip") XXX # this test makes next test to crash run = self.runner("gc_heap_stats") res = run([]) From fijal at codespeak.net Tue Dec 8 11:46:54 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 11:46:54 +0100 (CET) Subject: [pypy-svn] r69975 - pypy/branch/listcopyop/pypy/rpython/memory/gctransform Message-ID: <20091208104654.38E11168039@codespeak.net> Author: fijal Date: Tue Dec 8 11:46:53 2009 New Revision: 69975 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py Log: fix signatures Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py Tue Dec 8 11:46:53 2009 @@ -294,8 +294,7 @@ if hasattr(GCClass, 'arraycopy_writebarrier'): self.arraycopy_wb_p = getfn(GCClass.arraycopy_writebarrier.im_func, - [s_gc] + [annmodel.SomeAddress()] * 2 + - [annmodel.SomeInteger()] * 3, annmodel.SomeBool()) + [s_gc] + [annmodel.SomeAddress()] * 2, annmodel.s_None) # in some GCs we can inline the common case of # malloc_fixedsize(typeid, size, True, False, False) @@ -790,7 +789,7 @@ resulttype=llmemory.Address) dest_addr = hop.genop('cast_ptr_to_adr', [op.args[1]], resulttype=llmemory.Address) - hop.genop('direct_call', [self.arraycopy_ptr, self.c_const_gc, + hop.genop('direct_call', [self.arraycopy_wb_p, self.c_const_gc, source_addr, dest_addr]) def gct_weakref_create(self, hop): From fijal at codespeak.net Tue Dec 8 12:02:14 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 12:02:14 +0100 (CET) Subject: [pypy-svn] r69976 - pypy/branch/listcopyop/pypy/rpython/memory/gc Message-ID: <20091208110214.9F12A16803E@codespeak.net> Author: fijal Date: Tue Dec 8 12:02:14 2009 New Revision: 69976 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Log: Explanation + make the flag a bit nicer Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Tue Dec 8 12:02:14 2009 @@ -476,13 +476,23 @@ self.last_generation_root_objects.append(addr_struct) def arraycopy_writebarrier(self, source_addr, dest_addr): + """ For both GCFLAG_NO_YOUNG_PTRS and GCFLAG_NO_HEAP_PTRS following + yields true: + + if the flag on source is set to 0 (means source either contains + young pointers or is in nursery), we need to clear the same flag on dest, + provided it's set already + + assume_young_pointers contains already the interface for setting, + we just need to decide when we need to set it. + """ typeid = self.get_type_id(source_addr) assert self.is_gcarrayofgcptr(typeid) - need_set = False + need_clear = False if self.header(source_addr).tid & GCFLAG_NO_YOUNG_PTRS == 0: - need_set = True + need_clear = True if self.header(source_addr).tid & GCFLAG_NO_HEAP_PTRS == 0: - need_set = True + need_clear = True if need_set: self.assume_young_pointers(dest_addr) From afa at codespeak.net Tue Dec 8 13:23:23 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 8 Dec 2009 13:23:23 +0100 (CET) Subject: [pypy-svn] r69977 - pypy/trunk/pypy/module/__builtin__/test Message-ID: <20091208122323.AFD20168043@codespeak.net> Author: afa Date: Tue Dec 8 13:23:22 2009 New Revision: 69977 Modified: pypy/trunk/pypy/module/__builtin__/test/test_import.py Log: Open the test files in binary mode to write data, this fixes test_universal_newlines on Windows. Modified: pypy/trunk/pypy/module/__builtin__/test/test_import.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/test_import.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/test_import.py Tue Dec 8 13:23:22 2009 @@ -68,8 +68,10 @@ setuppkg("pkg_substituted", mod='') p = setuppkg("readonly", x='') p = setuppkg("pkg_univnewlines") - p.join('__init__.py').write('a=5\nb=6\rc="""hello\r\nworld"""\r') - p.join('mod.py').write('a=15\nb=16\rc="""foo\r\nbar"""\r') + p.join('__init__.py').write( + 'a=5\nb=6\rc="""hello\r\nworld"""\r', mode='wb') + p.join('mod.py').write( + 'a=15\nb=16\rc="""foo\r\nbar"""\r', mode='wb') # create compiled/x.py and a corresponding pyc file p = setuppkg("compiled", x = "x = 84") From afa at codespeak.net Tue Dec 8 13:43:30 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 8 Dec 2009 13:43:30 +0100 (CET) Subject: [pypy-svn] r69978 - pypy/trunk/pypy/module/__builtin__/test Message-ID: <20091208124330.6E0D7168047@codespeak.net> Author: afa Date: Tue Dec 8 13:43:29 2009 New Revision: 69978 Modified: pypy/trunk/pypy/module/__builtin__/test/test_import.py Log: Use binary mode to copy .pyc files, this fixes AppTestLonePycFile Modified: pypy/trunk/pypy/module/__builtin__/test/test_import.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/test_import.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/test_import.py Tue Dec 8 13:43:29 2009 @@ -96,7 +96,8 @@ stream.close() if space.config.objspace.usepycfiles: # also create a lone .pyc file - p.join('lone.pyc').write(p.join('x.pyc').read(), mode='wb') + p.join('lone.pyc').write(p.join('x.pyc').read(mode='rb'), + mode='wb') return str(root) From fijal at codespeak.net Tue Dec 8 16:29:32 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 16:29:32 +0100 (CET) Subject: [pypy-svn] r69979 - pypy/branch/listcopyop/pypy/rpython/memory/gc Message-ID: <20091208152932.E552E168043@codespeak.net> Author: fijal Date: Tue Dec 8 16:29:31 2009 New Revision: 69979 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Log: Improve comment, fix typo Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Tue Dec 8 16:29:31 2009 @@ -476,12 +476,17 @@ self.last_generation_root_objects.append(addr_struct) def arraycopy_writebarrier(self, source_addr, dest_addr): - """ For both GCFLAG_NO_YOUNG_PTRS and GCFLAG_NO_HEAP_PTRS following + """ This has the same effect as calling writebarrier over + each element in dest copied from source, except it might reset + one of the following flags a bit too eagerly, which means we'll have + a bit more objects to track, but being on the safe side. + + For both GCFLAG_NO_YOUNG_PTRS and GCFLAG_NO_HEAP_PTRS following yields true: if the flag on source is set to 0 (means source either contains - young pointers or is in nursery), we need to clear the same flag on dest, - provided it's set already + young pointers or is in nursery), we need to clear the same flag on + dest, provided it's set already assume_young_pointers contains already the interface for setting, we just need to decide when we need to set it. @@ -493,7 +498,7 @@ need_clear = True if self.header(source_addr).tid & GCFLAG_NO_HEAP_PTRS == 0: need_clear = True - if need_set: + if need_clear: self.assume_young_pointers(dest_addr) def write_into_last_generation_obj(self, addr_struct, addr): From fijal at codespeak.net Tue Dec 8 16:34:57 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 16:34:57 +0100 (CET) Subject: [pypy-svn] r69980 - pypy/branch/listcopyop/pypy/rpython/memory/gc Message-ID: <20091208153457.1B6C1168043@codespeak.net> Author: fijal Date: Tue Dec 8 16:34:56 2009 New Revision: 69980 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Log: Be a bit less eager when copying flags over. Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Tue Dec 8 16:34:56 2009 @@ -493,13 +493,15 @@ """ typeid = self.get_type_id(source_addr) assert self.is_gcarrayofgcptr(typeid) - need_clear = False + objhdr = self.header(dest_addr) if self.header(source_addr).tid & GCFLAG_NO_YOUNG_PTRS == 0: - need_clear = True + if objhdr.tid & GCFLAG_NO_YOUNG_PTRS: + self.old_objects_pointing_to_young.append(dest_addr) + objhdr.tid &= ~GCFLAG_NO_YOUNG_PTRS if self.header(source_addr).tid & GCFLAG_NO_HEAP_PTRS == 0: - need_clear = True - if need_clear: - self.assume_young_pointers(dest_addr) + if objhdr.tid & GCFLAG_NO_HEAP_PTRS: + objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS + self.last_generation_root_objects.append(dest_addr) def write_into_last_generation_obj(self, addr_struct, addr): objhdr = self.header(addr_struct) From pedronis at codespeak.net Tue Dec 8 16:49:13 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 8 Dec 2009 16:49:13 +0100 (CET) Subject: [pypy-svn] r69981 - in pypy/trunk/pypy/jit: backend/cli/test backend/llsupport backend/llsupport/test backend/llvm/test backend/x86 backend/x86/test metainterp Message-ID: <20091208154913.44C27168043@codespeak.net> Author: pedronis Date: Tue Dec 8 16:49:12 2009 New Revision: 69981 Modified: pypy/trunk/pypy/jit/backend/cli/test/conftest.py (props changed) pypy/trunk/pypy/jit/backend/llsupport/regalloc.py pypy/trunk/pypy/jit/backend/llsupport/test/test_gc.py pypy/trunk/pypy/jit/backend/llsupport/test/test_regalloc.py pypy/trunk/pypy/jit/backend/llvm/test/conftest.py (props changed) pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/jump.py pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py (contents, props changed) pypy/trunk/pypy/jit/backend/x86/test/test_jump.py pypy/trunk/pypy/jit/backend/x86/test/test_recompilation.py pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/trunk/pypy/jit/metainterp/logger.py (props changed) Log: merge esp-params, avoid esp adjustments around most emitted calls, instead reserve globally space for params after the frame locations Modified: pypy/trunk/pypy/jit/backend/llsupport/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/regalloc.py Tue Dec 8 16:49:12 2009 @@ -12,28 +12,28 @@ class NoVariableToSpill(Exception): pass -class StackManager(object): - """ Manage stack positions +class FrameManager(object): + """ Manage frame positions """ def __init__(self): - self.stack_bindings = {} - self.stack_depth = 0 + self.frame_bindings = {} + self.frame_depth = 0 def get(self, box): - return self.stack_bindings.get(box, None) + return self.frame_bindings.get(box, None) def loc(self, box, size): res = self.get(box) if res is not None: return res - newloc = self.stack_pos(self.stack_depth, size) - self.stack_bindings[box] = newloc - self.stack_depth += size + newloc = self.frame_pos(self.frame_depth, size) + self.frame_bindings[box] = newloc + self.frame_depth += size return newloc # abstract methods that need to be overwritten for specific assemblers @staticmethod - def stack_pos(loc, size): + def frame_pos(loc, size): raise NotImplementedError("Purely abstract") class RegisterManager(object): @@ -45,12 +45,12 @@ save_around_call_regs = [] reg_width = 1 # in terms of stack space eaten - def __init__(self, longevity, stack_manager=None, assembler=None): + def __init__(self, longevity, frame_manager=None, assembler=None): self.free_regs = self.all_regs[:] self.longevity = longevity self.reg_bindings = {} self.position = -1 - self.stack_manager = stack_manager + self.frame_manager = frame_manager self.assembler = assembler def stays_alive(self, v): @@ -147,8 +147,8 @@ selected_reg, need_lower_byte=need_lower_byte) loc = self.reg_bindings[v_to_spill] del self.reg_bindings[v_to_spill] - if self.stack_manager.get(v_to_spill) is None: - newloc = self.stack_manager.loc(v_to_spill, self.reg_width) + if self.frame_manager.get(v_to_spill) is None: + newloc = self.frame_manager.loc(v_to_spill, self.reg_width) self.assembler.regalloc_mov(loc, newloc) return loc @@ -204,7 +204,7 @@ try: return self.reg_bindings[box] except KeyError: - return self.stack_manager.loc(box, self.reg_width) + return self.frame_manager.loc(box, self.reg_width) def return_constant(self, v, forbidden_vars=[], selected_reg=None, imm_fine=True): @@ -260,7 +260,7 @@ self.reg_bindings[v] = loc self.assembler.regalloc_mov(prev_loc, loc) else: - loc = self.stack_manager.loc(v, self.reg_width) + loc = self.frame_manager.loc(v, self.reg_width) self.assembler.regalloc_mov(prev_loc, loc) def force_result_in_reg(self, result_v, v, forbidden_vars=[]): @@ -280,7 +280,7 @@ self.free_regs = [reg for reg in self.free_regs if reg is not loc] return loc if v not in self.reg_bindings: - prev_loc = self.stack_manager.loc(v, self.reg_width) + prev_loc = self.frame_manager.loc(v, self.reg_width) loc = self.force_allocate_reg(v, forbidden_vars) self.assembler.regalloc_mov(prev_loc, loc) assert v in self.reg_bindings @@ -289,7 +289,7 @@ # store result in the same place loc = self.reg_bindings[v] del self.reg_bindings[v] - if self.stack_manager.get(v) is None: + if self.frame_manager.get(v) is None: self._move_variable_away(v, loc) self.reg_bindings[result_v] = loc else: @@ -298,9 +298,9 @@ return loc def _sync_var(self, v): - if not self.stack_manager.get(v): + if not self.frame_manager.get(v): reg = self.reg_bindings[v] - to = self.stack_manager.loc(v, self.reg_width) + to = self.frame_manager.loc(v, self.reg_width) self.assembler.regalloc_mov(reg, to) # otherwise it's clean Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/test/test_gc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/test/test_gc.py Tue Dec 8 16:49:12 2009 @@ -61,11 +61,11 @@ assert addrs[i].address[0] == llmemory.cast_ptr_to_adr(allocs[i]) def test_GcRootMap_asmgcc(): - def stack_pos(n): + def frame_pos(n): return -4*(4+n) gcrootmap = GcRootMap_asmgcc() - num1 = stack_pos(1) - num2 = stack_pos(55) + num1 = frame_pos(1) + num2 = frame_pos(55) shape = gcrootmap.get_basic_shape() gcrootmap.add_ebp_offset(shape, num1) gcrootmap.add_ebp_offset(shape, num2) @@ -99,7 +99,7 @@ expected_shapeaddr = {} for i in range(1, 600): shape = gcrootmap.get_basic_shape() - gcrootmap.add_ebp_offset(shape, stack_pos(i)) + gcrootmap.add_ebp_offset(shape, frame_pos(i)) shapeaddr = gcrootmap.compress_callshape(shape) expected_shapeaddr[i] = shapeaddr retaddr = rffi.cast(llmemory.Address, 123456789 + i) Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/test/test_regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/test/test_regalloc.py Tue Dec 8 16:49:12 2009 @@ -1,6 +1,6 @@ from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat -from pypy.jit.backend.llsupport.regalloc import StackManager +from pypy.jit.backend.llsupport.regalloc import FrameManager from pypy.jit.backend.llsupport.regalloc import RegisterManager as BaseRegMan def newboxes(*values): @@ -26,8 +26,8 @@ def convert_to_imm(self, v): return v -class TStackManager(StackManager): - def stack_pos(self, i, size): +class TFrameManager(FrameManager): + def frame_pos(self, i, size): return i class MockAsm(object): @@ -110,13 +110,13 @@ def test_force_allocate_reg(self): boxes, longevity = boxes_and_longevity(5) b0, b1, b2, b3, b4 = boxes - sm = TStackManager() + fm = TFrameManager() class XRegisterManager(RegisterManager): no_lower_byte_regs = [r2, r3] rm = XRegisterManager(longevity, - stack_manager=sm, + frame_manager=fm, assembler=MockAsm()) rm.next_instruction() loc = rm.force_allocate_reg(b0) @@ -140,13 +140,13 @@ def test_make_sure_var_in_reg(self): boxes, longevity = boxes_and_longevity(5) - sm = TStackManager() - rm = RegisterManager(longevity, stack_manager=sm, + fm = TFrameManager() + rm = RegisterManager(longevity, frame_manager=fm, assembler=MockAsm()) rm.next_instruction() # allocate a stack position b0, b1, b2, b3, b4 = boxes - sp = sm.loc(b0, 1) + sp = fm.loc(b0, 1) assert sp == 0 loc = rm.make_sure_var_in_reg(b0) assert isinstance(loc, FakeReg) @@ -155,9 +155,9 @@ def test_force_result_in_reg_1(self): b0, b1 = newboxes(0, 0) longevity = {b0: (0, 1), b1: (1, 3)} - sm = TStackManager() + fm = TFrameManager() asm = MockAsm() - rm = RegisterManager(longevity, stack_manager=sm, assembler=asm) + rm = RegisterManager(longevity, frame_manager=fm, assembler=asm) rm.next_instruction() # first path, var is already in reg and dies loc0 = rm.force_allocate_reg(b0) @@ -171,9 +171,9 @@ def test_force_result_in_reg_2(self): b0, b1 = newboxes(0, 0) longevity = {b0: (0, 2), b1: (1, 3)} - sm = TStackManager() + fm = TFrameManager() asm = MockAsm() - rm = RegisterManager(longevity, stack_manager=sm, assembler=asm) + rm = RegisterManager(longevity, frame_manager=fm, assembler=asm) rm.next_instruction() loc0 = rm.force_allocate_reg(b0) rm._check_invariants() @@ -187,9 +187,9 @@ def test_force_result_in_reg_3(self): b0, b1, b2, b3, b4 = newboxes(0, 0, 0, 0, 0) longevity = {b0: (0, 2), b1: (0, 2), b3: (0, 2), b2: (0, 2), b4: (1, 3)} - sm = TStackManager() + fm = TFrameManager() asm = MockAsm() - rm = RegisterManager(longevity, stack_manager=sm, assembler=asm) + rm = RegisterManager(longevity, frame_manager=fm, assembler=asm) rm.next_instruction() for b in b0, b1, b2, b3: rm.force_allocate_reg(b) @@ -203,11 +203,11 @@ def test_force_result_in_reg_4(self): b0, b1 = newboxes(0, 0) longevity = {b0: (0, 1), b1: (0, 1)} - sm = TStackManager() + fm = TFrameManager() asm = MockAsm() - rm = RegisterManager(longevity, stack_manager=sm, assembler=asm) + rm = RegisterManager(longevity, frame_manager=fm, assembler=asm) rm.next_instruction() - sm.loc(b0, 1) + fm.loc(b0, 1) rm.force_result_in_reg(b1, b0) rm._check_invariants() loc = rm.loc(b1) @@ -219,9 +219,9 @@ def test_return_constant(self): asm = MockAsm() boxes, longevity = boxes_and_longevity(5) - sm = TStackManager() + fm = TFrameManager() rm = RegisterManager(longevity, assembler=asm, - stack_manager=sm) + frame_manager=fm) rm.next_instruction() loc = rm.return_constant(ConstInt(0), imm_fine=False) assert isinstance(loc, FakeReg) @@ -241,9 +241,9 @@ def test_force_result_in_reg_const(self): boxes, longevity = boxes_and_longevity(2) - sm = TStackManager() + fm = TFrameManager() asm = MockAsm() - rm = RegisterManager(longevity, stack_manager=sm, + rm = RegisterManager(longevity, frame_manager=fm, assembler=asm) rm.next_instruction() c = ConstInt(0) @@ -262,16 +262,16 @@ def call_result_location(self, v): return r1 - sm = TStackManager() + fm = TFrameManager() asm = MockAsm() boxes, longevity = boxes_and_longevity(5) - rm = XRegisterManager(longevity, stack_manager=sm, + rm = XRegisterManager(longevity, frame_manager=fm, assembler=asm) for b in boxes[:-1]: rm.force_allocate_reg(b) rm.before_call() assert len(rm.reg_bindings) == 2 - assert sm.stack_depth == 2 + assert fm.frame_depth == 2 assert len(asm.moves) == 2 rm._check_invariants() rm.after_call(boxes[-1]) @@ -285,16 +285,16 @@ def call_result_location(self, v): return r1 - sm = TStackManager() + fm = TFrameManager() asm = MockAsm() boxes, longevity = boxes_and_longevity(5) - rm = XRegisterManager(longevity, stack_manager=sm, + rm = XRegisterManager(longevity, frame_manager=fm, assembler=asm) for b in boxes[:-1]: rm.force_allocate_reg(b) rm.before_call(save_all_regs=True) assert len(rm.reg_bindings) == 0 - assert sm.stack_depth == 4 + assert fm.frame_depth == 4 assert len(asm.moves) == 4 rm._check_invariants() rm.after_call(boxes[-1]) @@ -302,20 +302,20 @@ rm._check_invariants() - def test_different_stack_width(self): + def test_different_frame_width(self): class XRegisterManager(RegisterManager): reg_width = 2 - sm = TStackManager() + fm = TFrameManager() b0 = BoxInt() longevity = {b0: (0, 1)} asm = MockAsm() - rm = RegisterManager(longevity, stack_manager=sm, assembler=asm) + rm = RegisterManager(longevity, frame_manager=fm, assembler=asm) f0 = BoxFloat() longevity = {f0: (0, 1)} - xrm = XRegisterManager(longevity, stack_manager=sm, assembler=asm) + xrm = XRegisterManager(longevity, frame_manager=fm, assembler=asm) xrm.loc(f0) rm.loc(b0) - assert sm.stack_depth == 3 + assert fm.frame_depth == 3 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Tue Dec 8 16:49:12 2009 @@ -157,7 +157,8 @@ """adds the following attributes to looptoken: _x86_loop_code (an integer giving an address) _x86_bootstrap_code (an integer giving an address) - _x86_stack_depth + _x86_frame_depth + _x86_param_depth _x86_arglocs """ self.make_sure_mc_exists() @@ -167,10 +168,12 @@ looptoken._x86_bootstrap_code = self.mc.tell() adr_stackadjust = self._assemble_bootstrap_code(inputargs, arglocs) looptoken._x86_loop_code = self.mc.tell() - looptoken._x86_stack_depth = -1 # temporarily - stack_depth = self._assemble(regalloc, operations) - self._patch_stackadjust(adr_stackadjust, stack_depth) - looptoken._x86_stack_depth = stack_depth + looptoken._x86_frame_depth = -1 # temporarily + looptoken._x86_param_depth = -1 # temporarily + frame_depth, param_depth = self._assemble(regalloc, operations) + self._patch_stackadjust(adr_stackadjust, frame_depth+param_depth) + looptoken._x86_frame_depth = frame_depth + looptoken._x86_param_depth = param_depth def assemble_bridge(self, faildescr, inputargs, operations): self.make_sure_mc_exists() @@ -180,16 +183,17 @@ assert ([loc.assembler() for loc in arglocs] == [loc.assembler() for loc in faildescr._x86_debug_faillocs]) regalloc = RegAlloc(self, self.cpu.translate_support_code) - fail_stack_depth = faildescr._x86_current_stack_depth - regalloc.prepare_bridge(fail_stack_depth, inputargs, arglocs, + fail_depths = faildescr._x86_current_depths + regalloc.prepare_bridge(fail_depths, inputargs, arglocs, operations) adr_bridge = self.mc.tell() adr_stackadjust = self._patchable_stackadjust() - stack_depth = self._assemble(regalloc, operations) - self._patch_stackadjust(adr_stackadjust, stack_depth) + frame_depth, param_depth = self._assemble(regalloc, operations) + self._patch_stackadjust(adr_stackadjust, frame_depth+param_depth) if not we_are_translated(): # for the benefit of tests - faildescr._x86_bridge_stack_depth = stack_depth + faildescr._x86_bridge_frame_depth = frame_depth + faildescr._x86_bridge_param_depth = param_depth # patch the jump from original guard self.patch_jump(faildescr, adr_bridge) @@ -207,26 +211,29 @@ self.mc2.done() if we_are_translated() or self.cpu.dont_keepalive_stuff: self._regalloc = None # else keep it around for debugging - stack_depth = regalloc.sm.stack_depth + frame_depth = regalloc.fm.frame_depth + param_depth = regalloc.param_depth jump_target_descr = regalloc.jump_target_descr if jump_target_descr is not None: - target_stack_depth = jump_target_descr._x86_stack_depth - stack_depth = max(stack_depth, target_stack_depth) - return stack_depth + target_frame_depth = jump_target_descr._x86_frame_depth + target_param_depth = jump_target_descr._x86_param_depth + frame_depth = max(frame_depth, target_frame_depth) + param_depth = max(param_depth, target_param_depth) + return frame_depth, param_depth def _patchable_stackadjust(self): # stack adjustment LEA self.mc.LEA(esp, fixedsize_ebp_ofs(0)) return self.mc.tell() - 4 - def _patch_stackadjust(self, adr_lea, stack_depth): + def _patch_stackadjust(self, adr_lea, reserved_depth): # patch stack adjustment LEA # possibly align, e.g. for Mac OS X mc = codebuf.InMemoryCodeBuilder(adr_lea, adr_lea + 4) # Compute the correct offset for the instruction LEA ESP, [EBP-4*words]. # Given that [EBP] is where we saved EBP, i.e. in the last word # of our fixed frame, then the 'words' value is: - words = (FRAME_FIXED_SIZE - 1) + stack_depth + words = (FRAME_FIXED_SIZE - 1) + reserved_depth mc.write(packimm32(-WORD * words)) mc.done() @@ -322,10 +329,10 @@ genop_discard_list[op.opnum](self, op, arglocs) def regalloc_perform_with_guard(self, op, guard_op, faillocs, - arglocs, resloc, current_stack_depth): + arglocs, resloc, current_depths): faildescr = guard_op.descr assert isinstance(faildescr, AbstractFailDescr) - faildescr._x86_current_stack_depth = current_stack_depth + faildescr._x86_current_depths = current_depths failargs = guard_op.fail_args guard_opnum = guard_op.opnum failaddr = self.implement_guard_recovery(guard_opnum, @@ -342,9 +349,9 @@ faildescr._x86_adr_jump_offset = adr_jump_offset def regalloc_perform_guard(self, guard_op, faillocs, arglocs, resloc, - current_stack_depth): + current_depths): self.regalloc_perform_with_guard(None, guard_op, faillocs, arglocs, - resloc, current_stack_depth) + resloc, current_depths) def load_effective_addr(self, sizereg, baseofs, scale, result): self.mc.LEA(result, addr_add(imm(0), sizereg, baseofs, scale)) @@ -407,14 +414,34 @@ ## self.mc.PUSH(imm(0)) --- or just use a single SUB(esp, imm) ## return extra_on_stack - def call(self, addr, args, res): - nargs = len(args) - extra_on_stack = nargs #self.align_stack_for_call(nargs) - for i in range(nargs-1, -1, -1): - self.mc.PUSH(args[i]) - self.mc.CALL(rel32(addr)) + def _emit_call(self, x, arglocs, start=0, tmp=eax): + p = 0 + n = len(arglocs) + for i in range(start, n): + loc = arglocs[i] + if isinstance(loc, REG): + if isinstance(loc, XMMREG): + self.mc.MOVSD(mem64(esp, p), loc) + else: + self.mc.MOV(mem(esp, p), loc) + p += round_up_to_4(loc.width) + p = 0 + for i in range(start, n): + loc = arglocs[i] + if not isinstance(loc, REG): + if isinstance(loc, MODRM64): + self.mc.MOVSD(xmm0, loc) + self.mc.MOVSD(mem64(esp, p), xmm0) + else: + self.mc.MOV(tmp, loc) + self.mc.MOV(mem(esp, p), tmp) + p += round_up_to_4(loc.width) + self._regalloc.reserve_param(p//WORD) + self.mc.CALL(x) self.mark_gc_roots() - self.mc.ADD(esp, imm(extra_on_stack * WORD)) + + def call(self, addr, args, res): + self._emit_call(rel32(addr), args) assert res is eax genop_int_neg = _unaryop("NEG") @@ -858,7 +885,7 @@ self.fail_boxes_float.get_addr_for_num(i) def rebuild_faillocs_from_descr(self, bytecode): - from pypy.jit.backend.x86.regalloc import X86StackManager + from pypy.jit.backend.x86.regalloc import X86FrameManager bytecode = rffi.cast(rffi.UCHARP, bytecode) arglocs = [] while 1: @@ -883,7 +910,7 @@ size = 2 else: size = 1 - loc = X86StackManager.stack_pos(code, size) + loc = X86FrameManager.frame_pos(code, size) elif code == self.CODE_STOP: break elif code == self.CODE_HOLE: @@ -1121,12 +1148,7 @@ sizeloc = arglocs[0] assert isinstance(sizeloc, IMM32) size = sizeloc.value - nargs = len(op.args)-1 - extra_on_stack = 0 - for arg in range(2, nargs + 2): - extra_on_stack += round_up_to_4(arglocs[arg].width) - #extra_on_stack = self.align_stack_for_call(extra_on_stack) - self.mc.SUB(esp, imm(extra_on_stack)) + if isinstance(op.args[0], Const): x = rel32(op.args[0].getint()) else: @@ -1135,29 +1157,9 @@ tmp = ecx else: tmp = eax - p = 0 - for i in range(2, nargs + 2): - loc = arglocs[i] - if isinstance(loc, REG): - if isinstance(loc, XMMREG): - self.mc.MOVSD(mem64(esp, p), loc) - else: - self.mc.MOV(mem(esp, p), loc) - p += round_up_to_4(loc.width) - p = 0 - for i in range(2, nargs + 2): - loc = arglocs[i] - if not isinstance(loc, REG): - if isinstance(loc, MODRM64): - self.mc.MOVSD(xmm0, loc) - self.mc.MOVSD(mem64(esp, p), xmm0) - else: - self.mc.MOV(tmp, loc) - self.mc.MOV(mem(esp, p), tmp) - p += round_up_to_4(loc.width) - self.mc.CALL(x) - self.mark_gc_roots() - self.mc.ADD(esp, imm(extra_on_stack)) + + self._emit_call(x, arglocs, 2, tmp=tmp) + if isinstance(resloc, MODRM64): self.mc.FSTP(resloc) elif size == 1: @@ -1245,14 +1247,13 @@ mc.CMP(edx, heap(nursery_top_adr)) mc.write(constlistofchars('\x76\x00')) # JNA after the block jmp_adr = mc.get_relative_pos() - mc.PUSH(imm(size)) - mc.CALL(rel32(slowpath_addr)) - self.mark_gc_roots() + self._emit_call(rel32(slowpath_addr), [imm(size)]) + # note that slowpath_addr returns a "long long", or more precisely # two results, which end up in eax and edx. # eax should contain the result of allocation, edx new value # of nursery_free_adr - mc.ADD(esp, imm(4)) + offset = mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 mc.overwrite(jmp_adr-1, [chr(offset)]) Modified: pypy/trunk/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/jump.py (original) +++ pypy/trunk/pypy/jit/backend/x86/jump.py Tue Dec 8 16:49:12 2009 @@ -5,7 +5,7 @@ class __extend__(OPERAND): __metaclass__ = extendabletype def _getregkey(self): - raise AssertionError("should only happen to registers and stack " + raise AssertionError("should only happen to registers and frame " "positions") class __extend__(REG): @@ -19,7 +19,7 @@ return self.position -def remap_stack_layout(assembler, src_locations, dst_locations, tmpreg): +def remap_frame_layout(assembler, src_locations, dst_locations, tmpreg): pending_dests = len(dst_locations) srccount = {} # maps dst_locations to how many times the same # location appears in src_locations Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Tue Dec 8 16:49:12 2009 @@ -11,11 +11,11 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib import rgc from pypy.jit.backend.llsupport import symbolic -from pypy.jit.backend.x86.jump import remap_stack_layout +from pypy.jit.backend.x86.jump import remap_frame_layout from pypy.jit.metainterp.resoperation import rop from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr -from pypy.jit.backend.llsupport.regalloc import StackManager, RegisterManager,\ +from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\ TempBox WORD = 4 @@ -67,8 +67,8 @@ return lltype.malloc(rffi.CArray(lltype.Float), BASE_CONSTANT_SIZE, flavor='raw') - def __init__(self, longevity, stack_manager=None, assembler=None): - RegisterManager.__init__(self, longevity, stack_manager=stack_manager, + def __init__(self, longevity, frame_manager=None, assembler=None): + RegisterManager.__init__(self, longevity, frame_manager=frame_manager, assembler=assembler) self.constant_arrays = [self.new_const_array()] self.constant_array_counter = 0 @@ -89,14 +89,14 @@ def after_call(self, v): # the result is stored in st0, but we don't have this around, - # so genop_call will move it to some stack location immediately + # so genop_call will move it to some frame location immediately # after the call - return self.stack_manager.loc(v, 2) + return self.frame_manager.loc(v, 2) -class X86StackManager(StackManager): +class X86FrameManager(FrameManager): @staticmethod - def stack_pos(i, size): + def frame_pos(i, size): if size == 1: res = mem(ebp, get_ebp_ofs(i)) elif size == 2: @@ -119,16 +119,17 @@ self.jump_target_descr = None def _prepare(self, inputargs, operations): - self.sm = X86StackManager() + self.fm = X86FrameManager() + self.param_depth = 0 cpu = self.assembler.cpu cpu.gc_ll_descr.rewrite_assembler(cpu, operations) # compute longevity of variables longevity = self._compute_vars_longevity(inputargs, operations) self.longevity = longevity self.rm = X86RegisterManager(longevity, - stack_manager = self.sm, + frame_manager = self.fm, assembler = self.assembler) - self.xrm = X86XMMRegisterManager(longevity, stack_manager = self.sm, + self.xrm = X86XMMRegisterManager(longevity, frame_manager = self.fm, assembler = self.assembler) def prepare_loop(self, inputargs, operations, looptoken): @@ -138,11 +139,15 @@ self.loop_consts = loop_consts return self._process_inputargs(inputargs) - def prepare_bridge(self, prev_stack_depth, inputargs, arglocs, operations): + def prepare_bridge(self, prev_depths, inputargs, arglocs, operations): self._prepare(inputargs, operations) self.loop_consts = {} self._update_bindings(arglocs, inputargs) - self.sm.stack_depth = prev_stack_depth + self.fm.frame_depth = prev_depths[0] + self.param_depth = prev_depths[1] + + def reserve_param(self, n): + self.param_depth = max(self.param_depth, n) def _process_inputargs(self, inputargs): # XXX we can sort out here by longevity if we need something @@ -170,7 +175,7 @@ if reg: loc = reg else: - loc = self.sm.loc(arg, width_of_type[arg.type]) + loc = self.fm.loc(arg, width_of_type[arg.type]) if arg.type == FLOAT: floatlocs[i] = loc else: @@ -241,13 +246,13 @@ self.xrm.reg_bindings[arg] = loc used[loc] = None else: - self.sm.stack_bindings[arg] = loc + self.fm.frame_bindings[arg] = loc else: if isinstance(loc, REG): self.rm.reg_bindings[arg] = loc used[loc] = None else: - self.sm.stack_bindings[arg] = loc + self.fm.frame_bindings[arg] = loc self.rm.free_regs = [] for reg in X86RegisterManager.all_regs: if reg not in used: @@ -274,9 +279,10 @@ faillocs = self.locs_for_fail(guard_op) self.rm.position += 1 self.xrm.position += 1 + current_depths = (self.fm.frame_depth, self.param_depth) self.assembler.regalloc_perform_with_guard(op, guard_op, faillocs, arglocs, result_loc, - self.sm.stack_depth) + current_depths) if op.result is not None: self.possibly_free_var(op.result) self.possibly_free_vars(guard_op.fail_args) @@ -289,9 +295,10 @@ arglocs)) else: self.assembler.dump('%s(%s)' % (guard_op, arglocs)) + current_depths = (self.fm.frame_depth, self.param_depth) self.assembler.regalloc_perform_guard(guard_op, faillocs, arglocs, result_loc, - self.sm.stack_depth) + current_depths) self.possibly_free_vars(guard_op.fail_args) def PerformDiscard(self, op, arglocs): @@ -888,11 +895,11 @@ src_locations = [self.loc(arg) for arg in op.args if arg.type != FLOAT] assert tmploc not in nonfloatlocs dst_locations = [loc for loc in nonfloatlocs if loc is not None] - remap_stack_layout(assembler, src_locations, dst_locations, tmploc) + remap_frame_layout(assembler, src_locations, dst_locations, tmploc) # Part about floats src_locations = [self.loc(arg) for arg in op.args if arg.type == FLOAT] dst_locations = [loc for loc in floatlocs if loc is not None] - remap_stack_layout(assembler, src_locations, dst_locations, xmmtmp) + remap_frame_layout(assembler, src_locations, dst_locations, xmmtmp) self.rm.possibly_free_var(box) self.xrm.possibly_free_var(box1) self.possibly_free_vars(op.args) @@ -903,7 +910,7 @@ def get_mark_gc_roots(self, gcrootmap): shape = gcrootmap.get_basic_shape() - for v, val in self.sm.stack_bindings.items(): + for v, val in self.fm.frame_bindings.items(): if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): assert isinstance(val, MODRM) gcrootmap.add_ebp_offset(shape, get_ebp_ofs(val.position)) @@ -937,7 +944,7 @@ oplist[num] = value def get_ebp_ofs(position): - # Argument is a stack position (0, 1, 2...). + # Argument is a frame position (0, 1, 2...). # Returns (ebp-20), (ebp-24), (ebp-28)... # i.e. the n'th word beyond the fixed frame size. return -WORD * (FRAME_FIXED_SIZE + position) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py Tue Dec 8 16:49:12 2009 @@ -1,6 +1,6 @@ from pypy.jit.backend.x86.ri386 import * from pypy.jit.backend.x86.assembler import Assembler386 -from pypy.jit.backend.x86.regalloc import X86StackManager, get_ebp_ofs +from pypy.jit.backend.x86.regalloc import X86FrameManager, get_ebp_ofs from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat from pypy.rlib.rarithmetic import intmask from pypy.rpython.lltypesystem import lltype, llmemory, rffi @@ -23,12 +23,12 @@ failargs = [BoxInt(), BoxPtr(), BoxFloat()] * 3 failargs.insert(6, None) failargs.insert(7, None) - locs = [X86StackManager.stack_pos(0, 1), - X86StackManager.stack_pos(1, 1), - X86StackManager.stack_pos(10, 2), - X86StackManager.stack_pos(100, 1), - X86StackManager.stack_pos(101, 1), - X86StackManager.stack_pos(110, 2), + locs = [X86FrameManager.frame_pos(0, 1), + X86FrameManager.frame_pos(1, 1), + X86FrameManager.frame_pos(10, 2), + X86FrameManager.frame_pos(100, 1), + X86FrameManager.frame_pos(101, 1), + X86FrameManager.frame_pos(110, 2), None, None, ebx, Modified: pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py Tue Dec 8 16:49:12 2009 @@ -19,7 +19,7 @@ from pypy.jit.backend.x86.test.test_regalloc import MockAssembler from pypy.jit.backend.x86.test.test_regalloc import BaseTestRegalloc -from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86StackManager,\ +from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86FrameManager,\ X86XMMRegisterManager from pypy.rpython.annlowlevel import llhelper @@ -64,10 +64,10 @@ longevity = {} for box in boxes: longevity[box] = (0, 1) - regalloc.sm = X86StackManager() - regalloc.rm = X86RegisterManager(longevity, regalloc.sm, + regalloc.fm = X86FrameManager() + regalloc.rm = X86RegisterManager(longevity, regalloc.fm, assembler=regalloc.assembler) - regalloc.xrm = X86XMMRegisterManager(longevity, regalloc.sm, + regalloc.xrm = X86XMMRegisterManager(longevity, regalloc.fm, assembler=regalloc.assembler) cpu = regalloc.assembler.cpu for box in boxes: Modified: pypy/trunk/pypy/jit/backend/x86/test/test_jump.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_jump.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_jump.py Tue Dec 8 16:49:12 2009 @@ -1,8 +1,8 @@ from pypy.jit.backend.x86.ri386 import * -from pypy.jit.backend.x86.regalloc import X86StackManager -from pypy.jit.backend.x86.jump import remap_stack_layout +from pypy.jit.backend.x86.regalloc import X86FrameManager +from pypy.jit.backend.x86.jump import remap_frame_layout -stack_pos = X86StackManager.stack_pos +frame_pos = X86FrameManager.frame_pos class MockAssembler: def __init__(self): @@ -36,33 +36,33 @@ def test_trivial(): assembler = MockAssembler() - remap_stack_layout(assembler, [], [], '?') + remap_frame_layout(assembler, [], [], '?') assert assembler.ops == [] - remap_stack_layout(assembler, [eax, ebx, ecx, edx, esi, edi], + remap_frame_layout(assembler, [eax, ebx, ecx, edx, esi, edi], [eax, ebx, ecx, edx, esi, edi], '?') assert assembler.ops == [] - s8 = stack_pos(1, 1) - s12 = stack_pos(31, 1) - s20 = stack_pos(6, 1) - remap_stack_layout(assembler, [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], + s8 = frame_pos(1, 1) + s12 = frame_pos(31, 1) + s20 = frame_pos(6, 1) + remap_frame_layout(assembler, [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], '?') assert assembler.ops == [] def test_simple_registers(): assembler = MockAssembler() - remap_stack_layout(assembler, [eax, ebx, ecx], [edx, esi, edi], '?') + remap_frame_layout(assembler, [eax, ebx, ecx], [edx, esi, edi], '?') assert assembler.ops == [('mov', eax, edx), ('mov', ebx, esi), ('mov', ecx, edi)] -def test_simple_stacklocs(): +def test_simple_framelocs(): assembler = MockAssembler() - s8 = stack_pos(0, 1) - s12 = stack_pos(13, 1) - s20 = stack_pos(20, 1) - s24 = stack_pos(221, 1) - remap_stack_layout(assembler, [s8, eax, s12], [s20, s24, edi], edx) + s8 = frame_pos(0, 1) + s12 = frame_pos(13, 1) + s20 = frame_pos(20, 1) + s24 = frame_pos(221, 1) + remap_frame_layout(assembler, [s8, eax, s12], [s20, s24, edi], edx) assert assembler.ops == [('mov', s8, edx), ('mov', edx, s20), ('mov', eax, s24), @@ -70,11 +70,11 @@ def test_reordering(): assembler = MockAssembler() - s8 = stack_pos(8, 1) - s12 = stack_pos(12, 1) - s20 = stack_pos(19, 1) - s24 = stack_pos(1, 1) - remap_stack_layout(assembler, [eax, s8, s20, ebx], + s8 = frame_pos(8, 1) + s12 = frame_pos(12, 1) + s20 = frame_pos(19, 1) + s24 = frame_pos(1, 1) + remap_frame_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, edi], '?') assert assembler.got([('mov', ebx, edi), ('mov', s8, ebx), @@ -83,11 +83,11 @@ def test_cycle(): assembler = MockAssembler() - s8 = stack_pos(8, 1) - s12 = stack_pos(12, 1) - s20 = stack_pos(19, 1) - s24 = stack_pos(1, 1) - remap_stack_layout(assembler, [eax, s8, s20, ebx], + s8 = frame_pos(8, 1) + s12 = frame_pos(12, 1) + s20 = frame_pos(19, 1) + s24 = frame_pos(1, 1) + remap_frame_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, s20], '?') assert assembler.got([('push', s8), ('mov', eax, s8), @@ -97,13 +97,13 @@ def test_cycle_2(): assembler = MockAssembler() - s8 = stack_pos(8, 1) - s12 = stack_pos(12, 1) - s20 = stack_pos(19, 1) - s24 = stack_pos(1, 1) - s2 = stack_pos(2, 1) - s3 = stack_pos(3, 1) - remap_stack_layout(assembler, + s8 = frame_pos(8, 1) + s12 = frame_pos(12, 1) + s20 = frame_pos(19, 1) + s24 = frame_pos(1, 1) + s2 = frame_pos(2, 1) + s3 = frame_pos(3, 1) + remap_frame_layout(assembler, [eax, s8, edi, s20, eax, s20, s24, esi, s2, s3], [s8, s20, edi, eax, edx, s24, ebx, s12, s3, s2], ecx) @@ -124,18 +124,18 @@ def test_constants(): assembler = MockAssembler() c3 = imm(3) - remap_stack_layout(assembler, [c3], [eax], '?') + remap_frame_layout(assembler, [c3], [eax], '?') assert assembler.ops == [('mov', c3, eax)] assembler = MockAssembler() - s12 = stack_pos(12, 1) - remap_stack_layout(assembler, [c3], [s12], '?') + s12 = frame_pos(12, 1) + remap_frame_layout(assembler, [c3], [s12], '?') assert assembler.ops == [('mov', c3, s12)] def test_constants_and_cycle(): assembler = MockAssembler() c3 = imm(3) - s12 = stack_pos(13, 1) - remap_stack_layout(assembler, [ebx, c3, s12], + s12 = frame_pos(13, 1) + remap_frame_layout(assembler, [ebx, c3, s12], [s12, eax, ebx], edi) assert assembler.ops == [('mov', c3, eax), ('push', s12), Modified: pypy/trunk/pypy/jit/backend/x86/test/test_recompilation.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_recompilation.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_recompilation.py Tue Dec 8 16:49:12 2009 @@ -33,7 +33,8 @@ jump(i1) ''' loop = self.interpret(ops, [0]) - previous = loop.token._x86_stack_depth + previous = loop.token._x86_frame_depth + assert loop.token._x86_param_depth == 0 assert self.getint(0) == 20 ops = ''' [i1] @@ -48,7 +49,8 @@ ''' bridge = self.attach_bridge(ops, loop, -2) descr = loop.operations[2].descr - new = descr._x86_bridge_stack_depth + new = descr._x86_bridge_frame_depth + assert descr._x86_bridge_param_depth == 0 assert new > previous self.cpu.set_future_value_int(0, 0) fail = self.run(loop) @@ -107,8 +109,10 @@ ''' bridge = self.attach_bridge(ops, loop, 5, looptoken=loop.token) guard_op = loop.operations[5] - loop_stack_depth = loop.token._x86_stack_depth - assert guard_op.descr._x86_bridge_stack_depth > loop_stack_depth + loop_frame_depth = loop.token._x86_frame_depth + assert loop.token._x86_param_depth == 0 + assert guard_op.descr._x86_bridge_frame_depth > loop_frame_depth + assert guard_op.descr._x86_bridge_param_depth == 0 self.cpu.set_future_value_int(0, 0) self.cpu.set_future_value_int(1, 0) self.cpu.set_future_value_int(2, 0) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py Tue Dec 8 16:49:12 2009 @@ -85,6 +85,20 @@ fdescr2 = BasicFailDescr(2) fdescr3 = BasicFailDescr(3) + def f1(x): + return x+1 + + def f2(x, y): + return x*y + + F1PTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Signed)) + F2PTR = lltype.Ptr(lltype.FuncType([lltype.Signed]*2, lltype.Signed)) + f1ptr = llhelper(F1PTR, f1) + f2ptr = llhelper(F2PTR, f2) + + f1_calldescr = cpu.calldescrof(F1PTR.TO, F1PTR.TO.ARGS, F1PTR.TO.RESULT) + f2_calldescr = cpu.calldescrof(F2PTR.TO, F2PTR.TO.ARGS, F2PTR.TO.RESULT) + namespace = locals().copy() type_system = 'lltype' @@ -522,3 +536,73 @@ ''' loop = self.interpret(ops, [0.0, .1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1] + +class TestRegAllocCallAndStackDepth(BaseTestRegalloc): + + def test_one_call(self): + ops = ''' + [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + i10 = call(ConstClass(f1ptr), i0, descr=f1_calldescr) + finish(i10, i1, i2, i3, i4, i5, i6, i7, i8, i9) + ''' + loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9, 9]) + assert self.getints(11) == [5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9] + assert loop.token._x86_param_depth == 1 + + def test_two_calls(self): + ops = ''' + [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + i10 = call(ConstClass(f1ptr), i0, descr=f1_calldescr) + i11 = call(ConstClass(f2ptr), i10, i1, descr=f2_calldescr) + finish(i11, i1, i2, i3, i4, i5, i6, i7, i8, i9) + ''' + loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9, 9]) + assert self.getints(11) == [5*7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9] + assert loop.token._x86_param_depth == 2 + + def test_bridge_calls_1(self): + ops = ''' + [i0, i1] + i2 = call(ConstClass(f1ptr), i0, descr=f1_calldescr) + guard_value(i2, 0, descr=fdescr1) [i2, i1] + finish(i1) + ''' + loop = self.interpret(ops, [4, 7]) + assert self.getint(0) == 5 + ops = ''' + [i2, i1] + i3 = call(ConstClass(f2ptr), i2, i1, descr=f2_calldescr) + finish(i3, descr=fdescr2) + ''' + bridge = self.attach_bridge(ops, loop, -2) + + assert loop.operations[-2].descr._x86_bridge_param_depth == 2 + + self.cpu.set_future_value_int(0, 4) + self.cpu.set_future_value_int(1, 7) + self.run(loop) + assert self.getint(0) == 5*7 + + def test_bridge_calls_2(self): + ops = ''' + [i0, i1] + i2 = call(ConstClass(f2ptr), i0, i1, descr=f2_calldescr) + guard_value(i2, 0, descr=fdescr1) [i2] + finish(i1) + ''' + loop = self.interpret(ops, [4, 7]) + assert self.getint(0) == 4*7 + ops = ''' + [i2] + i3 = call(ConstClass(f1ptr), i2, descr=f1_calldescr) + finish(i3, descr=fdescr2) + ''' + bridge = self.attach_bridge(ops, loop, -2) + + assert loop.operations[-2].descr._x86_bridge_param_depth == 2 + + self.cpu.set_future_value_int(0, 4) + self.cpu.set_future_value_int(1, 7) + self.run(loop) + assert self.getint(0) == 29 + Modified: pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py Tue Dec 8 16:49:12 2009 @@ -14,11 +14,8 @@ from pypy.rlib.jit import JitDriver, OPTIMIZER_SIMPLE, dont_look_inside from pypy.jit.backend.x86.runner import CPU386 from pypy.jit.backend.llsupport.gc import GcRefList, GcRootMap_asmgcc -from pypy.jit.backend.x86.regalloc import X86StackManager from pypy.tool.udir import udir -stack_pos = X86StackManager.stack_pos - class X(object): def __init__(self, x=0): self.x = x From pedronis at codespeak.net Tue Dec 8 16:50:14 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 8 Dec 2009 16:50:14 +0100 (CET) Subject: [pypy-svn] r69982 - in pypy/trunk/pypy: . module/exceptions objspace/std/test rlib/test translator/c/test Message-ID: <20091208155014.9DB50168043@codespeak.net> Author: pedronis Date: Tue Dec 8 16:50:13 2009 New Revision: 69982 Modified: pypy/trunk/pypy/ (props changed) pypy/trunk/pypy/module/exceptions/ (props changed) pypy/trunk/pypy/objspace/std/test/test_setobject.py (props changed) pypy/trunk/pypy/rlib/test/test_rcoroutine.py (props changed) pypy/trunk/pypy/translator/c/test/test_refcount.py (props changed) Log: oops, merge bookkeeping From fijal at codespeak.net Tue Dec 8 16:51:29 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 16:51:29 +0100 (CET) Subject: [pypy-svn] r69983 - pypy/branch/listcopyop/pypy/rpython/memory/gc Message-ID: <20091208155129.B44A0168043@codespeak.net> Author: fijal Date: Tue Dec 8 16:51:29 2009 New Revision: 69983 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Log: Rewrite it in the same way as writebarrier is written, comment Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Tue Dec 8 16:51:29 2009 @@ -480,29 +480,25 @@ each element in dest copied from source, except it might reset one of the following flags a bit too eagerly, which means we'll have a bit more objects to track, but being on the safe side. - - For both GCFLAG_NO_YOUNG_PTRS and GCFLAG_NO_HEAP_PTRS following - yields true: - - if the flag on source is set to 0 (means source either contains - young pointers or is in nursery), we need to clear the same flag on - dest, provided it's set already - - assume_young_pointers contains already the interface for setting, - we just need to decide when we need to set it. """ typeid = self.get_type_id(source_addr) assert self.is_gcarrayofgcptr(typeid) - objhdr = self.header(dest_addr) - if self.header(source_addr).tid & GCFLAG_NO_YOUNG_PTRS == 0: - if objhdr.tid & GCFLAG_NO_YOUNG_PTRS: - self.old_objects_pointing_to_young.append(dest_addr) - objhdr.tid &= ~GCFLAG_NO_YOUNG_PTRS - if self.header(source_addr).tid & GCFLAG_NO_HEAP_PTRS == 0: - if objhdr.tid & GCFLAG_NO_HEAP_PTRS: - objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS + source_hdr = self.header(source_addr) + dest_hdr = self.header(dest_addr) + if dest_hdr.tid & GCFLAG_NO_YOUNG_PTRS == 0: + return + # ^^^ a fast path of write-barrier + if source_hdr.tid & GCFLAG_NO_YOUNG_PTRS == 0: + # there might be an object in source that is in nursery + self.old_objects_pointing_to_young.append(dest_addr) + dest_hdr.tid &= ~GCFLAG_NO_YOUNG_PTRS + if dest_hdr.tid & GCFLAG_NO_HEAP_PTRS: + if source_hdr.tid & GCFLAG_NO_HEAP_PTRS == 0: + # ^^^ equivalend of addr from source not being in last + # gen + dest_hdr.tid &= ~GCFLAG_NO_HEAP_PTRS self.last_generation_root_objects.append(dest_addr) - + def write_into_last_generation_obj(self, addr_struct, addr): objhdr = self.header(addr_struct) if objhdr.tid & GCFLAG_NO_HEAP_PTRS: From pedronis at codespeak.net Tue Dec 8 16:53:12 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 8 Dec 2009 16:53:12 +0100 (CET) Subject: [pypy-svn] r69984 - pypy/branch/esp-params Message-ID: <20091208155312.F1D98168043@codespeak.net> Author: pedronis Date: Tue Dec 8 16:53:12 2009 New Revision: 69984 Removed: pypy/branch/esp-params/ Log: kill merged branch From arigo at codespeak.net Tue Dec 8 16:59:47 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 8 Dec 2009 16:59:47 +0100 (CET) Subject: [pypy-svn] r69985 - pypy/branch/listcopyop/pypy/rpython/memory/gc Message-ID: <20091208155947.CC13E168043@codespeak.net> Author: arigo Date: Tue Dec 8 16:59:47 2009 New Revision: 69985 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Log: * move write_into_last_generation_obj() closer to its call from remember_young_pointer(). * comment out an assert which is not strictly needed any more. Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Tue Dec 8 16:59:47 2009 @@ -466,6 +466,13 @@ remember_young_pointer._dont_inline_ = True self.remember_young_pointer = remember_young_pointer + def write_into_last_generation_obj(self, addr_struct, addr): + objhdr = self.header(addr_struct) + if objhdr.tid & GCFLAG_NO_HEAP_PTRS: + if not self.is_last_generation(addr): + objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS + self.last_generation_root_objects.append(addr_struct) + def assume_young_pointers(self, addr_struct): objhdr = self.header(addr_struct) if objhdr.tid & GCFLAG_NO_YOUNG_PTRS: @@ -481,8 +488,8 @@ one of the following flags a bit too eagerly, which means we'll have a bit more objects to track, but being on the safe side. """ - typeid = self.get_type_id(source_addr) - assert self.is_gcarrayofgcptr(typeid) + #typeid = self.get_type_id(source_addr) + #assert self.is_gcarrayofgcptr(typeid) ...? source_hdr = self.header(source_addr) dest_hdr = self.header(dest_addr) if dest_hdr.tid & GCFLAG_NO_YOUNG_PTRS == 0: @@ -498,13 +505,6 @@ # gen dest_hdr.tid &= ~GCFLAG_NO_HEAP_PTRS self.last_generation_root_objects.append(dest_addr) - - def write_into_last_generation_obj(self, addr_struct, addr): - objhdr = self.header(addr_struct) - if objhdr.tid & GCFLAG_NO_HEAP_PTRS: - if not self.is_last_generation(addr): - objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS - self.last_generation_root_objects.append(addr_struct) def is_last_generation(self, obj): # overridden by HybridGC From fijal at codespeak.net Tue Dec 8 17:07:53 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 17:07:53 +0100 (CET) Subject: [pypy-svn] r69986 - in pypy/branch/listcopyop/pypy: rlib rpython rpython/lltypesystem rpython/memory rpython/memory/gc rpython/memory/gctransform Message-ID: <20091208160753.DAB12168043@codespeak.net> Author: fijal Date: Tue Dec 8 17:07:52 2009 New Revision: 69986 Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py pypy/branch/listcopyop/pypy/rpython/llinterp.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py Log: Rename arraycopy_writebarrier to writebarrier_before_copy to reflect what it really does Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rlib/rgc.py (original) +++ pypy/branch/listcopyop/pypy/rlib/rgc.py Tue Dec 8 17:07:52 2009 @@ -338,7 +338,7 @@ if isinstance(TP.OF, lltype.Ptr) and TP.OF.TO._gckind == 'gc': # perform a write barrier that copies necessary flags from # source to dest - llop.gc_arraycopy_writebarrier(lltype.Void, source, dest) + llop.gc_writebarrier_before_copy(lltype.Void, source, dest) source_addr = llmemory.cast_ptr_to_adr(source) dest_addr = llmemory.cast_ptr_to_adr(dest) cp_source_addr = (source_addr + llmemory.itemoffsetof(TP, 0) + Modified: pypy/branch/listcopyop/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/llinterp.py (original) +++ pypy/branch/listcopyop/pypy/rpython/llinterp.py Tue Dec 8 17:07:52 2009 @@ -754,9 +754,9 @@ def op_zero_gc_pointers_inside(self, obj): raise NotImplementedError("zero_gc_pointers_inside") - def op_gc_arraycopy_writebarrier(self, source, dest): - if hasattr(self.heap, 'arraycopy_writebarrier'): - self.heap.arraycopy_writebarrier(source, dest) + def op_gc_writebarrier_before_copy(self, source, dest): + if hasattr(self.heap, 'writebarrier_before_copy'): + self.heap.writebarrier_before_copy(source, dest) def op_getfield(self, obj, field): checkptr(obj) Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py Tue Dec 8 17:07:52 2009 @@ -461,7 +461,7 @@ 'gc_thread_run' : LLOp(), 'gc_thread_die' : LLOp(), 'gc_assume_young_pointers': LLOp(canrun=True), - 'gc_arraycopy_writebarrier': LLOp(canrun=True), + 'gc_writebarrier_before_copy': LLOp(canrun=True), 'gc_heap_stats' : LLOp(canunwindgc=True), # ------- JIT & GC interaction, only for some GCs ---------- Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py Tue Dec 8 17:07:52 2009 @@ -394,7 +394,7 @@ checkadr(addr2) return addr1 - addr2 -def op_gc_arraycopy_writebarrier(source, dest): +def op_gc_writebarrier_before_copy(source, dest): A = lltype.typeOf(source) assert A == lltype.typeOf(dest) assert isinstance(A.TO, lltype.GcArray) Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Tue Dec 8 17:07:52 2009 @@ -482,7 +482,7 @@ objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS self.last_generation_root_objects.append(addr_struct) - def arraycopy_writebarrier(self, source_addr, dest_addr): + def writebarrier_before_copy(self, source_addr, dest_addr): """ This has the same effect as calling writebarrier over each element in dest copied from source, except it might reset one of the following flags a bit too eagerly, which means we'll have Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py Tue Dec 8 17:07:52 2009 @@ -292,8 +292,9 @@ [s_gc, annmodel.SomeInteger(knowntype=rffi.r_ushort)], annmodel.SomeInteger()) - if hasattr(GCClass, 'arraycopy_writebarrier'): - self.arraycopy_wb_p = getfn(GCClass.arraycopy_writebarrier.im_func, + if hasattr(GCClass, 'writebarrier_before_copy'): + self.wb_before_copy_ptr = \ + getfn(GCClass.writebarrier_before_copy.im_func, [s_gc] + [annmodel.SomeAddress()] * 2, annmodel.s_None) # in some GCs we can inline the common case of @@ -780,16 +781,16 @@ TYPE = v_ob.concretetype.TO gen_zero_gc_pointers(TYPE, v_ob, hop.llops) - def gct_gc_arraycopy_writebarrier(self, hop): - if not hasattr(self, 'arraycopy_wb_p'): + def gct_gc_writebarrier_before_copy(self, hop): + if not hasattr(self, 'wb_before_copy_ptr'): # should do nothing - return GCTransformer.gct_gc_arraycopy_writebarrier(self, hop) + return GCTransformer.gct_gc_writebarrier_before_copy(self, hop) op = hop.spaceop source_addr = hop.genop('cast_ptr_to_adr', [op.args[0]], resulttype=llmemory.Address) dest_addr = hop.genop('cast_ptr_to_adr', [op.args[1]], resulttype=llmemory.Address) - hop.genop('direct_call', [self.arraycopy_wb_p, self.c_const_gc, + hop.genop('direct_call', [self.wb_before_copy_ptr, self.c_const_gc, source_addr, dest_addr]) def gct_weakref_create(self, hop): Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py Tue Dec 8 17:07:52 2009 @@ -380,7 +380,7 @@ def gct_zero_gc_pointers_inside(self, hop): pass - def gct_gc_arraycopy_writebarrier(self, hop): + def gct_gc_writebarrier_before_copy(self, hop): pass def gct_gc_identityhash(self, hop): Modified: pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py Tue Dec 8 17:07:52 2009 @@ -128,11 +128,11 @@ ptr = lltype.cast_opaque_ptr(llmemory.GCREF, ptr) return self.gc.id(ptr) - def arraycopy_writebarrier(self, source, dest): - if hasattr(self.gc, 'arraycopy_writebarrier'): + def writebarrier_before_copy(self, source, dest): + if hasattr(self.gc, 'writebarrier_before_copy'): source_addr = llmemory.cast_ptr_to_adr(source) dest_addr = llmemory.cast_ptr_to_adr(dest) - return self.gc.arraycopy_writebarrier(source_addr, dest_addr) + return self.gc.writebarrier_before_copy(source_addr, dest_addr) # ____________________________________________________________ From fijal at codespeak.net Tue Dec 8 17:16:17 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 17:16:17 +0100 (CET) Subject: [pypy-svn] r69987 - pypy/branch/listcopyop/pypy/rpython/memory/gctransform Message-ID: <20091208161617.30C7D168043@codespeak.net> Author: fijal Date: Tue Dec 8 17:16:16 2009 New Revision: 69987 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py Log: complain loud if gc needs write barrier but does not provide writebarrier_before_copy Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py Tue Dec 8 17:16:16 2009 @@ -296,6 +296,8 @@ self.wb_before_copy_ptr = \ getfn(GCClass.writebarrier_before_copy.im_func, [s_gc] + [annmodel.SomeAddress()] * 2, annmodel.s_None) + elif GCClass.needs_write_barrier: + raise NotImplementedError("GC needs write barrier, but does not provide writebarrier_before_copy functionality") # in some GCs we can inline the common case of # malloc_fixedsize(typeid, size, True, False, False) From arigo at codespeak.net Tue Dec 8 17:24:51 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 8 Dec 2009 17:24:51 +0100 (CET) Subject: [pypy-svn] r69988 - pypy/branch/listcopyop/pypy/rlib Message-ID: <20091208162451.54CF2168043@codespeak.net> Author: arigo Date: Tue Dec 8 17:24:50 2009 New Revision: 69988 Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py Log: I don't see any reason left for which we must assert source != dest here. Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rlib/rgc.py (original) +++ pypy/branch/listcopyop/pypy/rlib/rgc.py Tue Dec 8 17:24:50 2009 @@ -333,8 +333,8 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.lltypesystem import lltype, llmemory - assert source != dest TP = lltype.typeOf(source).TO + assert TP == lltype.typeOf(dest).TO if isinstance(TP.OF, lltype.Ptr) and TP.OF.TO._gckind == 'gc': # perform a write barrier that copies necessary flags from # source to dest @@ -347,4 +347,3 @@ llmemory.sizeof(TP.OF) * dest_start) llmemory.raw_memcopy(cp_source_addr, cp_dest_addr, llmemory.sizeof(TP.OF) * length) - From arigo at codespeak.net Tue Dec 8 17:27:22 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 8 Dec 2009 17:27:22 +0100 (CET) Subject: [pypy-svn] r69989 - pypy/branch/listcopyop/pypy/rlib Message-ID: <20091208162722.A8B48168043@codespeak.net> Author: arigo Date: Tue Dec 8 17:27:19 2009 New Revision: 69989 Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py Log: Oups, there still is: raw_memcopy(). Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rlib/rgc.py (original) +++ pypy/branch/listcopyop/pypy/rlib/rgc.py Tue Dec 8 17:27:19 2009 @@ -333,6 +333,7 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.lltypesystem import lltype, llmemory + assert source != dest TP = lltype.typeOf(source).TO assert TP == lltype.typeOf(dest).TO if isinstance(TP.OF, lltype.Ptr) and TP.OF.TO._gckind == 'gc': From arigo at codespeak.net Tue Dec 8 17:27:39 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 8 Dec 2009 17:27:39 +0100 (CET) Subject: [pypy-svn] r69990 - in pypy/branch/listcopyop/pypy/rpython/memory: . test Message-ID: <20091208162739.75F11168043@codespeak.net> Author: arigo Date: Tue Dec 8 17:27:38 2009 New Revision: 69990 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py Log: Minor changes. Modified: pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py Tue Dec 8 17:27:38 2009 @@ -129,7 +129,7 @@ return self.gc.id(ptr) def writebarrier_before_copy(self, source, dest): - if hasattr(self.gc, 'writebarrier_before_copy'): + if self.gc.needs_write_barrier: source_addr = llmemory.cast_ptr_to_adr(source) dest_addr = llmemory.cast_ptr_to_adr(dest) return self.gc.writebarrier_before_copy(source_addr, dest_addr) Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py Tue Dec 8 17:27:38 2009 @@ -551,7 +551,7 @@ res = self.interpret(fn, [-1000], taggedpointers=True) assert res == 111 - def test_arraycopy_writebarrier(self): + def test_writebarrier_before_copy(self): import new ll_arraycopy = new.function(rgc.ll_arraycopy.func_code, {}) From fijal at codespeak.net Tue Dec 8 17:29:31 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 17:29:31 +0100 (CET) Subject: [pypy-svn] r69991 - pypy/branch/listcopyop/pypy/translator/c/test Message-ID: <20091208162931.72DFD168047@codespeak.net> Author: fijal Date: Tue Dec 8 17:29:30 2009 New Revision: 69991 Modified: pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py Log: Fix and improve test_newgc Modified: pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py Tue Dec 8 17:29:30 2009 @@ -1028,22 +1028,29 @@ assert res == expected - def define_listcopy(cls): + def define_arraycopy_writebarrier(cls): + import new + ll_arraycopy = new.function(rgc.ll_arraycopy.func_code, {}) + TP = lltype.GcArray(lltype.Signed) def fn(): l = lltype.malloc(TP, 100) for i in range(100): l[i] = 1 l2 = lltype.malloc(TP, 50) - llop.listcopy(lltype.Void, l, l2, 50, 0, 50) + ll_arraycopy(l, l2, 50, 0, 50) + # force a nursery collect + x = [] + for i in range(20): + x.append((1, lltype.malloc(S))) for i in range(50): assert l2[i] == 1 return 0 return fn - def test_listcopy(self): - self.run("listcopy") + def test_arraycopy_writebarrier(self): + self.run("arraycopy_writebarrier") from pypy.rlib.objectmodel import UnboxedValue From arigo at codespeak.net Tue Dec 8 17:32:52 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 8 Dec 2009 17:32:52 +0100 (CET) Subject: [pypy-svn] r69992 - in pypy/branch/listcopyop/pypy: rlib rpython/memory/test Message-ID: <20091208163252.3C10F168043@codespeak.net> Author: arigo Date: Tue Dec 8 17:32:51 2009 New Revision: 69992 Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py Log: Force the specialization to 'll', mostly for tests that call this function without a lowlevel or mixlevel annotator policy. Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rlib/rgc.py (original) +++ pypy/branch/listcopyop/pypy/rlib/rgc.py Tue Dec 8 17:32:51 2009 @@ -348,3 +348,4 @@ llmemory.sizeof(TP.OF) * dest_start) llmemory.raw_memcopy(cp_source_addr, cp_dest_addr, llmemory.sizeof(TP.OF) * length) +ll_arraycopy._annspecialcase_ = 'specialize:ll' Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py Tue Dec 8 17:32:51 2009 @@ -552,9 +552,6 @@ assert res == 111 def test_writebarrier_before_copy(self): - import new - ll_arraycopy = new.function(rgc.ll_arraycopy.func_code, {}) - S = lltype.GcStruct('S') TP = lltype.GcArray(lltype.Ptr(S)) def fn(): @@ -562,7 +559,7 @@ l2 = lltype.malloc(TP, 100) for i in range(100): l[i] = lltype.malloc(S) - ll_arraycopy(l, l2, 50, 0, 50) + rgc.ll_arraycopy(l, l2, 50, 0, 50) x = [] # force minor collect t = (1, lltype.malloc(S)) From xoraxax at codespeak.net Tue Dec 8 17:33:54 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 8 Dec 2009 17:33:54 +0100 (CET) Subject: [pypy-svn] r69993 - pypy/branch/sepcomp Message-ID: <20091208163354.88B2216804B@codespeak.net> Author: xoraxax Date: Tue Dec 8 17:33:52 2009 New Revision: 69993 Added: pypy/branch/sepcomp/ - copied from r55926, pypy/dist/ Log: Add base revision for separate compilation work. From fijal at codespeak.net Tue Dec 8 17:42:44 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 17:42:44 +0100 (CET) Subject: [pypy-svn] r69994 - pypy/branch/listcopyop/pypy/translator/c/test Message-ID: <20091208164244.3492C16804B@codespeak.net> Author: fijal Date: Tue Dec 8 17:42:42 2009 New Revision: 69994 Modified: pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py Log: provide additional symbol, why it passed for me before though??? Modified: pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py Tue Dec 8 17:42:42 2009 @@ -1033,6 +1033,7 @@ ll_arraycopy = new.function(rgc.ll_arraycopy.func_code, {}) TP = lltype.GcArray(lltype.Signed) + S = lltype.GcStruct('S') def fn(): l = lltype.malloc(TP, 100) for i in range(100): From fijal at codespeak.net Tue Dec 8 17:43:45 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 17:43:45 +0100 (CET) Subject: [pypy-svn] r69995 - pypy/branch/listcopyop/pypy/translator/c/test Message-ID: <20091208164345.3DD1516804B@codespeak.net> Author: fijal Date: Tue Dec 8 17:43:44 2009 New Revision: 69995 Modified: pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py Log: Move the test to base class, that's why... Modified: pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py Tue Dec 8 17:43:44 2009 @@ -835,6 +835,32 @@ res = self.run('hash_varsized') assert res != 0 + + def define_arraycopy_writebarrier(cls): + import new + ll_arraycopy = new.function(rgc.ll_arraycopy.func_code, {}) + + TP = lltype.GcArray(lltype.Signed) + S = lltype.GcStruct('S') + def fn(): + l = lltype.malloc(TP, 100) + for i in range(100): + l[i] = 1 + l2 = lltype.malloc(TP, 50) + ll_arraycopy(l, l2, 50, 0, 50) + # force a nursery collect + x = [] + for i in range(20): + x.append((1, lltype.malloc(S))) + for i in range(50): + assert l2[i] == 1 + return 0 + + return fn + + def test_arraycopy_writebarrier(self): + self.run("arraycopy_writebarrier") + class TestSemiSpaceGC(TestUsingFramework, snippet.SemiSpaceGCTestDefines): gcpolicy = "semispace" should_be_moving = True @@ -1027,32 +1053,6 @@ res = self.run("tagged") assert res == expected - - def define_arraycopy_writebarrier(cls): - import new - ll_arraycopy = new.function(rgc.ll_arraycopy.func_code, {}) - - TP = lltype.GcArray(lltype.Signed) - S = lltype.GcStruct('S') - def fn(): - l = lltype.malloc(TP, 100) - for i in range(100): - l[i] = 1 - l2 = lltype.malloc(TP, 50) - ll_arraycopy(l, l2, 50, 0, 50) - # force a nursery collect - x = [] - for i in range(20): - x.append((1, lltype.malloc(S))) - for i in range(50): - assert l2[i] == 1 - return 0 - - return fn - - def test_arraycopy_writebarrier(self): - self.run("arraycopy_writebarrier") - from pypy.rlib.objectmodel import UnboxedValue class TaggedBase(object): From fijal at codespeak.net Tue Dec 8 19:19:22 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 19:19:22 +0100 (CET) Subject: [pypy-svn] r69996 - pypy/branch/listcopyop/pypy/rlib/test Message-ID: <20091208181922.6ABD0168043@codespeak.net> Author: fijal Date: Tue Dec 8 19:19:21 2009 New Revision: 69996 Modified: pypy/branch/listcopyop/pypy/rlib/test/test_rgc.py Log: kill tabs Modified: pypy/branch/listcopyop/pypy/rlib/test/test_rgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rlib/test/test_rgc.py (original) +++ pypy/branch/listcopyop/pypy/rlib/test/test_rgc.py Tue Dec 8 19:19:21 2009 @@ -80,9 +80,9 @@ assert a1[i] == 100 + i for i in range(6): if 2 <= i < 5: - assert a2[i] == a1[i+2] - else: - assert a2[i] == 200 + i + assert a2[i] == a1[i+2] + else: + assert a2[i] == 200 + i def test_ll_arraycopy_2(): TYPE = lltype.GcArray(lltype.Void) @@ -105,9 +105,9 @@ assert a1[i] == org1[i] for i in range(6): if 2 <= i < 5: - assert a2[i] == a1[i+2] - else: - assert a2[i] == org2[i] + assert a2[i] == a1[i+2] + else: + assert a2[i] == org2[i] def test_ll_arraycopy_4(): S = lltype.GcStruct('S') @@ -123,6 +123,6 @@ assert a1[i] == org1[i] for i in range(6): if 2 <= i < 5: - assert a2[i] == a1[i+2] - else: - assert a2[i] == org2[i] + assert a2[i] == a1[i+2] + else: + assert a2[i] == org2[i] From fijal at codespeak.net Tue Dec 8 19:19:29 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 19:19:29 +0100 (CET) Subject: [pypy-svn] r69997 - pypy/branch/listcopyop/pypy/rpython/memory/gctransform/test Message-ID: <20091208181929.426F816804B@codespeak.net> Author: fijal Date: Tue Dec 8 19:19:28 2009 New Revision: 69997 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/test/test_framework.py Log: fix the test Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/test/test_framework.py Tue Dec 8 19:19:28 2009 @@ -119,6 +119,8 @@ GC_PARAMS = {} class GCClass(MarkSweepGC): needs_write_barrier = True + def writebarrier_before_copy(self, source, dest): + pass def write_barrier_check(spaceop, needs_write_barrier=True): t = TranslationContext() From fijal at codespeak.net Tue Dec 8 20:07:59 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 20:07:59 +0100 (CET) Subject: [pypy-svn] r69998 - pypy/branch/listcopyop/pypy/rlib/parsing/test Message-ID: <20091208190759.A14E616804B@codespeak.net> Author: fijal Date: Tue Dec 8 20:07:58 2009 New Revision: 69998 Modified: pypy/branch/listcopyop/pypy/rlib/parsing/test/test_pythonlexer.py Log: Be safe against pyc files Modified: pypy/branch/listcopyop/pypy/rlib/parsing/test/test_pythonlexer.py ============================================================================== --- pypy/branch/listcopyop/pypy/rlib/parsing/test/test_pythonlexer.py (original) +++ pypy/branch/listcopyop/pypy/rlib/parsing/test/test_pythonlexer.py Tue Dec 8 20:07:58 2009 @@ -230,7 +230,10 @@ assert tokens[i * 3].name == 'String' def test_self(): - s = py.path.local(__file__).read() + fname = __file__ + if fname.endswith('.pyc'): + fname = fname[:-1] + s = py.path.local(fname).read() tokens = pythonlexer.tokenize(s) print tokens From fijal at codespeak.net Tue Dec 8 20:08:45 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 20:08:45 +0100 (CET) Subject: [pypy-svn] r69999 - pypy/trunk/pypy/rlib/parsing/test Message-ID: <20091208190845.9C08B168047@codespeak.net> Author: fijal Date: Tue Dec 8 20:08:44 2009 New Revision: 69999 Modified: pypy/trunk/pypy/rlib/parsing/test/test_pythonlexer.py Log: port 69998 from branch, yak shaving Modified: pypy/trunk/pypy/rlib/parsing/test/test_pythonlexer.py ============================================================================== --- pypy/trunk/pypy/rlib/parsing/test/test_pythonlexer.py (original) +++ pypy/trunk/pypy/rlib/parsing/test/test_pythonlexer.py Tue Dec 8 20:08:44 2009 @@ -230,7 +230,10 @@ assert tokens[i * 3].name == 'String' def test_self(): - s = py.path.local(__file__).read() + fname = __file__ + if fname.endswith('.pyc'): + fname = fname[:-1] + s = py.path.local(fname).read() tokens = pythonlexer.tokenize(s) print tokens From dan at codespeak.net Tue Dec 8 22:02:04 2009 From: dan at codespeak.net (dan at codespeak.net) Date: Tue, 8 Dec 2009 22:02:04 +0100 (CET) Subject: [pypy-svn] r70001 - pypy/branch/micronumpy Message-ID: <20091208210204.B2CCF168043@codespeak.net> Author: dan Date: Tue Dec 8 22:02:04 2009 New Revision: 70001 Added: pypy/branch/micronumpy/ - copied from r70000, pypy/trunk/ Log: Micronumpy branch. From fijal at codespeak.net Tue Dec 8 22:48:28 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 8 Dec 2009 22:48:28 +0100 (CET) Subject: [pypy-svn] r70002 - in pypy/branch/listcopyop/pypy: rlib rpython rpython/lltypesystem rpython/memory/gctransform Message-ID: <20091208214828.3C80E168047@codespeak.net> Author: fijal Date: Tue Dec 8 22:48:24 2009 New Revision: 70002 Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py pypy/branch/listcopyop/pypy/rpython/llinterp.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py Log: Sort-of-solution for refcounting. Not too happy about it, any other idea? Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rlib/rgc.py (original) +++ pypy/branch/listcopyop/pypy/rlib/rgc.py Tue Dec 8 22:48:24 2009 @@ -332,6 +332,7 @@ def ll_arraycopy(source, dest, source_start, dest_start, length): from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.lltypesystem import lltype, llmemory + from pypy.rlib.objectmodel import keepalive_until_here assert source != dest TP = lltype.typeOf(source).TO @@ -340,12 +341,17 @@ # perform a write barrier that copies necessary flags from # source to dest llop.gc_writebarrier_before_copy(lltype.Void, source, dest) + # the hack below is a write barrier version for refcounting :-/ + for i in range(length): + llop.gc_push_alive(lltype.Void, source[source_start + i]) source_addr = llmemory.cast_ptr_to_adr(source) dest_addr = llmemory.cast_ptr_to_adr(dest) cp_source_addr = (source_addr + llmemory.itemoffsetof(TP, 0) + llmemory.sizeof(TP.OF) * source_start) cp_dest_addr = (dest_addr + llmemory.itemoffsetof(TP, 0) + llmemory.sizeof(TP.OF) * dest_start) + llmemory.raw_memcopy(cp_source_addr, cp_dest_addr, llmemory.sizeof(TP.OF) * length) + keepalive_until_here(source) ll_arraycopy._annspecialcase_ = 'specialize:ll' Modified: pypy/branch/listcopyop/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/llinterp.py (original) +++ pypy/branch/listcopyop/pypy/rpython/llinterp.py Tue Dec 8 22:48:24 2009 @@ -859,6 +859,9 @@ def op_gc_push_alive_pyobj(self, pyobj): raise NotImplementedError("gc_push_alive_pyobj") + def op_gc_push_alive(self, pyobj): + pass + def op_gc_pop_alive_pyobj(self, pyobj): raise NotImplementedError("gc_pop_alive_pyobj") Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py Tue Dec 8 22:48:24 2009 @@ -443,6 +443,7 @@ 'gc_restore_exception': LLOp(), 'gc_call_rtti_destructor': LLOp(), 'gc_deallocate': LLOp(), + 'gc_push_alive': LLOp(), 'gc_push_alive_pyobj': LLOp(), 'gc_pop_alive_pyobj': LLOp(), 'gc_reload_possibly_moved': LLOp(), Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/refcounting.py Tue Dec 8 22:48:24 2009 @@ -142,6 +142,9 @@ def var_needs_set_transform(self, var): return var_needsgc(var) + def gct_gc_push_alive(self, hop): + self.push_alive_nopyobj(hop.spaceop.args[0], hop.llops) + def push_alive_nopyobj(self, var, llops): v_adr = gen_cast(llops, llmemory.Address, var) llops.genop("direct_call", [self.increfptr, v_adr]) Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py Tue Dec 8 22:48:24 2009 @@ -377,6 +377,9 @@ gct_getfield = default + def gct_gc_push_alive(self, hop): + pass + def gct_zero_gc_pointers_inside(self, hop): pass From afa at codespeak.net Wed Dec 9 10:14:12 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 9 Dec 2009 10:14:12 +0100 (CET) Subject: [pypy-svn] r70003 - pypy/trunk/pypy/module/__builtin__/test Message-ID: <20091209091412.4BE6216804B@codespeak.net> Author: afa Date: Wed Dec 9 10:14:10 2009 New Revision: 70003 Modified: pypy/trunk/pypy/module/__builtin__/test/test_import.py Log: Fix the test when run with -A: When the function is geninterp'd, it runs in the '__builtin__ module When the function is run directly, imports are done in the context of the module.__builtin__.test package. Modified: pypy/trunk/pypy/module/__builtin__/test/test_import.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/test_import.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/test_import.py Wed Dec 9 10:14:10 2009 @@ -712,7 +712,11 @@ try: import datetime assert len(tried_imports) == 1 - assert tried_imports[0][0] == "datetime" + package_name = '.'.join(__name__.split('.')[:-1]) + if package_name: + assert tried_imports[0][0] == package_name + ".datetime" + else: + assert tried_imports[0][0] == "datetime" finally: sys.meta_path.pop() From xoraxax at codespeak.net Wed Dec 9 12:31:10 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 12:31:10 +0100 (CET) Subject: [pypy-svn] r70004 - pypy/branch/sepcomp/pypy/tool Message-ID: <20091209113110.8F09016804B@codespeak.net> Author: xoraxax Date: Wed Dec 9 12:31:09 2009 New Revision: 70004 Modified: pypy/branch/sepcomp/pypy/tool/descriptor.py Log: Add repr for instance method descriptor. Modified: pypy/branch/sepcomp/pypy/tool/descriptor.py ============================================================================== --- pypy/branch/sepcomp/pypy/tool/descriptor.py (original) +++ pypy/branch/sepcomp/pypy/tool/descriptor.py Wed Dec 9 12:31:09 2009 @@ -17,6 +17,9 @@ self.im_self = im_self self.im_class = im_class + def __repr__(self): + return "" % (self.im_class, self.im_self, self.im_func) + def __call__(self, *args, **kwds): firstarg = self.im_self if firstarg is None: From xoraxax at codespeak.net Wed Dec 9 12:32:02 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 12:32:02 +0100 (CET) Subject: [pypy-svn] r70005 - pypy/branch/sepcomp/pypy/rlib Message-ID: <20091209113202.40D5A16804B@codespeak.net> Author: xoraxax Date: Wed Dec 9 12:32:01 2009 New Revision: 70005 Modified: pypy/branch/sepcomp/pypy/rlib/libffi.py Log: Add support for mode passing and persistent dynamic modules. Modified: pypy/branch/sepcomp/pypy/rlib/libffi.py ============================================================================== --- pypy/branch/sepcomp/pypy/rlib/libffi.py (original) +++ pypy/branch/sepcomp/pypy/rlib/libffi.py Wed Dec 9 12:32:01 2009 @@ -12,6 +12,9 @@ import py import os + +DEBUG = True # writes dlerror() messages to stderr + _MS_WINDOWS = os.name == "nt" if _MS_WINDOWS: @@ -186,20 +189,33 @@ return "" return rffi.charp2str(res) - def dlopen(name): + def dlopen(name, mode=-1): """ Wrapper around C-level dlopen """ - if RTLD_LOCAL is not None: - mode = RTLD_LOCAL | RTLD_NOW - else: - mode = RTLD_NOW + if mode == -1: + if RTLD_LOCAL is not None: + mode = RTLD_LOCAL | RTLD_NOW + else: + mode = RTLD_NOW res = c_dlopen(name, rffi.cast(rffi.INT, mode)) if not res: - raise OSError(-1, dlerror()) + err = dlerror() + # because the message would be lost in a translated program (OSError only has an errno), + # we offer a way to write it to stderr + if DEBUG: + import os + os.write(2, err) + raise OSError(-1, err) return res dlclose = c_dlclose + def dlopen_global_persistent(name): + ll_libname = rffi.str2charp(name) + lib = dlopen(ll_libname, RTLD_GLOBAL | RTLD_NOW) + lltype.free(ll_libname, flavor='raw') + return lib + def dlsym(libhandle, name): """ Wrapper around C-level dlsym """ @@ -514,20 +530,18 @@ class CDLL: flags = FUNCFLAG_CDECL - - def __init__(self, libname): - self.ll_libname = lltype.nullptr(rffi.CCHARP.TO) + + def __init__(self, libname, unload_on_finalization=True): + self.unload_on_finalization = unload_on_finalization self.lib = lltype.nullptr(rffi.CCHARP.TO) - self.ll_libname = rffi.str2charp(libname) - self.lib = dlopen(self.ll_libname) + ll_libname = rffi.str2charp(libname) + self.lib = dlopen(ll_libname) + lltype.free(ll_libname, flavor='raw') def __del__(self): - if self.lib: + if self.lib and self.unload_on_finalization: dlclose(self.lib) self.lib = lltype.nullptr(rffi.CCHARP.TO) - if self.ll_libname: - lltype.free(self.ll_libname, flavor='raw') - self.ll_libname = lltype.nullptr(rffi.CCHARP.TO) def getpointer(self, name, argtypes, restype): # these arguments are already casted to proper ffi From xoraxax at codespeak.net Wed Dec 9 12:34:04 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 12:34:04 +0100 (CET) Subject: [pypy-svn] r70006 - pypy/branch/sepcomp/pypy/config Message-ID: <20091209113404.8BE5416804B@codespeak.net> Author: xoraxax Date: Wed Dec 9 12:34:04 2009 New Revision: 70006 Modified: pypy/branch/sepcomp/pypy/config/translationoption.py Log: Add new sepcomp options and default to -rdynamic for correct naive linkage of shared libs. Modified: pypy/branch/sepcomp/pypy/config/translationoption.py ============================================================================== --- pypy/branch/sepcomp/pypy/config/translationoption.py (original) +++ pypy/branch/sepcomp/pypy/config/translationoption.py Wed Dec 9 12:34:04 2009 @@ -100,6 +100,11 @@ BoolOption("rweakref", "The backend supports RPython-level weakrefs", default=True), + BoolOption("generatemodule", "Generate an extension module.", default=False, + requires=[("translation.backend", "c")]), + StrOption("exportpackage", "Specify which package to export.", default=""), # XXX only works with the c backend + + # misc StrOption("cc", "Specify compiler to use for compiling generated C", cmdline="--cc"), StrOption("profopt", "Specify profile based optimization script", @@ -125,7 +130,7 @@ StrOption("compilerflags", "Specify flags for the C compiler", cmdline="--cflags"), StrOption("linkerflags", "Specify flags for the linker (C backend only)", - cmdline="--ldflags"), + cmdline="--ldflags", default="-rdynamic"), # Flags of the TranslationContext: BoolOption("simplifying", "Simplify flow graphs", default=True), From xoraxax at codespeak.net Wed Dec 9 12:40:34 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 12:40:34 +0100 (CET) Subject: [pypy-svn] r70007 - in pypy/branch/sepcomp/pypy/annotation: . test Message-ID: <20091209114034.58C4616804B@codespeak.net> Author: xoraxax Date: Wed Dec 9 12:40:33 2009 New Revision: 70007 Modified: pypy/branch/sepcomp/pypy/annotation/description.py pypy/branch/sepcomp/pypy/annotation/test/test_annrpython.py Log: Add _attrs_this_ logic and other descriptor hacks. Modified: pypy/branch/sepcomp/pypy/annotation/description.py ============================================================================== --- pypy/branch/sepcomp/pypy/annotation/description.py (original) +++ pypy/branch/sepcomp/pypy/annotation/description.py Wed Dec 9 12:40:33 2009 @@ -114,6 +114,7 @@ class Desc(object): __metaclass__ = extendabletype + avoid_constantness = False def __init__(self, bookkeeper, pyobj=None): self.bookkeeper = bookkeeper @@ -360,10 +361,15 @@ name=None, basedesc=None, classdict=None, specialize=None): super(ClassDesc, self).__init__(bookkeeper, pyobj) + self.enable_virtual_access = (getattr(pyobj, '_exported_', False) or getattr(pyobj, '_force_virtual_', False) + ) # XXX should only look in dict if name is None: name = pyobj.__module__ + '.' + pyobj.__name__ + if hasattr(pyobj, '_force_name_'): + name = pyobj._force_name_ self.name = name + self.abstract = False self.basedesc = basedesc if classdict is None: classdict = {} # populated below @@ -400,6 +406,9 @@ "with _mixin_: %r" % (cls,)) base = b1 + if '_abstract_' in cls.__dict__: + self.abstract = bool(cls.__dict__['_abstract_']) + assert '_exported_' in cls.__dict__ self.add_sources_for_class(cls) if base is not object: self.basedesc = bookkeeper.getdesc(base) @@ -407,21 +416,27 @@ if '_settled_' in cls.__dict__: self.settled = bool(cls.__dict__['_settled_']) - if '__slots__' in cls.__dict__ or '_attrs_' in cls.__dict__: + if '__slots__' in cls.__dict__ or '_attrs_' in cls.__dict__ or '_attrs_this_' in cls.__dict__: attrs = {} - for decl in ('__slots__', '_attrs_'): + def fish_attrs(decl): decl = cls.__dict__.get(decl, []) if isinstance(decl, str): decl = (decl,) decl = dict.fromkeys(decl) - attrs.update(decl) + return decl + for decl in ('__slots__', '_attrs_'): + attrs.update(fish_attrs(decl)) + self.inheriting_enforced_attrs = attrs if self.basedesc is not None: if self.basedesc.all_enforced_attrs is None: raise Exception("%r has slots or _attrs_, " "but not its base class" % (pyobj,)) - attrs.update(self.basedesc.all_enforced_attrs) - self.all_enforced_attrs = attrs + attrs.update(self.basedesc.inheriting_enforced_attrs) + self.all_enforced_attrs = fish_attrs("_attrs_this_") + self.all_enforced_attrs.update(self.inheriting_enforced_attrs) + if self.abstract and not getattr(self, 'all_enforced_attrs', None): + self.all_enforced_attrs = () def add_source_attribute(self, name, value, mixin=False): if isinstance(value, types.FunctionType): @@ -464,6 +479,8 @@ def add_sources_for_class(self, cls, mixin=False): for name, value in cls.__dict__.items(): + if self.abstract: + assert name.startswith("_") or hasattr(value, '_inputtypes_') self.add_source_attribute(name, value, mixin) def getclassdef(self, key): @@ -682,6 +699,7 @@ def __init__(self, bookkeeper, funcdesc, originclassdef, selfclassdef, name, flags={}): super(MethodDesc, self).__init__(bookkeeper) + self.force_virtual_access = getattr(funcdesc.pyobj, '_force_virtual_', False) and originclassdef.classdesc.enable_virtual_access self.funcdesc = funcdesc self.originclassdef = originclassdef self.selfclassdef = selfclassdef Modified: pypy/branch/sepcomp/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/sepcomp/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/sepcomp/pypy/annotation/test/test_annrpython.py Wed Dec 9 12:40:33 2009 @@ -2482,7 +2482,7 @@ def fun(x, y): return y s_nonneg = annmodel.SomeInteger(nonneg=True) - fun._annenforceargs_ = policy.Sig(lambda s1,s2: s1, lambda s1,s2: s1) + fun._annenforceargs_ = policy.Sig(lambda s1, s2: s1, lambda s1, s2: s1) # means: the 2nd argument's annotation becomes the 1st argument's # input annotation @@ -2592,6 +2592,31 @@ from pypy.annotation.classdef import NoSuchAttrError py.test.raises(NoSuchAttrError, a.build_types, fun, [int]) + def test_enforced_attrs_this(self): + class Base(object): + _attrs_this_ = 'x' + def m(self): + return 65 + class A(Base): pass + for attrname, works1, works2 in [('x', True, True), + ('y', False, True)]: + def fun1(n): + o = Base() + setattr(o, attrname, 12) + return o.m() + def fun2(n): + o = A() + setattr(o, attrname, 12) + return o.m() + for works, fun in ((works1, fun1), (works2, fun2)): + a = self.RPythonAnnotator() + if works: + a.build_types(fun, [int]) + else: + from pypy.annotation.classdef import NoSuchAttrError + py.test.raises(NoSuchAttrError, a.build_types, fun, [int]) + + def test_attrs_enforce_attrs(self): class Superbase(object): _attrs_ = 'x' From xoraxax at codespeak.net Wed Dec 9 12:41:23 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 12:41:23 +0100 (CET) Subject: [pypy-svn] r70008 - pypy/branch/sepcomp/pypy/annotation Message-ID: <20091209114123.27EA216804B@codespeak.net> Author: xoraxax Date: Wed Dec 9 12:41:22 2009 New Revision: 70008 Modified: pypy/branch/sepcomp/pypy/annotation/model.py Log: Add make_acceptable_in_interface and force_virtual_access support for model.py. Modified: pypy/branch/sepcomp/pypy/annotation/model.py ============================================================================== --- pypy/branch/sepcomp/pypy/annotation/model.py (original) +++ pypy/branch/sepcomp/pypy/annotation/model.py Wed Dec 9 12:41:22 2009 @@ -150,6 +150,10 @@ def nonnoneify(self): return self + def make_acceptable_in_interface(self): + return None # we refuse + + class SomeFloat(SomeObject): "Stands for a float or an integer." knowntype = float # if we don't know if it's a float or an int, @@ -159,6 +163,9 @@ def can_be_none(self): return False + def make_acceptable_in_interface(self): + return SomeFloat() + class SomeSingleFloat(SomeObject): "Stands for an r_singlefloat." # No operation supported, not even union with a regular float @@ -168,6 +175,9 @@ def can_be_none(self): return False + def make_acceptable_in_interface(self): + return SomeSingleFloat() + class SomeInteger(SomeFloat): "Stands for an object which is known to be an integer." knowntype = int @@ -187,6 +197,9 @@ self.nonneg = unsigned or nonneg self.unsigned = unsigned # pypy.rlib.rarithmetic.r_uint + def make_acceptable_in_interface(self): + return SomeInteger(nonneg=False, knowntype=self.knowntype) + class SomeBool(SomeInteger): "Stands for true or false." knowntype = bool @@ -195,6 +208,7 @@ def __init__(self): pass + class SomeString(SomeObject): "Stands for an object which is known to be a string." knowntype = str @@ -208,6 +222,9 @@ def nonnoneify(self): return SomeString(can_be_None=False) + def make_acceptable_in_interface(self): + return SomeString(True) + class SomeUnicodeString(SomeObject): "Stands for an object which is known to be an unicode string" knowntype = unicode @@ -221,6 +238,9 @@ def nonnoneify(self): return SomeUnicodeString(can_be_None=False) + def make_acceptable_in_interface(self): + return SomeUnicodeString(True) + class SomeChar(SomeString): "Stands for an object known to be a string of length 1." @@ -351,18 +371,25 @@ def nonnoneify(self): return SomeInstance(self.classdef, can_be_None=False) + def make_acceptable_in_interface(self): + cdef = self.classdef.get_exported_cdef() + assert cdef is not None + return SomeInstance(cdef, True) class SomePBC(SomeObject): """Stands for a global user instance, built prior to the analysis, or a set of such instances.""" immutable = True - def __init__(self, descriptions, can_be_None=False, subset_of=None): + def __init__(self, descriptions, can_be_None=False, subset_of=None, force_virtual_access=False): # descriptions is a set of Desc instances. descriptions = dict.fromkeys(descriptions) self.descriptions = descriptions self.can_be_None = can_be_None self.subset_of = subset_of + self.force_virtual_access = force_virtual_access + if subset_of is not None: + self.force_virtual_access = subset_of.force_virtual_access or force_virtual_access self.simplify() if self.isNone(): self.knowntype = type(None) @@ -378,9 +405,13 @@ # hack for the convenience of direct callers to SomePBC(): # only if there is a single object in descriptions desc, = descriptions - if desc.pyobj is not None: + if desc.pyobj is not None and not desc.avoid_constantness and not self.force_virtual_access: self.const = desc.pyobj + def make_acceptable_in_interface(self): + if self.isNone(): + return self + def getKind(self): "Return the common Desc class of all descriptions in this PBC." kinds = {} @@ -420,7 +451,7 @@ if hasattr(self, 'const'): return None else: - return '{...%s...}'%(len(pbis),) + return '{%s}' % (str(pbis.keys())[1:-1], ) def fmt_knowntype(self, kt): if self.is_constant(): @@ -685,7 +716,7 @@ return s1 def isdegenerated(s_value): - return s_value.__class__ is SomeObject and s_value.knowntype is not type + return (s_value.__class__ is SomeObject and s_value.knowntype is not type) or (s_value.__class__ is SomeInstance and s_value.classdef is object) # make knowntypedata dictionary From xoraxax at codespeak.net Wed Dec 9 12:41:46 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 12:41:46 +0100 (CET) Subject: [pypy-svn] r70009 - pypy/branch/sepcomp/pypy/annotation Message-ID: <20091209114146.D2A3A16804B@codespeak.net> Author: xoraxax Date: Wed Dec 9 12:41:45 2009 New Revision: 70009 Modified: pypy/branch/sepcomp/pypy/annotation/classdef.py Log: Add pile of classdef hacks. Modified: pypy/branch/sepcomp/pypy/annotation/classdef.py ============================================================================== --- pypy/branch/sepcomp/pypy/annotation/classdef.py (original) +++ pypy/branch/sepcomp/pypy/annotation/classdef.py Wed Dec 9 12:41:45 2009 @@ -5,7 +5,7 @@ from __future__ import generators from pypy.annotation.model import SomePBC, s_ImpossibleValue, unionof from pypy.annotation.model import SomeInteger, isdegenerated, SomeTuple,\ - SomeString + SomeString, SomeInstance, UnionError from pypy.annotation import description @@ -75,6 +75,7 @@ self.readonly = True self.attr_allowed = True self.read_locations = {} + self.exported = False def add_constant_source(self, classdef, source): s_value = source.s_get_value(classdef, self.name) @@ -166,12 +167,79 @@ self.parentdefs = dict.fromkeys(self.getmro()) + def is_exported(self, recursive=True): + ecdef = self.get_exported_cdef(False) + if recursive: + return ecdef is not None + else: + return ecdef is self + + def get_exported_cdef(self, imported_as_well=True): # returns the most precise exported classdef + if '_exported_' in self.classdesc.classdict: + component = self.classdesc.classdict['_component_'].value + if self.bookkeeper.annotator.translator.config.translation.exportpackage == component.name: + return self + elif imported_as_well and '_importinfo_' in self.classdesc.classdict: + return self + if self.basedef is None or self.basedef.classdesc.pyobj is object: + return None + return self.basedef.get_exported_cdef() + + def get_import_data(self): + const = self.classdesc.classdict.get('_importinfo_', None) + if const is not None: + return const.value + def setup(self, sources): # collect the (supposed constant) class attributes for name, source in sources.items(): self.add_source_for_attribute(name, source) if self.bookkeeper: self.bookkeeper.event('classdef_setup', self) + self.call_imported_methods() + + def call_imported_methods(self): + from pypy.annotation.binaryop import unioncheck + for cdef in list(self.getmro())[:0:-1]: + impdata = cdef.get_import_data() + if impdata: + # add methods that override methods from imported base classes + for mangled_name, name, value in impdata.cls: + if not name: + continue + self.generalize_attr(name) # XXX ugly and probably non-sense + cdef.generalize_attr(name) + s_value = cdef.about_attribute(name) + if not s_value: + continue + s_func_subcls = self.lookup_filter(s_value, name) + s_func_impcls = cdef.lookup_filter(s_value, name) + #if s_func_subcls.descriptions.keys()[0].funcdesc is s_func_impcls.descriptions.keys()[0].funcdesc: + # continue # hm + bm_subcls_desc = s_func_subcls.descriptions.iterkeys().next() + bm_impcls_desc = s_func_impcls.descriptions.iterkeys().next() + smfc = bm_impcls_desc.funcdesc.pyobj.is_wrapping + smfc.compute_rebuilt_args_result(self.bookkeeper) + src_args_s, src_s_result = smfc.args_s_rebuilt, smfc.s_result_rebuilt + desc = bm_subcls_desc.bind_self(self) + args = self.bookkeeper.build_args("simple_call", src_args_s[1:]) + s_meth = SomePBC([desc.funcdesc]) + new_args_s = [SomeInstance(s_func_subcls.descriptions.keys()[0].originclassdef)] + src_args_s[1:] + self.bookkeeper.emulate_pbc_call(desc, s_meth, new_args_s) + self.bookkeeper.pending_specializations.append(lambda desc=desc.funcdesc, s_res=src_s_result: self.check_return_values(desc, s_res)) + # XXX remove + #desc.consider_call_site(self.bookkeeper, desc.getcallfamily(), [desc], args, s_ImpossibleValue) + + def check_return_values(self, desc, s_result): + bk = self.bookkeeper + graph = desc.getuniquegraph() + s_graph_result = bk.annotator.binding(graph.getreturnvar(), None) + if s_graph_result is None: + print "Annotation failed ..." + return + if not s_result.contains(s_graph_result): + raise UnionError("Invalid return value in overridden imported function!") + bk.annotator.setbinding(graph.getreturnvar(), s_result) def add_source_for_attribute(self, attr, source): """Adds information about a constant source for an attribute. From xoraxax at codespeak.net Wed Dec 9 12:48:20 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 12:48:20 +0100 (CET) Subject: [pypy-svn] r70010 - in pypy/branch/sepcomp/pypy/rpython: . lltypesystem lltypesystem/test test Message-ID: <20091209114820.E0B7D16804B@codespeak.net> Author: xoraxax Date: Wed Dec 9 12:48:20 2009 New Revision: 70010 Modified: pypy/branch/sepcomp/pypy/rpython/lltypesystem/lltype.py pypy/branch/sepcomp/pypy/rpython/lltypesystem/rclass.py pypy/branch/sepcomp/pypy/rpython/lltypesystem/rtagged.py pypy/branch/sepcomp/pypy/rpython/lltypesystem/test/test_lltype.py pypy/branch/sepcomp/pypy/rpython/normalizecalls.py pypy/branch/sepcomp/pypy/rpython/rclass.py pypy/branch/sepcomp/pypy/rpython/rpbc.py pypy/branch/sepcomp/pypy/rpython/rtyper.py pypy/branch/sepcomp/pypy/rpython/test/test_rpbc.py Log: Add HOp repr. Modified: pypy/branch/sepcomp/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/sepcomp/pypy/rpython/lltypesystem/lltype.py Wed Dec 9 12:48:20 2009 @@ -76,6 +76,12 @@ _is_compatible = __eq__ + def __getstate__(self): + return self.__dict__ + + def __setstate__(self, val): + self.__dict__.update(val) + def _enforce(self, value): if typeOf(value) != self: raise TypeError @@ -106,11 +112,6 @@ self.__cached_hash = result return result - # due to this dynamic hash value, we should forbid - # pickling, until we have an algorithm for that. - # but we just provide a tag for external help. - __hash_is_not_constant__ = True - def __repr__(self): return '<%s>' % (self,) @@ -1260,6 +1261,7 @@ class _container(object): __slots__ = () + _exported = False def _parentstructure(self, check=True): return None def _check(self): @@ -1280,7 +1282,7 @@ __slots__ = ('_TYPE', '_parent_type', '_parent_index', '_keepparent', - '_wrparent', + '_wrparent',"_exported", '__weakref__', '_storage') @@ -1651,7 +1653,9 @@ return id(self) def __setattr__(self, attr, value): - raise AttributeError("cannot change the attributes of %r" % (self,)) + if attr != "_exported": + raise AttributeError("cannot change the attributes of %r" % (self,)) + _container.__setattr__(self, attr, value) class _opaque(_parentable): def __init__(self, TYPE, parent=None, parentindex=None, **attrs): @@ -1696,8 +1700,18 @@ return _parentable._normalizedcontainer(self) +class _external_reference(_container): + def __init__(self, TYPE, name, component=None): + self._TYPE = TYPE + self.name = name + self.component = component + self.iddata = str(component) + name + + def _getid(self): + return id(self.iddata) + + class _pyobject(Hashable, _container): - __slots__ = [] # or we get in trouble with pickling _TYPE = PyObject @@ -1764,6 +1778,10 @@ o = _pyobject(obj) return _ptr(Ptr(PyObject), o) +def externalptr(TYPE, name, component=None): + o = _external_reference(TYPE, name, component) + return _ptr(Ptr(TYPE), o, solid=True) + def cast_ptr_to_int(ptr): return ptr._cast_to_int() Modified: pypy/branch/sepcomp/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/sepcomp/pypy/rpython/lltypesystem/rclass.py Wed Dec 9 12:48:20 2009 @@ -14,7 +14,8 @@ cast_pointer, cast_ptr_to_int, castable, nullptr, \ RuntimeTypeInfo, getRuntimeTypeInfo, typeOf, \ Array, Char, Void, attachRuntimeTypeInfo, \ - FuncType, Bool, Signed, functionptr, FuncType, PyObject + FuncType, Bool, Signed, functionptr, FuncType, PyObject, \ + normalizeptr from pypy.rpython.lltypesystem import lltype from pypy.rpython.robject import PyObjRepr, pyobj_repr from pypy.rpython.extregistry import ExtRegistryEntry @@ -62,8 +63,8 @@ OBJECTPTR = Ptr(OBJECT) OBJECT_VTABLE.become(Struct('object_vtable', #('parenttypeptr', CLASSTYPE), - ('subclassrange_min', Signed), - ('subclassrange_max', Signed), + ('level', Signed), + ('classrow', Ptr(Array(CLASSTYPE))), ('rtti', Ptr(RuntimeTypeInfo)), ('name', Ptr(Array(Char))), ('instantiate', Ptr(FuncType([], OBJECTPTR))), @@ -86,6 +87,50 @@ return vtable + +def weave_llfields(llfields, classinfo, types_only=False): + llfields_iter = iter(llfields) + result = [] + for mangled_name, name, value in classinfo: + if name is not None: + curfield = llfields_iter.next() + while curfield is not None and curfield[0] != mangled_name: # XXX does this make sense? + result.append((mangled_name, value.get_func())) + try: + curfield = llfields_iter.next() + except StopIteration: + curfield = None + if curfield is not None: + result.append(curfield) + else: + if not types_only: + value = typeOf(value) + result.append((mangled_name, value)) + try: + llfields_iter.next() + except StopIteration: + pass + else: + raise TyperError("Incorrect set of llfields") + return result + +def extend_exportinfo(exportinfo, mangled_name, name, r, classdesc=None): + T = r.lowleveltype + if classdesc is None: + llvalue = T + var = None + else: + var = classdesc.read_attribute(name, None) + llvalue = r.convert_desc_or_const(var) + #import pdb; pdb.set_trace() + if var is None or not getattr(var.value, '_export_', None): + name = None # hide + val = llvalue + else: + val = var.value + exportinfo.append((mangled_name, name, val)) + + class ClassRepr(AbstractClassRepr): def __init__(self, rtyper, classdef): AbstractClassRepr.__init__(self, rtyper, classdef) @@ -97,6 +142,7 @@ self.lowleveltype = Ptr(self.vtable_type) def _setup_repr(self): + exportinfo = None # NOTE: don't store mutable objects like the dicts below on 'self' # before they are fully built, to avoid strange bugs in case # of recursion where other code would uses these @@ -105,6 +151,9 @@ pbcfields = {} allmethods = {} if self.classdef is not None: + if self.classdef.is_exported(): + exportinfo = [] + importinfo = self.classdef.get_import_data() # class attributes llfields = [] attrs = self.classdef.attrs.items() @@ -117,9 +166,13 @@ allmethods[name] = True s_value = s_unboundmethod r = self.rtyper.getrepr(s_value) + #if "Multi" in repr(r) or ("Void" in repr(r.lowleveltype) and name in ("foo", "bar") and "foovoid" not in repr(self)): + # import pdb; pdb.set_trace() mangled_name = 'cls_' + name clsfields[name] = mangled_name, r llfields.append((mangled_name, r.lowleveltype)) + if exportinfo is not None and r.lowleveltype is not Void: + extend_exportinfo(exportinfo, mangled_name, name, r, self.classdef.classdesc) # attributes showing up in getattrs done on the class as a PBC extra_access_sets = self.rtyper.class_pbc_attributes.get( self.classdef, {}) @@ -128,7 +181,12 @@ mangled_name = mangle('pbc%d' % counter, attr) pbcfields[access_set, attr] = mangled_name, r llfields.append((mangled_name, r.lowleveltype)) + if exportinfo is not None: + assert 0, "not yet supported" # + if importinfo: + #import pdb; pdb.set_trace() + llfields = weave_llfields(llfields, importinfo.cls) self.rbase = getclassrepr(self.rtyper, self.classdef.basedef) self.rbase.setup() kwds = {'hints': {'immutable': True}} @@ -136,6 +194,8 @@ ('super', self.rbase.vtable_type), *llfields, **kwds) self.vtable_type.become(vtable_type) + if exportinfo is not None: + self.classdef.exportinfo_cls = exportinfo allmethods.update(self.rbase.allmethods) self.clsfields = clsfields self.pbcfields = pbcfields @@ -159,8 +219,20 @@ def getvtable(self, cast_to_typeptr=True): """Return a ptr to the vtable of this type.""" if self.vtable is None: - self.vtable = malloc(self.vtable_type, immortal=True) - self.setup_vtable(self.vtable, self) + if self.classdef and self.classdef.get_import_data(): + #from pypy.rpython.lltypesystem.rffi import CConstant + handle = self.classdef.get_import_data().vtablename + component = None + if isinstance(handle, tuple): + component, handle = handle + #T = lltype.Ptr(lltype.OpaqueType(handle, hints=dict(external_void=True))) + #val = CConstant("&" + handle, T) + self.vtable = lltype.externalptr(OBJECT_VTABLE, handle + "_vtable", component) + #self.vtable = val + cast_to_typeptr = False + else: + self.vtable = malloc(self.vtable_type, immortal=True) + self.setup_vtable(self.vtable, self) # vtable = self.vtable if cast_to_typeptr: @@ -175,11 +247,18 @@ # initialize the 'subclassrange_*' and 'name' fields if rsubcls.classdef is not None: #vtable.parenttypeptr = rsubcls.rbase.getvtable() - vtable.subclassrange_min = rsubcls.classdef.minid - vtable.subclassrange_max = rsubcls.classdef.maxid + vtable.level = rsubcls.classdef.level + vtable.classrow = malloc(Array(CLASSTYPE), rsubcls.classdef.level + 1, immortal=True) + cur_repr = rsubcls + for i in xrange(rsubcls.classdef.level, -1, -1): + vtable.classrow[i] = cur_repr.getvtable() + cur_repr = cur_repr.rbase + + if rsubcls.classdef.is_exported(): + rsubcls.classdef.exportinfo_vtableval = normalizeptr(vtable) else: #for the root class - vtable.subclassrange_min = 0 - vtable.subclassrange_max = sys.maxint + vtable.level = -1 + vtable.classrow = nullptr(Array(CLASSTYPE)) rinstance = getinstancerepr(self.rtyper, rsubcls.classdef) rinstance.setup() if rinstance.gcflavor == 'gc': @@ -198,6 +277,7 @@ #else: the classdef was created recently, so no instantiate() # could reach it else: + importinfo = self.classdef.get_import_data() # setup class attributes: for each attribute name at the level # of 'self', look up its value in the subclass rsubcls def assign(mangled_name, value): @@ -206,7 +286,7 @@ llvalue = r.convert_desc_or_const(value) setattr(vtable, mangled_name, llvalue) - mro = list(rsubcls.classdef.getmro()) + mro = list(rsubcls.classdef.getmro()) # XXX not needed? for fldname in self.clsfields: mangled_name, r = self.clsfields[fldname] if r.lowleveltype is Void: @@ -223,6 +303,10 @@ attrvalue = rsubcls.classdef.classdesc.read_attribute(attr, None) if attrvalue is not None: assign(mangled_name, attrvalue) + if importinfo: + for mangled_name, name, val in importinfo.cls: + if not name: + setattr(vtable, mangled_name, val) # then initialize the 'super' portion of the vtable self.rbase.setup_vtable(vtable.super, rsubcls) @@ -282,10 +366,8 @@ ## # a class with no subclass ## return hop.genop('ptr_eq', [v_cls1, v_cls2], resulttype=Bool) ## else: - minid = hop.inputconst(Signed, cls2.subclassrange_min) - maxid = hop.inputconst(Signed, cls2.subclassrange_max) - return hop.gendirectcall(ll_issubclass_const, v_cls1, minid, - maxid) + level = hop.inputconst(Signed, cls2.level) + return hop.gendirectcall(ll_issubclass_const, v_cls1, v_cls2, level) else: v_cls1, v_cls2 = hop.inputargs(class_repr, class_repr) return hop.gendirectcall(ll_issubclass, v_cls1, v_cls2) @@ -307,6 +389,7 @@ self.gcflavor = gcflavor def _setup_repr(self, llfields=None, hints=None, adtmeths=None): + exportinfo = None # NOTE: don't store mutable objects like the dicts below on 'self' # before they are fully built, to avoid strange bugs in case # of recursion where other code would uses these @@ -317,6 +400,8 @@ if self.classdef is None: fields['__class__'] = 'typeptr', get_type_repr(self.rtyper) else: + if self.classdef.is_exported(): + exportinfo = [] # instance attributes if llfields is None: llfields = [] @@ -328,12 +413,18 @@ mangled_name = 'inst_' + name fields[name] = mangled_name, r llfields.append((mangled_name, r.lowleveltype)) + if exportinfo is not None: + extend_exportinfo(exportinfo, mangled_name, name, r, None) + # # hash() support if self.rtyper.needs_hash_support(self.classdef): from pypy.rpython import rint fields['_hash_cache_'] = 'hash_cache', rint.signed_repr llfields.append(('hash_cache', Signed)) + importinfo = self.classdef.get_import_data() + if importinfo: + llfields = weave_llfields(llfields, importinfo.inst, True) self.rbase = getinstancerepr(self.rtyper, self.classdef.basedef, self.gcflavor) @@ -347,6 +438,11 @@ if '_immutable_' in self.classdef.classdesc.classdict: hints = hints.copy() hints['immutable'] = True + if self.classdef.is_exported(): + hints = hints.copy() + hints['_exported'] = True + if exportinfo is not None: + self.classdef.exportinfo_inst = exportinfo object_type = MkStruct(self.classdef.name, ('super', self.rbase.object_type), hints=hints, @@ -584,16 +680,15 @@ instance_repr = self.common_repr() v_obj, v_cls = hop.inputargs(instance_repr, class_repr) - if isinstance(v_cls, Constant): + if False and isinstance(v_cls, Constant): # XXX cls = v_cls.value # XXX re-implement the following optimization #if cls.subclassrange_max == cls.subclassrange_min: # # a class with no subclass # return hop.gendirectcall(rclass.ll_isinstance_exact, v_obj, v_cls) #else: - minid = hop.inputconst(Signed, cls.subclassrange_min) - maxid = hop.inputconst(Signed, cls.subclassrange_max) - return hop.gendirectcall(ll_isinstance_const, v_obj, minid, maxid) + level = hop.inputconst(Signed, cls.level) + return hop.gendirectcall(ll_isinstance_const, v_obj, v_cls, level) else: return hop.gendirectcall(ll_isinstance, v_obj, v_cls) @@ -679,11 +774,10 @@ return cast_pointer(OBJECTPTR, obj).typeptr def ll_issubclass(subcls, cls): - return cls.subclassrange_min <= subcls.subclassrange_min <= cls.subclassrange_max - -def ll_issubclass_const(subcls, minid, maxid): - return minid <= subcls.subclassrange_min <= maxid + return subcls.level >= cls.level and subcls.classrow[cls.level] == cls +def ll_issubclass_const(subcls, cls, level): + return subcls.level >= level and subcls.classrow[level] == cls def ll_isinstance(obj, cls): # obj should be cast to OBJECT or NONGCOBJECT if not obj: @@ -691,12 +785,12 @@ obj_cls = obj.typeptr return ll_issubclass(obj_cls, cls) -def ll_isinstance_const(obj, minid, maxid): +def ll_isinstance_const(obj, cls, level): if not obj: return False - return ll_issubclass_const(obj.typeptr, minid, maxid) + return ll_issubclass_const(obj.typeptr, cls, level) -def ll_isinstance_exact(obj, cls): +def ll_isinstance_exact(obj, cls): #unused if not obj: return False obj_cls = obj.typeptr Modified: pypy/branch/sepcomp/pypy/rpython/lltypesystem/rtagged.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/lltypesystem/rtagged.py (original) +++ pypy/branch/sepcomp/pypy/rpython/lltypesystem/rtagged.py Wed Dec 9 12:48:20 2009 @@ -129,10 +129,9 @@ cls = v_cls.value answer = self.unboxedclassdef.issubclass(classdef) c_answer_if_unboxed = hop.inputconst(lltype.Bool, answer) - minid = hop.inputconst(lltype.Signed, cls.subclassrange_min) - maxid = hop.inputconst(lltype.Signed, cls.subclassrange_max) - return hop.gendirectcall(ll_unboxed_isinstance_const, v_obj, - minid, maxid, c_answer_if_unboxed) + level = hop.inputconst(lltype.Signed, cls.level) + return hop.gendirectcall(ll_unboxed_isinstance_const, v_obj, v_cls, + level, c_answer_if_unboxed) def ll_int_to_unboxed(PTRTYPE, value): @@ -147,10 +146,10 @@ else: return instance.typeptr -def ll_unboxed_isinstance_const(obj, minid, maxid, answer_if_unboxed): +def ll_unboxed_isinstance_const(obj, cls, level, answer_if_unboxed): if not obj: return False if lltype.cast_ptr_to_int(obj) & 1: return answer_if_unboxed else: - return ll_issubclass_const(obj.typeptr, minid, maxid) + return ll_issubclass_const(obj.typeptr, cls, level) Modified: pypy/branch/sepcomp/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/branch/sepcomp/pypy/rpython/lltypesystem/test/test_lltype.py Wed Dec 9 12:48:20 2009 @@ -326,6 +326,19 @@ assert S == S1 assert hash(S1) == hash(S) + +def test_pickle(): + S = ForwardReference() + S.become(Struct('S', ('p', Ptr(S)))) + assert S == S + hash(S) + from pickle import dumps, loads + S_pickle = dumps(S) + S_loaded = loads(S_pickle) + assert S_loaded == S + assert hash(S_loaded) == hash(S) + + def test_array_with_non_container_elements(): As = GcArray(Signed) a = malloc(As, 3) Modified: pypy/branch/sepcomp/pypy/rpython/normalizecalls.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/normalizecalls.py (original) +++ pypy/branch/sepcomp/pypy/rpython/normalizecalls.py Wed Dec 9 12:48:20 2009 @@ -331,6 +331,14 @@ classdef.minid = TotalOrderSymbolic(witness, lst) classdef.maxid = TotalOrderSymbolic(witness + [MAX], lst) + +def assign_inheritance_levels(annotator): + bk = annotator.bookkeeper + for classdef in bk.classdefs: + if not hasattr(classdef, 'level'): + classdef.level = len(tuple(classdef.getmro())) - 1 + + MAX = 1E100 _cdef_id_counter = 0 def get_unique_cdef_id(cdef): @@ -350,7 +358,7 @@ try: normalize_call_familes(rtyper.annotator) merge_classpbc_getattr_into_classdef(rtyper) - assign_inheritance_ids(rtyper.annotator) + assign_inheritance_levels(rtyper.annotator) finally: rtyper.annotator.frozen -= 1 create_instantiate_functions(rtyper.annotator) Modified: pypy/branch/sepcomp/pypy/rpython/rclass.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/rclass.py (original) +++ pypy/branch/sepcomp/pypy/rpython/rclass.py Wed Dec 9 12:48:20 2009 @@ -77,7 +77,7 @@ if not s_value.isNone() and s_value.getKind() == description.MethodDesc: s_value = self.classdef.lookup_filter(s_value) funcdescs = [mdesc.funcdesc for mdesc in s_value.descriptions] - return annmodel.SomePBC(funcdescs) + return annmodel.SomePBC(funcdescs, force_virtual_access=any(mdesc.force_virtual_access for mdesc in s_value.descriptions)) return None # not a method def get_ll_eq_function(self): Modified: pypy/branch/sepcomp/pypy/rpython/rpbc.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/rpbc.py (original) +++ pypy/branch/sepcomp/pypy/rpython/rpbc.py Wed Dec 9 12:48:20 2009 @@ -177,12 +177,14 @@ class AbstractFunctionsPBCRepr(CanBeNull, Repr): """Representation selected for a PBC of function(s).""" - + force_virtual = False def __init__(self, rtyper, s_pbc): self.rtyper = rtyper self.s_pbc = s_pbc self.callfamily = s_pbc.descriptions.iterkeys().next().getcallfamily() - if len(s_pbc.descriptions) == 1 and not s_pbc.can_be_None: + + if len(s_pbc.descriptions) == 1 and not s_pbc.can_be_None and not ( + s_pbc.force_virtual_access): # a single function self.lowleveltype = Void else: Modified: pypy/branch/sepcomp/pypy/rpython/rtyper.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/rtyper.py (original) +++ pypy/branch/sepcomp/pypy/rpython/rtyper.py Wed Dec 9 12:48:20 2009 @@ -689,6 +689,9 @@ self.r_result = rtyper.getrepr(self.s_result) rtyper.call_all_setups() # compute ForwardReferences now + def __repr__(self): + return '' % (self.spaceop, len(self.llops)) + def copy(self): result = HighLevelOp(self.rtyper, self.spaceop, self.exceptionlinks, self.llops) Modified: pypy/branch/sepcomp/pypy/rpython/test/test_rpbc.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/test/test_rpbc.py (original) +++ pypy/branch/sepcomp/pypy/rpython/test/test_rpbc.py Wed Dec 9 12:48:20 2009 @@ -1595,7 +1595,33 @@ self.interpret(f, [int]) class TestLLtype(BaseTestRPBC, LLRtypeMixin): - pass + def test_devoid_exported_func(self): + from pypy.translator.translator import graphof + from pypy.objspace.flow.model import summary + class c: + _force_virtual_ = True + def foo(self): + return 42 + foo._force_virtual_ = True + def f(): + inst = c() + inst.foo() + return inst + def testfunc(x): + return x.foo() + def h(): + return testfunc(c()) + + t, r, g = self.gengraph(f) + bk = t.annotator.bookkeeper + getcdef = bk.getuniqueclassdef + classdef = getcdef(c) + vt = t.rtyper.class_reprs[classdef].getvtable() + assert isinstance(typeOf(t.rtyper.class_reprs[classdef].getvtable(False).cls_foo).TO, FuncType) + t, r, g2 = self.gengraph(h) + assert not summary(g).get('direct_call', 0) + assert not summary(graphof(t, testfunc)).get('direct_call', 0) + class TestOOtype(BaseTestRPBC, OORtypeMixin): pass From xoraxax at codespeak.net Wed Dec 9 12:49:12 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 12:49:12 +0100 (CET) Subject: [pypy-svn] r70011 - in pypy/branch/sepcomp/pypy/rpython: . lltypesystem lltypesystem/test test Message-ID: <20091209114912.D3467168072@codespeak.net> Author: xoraxax Date: Wed Dec 9 12:49:12 2009 New Revision: 70011 Modified: pypy/branch/sepcomp/pypy/rpython/lltypesystem/lltype.py pypy/branch/sepcomp/pypy/rpython/lltypesystem/rclass.py pypy/branch/sepcomp/pypy/rpython/lltypesystem/rtagged.py pypy/branch/sepcomp/pypy/rpython/lltypesystem/test/test_lltype.py pypy/branch/sepcomp/pypy/rpython/normalizecalls.py pypy/branch/sepcomp/pypy/rpython/rclass.py pypy/branch/sepcomp/pypy/rpython/rpbc.py pypy/branch/sepcomp/pypy/rpython/rtyper.py pypy/branch/sepcomp/pypy/rpython/test/test_rpbc.py Log: Revert last commit. Modified: pypy/branch/sepcomp/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/sepcomp/pypy/rpython/lltypesystem/lltype.py Wed Dec 9 12:49:12 2009 @@ -76,12 +76,6 @@ _is_compatible = __eq__ - def __getstate__(self): - return self.__dict__ - - def __setstate__(self, val): - self.__dict__.update(val) - def _enforce(self, value): if typeOf(value) != self: raise TypeError @@ -112,6 +106,11 @@ self.__cached_hash = result return result + # due to this dynamic hash value, we should forbid + # pickling, until we have an algorithm for that. + # but we just provide a tag for external help. + __hash_is_not_constant__ = True + def __repr__(self): return '<%s>' % (self,) @@ -1261,7 +1260,6 @@ class _container(object): __slots__ = () - _exported = False def _parentstructure(self, check=True): return None def _check(self): @@ -1282,7 +1280,7 @@ __slots__ = ('_TYPE', '_parent_type', '_parent_index', '_keepparent', - '_wrparent',"_exported", + '_wrparent', '__weakref__', '_storage') @@ -1653,9 +1651,7 @@ return id(self) def __setattr__(self, attr, value): - if attr != "_exported": - raise AttributeError("cannot change the attributes of %r" % (self,)) - _container.__setattr__(self, attr, value) + raise AttributeError("cannot change the attributes of %r" % (self,)) class _opaque(_parentable): def __init__(self, TYPE, parent=None, parentindex=None, **attrs): @@ -1700,18 +1696,8 @@ return _parentable._normalizedcontainer(self) -class _external_reference(_container): - def __init__(self, TYPE, name, component=None): - self._TYPE = TYPE - self.name = name - self.component = component - self.iddata = str(component) + name - - def _getid(self): - return id(self.iddata) - - class _pyobject(Hashable, _container): + __slots__ = [] # or we get in trouble with pickling _TYPE = PyObject @@ -1778,10 +1764,6 @@ o = _pyobject(obj) return _ptr(Ptr(PyObject), o) -def externalptr(TYPE, name, component=None): - o = _external_reference(TYPE, name, component) - return _ptr(Ptr(TYPE), o, solid=True) - def cast_ptr_to_int(ptr): return ptr._cast_to_int() Modified: pypy/branch/sepcomp/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/sepcomp/pypy/rpython/lltypesystem/rclass.py Wed Dec 9 12:49:12 2009 @@ -14,8 +14,7 @@ cast_pointer, cast_ptr_to_int, castable, nullptr, \ RuntimeTypeInfo, getRuntimeTypeInfo, typeOf, \ Array, Char, Void, attachRuntimeTypeInfo, \ - FuncType, Bool, Signed, functionptr, FuncType, PyObject, \ - normalizeptr + FuncType, Bool, Signed, functionptr, FuncType, PyObject from pypy.rpython.lltypesystem import lltype from pypy.rpython.robject import PyObjRepr, pyobj_repr from pypy.rpython.extregistry import ExtRegistryEntry @@ -63,8 +62,8 @@ OBJECTPTR = Ptr(OBJECT) OBJECT_VTABLE.become(Struct('object_vtable', #('parenttypeptr', CLASSTYPE), - ('level', Signed), - ('classrow', Ptr(Array(CLASSTYPE))), + ('subclassrange_min', Signed), + ('subclassrange_max', Signed), ('rtti', Ptr(RuntimeTypeInfo)), ('name', Ptr(Array(Char))), ('instantiate', Ptr(FuncType([], OBJECTPTR))), @@ -87,50 +86,6 @@ return vtable - -def weave_llfields(llfields, classinfo, types_only=False): - llfields_iter = iter(llfields) - result = [] - for mangled_name, name, value in classinfo: - if name is not None: - curfield = llfields_iter.next() - while curfield is not None and curfield[0] != mangled_name: # XXX does this make sense? - result.append((mangled_name, value.get_func())) - try: - curfield = llfields_iter.next() - except StopIteration: - curfield = None - if curfield is not None: - result.append(curfield) - else: - if not types_only: - value = typeOf(value) - result.append((mangled_name, value)) - try: - llfields_iter.next() - except StopIteration: - pass - else: - raise TyperError("Incorrect set of llfields") - return result - -def extend_exportinfo(exportinfo, mangled_name, name, r, classdesc=None): - T = r.lowleveltype - if classdesc is None: - llvalue = T - var = None - else: - var = classdesc.read_attribute(name, None) - llvalue = r.convert_desc_or_const(var) - #import pdb; pdb.set_trace() - if var is None or not getattr(var.value, '_export_', None): - name = None # hide - val = llvalue - else: - val = var.value - exportinfo.append((mangled_name, name, val)) - - class ClassRepr(AbstractClassRepr): def __init__(self, rtyper, classdef): AbstractClassRepr.__init__(self, rtyper, classdef) @@ -142,7 +97,6 @@ self.lowleveltype = Ptr(self.vtable_type) def _setup_repr(self): - exportinfo = None # NOTE: don't store mutable objects like the dicts below on 'self' # before they are fully built, to avoid strange bugs in case # of recursion where other code would uses these @@ -151,9 +105,6 @@ pbcfields = {} allmethods = {} if self.classdef is not None: - if self.classdef.is_exported(): - exportinfo = [] - importinfo = self.classdef.get_import_data() # class attributes llfields = [] attrs = self.classdef.attrs.items() @@ -166,13 +117,9 @@ allmethods[name] = True s_value = s_unboundmethod r = self.rtyper.getrepr(s_value) - #if "Multi" in repr(r) or ("Void" in repr(r.lowleveltype) and name in ("foo", "bar") and "foovoid" not in repr(self)): - # import pdb; pdb.set_trace() mangled_name = 'cls_' + name clsfields[name] = mangled_name, r llfields.append((mangled_name, r.lowleveltype)) - if exportinfo is not None and r.lowleveltype is not Void: - extend_exportinfo(exportinfo, mangled_name, name, r, self.classdef.classdesc) # attributes showing up in getattrs done on the class as a PBC extra_access_sets = self.rtyper.class_pbc_attributes.get( self.classdef, {}) @@ -181,12 +128,7 @@ mangled_name = mangle('pbc%d' % counter, attr) pbcfields[access_set, attr] = mangled_name, r llfields.append((mangled_name, r.lowleveltype)) - if exportinfo is not None: - assert 0, "not yet supported" # - if importinfo: - #import pdb; pdb.set_trace() - llfields = weave_llfields(llfields, importinfo.cls) self.rbase = getclassrepr(self.rtyper, self.classdef.basedef) self.rbase.setup() kwds = {'hints': {'immutable': True}} @@ -194,8 +136,6 @@ ('super', self.rbase.vtable_type), *llfields, **kwds) self.vtable_type.become(vtable_type) - if exportinfo is not None: - self.classdef.exportinfo_cls = exportinfo allmethods.update(self.rbase.allmethods) self.clsfields = clsfields self.pbcfields = pbcfields @@ -219,20 +159,8 @@ def getvtable(self, cast_to_typeptr=True): """Return a ptr to the vtable of this type.""" if self.vtable is None: - if self.classdef and self.classdef.get_import_data(): - #from pypy.rpython.lltypesystem.rffi import CConstant - handle = self.classdef.get_import_data().vtablename - component = None - if isinstance(handle, tuple): - component, handle = handle - #T = lltype.Ptr(lltype.OpaqueType(handle, hints=dict(external_void=True))) - #val = CConstant("&" + handle, T) - self.vtable = lltype.externalptr(OBJECT_VTABLE, handle + "_vtable", component) - #self.vtable = val - cast_to_typeptr = False - else: - self.vtable = malloc(self.vtable_type, immortal=True) - self.setup_vtable(self.vtable, self) + self.vtable = malloc(self.vtable_type, immortal=True) + self.setup_vtable(self.vtable, self) # vtable = self.vtable if cast_to_typeptr: @@ -247,18 +175,11 @@ # initialize the 'subclassrange_*' and 'name' fields if rsubcls.classdef is not None: #vtable.parenttypeptr = rsubcls.rbase.getvtable() - vtable.level = rsubcls.classdef.level - vtable.classrow = malloc(Array(CLASSTYPE), rsubcls.classdef.level + 1, immortal=True) - cur_repr = rsubcls - for i in xrange(rsubcls.classdef.level, -1, -1): - vtable.classrow[i] = cur_repr.getvtable() - cur_repr = cur_repr.rbase - - if rsubcls.classdef.is_exported(): - rsubcls.classdef.exportinfo_vtableval = normalizeptr(vtable) + vtable.subclassrange_min = rsubcls.classdef.minid + vtable.subclassrange_max = rsubcls.classdef.maxid else: #for the root class - vtable.level = -1 - vtable.classrow = nullptr(Array(CLASSTYPE)) + vtable.subclassrange_min = 0 + vtable.subclassrange_max = sys.maxint rinstance = getinstancerepr(self.rtyper, rsubcls.classdef) rinstance.setup() if rinstance.gcflavor == 'gc': @@ -277,7 +198,6 @@ #else: the classdef was created recently, so no instantiate() # could reach it else: - importinfo = self.classdef.get_import_data() # setup class attributes: for each attribute name at the level # of 'self', look up its value in the subclass rsubcls def assign(mangled_name, value): @@ -286,7 +206,7 @@ llvalue = r.convert_desc_or_const(value) setattr(vtable, mangled_name, llvalue) - mro = list(rsubcls.classdef.getmro()) # XXX not needed? + mro = list(rsubcls.classdef.getmro()) for fldname in self.clsfields: mangled_name, r = self.clsfields[fldname] if r.lowleveltype is Void: @@ -303,10 +223,6 @@ attrvalue = rsubcls.classdef.classdesc.read_attribute(attr, None) if attrvalue is not None: assign(mangled_name, attrvalue) - if importinfo: - for mangled_name, name, val in importinfo.cls: - if not name: - setattr(vtable, mangled_name, val) # then initialize the 'super' portion of the vtable self.rbase.setup_vtable(vtable.super, rsubcls) @@ -366,8 +282,10 @@ ## # a class with no subclass ## return hop.genop('ptr_eq', [v_cls1, v_cls2], resulttype=Bool) ## else: - level = hop.inputconst(Signed, cls2.level) - return hop.gendirectcall(ll_issubclass_const, v_cls1, v_cls2, level) + minid = hop.inputconst(Signed, cls2.subclassrange_min) + maxid = hop.inputconst(Signed, cls2.subclassrange_max) + return hop.gendirectcall(ll_issubclass_const, v_cls1, minid, + maxid) else: v_cls1, v_cls2 = hop.inputargs(class_repr, class_repr) return hop.gendirectcall(ll_issubclass, v_cls1, v_cls2) @@ -389,7 +307,6 @@ self.gcflavor = gcflavor def _setup_repr(self, llfields=None, hints=None, adtmeths=None): - exportinfo = None # NOTE: don't store mutable objects like the dicts below on 'self' # before they are fully built, to avoid strange bugs in case # of recursion where other code would uses these @@ -400,8 +317,6 @@ if self.classdef is None: fields['__class__'] = 'typeptr', get_type_repr(self.rtyper) else: - if self.classdef.is_exported(): - exportinfo = [] # instance attributes if llfields is None: llfields = [] @@ -413,18 +328,12 @@ mangled_name = 'inst_' + name fields[name] = mangled_name, r llfields.append((mangled_name, r.lowleveltype)) - if exportinfo is not None: - extend_exportinfo(exportinfo, mangled_name, name, r, None) - # # hash() support if self.rtyper.needs_hash_support(self.classdef): from pypy.rpython import rint fields['_hash_cache_'] = 'hash_cache', rint.signed_repr llfields.append(('hash_cache', Signed)) - importinfo = self.classdef.get_import_data() - if importinfo: - llfields = weave_llfields(llfields, importinfo.inst, True) self.rbase = getinstancerepr(self.rtyper, self.classdef.basedef, self.gcflavor) @@ -438,11 +347,6 @@ if '_immutable_' in self.classdef.classdesc.classdict: hints = hints.copy() hints['immutable'] = True - if self.classdef.is_exported(): - hints = hints.copy() - hints['_exported'] = True - if exportinfo is not None: - self.classdef.exportinfo_inst = exportinfo object_type = MkStruct(self.classdef.name, ('super', self.rbase.object_type), hints=hints, @@ -680,15 +584,16 @@ instance_repr = self.common_repr() v_obj, v_cls = hop.inputargs(instance_repr, class_repr) - if False and isinstance(v_cls, Constant): # XXX + if isinstance(v_cls, Constant): cls = v_cls.value # XXX re-implement the following optimization #if cls.subclassrange_max == cls.subclassrange_min: # # a class with no subclass # return hop.gendirectcall(rclass.ll_isinstance_exact, v_obj, v_cls) #else: - level = hop.inputconst(Signed, cls.level) - return hop.gendirectcall(ll_isinstance_const, v_obj, v_cls, level) + minid = hop.inputconst(Signed, cls.subclassrange_min) + maxid = hop.inputconst(Signed, cls.subclassrange_max) + return hop.gendirectcall(ll_isinstance_const, v_obj, minid, maxid) else: return hop.gendirectcall(ll_isinstance, v_obj, v_cls) @@ -774,10 +679,11 @@ return cast_pointer(OBJECTPTR, obj).typeptr def ll_issubclass(subcls, cls): - return subcls.level >= cls.level and subcls.classrow[cls.level] == cls + return cls.subclassrange_min <= subcls.subclassrange_min <= cls.subclassrange_max + +def ll_issubclass_const(subcls, minid, maxid): + return minid <= subcls.subclassrange_min <= maxid -def ll_issubclass_const(subcls, cls, level): - return subcls.level >= level and subcls.classrow[level] == cls def ll_isinstance(obj, cls): # obj should be cast to OBJECT or NONGCOBJECT if not obj: @@ -785,12 +691,12 @@ obj_cls = obj.typeptr return ll_issubclass(obj_cls, cls) -def ll_isinstance_const(obj, cls, level): +def ll_isinstance_const(obj, minid, maxid): if not obj: return False - return ll_issubclass_const(obj.typeptr, cls, level) + return ll_issubclass_const(obj.typeptr, minid, maxid) -def ll_isinstance_exact(obj, cls): #unused +def ll_isinstance_exact(obj, cls): if not obj: return False obj_cls = obj.typeptr Modified: pypy/branch/sepcomp/pypy/rpython/lltypesystem/rtagged.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/lltypesystem/rtagged.py (original) +++ pypy/branch/sepcomp/pypy/rpython/lltypesystem/rtagged.py Wed Dec 9 12:49:12 2009 @@ -129,9 +129,10 @@ cls = v_cls.value answer = self.unboxedclassdef.issubclass(classdef) c_answer_if_unboxed = hop.inputconst(lltype.Bool, answer) - level = hop.inputconst(lltype.Signed, cls.level) - return hop.gendirectcall(ll_unboxed_isinstance_const, v_obj, v_cls, - level, c_answer_if_unboxed) + minid = hop.inputconst(lltype.Signed, cls.subclassrange_min) + maxid = hop.inputconst(lltype.Signed, cls.subclassrange_max) + return hop.gendirectcall(ll_unboxed_isinstance_const, v_obj, + minid, maxid, c_answer_if_unboxed) def ll_int_to_unboxed(PTRTYPE, value): @@ -146,10 +147,10 @@ else: return instance.typeptr -def ll_unboxed_isinstance_const(obj, cls, level, answer_if_unboxed): +def ll_unboxed_isinstance_const(obj, minid, maxid, answer_if_unboxed): if not obj: return False if lltype.cast_ptr_to_int(obj) & 1: return answer_if_unboxed else: - return ll_issubclass_const(obj.typeptr, cls, level) + return ll_issubclass_const(obj.typeptr, minid, maxid) Modified: pypy/branch/sepcomp/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/branch/sepcomp/pypy/rpython/lltypesystem/test/test_lltype.py Wed Dec 9 12:49:12 2009 @@ -326,19 +326,6 @@ assert S == S1 assert hash(S1) == hash(S) - -def test_pickle(): - S = ForwardReference() - S.become(Struct('S', ('p', Ptr(S)))) - assert S == S - hash(S) - from pickle import dumps, loads - S_pickle = dumps(S) - S_loaded = loads(S_pickle) - assert S_loaded == S - assert hash(S_loaded) == hash(S) - - def test_array_with_non_container_elements(): As = GcArray(Signed) a = malloc(As, 3) Modified: pypy/branch/sepcomp/pypy/rpython/normalizecalls.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/normalizecalls.py (original) +++ pypy/branch/sepcomp/pypy/rpython/normalizecalls.py Wed Dec 9 12:49:12 2009 @@ -331,14 +331,6 @@ classdef.minid = TotalOrderSymbolic(witness, lst) classdef.maxid = TotalOrderSymbolic(witness + [MAX], lst) - -def assign_inheritance_levels(annotator): - bk = annotator.bookkeeper - for classdef in bk.classdefs: - if not hasattr(classdef, 'level'): - classdef.level = len(tuple(classdef.getmro())) - 1 - - MAX = 1E100 _cdef_id_counter = 0 def get_unique_cdef_id(cdef): @@ -358,7 +350,7 @@ try: normalize_call_familes(rtyper.annotator) merge_classpbc_getattr_into_classdef(rtyper) - assign_inheritance_levels(rtyper.annotator) + assign_inheritance_ids(rtyper.annotator) finally: rtyper.annotator.frozen -= 1 create_instantiate_functions(rtyper.annotator) Modified: pypy/branch/sepcomp/pypy/rpython/rclass.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/rclass.py (original) +++ pypy/branch/sepcomp/pypy/rpython/rclass.py Wed Dec 9 12:49:12 2009 @@ -77,7 +77,7 @@ if not s_value.isNone() and s_value.getKind() == description.MethodDesc: s_value = self.classdef.lookup_filter(s_value) funcdescs = [mdesc.funcdesc for mdesc in s_value.descriptions] - return annmodel.SomePBC(funcdescs, force_virtual_access=any(mdesc.force_virtual_access for mdesc in s_value.descriptions)) + return annmodel.SomePBC(funcdescs) return None # not a method def get_ll_eq_function(self): Modified: pypy/branch/sepcomp/pypy/rpython/rpbc.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/rpbc.py (original) +++ pypy/branch/sepcomp/pypy/rpython/rpbc.py Wed Dec 9 12:49:12 2009 @@ -177,14 +177,12 @@ class AbstractFunctionsPBCRepr(CanBeNull, Repr): """Representation selected for a PBC of function(s).""" - force_virtual = False + def __init__(self, rtyper, s_pbc): self.rtyper = rtyper self.s_pbc = s_pbc self.callfamily = s_pbc.descriptions.iterkeys().next().getcallfamily() - - if len(s_pbc.descriptions) == 1 and not s_pbc.can_be_None and not ( - s_pbc.force_virtual_access): + if len(s_pbc.descriptions) == 1 and not s_pbc.can_be_None: # a single function self.lowleveltype = Void else: Modified: pypy/branch/sepcomp/pypy/rpython/rtyper.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/rtyper.py (original) +++ pypy/branch/sepcomp/pypy/rpython/rtyper.py Wed Dec 9 12:49:12 2009 @@ -689,9 +689,6 @@ self.r_result = rtyper.getrepr(self.s_result) rtyper.call_all_setups() # compute ForwardReferences now - def __repr__(self): - return '' % (self.spaceop, len(self.llops)) - def copy(self): result = HighLevelOp(self.rtyper, self.spaceop, self.exceptionlinks, self.llops) Modified: pypy/branch/sepcomp/pypy/rpython/test/test_rpbc.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/test/test_rpbc.py (original) +++ pypy/branch/sepcomp/pypy/rpython/test/test_rpbc.py Wed Dec 9 12:49:12 2009 @@ -1595,33 +1595,7 @@ self.interpret(f, [int]) class TestLLtype(BaseTestRPBC, LLRtypeMixin): - def test_devoid_exported_func(self): - from pypy.translator.translator import graphof - from pypy.objspace.flow.model import summary - class c: - _force_virtual_ = True - def foo(self): - return 42 - foo._force_virtual_ = True - def f(): - inst = c() - inst.foo() - return inst - def testfunc(x): - return x.foo() - def h(): - return testfunc(c()) - - t, r, g = self.gengraph(f) - bk = t.annotator.bookkeeper - getcdef = bk.getuniqueclassdef - classdef = getcdef(c) - vt = t.rtyper.class_reprs[classdef].getvtable() - assert isinstance(typeOf(t.rtyper.class_reprs[classdef].getvtable(False).cls_foo).TO, FuncType) - t, r, g2 = self.gengraph(h) - assert not summary(g).get('direct_call', 0) - assert not summary(graphof(t, testfunc)).get('direct_call', 0) - + pass class TestOOtype(BaseTestRPBC, OORtypeMixin): pass From xoraxax at codespeak.net Wed Dec 9 12:50:23 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 12:50:23 +0100 (CET) Subject: [pypy-svn] r70012 - pypy/branch/sepcomp/pypy/rpython Message-ID: <20091209115023.B93E4168072@codespeak.net> Author: xoraxax Date: Wed Dec 9 12:50:23 2009 New Revision: 70012 Modified: pypy/branch/sepcomp/pypy/rpython/rtyper.py Log: Add HOp repr. Modified: pypy/branch/sepcomp/pypy/rpython/rtyper.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/rtyper.py (original) +++ pypy/branch/sepcomp/pypy/rpython/rtyper.py Wed Dec 9 12:50:23 2009 @@ -689,6 +689,9 @@ self.r_result = rtyper.getrepr(self.s_result) rtyper.call_all_setups() # compute ForwardReferences now + def __repr__(self): + return '' % (self.spaceop, len(self.llops)) + def copy(self): result = HighLevelOp(self.rtyper, self.spaceop, self.exceptionlinks, self.llops) From xoraxax at codespeak.net Wed Dec 9 12:50:46 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 12:50:46 +0100 (CET) Subject: [pypy-svn] r70013 - in pypy/branch/sepcomp/pypy/rpython: . lltypesystem/test test Message-ID: <20091209115046.CACB8168072@codespeak.net> Author: xoraxax Date: Wed Dec 9 12:50:46 2009 New Revision: 70013 Modified: pypy/branch/sepcomp/pypy/rpython/lltypesystem/test/test_lltype.py pypy/branch/sepcomp/pypy/rpython/rpbc.py pypy/branch/sepcomp/pypy/rpython/test/test_rpbc.py Log: Support for force_virtual_access, _exported attr of lltype containers, externalptr, _external_reference Allow pickling of lltypes. Modified: pypy/branch/sepcomp/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/branch/sepcomp/pypy/rpython/lltypesystem/test/test_lltype.py Wed Dec 9 12:50:46 2009 @@ -326,6 +326,19 @@ assert S == S1 assert hash(S1) == hash(S) + +def test_pickle(): + S = ForwardReference() + S.become(Struct('S', ('p', Ptr(S)))) + assert S == S + hash(S) + from pickle import dumps, loads + S_pickle = dumps(S) + S_loaded = loads(S_pickle) + assert S_loaded == S + assert hash(S_loaded) == hash(S) + + def test_array_with_non_container_elements(): As = GcArray(Signed) a = malloc(As, 3) Modified: pypy/branch/sepcomp/pypy/rpython/rpbc.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/rpbc.py (original) +++ pypy/branch/sepcomp/pypy/rpython/rpbc.py Wed Dec 9 12:50:46 2009 @@ -177,12 +177,14 @@ class AbstractFunctionsPBCRepr(CanBeNull, Repr): """Representation selected for a PBC of function(s).""" - + force_virtual = False def __init__(self, rtyper, s_pbc): self.rtyper = rtyper self.s_pbc = s_pbc self.callfamily = s_pbc.descriptions.iterkeys().next().getcallfamily() - if len(s_pbc.descriptions) == 1 and not s_pbc.can_be_None: + + if len(s_pbc.descriptions) == 1 and not s_pbc.can_be_None and not ( + s_pbc.force_virtual_access): # a single function self.lowleveltype = Void else: Modified: pypy/branch/sepcomp/pypy/rpython/test/test_rpbc.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/test/test_rpbc.py (original) +++ pypy/branch/sepcomp/pypy/rpython/test/test_rpbc.py Wed Dec 9 12:50:46 2009 @@ -1595,7 +1595,33 @@ self.interpret(f, [int]) class TestLLtype(BaseTestRPBC, LLRtypeMixin): - pass + def test_devoid_exported_func(self): + from pypy.translator.translator import graphof + from pypy.objspace.flow.model import summary + class c: + _force_virtual_ = True + def foo(self): + return 42 + foo._force_virtual_ = True + def f(): + inst = c() + inst.foo() + return inst + def testfunc(x): + return x.foo() + def h(): + return testfunc(c()) + + t, r, g = self.gengraph(f) + bk = t.annotator.bookkeeper + getcdef = bk.getuniqueclassdef + classdef = getcdef(c) + vt = t.rtyper.class_reprs[classdef].getvtable() + assert isinstance(typeOf(t.rtyper.class_reprs[classdef].getvtable(False).cls_foo).TO, FuncType) + t, r, g2 = self.gengraph(h) + assert not summary(g).get('direct_call', 0) + assert not summary(graphof(t, testfunc)).get('direct_call', 0) + class TestOOtype(BaseTestRPBC, OORtypeMixin): pass From xoraxax at codespeak.net Wed Dec 9 12:51:14 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 12:51:14 +0100 (CET) Subject: [pypy-svn] r70014 - pypy/branch/sepcomp/pypy/rpython/lltypesystem Message-ID: <20091209115114.0B072168072@codespeak.net> Author: xoraxax Date: Wed Dec 9 12:51:14 2009 New Revision: 70014 Modified: pypy/branch/sepcomp/pypy/rpython/lltypesystem/lltype.py Log: ... belongs to last commit. Modified: pypy/branch/sepcomp/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/sepcomp/pypy/rpython/lltypesystem/lltype.py Wed Dec 9 12:51:14 2009 @@ -76,6 +76,12 @@ _is_compatible = __eq__ + def __getstate__(self): + return self.__dict__ + + def __setstate__(self, val): + self.__dict__.update(val) + def _enforce(self, value): if typeOf(value) != self: raise TypeError @@ -106,11 +112,6 @@ self.__cached_hash = result return result - # due to this dynamic hash value, we should forbid - # pickling, until we have an algorithm for that. - # but we just provide a tag for external help. - __hash_is_not_constant__ = True - def __repr__(self): return '<%s>' % (self,) @@ -1260,6 +1261,7 @@ class _container(object): __slots__ = () + _exported = False def _parentstructure(self, check=True): return None def _check(self): @@ -1280,7 +1282,7 @@ __slots__ = ('_TYPE', '_parent_type', '_parent_index', '_keepparent', - '_wrparent', + '_wrparent',"_exported", '__weakref__', '_storage') @@ -1651,7 +1653,9 @@ return id(self) def __setattr__(self, attr, value): - raise AttributeError("cannot change the attributes of %r" % (self,)) + if attr != "_exported": + raise AttributeError("cannot change the attributes of %r" % (self,)) + _container.__setattr__(self, attr, value) class _opaque(_parentable): def __init__(self, TYPE, parent=None, parentindex=None, **attrs): @@ -1696,8 +1700,18 @@ return _parentable._normalizedcontainer(self) +class _external_reference(_container): + def __init__(self, TYPE, name, component=None): + self._TYPE = TYPE + self.name = name + self.component = component + self.iddata = str(component) + name + + def _getid(self): + return id(self.iddata) + + class _pyobject(Hashable, _container): - __slots__ = [] # or we get in trouble with pickling _TYPE = PyObject @@ -1764,6 +1778,10 @@ o = _pyobject(obj) return _ptr(Ptr(PyObject), o) +def externalptr(TYPE, name, component=None): + o = _external_reference(TYPE, name, component) + return _ptr(Ptr(TYPE), o, solid=True) + def cast_ptr_to_int(ptr): return ptr._cast_to_int() From xoraxax at codespeak.net Wed Dec 9 12:52:23 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 12:52:23 +0100 (CET) Subject: [pypy-svn] r70015 - pypy/branch/sepcomp/pypy/rpython Message-ID: <20091209115223.92F46168072@codespeak.net> Author: xoraxax Date: Wed Dec 9 12:52:23 2009 New Revision: 70015 Modified: pypy/branch/sepcomp/pypy/rpython/rclass.py Log: ... belongs to last commit. Modified: pypy/branch/sepcomp/pypy/rpython/rclass.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/rclass.py (original) +++ pypy/branch/sepcomp/pypy/rpython/rclass.py Wed Dec 9 12:52:23 2009 @@ -77,7 +77,7 @@ if not s_value.isNone() and s_value.getKind() == description.MethodDesc: s_value = self.classdef.lookup_filter(s_value) funcdescs = [mdesc.funcdesc for mdesc in s_value.descriptions] - return annmodel.SomePBC(funcdescs) + return annmodel.SomePBC(funcdescs, force_virtual_access=any(mdesc.force_virtual_access for mdesc in s_value.descriptions)) return None # not a method def get_ll_eq_function(self): From xoraxax at codespeak.net Wed Dec 9 12:52:53 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 12:52:53 +0100 (CET) Subject: [pypy-svn] r70016 - in pypy/branch/sepcomp/pypy/rpython: . lltypesystem Message-ID: <20091209115253.DA1C5168072@codespeak.net> Author: xoraxax Date: Wed Dec 9 12:52:52 2009 New Revision: 70016 Modified: pypy/branch/sepcomp/pypy/rpython/lltypesystem/rclass.py pypy/branch/sepcomp/pypy/rpython/lltypesystem/rtagged.py pypy/branch/sepcomp/pypy/rpython/normalizecalls.py Log: Add rtyper hacks and change the subclassing logic. Modified: pypy/branch/sepcomp/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/sepcomp/pypy/rpython/lltypesystem/rclass.py Wed Dec 9 12:52:52 2009 @@ -14,7 +14,8 @@ cast_pointer, cast_ptr_to_int, castable, nullptr, \ RuntimeTypeInfo, getRuntimeTypeInfo, typeOf, \ Array, Char, Void, attachRuntimeTypeInfo, \ - FuncType, Bool, Signed, functionptr, FuncType, PyObject + FuncType, Bool, Signed, functionptr, FuncType, PyObject, \ + normalizeptr from pypy.rpython.lltypesystem import lltype from pypy.rpython.robject import PyObjRepr, pyobj_repr from pypy.rpython.extregistry import ExtRegistryEntry @@ -62,8 +63,8 @@ OBJECTPTR = Ptr(OBJECT) OBJECT_VTABLE.become(Struct('object_vtable', #('parenttypeptr', CLASSTYPE), - ('subclassrange_min', Signed), - ('subclassrange_max', Signed), + ('level', Signed), + ('classrow', Ptr(Array(CLASSTYPE))), ('rtti', Ptr(RuntimeTypeInfo)), ('name', Ptr(Array(Char))), ('instantiate', Ptr(FuncType([], OBJECTPTR))), @@ -86,6 +87,50 @@ return vtable + +def weave_llfields(llfields, classinfo, types_only=False): + llfields_iter = iter(llfields) + result = [] + for mangled_name, name, value in classinfo: + if name is not None: + curfield = llfields_iter.next() + while curfield is not None and curfield[0] != mangled_name: # XXX does this make sense? + result.append((mangled_name, value.get_func())) + try: + curfield = llfields_iter.next() + except StopIteration: + curfield = None + if curfield is not None: + result.append(curfield) + else: + if not types_only: + value = typeOf(value) + result.append((mangled_name, value)) + try: + llfields_iter.next() + except StopIteration: + pass + else: + raise TyperError("Incorrect set of llfields") + return result + +def extend_exportinfo(exportinfo, mangled_name, name, r, classdesc=None): + T = r.lowleveltype + if classdesc is None: + llvalue = T + var = None + else: + var = classdesc.read_attribute(name, None) + llvalue = r.convert_desc_or_const(var) + #import pdb; pdb.set_trace() + if var is None or not getattr(var.value, '_export_', None): + name = None # hide + val = llvalue + else: + val = var.value + exportinfo.append((mangled_name, name, val)) + + class ClassRepr(AbstractClassRepr): def __init__(self, rtyper, classdef): AbstractClassRepr.__init__(self, rtyper, classdef) @@ -97,6 +142,7 @@ self.lowleveltype = Ptr(self.vtable_type) def _setup_repr(self): + exportinfo = None # NOTE: don't store mutable objects like the dicts below on 'self' # before they are fully built, to avoid strange bugs in case # of recursion where other code would uses these @@ -105,6 +151,9 @@ pbcfields = {} allmethods = {} if self.classdef is not None: + if self.classdef.is_exported(): + exportinfo = [] + importinfo = self.classdef.get_import_data() # class attributes llfields = [] attrs = self.classdef.attrs.items() @@ -117,9 +166,13 @@ allmethods[name] = True s_value = s_unboundmethod r = self.rtyper.getrepr(s_value) + #if "Multi" in repr(r) or ("Void" in repr(r.lowleveltype) and name in ("foo", "bar") and "foovoid" not in repr(self)): + # import pdb; pdb.set_trace() mangled_name = 'cls_' + name clsfields[name] = mangled_name, r llfields.append((mangled_name, r.lowleveltype)) + if exportinfo is not None and r.lowleveltype is not Void: + extend_exportinfo(exportinfo, mangled_name, name, r, self.classdef.classdesc) # attributes showing up in getattrs done on the class as a PBC extra_access_sets = self.rtyper.class_pbc_attributes.get( self.classdef, {}) @@ -128,7 +181,12 @@ mangled_name = mangle('pbc%d' % counter, attr) pbcfields[access_set, attr] = mangled_name, r llfields.append((mangled_name, r.lowleveltype)) + if exportinfo is not None: + assert 0, "not yet supported" # + if importinfo: + #import pdb; pdb.set_trace() + llfields = weave_llfields(llfields, importinfo.cls) self.rbase = getclassrepr(self.rtyper, self.classdef.basedef) self.rbase.setup() kwds = {'hints': {'immutable': True}} @@ -136,6 +194,8 @@ ('super', self.rbase.vtable_type), *llfields, **kwds) self.vtable_type.become(vtable_type) + if exportinfo is not None: + self.classdef.exportinfo_cls = exportinfo allmethods.update(self.rbase.allmethods) self.clsfields = clsfields self.pbcfields = pbcfields @@ -159,8 +219,20 @@ def getvtable(self, cast_to_typeptr=True): """Return a ptr to the vtable of this type.""" if self.vtable is None: - self.vtable = malloc(self.vtable_type, immortal=True) - self.setup_vtable(self.vtable, self) + if self.classdef and self.classdef.get_import_data(): + #from pypy.rpython.lltypesystem.rffi import CConstant + handle = self.classdef.get_import_data().vtablename + component = None + if isinstance(handle, tuple): + component, handle = handle + #T = lltype.Ptr(lltype.OpaqueType(handle, hints=dict(external_void=True))) + #val = CConstant("&" + handle, T) + self.vtable = lltype.externalptr(OBJECT_VTABLE, handle + "_vtable", component) + #self.vtable = val + cast_to_typeptr = False + else: + self.vtable = malloc(self.vtable_type, immortal=True) + self.setup_vtable(self.vtable, self) # vtable = self.vtable if cast_to_typeptr: @@ -175,11 +247,18 @@ # initialize the 'subclassrange_*' and 'name' fields if rsubcls.classdef is not None: #vtable.parenttypeptr = rsubcls.rbase.getvtable() - vtable.subclassrange_min = rsubcls.classdef.minid - vtable.subclassrange_max = rsubcls.classdef.maxid + vtable.level = rsubcls.classdef.level + vtable.classrow = malloc(Array(CLASSTYPE), rsubcls.classdef.level + 1, immortal=True) + cur_repr = rsubcls + for i in xrange(rsubcls.classdef.level, -1, -1): + vtable.classrow[i] = cur_repr.getvtable() + cur_repr = cur_repr.rbase + + if rsubcls.classdef.is_exported(): + rsubcls.classdef.exportinfo_vtableval = normalizeptr(vtable) else: #for the root class - vtable.subclassrange_min = 0 - vtable.subclassrange_max = sys.maxint + vtable.level = -1 + vtable.classrow = nullptr(Array(CLASSTYPE)) rinstance = getinstancerepr(self.rtyper, rsubcls.classdef) rinstance.setup() if rinstance.gcflavor == 'gc': @@ -198,6 +277,7 @@ #else: the classdef was created recently, so no instantiate() # could reach it else: + importinfo = self.classdef.get_import_data() # setup class attributes: for each attribute name at the level # of 'self', look up its value in the subclass rsubcls def assign(mangled_name, value): @@ -206,7 +286,7 @@ llvalue = r.convert_desc_or_const(value) setattr(vtable, mangled_name, llvalue) - mro = list(rsubcls.classdef.getmro()) + mro = list(rsubcls.classdef.getmro()) # XXX not needed? for fldname in self.clsfields: mangled_name, r = self.clsfields[fldname] if r.lowleveltype is Void: @@ -223,6 +303,10 @@ attrvalue = rsubcls.classdef.classdesc.read_attribute(attr, None) if attrvalue is not None: assign(mangled_name, attrvalue) + if importinfo: + for mangled_name, name, val in importinfo.cls: + if not name: + setattr(vtable, mangled_name, val) # then initialize the 'super' portion of the vtable self.rbase.setup_vtable(vtable.super, rsubcls) @@ -282,10 +366,8 @@ ## # a class with no subclass ## return hop.genop('ptr_eq', [v_cls1, v_cls2], resulttype=Bool) ## else: - minid = hop.inputconst(Signed, cls2.subclassrange_min) - maxid = hop.inputconst(Signed, cls2.subclassrange_max) - return hop.gendirectcall(ll_issubclass_const, v_cls1, minid, - maxid) + level = hop.inputconst(Signed, cls2.level) + return hop.gendirectcall(ll_issubclass_const, v_cls1, v_cls2, level) else: v_cls1, v_cls2 = hop.inputargs(class_repr, class_repr) return hop.gendirectcall(ll_issubclass, v_cls1, v_cls2) @@ -307,6 +389,7 @@ self.gcflavor = gcflavor def _setup_repr(self, llfields=None, hints=None, adtmeths=None): + exportinfo = None # NOTE: don't store mutable objects like the dicts below on 'self' # before they are fully built, to avoid strange bugs in case # of recursion where other code would uses these @@ -317,6 +400,8 @@ if self.classdef is None: fields['__class__'] = 'typeptr', get_type_repr(self.rtyper) else: + if self.classdef.is_exported(): + exportinfo = [] # instance attributes if llfields is None: llfields = [] @@ -328,12 +413,18 @@ mangled_name = 'inst_' + name fields[name] = mangled_name, r llfields.append((mangled_name, r.lowleveltype)) + if exportinfo is not None: + extend_exportinfo(exportinfo, mangled_name, name, r, None) + # # hash() support if self.rtyper.needs_hash_support(self.classdef): from pypy.rpython import rint fields['_hash_cache_'] = 'hash_cache', rint.signed_repr llfields.append(('hash_cache', Signed)) + importinfo = self.classdef.get_import_data() + if importinfo: + llfields = weave_llfields(llfields, importinfo.inst, True) self.rbase = getinstancerepr(self.rtyper, self.classdef.basedef, self.gcflavor) @@ -347,6 +438,11 @@ if '_immutable_' in self.classdef.classdesc.classdict: hints = hints.copy() hints['immutable'] = True + if self.classdef.is_exported(): + hints = hints.copy() + hints['_exported'] = True + if exportinfo is not None: + self.classdef.exportinfo_inst = exportinfo object_type = MkStruct(self.classdef.name, ('super', self.rbase.object_type), hints=hints, @@ -584,16 +680,15 @@ instance_repr = self.common_repr() v_obj, v_cls = hop.inputargs(instance_repr, class_repr) - if isinstance(v_cls, Constant): + if False and isinstance(v_cls, Constant): # XXX cls = v_cls.value # XXX re-implement the following optimization #if cls.subclassrange_max == cls.subclassrange_min: # # a class with no subclass # return hop.gendirectcall(rclass.ll_isinstance_exact, v_obj, v_cls) #else: - minid = hop.inputconst(Signed, cls.subclassrange_min) - maxid = hop.inputconst(Signed, cls.subclassrange_max) - return hop.gendirectcall(ll_isinstance_const, v_obj, minid, maxid) + level = hop.inputconst(Signed, cls.level) + return hop.gendirectcall(ll_isinstance_const, v_obj, v_cls, level) else: return hop.gendirectcall(ll_isinstance, v_obj, v_cls) @@ -679,11 +774,10 @@ return cast_pointer(OBJECTPTR, obj).typeptr def ll_issubclass(subcls, cls): - return cls.subclassrange_min <= subcls.subclassrange_min <= cls.subclassrange_max - -def ll_issubclass_const(subcls, minid, maxid): - return minid <= subcls.subclassrange_min <= maxid + return subcls.level >= cls.level and subcls.classrow[cls.level] == cls +def ll_issubclass_const(subcls, cls, level): + return subcls.level >= level and subcls.classrow[level] == cls def ll_isinstance(obj, cls): # obj should be cast to OBJECT or NONGCOBJECT if not obj: @@ -691,12 +785,12 @@ obj_cls = obj.typeptr return ll_issubclass(obj_cls, cls) -def ll_isinstance_const(obj, minid, maxid): +def ll_isinstance_const(obj, cls, level): if not obj: return False - return ll_issubclass_const(obj.typeptr, minid, maxid) + return ll_issubclass_const(obj.typeptr, cls, level) -def ll_isinstance_exact(obj, cls): +def ll_isinstance_exact(obj, cls): #unused if not obj: return False obj_cls = obj.typeptr Modified: pypy/branch/sepcomp/pypy/rpython/lltypesystem/rtagged.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/lltypesystem/rtagged.py (original) +++ pypy/branch/sepcomp/pypy/rpython/lltypesystem/rtagged.py Wed Dec 9 12:52:52 2009 @@ -129,10 +129,9 @@ cls = v_cls.value answer = self.unboxedclassdef.issubclass(classdef) c_answer_if_unboxed = hop.inputconst(lltype.Bool, answer) - minid = hop.inputconst(lltype.Signed, cls.subclassrange_min) - maxid = hop.inputconst(lltype.Signed, cls.subclassrange_max) - return hop.gendirectcall(ll_unboxed_isinstance_const, v_obj, - minid, maxid, c_answer_if_unboxed) + level = hop.inputconst(lltype.Signed, cls.level) + return hop.gendirectcall(ll_unboxed_isinstance_const, v_obj, v_cls, + level, c_answer_if_unboxed) def ll_int_to_unboxed(PTRTYPE, value): @@ -147,10 +146,10 @@ else: return instance.typeptr -def ll_unboxed_isinstance_const(obj, minid, maxid, answer_if_unboxed): +def ll_unboxed_isinstance_const(obj, cls, level, answer_if_unboxed): if not obj: return False if lltype.cast_ptr_to_int(obj) & 1: return answer_if_unboxed else: - return ll_issubclass_const(obj.typeptr, minid, maxid) + return ll_issubclass_const(obj.typeptr, cls, level) Modified: pypy/branch/sepcomp/pypy/rpython/normalizecalls.py ============================================================================== --- pypy/branch/sepcomp/pypy/rpython/normalizecalls.py (original) +++ pypy/branch/sepcomp/pypy/rpython/normalizecalls.py Wed Dec 9 12:52:52 2009 @@ -331,6 +331,14 @@ classdef.minid = TotalOrderSymbolic(witness, lst) classdef.maxid = TotalOrderSymbolic(witness + [MAX], lst) + +def assign_inheritance_levels(annotator): + bk = annotator.bookkeeper + for classdef in bk.classdefs: + if not hasattr(classdef, 'level'): + classdef.level = len(tuple(classdef.getmro())) - 1 + + MAX = 1E100 _cdef_id_counter = 0 def get_unique_cdef_id(cdef): @@ -350,7 +358,7 @@ try: normalize_call_familes(rtyper.annotator) merge_classpbc_getattr_into_classdef(rtyper) - assign_inheritance_ids(rtyper.annotator) + assign_inheritance_levels(rtyper.annotator) finally: rtyper.annotator.frozen -= 1 create_instantiate_functions(rtyper.annotator) From xoraxax at codespeak.net Wed Dec 9 12:55:30 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 12:55:30 +0100 (CET) Subject: [pypy-svn] r70017 - pypy/branch/sepcomp/pypy/interfaces Message-ID: <20091209115530.A6A03168051@codespeak.net> Author: xoraxax Date: Wed Dec 9 12:55:30 2009 New Revision: 70017 Added: pypy/branch/sepcomp/pypy/interfaces/ Log: Add empty interfaces directory. From xoraxax at codespeak.net Wed Dec 9 13:10:34 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 13:10:34 +0100 (CET) Subject: [pypy-svn] r70018 - pypy/branch/sepcomp/pypy/translator/cli/test Message-ID: <20091209121034.E1EAB168051@codespeak.net> Author: xoraxax Date: Wed Dec 9 13:10:33 2009 New Revision: 70018 Modified: pypy/branch/sepcomp/pypy/translator/cli/test/test_carbonpython.py Log: Remove skip, test does not fail for me. Modified: pypy/branch/sepcomp/pypy/translator/cli/test/test_carbonpython.py ============================================================================== --- pypy/branch/sepcomp/pypy/translator/cli/test/test_carbonpython.py (original) +++ pypy/branch/sepcomp/pypy/translator/cli/test/test_carbonpython.py Wed Dec 9 13:10:33 2009 @@ -106,7 +106,7 @@ assert entrypoints[1][1] == (MyClass, int) # foo inputtypes def test_compile_class(self): - py.test.skip('This test fails every other day. No clue why :-(') + #py.test.skip('This test fails every other day. No clue why :-(') class MyClass: @export(int) def __init__(self, x): From xoraxax at codespeak.net Wed Dec 9 13:11:57 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 13:11:57 +0100 (CET) Subject: [pypy-svn] r70019 - in pypy/branch/sepcomp/pypy/translator: c c/src llsupport Message-ID: <20091209121157.5FB6B16804B@codespeak.net> Author: xoraxax Date: Wed Dec 9 13:11:56 2009 New Revision: 70019 Modified: pypy/branch/sepcomp/pypy/translator/c/extfunc.py pypy/branch/sepcomp/pypy/translator/c/src/exception.h pypy/branch/sepcomp/pypy/translator/c/src/g_include.h pypy/branch/sepcomp/pypy/translator/c/src/support.h pypy/branch/sepcomp/pypy/translator/llsupport/wrapper.py Log: Add support for PYPY_EXTMODULE and config.translation.generatemodule. Modified: pypy/branch/sepcomp/pypy/translator/c/extfunc.py ============================================================================== --- pypy/branch/sepcomp/pypy/translator/c/extfunc.py (original) +++ pypy/branch/sepcomp/pypy/translator/c/extfunc.py Wed Dec 9 13:11:56 2009 @@ -106,7 +106,7 @@ yield ('RPYTHON_EXCEPTION_MATCH', exceptiondata.fn_exception_match) yield ('RPYTHON_TYPE_OF_EXC_INST', exceptiondata.fn_type_of_exc_inst) yield ('RPYTHON_RAISE_OSERROR', exceptiondata.fn_raise_OSError) - if not db.standalone: + if not db.standalone and not db.translator.config.translation.generatemodule: yield ('RPYTHON_PYEXCCLASS2EXC', exceptiondata.fn_pyexcclass2exc) yield ('RPyExceptionOccurred1', exctransformer.rpyexc_occured_ptr.value) Modified: pypy/branch/sepcomp/pypy/translator/c/src/exception.h ============================================================================== --- pypy/branch/sepcomp/pypy/translator/c/src/exception.h (original) +++ pypy/branch/sepcomp/pypy/translator/c/src/exception.h Wed Dec 9 13:11:56 2009 @@ -2,7 +2,7 @@ /************************************************************/ /*** C header subsection: exceptions ***/ -#if !defined(PYPY_STANDALONE) && !defined(PYPY_NOT_MAIN_FILE) +#if !defined(PYPY_STANDALONE) && !defined(PYPY_NOT_MAIN_FILE) && !defined(PYPY_EXTMODULE) PyObject *RPythonError; #endif @@ -59,7 +59,7 @@ #define RPyRaiseSimpleException(exc, msg) _RPyRaiseSimpleException(R##exc) void _RPyRaiseSimpleException(RPYTHON_EXCEPTION rexc); -#ifndef PYPY_STANDALONE +#if !defined(PYPY_STANDALONE) && !defined(PYPY_EXTMODULE) void RPyConvertExceptionFromCPython(void); void RPyConvertExceptionToCPython(void); #endif @@ -74,7 +74,7 @@ RPyRaiseException(RPYTHON_TYPE_OF_EXC_INST(rexc), rexc); } -#ifndef PYPY_STANDALONE +#if !defined(PYPY_STANDALONE) && !defined(PYPY_EXTMODULE) void RPyConvertExceptionFromCPython(void) { /* convert the CPython exception to an RPython one */ @@ -115,7 +115,7 @@ PyErr_NormalizeException(&pycls, &v, &tb); PyErr_Restore(pycls, v, tb); } -#endif /* !PYPY_STANDALONE */ +#endif /* !PYPY_STANDALONE && !PYPY_EXTMODULE */ #endif /* PYPY_NOT_MAIN_FILE */ Modified: pypy/branch/sepcomp/pypy/translator/c/src/g_include.h ============================================================================== --- pypy/branch/sepcomp/pypy/translator/c/src/g_include.h (original) +++ pypy/branch/sepcomp/pypy/translator/c/src/g_include.h Wed Dec 9 13:11:56 2009 @@ -2,7 +2,7 @@ /************************************************************/ /*** C header file for code produced by genc.py ***/ -#ifndef PYPY_STANDALONE +#if !defined(PYPY_STANDALONE) && !defined(PYPY_EXTMODULE) # include "Python.h" # include "compile.h" # include "frameobject.h" @@ -23,7 +23,7 @@ #define PY_LONG_LONG long long #endif -#ifndef PYPY_STANDALONE +#if !defined(PYPY_STANDALONE) && !defined(PYPY_EXTMODULE) # include "src/pyobj.h" #endif Modified: pypy/branch/sepcomp/pypy/translator/c/src/support.h ============================================================================== --- pypy/branch/sepcomp/pypy/translator/c/src/support.h (original) +++ pypy/branch/sepcomp/pypy/translator/c/src/support.h Wed Dec 9 13:11:56 2009 @@ -99,7 +99,7 @@ # define RPyBareItem(array, index) ((array)[index]) #endif -#ifndef PYPY_STANDALONE +#if !defined(PYPY_STANDALONE) && !defined(PYPY_EXTMODULE) /* prototypes */ Modified: pypy/branch/sepcomp/pypy/translator/llsupport/wrapper.py ============================================================================== --- pypy/branch/sepcomp/pypy/translator/llsupport/wrapper.py (original) +++ pypy/branch/sepcomp/pypy/translator/llsupport/wrapper.py Wed Dec 9 13:11:56 2009 @@ -19,6 +19,9 @@ graph = bk.getdesc(func).getuniquegraph() f = getfunctionptr(graph) + if translator.config.translation.generatemodule: + return f + FUNCTYPE = typeOf(f).TO newops = LowLevelOpList(translator.rtyper) @@ -74,5 +77,6 @@ PyObjPtr), wgraph.name, graph = wgraph, - exception_policy = "CPython") + exception_policy = "CPython", + _exported=True) From xoraxax at codespeak.net Wed Dec 9 13:12:41 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 13:12:41 +0100 (CET) Subject: [pypy-svn] r70020 - in pypy/branch/sepcomp/pypy/translator/c: . test Message-ID: <20091209121241.F3A3F168051@codespeak.net> Author: xoraxax Date: Wed Dec 9 13:12:41 2009 New Revision: 70020 Modified: pypy/branch/sepcomp/pypy/translator/c/database.py pypy/branch/sepcomp/pypy/translator/c/gc.py pypy/branch/sepcomp/pypy/translator/c/node.py pypy/branch/sepcomp/pypy/translator/c/support.py pypy/branch/sepcomp/pypy/translator/c/test/test_database.py Log: Prefix names by package name. Support for external_void, _external_reference, containers with external linkage. Add linkage information to generated C code. Modified: pypy/branch/sepcomp/pypy/translator/c/database.py ============================================================================== --- pypy/branch/sepcomp/pypy/translator/c/database.py (original) +++ pypy/branch/sepcomp/pypy/translator/c/database.py Wed Dec 9 13:12:41 2009 @@ -52,7 +52,11 @@ # assign to a constant object is something C doesn't think is # constant self.late_initializations = [] - self.namespace = CNameManager() + pkgname = "pypy" + if translator and translator.config.translation.exportpackage: + pkgname = translator.config.translation.exportpackage + self.namespace = CNameManager(pkgname + "_") + self.from_other_namespace = lambda ns, name, **kwargs: CNameManager(ns + "_").uniquename(name, **kwargs) if translator is None or translator.rtyper is None: self.exctransformer = None @@ -83,7 +87,7 @@ node = BareBoneArrayDefNode(self, T, varlength) else: node = ArrayDefNode(self, T, varlength) - elif isinstance(T, OpaqueType) and T.hints.get("render_structure", False): + elif isinstance(T, OpaqueType) and (T.hints.get("render_structure", False) or T.hints.get("external_void", False)): node = ExtTypeOpaqueDefNode(self, T) elif T == WeakRef: REALT = self.gcpolicy.get_real_weakref_type() @@ -130,6 +134,8 @@ elif isinstance(T, OpaqueType): if T == RuntimeTypeInfo: return self.gcpolicy.rtti_type() + elif T.hints.get("external_void", False): + return 'void @' elif T.hints.get("render_structure", False): node = self.gettypedefnode(T, varlength=varlength) if who_asks is not None: @@ -148,6 +154,8 @@ try: node = self.containernodes[container] except KeyError: + if isinstance(container, lltype._external_reference): + buildkwds.update(dict(external_name=container.name, external_component=container.component)) T = typeOf(container) if isinstance(T, (lltype.Array, lltype.Struct)): if hasattr(self.gctransformer, 'consider_constant'): Modified: pypy/branch/sepcomp/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/sepcomp/pypy/translator/c/gc.py (original) +++ pypy/branch/sepcomp/pypy/translator/c/gc.py Wed Dec 9 13:12:41 2009 @@ -253,7 +253,7 @@ return [] def implementation(self): - yield 'char %s /* uninitialized */;' % self.name + yield '%schar %s /* uninitialized */;' % (self.linkage(), self.name) class FrameworkGcRuntimeTypeInfo_OpaqueNode(BoehmGcRuntimeTypeInfo_OpaqueNode): nodekind = 'framework rtti' Modified: pypy/branch/sepcomp/pypy/translator/c/node.py ============================================================================== --- pypy/branch/sepcomp/pypy/translator/c/node.py (original) +++ pypy/branch/sepcomp/pypy/translator/c/node.py Wed Dec 9 13:12:41 2009 @@ -440,6 +440,9 @@ self.T = T self.dependencies = {} self.name = 'RPyOpaque_%s' % (T.tag,) + if T.hints.get("external_void", False): + self.typetag = "extern void*" + self.name = T.tag def setup(self): pass @@ -447,23 +450,34 @@ def definition(self): return [] + # ____________________________________________________________ class ContainerNode(object): if USESLOTS: - __slots__ = """db T obj + __slots__ = """db T obj inhibit_dependencies typename implementationtypename name ptrname - globalcontainer""".split() + globalcontainer external_name""".split() - def __init__(self, db, T, obj): + def __init__(self, db, T, obj, external_name=None, external_component=None): self.db = db self.T = T self.obj = obj - #self.dependencies = {} + self.external_name = external_name self.typename = db.gettype(T) #, who_asks=self) self.implementationtypename = db.gettype(T, varlength=self.getlength()) + self.inhibit_dependencies = False + if external_name: + self.inhibit_dependencies = True + self.globalcontainer = True + if not external_component: + self.name = external_name + else: + self.name = db.from_other_namespace(external_component, "g_" + external_name) + self.ptrname = '(&%s)' % (self.name, ) + return parent, parentindex = parentlink(obj) if parent is None: self.name = db.namespace.uniquename('g_' + self.basename()) @@ -483,13 +497,25 @@ return hasattr(self.T, "_hints") and self.T._hints.get('thread_local') def forward_declaration(self): - yield '%s;' % ( + yield '%s%s;' % (self.linkage(), forward_cdecl(self.implementationtypename, self.name, self.db.standalone, self.is_thread_local())) + def linkage(self): + linkage = "static " + if getattr(self.obj, "_exported", False): + linkage = "" + if getattr(self, "external_name", None): + linkage = "extern " + return linkage + + def implementation(self): + if self.external_name: + return [] + lines = list(self.initializationexpr()) - lines[0] = '%s = %s' % ( + lines[0] = '%s%s = %s' % (self.linkage(), cdecl(self.implementationtypename, self.name, self.is_thread_local()), lines[0]) lines[-1] += ';' @@ -512,6 +538,8 @@ return self.T._name def enum_dependencies(self): + if self.inhibit_dependencies: + return for name in self.T._names: yield getattr(self.obj, name) @@ -558,13 +586,14 @@ assert not USESLOTS or '__dict__' not in dir(StructNode) + class ArrayNode(ContainerNode): nodekind = 'array' if USESLOTS: __slots__ = () - def __init__(self, db, T, obj): - ContainerNode.__init__(self, db, T, obj) + def __init__(self, db, T, obj, **kwargs): + ContainerNode.__init__(self, db, T, obj, **kwargs) if barebonearray(T): self.ptrname = self.name @@ -572,6 +601,8 @@ return 'array' def enum_dependencies(self): + if self.inhibit_dependencies: + return return self.obj.items def getlength(self): @@ -630,6 +661,8 @@ return self.T._name def enum_dependencies(self): + if self.inhibit_dependencies: + return for i in range(self.obj.getlength()): yield self.obj.getitem(i) @@ -690,10 +723,17 @@ self.T = T self.obj = obj if getattr(obj, 'external', None) == 'C' and not db.need_sandboxing(obj): - self.name = forcename or self.basename() + comp = getattr(obj, 'component', None) + if comp: + self.name = db.from_other_namespace(comp, self.basename()) + else: + self.name = forcename or self.basename() else: - self.name = (forcename or - db.namespace.uniquename('g_' + self.basename())) + if getattr(obj, '_extname', None): + self.name = db.namespace.uniquename(obj._extname) + else: + self.name = (forcename or + db.namespace.uniquename('g_' + self.basename())) self.compilation_info = getattr(obj, 'compilation_info', ExternalCompilationInfo()) self.make_funcgens() @@ -717,7 +757,7 @@ def forward_declaration(self): for funcgen in self.funcgens: - yield '%s;' % ( + yield '%s%s;' % (self.linkage(), forward_cdecl(self.implementationtypename, funcgen.name(self.name), self.db.standalone)) @@ -736,7 +776,7 @@ # recompute implementationtypename as the argnames may have changed argnames = funcgen.argnames() implementationtypename = self.db.gettype(self.T, argnames=argnames) - yield '%s {' % cdecl(implementationtypename, funcgen.name(self.name)) + yield '%s%s {' % (self.linkage(), cdecl(implementationtypename, funcgen.name(self.name))) # # declare the local variables # Modified: pypy/branch/sepcomp/pypy/translator/c/support.py ============================================================================== --- pypy/branch/sepcomp/pypy/translator/c/support.py (original) +++ pypy/branch/sepcomp/pypy/translator/c/support.py Wed Dec 9 13:12:41 2009 @@ -38,7 +38,7 @@ __thread = "__thread " cdecl_str = __thread + cdecl(ctype, cname) - if standalone: + if standalone and False: # XXX why extern!? return 'extern ' + cdecl_str else: return cdecl_str Modified: pypy/branch/sepcomp/pypy/translator/c/test/test_database.py ============================================================================== --- pypy/branch/sepcomp/pypy/translator/c/test/test_database.py (original) +++ pypy/branch/sepcomp/pypy/translator/c/test/test_database.py Wed Dec 9 13:12:41 2009 @@ -1,5 +1,6 @@ import autopath, sys from pypy.rpython.lltypesystem.lltype import * +from pypy.rpython.lltypesystem import rffi from pypy.translator.translator import TranslationContext from pypy.translator.c.database import LowLevelDatabase from pypy.objspace.flow.model import Constant, Variable, SpaceOperation @@ -13,6 +14,13 @@ print '/*********************************/' structdeflist = database.getstructdeflist() for node in structdeflist: + if hasattr(node, 'forward_decl'): + if node.forward_decl: + print node.forward_decl + else: + print '%s %s;' % (node.typetag, node.name) + + for node in structdeflist: for line in node.definition(): print line print @@ -175,6 +183,20 @@ db.complete() dump_on_stdout(db) +def test_extdef(): + db = LowLevelDatabase() + handle = "handle" + S2 = GcStruct("testing2") + val = externalptr(S2, "foo") + #typ = Ptr(OpaqueType(handle, hints=dict(external_void=True))) + #val = rffi.CConstant(handle, typ) + S = GcStruct('testing', ('x', Ptr(S2)), ('y', Signed)) + s = malloc(S) + s.x = val + s.y = 1 + db.get(s) + db.complete() + dump_on_stdout(db) def test_malloc(): S = GcStruct('testing', ('x', Signed), ('y', Signed)) From xoraxax at codespeak.net Wed Dec 9 13:13:27 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 13:13:27 +0100 (CET) Subject: [pypy-svn] r70021 - in pypy/branch/sepcomp/pypy/translator: . c Message-ID: <20091209121327.70A76168051@codespeak.net> Author: xoraxax Date: Wed Dec 9 13:13:26 2009 New Revision: 70021 Modified: pypy/branch/sepcomp/pypy/translator/c/genc.py pypy/branch/sepcomp/pypy/translator/driver.py Log: Add support for secondary entrypoints and separate compilation. Write one file instead of multiple to ease linking (should be optimized). Modified: pypy/branch/sepcomp/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/sepcomp/pypy/translator/c/genc.py (original) +++ pypy/branch/sepcomp/pypy/translator/c/genc.py Wed Dec 9 13:13:26 2009 @@ -23,7 +23,7 @@ _compiled = False modulename = None - def __init__(self, translator, entrypoint, config, gcpolicy=None): + def __init__(self, translator, entrypoint, config, gcpolicy=None, secondary_entrypoints=()): self.translator = translator self.entrypoint = entrypoint self.entrypoint_name = self.entrypoint.func_name @@ -33,6 +33,7 @@ if gcpolicy is not None and gcpolicy.requires_stackless: config.translation.stackless = True self.eci = ExternalCompilationInfo() + self.secondary_entrypoints = secondary_entrypoints def build_database(self): translator = self.translator @@ -69,6 +70,11 @@ # build entrypoint and eventually other things to expose pf = self.getentrypointptr() pfname = db.get(pf) + + for func, _ in self.secondary_entrypoints: + bk = translator.annotator.bookkeeper + db.get(getfunctionptr(bk.getdesc(func).getuniquegraph())) + self.c_entrypoint_name = pfname db.complete() @@ -83,10 +89,14 @@ libraries=db.gcpolicy.gc_libraries())) all = [] + exported_names = [] for node in self.db.globalcontainers(): eci = getattr(node, 'compilation_info', None) if eci: all.append(eci) + if getattr(getattr(node, "obj", None), "export", None): # XXX this should be handled in the db + exported_names.append(node.name) + all.append(ExternalCompilationInfo(export_symbols=tuple(exported_names))) self.eci = self.eci.merge(*all) def get_gcpolicyclass(self): @@ -225,6 +235,7 @@ entrypoint_ptr = self.getentrypointptr() wrapped_entrypoint_c_name = self.db.get(entrypoint_ptr) + self.so_name = self.c_source_filename.new(ext=so_ext) CODE = """ import ctypes @@ -251,7 +262,7 @@ _rpython_startup = _lib.RPython_StartupCode _rpython_startup() -""" % {'so_name': self.c_source_filename.new(ext=so_ext), +""" % {'so_name': self.so_name, 'c_entrypoint_name': wrapped_entrypoint_c_name, 'nargs': len(lltype.typeOf(entrypoint_ptr).TO.ARGS)} modfile.write(CODE) @@ -492,7 +503,7 @@ othernodes.append(node) # for now, only split for stand-alone programs. if self.database.standalone: - self.one_source_file = False + self.one_source_file = True # XXX False self.funcnodes = funcnodes self.othernodes = othernodes self.path = path @@ -797,6 +808,8 @@ incfilename = targetdir.join('common_header.h') fi = incfilename.open('w') + if database.translator.config.translation.generatemodule: # XXX refactor the 2 src gen funcs + defines["PYPY_EXTMODULE"] = "1" # # Header # Modified: pypy/branch/sepcomp/pypy/translator/driver.py ============================================================================== --- pypy/branch/sepcomp/pypy/translator/driver.py (original) +++ pypy/branch/sepcomp/pypy/translator/driver.py Wed Dec 9 13:13:26 2009 @@ -4,12 +4,15 @@ from pypy.translator.tool.taskengine import SimpleTaskEngine from pypy.translator.goal import query from pypy.translator.goal.timing import Timer +from pypy.translator.sepcomp import ImportExportComponent, get_function_name from pypy.annotation import model as annmodel from pypy.annotation.listdef import s_list_of_strings +from pypy.annotation.signature import annotationoftype from pypy.annotation import policy as annpolicy from py.compat import optparse from pypy.tool.udir import udir + import py from pypy.tool.ansi_print import ansi_log log = py.log.Producer("translation") @@ -215,12 +218,14 @@ self.entry_point = entry_point self.translator = translator self.libdef = None - + self.secondary_entrypoints = () self.translator.driver_instrument_result = self.instrument_result def setup_library(self, libdef, policy=None, extra={}, empty_translator=None): + """ Used by carbonpython only. """ self.setup(None, None, policy, extra, empty_translator) self.libdef = libdef + self.secondary_entrypoints = libdef.functions def instrument_result(self, args): backend, ts = self.get_backend_and_type_system() @@ -302,22 +307,35 @@ annmodel.DEBUG = self.config.translation.debug annotator = translator.buildannotator(policy=policy) + if self.config.translation.exportpackage: + iep = ImportExportComponent.packages[self.config.translation.exportpackage] + self.secondary_entrypoints = iep.entry_points + if self.entry_point: s = annotator.build_types(self.entry_point, self.inputtypes) - - self.sanity_check_annotation() - if self.standalone and s.knowntype != int: - raise Exception("stand-alone program entry point must return an " - "int (and not, e.g., None or always raise an " - "exception).") - annotator.simplify() - return s else: - assert self.libdef is not None - for func, inputtypes in self.libdef.functions: - annotator.build_types(func, inputtypes) - self.sanity_check_annotation() - annotator.simplify() + s = None + + if self.secondary_entrypoints is not None: + for func, inputtypes in self.secondary_entrypoints: + if inputtypes == Ellipsis: + continue + rettype = annotator.build_types(func, inputtypes) + + self.sanity_check_annotation() + if self.entry_point and self.standalone and s.knowntype != int: + raise Exception("stand-alone program entry point must return an " + "int (and not, e.g., None or always raise an " + "exception).") + annotator.simplify() + + # XXX only supports a single set of exports per compilation run + generating_package = self.config.translation.exportpackage + if generating_package: + export_table = ImportExportComponent.packages[generating_package].new_table() + export_table.load_entrypoint_annotations(self.translator, self.secondary_entrypoints) + + return s # task_annotate = taskdef(task_annotate, [], "Annotating&simplifying") @@ -484,11 +502,24 @@ else: from pypy.translator.c.genc import CExtModuleBuilder as CBuilder cbuilder = CBuilder(self.translator, self.entry_point, - config=self.config) + config=self.config, secondary_entrypoints=self.secondary_entrypoints) + + # XXX make more general (for all backends) + # XXX only supports a single set of exports per compilation run + generating_package = self.config.translation.exportpackage + if generating_package: + export_table = ImportExportComponent.packages[generating_package].table + export_table.load_entrypoints(self.translator, self.secondary_entrypoints) + + # XXX messy cbuilder.stackless = self.config.translation.stackless - if not standalone: # xxx more messy + if not standalone: cbuilder.modulename = self.extmod_name + database = cbuilder.build_database() + if generating_package: + export_table.convert_llvalues(database) + ImportExportComponent.save_all() self.log.info("database for generating C source was created") self.cbuilder = cbuilder self.database = database @@ -507,15 +538,20 @@ # task_source_c = taskdef(task_source_c, ['database_c'], "Generating c source") - def create_exe(self): + def create_exe(self, is_so=False): if self.exe_name is not None: import shutil - exename = mkexename(self.c_entryp) - info = {'backend': self.config.translation.backend} + if is_so: + exename = self.c_entryp + else: + exename = mkexename(self.c_entryp) newexename = self.exe_name % self.get_info() + if is_so: + newexename += ".so" if '/' not in newexename and '\\' not in newexename: newexename = './' + newexename - newexename = mkexename(newexename) + if not is_so: + newexename = mkexename(newexename) shutil.copy(exename, newexename) self.c_entryp = newexename self.log.info("created: %s" % (self.c_entryp,)) @@ -528,7 +564,12 @@ self.c_entryp = cbuilder.executable_name self.create_exe() else: - self.c_entryp = cbuilder.get_entry_point() + if self.config.translation.generatemodule: + cbuilder._make_wrapper_module() + self.c_entryp = str(cbuilder.so_name) + self.create_exe(True) + else: + self.c_entryp = cbuilder.get_entry_point() # task_compile_c = taskdef(task_compile_c, ['source_c'], "Compiling c source") From xoraxax at codespeak.net Wed Dec 9 13:13:43 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 13:13:43 +0100 (CET) Subject: [pypy-svn] r70022 - pypy/branch/sepcomp/pypy/translator/goal Message-ID: <20091209121343.DB39B168051@codespeak.net> Author: xoraxax Date: Wed Dec 9 13:13:42 2009 New Revision: 70022 Added: pypy/branch/sepcomp/pypy/translator/goal/md5module.py pypy/branch/sepcomp/pypy/translator/goal/target_md5demo.py Log: Add demo module for sepcomp. Added: pypy/branch/sepcomp/pypy/translator/goal/md5module.py ============================================================================== --- (empty file) +++ pypy/branch/sepcomp/pypy/translator/goal/md5module.py Wed Dec 9 13:13:42 2009 @@ -0,0 +1,22 @@ +from pypy.translator.sepcomp import scimport, export, ImportExportComponent +from pypy.module.md5demo.interface import MD5Interface + + +class MD5Inherited(MD5Interface): + def __init__(self): + self.argument_cache = [] + + def update(self, x): + # keeps all strings that were fed into this hash object + self.argument_cache.append(x) + return MD5Interface.update(self, x) + + def digest(self): + return MD5Interface.digest(self) + "POSTF3" + + + at export(ret=MD5Interface, force_name="module_init") +def module_init(): + return MD5Inherited() + +ImportExportComponent("md5module", locals()) Added: pypy/branch/sepcomp/pypy/translator/goal/target_md5demo.py ============================================================================== --- (empty file) +++ pypy/branch/sepcomp/pypy/translator/goal/target_md5demo.py Wed Dec 9 13:13:42 2009 @@ -0,0 +1,5 @@ +from pypy.translator.goal import md5module + +def target(*args): + return lambda:None, () + From xoraxax at codespeak.net Wed Dec 9 13:13:55 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 13:13:55 +0100 (CET) Subject: [pypy-svn] r70023 - in pypy/branch/sepcomp/pypy/translator: . c/test Message-ID: <20091209121355.87716168051@codespeak.net> Author: xoraxax Date: Wed Dec 9 13:13:55 2009 New Revision: 70023 Added: pypy/branch/sepcomp/pypy/translator/c/test/test_separate.py pypy/branch/sepcomp/pypy/translator/sepcomp.py Log: Add separate compilation module and tests. Added: pypy/branch/sepcomp/pypy/translator/c/test/test_separate.py ============================================================================== --- (empty file) +++ pypy/branch/sepcomp/pypy/translator/c/test/test_separate.py Wed Dec 9 13:13:55 2009 @@ -0,0 +1,410 @@ +import sys +import os + +import autopath +import py +from pypy import conftest + +from pypy.rlib.libffi import CDLL, cast_type_to_ffitype, dlsym, dlopen_global_persistent, RTLD_GLOBAL +from pypy.annotation import model as annmodel +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.lltypesystem.lltype import Signed, Void +from pypy.translator.driver import TranslationDriver +from pypy.translator.interactive import Translation +from pypy.translator.sepcomp import ImportExportComponent, ExportTable, export, scimport, get_function_name, scimport + + +def compile(fn, argtypes, gcpolicy="boehm", backendopt=True, + annotatorpolicy=None, standalone=False, iep_name=''): + t = Translation(fn, argtypes, gc=gcpolicy, backend="c", + policy=annotatorpolicy, generatemodule=not standalone, verbose=False, exportpackage=iep_name) + if not backendopt: + t.disable(["backendopt_lltype"]) + t.ensure_setup(standalone=standalone) + t.annotate() + if conftest.option.view: + t.view() + t.source_c() + if conftest.option.view: + t.view() + cbuilder = t.driver.cbuilder + t.compile_c() + return cbuilder + + +def make_main_entrypoint(module_main_function_name, init_func=lambda:None): + @export(package="", force_name=module_main_function_name, ret=int) + def dummy(): + return 99 + main_func = scimport(dummy, dynamic=True, forward_ref=True) + def f_main(argv): + init_func() # so we can force calls to the constructor + so_filename = argv[1] + dlopen_global_persistent(so_filename) + print main_func() + return 0 + return f_main + + +class TestSeparateCompilation(object): + def setup_method(self, meth): + assert not getattr(self, 'packages_to_teardown', []) + self.packages_to_teardown = [] + + def teardown_method(self, meth): + for package in self.packages_to_teardown: + try: + package.dispose() + except OSError: + print "Could not dispose package information" + self.packages_to_teardown = [] + + def register_iep(self, package): + self.packages_to_teardown.append(package) + + def get_iep(self, *args): + frame = sys._getframe() + iep = ImportExportComponent(frame.f_back.f_code.co_name, *args) + self.register_iep(iep) + return iep + + def test_gather_llinfo(self): + @export(int, package="") + def f(x): + return x + 1 + def entry(): + return 0 + iep = self.get_iep(locals()) + driver = TranslationDriver(overrides={'translation.exportpackage': iep.name}) + driver.setup(entry, []) + driver.proceed(["database_c"]) + assert iep in ImportExportComponent.packages.values() + assert len(iep.entry_points) == 1 + assert len(iep.export_tables) == 1 + assert len(iep.export_tables.values()[0].functions) == 1 + + def test_export_wrong_rettype(self): + @export(int, package="", ret=str) + def f(x): + return x + 1 + iep = self.get_iep(locals()) + f_main = make_main_entrypoint("module_init") + py.test.raises(Exception, "compile(f_main, None, standalone=True, iep_name=iep.name)") + + def test_export_wrong_rettype2(self): + class A: + _package_ = "" + class B(A): + _package_ = "" + @export(A, package="", ret=B) + def f(x): + return x + iep = self.get_iep(locals()) + f_main = make_main_entrypoint("module_init") + py.test.raises(Exception, "compile(f_main, None, standalone=True, iep_name=iep.name)") + + + def test_import_export(self): + @export(int, package="") + def f(x): + return x + 1 + iep = self.get_iep(locals()) + f_main = make_main_entrypoint("module_init") + # compile f()+f_main() into an executable + builder = compile(f_main, None, standalone=True, iep_name=iep.name) + + f_imp = scimport(f, iep) + @export(force_name="module_init", package="") + def g(): # equivalent to an init function of a module + return f_imp(41) + builder2 = compile(g, []) + retdata = builder.cmdexec(str(builder2.so_name)) + print repr(retdata) + # XXX check whether function f_imp is correctly declared + assert int(retdata) == 42 + + def test_shaped_classes(self): + class foo: + _exported_ = True + _inheritable_ = True + _package_ = "" + + def __init__(self, x): + self.x = x + + def internal(self): # this method is internal + return self.x / 2 + + @export(package="") + def bar(self): + return self.x + 42 + + # main part of the program + @export(int, package="") + def create_foo(x): + return foo(x) + iep = self.get_iep(locals()) + f_main = make_main_entrypoint("module_init", lambda: foo(10).bar() ) + builder = compile(f_main, None, standalone=True, iep_name=iep.name) + + # module part + create_foo_imp = scimport(create_foo, iep) + @export(force_name="module_init", package="") + def g(): # equivalent to an init function of a module + return create_foo_imp(13).bar() + builder2 = compile(g, []) + retdata = builder.cmdexec(str(builder2.so_name)) + assert int(retdata) == 13 + 42 + + def test_check_broken_return_type(self): + class foo(object): + _exported_ = True + _package_ = "" + + def __init__(self, x): + self.x = x + + def internal(self): # this method is internal + return self.x / 2 + + @export(package="") + def bar(self): + return self.x + 2 + + @export(package="") + def foo(self): + return 16 + self.x + + class barinternal(foo): + def bar(self): + return self.x + 64 + + @export(foo, package="") + def call_bar_and_foo(x): + return x.bar() + x.foo() + x.internal() + iep = self.get_iep(locals()) + f_main = make_main_entrypoint("module_init", lambda: foo(256).bar() + barinternal(-7).bar()) + builder = compile(f_main, None, standalone=True, iep_name=iep.name) + foo_imported = scimport(foo) + class baz(foo_imported): + def __init__(self, x): + self.y = x + def bar(self): + return self # broken type + + # module part + bar_caller = scimport(call_bar_and_foo, iep) + @export(force_name="module_init") + def g(): # equivalent to an init function of a module + return bar_caller(baz(4096)) + py.test.raises(annmodel.UnionError, "compile(g, [])") + + def test_avoid_move_up(self): + py.test.skip("not fixed yet") + class abstractbase(object): + pass + class foo(abstractbase): + _exported_ = True + _package_ = "" + + def __init__(self, x): + self.x = x + + def internal(self): # this method is internal + return self.x / 2 + + @export(package="") + def bar(self): + return self.x + 2 + + @export(package="") + def foo(self): + return 16 + self.x + + class bar(abstractbase): + def foo(self): + return 20 + + def do_things(x): + return x.foo() + + def f1(x): + if x > 5: + c = foo + else: + c = bar + return do_things(c()) + + iep = self.get_iep(locals()) + builder = compile(f, None, standalone=True, iep_name=iep.name) + + def test_inheriting_classes(self): + class foo(object): + _exported_ = True + _package_ = "" + + def __init__(self, x): + self.x = x + + def internal(self): # this method is internal + return self.x / 2 + + @export(package="", ret=int) + def bar(self): + return int(self.x + 2) + + @export(Ellipsis, package="") + def foo(self, x): + return 16 + self.x + x + + class barinternal(foo): + def internal(self): + return 9 + def bar(self): + return self.x + 64 + + class foo2(foo): + _exported_ = True + _package_ = "" + + @export(foo, package="") + def call_bar_and_foo(x): + return x.bar() + x.foo(5) + x.internal() + @export(int, package="") + def get_internal_instance(x): + if x > 5: + return barinternal(42) + else: + return foo2(21) + iep = self.get_iep(locals()) + f_main = make_main_entrypoint("module_init", lambda: foo(256).foo(13) + barinternal(-7).bar()) + builder = compile(f_main, None, standalone=True, iep_name=iep.name) + foo_imported = scimport(foo) + foo2_imported = scimport(foo2) + class baz(foo_imported): + def __init__(self, x): + self.y = x + def bar(self): + return -self.y + 1024 + self.y * 2 + + # module part + bar_caller = scimport(call_bar_and_foo, iep) + get_internal_instance_imp = scimport(get_internal_instance) + @export(force_name="module_init") + def g(): # equivalent to an init function of a module + b = baz(4096) + i = bar_caller(b) + inst = get_internal_instance_imp(2) + if isinstance(inst, foo_imported): + i += 2**16 + if isinstance(inst, foo2_imported): + i += 2**17 + inst = get_internal_instance_imp(6) + if isinstance(inst, foo_imported): + i += 2**18 + if isinstance(inst, foo2_imported): + i += 2**19 + return i + + builder2 = compile(g, []) + retdata = builder.cmdexec(str(builder2.so_name)) + assert int(retdata) == 5 + 4096 + 1024 + 16 + 0 + 2**16 + 2**17 + 2**18 + + def test_isinstance(self): + class foo(object): + _exported_ = True + _package_ = "" + + def __init__(self, x): + self.x = x + + @export(package="") + def bar(self): + return self.x + 1 + + class barinternal(foo): + def bar(self): + return self.x + 2 + + @export(foo, package="") + def call_bar_and_foo(x): + r = 0 + if isinstance(x, foo): + r += 16 + if isinstance(x, barinternal): + r += 32 + return x.bar() + r + iep = self.get_iep(locals()) + f_main = make_main_entrypoint("module_init", lambda: foo(11).bar()) + builder = compile(f_main, None, standalone=True, iep_name=iep.name) + foo_imported = scimport(foo) + class baz(foo_imported): + def bar(self): + return 4 + + # module part + bar_caller = scimport(call_bar_and_foo, iep) + @export(force_name="module_init") + def g(): # equivalent to an init function of a module + return bar_caller(baz()) + builder2 = compile(g, []) + retdata = builder.cmdexec(str(builder2.so_name)) + assert int(retdata) == 20 + + def test_abstract_classes(self): + py.test.skip("currently not supported") + class foo(object): + _abstract_ = True + _exported_ = True + _package_ = "" + + @export(package="", ret=int) + def bar(self): + raise NotImplementedError + + @export(int, ret=int, package="") + def foo(self, x): + return 16 + + class barinternal(foo): + def __init__(self, x): + self.z = x + def internal(self): + return 9 + def bar(self): + return self.z + 64 + + @export(foo, ret=int, package="") + def call_bar_and_foo(x): + return x.bar() + x.foo(5) + @export(int, ret=foo, package="") + def get_internal_instance(x): + if x > 5: + return barinternal(42) + iep = self.get_iep(locals()) + f_main = make_main_entrypoint("module_init", lambda: barinternal(-7).bar()) + builder = compile(f_main, None, standalone=True, iep_name=iep.name) + foo_imported = scimport(foo) + class baz(foo_imported): + def __init__(self, x): + self.y = x + def bar(self): + return -self.y + 1024 + self.y * 2 + + # module part + bar_caller = scimport(call_bar_and_foo, iep) + get_internal_instance_imp = scimport(get_internal_instance) + @export(force_name="module_init") + def g(): # equivalent to an init function of a module + b = baz(4096) + i = bar_caller(b) + inst = get_internal_instance_imp(6) + if isinstance(inst, foo_imported): + i += 2**16 + return i + + builder2 = compile(g, []) + retdata = builder.cmdexec(str(builder2.so_name)) + assert int(retdata) == 4096 + 1024 + 16 + 2**16 + + Added: pypy/branch/sepcomp/pypy/translator/sepcomp.py ============================================================================== --- (empty file) +++ pypy/branch/sepcomp/pypy/translator/sepcomp.py Wed Dec 9 13:13:55 2009 @@ -0,0 +1,681 @@ +import os +import sys +import random +import pickle +import types + +import pypy +from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.libffi import dlsym, dlopen, RTLD_GLOBAL, RTLD_NOW +from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.rpython.lltypesystem import rffi +from pypy.rpython.lltypesystem.rffi import llexternal, CConstant, VOIDP +from pypy.rlib.objectmodel import Symbolic +from pypy.rpython.lltypesystem import lltype +from pypy.annotation.model import lltype_to_annotation, SomeString, SomeInstance, SomeObject, SomePBC +from pypy.rpython.annlowlevel import llhelper, PseudoHighLevelCallable, PseudoHighLevelCallableEntry, MixLevelHelperAnnotator +from pypy.rpython.typesystem import getfunctionptr +from pypy.annotation.signature import annotationoftype +from pypy.rpython.lltypesystem.lltype import LowLevelType +from pypy.annotation import model as annmodel +from pypy.annotation.bookkeeper import getbookkeeper +from pypy.objspace.flow.model import Constant +from pypy.translator.simplify import get_functype +from pypy.tool.sourcetools import compile_template + + + +class NonInheritable(type): + def __init__(self, clsname, bases, namespace): + for cls in bases: + if isinstance(cls, NonInheritable): + raise TypeError("While creating the class " + clsname + ": " + str(cls) + + " is not suitable as a base class") + super(NonInheritable, self).__init__(clsname, bases, namespace) + + +class ExportTablePickler(pickle.Pickler): + def persistent_id(self, obj): + if obj is type(None): # grrrr + return "NONETYPE" + return None + +class ExportTableUnpickler(pickle.Unpickler): + def persistent_load(self, key): + if key == "NONETYPE": + return type(None) + raise pickle.UnpicklingError, 'Invalid persistent id' + + + +class ExternalClassRegistryImpl(object): + classes_by_ci = {} + def get_class(self, classinfo): + try: + cls = self.classes_by_ci[classinfo.key()] + except KeyError: + self.classes_by_ci[classinfo.key()] = cls = classinfo._build_class() + return cls + +external_class_registry = ExternalClassRegistryImpl() + + +class ClassInfo(object): + def __init__(self, baseinfo, cls, inst, vtablename): + self.cls = cls + self.inst = inst + self.vtablename = vtablename + self.baseinfo = baseinfo + + def key(self): + return self.vtablename + + def __repr__(self): + return "" % (self.vtablename, self.cls, self.inst) + + def convert_llvalues(self, c_db): + for name in ("cls", "inst"): + new = [] + for key, attrname, val in getattr(self, name): + if attrname is None: + if isinstance(val, lltype._ptr): + llvalue = val._obj + assert isinstance(llvalue, lltype._container) + assert hasattr(llvalue, "_exported") + llvalue._exported = True + handle = c_db.get(val) + T = lltype.Ptr(lltype.OpaqueType(handle, hints=dict(external_void=True))) + val = CConstant("&" + handle, T) + elif not "Signed" in repr(val): # XXX + import pdb; pdb.set_trace() + + new.append((key, attrname, val)) + setattr(self, name, new) + + def _build_class(self): + if self.baseinfo is None: + base = object + else: + base = external_class_registry.get_class(self.baseinfo) + clsdict = dict(__slots__=(), _settled_=True, _force_virtual_=True, _importinfo_=self) + for key, attrname, typ in self.cls: + if attrname is None: # anonymous + continue + assert isinstance(typ, FunctionInfo) # XXX lift + extfunc = typ.get_func() + args_string = ", ".join(['arg_%i' % i for i in xrange(len(typ.s_args))]) + # XXX we dont have more of the sig than just the param count + wrap = compile_template("""def wrap_func(%s): return extfunc(%s)""" % (args_string, args_string), "wrap_func") + wrap.is_wrapping = extfunc + sig_helpers = [] + def make_helper(i): + def inputtypecalc(*arg): + extfunc.compute_rebuilt_args_result(getbookkeeper()) + return extfunc.args_s_rebuilt[i] + return inputtypecalc + for i in xrange(len(typ.s_args)): + sig_helpers.append(make_helper(i)) + wrap._annenforceargs_ = tuple(sig_helpers) + wrap._force_virtual_ = True + clsdict[attrname] = wrap + newcls = type(str(self.vtablename), (base,), clsdict) + return newcls + + +class Instance(object): + def __init__(self, classinfo): + self.classinfo = classinfo + def __repr__(self): + return "" % (self.classinfo, ) + + +class FunctionInfo(object): + def __init__(self, s_args, s_result, T_args, T_result): + self.T_args = T_args + self.T_result = T_result + self.s_args = s_args + self.s_result = s_result + self.link_key = None + + def __repr__(self): + return " %r, %r -> %r>" % (self.T_args, self.T_result, self.s_args, self.s_result) + + def get_func(self, dll=False): + smfc = SeparateModuleFunctionCallable(self.link_key, self.T_args, self.T_result, self.s_args, self.s_result, dll) + return smfc + + +class ExportTable(object): + """ A table with information about the exported symbols of a module compiled by pypy.""" + def __init__(self): + self.functions = None + self.classinfos_by_name = {} + + self.methods = set() + self.functions_by_obj = {} + self.classinfos_by_cdef = {} + self.classes_by_ci = {} + self.vtable_vals = {} + self.fi_llvalues = {} + + def __repr__(self): + return "" % (self.functions, self.classinfos_by_cdef, self.classinfos_by_name) + + def convert_llvalues(self, c_database): + for fi in self.functions_by_obj.values(): + dbname = c_database.get(self.fi_llvalues[fi]) + fi.link_key = dbname + for cdef, ci in self.classinfos_by_cdef.items(): + dbname = c_database.getcontainernode(self.vtable_vals[ci]._obj).name + ci.vtablename = (cdef.classdesc.pyobj._component_.name, cdef.name) + ci.convert_llvalues(c_database) + + def load_entrypoint_annotations(self, translator, entrypoints): + ann = translator.annotator + for func, types in entrypoints: + func = unwrap_meth_wrapper(func) + graph = ann.bookkeeper.getdesc(func).getuniquegraph() + inputargs = graph.getargs() + s_list = [] + for var in inputargs: + s_ann = ann.binding(var) + s_ann_exp = s_ann.make_acceptable_in_interface() + if s_ann_exp is None: + raise Exception("Unsuitable parameter type found: %r" % (s_ann, )) + if s_ann_exp != s_ann: + ann.setbinding(var, s_ann_exp) + s_list.append(s_ann_exp) + s_ret = ann.binding(graph.getreturnvar()) + s_ret_exp = s_ret.make_acceptable_in_interface() + if "str" in repr(s_ret): 1/0 + checktype = func._check_ret_type_ + if checktype is not None: + if not isinstance(checktype, annmodel.SomeObject): + checktype = annotationoftype(checktype, ann.bookkeeper) + checktype = checktype.make_acceptable_in_interface() + if s_ret != annmodel.s_ImpossibleValue: + assert checktype == s_ret_exp, "Wrong type!" + + if s_ret != annmodel.s_ImpossibleValue: + if s_ret_exp is None: + raise Exception("Unsuitable return type found: %r" % (s_ret,)) + if s_ret_exp is not s_ret: + ann.setbinding(graph.getreturnvar(), s_ret_exp) + else: + ann.setbinding(graph.getreturnvar(), checktype) + s_ret_exp = checktype + if "classdef=object" in repr(s_ret_exp): 1/0 + self.functions_by_obj[func] = FunctionInfo(s_list, s_ret_exp, None, None) + + def load_entrypoints(self, translator, entrypoints): + rtyper = translator.rtyper + from pypy.rpython.typesystem import getfunctionptr + for func, types in entrypoints: + func = unwrap_meth_wrapper(func) + finfo = self.functions_by_obj[func] + graph = translator.annotator.bookkeeper.getdesc(func).getuniquegraph() + llvalue = getfunctionptr(graph) + FT = llvalue._T + new_args = [] + for i, arg in enumerate(FT.ARGS): + new_args.append(self.opaquify_type(rtyper, arg, finfo.s_args[i])) + ARGS = tuple(new_args) + RESULT = self.opaquify_type(rtyper, FT.RESULT, finfo.s_result) + + finfo.T_args = ARGS + finfo.T_result = RESULT + self.fi_llvalues[finfo] = llvalue + finfo.s_args = [self.externalize_annotation(s_arg) for s_arg in finfo.s_args] + finfo.s_result = self.externalize_annotation(finfo.s_result) + # even if clsdefs dont show up in the external interface of a function, + # they might be exported and need to be available + for clsdef in translator.annotator.getuserclassdefinitions(): + if clsdef.is_exported(False): + self.get_classinfo(clsdef) + + + def lookup_function(self, funcname): + return self.functions[funcname] + + def lookup_class(self, classname): + return self.classinfos_by_name[classname] + + def __getstate__(self): + if self.functions is not None: + return self.__dict__.copy() + self.functions = {} + for func, fi in self.functions_by_obj.items(): + if func in self.methods: # are linked via classinfo + continue + name = get_function_name(func) + assert name not in self.functions + self.functions[name] = fi + odict = self.__dict__.copy() + del odict['classinfos_by_cdef'] + del odict['classes_by_ci'] + del odict['functions_by_obj'] + del odict['methods'] + del odict['vtable_vals'] + del odict['fi_llvalues'] + return odict + + def __setstate__(self, d): + self.__dict__.update(d) + self.classes_by_ci = {} + self.classinfos_by_cdef = None + self.functions_by_obj = None + + def convertexportinfo_to_finfo(self, ei_items): + result = [] + for key, name, typ in ei_items: + if is_exported(typ): + self.methods.add(typ) + typ = self.functions_by_obj[typ] + result.append((key, name, typ)) + return result + + def get_classinfo(self, cdef): + try: + return self.classinfos_by_cdef[cdef] + except KeyError: + if cdef is None: + return None + cdict = cdef.classdesc.classdict + assert '_exported_' in cdict, "Found non-exported class type in interface: %r" % (cdef, ) + obj = cdef.classdesc.pyobj + name = get_class_name(obj) + ci = ClassInfo(self.get_classinfo(cdef.basedef), self.convertexportinfo_to_finfo(cdef.exportinfo_cls), self.convertexportinfo_to_finfo(cdef.exportinfo_inst), None) + self.vtable_vals[ci] = cdef.exportinfo_vtableval + cdef.exportinfo_vtableval._obj._exported = True + self.classinfos_by_cdef[cdef] = ci + assert name not in self.classinfos_by_name + self.classinfos_by_name[name] = ci + return ci + + def opaquify_type(self, rtyper, T, s_ann): + if type(s_ann) in (SomeInstance, ): + return self.make_instance_wrapper(rtyper, s_ann) + if hasattr(T, "TO") and type(s_ann) not in (SomeString, ): + import pdb; pdb.set_trace() + return T + + def make_instance_wrapper(self, rtyper, s_ann): + cdef = s_ann.classdef + importinfo = s_ann.classdef.classdesc.classdict.get('_importinfo_') + if importinfo is not None: + return Instance(importinfo.value) + assert hasattr(cdef, "exportinfo_cls") and hasattr(cdef, "exportinfo_inst") + if rtyper is not None: + rtyper.getrepr(s_ann).get_reusable_prebuilt_instance() + return Instance(self.get_classinfo(s_ann.classdef)) + + def externalize_annotation(self, s_thing): + assert s_thing + if isinstance(s_thing, annmodel.SomeInstance): + return self.make_instance_wrapper(None, s_thing) + return s_thing + +def rebuild_annotation(info, bk): + if isinstance(info, SomeObject): + return info + if isinstance(info, Instance): + classinfo = info.classinfo + obj = external_class_registry.get_class(classinfo) + else: + obj = info + s_foo = annotationoftype(obj, bk) + return s_foo + + +class ImportExportComponent(object): + """ An ImportExportComponent holds separate compilation data and information about exported + functions. """ + interface_directory = os.path.join(os.path.dirname(pypy.__file__), "interfaces") + packages = {} + translation_key = random.randrange(0, 2**30) + + def __init__(self, name, *args): + assert name not in ImportExportComponent.packages + self.name = name + self.export_tables = None # {}: translation-run-id -> export_table + self.entry_points = [] + self.update(*args) + ImportExportComponent.packages[name] = self + self.dll = None + + def __repr__(self): + return "" % (self.name, + self.export_tables, len(self.entry_points)) + + def load_from_namespace(self, ns): + if isinstance(ns, dict): + dic = ns + else: + dic = ns.__dict__ + entrypoints = [] + for item in dic.itervalues(): + if is_exported(item): + item._component_ = self + llfnattrs = item._llfnobjattrs_ + if '_name' not in llfnattrs: + llfnattrs['_extname'] = get_function_name(item) + entrypoints.append((item, item._inputtypes_)) + elif isinstance(item, types.ClassType) or isinstance(item, type): + if getattr(item, '_exported_', None): + item._component_ = self + item._force_name_ = get_class_name(item) + entrypoints += collect_class_entrypoints(item, self) + self.entry_points.extend(entrypoints) + + def update(self, *args): + for arg in args: + self.load_from_namespace(arg) + + def ensure_load(self): + if self.export_tables is None: + self.load() + + @property + def interface_filename(self): + return os.path.join(self.interface_directory, self.name) + + def load(self): + if not os.path.exists(self.interface_filename): + self.export_tables = {} + else: + f = file(self.interface_filename, "rb") + unpickler = ExportTableUnpickler(f) + self.export_tables = unpickler.load() + f.close() + + def save(self): + assert self.export_tables is not None + if not os.path.exists(self.interface_directory): + os.mkdir(self.interface_directory) + f = file(self.interface_filename, "wb") + pickler = ExportTablePickler(f) + pickler.dump(self.export_tables) + f.close() + + def new_table(self): + self.export_tables = {} + table = ExportTable() + self.export_tables[self.translation_key] = table + return table + + @property + def table(self): + self.ensure_load() + if self.translation_key in self.export_tables: + return self.export_tables[self.translation_key] + return self.new_table() + + @property + def chosen_export_table(self): # XXX add invalidation + assert len(self.export_tables) == 1 + return self.export_tables[max(self.export_tables.keys())] + + @classmethod + def search_external_object_info(cls, obj): + for package in cls.packages.values(): + try: + return package, package.get_external_object_info(obj) + except KeyError: + pass + raise KeyError("Object %r not found in any export table (loaded: %r)." % (obj, cls.packages.keys())) + + def get_external_object_info(self, obj): + self.ensure_load() + if not self.export_tables: + raise KeyError("No export definitions found.") + if type(obj) is type: # forcing new style classes + classinfo = self.chosen_export_table.lookup_class(get_class_name(obj)) + return external_class_registry.get_class(classinfo) + assert isinstance(obj, type(lambda:None)) + funcname = get_function_name(obj) + info = self.chosen_export_table.lookup_function(funcname) + if info is not None: + return info.get_func() + raise KeyError("Object %r not found in export table." % (obj, )) + + @classmethod + def save_all(cls): + for package in cls.packages.values(): + if package.export_tables is not None: + package.save() + + def dispose(self): + os.remove(self.interface_filename) + del ImportExportComponent.packages[self.name] + + +class export(object): # based on code from carbonpython, XXX merge + def __new__(self, *args, **kwds): + if len(args) == 1 and isinstance(args[0], types.FunctionType): + func = args[0] + return export()(func) + return object.__new__(self, *args, **kwds) + + def __init__(self, *args, **kwds): + if args == (Ellipsis, ): + self.inputtypes = Ellipsis + else: + self.inputtypes = args + self.package = kwds.pop("package", None) + self.force_name = kwds.pop("force_name", None) + self.return_type = kwds.pop("ret", None) + if len(kwds) > 0: + raise TypeError, "unexpected keyword argument: '%s'" % kwds.keys()[0] + + def __call__(self, func): + func._inputtypes_ = self.inputtypes + func._package_ = self.package + llattrs = {'_exported': True} + if self.force_name: + llattrs.update({'_name': self.force_name, 'external': 'C'}) + func._llfnobjattrs_ = llattrs + func._export_ = True + func._force_virtual_ = True + func._check_ret_type_ = self.return_type + return func + +def inputtype_to_instance(it): + obj = synthesize_abstract_class_info(it) + if isinstance(obj, ClassInfo): + return Instance(obj) + return obj + +def scimport_forward_reference(obj): + inputtypes = obj._inputtypes_ + rettype = obj._check_ret_type_ + fi = FunctionInfo(inputtypes, rettype, None, None) + link_key = obj._llfnobjattrs_.get('_name') + assert link_key + fi.link_key = link_key + return fi + + +def synthesize_abstract_class_info(obj): + if obj is None: + return None + if type(obj) is not type: + return obj + if not obj.__dict__.get('_exported_'): + return obj + if not obj.__dict__.get('_abstract_'): + return None + if hasattr(obj, "_local_importinfo_"): + return obj._local_importinfo_ + name = get_class_name(obj) + bases = obj.__bases__ + base = None + if bases and bases[0] is not object: + base = bases[0] + ci = ClassInfo(synthesize_abstract_class_info(base), [], (), None) + obj._local_importinfo_ = ci + ci.vtablename = (obj._component_.name, name) + for name in obj.__dict__.keys(): + if name.startswith("_"): + continue + typ = obj.__dict__[name] + assert is_exported(typ) + ci.cls.append(("cls_" + name, name, synthesize_function(typ, obj))) + return ci + + +def scimport(obj, iep=None, dynamic=False, forward_ref=False): + if forward_ref: + return scimport_forward_reference(obj).get_func(dynamic) + if iep is not None: + info = iep.get_external_object_info(obj) + else: + iep, info = ImportExportComponent.search_external_object_info(obj) + + return info + +def instance_to_lltype(rtyper, val): + if isinstance(val, Instance): + return rtyper.getrepr(SomeInstance(rtyper.annotator.bookkeeper.getuniqueclassdef(external_class_registry.get_class(val.classinfo)))).lowleveltype + return val + + +def check_package(package_path, checkobj): + package, objname = package_path.rsplit(".", 1) + __import__(package) + mod = sys.modules[package] + objcheck = getattr(mod, objname) + assert checkobj is objcheck + + +def get_function_name(func, package=None, cls=None): + func_string = func + if not isinstance(func_string, str): + func_string = func.func_name + if package is None and hasattr(func, "_package_"): + package = func._package_ + if package is None: + if cls is not None: + package_path = get_class_name(cls) + "." + func_string + else: + package_path = func.__module__ + "." + func_string + check_package(package_path, func) + else: + if package != "": + package += "." + package_path = package + func_string + return "func_" + package_path + + +def get_class_name(cls): + if hasattr(cls, "_package_"): + classname = cls._package_ + cls.__name__ + else: + classname = cls.__module__ + "." + cls.__name__ + check_package(classname, cls) + return "class_" + classname + + +class SeparateModuleFunctionCallable(PseudoHighLevelCallable): + def __init__(self, link_key, T_args, T_ret, args_s, s_result, dll=False): + self.link_key = link_key + self.T_args = T_args + self.T_ret = T_ret + self.dll = dll + assert s_result + PseudoHighLevelCallable.__init__(self, None, args_s, s_result) + + def __repr__(self): + return "" % (self.link_key,) + + def compute_rebuilt_args_result(self, bk): + if hasattr(self, 'args_s_rebuilt') and bk is self.used_bk: + return + self.args_s_rebuilt = [rebuild_annotation(s_arg, bk) for s_arg in self.args_s] + self.s_result_rebuilt = rebuild_annotation(self.s_result, bk) + self.used_bk = bk + + +class SeparateModuleFunctionCallableEntry(PseudoHighLevelCallableEntry): + _type_ = SeparateModuleFunctionCallable + + def compute_result_annotation(self, *args_s): + bk = self.bookkeeper + self.instance.compute_rebuilt_args_result(bk) + s_args_given, s_result = self.instance.args_s_rebuilt, self.instance.s_result_rebuilt + assert len(s_args_given) == len(args_s) + for arg_s_1, arg_s_2 in zip(s_args_given, args_s): + assert arg_s_1.contains(arg_s_2) + return s_result + + def make_helper(self, rtyper, PFT, link_key, s_args, s_ret): + def caller(*args): # XXX breaks if the dlopen arguments are changed because the type is unknown + return rffi.cast(PFT, dlsym(dlopen(lltype.nullptr(rffi.CCHARP.TO), RTLD_GLOBAL | RTLD_NOW), link_key))(*args) + caller._annspecialcase_ = 'specialize:ll' + return rtyper.getannmixlevel().delayedfunction(caller, s_args, s_ret, True) + + def specialize_call(self, hop): + # XXX probably we can retrieve the old graph and make it usable for the llinterpreter? + args_r = [hop.rtyper.getrepr(s) for s in self.instance.args_s_rebuilt] + r_res = hop.rtyper.getrepr(self.instance.s_result_rebuilt) + if self.instance.T_args: + T_args = [instance_to_lltype(hop.rtyper, i) for i in self.instance.T_args] + T_ret = instance_to_lltype(hop.rtyper, self.instance.T_ret) + else: + T_args = [r.lowleveltype for r in args_r] + T_ret = r_res.lowleveltype + vlist = hop.inputargs(*args_r) + for r_arg, ARGTYPE in zip(args_r, T_args): + assert r_arg.lowleveltype == ARGTYPE + assert r_res.lowleveltype == T_ret + + component = None + lkey = self.instance.link_key + if isinstance(lkey, tuple): + component, lkey = lkey + FT = lltype.FuncType(T_args, T_ret) + + if self.instance.dll: + fnptr = self.make_helper(hop.rtyper, lltype.Ptr(FT), lkey, [annmodel.lltype_to_annotation(T) for T in T_args], annmodel.lltype_to_annotation(T_ret)) + else: + fnptr = lltype.functionptr(FT, lkey, component=component, external='C', canraise=True) + TYPE = lltype.typeOf(fnptr) + c_func = Constant(fnptr, TYPE) + hop.exception_is_here() + return hop.genop('direct_call', [c_func] + vlist, resulttype=r_res) + + +def is_exported(obj): + return isinstance(obj, (types.FunctionType, types.UnboundMethodType)) \ + and hasattr(obj, '_inputtypes_') + + +def collect_class_entrypoints(cls, component): + from pypy.translator.cli.carbonpython import wrap_method + entrypoints = [] + for item in cls.__dict__.itervalues(): + if is_exported(item): + inputtypes = item._inputtypes_ + if inputtypes != Ellipsis: + inputtypes = (cls,) + item._inputtypes_ + wrapped = wrap_method(item) + wrapped._meth_wrapper_info_ = (cls, item) + wrapped._check_ret_type_ = item._check_ret_type_ + wrapped._force_virtual_ = item._force_virtual_ + else: + wrapped = item + item._component_ = component + llfnattrs = item._llfnobjattrs_ + if '_name' not in llfnattrs: + llfnattrs['_extname'] = get_function_name(item, cls=cls) + entrypoints.append((wrapped, inputtypes)) + return entrypoints + + +def unwrap_meth_wrapper(func): + if hasattr(func, '_meth_wrapper_info_'): + mwi = func._meth_wrapper_info_ + assert mwi is not None # XXX ctor + func = mwi[1] + return func + + From xoraxax at codespeak.net Wed Dec 9 13:15:29 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 9 Dec 2009 13:15:29 +0100 (CET) Subject: [pypy-svn] r70024 - in pypy/branch/sepcomp/pypy/module: . md5demo md5demo/test Message-ID: <20091209121529.AAB04168051@codespeak.net> Author: xoraxax Date: Wed Dec 9 13:15:29 2009 New Revision: 70024 Added: pypy/branch/sepcomp/pypy/module/md5demo/ pypy/branch/sepcomp/pypy/module/md5demo/__init__.py pypy/branch/sepcomp/pypy/module/md5demo/interface.py pypy/branch/sepcomp/pypy/module/md5demo/interp_md5.py pypy/branch/sepcomp/pypy/module/md5demo/module_md5.py pypy/branch/sepcomp/pypy/module/md5demo/test/ pypy/branch/sepcomp/pypy/module/md5demo/test/test_md5.py Modified: pypy/branch/sepcomp/pypy/module/__init__.py Log: Add example md5demo module. Modified: pypy/branch/sepcomp/pypy/module/__init__.py ============================================================================== --- pypy/branch/sepcomp/pypy/module/__init__.py (original) +++ pypy/branch/sepcomp/pypy/module/__init__.py Wed Dec 9 13:15:29 2009 @@ -1 +1,5 @@ # empty +from pypy.module.md5demo import interp_md5 +from pypy.translator.sepcomp import scimport, ImportExportComponent, export + +ImportExportComponent("md5demo", interp_md5) Added: pypy/branch/sepcomp/pypy/module/md5demo/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/sepcomp/pypy/module/md5demo/__init__.py Wed Dec 9 13:15:29 2009 @@ -0,0 +1,30 @@ + +""" +Mixed-module definition for the md5 module. +Note that there is also a pure Python implementation in pypy/lib/md5.py; +the present mixed-module version of md5 takes precedence if it is enabled. +""" + +from pypy.interpreter.mixedmodule import MixedModule + + +class Module(MixedModule): + """\ +This module implements the interface to RSA's MD5 message digest +algorithm (see also Internet RFC 1321). Its use is quite +straightforward: use new() to create an md5 object. You can now feed +this object with arbitrary strings using the update() method, and at any +point you can ask it for the digest (a strong kind of 128-bit checksum, +a.k.a. ``fingerprint'') of the concatenation of the strings fed to it so +far using the digest() method.""" + + interpleveldefs = { + 'md5': 'interp_md5.W_MD5', + 'new': 'interp_md5.W_MD5', + 'MD5Type': 'interp_md5.W_MD5', + 'digest_size': 'space.wrap(16)', + 'load_module': 'interp_md5.load_module', + } + + appleveldefs = { + } Added: pypy/branch/sepcomp/pypy/module/md5demo/interface.py ============================================================================== --- (empty file) +++ pypy/branch/sepcomp/pypy/module/md5demo/interface.py Wed Dec 9 13:15:29 2009 @@ -0,0 +1,5 @@ +from pypy.translator.sepcomp import scimport +from pypy.module.md5demo.interp_md5 import RMD5Interface + +MD5Interface = scimport(RMD5Interface) +del RMD5Interface Added: pypy/branch/sepcomp/pypy/module/md5demo/interp_md5.py ============================================================================== --- (empty file) +++ pypy/branch/sepcomp/pypy/module/md5demo/interp_md5.py Wed Dec 9 13:15:29 2009 @@ -0,0 +1,88 @@ +from pypy.rlib import rmd5 +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root + +from pypy.rlib.objectmodel import instantiate, we_are_translated +from pypy.translator.sepcomp import scimport, ImportExportComponent, export +from pypy.rlib.libffi import dlopen_global_persistent + + +class RMD5Interface(rmd5.RMD5): + _exported_ = True + @export + def _init(self): + rmd5.RMD5._init(self) + return 1 + + @export + def digest(self): + return rmd5.RMD5.digest(self) + + @export(str) + def update(self, x): + return rmd5.RMD5.update(self, x) + + +class W_MD5(Wrappable): + """ + A subclass of RMD5 that can be exposed to app-level. + """ + + def __init__(self, space, md5obj): + self.space = space + self.md5obj = md5obj + md5obj._init() + + def update_w(self, string): + self.md5obj.update(string) + + def digest_w(self): + return self.space.wrap(self.md5obj.digest()) + + def hexdigest_w(self): + return self.space.wrap(self.md5obj.hexdigest()) + + +def W_MD5___new__(space, w_subtype, initialdata=''): + """ + Create a new md5 object and call its initializer. + """ + a = RMD5Interface() # hack + w_md5 = space.allocate_instance(W_MD5, w_subtype) + md5 = space.interp_w(W_MD5, w_md5) + W_MD5.__init__(md5, space, module_init()) + md5.md5obj.update(initialdata) + return w_md5 + +class MyRMD5(RMD5Interface): + def digest(self): + return RMD5Interface.digest(self) + "added string" + +def module_init(): + return get_dynamic_object() + #return MyRMD5() + + at export(ret=RMD5Interface, force_name="module_init") +def get_dynamic_object(foo): + raise NotImplementedError + +get_dynamic_object = scimport(get_dynamic_object, dynamic=True, forward_ref=True) + + + +def load_module(space): + dlopen_global_persistent("/tmp/testmodule.so") + return space.wrap(None) +load_module.unwrap_spec = [ObjSpace] + +W_MD5.typedef = TypeDef( + 'MD5Type', + __new__ = interp2app(W_MD5___new__, unwrap_spec=[ObjSpace, W_Root, + 'bufferstr']), + update = interp2app(W_MD5.update_w, unwrap_spec=['self', 'bufferstr']), + digest = interp2app(W_MD5.digest_w, unwrap_spec=['self']), + hexdigest = interp2app(W_MD5.hexdigest_w, unwrap_spec=['self']), + __doc__ = """md5(arg) -> return new md5 object. + +If arg is present, the method call update(arg) is made.""") Added: pypy/branch/sepcomp/pypy/module/md5demo/module_md5.py ============================================================================== --- (empty file) +++ pypy/branch/sepcomp/pypy/module/md5demo/module_md5.py Wed Dec 9 13:15:29 2009 @@ -0,0 +1,70 @@ +from pypy.rlib import rmd5 +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root + +from pypy.rlib.objectmodel import instantiate, we_are_translated +from pypy.translator.sepcomp import scimport, ImportExportComponent, export +from pypy.rlib.libffi import dlopen_global_persistent + +class RMD5_interface(rmd5.RMD5): + pass +class W_MD5(Wrappable): + """ + A subclass of RMD5 that can be exposed to app-level. + """ + _exported_ = True + + def __init__(self, space): + self.space = space + self._init() + + def update_w(self, string): + self.update(string) + + @export + def _init(self): + return rmd5.RMD5._init(self) + + @export + def digest(self): + return rmd5.RMD5.digest(self) + + @export(str) + def update(self, x): + return rmd5.RMD5.update(self, x) + + @export(W_Root) + def dostuff_w(self, w_arg): + return w_arg + +class W_MyMD5(W_MD5): + def __init__(self, spacewrapper): + self.spacewrapper = spacewrapper + self._init() + def digest(self): + return W_MD5.digest(self) + "added string" + +def module_init(spacewrapper, data): + w_mymd5 = instantiate(W_MyMD5) + if not we_are_translated(): + w_mymd5.space = spacewrapper.space + w_mymd5.__init__(spacewrapper) + w_mymd5.update(data) + return w_mymd5 + + at export(SpaceWrapper, str, ret=W_Root, force_name="module_init") +def get_dynamic_object(foo): + raise NotImplementedError + +get_dynamic_object = scimport(get_dynamic_object, dynamic=True, forward_ref=True) + + +def dynamic_md5(space, initialdata=""): + sw = SpaceWrapper(space) + w_obj = get_dynamic_object(sw, initialdata) + #w_obj = module_init(sw, initialdata) + return w_obj +dynamic_md5.unwrap_spec = [ObjSpace, str] + + Added: pypy/branch/sepcomp/pypy/module/md5demo/test/test_md5.py ============================================================================== --- (empty file) +++ pypy/branch/sepcomp/pypy/module/md5demo/test/test_md5.py Wed Dec 9 13:15:29 2009 @@ -0,0 +1,107 @@ +""" +Tests for the md5 module implemented at interp-level in pypy/module/md5. +""" + +import py +from pypy.conftest import gettestobjspace + + +class AppTestMD5DEMO(object): + + def setup_class(cls): + """ + Create a space with the md5 module and import it for use by the + tests. + """ + cls.space = gettestobjspace(usemodules=['md5demo']) + cls.w_md5 = cls.space.appexec([], """(): + import md5demo + return md5demo + """) + + def test_dynamic(self): + import md5 + assert self.md5.md5() + assert self.md5.md5("foo").digest() == md5.md5("foo").digest() + "added string" + self.md5.load_module() + + + def test_digest_size(self): + """ + md5.digest_size should be 16. + """ + assert self.md5.digest_size == 16 + + + def test_MD5Type(self): + """ + Test the two ways to construct an md5 object. + """ + md5 = self.md5 + d = md5.md5() + assert isinstance(d, md5.MD5Type) + d = md5.new() + assert isinstance(d, md5.MD5Type) + + + def test_md5object(self): + """ + Feed example strings into a md5 object and check the digest and + hexdigest. + """ + md5 = self.md5 + cases = ( + ("", + "d41d8cd98f00b204e9800998ecf8427e"), + ("a", + "0cc175b9c0f1b6a831c399e269772661"), + ("abc", + "900150983cd24fb0d6963f7d28e17f72"), + ("message digest", + "f96b697d7cb7938d525a2f31aaf161d0"), + ("abcdefghijklmnopqrstuvwxyz", + "c3fcd3d76192e4007dfb496cca67e13b"), + ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", + "d174ab98d277d9f5a5611c2c9f419d9f"), + ("1234567890"*8, + "57edf4a22be3c955ac49da2e2107b67a"), + ) + for input, expected in cases: + d = md5.new(input) + assert d.hexdigest() == expected + assert d.digest() == expected.decode('hex') + + + def test_copy(self): + """ + Test the copy() method. + """ + md5 = self.md5 + d1 = md5.md5() + d1.update("abcde") + d2 = d1.copy() + d2.update("fgh") + d1.update("jkl") + assert d1.hexdigest() == 'e570e7110ecef72fcb772a9c05d03373' + assert d2.hexdigest() == 'e8dc4081b13434b45189a720b77b6818' + + + def test_buffer(self): + """ + Test passing a buffer object. + """ + md5 = self.md5 + d1 = md5.md5(buffer("abcde")) + d1.update(buffer("jkl")) + assert d1.hexdigest() == 'e570e7110ecef72fcb772a9c05d03373' + + + def test_unicode(self): + """ + Test passing unicode strings. + """ + md5 = self.md5 + d1 = md5.md5(u"abcde") + d1.update(u"jkl") + assert d1.hexdigest() == 'e570e7110ecef72fcb772a9c05d03373' + raises(UnicodeEncodeError, d1.update, u'\xe9') From pedronis at codespeak.net Wed Dec 9 16:55:48 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 9 Dec 2009 16:55:48 +0100 (CET) Subject: [pypy-svn] r70025 - pypy/extradoc/planning Message-ID: <20091209155548.0A78D168072@codespeak.net> Author: pedronis Date: Wed Dec 9 16:55:47 2009 New Revision: 70025 Modified: pypy/extradoc/planning/jit.txt Log: this was done Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Wed Dec 9 16:55:47 2009 @@ -69,10 +69,6 @@ things we know are missing --------------------------- -metainterp/frontend: -- virtualizables are not finished: - fix 'test_external_read_sometimes' - backend: - speed of backend? From afa at codespeak.net Wed Dec 9 18:21:08 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 9 Dec 2009 18:21:08 +0100 (CET) Subject: [pypy-svn] r70026 - in pypy/trunk/pypy: interpreter module/__builtin__ module/__builtin__/test module/_demo module/_demo/test module/oracle Message-ID: <20091209172108.900A716800B@codespeak.net> Author: afa Date: Wed Dec 9 18:21:08 2009 New Revision: 70026 Removed: pypy/trunk/pypy/module/_demo/test/ Modified: pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/interpreter/mixedmodule.py pypy/trunk/pypy/interpreter/module.py pypy/trunk/pypy/module/__builtin__/importing.py pypy/trunk/pypy/module/__builtin__/test/test_import.py pypy/trunk/pypy/module/_demo/__init__.py pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_error.py Log: Revert again my recent changes to the import system. There are other consequences, like the "imp" module which must be rewritten to correctly handle built-in modules. Let's do this on a branch. (reverted r69947, r69956, r69969, r70003) Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Wed Dec 9 18:21:08 2009 @@ -239,8 +239,6 @@ config = get_pypy_config(translating=False) self.config = config - self.builtin_modules = {} - # import extra modules for side-effects import pypy.interpreter.nestedscope # register *_DEREF bytecodes @@ -268,34 +266,28 @@ def startup(self): # To be called before using the space - # Initialize already imported builtin modules + # Initialize all builtin modules from pypy.interpreter.module import Module - w_modules = self.sys.get('modules') for w_modname in self.unpackiterable( self.sys.get('builtin_module_names')): - try: - w_mod = self.getitem(w_modules, w_modname) - except OperationError, e: - if e.match(self, self.w_KeyError): - continue - raise modname = self.str_w(w_modname) - mod = self.interpclass_w(w_mod) - if isinstance(mod, Module) and not mod.startup_called: + mod = self.interpclass_w(self.getbuiltinmodule(modname)) + if isinstance(mod, Module): + import time self.timer.start("startup " + modname) mod.startup(self) - mod.startup_called = True self.timer.stop("startup " + modname) - def finish(self): w_exitfunc = self.sys.getdictvalue(self, 'exitfunc') if w_exitfunc is not None: self.call_function(w_exitfunc) from pypy.interpreter.module import Module - for w_mod in self.builtin_modules.values(): - mod = self.interpclass_w(w_mod) - if isinstance(mod, Module) and mod.startup_called: + for w_modname in self.unpackiterable( + self.sys.get('builtin_module_names')): + modname = self.str_w(w_modname) + mod = self.interpclass_w(self.getbuiltinmodule(modname)) + if isinstance(mod, Module): mod.shutdown(self) if self.config.objspace.std.withdictmeasurement: from pypy.objspace.std.dictmultiobject import report @@ -347,37 +339,14 @@ w_name = self.wrap(name) w_mod = self.wrap(Module(self, w_name)) - self.builtin_modules[name] = w_mod + w_modules = self.sys.get('modules') + self.setitem(w_modules, w_name, w_mod) return name def getbuiltinmodule(self, name): w_name = self.wrap(name) w_modules = self.sys.get('modules') - try: - return self.getitem(w_modules, w_name) - except OperationError, e: - if not e.match(self, self.w_KeyError): - raise - - # If the module is a builtin but not yet imported, - # retrieve it and initialize it - try: - w_mod = self.builtin_modules[name] - except KeyError: - raise e - else: - # Add the module to sys.modules - self.setitem(w_modules, w_name, w_mod) - - # And initialize it - from pypy.interpreter.module import Module - mod = self.interpclass_w(w_mod) - if isinstance(mod, Module) and not mod.startup_called: - self.timer.start("startup " + name) - mod.startup(self) - mod.startup_called = True - self.timer.stop("startup " + name) - return w_mod + return self.getitem(w_modules, w_name) def get_builtinmodule_to_install(self): """NOT_RPYTHON""" @@ -428,15 +397,16 @@ w_name = self.wrap('sys') self.sys = Module(self, w_name) w_modules = self.sys.get('modules') - self.builtin_modules['sys'] = self.wrap(self.sys) + self.setitem(w_modules, w_name, self.wrap(self.sys)) - self.builtin_modules['exceptions'] = self.wrap(self.exceptions_module) + self.setitem(w_modules, w_name_exceptions, + self.wrap(self.exceptions_module)) from pypy.module.__builtin__ import Module w_name = self.wrap('__builtin__') self.builtin = Module(self, w_name) w_builtin = self.wrap(self.builtin) - self.builtin_modules['__builtin__'] = self.wrap(w_builtin) + self.setitem(w_modules, w_name, w_builtin) self.setitem(self.builtin.w_dict, self.wrap('__builtins__'), w_builtin) bootstrap_modules = ['sys', '__builtin__', 'exceptions'] @@ -510,10 +480,12 @@ def setup_builtin_modules(self): "NOT_RPYTHON: only for initializing the space." - self.getbuiltinmodule('sys') - self.getbuiltinmodule('__builtin__') - for mod in self.builtin_modules.values(): - mod.setup_after_space_initialization() + from pypy.interpreter.module import Module + for w_modname in self.unpackiterable(self.sys.get('builtin_module_names')): + modname = self.unwrap(w_modname) + mod = self.getbuiltinmodule(modname) + if isinstance(mod, Module): + mod.setup_after_space_initialization() def initialize(self): """NOT_RPYTHON: Abstract method that should put some minimal Modified: pypy/trunk/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/trunk/pypy/interpreter/mixedmodule.py (original) +++ pypy/trunk/pypy/interpreter/mixedmodule.py Wed Dec 9 18:21:08 2009 @@ -87,7 +87,6 @@ def _freeze_(self): self.getdict() - self.startup_called = False # hint for the annotator: Modules can hold state, so they are # not constant return False Modified: pypy/trunk/pypy/interpreter/module.py ============================================================================== --- pypy/trunk/pypy/interpreter/module.py (original) +++ pypy/trunk/pypy/interpreter/module.py Wed Dec 9 18:21:08 2009 @@ -16,22 +16,21 @@ self.w_name = w_name if w_name is not None: space.setitem(w_dict, space.new_interned_str('__name__'), w_name) - self.startup_called = False def setup_after_space_initialization(self): """NOT_RPYTHON: to allow built-in modules to do some more setup after the space is fully initialized.""" def startup(self, space): - """This is called at runtime on import to allow the module to - do initialization when it is imported for the first time. + """This is called at runtime before the space gets uses to allow + the module to do initialization at runtime. """ def shutdown(self, space): """This is called when the space is shut down, just after - sys.exitfunc(), if the module has been imported. + sys.exitfunc(). """ - + def getdict(self): return self.w_dict Modified: pypy/trunk/pypy/module/__builtin__/importing.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/importing.py (original) +++ pypy/trunk/pypy/module/__builtin__/importing.py Wed Dec 9 18:21:08 2009 @@ -228,7 +228,7 @@ def _absolute_import(space, modulename, baselevel, w_fromlist, tentative): w = space.wrap - + w_mod = None parts = modulename.split('.') prefix = [] @@ -291,9 +291,6 @@ return w_mod - # check the builtin modules - if modulename in space.builtin_modules: - return space.getbuiltinmodule(modulename) if w_path is not None: for path in space.unpackiterable(w_path): Modified: pypy/trunk/pypy/module/__builtin__/test/test_import.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/test_import.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/test_import.py Wed Dec 9 18:21:08 2009 @@ -41,9 +41,7 @@ relative_a = "import a", abs_b = "import b", abs_x_y = "import x.y", - abs_sys = "import sys", string = "inpackage = 1", - errno = "", absolute = "from __future__ import absolute_import\nimport string", relative_b = "from __future__ import absolute_import\nfrom . import string", relative_c = "from __future__ import absolute_import\nfrom .string import inpackage", @@ -256,17 +254,6 @@ import pkg_r.inpkg raises(ImportError,imp) - def test_import_builtin_inpackage(self): - def imp(): - import pkg.sys - raises(ImportError,imp) - - import sys, pkg.abs_sys - assert pkg.abs_sys.sys is sys - - import errno, pkg.errno - assert pkg.errno is not errno - def test_import_Globals_Are_None(self): import sys m = __import__('sys') @@ -707,43 +694,14 @@ def find_module(self, fullname, path=None): tried_imports.append((fullname, path)) - import sys, math - sys.meta_path.append(Importer()) + import sys try: + sys.meta_path.append(Importer()) import datetime assert len(tried_imports) == 1 - package_name = '.'.join(__name__.split('.')[:-1]) - if package_name: - assert tried_imports[0][0] == package_name + ".datetime" - else: - assert tried_imports[0][0] == "datetime" - finally: - sys.meta_path.pop() - - def test_meta_path_block(self): - class ImportBlocker(object): - "Specified modules can't be imported, even if they are built-in" - def __init__(self, *namestoblock): - self.namestoblock = dict.fromkeys(namestoblock) - def find_module(self, fullname, path=None): - if fullname in self.namestoblock: - return self - def load_module(self, fullname): - raise ImportError, "blocked" - - import sys - modname = "errno" # an arbitrary harmless builtin module - mod = None - if modname in sys.modules: - mod = sys.modules - del sys.modules[modname] - sys.meta_path.append(ImportBlocker(modname)) - try: - raises(ImportError, __import__, modname) + tried_imports[0][0] == "datetime" finally: sys.meta_path.pop() - if mod: - sys.modules[modname] = mod def test_path_hooks_leaking(self): class Importer(object): Modified: pypy/trunk/pypy/module/_demo/__init__.py ============================================================================== --- pypy/trunk/pypy/module/_demo/__init__.py (original) +++ pypy/trunk/pypy/module/_demo/__init__.py Wed Dec 9 18:21:08 2009 @@ -12,13 +12,3 @@ appleveldefs = { 'DemoError' : 'app_demo.DemoError', } - - # Used in tests - demo_events = [] - def setup_after_space_initialization(self): - Module.demo_events.append('setup') - def startup(self, space): - Module.demo_events.append('startup') - def shutdown(self, space): - Module.demo_events.append('shutdown') - Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Wed Dec 9 18:21:08 2009 @@ -39,7 +39,6 @@ def startup(self, space): from pypy.module.oracle.interp_error import get state = get(space) - state.startup(space) (state.w_DecimalType, state.w_DateTimeType, state.w_DateType, state.w_TimedeltaType, ) = space.fixedview(space.appexec([], """(): Modified: pypy/trunk/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_error.py (original) +++ pypy/trunk/pypy/module/oracle/interp_error.py Wed Dec 9 18:21:08 2009 @@ -4,47 +4,32 @@ from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError - from pypy.module.oracle import roci, config -from pypy.rlib.unroll import unrolling_iterable - -exported_names = unrolling_iterable(""" - DatabaseError OperationalError InterfaceError ProgrammingError - NotSupportedError IntegrityError InternalError DataError - Variable Connection""".split()) class State: # XXX move to another file - def __init__(self, space): - "NOT_RPYTHON" - self.variableTypeByPythonType = {} - self.w_DecimalType = None - self.w_DateTimeType = None - self.w_DateType = None - self.w_TimedeltaType = None - - for name in exported_names: - setattr(self, 'w_' + name, None) - - def startup(self, space): w_module = space.getbuiltinmodule('cx_Oracle') - for name in exported_names: - setattr(self, 'w_' + name, space.getattr(w_module, space.wrap(name))) + def get(name): + return space.getattr(w_module, space.wrap(name)) + + self.w_DatabaseError = get('DatabaseError') + self.w_OperationalError = get('OperationalError') + self.w_InterfaceError = get('InterfaceError') + self.w_ProgrammingError = get('ProgrammingError') + self.w_NotSupportedError = get('NotSupportedError') + self.w_IntegrityError = get('IntegrityError') + self.w_InternalError = get('InternalError') + self.w_DataError = get('DataError') + self.w_Variable = get('Variable') + self.w_Connection = get('Connection') from pypy.module.oracle.interp_variable import all_variable_types + self.variableTypeByPythonType = {} for varType in all_variable_types: w_type = space.gettypeobject(varType.typedef) self.variableTypeByPythonType[w_type] = varType - (self.w_DecimalType, - self.w_DateTimeType, self.w_DateType, self.w_TimedeltaType, - ) = space.fixedview(space.appexec([], """(): - import decimal, datetime - return (decimal.Decimal, - datetime.datetime, datetime.date, datetime.timedelta) - """)) - def get(space): return space.fromcache(State) From afa at codespeak.net Wed Dec 9 18:28:22 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 9 Dec 2009 18:28:22 +0100 (CET) Subject: [pypy-svn] r70027 - pypy/branch/import-builtin Message-ID: <20091209172822.3E1BB16800B@codespeak.net> Author: afa Date: Wed Dec 9 18:28:21 2009 New Revision: 70027 Added: pypy/branch/import-builtin/ - copied from r69893, pypy/trunk/ Log: A branch to fix the implementation of the import system: - most built-in modules should not be imported when pypy starts - the imp module does not behave correctly wrt import hooks From afa at codespeak.net Wed Dec 9 18:31:19 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 9 Dec 2009 18:31:19 +0100 (CET) Subject: [pypy-svn] r70028 - pypy/branch/import-builtin Message-ID: <20091209173119.0DCFA16800B@codespeak.net> Author: afa Date: Wed Dec 9 18:31:18 2009 New Revision: 70028 Removed: pypy/branch/import-builtin/ Log: remove branch created with wrong revision From afa at codespeak.net Wed Dec 9 18:31:50 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 9 Dec 2009 18:31:50 +0100 (CET) Subject: [pypy-svn] r70029 - pypy/branch/import-builtin Message-ID: <20091209173150.10F7C16800B@codespeak.net> Author: afa Date: Wed Dec 9 18:31:49 2009 New Revision: 70029 Added: pypy/branch/import-builtin/ - copied from r70025, pypy/trunk/ Log: create again the branch to work on the import system From fijal at codespeak.net Wed Dec 9 21:43:09 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 9 Dec 2009 21:43:09 +0100 (CET) Subject: [pypy-svn] r70033 - in pypy/branch/listcopyop/pypy: rlib rpython rpython/lltypesystem rpython/memory/gc rpython/memory/gctransform Message-ID: <20091209204309.C59CD168015@codespeak.net> Author: fijal Date: Wed Dec 9 21:43:08 2009 New Revision: 70033 Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py pypy/branch/listcopyop/pypy/rpython/llinterp.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py Log: Revert the push_alive hack for refcounting. Instead make writebarrier_before_copy return a bool (again), otherwise go for the slow path. Used only by refcounting so far Modified: pypy/branch/listcopyop/pypy/rlib/rgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rlib/rgc.py (original) +++ pypy/branch/listcopyop/pypy/rlib/rgc.py Wed Dec 9 21:43:08 2009 @@ -340,10 +340,11 @@ if isinstance(TP.OF, lltype.Ptr) and TP.OF.TO._gckind == 'gc': # perform a write barrier that copies necessary flags from # source to dest - llop.gc_writebarrier_before_copy(lltype.Void, source, dest) - # the hack below is a write barrier version for refcounting :-/ - for i in range(length): - llop.gc_push_alive(lltype.Void, source[source_start + i]) + if not llop.gc_writebarrier_before_copy(lltype.Void, source, dest): + # if the write barrier is not supported, copy by hand + for i in range(length): + dest[i + dest_start] = source[i + source_start] + return source_addr = llmemory.cast_ptr_to_adr(source) dest_addr = llmemory.cast_ptr_to_adr(dest) cp_source_addr = (source_addr + llmemory.itemoffsetof(TP, 0) + @@ -354,4 +355,5 @@ llmemory.raw_memcopy(cp_source_addr, cp_dest_addr, llmemory.sizeof(TP.OF) * length) keepalive_until_here(source) + keepalive_until_here(dest) ll_arraycopy._annspecialcase_ = 'specialize:ll' Modified: pypy/branch/listcopyop/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/llinterp.py (original) +++ pypy/branch/listcopyop/pypy/rpython/llinterp.py Wed Dec 9 21:43:08 2009 @@ -859,9 +859,6 @@ def op_gc_push_alive_pyobj(self, pyobj): raise NotImplementedError("gc_push_alive_pyobj") - def op_gc_push_alive(self, pyobj): - pass - def op_gc_pop_alive_pyobj(self, pyobj): raise NotImplementedError("gc_pop_alive_pyobj") Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/lloperation.py Wed Dec 9 21:43:08 2009 @@ -443,7 +443,6 @@ 'gc_restore_exception': LLOp(), 'gc_call_rtti_destructor': LLOp(), 'gc_deallocate': LLOp(), - 'gc_push_alive': LLOp(), 'gc_push_alive_pyobj': LLOp(), 'gc_pop_alive_pyobj': LLOp(), 'gc_reload_possibly_moved': LLOp(), Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Wed Dec 9 21:43:08 2009 @@ -493,7 +493,7 @@ source_hdr = self.header(source_addr) dest_hdr = self.header(dest_addr) if dest_hdr.tid & GCFLAG_NO_YOUNG_PTRS == 0: - return + return True # ^^^ a fast path of write-barrier if source_hdr.tid & GCFLAG_NO_YOUNG_PTRS == 0: # there might be an object in source that is in nursery @@ -505,6 +505,7 @@ # gen dest_hdr.tid &= ~GCFLAG_NO_HEAP_PTRS self.last_generation_root_objects.append(dest_addr) + return True def is_last_generation(self, obj): # overridden by HybridGC Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py Wed Dec 9 21:43:08 2009 @@ -295,7 +295,7 @@ if hasattr(GCClass, 'writebarrier_before_copy'): self.wb_before_copy_ptr = \ getfn(GCClass.writebarrier_before_copy.im_func, - [s_gc] + [annmodel.SomeAddress()] * 2, annmodel.s_None) + [s_gc] + [annmodel.SomeAddress()] * 2, annmodel.SomeBool) elif GCClass.needs_write_barrier: raise NotImplementedError("GC needs write barrier, but does not provide writebarrier_before_copy functionality") @@ -793,7 +793,8 @@ dest_addr = hop.genop('cast_ptr_to_adr', [op.args[1]], resulttype=llmemory.Address) hop.genop('direct_call', [self.wb_before_copy_ptr, self.c_const_gc, - source_addr, dest_addr]) + source_addr, dest_addr], + resultvar=op.result) def gct_weakref_create(self, hop): op = hop.spaceop Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/refcounting.py Wed Dec 9 21:43:08 2009 @@ -142,9 +142,6 @@ def var_needs_set_transform(self, var): return var_needsgc(var) - def gct_gc_push_alive(self, hop): - self.push_alive_nopyobj(hop.spaceop.args[0], hop.llops) - def push_alive_nopyobj(self, var, llops): v_adr = gen_cast(llops, llmemory.Address, var) llops.genop("direct_call", [self.increfptr, v_adr]) @@ -162,6 +159,9 @@ lltype.typeOf(dealloc_fptr), dealloc_fptr) llops.genop("direct_call", [self.decref_ptr, v_adr, cdealloc_fptr]) + def gct_gc_writebarrier_before_copy(self, hop): + return rmodel.inputconst(lltype.Bool, False) + def gct_fv_gc_malloc(self, hop, flags, TYPE, c_size): v_raw = hop.genop("direct_call", [self.malloc_fixedsize_ptr, c_size], resulttype=llmemory.Address) Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py Wed Dec 9 21:43:08 2009 @@ -377,14 +377,13 @@ gct_getfield = default - def gct_gc_push_alive(self, hop): - pass - def gct_zero_gc_pointers_inside(self, hop): pass def gct_gc_writebarrier_before_copy(self, hop): - pass + # by default we don't need to do anything special with this, + # for exception see refcounting + return rmodel.inputconst(lltype.Bool, True) def gct_gc_identityhash(self, hop): # must be implemented in the various GCs From afa at codespeak.net Wed Dec 9 23:49:01 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 9 Dec 2009 23:49:01 +0100 (CET) Subject: [pypy-svn] r70037 - pypy/trunk/pypy/lib/test2 Message-ID: <20091209224901.5E4CC168015@codespeak.net> Author: afa Date: Wed Dec 9 23:49:00 2009 New Revision: 70037 Modified: pypy/trunk/pypy/lib/test2/test_grp_extra.py Log: skip the grp module on Windows Modified: pypy/trunk/pypy/lib/test2/test_grp_extra.py ============================================================================== --- pypy/trunk/pypy/lib/test2/test_grp_extra.py (original) +++ pypy/trunk/pypy/lib/test2/test_grp_extra.py Wed Dec 9 23:49:00 2009 @@ -1,5 +1,8 @@ import py -from pypy.lib import grp +try: + from pypy.lib import grp +except ImportError: + py.test.skip("No grp module on this platform") def test_extra(): py.test.raises(TypeError, grp.getgrnam, False) From arigo at codespeak.net Thu Dec 10 14:16:12 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 10 Dec 2009 14:16:12 +0100 (CET) Subject: [pypy-svn] r70039 - pypy/trunk/pypy/interpreter/test Message-ID: <20091210131612.96445168023@codespeak.net> Author: arigo Date: Thu Dec 10 14:16:12 2009 New Revision: 70039 Modified: pypy/trunk/pypy/interpreter/test/test_module.py Log: Windows friendlyness. Modified: pypy/trunk/pypy/interpreter/test/test_module.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_module.py (original) +++ pypy/trunk/pypy/interpreter/test/test_module.py Thu Dec 10 14:16:12 2009 @@ -57,7 +57,8 @@ import _pypy_interact # known to be in pypy/lib r = repr(_pypy_interact) assert r.startswith("') nofile = type(_pypy_interact)('nofile', 'foo') assert repr(nofile) == "" From arigo at codespeak.net Thu Dec 10 16:05:46 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 10 Dec 2009 16:05:46 +0100 (CET) Subject: [pypy-svn] r70040 - pypy/build/bot2/pypybuildbot Message-ID: <20091210150546.B5BC0168021@codespeak.net> Author: arigo Date: Thu Dec 10 16:05:45 2009 New Revision: 70040 Modified: pypy/build/bot2/pypybuildbot/master.py Log: Start the run of the LINUX32 tests first, to force wyvern to be considered busy and use cobra for at least some of the other runs. Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Thu Dec 10 16:05:45 2009 @@ -104,7 +104,9 @@ 'change_source': [], 'schedulers': [ - Nightly("nightly", [LINUX32, APPLVLLINUX32, APPLVLWIN32, + Nightly("nightly-first", [LINUX32], + hour=4, minute=44), + Nightly("nightly", [APPLVLLINUX32, APPLVLWIN32, STACKLESSAPPLVLLINUX32, JITLINUX32, OJITLINUX32, MACOSX32], hour=4, minute=45), From arigo at codespeak.net Thu Dec 10 16:22:58 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 10 Dec 2009 16:22:58 +0100 (CET) Subject: [pypy-svn] r70041 - in pypy/branch/listcopyop/pypy: rpython/memory/test translator/c/test Message-ID: <20091210152258.A4B8A168021@codespeak.net> Author: arigo Date: Thu Dec 10 16:22:58 2009 New Revision: 70041 Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py Log: Apply r69992 here too. Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Thu Dec 10 16:22:58 2009 @@ -847,9 +847,6 @@ # (and give fixedsize) def define_arraycopy(cls): - import new - ll_arraycopy = new.function(rgc.ll_arraycopy.func_code, {}) - S = lltype.GcStruct('S') TP = lltype.GcArray(lltype.Ptr(S)) def fn(): @@ -857,7 +854,7 @@ l2 = lltype.malloc(TP, 100) for i in range(100): l[i] = lltype.malloc(S) - ll_arraycopy(l, l2, 50, 0, 50) + rgc.ll_arraycopy(l, l2, 50, 0, 50) # force nursery collect x = [] for i in range(20): Modified: pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py Thu Dec 10 16:22:58 2009 @@ -837,9 +837,6 @@ def define_arraycopy_writebarrier(cls): - import new - ll_arraycopy = new.function(rgc.ll_arraycopy.func_code, {}) - TP = lltype.GcArray(lltype.Signed) S = lltype.GcStruct('S') def fn(): @@ -847,7 +844,7 @@ for i in range(100): l[i] = 1 l2 = lltype.malloc(TP, 50) - ll_arraycopy(l, l2, 50, 0, 50) + rgc.ll_arraycopy(l, l2, 50, 0, 50) # force a nursery collect x = [] for i in range(20): From arigo at codespeak.net Thu Dec 10 16:34:28 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 10 Dec 2009 16:34:28 +0100 (CET) Subject: [pypy-svn] r70042 - in pypy/branch/listcopyop/pypy/rpython: . lltypesystem memory memory/gctransform memory/gctransform/test memory/test Message-ID: <20091210153428.A816A168021@codespeak.net> Author: arigo Date: Thu Dec 10 16:34:27 2009 New Revision: 70042 Modified: pypy/branch/listcopyop/pypy/rpython/llinterp.py pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/support.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/test/test_framework.py pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Log: * fix some missing "return True". * revert a few changes that are not needed any more, as far as I can tell. Modified: pypy/branch/listcopyop/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/llinterp.py (original) +++ pypy/branch/listcopyop/pypy/rpython/llinterp.py Thu Dec 10 16:34:27 2009 @@ -756,7 +756,9 @@ def op_gc_writebarrier_before_copy(self, source, dest): if hasattr(self.heap, 'writebarrier_before_copy'): - self.heap.writebarrier_before_copy(source, dest) + return self.heap.writebarrier_before_copy(source, dest) + else: + return True def op_getfield(self, obj, field): checkptr(obj) Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/opimpl.py Thu Dec 10 16:34:27 2009 @@ -400,6 +400,7 @@ assert isinstance(A.TO, lltype.GcArray) assert isinstance(A.TO.OF, lltype.Ptr) assert A.TO.OF.TO._gckind == 'gc' + return True def op_getfield(p, name): checkptr(p) Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/refcounting.py Thu Dec 10 16:34:27 2009 @@ -160,6 +160,8 @@ llops.genop("direct_call", [self.decref_ptr, v_adr, cdealloc_fptr]) def gct_gc_writebarrier_before_copy(self, hop): + # Not supported, so return False. Means that rgc.ll_arraycopy() + # will do the copy manually (with a 'for' loop). return rmodel.inputconst(lltype.Bool, False) def gct_fv_gc_malloc(self, hop, flags, TYPE, c_size): Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/support.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/support.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/support.py Thu Dec 10 16:34:27 2009 @@ -1,7 +1,6 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.extregistry import ExtRegistryEntry from pypy.annotation import model as annmodel -from pypy.rlib.objectmodel import specialize import os def var_ispyobj(var): Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/test/test_framework.py Thu Dec 10 16:34:27 2009 @@ -120,7 +120,7 @@ class GCClass(MarkSweepGC): needs_write_barrier = True def writebarrier_before_copy(self, source, dest): - pass + return True def write_barrier_check(spaceop, needs_write_barrier=True): t = TranslationContext() Modified: pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gcwrapper.py Thu Dec 10 16:34:27 2009 @@ -133,6 +133,8 @@ source_addr = llmemory.cast_ptr_to_adr(source) dest_addr = llmemory.cast_ptr_to_adr(dest) return self.gc.writebarrier_before_copy(source_addr, dest_addr) + else: + return True # ____________________________________________________________ Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Thu Dec 10 16:34:27 2009 @@ -835,8 +835,7 @@ return f def test_gc_heap_stats(self): - py.test.skip("XFail fails, so skip") - XXX # this test makes next test to crash + py.test.skip("this test makes the following test crash. Investigate.") run = self.runner("gc_heap_stats") res = run([]) assert res % 10000 == 2611 @@ -877,7 +876,7 @@ class transformerclass(framework.FrameworkGCTransformer): GC_PARAMS = {'start_heap_size': 4096 } root_stack_depth = 200 - from pypy.rpython.memory.gc.marksweep import MarkSweepGC as GCClass + def define_cloning(cls): B = lltype.GcStruct('B', ('x', lltype.Signed)) From arigo at codespeak.net Thu Dec 10 16:37:46 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 10 Dec 2009 16:37:46 +0100 (CET) Subject: [pypy-svn] r70043 - pypy/branch/listcopyop/pypy/translator/c/test Message-ID: <20091210153746.B4671168021@codespeak.net> Author: arigo Date: Thu Dec 10 16:37:46 2009 New Revision: 70043 Modified: pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py Log: Improve that test coverage from bad to slightly less bad. Modified: pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py Thu Dec 10 16:37:46 2009 @@ -842,15 +842,15 @@ def fn(): l = lltype.malloc(TP, 100) for i in range(100): - l[i] = 1 + l[i] = i * 3 l2 = lltype.malloc(TP, 50) - rgc.ll_arraycopy(l, l2, 50, 0, 50) + rgc.ll_arraycopy(l, l2, 40, 0, 50) # force a nursery collect x = [] for i in range(20): x.append((1, lltype.malloc(S))) for i in range(50): - assert l2[i] == 1 + assert l2[i] == (40 + i) * 3 return 0 return fn @@ -858,6 +858,7 @@ def test_arraycopy_writebarrier(self): self.run("arraycopy_writebarrier") + class TestSemiSpaceGC(TestUsingFramework, snippet.SemiSpaceGCTestDefines): gcpolicy = "semispace" should_be_moving = True From arigo at codespeak.net Thu Dec 10 16:49:22 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 10 Dec 2009 16:49:22 +0100 (CET) Subject: [pypy-svn] r70044 - pypy/branch/listcopyop/pypy/rpython/memory/gctransform Message-ID: <20091210154922.EEABC168022@codespeak.net> Author: arigo Date: Thu Dec 10 16:49:22 2009 New Revision: 70044 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py Log: Grrr. Running tests is hard, so let's not run them and expect arigo to merge the branch anyway. I'm not particularly happy. Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py Thu Dec 10 16:49:22 2009 @@ -295,7 +295,7 @@ if hasattr(GCClass, 'writebarrier_before_copy'): self.wb_before_copy_ptr = \ getfn(GCClass.writebarrier_before_copy.im_func, - [s_gc] + [annmodel.SomeAddress()] * 2, annmodel.SomeBool) + [s_gc] + [annmodel.SomeAddress()] * 2, annmodel.SomeBool()) elif GCClass.needs_write_barrier: raise NotImplementedError("GC needs write barrier, but does not provide writebarrier_before_copy functionality") From arigo at codespeak.net Thu Dec 10 16:52:10 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 10 Dec 2009 16:52:10 +0100 (CET) Subject: [pypy-svn] r70045 - pypy/branch/listcopyop/pypy/translator/c/test Message-ID: <20091210155210.30BA8168022@codespeak.net> Author: arigo Date: Thu Dec 10 16:52:09 2009 New Revision: 70045 Modified: pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py Log: More grrr-ness. Realized that the test in test_newgc was only about arrays of ints, i.e. it was not testing the gc_writebarrier_before_copy operation *at all*. Modified: pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py Thu Dec 10 16:52:09 2009 @@ -836,7 +836,7 @@ assert res != 0 - def define_arraycopy_writebarrier(cls): + def define_arraycopy_writebarrier_int(cls): TP = lltype.GcArray(lltype.Signed) S = lltype.GcStruct('S') def fn(): @@ -855,8 +855,27 @@ return fn - def test_arraycopy_writebarrier(self): - self.run("arraycopy_writebarrier") + def test_arraycopy_writebarrier_int(self): + self.run("arraycopy_writebarrier_int") + + def define_arraycopy_writebarrier_ptr(cls): + TP = lltype.GcArray(lltype.Ptr(lltype.GcArray(lltype.Signed))) + S = lltype.GcStruct('S') + def fn(): + l = lltype.malloc(TP, 100) + for i in range(100): + l[i] = lltype.malloc(TP.OF.TO, i) + l2 = lltype.malloc(TP, 50) + rgc.ll_arraycopy(l, l2, 40, 0, 50) + rgc.collect() + for i in range(50): + assert l2[i] == l[40 + i] + return 0 + + return fn + + def test_arraycopy_writebarrier_ptr(self): + self.run("arraycopy_writebarrier_ptr") class TestSemiSpaceGC(TestUsingFramework, snippet.SemiSpaceGCTestDefines): From arigo at codespeak.net Thu Dec 10 16:53:48 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 10 Dec 2009 16:53:48 +0100 (CET) Subject: [pypy-svn] r70046 - pypy/branch/listcopyop/pypy/translator/c/test Message-ID: <20091210155348.67B2A168022@codespeak.net> Author: arigo Date: Thu Dec 10 16:53:48 2009 New Revision: 70046 Modified: pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py Log: Kill unused line. Modified: pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/listcopyop/pypy/translator/c/test/test_newgc.py Thu Dec 10 16:53:48 2009 @@ -860,7 +860,6 @@ def define_arraycopy_writebarrier_ptr(cls): TP = lltype.GcArray(lltype.Ptr(lltype.GcArray(lltype.Signed))) - S = lltype.GcStruct('S') def fn(): l = lltype.malloc(TP, 100) for i in range(100): From arigo at codespeak.net Thu Dec 10 16:56:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 10 Dec 2009 16:56:29 +0100 (CET) Subject: [pypy-svn] r70047 - pypy/branch/listcopyop/pypy/rpython/memory/test Message-ID: <20091210155629.4BD13168022@codespeak.net> Author: arigo Date: Thu Dec 10 16:56:28 2009 New Revision: 70047 Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py Log: Improve the test. Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_gc.py Thu Dec 10 16:56:28 2009 @@ -566,7 +566,7 @@ for i in range(20): x.append(t) for i in range(50): - assert l2[i] + assert l2[i] == l[50 + i] return 0 self.interpret(fn, []) From arigo at codespeak.net Thu Dec 10 16:57:52 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 10 Dec 2009 16:57:52 +0100 (CET) Subject: [pypy-svn] r70048 - pypy/branch/listcopyop/pypy/rpython/memory/test Message-ID: <20091210155752.0E5C8168022@codespeak.net> Author: arigo Date: Thu Dec 10 16:57:51 2009 New Revision: 70048 Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Log: Rename that test to be consistent. Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Thu Dec 10 16:57:51 2009 @@ -845,7 +845,7 @@ # ^^^ a crude assumption that totsize - varsize would be dividable by 4 # (and give fixedsize) - def define_arraycopy(cls): + def define_writebarrier_before_copy(cls): S = lltype.GcStruct('S') TP = lltype.GcArray(lltype.Ptr(S)) def fn(): @@ -864,8 +864,8 @@ return fn - def test_arraycopy(self): - run = self.runner("arraycopy") + def test_writebarrier_before_copy(self): + run = self.runner("writebarrier_before_copy") run([]) # ________________________________________________________________ From arigo at codespeak.net Thu Dec 10 16:59:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 10 Dec 2009 16:59:42 +0100 (CET) Subject: [pypy-svn] r70049 - pypy/branch/listcopyop/pypy/rpython/memory/test Message-ID: <20091210155942.B926C168022@codespeak.net> Author: arigo Date: Thu Dec 10 16:59:41 2009 New Revision: 70049 Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Log: Improve the test. Modified: pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/test/test_transformed_gc.py Thu Dec 10 16:59:41 2009 @@ -859,7 +859,7 @@ for i in range(20): x.append((1, lltype.malloc(S))) for i in range(50): - assert l2[i] + assert l2[i] == l[50 + i] return 0 return fn From arigo at codespeak.net Thu Dec 10 17:07:58 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 10 Dec 2009 17:07:58 +0100 (CET) Subject: [pypy-svn] r70050 - pypy/branch/listcopyop/pypy/rpython/memory/gctransform Message-ID: <20091210160758.C0B88168022@codespeak.net> Author: arigo Date: Thu Dec 10 17:07:58 2009 New Revision: 70050 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/boehm.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py Log: Take a conservative default instead of one that requires care in potentially all subclasses. Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/boehm.py Thu Dec 10 17:07:58 2009 @@ -167,6 +167,9 @@ resulttype=llmemory.Address) hop.cast_result(v_addr) + def gct_gc_writebarrier_before_copy(self, hop): + return rmodel.inputconst(lltype.Bool, True) # no write barrier needed + def gct_gc_identityhash(self, hop): v_obj = hop.spaceop.args[0] v_adr = hop.genop("cast_ptr_to_adr", [v_obj], Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/framework.py Thu Dec 10 17:07:58 2009 @@ -785,8 +785,8 @@ def gct_gc_writebarrier_before_copy(self, hop): if not hasattr(self, 'wb_before_copy_ptr'): - # should do nothing - return GCTransformer.gct_gc_writebarrier_before_copy(self, hop) + # no write barrier needed in that case + return rmodel.inputconst(lltype.Bool, True) op = hop.spaceop source_addr = hop.genop('cast_ptr_to_adr', [op.args[0]], resulttype=llmemory.Address) Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/refcounting.py Thu Dec 10 17:07:58 2009 @@ -159,11 +159,6 @@ lltype.typeOf(dealloc_fptr), dealloc_fptr) llops.genop("direct_call", [self.decref_ptr, v_adr, cdealloc_fptr]) - def gct_gc_writebarrier_before_copy(self, hop): - # Not supported, so return False. Means that rgc.ll_arraycopy() - # will do the copy manually (with a 'for' loop). - return rmodel.inputconst(lltype.Bool, False) - def gct_fv_gc_malloc(self, hop, flags, TYPE, c_size): v_raw = hop.genop("direct_call", [self.malloc_fixedsize_ptr, c_size], resulttype=llmemory.Address) Modified: pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gctransform/transform.py Thu Dec 10 17:07:58 2009 @@ -381,9 +381,11 @@ pass def gct_gc_writebarrier_before_copy(self, hop): - # by default we don't need to do anything special with this, - # for exception see refcounting - return rmodel.inputconst(lltype.Bool, True) + # We take the conservative default and return False here, meaning + # that rgc.ll_arraycopy() will do the copy by hand (i.e. with a + # 'for' loop). Subclasses that have their own logic, or that don't + # need any kind of write barrier, may return True. + return rmodel.inputconst(lltype.Bool, False) def gct_gc_identityhash(self, hop): # must be implemented in the various GCs From arigo at codespeak.net Thu Dec 10 17:10:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 10 Dec 2009 17:10:42 +0100 (CET) Subject: [pypy-svn] r70051 - pypy/branch/listcopyop/pypy/rpython/memory/gc Message-ID: <20091210161042.8F9A3168022@codespeak.net> Author: arigo Date: Thu Dec 10 17:10:41 2009 New Revision: 70051 Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Log: Cleanups. Modified: pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/listcopyop/pypy/rpython/memory/gc/generation.py Thu Dec 10 17:10:41 2009 @@ -5,7 +5,7 @@ from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE -from pypy.rlib.objectmodel import free_non_gc_object, specialize +from pypy.rlib.objectmodel import free_non_gc_object from pypy.rlib.debug import ll_assert from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.lltypesystem.lloperation import llop @@ -488,8 +488,6 @@ one of the following flags a bit too eagerly, which means we'll have a bit more objects to track, but being on the safe side. """ - #typeid = self.get_type_id(source_addr) - #assert self.is_gcarrayofgcptr(typeid) ...? source_hdr = self.header(source_addr) dest_hdr = self.header(dest_addr) if dest_hdr.tid & GCFLAG_NO_YOUNG_PTRS == 0: From arigo at codespeak.net Thu Dec 10 18:02:58 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 10 Dec 2009 18:02:58 +0100 (CET) Subject: [pypy-svn] r70052 - pypy/branch/listcopyop/pypy/rpython/lltypesystem Message-ID: <20091210170258.713A3168024@codespeak.net> Author: arigo Date: Thu Dec 10 18:02:58 2009 New Revision: 70052 Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py Log: Fix two issues: * In case of list shrinking, don't use new_allocated, but newsize. This avoids putting random items in the extra space. (I think a test could show the problem if needed.) * Don't call ll_arraycopy() at all if there were no items previously, to avoid copying the GC flags of the prebuilt empty array. Modified: pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/branch/listcopyop/pypy/rpython/lltypesystem/rlist.py Thu Dec 10 18:02:58 2009 @@ -207,14 +207,17 @@ except OverflowError: raise MemoryError # XXX consider to have a real realloc + # new_allocated is a bit more than newsize, enough to ensure an amortized + # linear complexity for e.g. repeated usage of l.append(). items = l.items newitems = malloc(typeOf(l).TO.items.TO, new_allocated) before_len = l.length - if before_len < new_allocated: - p = before_len - else: - p = new_allocated - rgc.ll_arraycopy(items, newitems, 0, 0, p) + if before_len: # avoids copying GC flags from the prebuilt_empty_array + if before_len < newsize: + p = before_len + else: + p = newsize + rgc.ll_arraycopy(items, newitems, 0, 0, p) l.length = newsize l.items = newitems _ll_list_resize_really._annenforceargs_ = (None, int) From afa at codespeak.net Thu Dec 10 18:05:45 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 10 Dec 2009 18:05:45 +0100 (CET) Subject: [pypy-svn] r70053 - in pypy/branch/import-builtin/pypy: interpreter module/__builtin__ module/__builtin__/test module/__pypy__ module/imp module/imp/test Message-ID: <20091210170545.09A2F168024@codespeak.net> Author: afa Date: Thu Dec 10 18:05:42 2009 New Revision: 70053 Added: pypy/branch/import-builtin/pypy/module/imp/ (props changed) pypy/branch/import-builtin/pypy/module/imp/__init__.py (contents, props changed) pypy/branch/import-builtin/pypy/module/imp/importing.py - copied, changed from r70029, pypy/branch/import-builtin/pypy/module/__builtin__/importing.py pypy/branch/import-builtin/pypy/module/imp/test/ (props changed) pypy/branch/import-builtin/pypy/module/imp/test/test_import.py - copied, changed from r70029, pypy/branch/import-builtin/pypy/module/__builtin__/test/test_import.py Removed: pypy/branch/import-builtin/pypy/module/__builtin__/app_misc.py pypy/branch/import-builtin/pypy/module/__builtin__/importing.py pypy/branch/import-builtin/pypy/module/__builtin__/test/test_import.py Modified: pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py pypy/branch/import-builtin/pypy/module/__builtin__/__init__.py pypy/branch/import-builtin/pypy/module/__pypy__/__init__.py Log: A large rewrite of the import mechanisms, this ensures that different hooks are processed in the correct order, and the imp module correctly handles builtin modules. Modified: pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py Thu Dec 10 18:05:42 2009 @@ -421,16 +421,18 @@ "NOT_RPYTHON: only for initializing the space." from pypy.module.exceptions import Module - w_name_exceptions = self.wrap('exceptions') - self.exceptions_module = Module(self, w_name_exceptions) + w_name = self.wrap('exceptions') + self.exceptions_module = Module(self, w_name) + self.builtin_modules['exceptions'] = self.wrap(self.exceptions_module) from pypy.module.sys import Module w_name = self.wrap('sys') self.sys = Module(self, w_name) - w_modules = self.sys.get('modules') self.builtin_modules['sys'] = self.wrap(self.sys) - self.builtin_modules['exceptions'] = self.wrap(self.exceptions_module) + from pypy.module.imp import Module + w_name = self.wrap('imp') + self.builtin_modules['imp'] = self.wrap(Module(self, w_name)) from pypy.module.__builtin__ import Module w_name = self.wrap('__builtin__') @@ -439,7 +441,7 @@ self.builtin_modules['__builtin__'] = self.wrap(w_builtin) self.setitem(self.builtin.w_dict, self.wrap('__builtins__'), w_builtin) - bootstrap_modules = ['sys', '__builtin__', 'exceptions'] + bootstrap_modules = ['sys', 'imp', '__builtin__', 'exceptions'] installed_builtin_modules = bootstrap_modules[:] self.export_builtin_exceptions() @@ -511,6 +513,7 @@ def setup_builtin_modules(self): "NOT_RPYTHON: only for initializing the space." self.getbuiltinmodule('sys') + self.getbuiltinmodule('imp') self.getbuiltinmodule('__builtin__') for mod in self.builtin_modules.values(): mod.setup_after_space_initialization() Modified: pypy/branch/import-builtin/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/__builtin__/__init__.py (original) +++ pypy/branch/import-builtin/pypy/module/__builtin__/__init__.py Thu Dec 10 18:05:42 2009 @@ -1,6 +1,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter import module from pypy.interpreter.mixedmodule import MixedModule +import pypy.module.imp.importing # put builtins here that should be optimized somehow @@ -35,9 +36,6 @@ 'vars' : 'app_inspect.vars', 'dir' : 'app_inspect.dir', - '_find_module' : 'app_misc.find_module', - 'reload' : 'app_misc.reload', - '__filestub' : 'app_file_stub.file', } @@ -88,7 +86,8 @@ 'compile' : 'compiling.compile', 'eval' : 'compiling.eval', - '__import__' : 'importing.importhook', + '__import__' : 'pypy.module.imp.importing.importhook', + 'reload' : 'pypy.module.imp.importing.reload', 'range' : 'functional.range_int', 'xrange' : 'functional.W_XRange', Modified: pypy/branch/import-builtin/pypy/module/__pypy__/__init__.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/__pypy__/__init__.py (original) +++ pypy/branch/import-builtin/pypy/module/__pypy__/__init__.py Thu Dec 10 18:05:42 2009 @@ -1,7 +1,7 @@ # Package initialisation from pypy.interpreter.mixedmodule import MixedModule -from pypy.module.__builtin__.importing import get_pyc_magic +from pypy.module.imp.importing import get_pyc_magic class Module(MixedModule): appleveldefs = { Added: pypy/branch/import-builtin/pypy/module/imp/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/import-builtin/pypy/module/imp/__init__.py Thu Dec 10 18:05:42 2009 @@ -0,0 +1,179 @@ +from pypy.interpreter.mixedmodule import MixedModule + +names = ['C_BUILTIN', 'C_EXTENSION', 'IMP_HOOK', 'NullImporter', 'PKG_DIRECTORY', + 'PY_CODERESOURCE', 'PY_COMPILED', 'PY_FROZEN', 'PY_RESOURCE', 'PY_SOURCE', + 'SEARCH_ERROR', '__doc__', '__name__', '__package__', 'acquire_lock', 'find_module', + 'get_frozen_object', 'get_magic', 'get_suffixes', 'init_builtin', 'init_frozen', + 'is_builtin', 'is_frozen', 'load_compiled', 'load_dynamic', 'load_module', + 'load_package', 'load_source', 'lock_held', 'new_module', 'release_lock', 'reload'] + + +class Module(MixedModule): + """ + This module provides the components needed to build your own + __import__ function. + """ + interpleveldefs = { + 'PY_SOURCE': 'space.wrap(importing.PY_SOURCE)', + 'PY_COMPILED': 'space.wrap(importing.PY_COMPILED)', + 'PKG_DIRECTORY': 'space.wrap(importing.PKG_DIRECTORY)', + 'C_BUILTIN': 'space.wrap(importing.C_BUILTIN)', + + 'get_magic': 'interp_imp.get_magic', + 'find_module': 'interp_imp.find_module', + 'load_module': 'interp_imp.load_module', + 'new_module': 'interp_imp.new_module', + # XXX CPython testing hack: delegate to the real imp.get_magic + # get_magic = __import__('imp').get_magic + } + + appleveldefs = { + } + +# PyPy-specific interface +try: + import __pypy__ + def get_magic(): + """Return the magic number for .pyc or .pyo files.""" + import struct + return struct.pack(' (file, filename, (suffix, mode, type)) + Search for a module. If path is omitted or None, search for a + built-in, frozen or special module and continue search in sys.path. + The module name cannot contain '.'; to search for a submodule of a + package, pass the submodule name and the package's __path__. + """ + if path is None: + if name in sys.builtin_module_names: + return (None, name, ('', '', C_BUILTIN)) + path = sys.path + for base in path: + filename = os.path.join(base, name) + if os.path.isdir(filename): + return (None, filename, ('', '', PKG_DIRECTORY)) + for ext, mode, kind in get_suffixes(): + if os.path.exists(filename+ext): + return (file(filename+ext, mode), filename+ext, (ext, mode, kind)) + raise ImportError, 'No module named %s' % (name,) + + +def load_module(name, file, filename, description): + """Load a module, given information returned by find_module(). + The module name must include the full package name, if any. + """ + suffix, mode, type = description + + if type == PY_SOURCE: + return load_source(name, filename, file) + + if type == PKG_DIRECTORY: + initfilename = os.path.join(filename, '__init__.py') + module = sys.modules.setdefault(name, new_module(name)) + module.__name__ = name + module.__doc__ = None + module.__file__ = initfilename + module.__path__ = [filename] + execfile(initfilename, module.__dict__) + return module + + if type == C_BUILTIN: + module = __import__(name, {}, {}, None) + return module + if type == PY_COMPILED: + return load_compiled(name, filename, file) + raise ValueError, 'invalid description argument: %r' % (description,) + +def load_dynamic(name, *args, **kwds): + raise ImportError(name) + +def load_source(name, pathname, file=None): + autoopen = file is None + if autoopen: + file = open(pathname, 'U') + source = file.read() + if autoopen: + file.close() + co = compile(source, pathname, 'exec') + return run_module(name, pathname, co) + +def load_compiled(name, pathname, file=None): + import marshal + autoopen = file is None + if autoopen: + file = open(pathname, 'rb') + magic = file.read(4) + if magic != get_magic(): + raise ImportError("Bad magic number in %s" % pathname) + file.read(4) # skip timestamp + co = marshal.load(file) + if autoopen: + file.close() + return run_module(name, pathname, co) + +def run_module(name, pathname, co): + module = sys.modules.setdefault(name, new_module(name)) + module.__name__ = name + module.__doc__ = None + module.__file__ = pathname + try: + exec co in module.__dict__ + except : + sys.modules.pop(name,None) + raise + return module + + +def new_module(name): + """Create a new module. Do not enter it in sys.modules. + The module name must include the full package name, if any. + """ + return new.module(name) + + +def init_builtin(name): + if name not in sys.builtin_module_names: + return None + if name in sys.modules: + raise ImportError("cannot initialize a built-in module twice " + "in PyPy") + return __import__(name) + +def init_frozen(name): + return None + +def is_builtin(name): + if name in sys.builtin_module_names: + return -1 # cannot be initialized again + else: + return 0 + +def is_frozen(name): + return False + +# ____________________________________________________________ + +try: + # PyPy-specific interface + from thread import _importlock_held as lock_held + from thread import _importlock_acquire as acquire_lock + from thread import _importlock_release as release_lock +except ImportError: + def lock_held(): + """On platforms without threads, return False.""" + return False + def acquire_lock(): + """On platforms without threads, this function does nothing.""" + def release_lock(): + """On platforms without threads, this function does nothing.""" Copied: pypy/branch/import-builtin/pypy/module/imp/importing.py (from r70029, pypy/branch/import-builtin/pypy/module/__builtin__/importing.py) ============================================================================== --- pypy/branch/import-builtin/pypy/module/__builtin__/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Thu Dec 10 18:05:42 2009 @@ -14,14 +14,16 @@ from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import we_are_translated -NOFILE = 0 -PYFILE = 1 -PYCFILE = 2 +SEARCH_ERROR = 0 +PY_SOURCE = 1 +PY_COMPILED = 2 +PKG_DIRECTORY = 3 +C_BUILTIN = 4 def find_modtype(space, filepart): """Check which kind of module to import for the given filepart, - which is a path without extension. Returns PYFILE, PYCFILE or - NOFILE. + which is a path without extension. Returns PY_SOURCE, PY_COMPILED or + SEARCH_ERROR. """ # check the .py file pyfile = filepart + ".py" @@ -33,22 +35,23 @@ # is False: if a .py file does not exist, we don't even try to # look for a lone .pyc file. if not space.config.objspace.lonepycfiles: - return NOFILE + return SEARCH_ERROR, None, None pyfile_ts = 0 pyfile_exists = False # check the .pyc file if space.config.objspace.usepycfiles: - pycfile = filepart + ".pyc" + pycfile = filepart + ".pyc" if case_ok(pycfile): if check_compiled_module(space, pycfile, pyfile_ts): - return PYCFILE # existing and up-to-date .pyc file + # existing and up-to-date .pyc file + return PY_COMPILED, ".pyc", "rb" # no .pyc file, use the .py file if it exists if pyfile_exists: - return PYFILE + return PY_SOURCE, ".py", "rU" else: - return NOFILE + return SEARCH_ERROR, None, None if sys.platform in ['linux2', 'freebsd']: def case_ok(filename): @@ -67,61 +70,6 @@ except OSError: return False -def _prepare_module(space, w_mod, filename, pkgdir): - w = space.wrap - space.sys.setmodule(w_mod) - space.setattr(w_mod, w('__file__'), space.wrap(filename)) - space.setattr(w_mod, w('__doc__'), space.w_None) - if pkgdir is not None: - space.setattr(w_mod, w('__path__'), space.newlist([w(pkgdir)])) - -def try_import_mod(space, w_modulename, filepart, w_parent, w_name, pkgdir=None): - - # decide what type we want (pyc/py) - modtype = find_modtype(space, filepart) - - if modtype == NOFILE: - return None - - w = space.wrap - w_mod = w(Module(space, w_modulename)) - - try: - if modtype == PYFILE: - filename = filepart + ".py" - stream = streamio.open_file_as_stream(filename, "rU") - else: - assert modtype == PYCFILE - filename = filepart + ".pyc" - stream = streamio.open_file_as_stream(filename, "rb") - - try: - _prepare_module(space, w_mod, filename, pkgdir) - try: - if modtype == PYFILE: - load_source_module(space, w_modulename, w_mod, filename, - stream.readall()) - else: - magic = _r_long(stream) - timestamp = _r_long(stream) - load_compiled_module(space, w_modulename, w_mod, filename, - magic, timestamp, stream.readall()) - - except OperationError, e: - w_mods = space.sys.get('modules') - space.call_method(w_mods,'pop', w_modulename, space.w_None) - raise - finally: - stream.close() - - except StreamErrors: - return None - - w_mod = check_sys_modules(space, w_modulename) - if w_mod is not None and w_parent is not None: - space.setattr(w_parent, w_name, w_mod) - return w_mod - def try_getattr(space, w_obj, w_name): try: return space.getattr(w_obj, w_name) @@ -267,8 +215,140 @@ else: return first +def find_in_meta_path(space, w_modulename, w_path): + assert w_modulename is not None + for w_hook in space.unpackiterable(space.sys.get("meta_path")): + w_loader = space.call_method(w_hook, "find_module", + w_modulename, w_path) + if space.is_true(w_loader): + return w_loader + +def find_in_path_hooks(space, w_modulename, w_pathitem): + w_path_importer_cache = space.sys.get("path_importer_cache") + try: + w_importer = space.getitem(w_path_importer_cache, w_pathitem) + except OperationError, e: + if not e.match(space, space.w_KeyError): + raise + w_importer = space.w_None + space.setitem(w_path_importer_cache, w_pathitem, w_importer) + for w_hook in space.unpackiterable(space.sys.get("path_hooks")): + try: + w_importer = space.call_function(w_hook, w_pathitem) + except OperationError, e: + if not e.match(space, space.w_ImportError): + raise + else: + break + if space.is_true(w_importer): + space.setitem(w_path_importer_cache, w_pathitem, w_importer) + if space.is_true(w_importer): + w_loader = space.call_method(w_importer, "find_module", w_modulename) + if space.is_true(w_loader): + return w_loader + +def ImportInfo(type, name, stream, suffix="", filemode=""): + return type, name, stream, suffix, filemode + +def find_module(space, modulename, w_modulename, partname, w_path, use_loader=True): + """If use_loader is False, returns (import_info, None) + if use_loader is True, may return (None, w_loader)""" + # Examin importhooks (PEP302) before doing the import + if use_loader: + w_loader = find_in_meta_path(space, w_modulename, w_path) + if w_loader: + return None, w_loader + + # XXX Check for frozen modules? + # when w_path is a string + + # check the builtin modules + if modulename in space.builtin_modules: + return ImportInfo(C_BUILTIN, modulename, None), None + + # XXX check frozen modules? + # when w_path is null + + if w_path is None: + w_path = space.sys.get("path") + + for w_pathitem in space.unpackiterable(w_path): + # sys.path_hooks import hook + if use_loader: + w_loader = find_in_path_hooks(space, w_modulename, w_pathitem) + if w_loader: + return None, w_loader + + path = space.str_w(w_pathitem) + filepart = os.path.join(path, partname) + if os.path.isdir(filepart) and case_ok(filepart): + initfile = os.path.join(filepart, '__init__') + modtype, _, _ = find_modtype(space, initfile) + if modtype in (PY_SOURCE, PY_COMPILED): + return ImportInfo(PKG_DIRECTORY, filepart, None), None + else: + msg = "Not importing directory " +\ + "'%s' missing __init__.py" % (filepart,) + space.warn(msg, space.w_ImportWarning) + modtype, suffix, filemode = find_modtype(space, filepart) + try: + if modtype in (PY_SOURCE, PY_COMPILED): + filename = filepart + suffix + stream = streamio.open_file_as_stream(filename, filemode) + try: + return ImportInfo(modtype, filename, stream, suffix, filemode), None + except: + stream.close() + raise + except StreamErrors: + pass + return None, None + +def load_module(space, w_modulename, import_info, w_loader, ispkg=False): + if w_loader: + return space.call_method(w_loader, "load_module", w_modulename) + if import_info is None: + return + modtype, filename, stream, _, _ = import_info + if modtype == C_BUILTIN: + return space.getbuiltinmodule(filename) + + if modtype in (PY_SOURCE, PY_COMPILED, PKG_DIRECTORY): + if ispkg: + w_mod = space.getitem(space.sys.get('modules'), w_modulename) + else: + w_mod = space.wrap(Module(space, w_modulename)) + space.sys.setmodule(w_mod) + space.setattr(w_mod, space.wrap('__file__'), space.wrap(filename)) + space.setattr(w_mod, space.wrap('__doc__'), space.w_None) + + try: + if modtype == PY_SOURCE: + load_source_module(space, w_modulename, w_mod, filename, + stream.readall()) + return w_mod + elif modtype == PY_COMPILED: + magic = _r_long(stream) + timestamp = _r_long(stream) + load_compiled_module(space, w_modulename, w_mod, filename, + magic, timestamp, stream.readall()) + return w_mod + elif modtype == PKG_DIRECTORY: + w_path = space.newlist([space.wrap(filename)]) + space.setattr(w_mod, space.wrap('__path__'), w_path) + import_info, _ = find_module(space, "__init__", None, "__init__", + w_path, use_loader=False) + if import_info is None: + return w_mod + load_module(space, w_modulename, import_info, None, ispkg=True) + w_mod = check_sys_modules(space, w_modulename) # in case of "substitution" + return w_mod + except OperationError: + w_mods = space.sys.get('modules') + space.call_method(w_mods, 'pop', w_modulename, space.w_None) + raise + def load_part(space, w_path, prefix, partname, w_parent, tentative): - w_find_module = space.getattr(space.builtin, space.wrap("_find_module")) w = space.wrap modulename = '.'.join(prefix + [partname]) w_modulename = w(modulename) @@ -277,44 +357,20 @@ if not space.is_w(w_mod, space.w_None): return w_mod else: - # Examin importhooks (PEP302) before doing the import - if w_path is not None: - w_loader = space.call_function(w_find_module, w_modulename, w_path) - else: - w_loader = space.call_function(w_find_module, w_modulename, - space.w_None) - if not space.is_w(w_loader, space.w_None): - w_mod = space.call_method(w_loader, "load_module", w_modulename) - #w_mod_ = check_sys_modules(space, w_modulename) - if w_mod is not None and w_parent is not None: - space.setattr(w_parent, w(partname), w_mod) - - return w_mod - - # check the builtin modules - if modulename in space.builtin_modules: - return space.getbuiltinmodule(modulename) + import_info, w_loader = find_module( + space, modulename, w_modulename, partname, w_path) - if w_path is not None: - for path in space.unpackiterable(w_path): - path = space.str_w(path) - dir = os.path.join(path, partname) - if os.path.isdir(dir) and case_ok(dir): - fn = os.path.join(dir, '__init__') - w_mod = try_import_mod(space, w_modulename, fn, - w_parent, w(partname), - pkgdir=dir) - if w_mod is not None: - return w_mod - else: - msg = "Not importing directory " +\ - "'%s' missing __init__.py" % dir - space.warn(msg, space.w_ImportWarning) - fn = dir - w_mod = try_import_mod(space, w_modulename, fn, w_parent, - w(partname)) - if w_mod is not None: - return w_mod + try: + if (import_info, w_loader) != (None, None): + w_mod = load_module(space, w_modulename, import_info, w_loader) + if w_parent is not None: + space.setattr(w_parent, space.wrap(partname), w_mod) + return w_mod + finally: + if import_info: + _, _, stream, _, _ = import_info + if stream: + stream.close() if tentative: return None @@ -323,6 +379,45 @@ msg = "No module named %s" % modulename raise OperationError(space.w_ImportError, w(msg)) +def reload(space, w_module): + """Reload the module. + The module must have been successfully imported before.""" + if not space.is_w(space.type(w_module), space.type(space.sys)): + raise OperationError( + space.w_TypeError, + space.wrap("reload() argument must be module")) + + w_modulename = space.getattr(w_module, space.wrap("__name__")) + modulename = space.str_w(w_modulename) + if not space.is_w(check_sys_modules(space, w_modulename), w_module): + raise OperationError( + space.w_ImportError, + space.wrap("reload(): module %s not in sys.modules" % (modulename,))) + + namepath = modulename.split('.') + subname = namepath[-1] + parent_name = '.'.join(namepath[:-1]) + parent = None + w_path = None + if parent_name: + w_parent = check_sys_modules(space, space.wrap(parent_name)) + if w_parent is None: + raise OperationError( + space.w_ImportError, + space.wrap("reload(): parent %s not in sys.modules" % ( + parent_name,))) + w_path = space.getitem(w_parent, space.wrap("__path")) + + import_info, w_loader = find_module( + space, modulename, w_modulename, subname, w_path) + + if (import_info, w_loader) == (None, None): + # ImportError + msg = "No module named %s" % modulename + raise OperationError(space.w_ImportError, space.wrap(msg)) + + return load_module(space, w_modulename, import_info, w_loader) + # __________________________________________________________________ # # import lock, to prevent two threads from running module-level code in Copied: pypy/branch/import-builtin/pypy/module/imp/test/test_import.py (from r70029, pypy/branch/import-builtin/pypy/module/__builtin__/test/test_import.py) ============================================================================== --- pypy/branch/import-builtin/pypy/module/__builtin__/test/test_import.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Thu Dec 10 18:05:42 2009 @@ -9,7 +9,7 @@ import sys, os import tempfile, marshal -from pypy.module.__builtin__ import importing +from pypy.module.imp import importing from pypy import conftest @@ -64,6 +64,7 @@ ) setuppkg("pkg_substituting", __init__ = "import sys, pkg_substituted\n" + "print 'TOTO', __name__\n" "sys.modules[__name__] = pkg_substituted") setuppkg("pkg_substituted", mod='') p = setuppkg("readonly", x='') @@ -726,12 +727,13 @@ def __init__(self, *namestoblock): self.namestoblock = dict.fromkeys(namestoblock) def find_module(self, fullname, path=None): + print "AFA FIND_MODULE", fullname if fullname in self.namestoblock: return self def load_module(self, fullname): raise ImportError, "blocked" - import sys + import sys, imp modname = "errno" # an arbitrary harmless builtin module mod = None if modname in sys.modules: @@ -740,6 +742,10 @@ sys.meta_path.append(ImportBlocker(modname)) try: raises(ImportError, __import__, modname) + # the imp module doesn't use meta_path, and is not blocked + # (until imp.get_loader is implemented, see PEP302) + file, filename, stuff = imp.find_module(modname) + imp.load_module(modname, file, filename, stuff) finally: sys.meta_path.pop() if mod: @@ -774,6 +780,64 @@ sys.path.pop(0) sys.path_hooks.pop() + def test_imp_wrapper(self): + import sys, os, imp + class ImpWrapper: + + def __init__(self, path=None): + if path is not None and not os.path.isdir(path): + raise ImportError + self.path = path + + def find_module(self, fullname, path=None): + subname = fullname.split(".")[-1] + if subname != fullname and self.path is None: + return None + if self.path is None: + path = None + else: + path = [self.path] + try: + file, filename, stuff = imp.find_module(subname, path) + except ImportError, e: + return None + return ImpLoader(file, filename, stuff) + + class ImpLoader: + + def __init__(self, file, filename, stuff): + self.file = file + self.filename = filename + self.stuff = stuff + + def load_module(self, fullname): + mod = imp.load_module(fullname, self.file, self.filename, self.stuff) + if self.file: + self.file.close() + mod.__loader__ = self # for introspection + return mod + + i = ImpWrapper() + sys.meta_path.append(i) + sys.path_hooks.append(ImpWrapper) + sys.path_importer_cache.clear() + mnames = ("colorsys", "urlparse", "distutils.core", "compiler.misc") + for mname in mnames: + parent = mname.split(".")[0] + for n in sys.modules.keys(): + if n.startswith(parent): + del sys.modules[n] + for mname in mnames: + m = __import__(mname, globals(), locals(), ["__dummy__"]) + m.__loader__ # to make sure we actually handled the import + # Delete urllib from modules because urlparse was imported above. + # Without this hack, test_socket_ssl fails if run in this order: + # regrtest.py test_codecmaps_tw test_importhooks test_socket_ssl + try: + del sys.modules['urllib'] + except KeyError: + pass + class AppTestNoPycFile(object): usepycfiles = False lonepycfiles = False From afa at codespeak.net Thu Dec 10 19:13:17 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 10 Dec 2009 19:13:17 +0100 (CET) Subject: [pypy-svn] r70054 - in pypy/branch/import-builtin/pypy: lib lib/app_test module/imp module/imp/test Message-ID: <20091210181317.29FFD168024@codespeak.net> Author: afa Date: Thu Dec 10 19:13:16 2009 New Revision: 70054 Added: pypy/branch/import-builtin/pypy/module/imp/test/test_app.py Removed: pypy/branch/import-builtin/pypy/lib/app_test/test_imp_extra.py pypy/branch/import-builtin/pypy/lib/imp.py Modified: pypy/branch/import-builtin/pypy/module/imp/__init__.py pypy/branch/import-builtin/pypy/module/imp/importing.py Log: Migrate tests for the imp module, and implement missing functions Modified: pypy/branch/import-builtin/pypy/module/imp/__init__.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/__init__.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/__init__.py Thu Dec 10 19:13:16 2009 @@ -18,13 +18,19 @@ 'PY_COMPILED': 'space.wrap(importing.PY_COMPILED)', 'PKG_DIRECTORY': 'space.wrap(importing.PKG_DIRECTORY)', 'C_BUILTIN': 'space.wrap(importing.C_BUILTIN)', + 'get_suffixes': 'interp_imp.get_suffixes', 'get_magic': 'interp_imp.get_magic', 'find_module': 'interp_imp.find_module', 'load_module': 'interp_imp.load_module', + 'load_source': 'interp_imp.load_source', + 'load_compiled': 'interp_imp.load_compiled', + #'run_module': 'interp_imp.run_module', 'new_module': 'interp_imp.new_module', - # XXX CPython testing hack: delegate to the real imp.get_magic - # get_magic = __import__('imp').get_magic + 'init_builtin': 'interp_imp.init_builtin', + 'init_frozen': 'interp_imp.init_frozen', + 'is_builtin': 'interp_imp.is_builtin', + 'is_frozen': 'interp_imp.is_frozen', } appleveldefs = { Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Thu Dec 10 19:13:16 2009 @@ -49,7 +49,7 @@ # no .pyc file, use the .py file if it exists if pyfile_exists: - return PY_SOURCE, ".py", "rU" + return PY_SOURCE, ".py", "U" else: return SEARCH_ERROR, None, None @@ -269,39 +269,37 @@ # XXX check frozen modules? # when w_path is null - if w_path is None: - w_path = space.sys.get("path") - - for w_pathitem in space.unpackiterable(w_path): - # sys.path_hooks import hook - if use_loader: - w_loader = find_in_path_hooks(space, w_modulename, w_pathitem) - if w_loader: - return None, w_loader - - path = space.str_w(w_pathitem) - filepart = os.path.join(path, partname) - if os.path.isdir(filepart) and case_ok(filepart): - initfile = os.path.join(filepart, '__init__') - modtype, _, _ = find_modtype(space, initfile) - if modtype in (PY_SOURCE, PY_COMPILED): - return ImportInfo(PKG_DIRECTORY, filepart, None), None - else: - msg = "Not importing directory " +\ - "'%s' missing __init__.py" % (filepart,) - space.warn(msg, space.w_ImportWarning) - modtype, suffix, filemode = find_modtype(space, filepart) - try: - if modtype in (PY_SOURCE, PY_COMPILED): - filename = filepart + suffix - stream = streamio.open_file_as_stream(filename, filemode) - try: - return ImportInfo(modtype, filename, stream, suffix, filemode), None - except: - stream.close() - raise - except StreamErrors: - pass + if w_path is not None: + for w_pathitem in space.unpackiterable(w_path): + # sys.path_hooks import hook + if use_loader: + w_loader = find_in_path_hooks(space, w_modulename, w_pathitem) + if w_loader: + return None, w_loader + + path = space.str_w(w_pathitem) + filepart = os.path.join(path, partname) + if os.path.isdir(filepart) and case_ok(filepart): + initfile = os.path.join(filepart, '__init__') + modtype, _, _ = find_modtype(space, initfile) + if modtype in (PY_SOURCE, PY_COMPILED): + return ImportInfo(PKG_DIRECTORY, filepart, None), None + else: + msg = "Not importing directory " +\ + "'%s' missing __init__.py" % (filepart,) + space.warn(msg, space.w_ImportWarning) + modtype, suffix, filemode = find_modtype(space, filepart) + try: + if modtype in (PY_SOURCE, PY_COMPILED): + filename = filepart + suffix + stream = streamio.open_file_as_stream(filename, filemode) + try: + return ImportInfo(modtype, filename, stream, suffix, filemode), None + except: + stream.close() + raise + except StreamErrors: + pass return None, None def load_module(space, w_modulename, import_info, w_loader, ispkg=False): @@ -524,6 +522,13 @@ MARSHAL_VERSION_FOR_PYC = 2 def get_pyc_magic(space): + # XXX CPython testing hack: delegate to the real imp.get_magic + if not we_are_translated(): + if '__pypy__' not in space.builtin_modules: + import struct + magic = __import__('imp').get_magic() + return struct.unpack(' Author: pedronis Date: Thu Dec 10 19:17:55 2009 New Revision: 70055 Added: pypy/branch/fixed-fail-boxes/ - copied from r70054, pypy/trunk/ Log: branch to run the all jit tests after having removing the support for growing fail boxes lists in x86 From arigo at codespeak.net Thu Dec 10 19:19:07 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 10 Dec 2009 19:19:07 +0100 (CET) Subject: [pypy-svn] r70056 - pypy/build/bot2/pypybuildbot Message-ID: <20091210181907.ED77C168024@codespeak.net> Author: arigo Date: Thu Dec 10 19:19:07 2009 New Revision: 70056 Modified: pypy/build/bot2/pypybuildbot/summary.py Log: Change the target of the link behind the 2nd part of the titles. E.g. in "{own} branch/listcopyop": * clicking "{own}" shows all entries about "{own}" (as before); * clicking "branch/listcopyop" shows all entries about "branch/listcopyop" (new behavior). Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Thu Dec 10 19:19:07 2009 @@ -342,15 +342,14 @@ branch = trunk_name(branch) category = category_name(category) - cat_branch = self.cur_cat_branch = (category, branch) + self.cur_cat_branch = (category, branch) cat_anchor = html.a("{%s}" % category, href="/summary?category=%s" % category, class_="failSummary branch") branch_anchor = html.a(branch, - href="/summary?category=%s&branch=%s" % - cat_branch, + href="/summary?branch=%s" % branch, class_="failSummary branch") if fine: extra = html.img(alt=":-)", src="success.png") From pedronis at codespeak.net Thu Dec 10 19:19:50 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 10 Dec 2009 19:19:50 +0100 (CET) Subject: [pypy-svn] r70057 - in pypy/branch/fixed-fail-boxes/pypy/jit: backend/cli backend/llgraph backend/llsupport backend/llsupport/test backend/x86 backend/x86/test metainterp metainterp/test Message-ID: <20091210181950.BDA51168024@codespeak.net> Author: pedronis Date: Thu Dec 10 19:19:50 2009 New Revision: 70057 Modified: pypy/branch/fixed-fail-boxes/pypy/jit/backend/cli/runner.py pypy/branch/fixed-fail-boxes/pypy/jit/backend/llgraph/runner.py pypy/branch/fixed-fail-boxes/pypy/jit/backend/llsupport/llmodel.py pypy/branch/fixed-fail-boxes/pypy/jit/backend/llsupport/test/test_runner.py pypy/branch/fixed-fail-boxes/pypy/jit/backend/x86/assembler.py pypy/branch/fixed-fail-boxes/pypy/jit/backend/x86/runner.py pypy/branch/fixed-fail-boxes/pypy/jit/backend/x86/support.py pypy/branch/fixed-fail-boxes/pypy/jit/backend/x86/test/test_support.py pypy/branch/fixed-fail-boxes/pypy/jit/metainterp/test/test_basic.py pypy/branch/fixed-fail-boxes/pypy/jit/metainterp/warmspot.py Log: remove support for growing fail boxes lists Modified: pypy/branch/fixed-fail-boxes/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/branch/fixed-fail-boxes/pypy/jit/backend/cli/runner.py (original) +++ pypy/branch/fixed-fail-boxes/pypy/jit/backend/cli/runner.py Thu Dec 10 19:19:50 2009 @@ -43,7 +43,7 @@ supports_floats = True ts = oohelper - def __init__(self, rtyper, stats, translate_support_code=False, + def __init__(self, rtyper, stats, opts=None, translate_support_code=False, mixlevelann=None, gcdescr=None): model.AbstractCPU.__init__(self) self.rtyper = rtyper Modified: pypy/branch/fixed-fail-boxes/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/fixed-fail-boxes/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/fixed-fail-boxes/pypy/jit/backend/llgraph/runner.py Thu Dec 10 19:19:50 2009 @@ -74,8 +74,9 @@ class BaseCPU(model.AbstractCPU): supports_floats = True - def __init__(self, rtyper, stats=None, translate_support_code=False, + def __init__(self, rtyper, stats=None, opts=None, translate_support_code=False, annmixlevel=None, gcdescr=None): + assert type(opts) is not bool model.AbstractCPU.__init__(self) self.rtyper = rtyper self.translate_support_code = translate_support_code Modified: pypy/branch/fixed-fail-boxes/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/fixed-fail-boxes/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/fixed-fail-boxes/pypy/jit/backend/llsupport/llmodel.py Thu Dec 10 19:19:50 2009 @@ -17,8 +17,11 @@ class AbstractLLCPU(AbstractCPU): from pypy.jit.metainterp.typesystem import llhelper as ts - def __init__(self, rtyper, stats, translate_support_code=False, + def __init__(self, rtyper, stats, opts, translate_support_code=False, gcdescr=None): + assert type(opts) is not bool + self.opts = opts + from pypy.jit.backend.llsupport.gc import get_ll_description AbstractCPU.__init__(self) self.rtyper = rtyper Modified: pypy/branch/fixed-fail-boxes/pypy/jit/backend/llsupport/test/test_runner.py ============================================================================== --- pypy/branch/fixed-fail-boxes/pypy/jit/backend/llsupport/test/test_runner.py (original) +++ pypy/branch/fixed-fail-boxes/pypy/jit/backend/llsupport/test/test_runner.py Thu Dec 10 19:19:50 2009 @@ -18,4 +18,4 @@ # ====> ../../test/runner_test.py def setup_class(cls): - cls.cpu = MyLLCPU(rtyper=None, stats=FakeStats()) + cls.cpu = MyLLCPU(rtyper=None, stats=FakeStats(), opts=None) Modified: pypy/branch/fixed-fail-boxes/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/fixed-fail-boxes/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/fixed-fail-boxes/pypy/jit/backend/x86/assembler.py Thu Dec 10 19:19:50 2009 @@ -15,9 +15,7 @@ from pypy.jit.backend.x86 import codebuf from pypy.jit.backend.x86.ri386 import * from pypy.jit.metainterp.resoperation import rop -from pypy.jit.backend.x86.support import NonmovableGrowableArrayFloat,\ - NonmovableGrowableArraySigned, NonmovableGrowableArrayGCREF,\ - CHUNK_SIZE +from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print # our calling convention - we pass first 6 args in registers @@ -78,7 +76,8 @@ mc2 = None mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE - def __init__(self, cpu, translate_support_code=False): + def __init__(self, cpu, translate_support_code=False, + failargs_limit=1000): self.cpu = cpu self.verbose = False self.rtyper = cpu.rtyper @@ -86,21 +85,18 @@ self.malloc_array_func_addr = 0 self.malloc_str_func_addr = 0 self.malloc_unicode_func_addr = 0 - self.fail_boxes_int = NonmovableGrowableArraySigned() - self.fail_boxes_ptr = NonmovableGrowableArrayGCREF() - self.fail_boxes_float = NonmovableGrowableArrayFloat() + self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) + self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) + self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 self.loc_float_const_neg = None self.loc_float_const_abs = None self.setup_failure_recovery() def leave_jitted_hook(self): - i = 0 - while i < self.fail_boxes_ptr.lgt: - chunk = self.fail_boxes_ptr.chunks[i] - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(chunk)) - i += 1 + ptrs = self.fail_boxes_ptr.ar + llop.gc_assume_young_pointers(lltype.Void, + llmemory.cast_ptr_to_adr(ptrs)) def make_sure_mc_exists(self): if self.mc is None: Modified: pypy/branch/fixed-fail-boxes/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/fixed-fail-boxes/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/fixed-fail-boxes/pypy/jit/backend/x86/runner.py Thu Dec 10 19:19:50 2009 @@ -16,14 +16,18 @@ BOOTSTRAP_TP = lltype.FuncType([], lltype.Signed) dont_keepalive_stuff = False # for tests - def __init__(self, rtyper, stats, translate_support_code=False, + def __init__(self, rtyper, stats, opts=None, translate_support_code=False, gcdescr=None): - AbstractLLCPU.__init__(self, rtyper, stats, translate_support_code, + AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) - self._bootstrap_cache = {} def setup(self): - self.assembler = Assembler386(self, self.translate_support_code) + if self.opts is not None: + failargs_limit = self.opts.failargs_limit + else: + failargs_limit = 1000 + self.assembler = Assembler386(self, self.translate_support_code, + failargs_limit) def get_on_leave_jitted_hook(self): return self.assembler.leave_jitted_hook Modified: pypy/branch/fixed-fail-boxes/pypy/jit/backend/x86/support.py ============================================================================== --- pypy/branch/fixed-fail-boxes/pypy/jit/backend/x86/support.py (original) +++ pypy/branch/fixed-fail-boxes/pypy/jit/backend/x86/support.py Thu Dec 10 19:19:50 2009 @@ -1,53 +1,23 @@ - from pypy.rpython.lltypesystem import lltype, rffi, llmemory -from pypy.rlib import rgc -from pypy.rlib.objectmodel import we_are_translated - -CHUNK_SIZE_BITS = 8 -CHUNK_SIZE = 1 << CHUNK_SIZE_BITS -def new_nonmovable_growable_array(TP): +def values_array(TP, size): ATP = lltype.GcArray(TP) - class NonmovableGrowableArray(object): + class ValuesArray(object): def __init__(self): - self.chunks = [] - self.lgt = 0 - - def _grow(self): - # XXX workaround for a fact that rgc.malloc_nonmovable always - # returns nullptr when run on top of python - if we_are_translated(): - new_item = rgc.malloc_nonmovable(ATP, CHUNK_SIZE, zero=True) - else: - new_item = lltype.malloc(ATP, CHUNK_SIZE, zero=True) - self.chunks.append(new_item) - self.lgt += 1 - _grow._dont_inline_ = True + self.ar = lltype.malloc(ATP, size, zero=True, immortal=True) def get_addr_for_num(self, i): - chunk_no, ofs = self._no_of(i) - chunk = self.chunks[chunk_no] - rffi.cast(lltype.Signed, chunk) return rffi.cast(lltype.Signed, lltype.direct_ptradd( - lltype.direct_arrayitems(chunk), ofs)) - - def _no_of(self, i): - while i >= len(self.chunks) * CHUNK_SIZE: - self._grow() - return i >> CHUNK_SIZE_BITS, i & (CHUNK_SIZE-1) - _no_of._always_inline_ = True + lltype.direct_arrayitems(self.ar), i)) def setitem(self, i, v): - chunk_no, ofs = self._no_of(i) - self.chunks[chunk_no][ofs] = v + self.ar[i] = v def getitem(self, i): - chunk_no, ofs = self._no_of(i) - return self.chunks[chunk_no][ofs] + return self.ar[i] - return NonmovableGrowableArray + def _freeze_(self): + return True -NonmovableGrowableArrayFloat = new_nonmovable_growable_array(lltype.Float) -NonmovableGrowableArraySigned = new_nonmovable_growable_array(lltype.Signed) -NonmovableGrowableArrayGCREF = new_nonmovable_growable_array(llmemory.GCREF) + return ValuesArray() Modified: pypy/branch/fixed-fail-boxes/pypy/jit/backend/x86/test/test_support.py ============================================================================== --- pypy/branch/fixed-fail-boxes/pypy/jit/backend/x86/test/test_support.py (original) +++ pypy/branch/fixed-fail-boxes/pypy/jit/backend/x86/test/test_support.py Thu Dec 10 19:19:50 2009 @@ -1,27 +1,21 @@ -from pypy.jit.backend.x86.support import NonmovableGrowableArraySigned, CHUNK_SIZE +from pypy.jit.backend.x86.support import values_array from pypy.rpython.lltypesystem import lltype, llmemory, rffi -def test_nonmovable_growable_array(): - ar = NonmovableGrowableArraySigned() +def test_values_array_signed(): + ar = values_array(lltype.Signed, 50) adr = ar.get_addr_for_num(10) rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] = 42 assert ar.getitem(10) == 42 ar.setitem(42, 38) adr = ar.get_addr_for_num(42) assert rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] == 38 - adr = ar.get_addr_for_num(CHUNK_SIZE + 10) - rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] = 42 - assert ar.getitem(CHUNK_SIZE + 10) == 42 - ar.setitem(CHUNK_SIZE + 42, 38) - adr = ar.get_addr_for_num(CHUNK_SIZE + 42) - assert rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] == 38 - adr = ar.get_addr_for_num(3 * CHUNK_SIZE + 10) - rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] = 42 - assert ar.getitem(3 * CHUNK_SIZE + 10) == 42 - ar.setitem(3 * CHUNK_SIZE + 42, 38) - adr = ar.get_addr_for_num(3 * CHUNK_SIZE + 42) - assert rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] == 38 - ar.setitem(8*CHUNK_SIZE, 13) - assert ar.getitem(8*CHUNK_SIZE) == 13 - + +def test_values_array_float(): + ar = values_array(lltype.Float, 50) + adr = ar.get_addr_for_num(10) + rffi.cast(rffi.CArrayPtr(lltype.Float), adr)[0] = 42.5 + assert ar.getitem(10) == 42.5 + ar.setitem(42, 38.5) + adr = ar.get_addr_for_num(42) + assert rffi.cast(rffi.CArrayPtr(lltype.Float), adr)[0] == 38.5 Modified: pypy/branch/fixed-fail-boxes/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/fixed-fail-boxes/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/fixed-fail-boxes/pypy/jit/metainterp/test/test_basic.py Thu Dec 10 19:19:50 2009 @@ -21,7 +21,7 @@ rtyper = support.annotate(func, values, type_system=type_system) stats = history.Stats() - cpu = CPUClass(rtyper, stats, False) + cpu = CPUClass(rtyper, stats, None, False) graphs = rtyper.annotator.translator.graphs opt = history.Options(listops=listops) metainterp_sd = pyjitpl.MetaInterpStaticData(graphs[0], cpu, stats, opt) Modified: pypy/branch/fixed-fail-boxes/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/fixed-fail-boxes/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/fixed-fail-boxes/pypy/jit/metainterp/warmspot.py Thu Dec 10 19:19:50 2009 @@ -256,7 +256,7 @@ annhelper = self.annhelper else: annhelper = None - cpu = CPUClass(self.translator.rtyper, self.stats, + cpu = CPUClass(self.translator.rtyper, self.stats, opt, translate_support_code, gcdescr=self.gcdescr) self.cpu = cpu self.metainterp_sd = MetaInterpStaticData(self.portal_graph, # xxx From afa at codespeak.net Thu Dec 10 19:20:28 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 10 Dec 2009 19:20:28 +0100 (CET) Subject: [pypy-svn] r70058 - in pypy/branch/import-builtin/pypy/module: imp thread zipimport Message-ID: <20091210182028.5F7BD168024@codespeak.net> Author: afa Date: Thu Dec 10 19:20:26 2009 New Revision: 70058 Added: pypy/branch/import-builtin/pypy/module/imp/interp_imp.py Modified: pypy/branch/import-builtin/pypy/module/thread/importlock.py pypy/branch/import-builtin/pypy/module/zipimport/interp_zipimport.py Log: Fixes Added: pypy/branch/import-builtin/pypy/module/imp/interp_imp.py ============================================================================== --- (empty file) +++ pypy/branch/import-builtin/pypy/module/imp/interp_imp.py Thu Dec 10 19:20:26 2009 @@ -0,0 +1,131 @@ +from pypy.module.imp import importing +from pypy.module._file.interp_file import W_File +from pypy.rlib import streamio +from pypy.interpreter.error import OperationError +from pypy.interpreter.module import Module +from pypy.interpreter.gateway import NoneNotWrapped +import struct + +def get_suffixes(space): + w = space.wrap + return space.newlist([ + space.newtuple([w('.py'), w('U'), w(importing.PY_SOURCE)]), + space.newtuple([w('.pyc'), w('rb'), w(importing.PY_COMPILED)]), + ]) + +def get_magic(space): + return space.wrap(struct.pack(' Author: arigo Date: Thu Dec 10 21:07:02 2009 New Revision: 70059 Modified: pypy/trunk/pypy/rlib/rgc.py pypy/trunk/pypy/rlib/test/test_rgc.py pypy/trunk/pypy/rpython/llinterp.py pypy/trunk/pypy/rpython/lltypesystem/llmemory.py pypy/trunk/pypy/rpython/lltypesystem/lloperation.py pypy/trunk/pypy/rpython/lltypesystem/opimpl.py pypy/trunk/pypy/rpython/lltypesystem/rlist.py pypy/trunk/pypy/rpython/memory/gc/generation.py pypy/trunk/pypy/rpython/memory/gctransform/boehm.py pypy/trunk/pypy/rpython/memory/gctransform/framework.py pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py pypy/trunk/pypy/rpython/memory/gctransform/transform.py pypy/trunk/pypy/rpython/memory/gcwrapper.py pypy/trunk/pypy/rpython/memory/test/test_gc.py pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py pypy/trunk/pypy/translator/c/test/test_newgc.py Log: Merge 'branch/listcopyop': add the rgc.ll_arraycopy() helper to copy (part of) an array into another without the penalty of the write barrier for each item. Use it for _ll_list_resize(). It could be used more, but this is already an important use case. Modified: pypy/trunk/pypy/rlib/rgc.py ============================================================================== --- pypy/trunk/pypy/rlib/rgc.py (original) +++ pypy/trunk/pypy/rlib/rgc.py Thu Dec 10 21:07:02 2009 @@ -328,3 +328,32 @@ hop.exception_cannot_occur() return hop.genop('finish_building_buffer', vlist, resulttype=hop.r_result.lowleveltype) + +def ll_arraycopy(source, dest, source_start, dest_start, length): + from pypy.rpython.lltypesystem.lloperation import llop + from pypy.rpython.lltypesystem import lltype, llmemory + from pypy.rlib.objectmodel import keepalive_until_here + + assert source != dest + TP = lltype.typeOf(source).TO + assert TP == lltype.typeOf(dest).TO + if isinstance(TP.OF, lltype.Ptr) and TP.OF.TO._gckind == 'gc': + # perform a write barrier that copies necessary flags from + # source to dest + if not llop.gc_writebarrier_before_copy(lltype.Void, source, dest): + # if the write barrier is not supported, copy by hand + for i in range(length): + dest[i + dest_start] = source[i + source_start] + return + source_addr = llmemory.cast_ptr_to_adr(source) + dest_addr = llmemory.cast_ptr_to_adr(dest) + cp_source_addr = (source_addr + llmemory.itemoffsetof(TP, 0) + + llmemory.sizeof(TP.OF) * source_start) + cp_dest_addr = (dest_addr + llmemory.itemoffsetof(TP, 0) + + llmemory.sizeof(TP.OF) * dest_start) + + llmemory.raw_memcopy(cp_source_addr, cp_dest_addr, + llmemory.sizeof(TP.OF) * length) + keepalive_until_here(source) + keepalive_until_here(dest) +ll_arraycopy._annspecialcase_ = 'specialize:ll' Modified: pypy/trunk/pypy/rlib/test/test_rgc.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_rgc.py (original) +++ pypy/trunk/pypy/rlib/test/test_rgc.py Thu Dec 10 21:07:02 2009 @@ -68,3 +68,61 @@ return hlstr(rgc.finish_building_buffer(ptr, 2)) assert f() == 'ab' + +def test_ll_arraycopy_1(): + TYPE = lltype.GcArray(lltype.Signed) + a1 = lltype.malloc(TYPE, 10) + a2 = lltype.malloc(TYPE, 6) + for i in range(10): a1[i] = 100 + i + for i in range(6): a2[i] = 200 + i + rgc.ll_arraycopy(a1, a2, 4, 2, 3) + for i in range(10): + assert a1[i] == 100 + i + for i in range(6): + if 2 <= i < 5: + assert a2[i] == a1[i+2] + else: + assert a2[i] == 200 + i + +def test_ll_arraycopy_2(): + TYPE = lltype.GcArray(lltype.Void) + a1 = lltype.malloc(TYPE, 10) + a2 = lltype.malloc(TYPE, 6) + rgc.ll_arraycopy(a1, a2, 4, 2, 3) + # nothing to assert here, should not crash... + +def test_ll_arraycopy_3(): + S = lltype.Struct('S') # non-gc + TYPE = lltype.GcArray(lltype.Ptr(S)) + a1 = lltype.malloc(TYPE, 10) + a2 = lltype.malloc(TYPE, 6) + org1 = [None] * 10 + org2 = [None] * 6 + for i in range(10): a1[i] = org1[i] = lltype.malloc(S, immortal=True) + for i in range(6): a2[i] = org2[i] = lltype.malloc(S, immortal=True) + rgc.ll_arraycopy(a1, a2, 4, 2, 3) + for i in range(10): + assert a1[i] == org1[i] + for i in range(6): + if 2 <= i < 5: + assert a2[i] == a1[i+2] + else: + assert a2[i] == org2[i] + +def test_ll_arraycopy_4(): + S = lltype.GcStruct('S') + TYPE = lltype.GcArray(lltype.Ptr(S)) + a1 = lltype.malloc(TYPE, 10) + a2 = lltype.malloc(TYPE, 6) + org1 = [None] * 10 + org2 = [None] * 6 + for i in range(10): a1[i] = org1[i] = lltype.malloc(S) + for i in range(6): a2[i] = org2[i] = lltype.malloc(S) + rgc.ll_arraycopy(a1, a2, 4, 2, 3) + for i in range(10): + assert a1[i] == org1[i] + for i in range(6): + if 2 <= i < 5: + assert a2[i] == a1[i+2] + else: + assert a2[i] == org2[i] Modified: pypy/trunk/pypy/rpython/llinterp.py ============================================================================== --- pypy/trunk/pypy/rpython/llinterp.py (original) +++ pypy/trunk/pypy/rpython/llinterp.py Thu Dec 10 21:07:02 2009 @@ -754,6 +754,12 @@ def op_zero_gc_pointers_inside(self, obj): raise NotImplementedError("zero_gc_pointers_inside") + def op_gc_writebarrier_before_copy(self, source, dest): + if hasattr(self.heap, 'writebarrier_before_copy'): + return self.heap.writebarrier_before_copy(source, dest) + else: + return True + def op_getfield(self, obj, field): checkptr(obj) # check the difference between op_getfield and op_getsubstruct: Modified: pypy/trunk/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/llmemory.py Thu Dec 10 21:07:02 2009 @@ -118,6 +118,9 @@ return if isinstance(self.TYPE, lltype.ContainerType): PTR = lltype.Ptr(self.TYPE) + elif self.TYPE == GCREF: + self._raw_memcopy_gcrefs(srcadr, dstadr) + return else: PTR = lltype.Ptr(lltype.FixedSizeArray(self.TYPE, 1)) while True: @@ -130,6 +133,18 @@ srcadr += ItemOffset(self.TYPE) dstadr += ItemOffset(self.TYPE) + def _raw_memcopy_gcrefs(self, srcadr, dstadr): + # special case to handle arrays of any GC pointers + repeat = self.repeat + while True: + data = srcadr.address[0] + dstadr.address[0] = data + repeat -= 1 + if repeat <= 0: + break + srcadr += ItemOffset(self.TYPE) + dstadr += ItemOffset(self.TYPE) + _end_markers = weakref.WeakKeyDictionary() # -> _endmarker class _endmarker_struct(lltype._struct): __slots__ = () Modified: pypy/trunk/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/lloperation.py Thu Dec 10 21:07:02 2009 @@ -461,6 +461,7 @@ 'gc_thread_run' : LLOp(), 'gc_thread_die' : LLOp(), 'gc_assume_young_pointers': LLOp(canrun=True), + 'gc_writebarrier_before_copy': LLOp(canrun=True), 'gc_heap_stats' : LLOp(canunwindgc=True), # ------- JIT & GC interaction, only for some GCs ---------- Modified: pypy/trunk/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/opimpl.py Thu Dec 10 21:07:02 2009 @@ -394,6 +394,14 @@ checkadr(addr2) return addr1 - addr2 +def op_gc_writebarrier_before_copy(source, dest): + A = lltype.typeOf(source) + assert A == lltype.typeOf(dest) + assert isinstance(A.TO, lltype.GcArray) + assert isinstance(A.TO.OF, lltype.Ptr) + assert A.TO.OF.TO._gckind == 'gc' + return True + def op_getfield(p, name): checkptr(p) TYPE = lltype.typeOf(p).TO Modified: pypy/trunk/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rlist.py Thu Dec 10 21:07:02 2009 @@ -15,10 +15,10 @@ from pypy.rpython import robject from pypy.rlib.debug import ll_assert from pypy.rlib.rarithmetic import ovfcheck -from pypy.rpython.lltypesystem.llmemory import cast_ptr_to_adr, raw_memclear,\ - raw_memcopy, sizeof, itemoffsetof, offsetof from pypy.rpython.lltypesystem import rffi from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rlib import rgc # ____________________________________________________________ # @@ -207,25 +207,17 @@ except OverflowError: raise MemoryError # XXX consider to have a real realloc + # new_allocated is a bit more than newsize, enough to ensure an amortized + # linear complexity for e.g. repeated usage of l.append(). items = l.items newitems = malloc(typeOf(l).TO.items.TO, new_allocated) before_len = l.length - if before_len < new_allocated: - p = before_len - 1 - else: - p = new_allocated - 1 - ITEM = typeOf(l).TO.ITEM - if isinstance(ITEM, Ptr): - while p >= 0: - newitems[p] = items[p] - items[p] = nullptr(ITEM.TO) - p -= 1 - else: - source = cast_ptr_to_adr(items) + itemoffsetof(typeOf(l.items).TO, 0) - dest = cast_ptr_to_adr(newitems) + itemoffsetof(typeOf(l.items).TO, 0) - s = p + 1 - raw_memcopy(source, dest, sizeof(ITEM) * s) - keepalive_until_here(items) + if before_len: # avoids copying GC flags from the prebuilt_empty_array + if before_len < newsize: + p = before_len + else: + p = newsize + rgc.ll_arraycopy(items, newitems, 0, 0, p) l.length = newsize l.items = newitems _ll_list_resize_really._annenforceargs_ = (None, int) Modified: pypy/trunk/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/generation.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/generation.py Thu Dec 10 21:07:02 2009 @@ -466,6 +466,13 @@ remember_young_pointer._dont_inline_ = True self.remember_young_pointer = remember_young_pointer + def write_into_last_generation_obj(self, addr_struct, addr): + objhdr = self.header(addr_struct) + if objhdr.tid & GCFLAG_NO_HEAP_PTRS: + if not self.is_last_generation(addr): + objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS + self.last_generation_root_objects.append(addr_struct) + def assume_young_pointers(self, addr_struct): objhdr = self.header(addr_struct) if objhdr.tid & GCFLAG_NO_YOUNG_PTRS: @@ -475,12 +482,28 @@ objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS self.last_generation_root_objects.append(addr_struct) - def write_into_last_generation_obj(self, addr_struct, addr): - objhdr = self.header(addr_struct) - if objhdr.tid & GCFLAG_NO_HEAP_PTRS: - if not self.is_last_generation(addr): - objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS - self.last_generation_root_objects.append(addr_struct) + def writebarrier_before_copy(self, source_addr, dest_addr): + """ This has the same effect as calling writebarrier over + each element in dest copied from source, except it might reset + one of the following flags a bit too eagerly, which means we'll have + a bit more objects to track, but being on the safe side. + """ + source_hdr = self.header(source_addr) + dest_hdr = self.header(dest_addr) + if dest_hdr.tid & GCFLAG_NO_YOUNG_PTRS == 0: + return True + # ^^^ a fast path of write-barrier + if source_hdr.tid & GCFLAG_NO_YOUNG_PTRS == 0: + # there might be an object in source that is in nursery + self.old_objects_pointing_to_young.append(dest_addr) + dest_hdr.tid &= ~GCFLAG_NO_YOUNG_PTRS + if dest_hdr.tid & GCFLAG_NO_HEAP_PTRS: + if source_hdr.tid & GCFLAG_NO_HEAP_PTRS == 0: + # ^^^ equivalend of addr from source not being in last + # gen + dest_hdr.tid &= ~GCFLAG_NO_HEAP_PTRS + self.last_generation_root_objects.append(dest_addr) + return True def is_last_generation(self, obj): # overridden by HybridGC Modified: pypy/trunk/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/boehm.py Thu Dec 10 21:07:02 2009 @@ -167,6 +167,9 @@ resulttype=llmemory.Address) hop.cast_result(v_addr) + def gct_gc_writebarrier_before_copy(self, hop): + return rmodel.inputconst(lltype.Bool, True) # no write barrier needed + def gct_gc_identityhash(self, hop): v_obj = hop.spaceop.args[0] v_adr = hop.genop("cast_ptr_to_adr", [v_obj], Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/framework.py Thu Dec 10 21:07:02 2009 @@ -292,6 +292,13 @@ [s_gc, annmodel.SomeInteger(knowntype=rffi.r_ushort)], annmodel.SomeInteger()) + if hasattr(GCClass, 'writebarrier_before_copy'): + self.wb_before_copy_ptr = \ + getfn(GCClass.writebarrier_before_copy.im_func, + [s_gc] + [annmodel.SomeAddress()] * 2, annmodel.SomeBool()) + elif GCClass.needs_write_barrier: + raise NotImplementedError("GC needs write barrier, but does not provide writebarrier_before_copy functionality") + # in some GCs we can inline the common case of # malloc_fixedsize(typeid, size, True, False, False) if getattr(GCClass, 'inline_simple_malloc', False): @@ -776,6 +783,19 @@ TYPE = v_ob.concretetype.TO gen_zero_gc_pointers(TYPE, v_ob, hop.llops) + def gct_gc_writebarrier_before_copy(self, hop): + if not hasattr(self, 'wb_before_copy_ptr'): + # no write barrier needed in that case + return rmodel.inputconst(lltype.Bool, True) + op = hop.spaceop + source_addr = hop.genop('cast_ptr_to_adr', [op.args[0]], + resulttype=llmemory.Address) + dest_addr = hop.genop('cast_ptr_to_adr', [op.args[1]], + resulttype=llmemory.Address) + hop.genop('direct_call', [self.wb_before_copy_ptr, self.c_const_gc, + source_addr, dest_addr], + resultvar=op.result) + def gct_weakref_create(self, hop): op = hop.spaceop Modified: pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py Thu Dec 10 21:07:02 2009 @@ -119,6 +119,8 @@ GC_PARAMS = {} class GCClass(MarkSweepGC): needs_write_barrier = True + def writebarrier_before_copy(self, source, dest): + return True def write_barrier_check(spaceop, needs_write_barrier=True): t = TranslationContext() Modified: pypy/trunk/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/transform.py Thu Dec 10 21:07:02 2009 @@ -12,7 +12,7 @@ from pypy.translator.backendopt.ssa import DataFlowFamilyBuilder from pypy.translator.backendopt.constfold import constant_fold_graph from pypy.annotation import model as annmodel -from pypy.rpython import rmodel, annlowlevel +from pypy.rpython import rmodel from pypy.rpython.memory import gc from pypy.rpython.memory.gctransform.support import var_ispyobj from pypy.rpython.annlowlevel import MixLevelHelperAnnotator @@ -380,6 +380,13 @@ def gct_zero_gc_pointers_inside(self, hop): pass + def gct_gc_writebarrier_before_copy(self, hop): + # We take the conservative default and return False here, meaning + # that rgc.ll_arraycopy() will do the copy by hand (i.e. with a + # 'for' loop). Subclasses that have their own logic, or that don't + # need any kind of write barrier, may return True. + return rmodel.inputconst(lltype.Bool, False) + def gct_gc_identityhash(self, hop): # must be implemented in the various GCs raise NotImplementedError Modified: pypy/trunk/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/trunk/pypy/rpython/memory/gcwrapper.py Thu Dec 10 21:07:02 2009 @@ -128,6 +128,13 @@ ptr = lltype.cast_opaque_ptr(llmemory.GCREF, ptr) return self.gc.id(ptr) + def writebarrier_before_copy(self, source, dest): + if self.gc.needs_write_barrier: + source_addr = llmemory.cast_ptr_to_adr(source) + dest_addr = llmemory.cast_ptr_to_adr(dest) + return self.gc.writebarrier_before_copy(source_addr, dest_addr) + else: + return True # ____________________________________________________________ Modified: pypy/trunk/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/trunk/pypy/rpython/memory/test/test_gc.py Thu Dec 10 21:07:02 2009 @@ -551,6 +551,26 @@ res = self.interpret(fn, [-1000], taggedpointers=True) assert res == 111 + def test_writebarrier_before_copy(self): + S = lltype.GcStruct('S') + TP = lltype.GcArray(lltype.Ptr(S)) + def fn(): + l = lltype.malloc(TP, 100) + l2 = lltype.malloc(TP, 100) + for i in range(100): + l[i] = lltype.malloc(S) + rgc.ll_arraycopy(l, l2, 50, 0, 50) + x = [] + # force minor collect + t = (1, lltype.malloc(S)) + for i in range(20): + x.append(t) + for i in range(50): + assert l2[i] == l[50 + i] + return 0 + + self.interpret(fn, []) + from pypy.rlib.objectmodel import UnboxedValue Modified: pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py Thu Dec 10 21:07:02 2009 @@ -835,6 +835,7 @@ return f def test_gc_heap_stats(self): + py.test.skip("this test makes the following test crash. Investigate.") run = self.runner("gc_heap_stats") res = run([]) assert res % 10000 == 2611 @@ -844,26 +845,7 @@ # ^^^ a crude assumption that totsize - varsize would be dividable by 4 # (and give fixedsize) - - def define_listcopy(cls): - TP = lltype.GcArray(lltype.Signed) - def fn(): - l = lltype.malloc(TP, 100) - for i in range(100): - l[i] = 1 - l2 = lltype.malloc(TP, 50) - #llop.listcopy(lltype.Void, l, l2, 50, 0, 50) - #for i in range(50): - # assert l2[i] == 1 - return 0 - - return fn - - def test_listcopy(self): - run = self.runner("listcopy") - run([]) - - def define_listcopy_ptr(cls): + def define_writebarrier_before_copy(cls): S = lltype.GcStruct('S') TP = lltype.GcArray(lltype.Ptr(S)) def fn(): @@ -871,20 +853,19 @@ l2 = lltype.malloc(TP, 100) for i in range(100): l[i] = lltype.malloc(S) - #llop.listcopy(lltype.Void, l, l2, 50, 0, 50) + rgc.ll_arraycopy(l, l2, 50, 0, 50) # force nursery collect x = [] for i in range(20): x.append((1, lltype.malloc(S))) - #for i in range(50): - # assert l2[i] + for i in range(50): + assert l2[i] == l[50 + i] return 0 return fn - def test_listcopy_ptr(self): - py.test.skip("explodes") - run = self.runner("listcopy_ptr") + def test_writebarrier_before_copy(self): + run = self.runner("writebarrier_before_copy") run([]) # ________________________________________________________________ Modified: pypy/trunk/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_newgc.py (original) +++ pypy/trunk/pypy/translator/c/test/test_newgc.py Thu Dec 10 21:07:02 2009 @@ -835,6 +835,48 @@ res = self.run('hash_varsized') assert res != 0 + + def define_arraycopy_writebarrier_int(cls): + TP = lltype.GcArray(lltype.Signed) + S = lltype.GcStruct('S') + def fn(): + l = lltype.malloc(TP, 100) + for i in range(100): + l[i] = i * 3 + l2 = lltype.malloc(TP, 50) + rgc.ll_arraycopy(l, l2, 40, 0, 50) + # force a nursery collect + x = [] + for i in range(20): + x.append((1, lltype.malloc(S))) + for i in range(50): + assert l2[i] == (40 + i) * 3 + return 0 + + return fn + + def test_arraycopy_writebarrier_int(self): + self.run("arraycopy_writebarrier_int") + + def define_arraycopy_writebarrier_ptr(cls): + TP = lltype.GcArray(lltype.Ptr(lltype.GcArray(lltype.Signed))) + def fn(): + l = lltype.malloc(TP, 100) + for i in range(100): + l[i] = lltype.malloc(TP.OF.TO, i) + l2 = lltype.malloc(TP, 50) + rgc.ll_arraycopy(l, l2, 40, 0, 50) + rgc.collect() + for i in range(50): + assert l2[i] == l[40 + i] + return 0 + + return fn + + def test_arraycopy_writebarrier_ptr(self): + self.run("arraycopy_writebarrier_ptr") + + class TestSemiSpaceGC(TestUsingFramework, snippet.SemiSpaceGCTestDefines): gcpolicy = "semispace" should_be_moving = True From arigo at codespeak.net Thu Dec 10 21:07:22 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 10 Dec 2009 21:07:22 +0100 (CET) Subject: [pypy-svn] r70060 - pypy/branch/listcopyop Message-ID: <20091210200722.65A76168021@codespeak.net> Author: arigo Date: Thu Dec 10 21:07:19 2009 New Revision: 70060 Removed: pypy/branch/listcopyop/ Log: Remove merged branch. From dan at codespeak.net Thu Dec 10 21:48:56 2009 From: dan at codespeak.net (dan at codespeak.net) Date: Thu, 10 Dec 2009 21:48:56 +0100 (CET) Subject: [pypy-svn] r70061 - in pypy/branch/micronumpy/pypy/module/micronumpy: . test Message-ID: <20091210204856.CC074168021@codespeak.net> Author: dan Date: Thu Dec 10 21:48:55 2009 New Revision: 70061 Added: pypy/branch/micronumpy/pypy/module/micronumpy/app_numarray.py Modified: pypy/branch/micronumpy/pypy/module/micronumpy/__init__.py pypy/branch/micronumpy/pypy/module/micronumpy/numarray.py pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py Log: Added test cases to fail and a fair amount of non-functional code. Modified: pypy/branch/micronumpy/pypy/module/micronumpy/__init__.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/__init__.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/__init__.py Thu Dec 10 21:48:55 2009 @@ -4,10 +4,14 @@ class Module(MixedModule): applevel_name = 'numpy' + appleveldefs = { + 'array' : 'app_numarray.array', + } interpleveldefs = { 'zeros' : 'numarray.zeros', 'minimum' : 'ufunc.minimum', + 'IntArray' : 'numarray.IntArray', + 'FloatArray' : 'numarray.FloatArray', } - appleveldefs = {} Added: pypy/branch/micronumpy/pypy/module/micronumpy/app_numarray.py ============================================================================== --- (empty file) +++ pypy/branch/micronumpy/pypy/module/micronumpy/app_numarray.py Thu Dec 10 21:48:55 2009 @@ -0,0 +1,27 @@ +types_list = [object, complex, float, int] + +def lowest_type(x): + result = object + for type in types_list: + if isinstance(x, type): + result = type + return result + +def lowest_common_type(xs): + types = [type(x) for x in xs] + result = int + for t in types: + if types_list.index(t) < types_list.index(result): + result = t + return result + +def array(xs, dtype=None): + import numpy + arrays = { + int: numpy.IntArray, + float: numpy.FloatArray, + #complex: ComplexNumArray, + } + #type = lowest_common_type(xs) + #return arrays[type](xs) + return numpy.zeros(len(xs), dtype=int) #FIXME Modified: pypy/branch/micronumpy/pypy/module/micronumpy/numarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/numarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/numarray.py Thu Dec 10 21:48:55 2009 @@ -1,50 +1,140 @@ - from pypy.interpreter.baseobjspace import ObjSpace, W_Root, Wrappable from pypy.interpreter.error import OperationError from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app, NoneNotWrapped from pypy.rlib.debug import make_sure_not_resized -class BaseNumArray(Wrappable): - pass +from pypy.rlib.objectmodel import specialize -class NumArray(BaseNumArray): - def __init__(self, space, dim, dtype): - self.dim = dim - self.space = space - # ignore dtype for now - self.storage = [0] * dim - make_sure_not_resized(self.storage) - - def descr_getitem(self, index): - space = self.space - try: - return space.wrap(self.storage[index]) - except IndexError: - raise OperationError(space.w_IndexError, - space.wrap("list index out of range")) - descr_getitem.unwrap_spec = ['self', int] +result_types = { + (int, int): int, + (int, float): float, + (float, int): float, + (float, float): float + } - def descr_setitem(self, index, value): - space = self.space - try: - self.storage[index] = value - except IndexError: - raise OperationError(space.w_IndexError, - space.wrap("list index out of range")) - return space.w_None - descr_setitem.unwrap_spec = ['self', int, int] - - def descr_len(self): - return self.space.wrap(len(self.storage)) - descr_len.unwrap_spec = ['self'] +class BaseNumArray(Wrappable): + pass -NumArray.typedef = TypeDef( - 'NumArray', - __getitem__ = interp2app(NumArray.descr_getitem), - __setitem__ = interp2app(NumArray.descr_setitem), - __len__ = interp2app(NumArray.descr_len), -) +def iterable_type(space, w_xs): + xs = space.fixedview(w_xs) + type = int + for i in range(len(xs)): + type = result_types[type, xs[i]] + return type + +def int_unwrapper(space, w_x): + return space.int_w(space.int(w_x)) + +def float_unwrapper(space, w_x): + return space.float_w(space.float(w_x)) + +def create_numarray(type, unwrapper, name): + class NumArray(BaseNumArray): + def __init__(self, space, length, dtype): + #ignore dtype, irrelevant to optimized numarray implementations too + self.length = length + self.space = space + self.storage = [type(0.0)] * length + make_sure_not_resized(self.storage) + + + def _dup_size(self, type): + return self.__class__(space, result_types[self.type, type], self.length) + + def create_scalar_op(unwrap, f): + def scalar_operation(self, w_x): + space = self.space + x = unwrap(w_x) + result = self._dup_size(type(x)) + for i in range(self.length): + result[i] = f(self.storage[i], x) + return space.wrap(result) + return scalar_operation + + def create_fixedview_op(unwrap, f): + def fixedview_operation(self, w_xs): + space = self.space + + try: + xs = space.fixedview(w_xs, len(self.storage)) + except UnpackValueError, e: + # w_xs is of the wrong size + raise OperationError(space.w_ValueError, + space.wrap("shape mismatch: objects cannot be broadcast to the same shape")) + + result = self._dup_size(iterable_type(space, xs)) + + i = 0 + for w_x in xs: + result[i] = f(self.storage[i], unwrap(w_x)) + i += 1 + return result + return fixedview_operation + + #def mul_iterable(self, w_xs): + #return self.fixedview_operation(w_xs, mul) + +# def descr_mul(self, w_x): +# space = self.space +# if space.type(w_x) in [W_Int, W_Float]: #complex, long +# try: +# return self.mul_scalar(space.int_w(w_x)) +# except TypeError: +# return self.mul_scalar(space.float_w(w_x)) +# else: +# return self.mul_iterable(w_x) +# descr_mul.unwrap_spec = ['self', W_Root] + + def descr_getitem(self, index): + space = self.space + try: + return space.wrap(self.storage[index]) + except IndexError: + raise OperationError(space.w_IndexError, + space.wrap("list index out of range")) + descr_getitem.unwrap_spec = ['self', int] + + def descr_setitem(self, index, w_value): + space = self.space + try: + self.storage[index] = unwrapper(space, w_value) + except IndexError: + raise OperationError(space.w_IndexError, + space.wrap("list index out of range")) + return space.w_None + descr_setitem.unwrap_spec = ['self', int, W_Root] + + def descr_len(self): + return self.space.wrap(len(self.storage)) + descr_len.unwrap_spec = ['self'] + + def descr_init(xs): pass + + NumArray.typedef = TypeDef( + name, + #__init__ = interp2app(descr_init), #FIXME + __getitem__ = interp2app(NumArray.descr_getitem), + __setitem__ = interp2app(NumArray.descr_setitem), + __len__ = interp2app(NumArray.descr_len), + ) + return NumArray + +IntArray = create_numarray(int, int_unwrapper, 'IntArray') +NumArray = IntArray # FIXME: compatibility for now +FloatArray = create_numarray(float, float_unwrapper, 'FloatArray') + +arrays = { + int: IntArray, + float: FloatArray + } + +#def array(space, w_xs): +# w_length = space.len(w_xs) +# length = space.int_w(w_length) +# #TODO: discover type +# result = NumArray(space, type, length) +#array.unwrap_spec = [ObjSpace, W_Root] def compute_pos(space, indexes, dim): current = 1 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Thu Dec 10 21:48:55 2009 @@ -4,7 +4,34 @@ class AppTestNumpy(object): def setup_class(cls): cls.space = gettestobjspace(usemodules=('micronumpy',)) - + cls.w_compare = cls.space.appexec([], + """(): + def compare(a, b): + for x, y in zip(a, b): + if x != y: return False + return True + return compare""") + + def create_type_test(type): + def test_type_array(self): + compare = self.compare + from numpy import array + data = [type(x) for x in xrange(4)] + ar = array(data) + + assert compare(ar, data) + return test_type_array + + test_int_array = create_type_test(int) + test_float_array = create_type_test(float) + + def test_iterable_construction(self): + compare = self.compare + from numpy import array + ar = array(xrange(4)) + + assert compare(ar, xrange(4)) + def test_zeroes(self): from numpy import zeros ar = zeros(3, dtype=int) Modified: pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py Thu Dec 10 21:48:55 2009 @@ -6,11 +6,11 @@ def minimum(space, w_a, w_b): if not isinstance(w_a, NumArray) or not isinstance(w_b, NumArray): raise OperationError(space.w_TypeError, - space.wrap("expecting NumArrat object")) - if w_a.dim != w_b.dim: + space.wrap("expecting NumArray object")) + if w_a.length != w_b.length: raise OperationError(space.w_ValueError, space.wrap("minimum of arrays of different length")) - res = NumArray(space, w_a.dim, 'i') + res = NumArray(space, w_a.length, 'i') for i in range(len(w_a.storage)): one = w_a.storage[i] two = w_b.storage[i] From pedronis at codespeak.net Thu Dec 10 22:32:44 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 10 Dec 2009 22:32:44 +0100 (CET) Subject: [pypy-svn] r70062 - in pypy/trunk/pypy: . jit/backend/cli jit/backend/cli/test jit/backend/llgraph jit/backend/llsupport jit/backend/llsupport/test jit/backend/llvm/test jit/backend/x86 jit/backend/x86/test jit/metainterp jit/metainterp/test module/exceptions objspace/std/test rlib/test translator/c/test Message-ID: <20091210213244.24A73168021@codespeak.net> Author: pedronis Date: Thu Dec 10 22:32:42 2009 New Revision: 70062 Modified: pypy/trunk/pypy/ (props changed) pypy/trunk/pypy/jit/backend/cli/runner.py pypy/trunk/pypy/jit/backend/cli/test/conftest.py (props changed) pypy/trunk/pypy/jit/backend/llgraph/runner.py pypy/trunk/pypy/jit/backend/llsupport/llmodel.py pypy/trunk/pypy/jit/backend/llsupport/test/test_runner.py pypy/trunk/pypy/jit/backend/llvm/test/conftest.py (props changed) pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/runner.py pypy/trunk/pypy/jit/backend/x86/support.py pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py (props changed) pypy/trunk/pypy/jit/backend/x86/test/test_support.py pypy/trunk/pypy/jit/metainterp/logger.py (props changed) pypy/trunk/pypy/jit/metainterp/test/test_basic.py pypy/trunk/pypy/jit/metainterp/warmspot.py pypy/trunk/pypy/module/exceptions/ (props changed) pypy/trunk/pypy/objspace/std/test/test_setobject.py (props changed) pypy/trunk/pypy/rlib/test/test_rcoroutine.py (props changed) pypy/trunk/pypy/translator/c/test/test_refcount.py (props changed) Log: merge fixed-fail-boxes, remove unneeded support for growing fail boxes lists Modified: pypy/trunk/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/runner.py (original) +++ pypy/trunk/pypy/jit/backend/cli/runner.py Thu Dec 10 22:32:42 2009 @@ -43,7 +43,7 @@ supports_floats = True ts = oohelper - def __init__(self, rtyper, stats, translate_support_code=False, + def __init__(self, rtyper, stats, opts=None, translate_support_code=False, mixlevelann=None, gcdescr=None): model.AbstractCPU.__init__(self) self.rtyper = rtyper Modified: pypy/trunk/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/runner.py Thu Dec 10 22:32:42 2009 @@ -74,8 +74,9 @@ class BaseCPU(model.AbstractCPU): supports_floats = True - def __init__(self, rtyper, stats=None, translate_support_code=False, + def __init__(self, rtyper, stats=None, opts=None, translate_support_code=False, annmixlevel=None, gcdescr=None): + assert type(opts) is not bool model.AbstractCPU.__init__(self) self.rtyper = rtyper self.translate_support_code = translate_support_code Modified: pypy/trunk/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/llmodel.py Thu Dec 10 22:32:42 2009 @@ -17,8 +17,11 @@ class AbstractLLCPU(AbstractCPU): from pypy.jit.metainterp.typesystem import llhelper as ts - def __init__(self, rtyper, stats, translate_support_code=False, + def __init__(self, rtyper, stats, opts, translate_support_code=False, gcdescr=None): + assert type(opts) is not bool + self.opts = opts + from pypy.jit.backend.llsupport.gc import get_ll_description AbstractCPU.__init__(self) self.rtyper = rtyper Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/test/test_runner.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/test/test_runner.py Thu Dec 10 22:32:42 2009 @@ -18,4 +18,4 @@ # ====> ../../test/runner_test.py def setup_class(cls): - cls.cpu = MyLLCPU(rtyper=None, stats=FakeStats()) + cls.cpu = MyLLCPU(rtyper=None, stats=FakeStats(), opts=None) Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Thu Dec 10 22:32:42 2009 @@ -15,9 +15,7 @@ from pypy.jit.backend.x86 import codebuf from pypy.jit.backend.x86.ri386 import * from pypy.jit.metainterp.resoperation import rop -from pypy.jit.backend.x86.support import NonmovableGrowableArrayFloat,\ - NonmovableGrowableArraySigned, NonmovableGrowableArrayGCREF,\ - CHUNK_SIZE +from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print # our calling convention - we pass first 6 args in registers @@ -78,7 +76,8 @@ mc2 = None mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE - def __init__(self, cpu, translate_support_code=False): + def __init__(self, cpu, translate_support_code=False, + failargs_limit=1000): self.cpu = cpu self.verbose = False self.rtyper = cpu.rtyper @@ -86,21 +85,18 @@ self.malloc_array_func_addr = 0 self.malloc_str_func_addr = 0 self.malloc_unicode_func_addr = 0 - self.fail_boxes_int = NonmovableGrowableArraySigned() - self.fail_boxes_ptr = NonmovableGrowableArrayGCREF() - self.fail_boxes_float = NonmovableGrowableArrayFloat() + self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) + self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) + self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 self.loc_float_const_neg = None self.loc_float_const_abs = None self.setup_failure_recovery() def leave_jitted_hook(self): - i = 0 - while i < self.fail_boxes_ptr.lgt: - chunk = self.fail_boxes_ptr.chunks[i] - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(chunk)) - i += 1 + ptrs = self.fail_boxes_ptr.ar + llop.gc_assume_young_pointers(lltype.Void, + llmemory.cast_ptr_to_adr(ptrs)) def make_sure_mc_exists(self): if self.mc is None: Modified: pypy/trunk/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/runner.py Thu Dec 10 22:32:42 2009 @@ -16,14 +16,18 @@ BOOTSTRAP_TP = lltype.FuncType([], lltype.Signed) dont_keepalive_stuff = False # for tests - def __init__(self, rtyper, stats, translate_support_code=False, + def __init__(self, rtyper, stats, opts=None, translate_support_code=False, gcdescr=None): - AbstractLLCPU.__init__(self, rtyper, stats, translate_support_code, + AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) - self._bootstrap_cache = {} def setup(self): - self.assembler = Assembler386(self, self.translate_support_code) + if self.opts is not None: + failargs_limit = self.opts.failargs_limit + else: + failargs_limit = 1000 + self.assembler = Assembler386(self, self.translate_support_code, + failargs_limit) def get_on_leave_jitted_hook(self): return self.assembler.leave_jitted_hook Modified: pypy/trunk/pypy/jit/backend/x86/support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/support.py (original) +++ pypy/trunk/pypy/jit/backend/x86/support.py Thu Dec 10 22:32:42 2009 @@ -1,53 +1,23 @@ - from pypy.rpython.lltypesystem import lltype, rffi, llmemory -from pypy.rlib import rgc -from pypy.rlib.objectmodel import we_are_translated - -CHUNK_SIZE_BITS = 8 -CHUNK_SIZE = 1 << CHUNK_SIZE_BITS -def new_nonmovable_growable_array(TP): +def values_array(TP, size): ATP = lltype.GcArray(TP) - class NonmovableGrowableArray(object): + class ValuesArray(object): def __init__(self): - self.chunks = [] - self.lgt = 0 - - def _grow(self): - # XXX workaround for a fact that rgc.malloc_nonmovable always - # returns nullptr when run on top of python - if we_are_translated(): - new_item = rgc.malloc_nonmovable(ATP, CHUNK_SIZE, zero=True) - else: - new_item = lltype.malloc(ATP, CHUNK_SIZE, zero=True) - self.chunks.append(new_item) - self.lgt += 1 - _grow._dont_inline_ = True + self.ar = lltype.malloc(ATP, size, zero=True, immortal=True) def get_addr_for_num(self, i): - chunk_no, ofs = self._no_of(i) - chunk = self.chunks[chunk_no] - rffi.cast(lltype.Signed, chunk) return rffi.cast(lltype.Signed, lltype.direct_ptradd( - lltype.direct_arrayitems(chunk), ofs)) - - def _no_of(self, i): - while i >= len(self.chunks) * CHUNK_SIZE: - self._grow() - return i >> CHUNK_SIZE_BITS, i & (CHUNK_SIZE-1) - _no_of._always_inline_ = True + lltype.direct_arrayitems(self.ar), i)) def setitem(self, i, v): - chunk_no, ofs = self._no_of(i) - self.chunks[chunk_no][ofs] = v + self.ar[i] = v def getitem(self, i): - chunk_no, ofs = self._no_of(i) - return self.chunks[chunk_no][ofs] + return self.ar[i] - return NonmovableGrowableArray + def _freeze_(self): + return True -NonmovableGrowableArrayFloat = new_nonmovable_growable_array(lltype.Float) -NonmovableGrowableArraySigned = new_nonmovable_growable_array(lltype.Signed) -NonmovableGrowableArrayGCREF = new_nonmovable_growable_array(llmemory.GCREF) + return ValuesArray() Modified: pypy/trunk/pypy/jit/backend/x86/test/test_support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_support.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_support.py Thu Dec 10 22:32:42 2009 @@ -1,27 +1,21 @@ -from pypy.jit.backend.x86.support import NonmovableGrowableArraySigned, CHUNK_SIZE +from pypy.jit.backend.x86.support import values_array from pypy.rpython.lltypesystem import lltype, llmemory, rffi -def test_nonmovable_growable_array(): - ar = NonmovableGrowableArraySigned() +def test_values_array_signed(): + ar = values_array(lltype.Signed, 50) adr = ar.get_addr_for_num(10) rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] = 42 assert ar.getitem(10) == 42 ar.setitem(42, 38) adr = ar.get_addr_for_num(42) assert rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] == 38 - adr = ar.get_addr_for_num(CHUNK_SIZE + 10) - rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] = 42 - assert ar.getitem(CHUNK_SIZE + 10) == 42 - ar.setitem(CHUNK_SIZE + 42, 38) - adr = ar.get_addr_for_num(CHUNK_SIZE + 42) - assert rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] == 38 - adr = ar.get_addr_for_num(3 * CHUNK_SIZE + 10) - rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] = 42 - assert ar.getitem(3 * CHUNK_SIZE + 10) == 42 - ar.setitem(3 * CHUNK_SIZE + 42, 38) - adr = ar.get_addr_for_num(3 * CHUNK_SIZE + 42) - assert rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] == 38 - ar.setitem(8*CHUNK_SIZE, 13) - assert ar.getitem(8*CHUNK_SIZE) == 13 - + +def test_values_array_float(): + ar = values_array(lltype.Float, 50) + adr = ar.get_addr_for_num(10) + rffi.cast(rffi.CArrayPtr(lltype.Float), adr)[0] = 42.5 + assert ar.getitem(10) == 42.5 + ar.setitem(42, 38.5) + adr = ar.get_addr_for_num(42) + assert rffi.cast(rffi.CArrayPtr(lltype.Float), adr)[0] == 38.5 Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Thu Dec 10 22:32:42 2009 @@ -21,7 +21,7 @@ rtyper = support.annotate(func, values, type_system=type_system) stats = history.Stats() - cpu = CPUClass(rtyper, stats, False) + cpu = CPUClass(rtyper, stats, None, False) graphs = rtyper.annotator.translator.graphs opt = history.Options(listops=listops) metainterp_sd = pyjitpl.MetaInterpStaticData(graphs[0], cpu, stats, opt) Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmspot.py Thu Dec 10 22:32:42 2009 @@ -256,7 +256,7 @@ annhelper = self.annhelper else: annhelper = None - cpu = CPUClass(self.translator.rtyper, self.stats, + cpu = CPUClass(self.translator.rtyper, self.stats, opt, translate_support_code, gcdescr=self.gcdescr) self.cpu = cpu self.metainterp_sd = MetaInterpStaticData(self.portal_graph, # xxx From pedronis at codespeak.net Thu Dec 10 22:34:03 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 10 Dec 2009 22:34:03 +0100 (CET) Subject: [pypy-svn] r70063 - pypy/branch/fixed-fail-boxes Message-ID: <20091210213403.1B1AE168021@codespeak.net> Author: pedronis Date: Thu Dec 10 22:34:02 2009 New Revision: 70063 Removed: pypy/branch/fixed-fail-boxes/ Log: merged From afa at codespeak.net Fri Dec 11 01:02:25 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 11 Dec 2009 01:02:25 +0100 (CET) Subject: [pypy-svn] r70064 - pypy/branch/import-builtin/pypy/module/imp Message-ID: <20091211000225.1AF5F168024@codespeak.net> Author: afa Date: Fri Dec 11 01:02:24 2009 New Revision: 70064 Modified: pypy/branch/import-builtin/pypy/module/imp/__init__.py pypy/branch/import-builtin/pypy/module/imp/importing.py pypy/branch/import-builtin/pypy/module/imp/interp_imp.py Log: Translation fix and some clean-ups Modified: pypy/branch/import-builtin/pypy/module/imp/__init__.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/__init__.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/__init__.py Fri Dec 11 01:02:24 2009 @@ -1,13 +1,5 @@ from pypy.interpreter.mixedmodule import MixedModule -names = ['C_BUILTIN', 'C_EXTENSION', 'IMP_HOOK', 'NullImporter', 'PKG_DIRECTORY', - 'PY_CODERESOURCE', 'PY_COMPILED', 'PY_FROZEN', 'PY_RESOURCE', 'PY_SOURCE', - 'SEARCH_ERROR', '__doc__', '__name__', '__package__', 'acquire_lock', 'find_module', - 'get_frozen_object', 'get_magic', 'get_suffixes', 'init_builtin', 'init_frozen', - 'is_builtin', 'is_frozen', 'load_compiled', 'load_dynamic', 'load_module', - 'load_package', 'load_source', 'lock_held', 'new_module', 'release_lock', 'reload'] - - class Module(MixedModule): """ This module provides the components needed to build your own @@ -20,7 +12,7 @@ 'C_BUILTIN': 'space.wrap(importing.C_BUILTIN)', 'get_suffixes': 'interp_imp.get_suffixes', - 'get_magic': 'interp_imp.get_magic', + #'get_magic': 'interp_imp.get_magic', 'find_module': 'interp_imp.find_module', 'load_module': 'interp_imp.load_module', 'load_source': 'interp_imp.load_source', @@ -35,151 +27,3 @@ appleveldefs = { } - -# PyPy-specific interface -try: - import __pypy__ - def get_magic(): - """Return the magic number for .pyc or .pyo files.""" - import struct - return struct.pack(' (file, filename, (suffix, mode, type)) - Search for a module. If path is omitted or None, search for a - built-in, frozen or special module and continue search in sys.path. - The module name cannot contain '.'; to search for a submodule of a - package, pass the submodule name and the package's __path__. - """ - if path is None: - if name in sys.builtin_module_names: - return (None, name, ('', '', C_BUILTIN)) - path = sys.path - for base in path: - filename = os.path.join(base, name) - if os.path.isdir(filename): - return (None, filename, ('', '', PKG_DIRECTORY)) - for ext, mode, kind in get_suffixes(): - if os.path.exists(filename+ext): - return (file(filename+ext, mode), filename+ext, (ext, mode, kind)) - raise ImportError, 'No module named %s' % (name,) - - -def load_module(name, file, filename, description): - """Load a module, given information returned by find_module(). - The module name must include the full package name, if any. - """ - suffix, mode, type = description - - if type == PY_SOURCE: - return load_source(name, filename, file) - - if type == PKG_DIRECTORY: - initfilename = os.path.join(filename, '__init__.py') - module = sys.modules.setdefault(name, new_module(name)) - module.__name__ = name - module.__doc__ = None - module.__file__ = initfilename - module.__path__ = [filename] - execfile(initfilename, module.__dict__) - return module - - if type == C_BUILTIN: - module = __import__(name, {}, {}, None) - return module - if type == PY_COMPILED: - return load_compiled(name, filename, file) - raise ValueError, 'invalid description argument: %r' % (description,) - -def load_dynamic(name, *args, **kwds): - raise ImportError(name) - -def load_source(name, pathname, file=None): - autoopen = file is None - if autoopen: - file = open(pathname, 'U') - source = file.read() - if autoopen: - file.close() - co = compile(source, pathname, 'exec') - return run_module(name, pathname, co) - -def load_compiled(name, pathname, file=None): - import marshal - autoopen = file is None - if autoopen: - file = open(pathname, 'rb') - magic = file.read(4) - if magic != get_magic(): - raise ImportError("Bad magic number in %s" % pathname) - file.read(4) # skip timestamp - co = marshal.load(file) - if autoopen: - file.close() - return run_module(name, pathname, co) - -def run_module(name, pathname, co): - module = sys.modules.setdefault(name, new_module(name)) - module.__name__ = name - module.__doc__ = None - module.__file__ = pathname - try: - exec co in module.__dict__ - except : - sys.modules.pop(name,None) - raise - return module - - -def new_module(name): - """Create a new module. Do not enter it in sys.modules. - The module name must include the full package name, if any. - """ - return new.module(name) - - -def init_builtin(name): - if name not in sys.builtin_module_names: - return None - if name in sys.modules: - raise ImportError("cannot initialize a built-in module twice " - "in PyPy") - return __import__(name) - -def init_frozen(name): - return None - -def is_builtin(name): - if name in sys.builtin_module_names: - return -1 # cannot be initialized again - else: - return 0 - -def is_frozen(name): - return False - -# ____________________________________________________________ - -try: - # PyPy-specific interface - from thread import _importlock_held as lock_held - from thread import _importlock_acquire as acquire_lock - from thread import _importlock_release as release_lock -except ImportError: - def lock_held(): - """On platforms without threads, return False.""" - return False - def acquire_lock(): - """On platforms without threads, this function does nothing.""" - def release_lock(): - """On platforms without threads, this function does nothing.""" Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Fri Dec 11 01:02:24 2009 @@ -19,6 +19,7 @@ PY_COMPILED = 2 PKG_DIRECTORY = 3 C_BUILTIN = 4 +IMP_HOOK = 5 def find_modtype(space, filepart): """Check which kind of module to import for the given filepart, @@ -247,24 +248,33 @@ if space.is_true(w_loader): return w_loader -def ImportInfo(type, name, stream, suffix="", filemode=""): - return type, name, stream, suffix, filemode +class FindInfo: + def __init__(self, modtype, filename, stream, + suffix="", filemode="", w_loader=None): + self.modtype = modtype + self.filename = filename + self.stream = stream + self.suffix = suffix + self.filemode = filemode + self.w_loader = w_loader + + @staticmethod + def fromLoader(w_loader): + return FindInfo(IMP_HOOK, '', None, w_loader=w_loader) def find_module(space, modulename, w_modulename, partname, w_path, use_loader=True): - """If use_loader is False, returns (import_info, None) - if use_loader is True, may return (None, w_loader)""" # Examin importhooks (PEP302) before doing the import if use_loader: w_loader = find_in_meta_path(space, w_modulename, w_path) if w_loader: - return None, w_loader + return FindInfo.fromLoader(w_loader) # XXX Check for frozen modules? # when w_path is a string # check the builtin modules if modulename in space.builtin_modules: - return ImportInfo(C_BUILTIN, modulename, None), None + return FindInfo(C_BUILTIN, modulename, None) # XXX check frozen modules? # when w_path is null @@ -275,7 +285,7 @@ if use_loader: w_loader = find_in_path_hooks(space, w_modulename, w_pathitem) if w_loader: - return None, w_loader + return FindInfo.fromLoader(w_loader) path = space.str_w(w_pathitem) filepart = os.path.join(path, partname) @@ -283,7 +293,7 @@ initfile = os.path.join(filepart, '__init__') modtype, _, _ = find_modtype(space, initfile) if modtype in (PY_SOURCE, PY_COMPILED): - return ImportInfo(PKG_DIRECTORY, filepart, None), None + return FindInfo(PKG_DIRECTORY, filepart, None) else: msg = "Not importing directory " +\ "'%s' missing __init__.py" % (filepart,) @@ -294,57 +304,60 @@ filename = filepart + suffix stream = streamio.open_file_as_stream(filename, filemode) try: - return ImportInfo(modtype, filename, stream, suffix, filemode), None + return FindInfo(modtype, filename, stream, suffix, filemode) except: stream.close() raise except StreamErrors: pass - return None, None -def load_module(space, w_modulename, import_info, w_loader, ispkg=False): - if w_loader: - return space.call_method(w_loader, "load_module", w_modulename) - if import_info is None: + # not found + return None + +def load_module(space, w_modulename, find_info, ispkg=False): + if find_info is None: return - modtype, filename, stream, _, _ = import_info - if modtype == C_BUILTIN: - return space.getbuiltinmodule(filename) + if find_info.w_loader: + return space.call_method(find_info.w_loader, "load_module", w_modulename) - if modtype in (PY_SOURCE, PY_COMPILED, PKG_DIRECTORY): + if find_info.modtype == C_BUILTIN: + return space.getbuiltinmodule(find_info.filename) + + if find_info.modtype in (PY_SOURCE, PY_COMPILED, PKG_DIRECTORY): if ispkg: w_mod = space.getitem(space.sys.get('modules'), w_modulename) else: w_mod = space.wrap(Module(space, w_modulename)) space.sys.setmodule(w_mod) - space.setattr(w_mod, space.wrap('__file__'), space.wrap(filename)) + space.setattr(w_mod, space.wrap('__file__'), space.wrap(find_info.filename)) space.setattr(w_mod, space.wrap('__doc__'), space.w_None) - try: - if modtype == PY_SOURCE: - load_source_module(space, w_modulename, w_mod, filename, - stream.readall()) - return w_mod - elif modtype == PY_COMPILED: - magic = _r_long(stream) - timestamp = _r_long(stream) - load_compiled_module(space, w_modulename, w_mod, filename, - magic, timestamp, stream.readall()) - return w_mod - elif modtype == PKG_DIRECTORY: - w_path = space.newlist([space.wrap(filename)]) - space.setattr(w_mod, space.wrap('__path__'), w_path) - import_info, _ = find_module(space, "__init__", None, "__init__", - w_path, use_loader=False) - if import_info is None: + try: + if find_info.modtype == PY_SOURCE: + load_source_module(space, w_modulename, w_mod, find_info.filename, + find_info.stream.readall()) return w_mod - load_module(space, w_modulename, import_info, None, ispkg=True) - w_mod = check_sys_modules(space, w_modulename) # in case of "substitution" - return w_mod - except OperationError: - w_mods = space.sys.get('modules') - space.call_method(w_mods, 'pop', w_modulename, space.w_None) - raise + elif find_info.modtype == PY_COMPILED: + magic = _r_long(find_info.stream) + timestamp = _r_long(find_info.stream) + load_compiled_module(space, w_modulename, w_mod, find_info.filename, + magic, timestamp, find_info.stream.readall()) + return w_mod + elif find_info.modtype == PKG_DIRECTORY: + w_path = space.newlist([space.wrap(find_info.filename)]) + space.setattr(w_mod, space.wrap('__path__'), w_path) + find_info = find_module(space, "__init__", None, "__init__", + w_path, use_loader=False) + if find_info is None: + return w_mod + load_module(space, w_modulename, find_info, ispkg=True) + # fetch the module again, in case of "substitution" + w_mod = check_sys_modules(space, w_modulename) + return w_mod + except OperationError: + w_mods = space.sys.get('modules') + space.call_method(w_mods, 'pop', w_modulename, space.w_None) + raise def load_part(space, w_path, prefix, partname, w_parent, tentative): w = space.wrap @@ -355,18 +368,18 @@ if not space.is_w(w_mod, space.w_None): return w_mod else: - import_info, w_loader = find_module( + find_info = find_module( space, modulename, w_modulename, partname, w_path) try: - if (import_info, w_loader) != (None, None): - w_mod = load_module(space, w_modulename, import_info, w_loader) + if find_info: + w_mod = load_module(space, w_modulename, find_info) if w_parent is not None: space.setattr(w_parent, space.wrap(partname), w_mod) return w_mod finally: - if import_info: - _, _, stream, _, _ = import_info + if find_info: + stream = find_info.stream if stream: stream.close() @@ -406,15 +419,15 @@ parent_name,))) w_path = space.getitem(w_parent, space.wrap("__path")) - import_info, w_loader = find_module( + find_info = find_module( space, modulename, w_modulename, subname, w_path) - if (import_info, w_loader) == (None, None): + if not find_info: # ImportError msg = "No module named %s" % modulename raise OperationError(space.w_ImportError, space.wrap(msg)) - return load_module(space, w_modulename, import_info, w_loader) + return load_module(space, w_modulename, find_info) # __________________________________________________________________ # Modified: pypy/branch/import-builtin/pypy/module/imp/interp_imp.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/interp_imp.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/interp_imp.py Fri Dec 11 01:02:24 2009 @@ -14,7 +14,15 @@ ]) def get_magic(space): - return space.wrap(struct.pack('>= 8 + b = x & 0xff + x >>= 8 + c = x & 0xff + x >>= 8 + d = x & 0xff + return space.wrap(chr(a) + chr(b) + chr(c) + chr(d)) def get_file(space, w_file, filename, filemode): if w_file is None or space.is_w(w_file, space.w_None): @@ -27,26 +35,28 @@ if space.is_w(w_path, space.w_None): w_path = space.sys.get('path') - import_info, _ = importing.find_module( + find_info = importing.find_module( space, name, w_name, name, w_path, use_loader=False) - if import_info is None: + if not find_info: raise OperationError( space.w_ImportError, space.wrap("No module named %s" % (name,))) - modtype, filename, stream, suffix, filemode = import_info - w_filename = space.wrap(filename) + w_filename = space.wrap(find_info.filename) + stream = find_info.stream if stream is not None: fileobj = W_File(space) fileobj.fdopenstream( stream, stream.try_to_find_file_descriptor(), - filemode, w_filename) + find_info.filemode, w_filename) w_fileobj = space.wrap(fileobj) else: w_fileobj = space.w_None w_import_info = space.newtuple( - [space.wrap(suffix), space.wrap(filemode), space.wrap(modtype)]) + [space.wrap(find_info.suffix), + space.wrap(find_info.filemode), + space.wrap(find_info.modtype)]) return space.newtuple([w_fileobj, w_filename, w_import_info]) def load_module(space, w_name, w_file, w_filename, w_info): @@ -59,13 +69,14 @@ else: stream = get_file(space, w_file, filename, filemode) - import_info = (space.int_w(w_modtype), - filename, - stream, - space.str_w(w_suffix), - filemode) + find_info = importing.FindInfo( + space.int_w(w_modtype), + filename, + stream, + space.str_w(w_suffix), + filemode) return importing.load_module( - space, w_name, import_info, None) + space, w_name, find_info) def load_source(space, w_modulename, w_filename, w_file=None): filename = space.str_w(w_filename) From afa at codespeak.net Fri Dec 11 11:36:31 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 11 Dec 2009 11:36:31 +0100 (CET) Subject: [pypy-svn] r70065 - in pypy/branch/import-builtin/pypy/module: imp thread Message-ID: <20091211103631.D2CC116801F@codespeak.net> Author: afa Date: Fri Dec 11 11:36:30 2009 New Revision: 70065 Removed: pypy/branch/import-builtin/pypy/module/thread/importlock.py Modified: pypy/branch/import-builtin/pypy/module/imp/__init__.py pypy/branch/import-builtin/pypy/module/imp/importing.py pypy/branch/import-builtin/pypy/module/imp/interp_imp.py pypy/branch/import-builtin/pypy/module/thread/__init__.py Log: More translation fixes. Also move the thread._importlock_* functions to the imp module, as suggested by the XXX comment. Modified: pypy/branch/import-builtin/pypy/module/imp/__init__.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/__init__.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/__init__.py Fri Dec 11 11:36:30 2009 @@ -23,6 +23,10 @@ 'init_frozen': 'interp_imp.init_frozen', 'is_builtin': 'interp_imp.is_builtin', 'is_frozen': 'interp_imp.is_frozen', + + 'lock_held': 'interp_imp.lock_held', + 'acquire_lock': 'interp_imp.acquire_lock', + 'release_lock': 'interp_imp.release_lock', } appleveldefs = { Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Fri Dec 11 11:36:30 2009 @@ -314,6 +314,14 @@ # not found return None +def _prepare_module(space, w_mod, filename, pkgdir): + w = space.wrap + space.sys.setmodule(w_mod) + space.setattr(w_mod, w('__file__'), space.wrap(filename)) + space.setattr(w_mod, w('__doc__'), space.w_None) + if pkgdir is not None: + space.setattr(w_mod, w('__path__'), space.newlist([w(pkgdir)])) + def load_module(space, w_modulename, find_info, ispkg=False): if find_info is None: return @@ -328,9 +336,11 @@ w_mod = space.getitem(space.sys.get('modules'), w_modulename) else: w_mod = space.wrap(Module(space, w_modulename)) - space.sys.setmodule(w_mod) - space.setattr(w_mod, space.wrap('__file__'), space.wrap(find_info.filename)) - space.setattr(w_mod, space.wrap('__doc__'), space.w_None) + if find_info.modtype == PKG_DIRECTORY: + pkgdir = find_info.filename + else: + pkgdir = None + _prepare_module(space, w_mod, find_info.filename, pkgdir) try: if find_info.modtype == PY_SOURCE: Modified: pypy/branch/import-builtin/pypy/module/imp/interp_imp.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/interp_imp.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/interp_imp.py Fri Dec 11 11:36:30 2009 @@ -84,9 +84,7 @@ stream = get_file(space, w_file, filename, 'U') w_mod = space.wrap(Module(space, w_modulename)) - space.sys.setmodule(w_mod) - space.setattr(w_mod, space.wrap('__file__'), w_filename) - space.setattr(w_mod, space.wrap('__doc__'), space.w_None) + importing._prepare_module(space, w_mod, filename, None) importing.load_source_module( space, w_modulename, w_mod, filename, stream.readall()) @@ -100,9 +98,7 @@ stream = get_file(space, w_file, filename, 'rb') w_mod = space.wrap(Module(space, w_modulename)) - space.sys.setmodule(w_mod) - space.setattr(w_mod, space.wrap('__file__'), w_filename) - space.setattr(w_mod, space.wrap('__doc__'), space.w_None) + importing._prepare_module(space, w_mod, filename, None) magic = importing._r_long(stream) timestamp = importing._r_long(stream) @@ -140,3 +136,19 @@ def is_frozen(space, w_name): return space.w_False + +#__________________________________________________________________ + +def lock_held(space): + if space.config.objspace.usemodules.thread: + return space.wrap(importing.getimportlock(space).lock_held()) + else: + return False + +def acquire_lock(space): + if space.config.objspace.usemodules.thread: + importing.getimportlock(space).acquire_lock() + +def release_lock(space): + if space.config.objspace.usemodules.thread: + importing.getimportlock(space).release_lock() Modified: pypy/branch/import-builtin/pypy/module/thread/__init__.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/thread/__init__.py (original) +++ pypy/branch/import-builtin/pypy/module/thread/__init__.py Fri Dec 11 11:36:30 2009 @@ -18,11 +18,6 @@ 'allocate': 'os_lock.allocate_lock', # obsolete synonym 'LockType': 'os_lock.getlocktype(space)', '_local': 'os_local.getlocaltype(space)', - - # custom interface for the 'imp' module - '_importlock_held': 'importlock.held', - '_importlock_acquire': 'importlock.acquire', - '_importlock_release': 'importlock.release', } def __init__(self, space, *args): From arigo at codespeak.net Fri Dec 11 12:10:22 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 11 Dec 2009 12:10:22 +0100 (CET) Subject: [pypy-svn] r70066 - in pypy/branch/virtual-forcing/pypy/jit: backend/test metainterp metainterp/test Message-ID: <20091211111022.6F22216801F@codespeak.net> Author: arigo Date: Fri Dec 11 12:10:21 2009 New Revision: 70066 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/support.py pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_basic.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Log: Finally found a simple workaround for the next issue in the pypy-c-jit. It's quite a bit hackish but it is easy because it works purely locally in pyjitpl.py. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/support.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/test/support.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/test/support.py Fri Dec 11 12:10:21 2009 @@ -100,6 +100,9 @@ def check_aborted_count(self, *args, **kwds): pass + def check_aborted_count_at_least(self, *args, **kwds): + pass + def interp_operations(self, *args, **kwds): py.test.skip("interp_operations test skipped") Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py Fri Dec 11 12:10:21 2009 @@ -750,7 +750,7 @@ descr_virtualref_index = virtualref.get_descr_virtualref_index(cpu) # # Replace the VIRTUAL_REF operation with a virtual structure of type - # 'vref.JIT_VIRTUAL_REF'. The virtual structure may be forced soon, + # 'jit_virtual_ref'. The jit_virtual_ref structure may be forced soon, # but the point is that doing so does not force the original structure. op = ResOperation(rop.NEW_WITH_VTABLE, [c_cls], op.result) vrefvalue = self.make_virtual(c_cls, op.result, op) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Fri Dec 11 12:10:21 2009 @@ -917,8 +917,30 @@ lastbox = metainterp.virtualref_boxes.pop() assert box.getref_base() == lastbox.getref_base() if not metainterp.is_blackholing(): + if vrefbox.getref_base(): + metainterp.history.record(rop.VIRTUAL_REF_FINISH, + [vrefbox, lastbox], None) + + def cancel_tracking_virtual_ref(self, argboxes): + # if the most recent vref points to an object that escapes in a + # residual call (i.e. is in argboxes), then cancel its special + # treatment and allow it to escape. XXX check and document more, or + # find another approach -- see test_recursive_call in test_virtualref + metainterp = self.metainterp + if metainterp.is_blackholing(): + return + if len(metainterp.virtualref_boxes) == 0: + return + lastbox = metainterp.virtualref_boxes[-2] + if lastbox not in argboxes: + return + vrefbox = metainterp.virtualref_boxes[-1] + if vrefbox.getref_base(): metainterp.history.record(rop.VIRTUAL_REF_FINISH, [vrefbox, lastbox], None) + ConstRef = metainterp.cpu.ts.ConstRef + nullbox = ConstRef(ConstRef.value) + metainterp.virtualref_boxes[-1] = nullbox # ------------------------------ @@ -1023,6 +1045,7 @@ def do_residual_call(self, argboxes, descr, exc): effectinfo = descr.get_extra_info() if effectinfo is None or effectinfo.forces_virtual_or_virtualizable: + self.cancel_tracking_virtual_ref(argboxes) # residual calls require attention to keep virtualizables in-sync self.metainterp.vable_and_vrefs_before_residual_call() # xxx do something about code duplication @@ -1433,6 +1456,7 @@ self.staticdata.stats.aborted() self.staticdata.profiler.end_tracing() self.staticdata.profiler.start_blackhole() + switch_to_blackhole._dont_inline_ = True def switch_to_blackhole_if_trace_too_long(self): if not self.is_blackholing(): @@ -1766,9 +1790,11 @@ for i in range(1, len(self.virtualref_boxes), 2): vrefbox = self.virtualref_boxes[i] vref = vrefbox.getref_base() + if not vref: + continue virtualref.tracing_before_residual_call(vref) - # the FORCE_TOKEN is already set at runtime in each vrefs when - # they are created, by optimizeopt.py. + # the FORCE_TOKEN is already set at runtime in each vref when + # it is created, by optimizeopt.py. # vinfo = self.staticdata.virtualizable_info if vinfo is not None: @@ -1791,6 +1817,8 @@ for i in range(1, len(self.virtualref_boxes), 2): vrefbox = self.virtualref_boxes[i] vref = vrefbox.getref_base() + if not vref: + continue if virtualref.tracing_after_residual_call(vref): # this vref escaped during CALL_MAY_FORCE. escapes = True @@ -1849,6 +1877,8 @@ for i in range(0, len(virtualref_boxes), 2): virtualbox = virtualref_boxes[i] vrefbox = virtualref_boxes[i+1] + if not vrefbox.getref_base(): + continue virtualref.continue_tracing(vrefbox.getref_base(), virtualbox.getref_base()) # Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_basic.py Fri Dec 11 12:10:21 2009 @@ -52,6 +52,8 @@ assert get_stats().exec_jumps <= maxcount def check_aborted_count(self, count): assert get_stats().aborted_count == count + def check_aborted_count_at_least(self, count): + assert get_stats().aborted_count >= count def meta_interp(self, *args, **kwds): kwds['CPUClass'] = self.CPUClass Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Fri Dec 11 12:10:21 2009 @@ -106,7 +106,7 @@ # we call virtual_ref(x).) exctx.topframeref = vref_None virtual_ref_finish(x) - # 'vref' is allowed to escape, and even be forced, even after + # 'x' is allowed to escape, and even be forced, even after # the call to finish(). g(vref) n -= 1 @@ -144,6 +144,7 @@ res = self.meta_interp(f, [-4]) assert res == 16 * 19 self.check_loops({}) # because we aborted tracing + self.check_aborted_count_at_least(1) def test_simple_no_access(self): myjitdriver = JitDriver(greens = [], reds = ['n']) @@ -208,6 +209,7 @@ # self.meta_interp(f, [15]) self.check_loops({}) # because we aborted tracing + self.check_aborted_count_at_least(1) def test_simple_force_sometimes(self): myjitdriver = JitDriver(greens = [], reds = ['n']) @@ -344,7 +346,6 @@ exctx = ExCtx() # def f(n): - later = None while n > 0: myjitdriver.can_enter_jit(n=n) myjitdriver.jit_merge_point(n=n) @@ -359,6 +360,70 @@ res = self.meta_interp(f, [15]) assert res == 1 self.check_loops({}) # because we aborted tracing + self.check_aborted_count_at_least(1) + + def test_recursive_call_1(self): + myjitdriver = JitDriver(greens = [], reds = ['n', 'frame', 'rec']) + # + class XY: + pass + class ExCtx: + pass + exctx = ExCtx() + # + def f(frame, n, reclevel): + while n > 0: + myjitdriver.can_enter_jit(n=n, frame=frame, rec=reclevel) + myjitdriver.jit_merge_point(n=n, frame=frame, rec=reclevel) + if reclevel == 0: + return n + xy = XY() + exctx.topframeref = virtual_ref(xy) + m = f(xy, n, reclevel-1) + assert m == n + n -= 1 + exctx.topframeref = vref_None + virtual_ref_finish(xy) + return 2 + def main(n, reclevel): + return f(XY(), n, reclevel) + # + res = self.meta_interp(main, [15, 1]) + assert res == main(15, 1) + self.check_aborted_count(0) + + def test_recursive_call_2(self): + myjitdriver = JitDriver(greens = [], reds = ['n', 'frame', 'rec']) + # + class XY: + n = 0 + class ExCtx: + pass + exctx = ExCtx() + # + def f(frame, n, reclevel): + while n > 0: + myjitdriver.can_enter_jit(n=n, frame=frame, rec=reclevel) + myjitdriver.jit_merge_point(n=n, frame=frame, rec=reclevel) + frame.n += 1 + xy = XY() + xy.n = n + exctx.topframeref = virtual_ref(xy) + if reclevel > 0: + m = f(xy, frame.n, reclevel-1) + assert xy.n == m + n -= 1 + else: + n -= 2 + exctx.topframeref = vref_None + virtual_ref_finish(xy) + return frame.n + def main(n, reclevel): + return f(XY(), n, reclevel) + # + res = self.meta_interp(main, [10, 2]) + assert res == main(10, 2) + self.check_aborted_count(0) class TestLLtype(VRefTests, LLJitMixin): From arigo at codespeak.net Fri Dec 11 12:17:00 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 11 Dec 2009 12:17:00 +0100 (CET) Subject: [pypy-svn] r70067 - pypy/trunk/pypy/interpreter/test Message-ID: <20091211111700.55FAD16801F@codespeak.net> Author: arigo Date: Fri Dec 11 12:16:59 2009 New Revision: 70067 Modified: pypy/trunk/pypy/interpreter/test/test_module.py Log: Replace continuation lines ending with '\' with parenthesis. Seems that py.test is confused by '\' when repeating the evaluation. Modified: pypy/trunk/pypy/interpreter/test/test_module.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_module.py (original) +++ pypy/trunk/pypy/interpreter/test/test_module.py Fri Dec 11 12:16:59 2009 @@ -56,10 +56,10 @@ import _pypy_interact # known to be in pypy/lib r = repr(_pypy_interact) - assert r.startswith("') + assert (r.startswith("')) nofile = type(_pypy_interact)('nofile', 'foo') assert repr(nofile) == "" From afa at codespeak.net Fri Dec 11 14:01:55 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 11 Dec 2009 14:01:55 +0100 (CET) Subject: [pypy-svn] r70068 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091211130155.C9ADB16801F@codespeak.net> Author: afa Date: Fri Dec 11 14:01:55 2009 New Revision: 70068 Modified: pypy/trunk/pypy/module/oracle/interp_connect.py pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/interp_object.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_connect.py pypy/trunk/pypy/module/oracle/transform.py Log: Translation fixes for the Oracle module, on a 64bit platform with the oracle9 client Modified: pypy/trunk/pypy/module/oracle/interp_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_connect.py (original) +++ pypy/trunk/pypy/module/oracle/interp_connect.py Fri Dec 11 14:01:55 2009 @@ -315,23 +315,26 @@ mode |= roci.OCI_SESSGET_CREDEXT # set the connection class, if applicable - stringBuffer.fill(space, w_cclass) - try: - if stringBuffer.size > 0: - externalCredentials = False - status = roci.OCIAttrSet( - authInfo, - roci.OCI_HTYPE_AUTHINFO, - stringBuffer.ptr, stringBuffer.size, - roci.OCI_ATTR_CONNECTION_CLASS, - self.environment.errorHandle) - self.environment.checkForError( - status, "Connection_GetConnection(): set connection class") - finally: - stringBuffer.clear() + if roci.OCI_ATTR_CONNECTION_CLASS is not None: + stringBuffer.fill(space, w_cclass) + try: + if stringBuffer.size > 0: + externalCredentials = False + status = roci.OCIAttrSet( + authInfo, + roci.OCI_HTYPE_AUTHINFO, + stringBuffer.ptr, stringBuffer.size, + roci.OCI_ATTR_CONNECTION_CLASS, + self.environment.errorHandle) + self.environment.checkForError( + status, + "Connection_GetConnection(): set connection class") + finally: + stringBuffer.clear() # set the purity, if applicable - if purity != roci.OCI_ATTR_PURITY_DEFAULT: + if (roci.OCI_ATTR_PURITY is not None + and purity != roci.OCI_ATTR_PURITY_DEFAULT): purityptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, 1, flavor='raw') purityptr[0] = rffi.cast(roci.ub4, purity) Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Fri Dec 11 14:01:55 2009 @@ -927,12 +927,13 @@ status, "Cursor_GetBindNames()") # Too few elements allocated - if foundElementsPtr[0] < 0: - return -foundElementsPtr[0], None + foundElements = rffi.cast(lltype.Signed, foundElementsPtr[0]) + if foundElements < 0: + return -foundElements, None names_w = [] # process the bind information returned - for i in range(foundElementsPtr[0]): + for i in range(foundElements): if rffi.cast(lltype.Signed, duplicate[i]): continue names_w.append( Modified: pypy/trunk/pypy/module/oracle/interp_object.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_object.py (original) +++ pypy/trunk/pypy/module/oracle/interp_object.py Fri Dec 11 14:01:55 2009 @@ -491,7 +491,7 @@ status, "ExternalObjectVar_ConvertCollection(): get next") - if eofptr[0]: + if rffi.cast(lltype.Signed, eofptr[0]): break element = convertObject( space, environment, Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Fri Dec 11 14:01:55 2009 @@ -1018,7 +1018,7 @@ self.environment.checkForError( status, "LobVar_SetValue(): check temporary") - temporary = temporaryptr[0] + temporary = rffi.cast(lltype.Signed, temporaryptr[0]) finally: lltype.free(temporaryptr, flavor='raw') @@ -1373,8 +1373,17 @@ variableTypeNChar = { roci.SQLT_AFC: VT_FixedNationalChar, roci.SQLT_CHR: VT_NationalCharString, - roci.SQLT_CLOB: VT_NCLOB, + roci.SQLT_CLOB: VT_NCLOB, } +# remove eventual undefined types +try: + del variableType[None] +except KeyError: + pass +try: + del variableTypeNChar[None] +except KeyError: + pass def _typeByOracleDataType(dataType, charsetForm): if charsetForm == roci.SQLCS_NCHAR: Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Fri Dec 11 14:01:55 2009 @@ -18,7 +18,10 @@ libraries = ['oci'] library_dirs = [str(ORACLE_HOME.join('OCI', 'lib', 'MSVC'))] else: - include_dirs = [str(ORACLE_HOME.join('sdk', 'include'))] + include_dirs = [str(ORACLE_HOME.join('sdk', 'include')), # Oracle 11 + str(ORACLE_HOME.join('rdbms', 'demo')), # Oracle 9 + str(ORACLE_HOME.join('rdbms', 'public')), # Oracle 9 + ] libraries = ['clntsh'] library_dirs = [str(ORACLE_HOME.join('lib'))] @@ -67,14 +70,7 @@ ('OCIDateTime', OCITime), ]) - constants = ''' - OCI_DEFAULT OCI_OBJECT OCI_THREADED OCI_EVENTS - OCI_SUCCESS OCI_SUCCESS_WITH_INFO OCI_INVALID_HANDLE OCI_NO_DATA - OCI_HTYPE_ERROR OCI_HTYPE_SVCCTX OCI_HTYPE_SERVER OCI_HTYPE_SESSION - OCI_HTYPE_STMT OCI_HTYPE_DESCRIBE OCI_HTYPE_BIND OCI_HTYPE_DEFINE - OCI_HTYPE_ENV OCI_HTYPE_SPOOL OCI_HTYPE_AUTHINFO OCI_ATTR_CONNECTION_CLASS - OCI_DTYPE_PARAM OCI_DTYPE_TIMESTAMP OCI_DTYPE_INTERVAL_DS OCI_DTYPE_LOB - OCI_CRED_RDBMS OCI_CRED_EXT OCI_SPOOL_ATTRVAL_NOWAIT + defines = ''' OCI_ATTR_SERVER OCI_ATTR_SESSION OCI_ATTR_USERNAME OCI_ATTR_PASSWORD OCI_ATTR_STMT_TYPE OCI_ATTR_PARAM OCI_ATTR_PARAM_COUNT OCI_ATTR_ROW_COUNT OCI_ATTR_NAME OCI_ATTR_SCALE OCI_ATTR_PRECISION OCI_ATTR_IS_NULL @@ -85,7 +81,23 @@ OCI_ATTR_CHARSET_FORM OCI_ATTR_CHARSET_ID OCI_ATTR_ENV_CHARSET_ID OCI_ATTR_PARSE_ERROR_OFFSET OCI_ATTR_SPOOL_OPEN_COUNT OCI_ATTR_SPOOL_BUSY_COUNT OCI_ATTR_SPOOL_TIMEOUT - OCI_ATTR_SPOOL_GETMODE OCI_ATTR_PURITY + OCI_ATTR_SPOOL_GETMODE OCI_ATTR_PURITY OCI_ATTR_CONNECTION_CLASS + OCI_ATTR_PURITY_DEFAULT + SQLT_CHR SQLT_LNG SQLT_AFC SQLT_RDD SQLT_BIN SQLT_LBI SQLT_LVC SQLT_LVB + SQLT_BFLOAT SQLT_IBFLOAT SQLT_BDOUBLE SQLT_IBDOUBLE + SQLT_NUM SQLT_VNU SQLT_DAT SQLT_ODT SQLT_DATE SQLT_TIMESTAMP + SQLT_TIMESTAMP_TZ SQLT_TIMESTAMP_LTZ SQLT_INTERVAL_DS + SQLT_CLOB SQLT_CLOB SQLT_BLOB SQLT_BFILE SQLT_RSET SQLT_NTY + '''.split() + + constants = ''' + OCI_DEFAULT OCI_OBJECT OCI_THREADED OCI_EVENTS + OCI_SUCCESS OCI_SUCCESS_WITH_INFO OCI_INVALID_HANDLE OCI_NO_DATA + OCI_HTYPE_ERROR OCI_HTYPE_SVCCTX OCI_HTYPE_SERVER OCI_HTYPE_SESSION + OCI_HTYPE_STMT OCI_HTYPE_DESCRIBE OCI_HTYPE_BIND OCI_HTYPE_DEFINE + OCI_HTYPE_ENV OCI_HTYPE_SPOOL OCI_HTYPE_AUTHINFO + OCI_DTYPE_PARAM OCI_DTYPE_TIMESTAMP OCI_DTYPE_INTERVAL_DS OCI_DTYPE_LOB + OCI_CRED_RDBMS OCI_CRED_EXT OCI_SPOOL_ATTRVAL_NOWAIT OCI_NTV_SYNTAX OCI_COMMIT_ON_SUCCESS OCI_FETCH_NEXT OCI_IND_NULL OCI_IND_NOTNULL @@ -93,11 +105,6 @@ OCI_OTYPE_PTR OCI_PTYPE_TYPE OCI_STMT_SELECT OCI_STMT_CREATE OCI_STMT_DROP OCI_STMT_ALTER OCI_STMT_INSERT OCI_STMT_DELETE OCI_STMT_UPDATE - SQLT_CHR SQLT_LNG SQLT_AFC SQLT_RDD SQLT_BIN SQLT_LBI SQLT_LVC SQLT_LVB - SQLT_BFLOAT SQLT_IBFLOAT SQLT_BDOUBLE SQLT_IBDOUBLE - SQLT_NUM SQLT_VNU SQLT_DAT SQLT_ODT SQLT_DATE SQLT_TIMESTAMP - SQLT_TIMESTAMP_TZ SQLT_TIMESTAMP_LTZ SQLT_INTERVAL_DS - SQLT_CLOB SQLT_CLOB SQLT_BLOB SQLT_BFILE SQLT_RSET SQLT_NTY SQLCS_IMPLICIT SQLCS_NCHAR OCI_TEMP_CLOB OCI_TEMP_BLOB OCI_DURATION_SESSION OCI_ONE_PIECE OCI_NUMBER_SIGNED @@ -107,9 +114,11 @@ OCI_NLS_MAXBUFSZ OCI_NLS_CS_ORA_TO_IANA OCI_UTF16ID OCI_SPC_STMTCACHE OCI_SPC_HOMOGENEOUS OCI_SESSGET_SPOOL OCI_SESSGET_CREDPROXY OCI_SESSGET_STMTCACHE - OCI_SESSGET_CREDEXT OCI_SESSRLS_DROPSESS OCI_ATTR_PURITY_DEFAULT + OCI_SESSGET_CREDEXT OCI_SESSRLS_DROPSESS '''.split() + for c in defines: + locals()[c] = platform.DefinedConstantInteger(c) for c in constants: locals()[c] = platform.ConstantInteger(c) Modified: pypy/trunk/pypy/module/oracle/test/test_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_connect.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_connect.py Fri Dec 11 14:01:55 2009 @@ -160,4 +160,6 @@ e = raises(oracle.DatabaseError, pool.acquire, user="proxyuser") # ORA-01017: invalid username/password; logon denied # ORA-28150: proxy not authorized to connect as client - assert e.value[0].code in (1017, 28150) + # ORA-01031: insufficient privileges + print "Error code", e.value[0].code + assert e.value[0].code in (1017, 28150, 1031) Modified: pypy/trunk/pypy/module/oracle/transform.py ============================================================================== --- pypy/trunk/pypy/module/oracle/transform.py (original) +++ pypy/trunk/pypy/module/oracle/transform.py Fri Dec 11 14:01:55 2009 @@ -89,7 +89,7 @@ w_datetime, w(yearptr[0]), w(monthptr[0]), w(dayptr[0]), w(hourptr[0]), w(minuteptr[0]), w(secondptr[0]), - w(fsecondptr[0] / 1000)) + w(rffi.cast(lltype.Signed, fsecondptr[0]) / 1000)) finally: lltype.free(yearptr, flavor='raw') lltype.free(monthptr, flavor='raw') @@ -121,8 +121,10 @@ w('timedelta')) days = daysptr[0] - seconds = hoursptr[0] * 3600 + minutesptr[0] * 60 + secondsptr[0] - microseconds = fsecondsptr[0] / 1000 + seconds = (rffi.cast(lltype.Signed, hoursptr[0]) * 3600 + + rffi.cast(lltype.Signed, minutesptr[0]) * 60 + + rffi.cast(lltype.Signed, secondsptr[0])) + microseconds = rffi.cast(lltype.Signed, fsecondsptr[0]) / 1000 return space.call_function( w_timedelta, From afa at codespeak.net Fri Dec 11 14:37:51 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 11 Dec 2009 14:37:51 +0100 (CET) Subject: [pypy-svn] r70069 - pypy/branch/import-builtin/pypy/module/imp Message-ID: <20091211133751.B6E25168024@codespeak.net> Author: afa Date: Fri Dec 11 14:37:51 2009 New Revision: 70069 Modified: pypy/branch/import-builtin/pypy/module/imp/__init__.py pypy/branch/import-builtin/pypy/module/imp/importing.py Log: fixes for python test suite Modified: pypy/branch/import-builtin/pypy/module/imp/__init__.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/__init__.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/__init__.py Fri Dec 11 14:37:51 2009 @@ -12,7 +12,7 @@ 'C_BUILTIN': 'space.wrap(importing.C_BUILTIN)', 'get_suffixes': 'interp_imp.get_suffixes', - #'get_magic': 'interp_imp.get_magic', + 'get_magic': 'interp_imp.get_magic', 'find_module': 'interp_imp.find_module', 'load_module': 'interp_imp.load_module', 'load_source': 'interp_imp.load_source', Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Fri Dec 11 14:37:51 2009 @@ -419,7 +419,6 @@ subname = namepath[-1] parent_name = '.'.join(namepath[:-1]) parent = None - w_path = None if parent_name: w_parent = check_sys_modules(space, space.wrap(parent_name)) if w_parent is None: @@ -428,6 +427,8 @@ space.wrap("reload(): parent %s not in sys.modules" % ( parent_name,))) w_path = space.getitem(w_parent, space.wrap("__path")) + else: + w_path = space.sys.get('path') find_info = find_module( space, modulename, w_modulename, subname, w_path) From afa at codespeak.net Fri Dec 11 17:43:08 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 11 Dec 2009 17:43:08 +0100 (CET) Subject: [pypy-svn] r70071 - in pypy/branch/import-builtin/pypy: config doc/config module/imp module/zipimport/test Message-ID: <20091211164308.4230116801D@codespeak.net> Author: afa Date: Fri Dec 11 17:43:07 2009 New Revision: 70071 Added: pypy/branch/import-builtin/pypy/doc/config/objspace.usemodules.imp.txt Modified: pypy/branch/import-builtin/pypy/config/pypyoption.py pypy/branch/import-builtin/pypy/module/imp/__init__.py pypy/branch/import-builtin/pypy/module/imp/importing.py pypy/branch/import-builtin/pypy/module/zipimport/test/test_zipimport.py Log: More fixes after running tests on builtbot Modified: pypy/branch/import-builtin/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/import-builtin/pypy/config/pypyoption.py (original) +++ pypy/branch/import-builtin/pypy/config/pypyoption.py Fri Dec 11 17:43:07 2009 @@ -16,7 +16,7 @@ default_modules = essential_modules.copy() default_modules.update(dict.fromkeys( - ["_codecs", "gc", "_weakref", "marshal", "errno", + ["_codecs", "gc", "_weakref", "marshal", "errno", "imp", "math", "_sre", "_pickle_support", "operator", "parser", "symbol", "token", "_ast", "_random", "__pypy__", "_testing"])) Added: pypy/branch/import-builtin/pypy/doc/config/objspace.usemodules.imp.txt ============================================================================== --- (empty file) +++ pypy/branch/import-builtin/pypy/doc/config/objspace.usemodules.imp.txt Fri Dec 11 17:43:07 2009 @@ -0,0 +1,2 @@ +Use the 'imp' module. +This module is included by default. Modified: pypy/branch/import-builtin/pypy/module/imp/__init__.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/__init__.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/__init__.py Fri Dec 11 17:43:07 2009 @@ -10,6 +10,7 @@ 'PY_COMPILED': 'space.wrap(importing.PY_COMPILED)', 'PKG_DIRECTORY': 'space.wrap(importing.PKG_DIRECTORY)', 'C_BUILTIN': 'space.wrap(importing.C_BUILTIN)', + 'C_EXTENSION': 'space.wrap(importing.C_EXTENSION)', 'get_suffixes': 'interp_imp.get_suffixes', 'get_magic': 'interp_imp.get_magic', Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Fri Dec 11 17:43:07 2009 @@ -17,9 +17,13 @@ SEARCH_ERROR = 0 PY_SOURCE = 1 PY_COMPILED = 2 -PKG_DIRECTORY = 3 -C_BUILTIN = 4 -IMP_HOOK = 5 +C_EXTENSION = 3 +# PY_RESOURCE = 4 +PKG_DIRECTORY = 5 +C_BUILTIN = 6 +# PY_FROZEN = 7 +# PY_CODERESOURCE = 8 +IMP_HOOK = 9 def find_modtype(space, filepart): """Check which kind of module to import for the given filepart, Modified: pypy/branch/import-builtin/pypy/module/zipimport/test/test_zipimport.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/zipimport/test/test_zipimport.py (original) +++ pypy/branch/import-builtin/pypy/module/zipimport/test/test_zipimport.py Fri Dec 11 17:43:07 2009 @@ -4,7 +4,7 @@ import py import time import struct -from pypy.module.__builtin__.importing import get_pyc_magic, _w_long +from pypy.module.imp.importing import get_pyc_magic, _w_long from StringIO import StringIO from pypy.tool.udir import udir From arigo at codespeak.net Fri Dec 11 17:58:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 11 Dec 2009 17:58:42 +0100 (CET) Subject: [pypy-svn] r70072 - in pypy/branch/virtual-forcing/pypy: jit/backend/llgraph jit/backend/test jit/backend/x86 jit/metainterp jit/metainterp/test rlib Message-ID: <20091211165842.B1D7E168012@codespeak.net> Author: arigo Date: Fri Dec 11 17:58:41 2009 New Revision: 70072 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py pypy/branch/virtual-forcing/pypy/rlib/jit.py Log: Add a new operation 'virtual_ref_check', documented in rlib/jit. Implement it to replace the hack of the previous checkin. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py Fri Dec 11 17:58:41 2009 @@ -154,7 +154,7 @@ 'call_may_force' : (('int', 'varargs'), 'intorptr'), 'guard_not_forced': ((), None), 'virtual_ref' : (('ref', 'int'), 'ref'), - 'virtual_ref_finish':(('ref', 'ref'), None), + 'virtual_ref_check': (('varargs',), None), #'getitem' : (('void', 'ref', 'int'), 'int'), #'setitem' : (('void', 'ref', 'int', 'int'), None), #'newlist' : (('void', 'varargs'), 'ref'), @@ -818,7 +818,7 @@ def op_virtual_ref(self, _, virtual, index): return virtual - def op_virtual_ref_finish(self, _, vref, virtual): + def op_virtual_ref_check(self, _, *args): pass Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Fri Dec 11 17:58:41 2009 @@ -786,6 +786,12 @@ 'ref') assert r.value == u_box.value + def test_virtual_ref_check(self): + # if VIRTUAL_REF_CHECK reaches the backend, it is a no-op + self.execute_operation(rop.VIRTUAL_REF_CHECK, + [BoxInt(123), BoxInt(234)], + 'void') + def test_jump(self): # this test generates small loops where the JUMP passes many # arguments of various types, shuffling them around. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py Fri Dec 11 17:58:41 2009 @@ -928,6 +928,9 @@ def consider_debug_merge_point(self, op, ignored): pass + def consider_virtual_ref_check(self, op, ignored): + self.possibly_free_vars(op.args) + def get_mark_gc_roots(self, gcrootmap): shape = gcrootmap.get_basic_shape() for v, val in self.sm.stack_bindings.items(): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py Fri Dec 11 17:58:41 2009 @@ -1158,16 +1158,20 @@ self.var_position(op.args[3])) def serialize_op_jit_marker(self, op): - if op.args[0].value == 'jit_merge_point': - assert self.portal, "jit_merge_point in non-main graph!" - self.emit('jit_merge_point') - assert ([self.var_position(i) for i in op.args[2:]] == - range(0, 2*(len(op.args) - 2), 2)) - #for i in range(2, len(op.args)): - # arg = op.args[i] - # self._eventualy_builtin(arg) - elif op.args[0].value == 'can_enter_jit': - self.emit('can_enter_jit') + key = op.args[0].value + getattr(self, 'handle_jit_marker__%s' % key)(op) + + def handle_jit_marker__jit_merge_point(self, op): + assert self.portal, "jit_merge_point in non-main graph!" + self.emit('jit_merge_point') + assert ([self.var_position(i) for i in op.args[2:]] == + range(0, 2*(len(op.args) - 2), 2)) + + def handle_jit_marker__can_enter_jit(self, op): + self.emit('can_enter_jit') + + def handle_jit_marker__virtual_ref_check(self, op): + self.emit('virtual_ref_check') def serialize_op_direct_call(self, op): kind = self.codewriter.guess_call_kind(op) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py Fri Dec 11 17:58:41 2009 @@ -226,7 +226,7 @@ def do_virtual_ref(cpu, box1, box2): raise NotImplementedError -def do_virtual_ref_finish(cpu, box1, box2): +def do_virtual_ref_check(cpu): raise NotImplementedError def do_debug_merge_point(cpu, box1): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py Fri Dec 11 17:58:41 2009 @@ -759,26 +759,29 @@ vrefvalue.setfield(descr_virtual_token, self.getvalue(tokenbox)) vrefvalue.setfield(descr_virtualref_index, self.getvalue(indexbox)) - def optimize_VIRTUAL_REF_FINISH(self, op): - value = self.getvalue(op.args[1]) - if not value.is_virtual(): # virtual_ref(non-virtual) gives bad - raise compile.GiveUp # results, so don't bother compiling it - # - # Set the 'forced' field of the virtual_ref. - # In good cases, this is all virtual, so has no effect. - # Otherwise, this forces the real object -- but only now, as - # opposed to much earlier. This is important because the object is - # typically a PyPy PyFrame, and now is the end of its execution, so - # forcing it now does not have catastrophic effects. - from pypy.jit.metainterp import virtualref - # - set 'forced' to point to the real object - op1 = ResOperation(rop.SETFIELD_GC, op.args, None, - descr = virtualref.get_descr_forced(self.cpu)) - self.optimize_SETFIELD_GC(op1) - # - set 'virtual_token' to TOKEN_NONE - op1 = ResOperation(rop.SETFIELD_GC, [op.args[0], ConstInt(0)], None, + def optimize_VIRTUAL_REF_CHECK(self, op): + for i in range(0, len(op.args), 2): + value = self.getvalue(op.args[i]) + if not value.is_virtual(): # virtual_ref(non-virtual) means it + raise compile.GiveUp # escaped already, which is bad + # + # Set the 'forced' field of the virtual_ref. + # In good cases, this is all virtual, so has no effect. + # Otherwise, this forces the real object -- but only now, as + # opposed to much earlier. This is important because the object is + # typically a PyPy PyFrame, and now is the end of its execution, so + # forcing it now does not have catastrophic effects. + from pypy.jit.metainterp import virtualref + # - set 'forced' to point to the real object + args = [op.args[i+1], op.args[i]] + op1 = ResOperation(rop.SETFIELD_GC, args, None, + descr = virtualref.get_descr_forced(self.cpu)) + self.optimize_SETFIELD_GC(op1) + # - set 'virtual_token' to TOKEN_NONE + args = [op.args[i+1], ConstInt(0)] + op1 = ResOperation(rop.SETFIELD_GC, args, None, descr = virtualref.get_descr_virtual_token(self.cpu)) - self.optimize_SETFIELD_GC(op1) + self.optimize_SETFIELD_GC(op1) def optimize_GETFIELD_GC(self, op): value = self.getvalue(op.args[0]) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Fri Dec 11 17:58:41 2009 @@ -836,6 +836,7 @@ except GiveUp: self.metainterp.staticdata.profiler.count(ABORT_BRIDGE) self.metainterp.switch_to_blackhole() + self.metainterp.generate_virtual_ref_check() if self.metainterp.is_blackholing(): self.blackhole_reached_merge_point(self.env) return True @@ -888,6 +889,22 @@ @arguments("box") def opimpl_virtual_ref(self, box): + # Details on the content of metainterp.virtualref_boxes: + # + # * it's a list whose items go two by two, containing first the + # virtual box (e.g. the PyFrame) and then the vref box (e.g. + # the 'virtual_ref(frame)'). + # + # * if we detect that the virtual box escapes during tracing + # already (by generating a CALl_MAY_FORCE that marks the flags + # in the vref), then we replace the vref in the list with + # ConstPtr(NULL). + # + # * at the next virtual_ref_check() or jit_merge_point, any such + # NULL causes tracing to abort. So escaping is fine if there + # is no virtual_ref_check() or jit_merge_point before the final + # call to virtual_ref_finish(). + # metainterp = self.metainterp if metainterp.is_blackholing(): resbox = box # good enough when blackholing @@ -913,34 +930,19 @@ # virtual_ref_finish() assumes that we have a stack-like, last-in # first-out order. metainterp = self.metainterp + if not metainterp.is_blackholing(): + vrefbox = metainterp.virtualref_boxes[-1] + vref = vrefbox.getref_base() + if virtualref.is_virtual_ref(vref): + metainterp.stop_tracking_virtualref(-2) + # vrefbox = metainterp.virtualref_boxes.pop() lastbox = metainterp.virtualref_boxes.pop() assert box.getref_base() == lastbox.getref_base() - if not metainterp.is_blackholing(): - if vrefbox.getref_base(): - metainterp.history.record(rop.VIRTUAL_REF_FINISH, - [vrefbox, lastbox], None) - - def cancel_tracking_virtual_ref(self, argboxes): - # if the most recent vref points to an object that escapes in a - # residual call (i.e. is in argboxes), then cancel its special - # treatment and allow it to escape. XXX check and document more, or - # find another approach -- see test_recursive_call in test_virtualref - metainterp = self.metainterp - if metainterp.is_blackholing(): - return - if len(metainterp.virtualref_boxes) == 0: - return - lastbox = metainterp.virtualref_boxes[-2] - if lastbox not in argboxes: - return - vrefbox = metainterp.virtualref_boxes[-1] - if vrefbox.getref_base(): - metainterp.history.record(rop.VIRTUAL_REF_FINISH, - [vrefbox, lastbox], None) - ConstRef = metainterp.cpu.ts.ConstRef - nullbox = ConstRef(ConstRef.value) - metainterp.virtualref_boxes[-1] = nullbox + + @arguments() + def opimpl_virtual_ref_check(self): + self.metainterp.generate_virtual_ref_check() # ------------------------------ @@ -1045,7 +1047,6 @@ def do_residual_call(self, argboxes, descr, exc): effectinfo = descr.get_extra_info() if effectinfo is None or effectinfo.forces_virtual_or_virtualizable: - self.cancel_tracking_virtual_ref(argboxes) # residual calls require attention to keep virtualizables in-sync self.metainterp.vable_and_vrefs_before_residual_call() # xxx do something about code duplication @@ -1790,8 +1791,6 @@ for i in range(1, len(self.virtualref_boxes), 2): vrefbox = self.virtualref_boxes[i] vref = vrefbox.getref_base() - if not vref: - continue virtualref.tracing_before_residual_call(vref) # the FORCE_TOKEN is already set at runtime in each vref when # it is created, by optimizeopt.py. @@ -1817,11 +1816,9 @@ for i in range(1, len(self.virtualref_boxes), 2): vrefbox = self.virtualref_boxes[i] vref = vrefbox.getref_base() - if not vref: - continue if virtualref.tracing_after_residual_call(vref): # this vref escaped during CALL_MAY_FORCE. - escapes = True + self.stop_tracking_virtualref(i-1) # vinfo = self.staticdata.virtualizable_info if vinfo is not None: @@ -1837,6 +1834,44 @@ if escapes: self.load_fields_from_virtualizable() + def generate_virtual_ref_check(self): + if self.is_blackholing(): + return + # first, abort tracing if we reach this point with an escaped virtual + for i in range(1, len(self.virtualref_boxes), 2): + vrefbox = self.virtualref_boxes[i] + vref = vrefbox.getref_base() + if not virtualref.is_virtual_ref(vref): + self.switch_to_blackhole() + return + # then record VIRTUAL_REF_CHECK, with no argument so far. + # Arguments may be added by stop_tracking_virtualref(). + self.history.record(rop.VIRTUAL_REF_CHECK, [], None) + + def stop_tracking_virtualref(self, j): + # we look for the most recent VIRTUAL_REF_CHECK marker, and add + # there the pair virtualbox/vrefbox as arguments. + virtualbox = self.virtualref_boxes[j] + vrefbox = self.virtualref_boxes[j+1] + assert virtualref.is_virtual_ref(vrefbox.getref_base()) + self.virtualref_boxes[j+1] = self.cpu.ts.CONST_NULL + operations = self.history.operations + for i in range(len(operations)-1, -1, -1): + op = operations[i] + if op.opnum == rop.VIRTUAL_REF_CHECK: + # found + op.args = op.args + [virtualbox, vrefbox] + break + if op.result is vrefbox: + # no VIRTUAL_REF_CHECK exists before the VIRTUAL_REF + # that created this vref. Replace it with a mere SAME_AS. + if op.opnum == rop.VIRTUAL_REF: + op.opnum = rop.SAME_AS + op.args = [op.args[0]] + break + else: + pass # not found at all! nothing to do, just ignore it + def handle_exception(self): etype = self.cpu.get_exception() evalue = self.cpu.get_exc_value() @@ -1877,8 +1912,6 @@ for i in range(0, len(virtualref_boxes), 2): virtualbox = virtualref_boxes[i] vrefbox = virtualref_boxes[i+1] - if not vrefbox.getref_base(): - continue virtualref.continue_tracing(vrefbox.getref_base(), virtualbox.getref_base()) # Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py Fri Dec 11 17:58:41 2009 @@ -223,7 +223,7 @@ 'COND_CALL_GC_MALLOC', # [a, b, if_(a<=b)_result, if_(a>b)_call, args...] # => result (for mallocs) 'DEBUG_MERGE_POINT/1', # debugging only - 'VIRTUAL_REF_FINISH/2', + 'VIRTUAL_REF_CHECK', '_CANRAISE_FIRST', # ----- start of can_raise operations ----- 'CALL', Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Fri Dec 11 17:58:41 2009 @@ -1,7 +1,7 @@ import py from pypy.rpython.lltypesystem import lltype, llmemory, lloperation -from pypy.rlib.jit import JitDriver, dont_look_inside -from pypy.rlib.jit import virtual_ref, virtual_ref_finish, vref_None +from pypy.rlib.jit import JitDriver, dont_look_inside, vref_None +from pypy.rlib.jit import virtual_ref, virtual_ref_check, virtual_ref_finish from pypy.rlib.objectmodel import compute_unique_id from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.jit.metainterp.resoperation import rop @@ -22,6 +22,7 @@ def f(): x = X() exctx.topframeref = virtual_ref(x) + virtual_ref_check() exctx.topframeref = vref_None virtual_ref_finish(x) return 1 @@ -29,7 +30,7 @@ self.interp_operations(f, []) self.check_operations_history(new_with_vtable=1, # X() virtual_ref=1, - virtual_ref_finish=1) + virtual_ref_check=1) def test_make_vref_guard(self): if not isinstance(self, TestLLtype): @@ -53,6 +54,7 @@ exctx.topframeref = virtual_ref(x) def leave(): exctx.topframeref = vref_None + virtual_ref_check() virtual_ref_finish(exctx._frame) def f(n): enter(n) @@ -105,6 +107,7 @@ # here, 'x' should be virtual. (This is ensured because # we call virtual_ref(x).) exctx.topframeref = vref_None + virtual_ref_check() virtual_ref_finish(x) # 'x' is allowed to escape, and even be forced, even after # the call to finish(). @@ -114,8 +117,72 @@ # self.meta_interp(f, [10]) self.check_loops(new_with_vtable=2) # the vref, and later the X + self.check_aborted_count(0) + + def test_make_vref_and_force_nocheck_1(self): + jitdriver = JitDriver(greens = [], reds = ['total', 'n']) + # + class X: + pass + class ExCtx: + pass + exctx = ExCtx() + # + @dont_look_inside + def force_me(): + return exctx.topframeref().n + # + def f(n): + total = 0 + while total < 300: + jitdriver.can_enter_jit(total=total, n=n) + jitdriver.jit_merge_point(total=total, n=n) + x = X() + x.n = n + 123 + exctx.topframeref = virtual_ref(x) + # --- no virtual_ref_check() here --- + total += force_me() - 100 + virtual_ref_finish(x) + exctx.topframeref = vref_None + return total + # + res = self.meta_interp(f, [-4]) + assert res == 16 * 19 + self.check_aborted_count(0) + + def test_make_vref_and_force_nocheck_2(self): + jitdriver = JitDriver(greens = [], reds = ['total', 'n']) + # + class X: + pass + class ExCtx: + pass + exctx = ExCtx() + # + @dont_look_inside + def force_me(): + return exctx.topframeref().n + # + def f(n): + total = 0 + while total < 300: + jitdriver.can_enter_jit(total=total, n=n) + jitdriver.jit_merge_point(total=total, n=n) + x = X() + x.n = n + 123 + exctx.topframeref = virtual_ref(x) + virtual_ref_check() + total += force_me() - 100 + # --- but no virtual_ref_check() there --- + virtual_ref_finish(x) + exctx.topframeref = vref_None + return total + # + res = self.meta_interp(f, [-4]) + assert res == 16 * 19 + self.check_aborted_count(0) - def test_make_vref_and_force(self): + def test_make_vref_and_force_check(self): jitdriver = JitDriver(greens = [], reds = ['total', 'n']) # class X: @@ -137,6 +204,7 @@ x.n = n + 123 exctx.topframeref = virtual_ref(x) total += force_me() - 100 + virtual_ref_check() virtual_ref_finish(x) exctx.topframeref = vref_None return total @@ -175,11 +243,13 @@ xy.next1 = None xy.next2 = None xy.next3 = None + virtual_ref_check() virtual_ref_finish(xy) # self.meta_interp(f, [15]) self.check_loops(new_with_vtable=2) # the vref, and xy so far, # but not xy.next1/2/3 + self.check_aborted_count(0) def test_simple_force_always(self): myjitdriver = JitDriver(greens = [], reds = ['n']) @@ -204,6 +274,7 @@ xy.n = n exctx.topframeref = virtual_ref(xy) n -= externalfn(n) + virtual_ref_check() virtual_ref_finish(xy) exctx.topframeref = vref_None # @@ -234,6 +305,7 @@ xy.n = n exctx.topframeref = virtual_ref(xy) n -= externalfn(n) + virtual_ref_check() virtual_ref_finish(xy) exctx.topframeref = vref_None return exctx.m @@ -241,6 +313,7 @@ res = self.meta_interp(f, [30]) assert res == 13 self.check_loop_count(1) + self.check_aborted_count(0) def test_blackhole_forces(self): myjitdriver = JitDriver(greens = [], reds = ['n']) @@ -267,12 +340,14 @@ externalfn(n) n -= 1 exctx.topframeref = vref_None + virtual_ref_check() virtual_ref_finish(xy) return exctx.m # res = self.meta_interp(f, [30]) assert res == 13 self.check_loop_count(1) + self.check_aborted_count(0) def test_bridge_forces(self): myjitdriver = JitDriver(greens = [], reds = ['n']) @@ -299,12 +374,14 @@ externalfn(n) n -= 1 exctx.topframeref = vref_None + virtual_ref_check() virtual_ref_finish(xy) return exctx.m # res = self.meta_interp(f, [72]) assert res == 6 self.check_loop_count(1) # the bridge should not be compiled + self.check_aborted_count_at_least(1) def test_access_vref_later(self): myjitdriver = JitDriver(greens = [], reds = ['n']) @@ -320,7 +397,6 @@ return exctx.later().n # def f(n): - later = None while n > 0: myjitdriver.can_enter_jit(n=n) myjitdriver.jit_merge_point(n=n) @@ -330,11 +406,13 @@ exctx.later = exctx.topframeref n -= 1 exctx.topframeref = vref_None + virtual_ref_check() virtual_ref_finish(xy) return g() # res = self.meta_interp(f, [15]) assert res == 1 + self.check_aborted_count(0) def test_jit_force_virtual_seen(self): myjitdriver = JitDriver(greens = [], reds = ['n']) @@ -354,6 +432,7 @@ exctx.topframeref = virtual_ref(xy) n = exctx.topframeref().n - 1 exctx.topframeref = vref_None + virtual_ref_check() virtual_ref_finish(xy) return 1 # Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py Fri Dec 11 17:58:41 2009 @@ -76,12 +76,22 @@ vref.forced = lltype.cast_opaque_ptr(rclass.OBJECTPTR, real_object) return lltype.cast_opaque_ptr(llmemory.GCREF, vref) +def is_virtual_ref(gcref): + if not gcref: + return False + inst = lltype.cast_opaque_ptr(rclass.OBJECTPTR, gcref) + return inst.typeptr == jit_virtual_ref_vtable + def tracing_before_residual_call(gcref): + if not is_virtual_ref(gcref): + return vref = lltype.cast_opaque_ptr(lltype.Ptr(JIT_VIRTUAL_REF), gcref) assert not vref.virtual_token vref.virtual_token = TOKEN_TRACING_RESCALL def tracing_after_residual_call(gcref): + if not is_virtual_ref(gcref): + return False vref = lltype.cast_opaque_ptr(lltype.Ptr(JIT_VIRTUAL_REF), gcref) if vref.virtual_token: # not modified by the residual call; assert that it is still @@ -94,6 +104,8 @@ return True def forced_single_vref(gcref, real_object): + if not is_virtual_ref(gcref): + return assert real_object vref = lltype.cast_opaque_ptr(lltype.Ptr(JIT_VIRTUAL_REF), gcref) assert (vref.virtual_token != TOKEN_NONE and @@ -102,6 +114,8 @@ vref.forced = lltype.cast_opaque_ptr(rclass.OBJECTPTR, real_object) def continue_tracing(gcref, real_object): + if not is_virtual_ref(gcref): + return vref = lltype.cast_opaque_ptr(lltype.Ptr(JIT_VIRTUAL_REF), gcref) assert vref.virtual_token != TOKEN_TRACING_RESCALL vref.virtual_token = TOKEN_NONE Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/jit.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/jit.py Fri Dec 11 17:58:41 2009 @@ -112,11 +112,18 @@ dereferenced (by the call syntax 'vref()') before the virtual_ref_finish, then we get out of the assembler. If it is not dereferenced at all, or only after the virtual_ref_finish, then - nothing special occurs. + nothing special occurs. Note that the checks for 'being virtual' + only occurs when virtual_ref_check() is called (mostly for testing), + or when jit_merge_point is called by JITted code in a recursive call. """ return DirectJitVRef(x) virtual_ref.oopspec = 'virtual_ref(x)' +def virtual_ref_check(): + from pypy.rpython.lltypesystem import lltype, lloperation + lloperation.llop.jit_marker(lltype.Void, + lloperation.void('virtual_ref_check')) + def virtual_ref_finish(x): """See docstring in virtual_ref(x). Note that virtual_ref_finish takes as argument the real object, not the vref.""" From arigo at codespeak.net Fri Dec 11 18:24:15 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 11 Dec 2009 18:24:15 +0100 (CET) Subject: [pypy-svn] r70073 - pypy/branch/virtual-forcing/pypy/jit/metainterp Message-ID: <20091211172415.7C1CA16801D@codespeak.net> Author: arigo Date: Fri Dec 11 18:24:15 2009 New Revision: 70073 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py Log: Translation fix. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py Fri Dec 11 18:24:15 2009 @@ -226,7 +226,7 @@ def do_virtual_ref(cpu, box1, box2): raise NotImplementedError -def do_virtual_ref_check(cpu): +def do_virtual_ref_check(cpu, *boxes): raise NotImplementedError def do_debug_merge_point(cpu, box1): From arigo at codespeak.net Fri Dec 11 18:42:47 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 11 Dec 2009 18:42:47 +0100 (CET) Subject: [pypy-svn] r70074 - pypy/branch/virtual-forcing/pypy/jit/metainterp Message-ID: <20091211174247.4516516801F@codespeak.net> Author: arigo Date: Fri Dec 11 18:42:46 2009 New Revision: 70074 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Log: I think it must switch to a blackhole in that case. Not sure :-( Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Fri Dec 11 18:42:46 2009 @@ -1808,17 +1808,20 @@ None, descr=vinfo.vable_token_descr) def vable_and_vrefs_after_residual_call(self): + for i in range(1, len(self.virtualref_boxes), 2): + if self.is_blackholing(): + break + vrefbox = self.virtualref_boxes[i] + vref = vrefbox.getref_base() + if virtualref.tracing_after_residual_call(vref): + # this vref escaped during CALL_MAY_FORCE. + self.stop_tracking_virtualref(i-1) + # if self.is_blackholing(): escapes = True else: escapes = False # - for i in range(1, len(self.virtualref_boxes), 2): - vrefbox = self.virtualref_boxes[i] - vref = vrefbox.getref_base() - if virtualref.tracing_after_residual_call(vref): - # this vref escaped during CALL_MAY_FORCE. - self.stop_tracking_virtualref(i-1) # vinfo = self.staticdata.virtualizable_info if vinfo is not None: @@ -1870,7 +1873,8 @@ op.args = [op.args[0]] break else: - pass # not found at all! nothing to do, just ignore it + # not found at all! + self.switch_to_blackhole() def handle_exception(self): etype = self.cpu.get_exception() From arigo at codespeak.net Fri Dec 11 18:49:39 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 11 Dec 2009 18:49:39 +0100 (CET) Subject: [pypy-svn] r70075 - pypy/branch/virtual-forcing/pypy/jit/metainterp Message-ID: <20091211174939.9B04716801F@codespeak.net> Author: arigo Date: Fri Dec 11 18:49:39 2009 New Revision: 70075 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Log: Looks maybe a bit better. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Fri Dec 11 18:49:39 2009 @@ -1808,20 +1808,19 @@ None, descr=vinfo.vable_token_descr) def vable_and_vrefs_after_residual_call(self): - for i in range(1, len(self.virtualref_boxes), 2): - if self.is_blackholing(): - break - vrefbox = self.virtualref_boxes[i] - vref = vrefbox.getref_base() - if virtualref.tracing_after_residual_call(vref): - # this vref escaped during CALL_MAY_FORCE. - self.stop_tracking_virtualref(i-1) - # if self.is_blackholing(): escapes = True else: escapes = False # + for i in range(1, len(self.virtualref_boxes), 2): + if self.is_blackholing(): + break + vrefbox = self.virtualref_boxes[i] + vref = vrefbox.getref_base() + if virtualref.tracing_after_residual_call(vref): + # this vref escaped during CALL_MAY_FORCE. + self.stop_tracking_virtualref(i-1) # vinfo = self.staticdata.virtualizable_info if vinfo is not None: @@ -1873,8 +1872,7 @@ op.args = [op.args[0]] break else: - # not found at all! - self.switch_to_blackhole() + assert 0, "not found the VIRTUAL_REF nor VIRTUAL_REF_CHECK at all!" def handle_exception(self): etype = self.cpu.get_exception() @@ -1912,6 +1910,8 @@ self, newboxes, resumedescr, expect_virtualizable) # # virtual refs: make the vrefs point to the freshly allocated virtuals + if not self.is_blackholing(): + self.history.record(rop.VIRTUAL_REF_CHECK, [], None) self.virtualref_boxes = virtualref_boxes for i in range(0, len(virtualref_boxes), 2): virtualbox = virtualref_boxes[i] From afa at codespeak.net Fri Dec 11 22:21:43 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 11 Dec 2009 22:21:43 +0100 (CET) Subject: [pypy-svn] r70076 - pypy/branch/import-builtin/pypy/module/imp Message-ID: <20091211212143.C69A4168020@codespeak.net> Author: afa Date: Fri Dec 11 22:21:42 2009 New Revision: 70076 Modified: pypy/branch/import-builtin/pypy/module/imp/__init__.py pypy/branch/import-builtin/pypy/module/imp/importing.py Log: try to fix another test Modified: pypy/branch/import-builtin/pypy/module/imp/__init__.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/__init__.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/__init__.py Fri Dec 11 22:21:42 2009 @@ -8,9 +8,10 @@ interpleveldefs = { 'PY_SOURCE': 'space.wrap(importing.PY_SOURCE)', 'PY_COMPILED': 'space.wrap(importing.PY_COMPILED)', + 'C_EXTENSION': 'space.wrap(importing.C_EXTENSION)', 'PKG_DIRECTORY': 'space.wrap(importing.PKG_DIRECTORY)', 'C_BUILTIN': 'space.wrap(importing.C_BUILTIN)', - 'C_EXTENSION': 'space.wrap(importing.C_EXTENSION)', + 'PY_FROZEN': 'space.wrap(importing.PY_FROZEN)', 'get_suffixes': 'interp_imp.get_suffixes', 'get_magic': 'interp_imp.get_magic', Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Fri Dec 11 22:21:42 2009 @@ -21,7 +21,7 @@ # PY_RESOURCE = 4 PKG_DIRECTORY = 5 C_BUILTIN = 6 -# PY_FROZEN = 7 +PY_FROZEN = 7 # PY_CODERESOURCE = 8 IMP_HOOK = 9 From afa at codespeak.net Sat Dec 12 11:39:13 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 12 Dec 2009 11:39:13 +0100 (CET) Subject: [pypy-svn] r70077 - pypy/branch/import-builtin/pypy/module/imp Message-ID: <20091212103913.D87F8168024@codespeak.net> Author: afa Date: Sat Dec 12 11:39:12 2009 New Revision: 70077 Modified: pypy/branch/import-builtin/pypy/module/imp/interp_imp.py Log: fix translation when withmod-thread==False Modified: pypy/branch/import-builtin/pypy/module/imp/interp_imp.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/interp_imp.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/interp_imp.py Sat Dec 12 11:39:12 2009 @@ -143,7 +143,7 @@ if space.config.objspace.usemodules.thread: return space.wrap(importing.getimportlock(space).lock_held()) else: - return False + return space.w_False def acquire_lock(space): if space.config.objspace.usemodules.thread: From arigo at codespeak.net Sat Dec 12 13:59:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 12 Dec 2009 13:59:55 +0100 (CET) Subject: [pypy-svn] r70078 - in pypy/branch/virtual-forcing/pypy/jit: backend/llgraph backend/test backend/x86 metainterp metainterp/test Message-ID: <20091212125955.E8686168024@codespeak.net> Author: arigo Date: Sat Dec 12 13:59:55 2009 New Revision: 70078 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Log: Do it another way. Much simpler and does not suffer from the issue that virtualref_boxes is out-of-sync if we mangle past VIRTUAL_REF_CHECK instructions in the trace. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py Sat Dec 12 13:59:55 2009 @@ -154,7 +154,7 @@ 'call_may_force' : (('int', 'varargs'), 'intorptr'), 'guard_not_forced': ((), None), 'virtual_ref' : (('ref', 'int'), 'ref'), - 'virtual_ref_check': (('varargs',), None), + 'virtual_ref_finish': (('ref', 'ref'), None), #'getitem' : (('void', 'ref', 'int'), 'int'), #'setitem' : (('void', 'ref', 'int', 'int'), None), #'newlist' : (('void', 'varargs'), 'ref'), @@ -818,7 +818,7 @@ def op_virtual_ref(self, _, virtual, index): return virtual - def op_virtual_ref_check(self, _, *args): + def op_virtual_ref_finish(self, _, vref, virtual): pass Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Sat Dec 12 13:59:55 2009 @@ -786,9 +786,9 @@ 'ref') assert r.value == u_box.value - def test_virtual_ref_check(self): - # if VIRTUAL_REF_CHECK reaches the backend, it is a no-op - self.execute_operation(rop.VIRTUAL_REF_CHECK, + def test_virtual_ref_finish(self): + # if VIRTUAL_REF_FINISH reaches the backend, it is a no-op + self.execute_operation(rop.VIRTUAL_REF_FINISH, [BoxInt(123), BoxInt(234)], 'void') Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py Sat Dec 12 13:59:55 2009 @@ -928,7 +928,7 @@ def consider_debug_merge_point(self, op, ignored): pass - def consider_virtual_ref_check(self, op, ignored): + def consider_virtual_ref_finish(self, op, ignored): self.possibly_free_vars(op.args) def get_mark_gc_roots(self, gcrootmap): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py Sat Dec 12 13:59:55 2009 @@ -226,7 +226,7 @@ def do_virtual_ref(cpu, box1, box2): raise NotImplementedError -def do_virtual_ref_check(cpu, *boxes): +def do_virtual_ref_finish(cpu, box1, box2): raise NotImplementedError def do_debug_merge_point(cpu, box1): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py Sat Dec 12 13:59:55 2009 @@ -759,29 +759,27 @@ vrefvalue.setfield(descr_virtual_token, self.getvalue(tokenbox)) vrefvalue.setfield(descr_virtualref_index, self.getvalue(indexbox)) - def optimize_VIRTUAL_REF_CHECK(self, op): - for i in range(0, len(op.args), 2): - value = self.getvalue(op.args[i]) - if not value.is_virtual(): # virtual_ref(non-virtual) means it - raise compile.GiveUp # escaped already, which is bad - # - # Set the 'forced' field of the virtual_ref. - # In good cases, this is all virtual, so has no effect. - # Otherwise, this forces the real object -- but only now, as - # opposed to much earlier. This is important because the object is - # typically a PyPy PyFrame, and now is the end of its execution, so - # forcing it now does not have catastrophic effects. - from pypy.jit.metainterp import virtualref - # - set 'forced' to point to the real object - args = [op.args[i+1], op.args[i]] - op1 = ResOperation(rop.SETFIELD_GC, args, None, - descr = virtualref.get_descr_forced(self.cpu)) - self.optimize_SETFIELD_GC(op1) - # - set 'virtual_token' to TOKEN_NONE - args = [op.args[i+1], ConstInt(0)] - op1 = ResOperation(rop.SETFIELD_GC, args, None, - descr = virtualref.get_descr_virtual_token(self.cpu)) - self.optimize_SETFIELD_GC(op1) + def optimize_VIRTUAL_REF_FINISH(self, op): + value = self.getvalue(op.args[1]) + if not value.is_virtual(): # virtual_ref(non-virtual) means it + raise compile.GiveUp # escaped already, which is bad + # + # Set the 'forced' field of the virtual_ref. + # In good cases, this is all virtual, so has no effect. + # Otherwise, this forces the real object -- but only now, as + # opposed to much earlier. This is important because the object is + # typically a PyPy PyFrame, and now is the end of its execution, so + # forcing it now does not have catastrophic effects. + from pypy.jit.metainterp import virtualref + # - set 'forced' to point to the real object + op1 = ResOperation(rop.SETFIELD_GC, op.args, None, + descr = virtualref.get_descr_forced(self.cpu)) + self.optimize_SETFIELD_GC(op1) + # - set 'virtual_token' to TOKEN_NONE + args = [op.args[0], ConstInt(0)] + op1 = ResOperation(rop.SETFIELD_GC, args, None, + descr = virtualref.get_descr_virtual_token(self.cpu)) + self.optimize_SETFIELD_GC(op1) def optimize_GETFIELD_GC(self, op): value = self.getvalue(op.args[0]) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Sat Dec 12 13:59:55 2009 @@ -836,7 +836,7 @@ except GiveUp: self.metainterp.staticdata.profiler.count(ABORT_BRIDGE) self.metainterp.switch_to_blackhole() - self.metainterp.generate_virtual_ref_check() + self.metainterp.virtual_ref_check() if self.metainterp.is_blackholing(): self.blackhole_reached_merge_point(self.env) return True @@ -900,9 +900,9 @@ # in the vref), then we replace the vref in the list with # ConstPtr(NULL). # - # * at the next virtual_ref_check() or jit_merge_point, any such + # * at the next virtual_ref_check() or jit_merge_point(), any such # NULL causes tracing to abort. So escaping is fine if there - # is no virtual_ref_check() or jit_merge_point before the final + # is no virtual_ref_check()/jit_merge_point() before the final # call to virtual_ref_finish(). # metainterp = self.metainterp @@ -930,19 +930,18 @@ # virtual_ref_finish() assumes that we have a stack-like, last-in # first-out order. metainterp = self.metainterp - if not metainterp.is_blackholing(): - vrefbox = metainterp.virtualref_boxes[-1] - vref = vrefbox.getref_base() - if virtualref.is_virtual_ref(vref): - metainterp.stop_tracking_virtualref(-2) - # vrefbox = metainterp.virtualref_boxes.pop() lastbox = metainterp.virtualref_boxes.pop() assert box.getref_base() == lastbox.getref_base() + if not metainterp.is_blackholing(): + vref = vrefbox.getref_base() + if virtualref.is_virtual_ref(vref): + metainterp.history.record(rop.VIRTUAL_REF_FINISH, + [vrefbox, lastbox], None) @arguments() def opimpl_virtual_ref_check(self): - self.metainterp.generate_virtual_ref_check() + self.metainterp.virtual_ref_check() # ------------------------------ @@ -1052,7 +1051,7 @@ # xxx do something about code duplication resbox = self.metainterp.execute_and_record_varargs( rop.CALL_MAY_FORCE, argboxes, descr=descr) - self.metainterp.vable_and_vrefs_after_residual_call() + self.metainterp.vable_and_vrefs_after_residual_call(argboxes) if resbox is not None: self.make_result_box(resbox) self.generate_guard(self.pc, rop.GUARD_NOT_FORCED, None, []) @@ -1807,20 +1806,23 @@ force_token_box], None, descr=vinfo.vable_token_descr) - def vable_and_vrefs_after_residual_call(self): + def vable_and_vrefs_after_residual_call(self, argboxes): if self.is_blackholing(): escapes = True else: escapes = False # - for i in range(1, len(self.virtualref_boxes), 2): - if self.is_blackholing(): - break - vrefbox = self.virtualref_boxes[i] + for i in range(0, len(self.virtualref_boxes), 2): + virtualbox = self.virtualref_boxes[i] + vrefbox = self.virtualref_boxes[i+1] vref = vrefbox.getref_base() - if virtualref.tracing_after_residual_call(vref): - # this vref escaped during CALL_MAY_FORCE. - self.stop_tracking_virtualref(i-1) + if (virtualref.tracing_after_residual_call(vref) or + virtualbox in argboxes): # <-- XXX hack + # this vref was really a virtual_ref, but it escaped + # during this CALL_MAY_FORCE. Mark this fact by + # generating a VIRTUAL_REF_FINISH on it and replacing + # it by ConstPtr(NULL). + self.stop_tracking_virtualref(i) # vinfo = self.staticdata.virtualizable_info if vinfo is not None: @@ -1836,43 +1838,28 @@ if escapes: self.load_fields_from_virtualizable() - def generate_virtual_ref_check(self): + def stop_tracking_virtualref(self, i): + virtualbox = self.virtualref_boxes[i] + vrefbox = self.virtualref_boxes[i+1] + # record VIRTUAL_REF_FINISH just before the current CALL_MAY_FORCE + call_may_force_op = self.history.operations.pop() + assert call_may_force_op.opnum == rop.CALL_MAY_FORCE + self.history.record(rop.VIRTUAL_REF_FINISH, + [vrefbox, virtualbox], None) + self.history.operations.append(call_may_force_op) + # mark by replacing it with ConstPtr(NULL) + self.virtualref_boxes[i+1] = self.cpu.ts.CONST_NULL + + def virtual_ref_check(self): if self.is_blackholing(): return - # first, abort tracing if we reach this point with an escaped virtual + # abort tracing if we reach this point with an escaped virtual for i in range(1, len(self.virtualref_boxes), 2): vrefbox = self.virtualref_boxes[i] vref = vrefbox.getref_base() if not virtualref.is_virtual_ref(vref): self.switch_to_blackhole() - return - # then record VIRTUAL_REF_CHECK, with no argument so far. - # Arguments may be added by stop_tracking_virtualref(). - self.history.record(rop.VIRTUAL_REF_CHECK, [], None) - - def stop_tracking_virtualref(self, j): - # we look for the most recent VIRTUAL_REF_CHECK marker, and add - # there the pair virtualbox/vrefbox as arguments. - virtualbox = self.virtualref_boxes[j] - vrefbox = self.virtualref_boxes[j+1] - assert virtualref.is_virtual_ref(vrefbox.getref_base()) - self.virtualref_boxes[j+1] = self.cpu.ts.CONST_NULL - operations = self.history.operations - for i in range(len(operations)-1, -1, -1): - op = operations[i] - if op.opnum == rop.VIRTUAL_REF_CHECK: - # found - op.args = op.args + [virtualbox, vrefbox] break - if op.result is vrefbox: - # no VIRTUAL_REF_CHECK exists before the VIRTUAL_REF - # that created this vref. Replace it with a mere SAME_AS. - if op.opnum == rop.VIRTUAL_REF: - op.opnum = rop.SAME_AS - op.args = [op.args[0]] - break - else: - assert 0, "not found the VIRTUAL_REF nor VIRTUAL_REF_CHECK at all!" def handle_exception(self): etype = self.cpu.get_exception() @@ -1910,8 +1897,6 @@ self, newboxes, resumedescr, expect_virtualizable) # # virtual refs: make the vrefs point to the freshly allocated virtuals - if not self.is_blackholing(): - self.history.record(rop.VIRTUAL_REF_CHECK, [], None) self.virtualref_boxes = virtualref_boxes for i in range(0, len(virtualref_boxes), 2): virtualbox = virtualref_boxes[i] Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py Sat Dec 12 13:59:55 2009 @@ -223,7 +223,7 @@ 'COND_CALL_GC_MALLOC', # [a, b, if_(a<=b)_result, if_(a>b)_call, args...] # => result (for mallocs) 'DEBUG_MERGE_POINT/1', # debugging only - 'VIRTUAL_REF_CHECK', + 'VIRTUAL_REF_FINISH/2', '_CANRAISE_FIRST', # ----- start of can_raise operations ----- 'CALL', Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Sat Dec 12 13:59:55 2009 @@ -30,7 +30,7 @@ self.interp_operations(f, []) self.check_operations_history(new_with_vtable=1, # X() virtual_ref=1, - virtual_ref_check=1) + virtual_ref_finish=1) def test_make_vref_guard(self): if not isinstance(self, TestLLtype): From arigo at codespeak.net Sat Dec 12 16:43:12 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 12 Dec 2009 16:43:12 +0100 (CET) Subject: [pypy-svn] r70080 - pypy/branch/virtual-forcing/pypy/jit/metainterp Message-ID: <20091212154312.84249168027@codespeak.net> Author: arigo Date: Sat Dec 12 16:43:12 2009 New Revision: 70080 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Log: Obscure. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Sat Dec 12 16:43:12 2009 @@ -508,6 +508,13 @@ @arguments("box", "descr", "box") def opimpl_setfield_gc(self, box, fielddesc, valuebox): self.execute_with_descr(rop.SETFIELD_GC, fielddesc, box, valuebox) + # XXX + metainterp = self.metainterp + if (len(metainterp.virtualref_boxes) >= 2 and + metainterp.virtualref_boxes[-2] == valuebox and + virtualref.is_virtual_ref(metainterp.virtualref_boxes[-1].getref_base())): + metainterp.stop_tracking_virtualref(-2) + # @arguments("box", "descr") def opimpl_getfield_raw(self, box, fielddesc): @@ -1841,12 +1848,11 @@ def stop_tracking_virtualref(self, i): virtualbox = self.virtualref_boxes[i] vrefbox = self.virtualref_boxes[i+1] - # record VIRTUAL_REF_FINISH just before the current CALL_MAY_FORCE - call_may_force_op = self.history.operations.pop() - assert call_may_force_op.opnum == rop.CALL_MAY_FORCE + # record VIRTUAL_REF_FINISH just before the current op + current_op = self.history.operations.pop() self.history.record(rop.VIRTUAL_REF_FINISH, [vrefbox, virtualbox], None) - self.history.operations.append(call_may_force_op) + self.history.operations.append(current_op) # mark by replacing it with ConstPtr(NULL) self.virtualref_boxes[i+1] = self.cpu.ts.CONST_NULL From arigo at codespeak.net Sun Dec 13 12:53:08 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 13 Dec 2009 12:53:08 +0100 (CET) Subject: [pypy-svn] r70082 - in pypy/trunk/pypy: rpython/lltypesystem translator/c Message-ID: <20091213115308.EAC0D168035@codespeak.net> Author: arigo Date: Sun Dec 13 12:53:07 2009 New Revision: 70082 Modified: pypy/trunk/pypy/rpython/lltypesystem/rffi.py pypy/trunk/pypy/translator/c/genc.py Log: Check the 32-bit-versus-64-bit configuration: * in rffi, verify that we are not trying to run on a 32-bit CPython with a 64-bit C compiler or vice-versa; * in genc, produce in the C code a check that will give a compile-time error if the C compiler uses a different word size. Modified: pypy/trunk/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rffi.py Sun Dec 13 12:53:07 2009 @@ -801,6 +801,13 @@ return llmemory.offsetof(STRUCT, fieldname) offsetof._annspecialcase_ = 'specialize:memo' +# check that we have a sane configuration +assert sys.maxint == (1 << (8 * sizeof(lltype.Signed) - 1)) - 1, ( + "Mixed configuration of the word size of the machine:\n\t" + "the underlying Python was compiled with maxint=%d,\n\t" + "but the C compiler says that 'long' is %d bytes" % ( + sys.maxint, sizeof(lltype.Signed))) + # ********************** some helpers ******************* def make(STRUCT, **fields): Modified: pypy/trunk/pypy/translator/c/genc.py ============================================================================== --- pypy/trunk/pypy/translator/c/genc.py (original) +++ pypy/trunk/pypy/translator/c/genc.py Sun Dec 13 12:53:07 2009 @@ -729,7 +729,16 @@ print >> f +def gen_size_check(f): + from pypy.rlib.rarithmetic import LONG_BIT + print >> f, '#if 8 * SIZEOF_LONG != %d' % (LONG_BIT,) + print >> f, '# error "C files are generated for a %d-bit platform"' % ( + LONG_BIT,) + print >> f, '#endif' + print >> f + def gen_structdef(f, database): + gen_size_check(f) structdeflist = database.getstructdeflist() print >> f, '/***********************************************************/' print >> f, '/*** Structure definitions ***/' From arigo at codespeak.net Sun Dec 13 13:04:52 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 13 Dec 2009 13:04:52 +0100 (CET) Subject: [pypy-svn] r70083 - pypy/trunk/pypy/translator/c/src Message-ID: <20091213120452.97018168035@codespeak.net> Author: arigo Date: Sun Dec 13 13:04:52 2009 New Revision: 70083 Modified: pypy/trunk/pypy/translator/c/src/debug.h Log: Fix for OSes that don't run on a 32-bit Intel and that don't have the POSIX function clock_gettime(). This includes notably Mac OS/X on 64-bit platforms. Modified: pypy/trunk/pypy/translator/c/src/debug.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/debug.h (original) +++ pypy/trunk/pypy/translator/c/src/debug.h Sun Dec 13 13:04:52 2009 @@ -119,13 +119,21 @@ # define READ_TIMESTAMP(val) QueryPerformanceCounter(&(val)) # else # include +# include # define READ_TIMESTAMP(val) (val) = pypy_read_timestamp() static long long pypy_read_timestamp(void) { +# ifdef CLOCK_THREAD_CPUTIME_ID struct timespec tspec; clock_gettime(CLOCK_THREAD_CPUTIME_ID, &tspec); return ((long long)tspec.tv_sec) * 1000000000LL + tspec.tv_nsec; +# else + /* argh, we don't seem to have clock_gettime(). Bad OS. */ + struct timeval tv; + gettimeofday(tv, NULL); + return ((long long)tv.tv_sec) * 1000000LL + tv.tv_usec; +# endif } # endif #endif From arigo at codespeak.net Sun Dec 13 17:02:46 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 13 Dec 2009 17:02:46 +0100 (CET) Subject: [pypy-svn] r70084 - in pypy/branch/virtual-forcing/pypy: jit/metainterp jit/metainterp/test rlib Message-ID: <20091213160246.961A8168030@codespeak.net> Author: arigo Date: Sun Dec 13 17:02:46 2009 New Revision: 70084 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py pypy/branch/virtual-forcing/pypy/rlib/jit.py Log: Kill the virtual_ref_check() operation and simplify again the semantics of virtualrefs. Now it will no longer cause tracing to abort. It might simplify things quite a lot, and give hopefully good results. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py Sun Dec 13 17:02:46 2009 @@ -1170,9 +1170,6 @@ def handle_jit_marker__can_enter_jit(self, op): self.emit('can_enter_jit') - def handle_jit_marker__virtual_ref_check(self, op): - self.emit('virtual_ref_check') - def serialize_op_direct_call(self, op): kind = self.codewriter.guess_call_kind(op) return getattr(self, 'handle_%s_call' % kind)(op) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py Sun Dec 13 17:02:46 2009 @@ -737,9 +737,6 @@ self._optimize_oois_ooisnot(op, False) def optimize_VIRTUAL_REF(self, op): - value = self.getvalue(op.args[0]) - if not value.is_virtual(): # virtual_ref(non-virtual) gives bad - raise compile.GiveUp # results, so don't bother compiling it indexbox = op.args[1] # # get some constants (these calls are all 'memo') @@ -760,10 +757,6 @@ vrefvalue.setfield(descr_virtualref_index, self.getvalue(indexbox)) def optimize_VIRTUAL_REF_FINISH(self, op): - value = self.getvalue(op.args[1]) - if not value.is_virtual(): # virtual_ref(non-virtual) means it - raise compile.GiveUp # escaped already, which is bad - # # Set the 'forced' field of the virtual_ref. # In good cases, this is all virtual, so has no effect. # Otherwise, this forces the real object -- but only now, as @@ -780,6 +773,12 @@ op1 = ResOperation(rop.SETFIELD_GC, args, None, descr = virtualref.get_descr_virtual_token(self.cpu)) self.optimize_SETFIELD_GC(op1) + # Note that in some cases the virtual in op.args[1] has been forced + # already. This is fine. In that case, and *if* a residual + # CALL_MAY_FORCE suddenly turns out to access it, then it will + # trigger a ResumeGuardForcedDescr.handle_async_forcing() which + # will work too (but just be a little pointless, as the structure + # was already forced). def optimize_GETFIELD_GC(self, op): value = self.getvalue(op.args[0]) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Sun Dec 13 17:02:46 2009 @@ -508,13 +508,6 @@ @arguments("box", "descr", "box") def opimpl_setfield_gc(self, box, fielddesc, valuebox): self.execute_with_descr(rop.SETFIELD_GC, fielddesc, box, valuebox) - # XXX - metainterp = self.metainterp - if (len(metainterp.virtualref_boxes) >= 2 and - metainterp.virtualref_boxes[-2] == valuebox and - virtualref.is_virtual_ref(metainterp.virtualref_boxes[-1].getref_base())): - metainterp.stop_tracking_virtualref(-2) - # @arguments("box", "descr") def opimpl_getfield_raw(self, box, fielddesc): @@ -843,7 +836,6 @@ except GiveUp: self.metainterp.staticdata.profiler.count(ABORT_BRIDGE) self.metainterp.switch_to_blackhole() - self.metainterp.virtual_ref_check() if self.metainterp.is_blackholing(): self.blackhole_reached_merge_point(self.env) return True @@ -907,11 +899,6 @@ # in the vref), then we replace the vref in the list with # ConstPtr(NULL). # - # * at the next virtual_ref_check() or jit_merge_point(), any such - # NULL causes tracing to abort. So escaping is fine if there - # is no virtual_ref_check()/jit_merge_point() before the final - # call to virtual_ref_finish(). - # metainterp = self.metainterp if metainterp.is_blackholing(): resbox = box # good enough when blackholing @@ -946,10 +933,6 @@ metainterp.history.record(rop.VIRTUAL_REF_FINISH, [vrefbox, lastbox], None) - @arguments() - def opimpl_virtual_ref_check(self): - self.metainterp.virtual_ref_check() - # ------------------------------ def setup_call(self, argboxes): @@ -1058,7 +1041,7 @@ # xxx do something about code duplication resbox = self.metainterp.execute_and_record_varargs( rop.CALL_MAY_FORCE, argboxes, descr=descr) - self.metainterp.vable_and_vrefs_after_residual_call(argboxes) + self.metainterp.vable_and_vrefs_after_residual_call() if resbox is not None: self.make_result_box(resbox) self.generate_guard(self.pc, rop.GUARD_NOT_FORCED, None, []) @@ -1813,7 +1796,7 @@ force_token_box], None, descr=vinfo.vable_token_descr) - def vable_and_vrefs_after_residual_call(self, argboxes): + def vable_and_vrefs_after_residual_call(self): if self.is_blackholing(): escapes = True else: @@ -1823,8 +1806,7 @@ virtualbox = self.virtualref_boxes[i] vrefbox = self.virtualref_boxes[i+1] vref = vrefbox.getref_base() - if (virtualref.tracing_after_residual_call(vref) or - virtualbox in argboxes): # <-- XXX hack + if virtualref.tracing_after_residual_call(vref): # this vref was really a virtual_ref, but it escaped # during this CALL_MAY_FORCE. Mark this fact by # generating a VIRTUAL_REF_FINISH on it and replacing @@ -1848,25 +1830,15 @@ def stop_tracking_virtualref(self, i): virtualbox = self.virtualref_boxes[i] vrefbox = self.virtualref_boxes[i+1] - # record VIRTUAL_REF_FINISH just before the current op - current_op = self.history.operations.pop() + # record VIRTUAL_REF_FINISH just before the current CALL_MAY_FORCE + call_may_force_op = self.history.operations.pop() + assert call_may_force_op.opnum == rop.CALL_MAY_FORCE self.history.record(rop.VIRTUAL_REF_FINISH, [vrefbox, virtualbox], None) - self.history.operations.append(current_op) + self.history.operations.append(call_may_force_op) # mark by replacing it with ConstPtr(NULL) self.virtualref_boxes[i+1] = self.cpu.ts.CONST_NULL - def virtual_ref_check(self): - if self.is_blackholing(): - return - # abort tracing if we reach this point with an escaped virtual - for i in range(1, len(self.virtualref_boxes), 2): - vrefbox = self.virtualref_boxes[i] - vref = vrefbox.getref_base() - if not virtualref.is_virtual_ref(vref): - self.switch_to_blackhole() - break - def handle_exception(self): etype = self.cpu.get_exception() evalue = self.cpu.get_exc_value() Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Sun Dec 13 17:02:46 2009 @@ -1,7 +1,7 @@ import py from pypy.rpython.lltypesystem import lltype, llmemory, lloperation from pypy.rlib.jit import JitDriver, dont_look_inside, vref_None -from pypy.rlib.jit import virtual_ref, virtual_ref_check, virtual_ref_finish +from pypy.rlib.jit import virtual_ref, virtual_ref_finish from pypy.rlib.objectmodel import compute_unique_id from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.jit.metainterp.resoperation import rop @@ -22,7 +22,6 @@ def f(): x = X() exctx.topframeref = virtual_ref(x) - virtual_ref_check() exctx.topframeref = vref_None virtual_ref_finish(x) return 1 @@ -54,7 +53,6 @@ exctx.topframeref = virtual_ref(x) def leave(): exctx.topframeref = vref_None - virtual_ref_check() virtual_ref_finish(exctx._frame) def f(n): enter(n) @@ -107,10 +105,9 @@ # here, 'x' should be virtual. (This is ensured because # we call virtual_ref(x).) exctx.topframeref = vref_None - virtual_ref_check() virtual_ref_finish(x) - # 'x' is allowed to escape, and even be forced, even after - # the call to finish(). + # 'x' and 'vref' can randomly escape after the call to + # finish(). g(vref) n -= 1 return 1 @@ -119,104 +116,10 @@ self.check_loops(new_with_vtable=2) # the vref, and later the X self.check_aborted_count(0) - def test_make_vref_and_force_nocheck_1(self): - jitdriver = JitDriver(greens = [], reds = ['total', 'n']) - # - class X: - pass - class ExCtx: - pass - exctx = ExCtx() - # - @dont_look_inside - def force_me(): - return exctx.topframeref().n - # - def f(n): - total = 0 - while total < 300: - jitdriver.can_enter_jit(total=total, n=n) - jitdriver.jit_merge_point(total=total, n=n) - x = X() - x.n = n + 123 - exctx.topframeref = virtual_ref(x) - # --- no virtual_ref_check() here --- - total += force_me() - 100 - virtual_ref_finish(x) - exctx.topframeref = vref_None - return total - # - res = self.meta_interp(f, [-4]) - assert res == 16 * 19 - self.check_aborted_count(0) - - def test_make_vref_and_force_nocheck_2(self): - jitdriver = JitDriver(greens = [], reds = ['total', 'n']) - # - class X: - pass - class ExCtx: - pass - exctx = ExCtx() - # - @dont_look_inside - def force_me(): - return exctx.topframeref().n - # - def f(n): - total = 0 - while total < 300: - jitdriver.can_enter_jit(total=total, n=n) - jitdriver.jit_merge_point(total=total, n=n) - x = X() - x.n = n + 123 - exctx.topframeref = virtual_ref(x) - virtual_ref_check() - total += force_me() - 100 - # --- but no virtual_ref_check() there --- - virtual_ref_finish(x) - exctx.topframeref = vref_None - return total - # - res = self.meta_interp(f, [-4]) - assert res == 16 * 19 - self.check_aborted_count(0) - - def test_make_vref_and_force_check(self): - jitdriver = JitDriver(greens = [], reds = ['total', 'n']) - # - class X: - pass - class ExCtx: - pass - exctx = ExCtx() - # - @dont_look_inside - def force_me(): - return exctx.topframeref().n - # - def f(n): - total = 0 - while total < 300: - jitdriver.can_enter_jit(total=total, n=n) - jitdriver.jit_merge_point(total=total, n=n) - x = X() - x.n = n + 123 - exctx.topframeref = virtual_ref(x) - total += force_me() - 100 - virtual_ref_check() - virtual_ref_finish(x) - exctx.topframeref = vref_None - return total - # - res = self.meta_interp(f, [-4]) - assert res == 16 * 19 - self.check_loops({}) # because we aborted tracing - self.check_aborted_count_at_least(1) - def test_simple_no_access(self): myjitdriver = JitDriver(greens = [], reds = ['n']) # + A = lltype.GcArray(lltype.Signed) class XY: pass class ExCtx: @@ -234,26 +137,26 @@ myjitdriver.can_enter_jit(n=n) myjitdriver.jit_merge_point(n=n) xy = XY() - xy.next1 = XY() - xy.next2 = XY() - xy.next3 = XY() + xy.next1 = lltype.malloc(A, 0) + xy.next2 = lltype.malloc(A, 0) + xy.next3 = lltype.malloc(A, 0) exctx.topframeref = virtual_ref(xy) n -= externalfn(n) exctx.topframeref = vref_None - xy.next1 = None - xy.next2 = None - xy.next3 = None - virtual_ref_check() + xy.next1 = lltype.nullptr(A) + xy.next2 = lltype.nullptr(A) + xy.next3 = lltype.nullptr(A) virtual_ref_finish(xy) # self.meta_interp(f, [15]) - self.check_loops(new_with_vtable=2) # the vref, and xy so far, - # but not xy.next1/2/3 + self.check_loops(new_with_vtable=2, # the vref, and xy so far, + new_array=0) # but not xy.next1/2/3 self.check_aborted_count(0) def test_simple_force_always(self): myjitdriver = JitDriver(greens = [], reds = ['n']) # + A = lltype.GcArray(lltype.Signed) class XY: pass class ExCtx: @@ -271,20 +174,27 @@ myjitdriver.can_enter_jit(n=n) myjitdriver.jit_merge_point(n=n) xy = XY() + xy.next1 = lltype.malloc(A, 0) + xy.next2 = lltype.malloc(A, 0) + xy.next3 = lltype.malloc(A, 0) xy.n = n exctx.topframeref = virtual_ref(xy) n -= externalfn(n) - virtual_ref_check() + xy.next1 = lltype.nullptr(A) + xy.next2 = lltype.nullptr(A) + xy.next3 = lltype.nullptr(A) virtual_ref_finish(xy) exctx.topframeref = vref_None # self.meta_interp(f, [15]) - self.check_loops({}) # because we aborted tracing - self.check_aborted_count_at_least(1) + self.check_loops(new_with_vtable=2, # XY(), the vref + new_array=3) # next1/2/3 + self.check_aborted_count(0) def test_simple_force_sometimes(self): myjitdriver = JitDriver(greens = [], reds = ['n']) # + A = lltype.GcArray(lltype.Signed) class XY: pass class ExCtx: @@ -302,22 +212,30 @@ myjitdriver.can_enter_jit(n=n) myjitdriver.jit_merge_point(n=n) xy = XY() + xy.next1 = lltype.malloc(A, 0) + xy.next2 = lltype.malloc(A, 0) + xy.next3 = lltype.malloc(A, 0) xy.n = n exctx.topframeref = virtual_ref(xy) n -= externalfn(n) - virtual_ref_check() + xy.next1 = lltype.nullptr(A) + xy.next2 = lltype.nullptr(A) + xy.next3 = lltype.nullptr(A) virtual_ref_finish(xy) exctx.topframeref = vref_None return exctx.m # res = self.meta_interp(f, [30]) assert res == 13 + self.check_loops(new_with_vtable=2, # the vref, XY() at the end + new_array=0) # but not next1/2/3 self.check_loop_count(1) self.check_aborted_count(0) def test_blackhole_forces(self): myjitdriver = JitDriver(greens = [], reds = ['n']) # + A = lltype.GcArray(lltype.Signed) class XY: pass class ExCtx: @@ -334,24 +252,32 @@ myjitdriver.can_enter_jit(n=n) myjitdriver.jit_merge_point(n=n) xy = XY() + xy.next1 = lltype.malloc(A, 0) + xy.next2 = lltype.malloc(A, 0) + xy.next3 = lltype.malloc(A, 0) xy.n = n exctx.topframeref = virtual_ref(xy) if n == 13: externalfn(n) n -= 1 exctx.topframeref = vref_None - virtual_ref_check() + xy.next1 = lltype.nullptr(A) + xy.next2 = lltype.nullptr(A) + xy.next3 = lltype.nullptr(A) virtual_ref_finish(xy) return exctx.m # res = self.meta_interp(f, [30]) assert res == 13 + self.check_loops(new_with_vtable=2, # the vref, XY() at the end + new_array=0) # but not next1/2/3 self.check_loop_count(1) self.check_aborted_count(0) def test_bridge_forces(self): myjitdriver = JitDriver(greens = [], reds = ['n']) # + A = lltype.GcArray(lltype.Signed) class XY: pass class ExCtx: @@ -368,20 +294,34 @@ myjitdriver.can_enter_jit(n=n) myjitdriver.jit_merge_point(n=n) xy = XY() + xy.next1 = lltype.malloc(A, 0) + xy.next2 = lltype.malloc(A, 0) + xy.next3 = lltype.malloc(A, 0) + xy.next4 = lltype.malloc(A, 0) + xy.next5 = lltype.malloc(A, 0) xy.n = n exctx.topframeref = virtual_ref(xy) if n % 6 == 0: + xy.next1 = lltype.nullptr(A) + xy.next2 = lltype.nullptr(A) + xy.next3 = lltype.nullptr(A) externalfn(n) n -= 1 exctx.topframeref = vref_None - virtual_ref_check() + xy.next1 = lltype.nullptr(A) + xy.next2 = lltype.nullptr(A) + xy.next3 = lltype.nullptr(A) + xy.next4 = lltype.nullptr(A) + xy.next5 = lltype.nullptr(A) virtual_ref_finish(xy) return exctx.m # res = self.meta_interp(f, [72]) assert res == 6 - self.check_loop_count(1) # the bridge should not be compiled - self.check_aborted_count_at_least(1) + self.check_loop_count(2) # the loop and the bridge + self.check_loops(new_with_vtable=3, # loop: vref, xy; bridge: xy + new_array=2) # bridge: next4, next5 + self.check_aborted_count(0) def test_access_vref_later(self): myjitdriver = JitDriver(greens = [], reds = ['n']) @@ -406,7 +346,6 @@ exctx.later = exctx.topframeref n -= 1 exctx.topframeref = vref_None - virtual_ref_check() virtual_ref_finish(xy) return g() # @@ -417,6 +356,7 @@ def test_jit_force_virtual_seen(self): myjitdriver = JitDriver(greens = [], reds = ['n']) # + A = lltype.GcArray(lltype.Signed) class XY: pass class ExCtx: @@ -430,16 +370,18 @@ xy = XY() xy.n = n exctx.topframeref = virtual_ref(xy) + xy.next1 = lltype.malloc(A, 0) n = exctx.topframeref().n - 1 + xy.next1 = lltype.nullptr(A) exctx.topframeref = vref_None - virtual_ref_check() virtual_ref_finish(xy) return 1 # res = self.meta_interp(f, [15]) assert res == 1 - self.check_loops({}) # because we aborted tracing - self.check_aborted_count_at_least(1) + self.check_loops(new_with_vtable=2, # vref, xy + new_array=1) # next1 + self.check_aborted_count(0) def test_recursive_call_1(self): myjitdriver = JitDriver(greens = [], reds = ['n', 'frame', 'rec']) Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/jit.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/jit.py Sun Dec 13 17:02:46 2009 @@ -103,27 +103,17 @@ # VRefs def virtual_ref(x): + """Creates a 'vref' object that contains a reference to 'x'. Calls to virtual_ref/virtual_ref_finish must be properly nested. The idea - is that the object 'x' is supposed to be JITted as a virtual (the - JIT will abort if it is not), at least between the calls to - virtual_ref and virtual_ref_finish. The point is that the 'vref' - returned by virtual_ref may escape early. If at runtime it is - dereferenced (by the call syntax 'vref()') before the - virtual_ref_finish, then we get out of the assembler. If it is not - dereferenced at all, or only after the virtual_ref_finish, then - nothing special occurs. Note that the checks for 'being virtual' - only occurs when virtual_ref_check() is called (mostly for testing), - or when jit_merge_point is called by JITted code in a recursive call. - """ + is that the object 'x' is supposed to be JITted as a virtual between + the calls to virtual_ref and virtual_ref_finish, but the 'vref' + object can escape at any point in time. If at runtime it is + dereferenced (by the call syntax 'vref()'), it returns 'x', which is + then forced.""" return DirectJitVRef(x) virtual_ref.oopspec = 'virtual_ref(x)' -def virtual_ref_check(): - from pypy.rpython.lltypesystem import lltype, lloperation - lloperation.llop.jit_marker(lltype.Void, - lloperation.void('virtual_ref_check')) - def virtual_ref_finish(x): """See docstring in virtual_ref(x). Note that virtual_ref_finish takes as argument the real object, not the vref.""" From arigo at codespeak.net Sun Dec 13 17:05:14 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 13 Dec 2009 17:05:14 +0100 (CET) Subject: [pypy-svn] r70085 - pypy/trunk/pypy/translator/c/src Message-ID: <20091213160514.7383A168038@codespeak.net> Author: arigo Date: Sun Dec 13 17:05:13 2009 New Revision: 70085 Modified: pypy/trunk/pypy/translator/c/src/debug.h Log: Bah. Modified: pypy/trunk/pypy/translator/c/src/debug.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/debug.h (original) +++ pypy/trunk/pypy/translator/c/src/debug.h Sun Dec 13 17:05:13 2009 @@ -131,7 +131,7 @@ # else /* argh, we don't seem to have clock_gettime(). Bad OS. */ struct timeval tv; - gettimeofday(tv, NULL); + gettimeofday(&tv, NULL); return ((long long)tv.tv_sec) * 1000000LL + tv.tv_usec; # endif } From arigo at codespeak.net Sun Dec 13 18:06:16 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 13 Dec 2009 18:06:16 +0100 (CET) Subject: [pypy-svn] r70086 - in pypy/branch/virtual-forcing/pypy/jit/metainterp: . test Message-ID: <20091213170616.CFCCF168035@codespeak.net> Author: arigo Date: Sun Dec 13 18:06:15 2009 New Revision: 70086 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/jitprof.py pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_jitprof.py Log: Also count the number of virtualizable escapes. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/jitprof.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/jitprof.py Sun Dec 13 18:06:15 2009 @@ -20,6 +20,7 @@ OPT_FORCINGS ABORT_TOO_LONG ABORT_BRIDGE +ABORT_ESCAPE NVIRTUALS NVHOLES NVREUSED @@ -176,8 +177,9 @@ self._print_intline("opt ops", cnt[OPT_OPS]) self._print_intline("opt guards", cnt[OPT_GUARDS]) self._print_intline("forcings", cnt[OPT_FORCINGS]) - self._print_intline("trace too long", cnt[ABORT_TOO_LONG]) - self._print_intline("bridge abort", cnt[ABORT_BRIDGE]) + self._print_intline("abort: trace too long", cnt[ABORT_TOO_LONG]) + self._print_intline("abort: compiling", cnt[ABORT_BRIDGE]) + self._print_intline("abort: vable escape", cnt[ABORT_ESCAPE]) self._print_intline("nvirtuals", cnt[NVIRTUALS]) self._print_intline("nvholes", cnt[NVHOLES]) self._print_intline("nvreused", cnt[NVREUSED]) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Sun Dec 13 18:06:15 2009 @@ -834,8 +834,7 @@ try: self.metainterp.reached_can_enter_jit(self.env) except GiveUp: - self.metainterp.staticdata.profiler.count(ABORT_BRIDGE) - self.metainterp.switch_to_blackhole() + self.metainterp.switch_to_blackhole(ABORT_BRIDGE) if self.metainterp.is_blackholing(): self.blackhole_reached_merge_point(self.env) return True @@ -1281,8 +1280,7 @@ try: self.compile_done_with_this_frame(resultbox) except GiveUp: - self.staticdata.profiler.count(ABORT_BRIDGE) - self.switch_to_blackhole() + self.switch_to_blackhole(ABORT_BRIDGE) sd = self.staticdata if sd.result_type == 'void': assert resultbox is None @@ -1319,8 +1317,7 @@ try: self.compile_exit_frame_with_exception(excvaluebox) except GiveUp: - self.staticdata.profiler.count(ABORT_BRIDGE) - self.switch_to_blackhole() + self.switch_to_blackhole(ABORT_BRIDGE) raise self.staticdata.ExitFrameWithExceptionRef(self.cpu, excvaluebox.getref_base()) def check_recursion_invariant(self): @@ -1438,7 +1435,8 @@ op.pc = self.framestack[-1].pc op.name = self.framestack[-1].jitcode.name - def switch_to_blackhole(self): + def switch_to_blackhole(self, reason): + self.staticdata.profiler.count(reason) debug_print('~~~ ABORTING TRACING') debug_stop('jit-tracing') debug_start('jit-blackhole') @@ -1452,10 +1450,9 @@ if not self.is_blackholing(): warmrunnerstate = self.staticdata.state if len(self.history.operations) > warmrunnerstate.trace_limit: - self.staticdata.profiler.count(ABORT_TOO_LONG) self.greenkey_of_huge_function = self.find_biggest_function() self.portal_trace_positions = None - self.switch_to_blackhole() + self.switch_to_blackhole(ABORT_TOO_LONG) def _interpret(self): # Execute the frames forward until we raise a DoneWithThisFrame, @@ -1822,7 +1819,7 @@ escapes = True # if escapes: - self.switch_to_blackhole() + self.switch_to_blackhole(ABORT_ESCAPE) # if escapes: self.load_fields_from_virtualizable() Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_jitprof.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_jitprof.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_jitprof.py Sun Dec 13 18:06:15 2009 @@ -64,7 +64,7 @@ assert profiler.events == expected assert profiler.times == [2, 1, 1, 1] assert profiler.counters == [1, 1, 1, 1, 4, 3, 1, 1, 7, 1, 0, 0, 0, - 0, 0, 0] + 0, 0, 0, 0] def test_simple_loop_with_call(self): @dont_look_inside From arigo at codespeak.net Sun Dec 13 18:18:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 13 Dec 2009 18:18:42 +0100 (CET) Subject: [pypy-svn] r70087 - pypy/branch/virtual-forcing/pypy/jit/metainterp Message-ID: <20091213171842.2F93E16802D@codespeak.net> Author: arigo Date: Sun Dec 13 18:18:41 2009 New Revision: 70087 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Log: Uh. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Sun Dec 13 18:18:41 2009 @@ -11,7 +11,7 @@ from pypy.jit.metainterp import codewriter, executor from pypy.jit.metainterp.logger import Logger from pypy.jit.metainterp.jitprof import BLACKHOLED_OPS, EmptyProfiler -from pypy.jit.metainterp.jitprof import GUARDS, RECORDED_OPS +from pypy.jit.metainterp.jitprof import GUARDS, RECORDED_OPS, ABORT_ESCAPE from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG, ABORT_BRIDGE from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import specialize From afa at codespeak.net Mon Dec 14 10:56:36 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 14 Dec 2009 10:56:36 +0100 (CET) Subject: [pypy-svn] r70089 - in pypy/branch/import-builtin/pypy/module/imp: . test Message-ID: <20091214095636.9D59E16802A@codespeak.net> Author: afa Date: Mon Dec 14 10:56:35 2009 New Revision: 70089 Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Log: Add a test + fix for the reload() function Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Mon Dec 14 10:56:35 2009 @@ -326,7 +326,7 @@ if pkgdir is not None: space.setattr(w_mod, w('__path__'), space.newlist([w(pkgdir)])) -def load_module(space, w_modulename, find_info, ispkg=False): +def load_module(space, w_modulename, find_info, reuse=False): if find_info is None: return if find_info.w_loader: @@ -336,7 +336,7 @@ return space.getbuiltinmodule(find_info.filename) if find_info.modtype in (PY_SOURCE, PY_COMPILED, PKG_DIRECTORY): - if ispkg: + if reuse: w_mod = space.getitem(space.sys.get('modules'), w_modulename) else: w_mod = space.wrap(Module(space, w_modulename)) @@ -364,7 +364,10 @@ w_path, use_loader=False) if find_info is None: return w_mod - load_module(space, w_modulename, find_info, ispkg=True) + try: + load_module(space, w_modulename, find_info, reuse=True) + finally: + find_info.stream.close() # fetch the module again, in case of "substitution" w_mod = check_sys_modules(space, w_modulename) return w_mod @@ -442,7 +445,11 @@ msg = "No module named %s" % modulename raise OperationError(space.w_ImportError, space.wrap(msg)) - return load_module(space, w_modulename, find_info) + try: + return load_module(space, w_modulename, find_info, reuse=True) + finally: + find_info.stream.close() + # __________________________________________________________________ # Modified: pypy/branch/import-builtin/pypy/module/imp/test/test_import.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/test/test_import.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Mon Dec 14 10:56:35 2009 @@ -34,6 +34,7 @@ a = "imamodule = 1\ninpackage = 0", b = "imamodule = 1\ninpackage = 0", ambig = "imamodule = 1", + test_reload = "def test():\n raise ValueError\n", ) root.ensure("notapackage", dir=1) # empty, no __init__.py setuppkg("pkg", @@ -383,6 +384,34 @@ assert mod.b == 16 assert mod.c == "foo\nbar" + def test_reload(self): + import test_reload + try: + test_reload.test() + except ValueError: + pass + + # If this test runs too quickly, test_reload.py's mtime + # attribute will remain unchanged even if the file is rewritten. + # Consequently, the file would not reload. So, added a sleep() + # delay to assure that a new, distinct timestamp is written. + import time + time.sleep(1) + + f = open(test_reload.__file__, "w") + f.write("def test():\n raise NotImplementedError\n") + f.close() + reload(test_reload) + try: + test_reload.test() + except NotImplementedError: + pass + + # Ensure that the file is closed + # (on windows at least) + import os + os.unlink(test_reload.__file__) + def _getlong(data): x = marshal.dumps(data) return x[-4:] From afa at codespeak.net Mon Dec 14 13:39:04 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 14 Dec 2009 13:39:04 +0100 (CET) Subject: [pypy-svn] r70091 - pypy/branch/import-builtin/pypy/module/imp Message-ID: <20091214123904.1422F16802F@codespeak.net> Author: afa Date: Mon Dec 14 13:39:03 2009 New Revision: 70091 Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py pypy/branch/import-builtin/pypy/module/imp/interp_imp.py Log: Don't use sys.path when loading a submodule Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Mon Dec 14 13:39:03 2009 @@ -185,9 +185,7 @@ w_mod = None parts = modulename.split('.') prefix = [] - # it would be nice if we could do here: w_path = space.sys.w_path - # instead: - w_path = space.sys.get('path') + w_path = None first = None level = 0 @@ -222,6 +220,8 @@ def find_in_meta_path(space, w_modulename, w_path): assert w_modulename is not None + if w_path is None: + w_path = space.w_None for w_hook in space.unpackiterable(space.sys.get("meta_path")): w_loader = space.call_method(w_hook, "find_module", w_modulename, w_path) @@ -276,9 +276,11 @@ # XXX Check for frozen modules? # when w_path is a string - # check the builtin modules - if modulename in space.builtin_modules: - return FindInfo(C_BUILTIN, modulename, None) + if w_path is None: + # check the builtin modules + if modulename in space.builtin_modules: + return FindInfo(C_BUILTIN, modulename, None) + w_path = space.sys.get('path') # XXX check frozen modules? # when w_path is null @@ -381,10 +383,11 @@ modulename = '.'.join(prefix + [partname]) w_modulename = w(modulename) w_mod = check_sys_modules(space, w_modulename) + if w_mod is not None: if not space.is_w(w_mod, space.w_None): return w_mod - else: + elif not prefix or w_path is not None: find_info = find_module( space, modulename, w_modulename, partname, w_path) @@ -433,9 +436,9 @@ space.w_ImportError, space.wrap("reload(): parent %s not in sys.modules" % ( parent_name,))) - w_path = space.getitem(w_parent, space.wrap("__path")) + w_path = space.getitem(w_parent, space.wrap("__path__")) else: - w_path = space.sys.get('path') + w_path = None find_info = find_module( space, modulename, w_modulename, subname, w_path) @@ -448,7 +451,8 @@ try: return load_module(space, w_modulename, find_info, reuse=True) finally: - find_info.stream.close() + if find_info.stream: + find_info.stream.close() # __________________________________________________________________ Modified: pypy/branch/import-builtin/pypy/module/imp/interp_imp.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/interp_imp.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/interp_imp.py Mon Dec 14 13:39:03 2009 @@ -33,7 +33,7 @@ def find_module(space, w_name, w_path=None): name = space.str_w(w_name) if space.is_w(w_path, space.w_None): - w_path = space.sys.get('path') + w_path = None find_info = importing.find_module( space, name, w_name, name, w_path, use_loader=False) From afa at codespeak.net Mon Dec 14 14:39:25 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 14 Dec 2009 14:39:25 +0100 (CET) Subject: [pypy-svn] r70092 - in pypy/branch/import-builtin/pypy/module/imp: . test Message-ID: <20091214133925.CA8CF168027@codespeak.net> Author: afa Date: Mon Dec 14 14:39:25 2009 New Revision: 70092 Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py pypy/branch/import-builtin/pypy/module/imp/test/test_app.py pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Log: imp.find_module() always returns the .py when it exists, even when the .pyc is up to date. Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Mon Dec 14 14:39:25 2009 @@ -33,30 +33,22 @@ # check the .py file pyfile = filepart + ".py" if os.path.exists(pyfile) and case_ok(pyfile): - pyfile_ts = os.stat(pyfile)[stat.ST_MTIME] - pyfile_exists = True - else: - # The .py file does not exist. By default on PyPy, lonepycfiles - # is False: if a .py file does not exist, we don't even try to - # look for a lone .pyc file. - if not space.config.objspace.lonepycfiles: - return SEARCH_ERROR, None, None - pyfile_ts = 0 - pyfile_exists = False + return PY_SOURCE, ".py", "U" + + # The .py file does not exist. By default on PyPy, lonepycfiles + # is False: if a .py file does not exist, we don't even try to + # look for a lone .pyc file. + if not space.config.objspace.lonepycfiles: + return SEARCH_ERROR, None, None # check the .pyc file if space.config.objspace.usepycfiles: pycfile = filepart + ".pyc" - if case_ok(pycfile): - if check_compiled_module(space, pycfile, pyfile_ts): - # existing and up-to-date .pyc file - return PY_COMPILED, ".pyc", "rb" + if os.path.exists(pycfile) and case_ok(pycfile): + # existing .pyc file + return PY_COMPILED, ".pyc", "rb" - # no .pyc file, use the .py file if it exists - if pyfile_exists: - return PY_SOURCE, ".py", "U" - else: - return SEARCH_ERROR, None, None + return SEARCH_ERROR, None, None if sys.platform in ['linux2', 'freebsd']: def case_ok(filename): @@ -588,19 +580,35 @@ Load a source module from a given file and return its module object. """ - pycode = parse_source_module(space, pathname, source) - - if space.config.objspace.usepycfiles and write_pyc: - mtime = int(os.stat(pathname)[stat.ST_MTIME]) + print "AFA LOAD SOURCE MODULE", pathname + if space.config.objspace.usepycfiles: cpathname = pathname + 'c' - write_compiled_module(space, pycode, cpathname, mtime) + mtime = int(os.stat(pathname)[stat.ST_MTIME]) + stream = check_compiled_module(space, cpathname, mtime) + print "CHECKED", stream + else: + stream = None w = space.wrap + + if stream: + # existing and up-to-date .pyc file + try: + code_w = read_compiled_module(space, cpathname, stream.readall()) + finally: + stream.close() + space.setattr(w_mod, w('__file__'), w(cpathname)) + else: + code_w = parse_source_module(space, pathname, source) + + if space.config.objspace.usepycfiles and write_pyc: + write_compiled_module(space, code_w, cpathname, mtime) + w_dict = space.getattr(w_mod, w('__dict__')) space.call_method(w_dict, 'setdefault', w('__builtins__'), w(space.builtin)) - pycode.exec_code(space, w_dict, w_dict) + code_w.exec_code(space, w_dict, w_dict) return w_mod @@ -640,21 +648,23 @@ """ Check if a pyc file's magic number and (optionally) mtime match. """ + stream = None try: stream = streamio.open_file_as_stream(pycfilename, "rb") - try: - magic = _r_long(stream) - if magic != get_pyc_magic(space): - return False - if expected_mtime != 0: - pyc_mtime = _r_long(stream) - if pyc_mtime != expected_mtime: - return False - finally: + magic = _r_long(stream) + if magic != get_pyc_magic(space): stream.close() + return None + if expected_mtime != 0: + pyc_mtime = _r_long(stream) + if pyc_mtime != expected_mtime: + stream.close() + return None + return stream except StreamErrors: - return False - return True + if stream: + stream.close() + return None def read_compiled_module(space, cpathname, strbuf): """ Read a code object from a file and check it for validity """ @@ -679,14 +689,12 @@ "Bad magic number in %s" % cpathname)) #print "loading pyc file:", cpathname code_w = read_compiled_module(space, cpathname, source) - #if (Py_VerboseFlag) - # PySys_WriteStderr("import %s # precompiled from %s\n", - # name, cpathname); - w_dic = space.getattr(w_mod, w('__dict__')) - space.call_method(w_dic, 'setdefault', - w('__builtins__'), + + w_dict = space.getattr(w_mod, w('__dict__')) + space.call_method(w_dict, 'setdefault', + w('__builtins__'), w(space.builtin)) - code_w.exec_code(space, w_dic, w_dic) + code_w.exec_code(space, w_dict, w_dict) return w_mod Modified: pypy/branch/import-builtin/pypy/module/imp/test/test_app.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/test/test_app.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/test/test_app.py Mon Dec 14 14:39:25 2009 @@ -35,8 +35,7 @@ file.close() assert os.path.exists(pathname) pathname = pathname.lower() - assert (pathname.endswith('.py') or pathname.endswith('.pyc') - or pathname.endswith('.pyo')) + assert pathname.endswith('.py') # even if .pyc is up-to-date assert description in self.imp.get_suffixes() Modified: pypy/branch/import-builtin/pypy/module/imp/test/test_import.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/test/test_import.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Mon Dec 14 14:39:25 2009 @@ -443,13 +443,14 @@ ret = importing.check_compiled_module(space, cpathname, mtime) - assert ret is True + assert ret is not None + ret.close() # check for wrong mtime ret = importing.check_compiled_module(space, cpathname, mtime+1) - assert ret is False + assert ret is None os.remove(cpathname) # check for wrong version @@ -457,7 +458,7 @@ ret = importing.check_compiled_module(space, cpathname, mtime) - assert ret is False + assert ret is None # check for empty .pyc file f = open(cpathname, 'wb') @@ -465,7 +466,7 @@ ret = importing.check_compiled_module(space, cpathname, mtime) - assert ret is False + assert ret is None os.remove(cpathname) def test_read_compiled_module(self): @@ -659,7 +660,8 @@ ret = importing.check_compiled_module(space, cpathname, mtime) - assert ret is True + assert ret is not None + ret.close() # read compiled module stream = streamio.open_file_as_stream(cpathname, "rb") From arigo at codespeak.net Mon Dec 14 15:14:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 14 Dec 2009 15:14:55 +0100 (CET) Subject: [pypy-svn] r70095 - pypy/branch/virtual-forcing/pypy/jit/metainterp Message-ID: <20091214141455.657D616802A@codespeak.net> Author: arigo Date: Mon Dec 14 15:14:54 2009 New Revision: 70095 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Log: Debug_print the next opcode too, in addition to recording it in the trace. Useful in case of abort, to know where the abort took place. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Mon Dec 14 15:14:54 2009 @@ -845,6 +845,7 @@ greenkey = self.env[:num_green_args] sd = self.metainterp.staticdata loc = sd.state.get_location_str(greenkey) + debug_print(loc) constloc = self.metainterp.cpu.ts.conststr(loc) self.metainterp.history.record(rop.DEBUG_MERGE_POINT, [constloc], None) From afa at codespeak.net Mon Dec 14 15:16:19 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 14 Dec 2009 15:16:19 +0100 (CET) Subject: [pypy-svn] r70096 - pypy/branch/import-builtin/pypy/module/imp Message-ID: <20091214141619.757B916802A@codespeak.net> Author: afa Date: Mon Dec 14 15:16:18 2009 New Revision: 70096 Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py Log: fix translation, and remove debug prints Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Mon Dec 14 15:16:18 2009 @@ -580,14 +580,13 @@ Load a source module from a given file and return its module object. """ - print "AFA LOAD SOURCE MODULE", pathname if space.config.objspace.usepycfiles: cpathname = pathname + 'c' mtime = int(os.stat(pathname)[stat.ST_MTIME]) stream = check_compiled_module(space, cpathname, mtime) - print "CHECKED", stream else: stream = None + cpathname = None w = space.wrap From afa at codespeak.net Mon Dec 14 16:03:55 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 14 Dec 2009 16:03:55 +0100 (CET) Subject: [pypy-svn] r70098 - pypy/branch/import-builtin/pypy/module/imp Message-ID: <20091214150355.DF0F516802A@codespeak.net> Author: afa Date: Mon Dec 14 16:03:55 2009 New Revision: 70098 Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py Log: translation fixes Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Mon Dec 14 16:03:55 2009 @@ -580,15 +580,16 @@ Load a source module from a given file and return its module object. """ + w = space.wrap + if space.config.objspace.usepycfiles: cpathname = pathname + 'c' mtime = int(os.stat(pathname)[stat.ST_MTIME]) stream = check_compiled_module(space, cpathname, mtime) else: - stream = None cpathname = None - - w = space.wrap + mtime = 0 + stream = None if stream: # existing and up-to-date .pyc file From cfbolz at codespeak.net Mon Dec 14 16:04:45 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 14 Dec 2009 16:04:45 +0100 (CET) Subject: [pypy-svn] r70099 - in pypy/trunk/pypy: lang/gameboy translator/goal Message-ID: <20091214150445.4A49A16802F@codespeak.net> Author: cfbolz Date: Mon Dec 14 16:04:44 2009 New Revision: 70099 Removed: pypy/trunk/pypy/lang/gameboy/ pypy/trunk/pypy/translator/goal/targetgbfullprofiling.py pypy/trunk/pypy/translator/goal/targetgbimplementation.py pypy/trunk/pypy/translator/goal/targetgbprofiling.py pypy/trunk/pypy/translator/goal/targetgbrom4.py Log: Kill gameboy interp here, I moved it to http://codespeak.net/svn/pypy/lang/gameboy From cfbolz at codespeak.net Mon Dec 14 16:10:42 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 14 Dec 2009 16:10:42 +0100 (CET) Subject: [pypy-svn] r70101 - in pypy/trunk/pypy/lang: automata malbolge Message-ID: <20091214151042.8292A16802A@codespeak.net> Author: cfbolz Date: Mon Dec 14 16:10:42 2009 New Revision: 70101 Removed: pypy/trunk/pypy/lang/automata/ pypy/trunk/pypy/lang/malbolge/ Log: Simply kill the automata and malbolge implementations, I think they have outlived their usefulness. If anybody wants them back, please resurrect them in http://codespeak.net/svn/pypy/lang From cfbolz at codespeak.net Mon Dec 14 16:17:27 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 14 Dec 2009 16:17:27 +0100 (CET) Subject: [pypy-svn] r70103 - in pypy/trunk/pypy: lang/scheme translator/goal Message-ID: <20091214151727.33AE016802A@codespeak.net> Author: cfbolz Date: Mon Dec 14 16:17:26 2009 New Revision: 70103 Removed: pypy/trunk/pypy/lang/scheme/ pypy/trunk/pypy/translator/goal/targetscheme.py Log: scheme moves too From cfbolz at codespeak.net Mon Dec 14 17:09:53 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 14 Dec 2009 17:09:53 +0100 (CET) Subject: [pypy-svn] r70108 - pypy/trunk/pypy/doc Message-ID: <20091214160953.9F9F916802A@codespeak.net> Author: cfbolz Date: Mon Dec 14 17:09:53 2009 New Revision: 70108 Removed: pypy/trunk/pypy/doc/javascript-interpreter.txt Modified: pypy/trunk/pypy/doc/_ref.txt Log: kill moved docs, remove unused reference Modified: pypy/trunk/pypy/doc/_ref.txt ============================================================================== --- pypy/trunk/pypy/doc/_ref.txt (original) +++ pypy/trunk/pypy/doc/_ref.txt Mon Dec 14 17:09:53 2009 @@ -45,7 +45,6 @@ .. _`pypy/interpreter/astcompiler/asthelpers.py`: ../../pypy/interpreter/astcompiler/asthelpers.py .. _`pypy/interpreter/astcompiler/ast.py`: ../../pypy/interpreter/astcompiler/ast.py .. _`pypy/interpreter/typedef.py`: ../../pypy/interpreter/typedef.py -.. _`lang/`: ../../pypy/lang .. _`lib/`: .. _`pypy/lib/`: ../../pypy/lib .. _`lib/app_test/`: ../../pypy/lib/app_test From cfbolz at codespeak.net Mon Dec 14 17:21:55 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 14 Dec 2009 17:21:55 +0100 (CET) Subject: [pypy-svn] r70109 - pypy/branch/change-celldict Message-ID: <20091214162155.0E1CB16802A@codespeak.net> Author: cfbolz Date: Mon Dec 14 17:21:54 2009 New Revision: 70109 Added: pypy/branch/change-celldict/ - copied from r70108, pypy/trunk/ Log: a branch where I want to experiment with a radically simplified cell-dict From cfbolz at codespeak.net Mon Dec 14 18:01:15 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 14 Dec 2009 18:01:15 +0100 (CET) Subject: [pypy-svn] r70110 - in pypy/branch/change-celldict/pypy: interpreter objspace/std objspace/std/test Message-ID: <20091214170115.1A40B168027@codespeak.net> Author: cfbolz Date: Mon Dec 14 18:01:14 2009 New Revision: 70110 Modified: pypy/branch/change-celldict/pypy/interpreter/pycode.py pypy/branch/change-celldict/pypy/objspace/std/celldict.py pypy/branch/change-celldict/pypy/objspace/std/objspace.py pypy/branch/change-celldict/pypy/objspace/std/test/test_celldict.py Log: Strip the cell dict down to something radically simple. Now it is really just a dict mapping names to cells. For the non-JIT case this makes global-lookups non-optimized (but the caching seems to not have helped much anyway). For the JIT-case the effect is nearly as good as the old code, and even better, if the same key is looked up many times in the same module (which is quite common for libraries). Modified: pypy/branch/change-celldict/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/change-celldict/pypy/interpreter/pycode.py (original) +++ pypy/branch/change-celldict/pypy/interpreter/pycode.py Mon Dec 14 18:01:14 2009 @@ -117,9 +117,6 @@ self._compute_flatcall() - if self.space.config.objspace.std.withcelldict: - from pypy.objspace.std.celldict import init_code - init_code(self) def _init_flags(self): co_code = self.co_code Modified: pypy/branch/change-celldict/pypy/objspace/std/celldict.py ============================================================================== --- pypy/branch/change-celldict/pypy/objspace/std/celldict.py (original) +++ pypy/branch/change-celldict/pypy/objspace/std/celldict.py Mon Dec 14 18:01:14 2009 @@ -1,4 +1,8 @@ -from pypy.interpreter.pycode import CO_CONTAINSGLOBALS +""" A very simple cell dict implementation. The dictionary maps keys to cell. +This ensures that the function (dict, key) -> cell is pure. By itself, this +optimization is not helping at all, but in conjunction with the JIT it can +speed up global lookups a lot.""" + from pypy.objspace.std.dictmultiobject import IteratorImplementation from pypy.objspace.std.dictmultiobject import W_DictMultiObject, _is_sane_hash from pypy.rlib import jit @@ -19,31 +23,22 @@ def __init__(self, space): self.space = space self.content = {} - self.unshadowed_builtins = {} - def getcell(self, key, make_new=True): + def getcell(self, key, makenew): + if makenew or jit.we_are_jitted(): + # when we are jitting, we always go through the pure function + # below, to ensure that we have no residual dict lookup + return self._getcell_makenew(key) + return self.content.get(key, None) + + @jit.purefunction_promote + def _getcell_makenew(self, key): res = self.content.get(key, None) if res is not None: return res - if not make_new: - return None result = self.content[key] = ModuleCell() return result - def add_unshadowed_builtin(self, name, builtin_impl): - assert isinstance(builtin_impl, ModuleDictImplementation) - self.unshadowed_builtins[name] = builtin_impl - - def invalidate_unshadowed_builtin(self, name): - impl = self.unshadowed_builtins[name] - try: - cell = impl.content[name] - except KeyError: - pass - else: - w_value = cell.invalidate() - cell = impl.content[name] = ModuleCell(w_value) - def impl_setitem(self, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_str): @@ -52,11 +47,7 @@ self._as_rdict().setitem(w_key, w_value) def impl_setitem_str(self, name, w_value, shadows_type=True): - self.getcell(name).w_value = w_value - - if name in self.unshadowed_builtins: - self.invalidate_unshadowed_builtin(name) - del self.unshadowed_builtins[name] + self.getcell(name, True).w_value = w_value def impl_delitem(self, w_key): space = self.space @@ -64,17 +55,25 @@ if space.is_w(w_key_type, space.w_str): key = space.str_w(w_key) cell = self.getcell(key, False) - if cell is None: + if cell is None or cell.w_value is None: raise KeyError + # note that we don't remove the cell from self.content, to make + # sure that a key that was found at any point in the dict, still + # maps to the same cell later (even if this cell no longer + # represents a key) cell.invalidate() - del self.content[key] elif _is_sane_hash(space, w_key_type): raise KeyError else: self._as_rdict().delitem(w_key) def impl_length(self): - return len(self.content) + # inefficient, but do we care? + res = 0 + for cell in self.content.itervalues(): + if cell.w_value is not None: + res += 1 + return res def impl_getitem(self, w_lookup): space = self.space @@ -91,6 +90,7 @@ res = self.getcell(lookup, False) if res is None: return None + # note that even if the res.w_value is None, the next line is fine return res.w_value def impl_iter(self): @@ -98,39 +98,34 @@ def impl_keys(self): space = self.space - return [space.wrap(key) for key in self.content.iterkeys()] + return [space.wrap(key) for key, cell in self.content.iteritems() + if cell.w_value is not None] def impl_values(self): - return [cell.w_value for cell in self.content.itervalues()] + return [cell.w_value for cell in self.content.itervalues() + if cell.w_value is not None] def impl_items(self): space = self.space return [space.newtuple([space.wrap(key), cell.w_value]) - for (key, cell) in self.content.iteritems()] + for (key, cell) in self.content.iteritems() + if cell.w_value is not None] def impl_clear(self): - # inefficient, but who cares for k, cell in self.content.iteritems(): cell.invalidate() - for k in self.unshadowed_builtins: - self.invalidate_unshadowed_builtin(k) - self.content.clear() - self.unshadowed_builtins.clear() - def _as_rdict(self): r_dict_content = self.initialize_as_rdict() for k, cell in self.content.iteritems(): - r_dict_content[self.space.wrap(k)] = cell.w_value + if cell.w_value is not None: + r_dict_content[self.space.wrap(k)] = cell.w_value cell.invalidate() - for k in self.unshadowed_builtins: - self.invalidate_unshadowed_builtin(k) self._clear_fields() return self def _clear_fields(self): self.content = None - self.unshadowed_builtins = None class ModuleDictIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): @@ -138,99 +133,8 @@ self.iterator = dictimplementation.content.iteritems() def next_entry(self): - # note that this 'for' loop only runs once, at most for key, cell in self.iterator: - return (self.space.wrap(key), cell.w_value) + if cell.w_value is not None: + return (self.space.wrap(key), cell.w_value) else: return None, None - - -class State(object): - def __init__(self, space): - self.space = space - self.invalidcell = ModuleCell() - self.always_invalid_cache = [] - self.neverused_dictcontent = {} - -class GlobalCacheHolder(object): - def __init__(self, space): - self.cache = None - state = space.fromcache(State) - self.dictcontent = state.neverused_dictcontent - - def getcache(self, space, code, w_globals): - if type(w_globals) is ModuleDictImplementation: - content = w_globals.content - else: - content = None - if self.dictcontent is content: - return self.cache - return self.getcache_slow(space, code, w_globals, content) - getcache._always_inline_ = True - - def getcache_slow(self, space, code, w_globals, content): - state = space.fromcache(State) - if content is None: - cache = state.always_invalid_cache - if len(code.co_names_w) > len(cache): - cache = [state.invalidcell] * len(code.co_names_w) - state.always_invalid_cache = cache - else: - cache = [state.invalidcell] * len(code.co_names_w) - self.cache = cache - self.dictcontent = content - return cache - getcache_slow._dont_inline_ = True - -def init_code(code): - if code.co_flags & CO_CONTAINSGLOBALS: - code.globalcacheholder = GlobalCacheHolder(code.space) - else: - code.globalcacheholder = None - - -def get_global_cache(space, code, w_globals): - from pypy.interpreter.pycode import PyCode - assert isinstance(code, PyCode) - holder = code.globalcacheholder - if holder is not None: - return holder.getcache(space, code, w_globals) - return None - -def getimplementation(w_dict): - if type(w_dict) is ModuleDictImplementation and w_dict.r_dict_content is None: - return w_dict - else: - return None - -def LOAD_GLOBAL(f, nameindex, *ignored): - cell = f.cache_for_globals[nameindex] - w_value = cell.w_value - if w_value is None: - # slow path - w_value = load_global_fill_cache(f, nameindex) - f.pushvalue(w_value) -LOAD_GLOBAL._always_inline_ = True - -def find_cell_from_dict(implementation, name): - if implementation is not None: - return implementation.getcell(name, False) - return None - - at jit.dont_look_inside -def load_global_fill_cache(f, nameindex): - name = f.space.str_w(f.getname_w(nameindex)) - implementation = getimplementation(f.w_globals) - if implementation is not None: - cell = implementation.getcell(name, False) - if cell is None: - builtin_impl = getimplementation(f.get_builtin().getdict()) - cell = find_cell_from_dict(builtin_impl, name) - if cell is not None: - implementation.add_unshadowed_builtin(name, builtin_impl) - - if cell is not None: - f.cache_for_globals[nameindex] = cell - return cell.w_value - return f._load_global(f.getname_u(nameindex)) -load_global_fill_cache._dont_inline_ = True Modified: pypy/branch/change-celldict/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/change-celldict/pypy/objspace/std/objspace.py (original) +++ pypy/branch/change-celldict/pypy/objspace/std/objspace.py Mon Dec 14 18:01:14 2009 @@ -71,16 +71,8 @@ # Import all the object types and implementations self.model = StdTypeModel(self.config) - from pypy.objspace.std.celldict import get_global_cache class StdObjSpaceFrame(pyframe.PyFrame): - if self.config.objspace.std.withcelldict: - def __init__(self, space, code, w_globals, closure): - pyframe.PyFrame.__init__(self, space, code, w_globals, closure) - self.cache_for_globals = get_global_cache(space, code, w_globals) - - from pypy.objspace.std.celldict import LOAD_GLOBAL - if self.config.objspace.std.optimized_int_add: if self.config.objspace.std.withsmallint: def BINARY_ADD(f, oparg, *ignored): Modified: pypy/branch/change-celldict/pypy/objspace/std/test/test_celldict.py ============================================================================== --- pypy/branch/change-celldict/pypy/objspace/std/test/test_celldict.py (original) +++ pypy/branch/change-celldict/pypy/objspace/std/test/test_celldict.py Mon Dec 14 18:01:14 2009 @@ -1,262 +1,31 @@ import py from pypy.conftest import gettestobjspace, option -from pypy.objspace.std.celldict import get_global_cache, ModuleCell, ModuleDictImplementation +from pypy.objspace.std.celldict import ModuleCell, ModuleDictImplementation +from pypy.objspace.std.test.test_dictmultiobject import FakeSpace from pypy.interpreter import gateway -# this file tests mostly the effects of caching global lookup. The dict -# implementation itself is tested in test_dictmultiobject.py - - -class AppTestCellDict(object): - def setup_class(cls): - if option.runappdirect: - py.test.skip("not appdirect tests") - cls.space = gettestobjspace(**{"objspace.std.withcelldict": True}) - cls.w_impl_used = cls.space.appexec([], """(): - import __pypy__ - def impl_used(obj): - assert "ModuleDictImplementation" in __pypy__.internal_repr(obj) - return impl_used - """) - def is_in_cache(space, w_code, w_globals, w_name): - name = space.str_w(w_name) - cache = get_global_cache(space, w_code, w_globals) - index = [space.str_w(w_n) for w_n in w_code.co_names_w].index(name) - return space.wrap(cache[index].w_value is not None) - is_in_cache = gateway.interp2app(is_in_cache) - cls.w_is_in_cache = cls.space.wrap(is_in_cache) - stored_builtins = [] - def rescue_builtins(space): - w_dict = space.builtin.getdict() - content = {} - for key, cell in w_dict.content.iteritems(): - newcell = ModuleCell() - newcell.w_value = cell.w_value - content[key] = newcell - stored_builtins.append(content) - rescue_builtins = gateway.interp2app(rescue_builtins) - cls.w_rescue_builtins = cls.space.wrap(rescue_builtins) - def restore_builtins(space): - w_dict = space.builtin.getdict() - assert isinstance(w_dict, ModuleDictImplementation) - w_dict.content = stored_builtins.pop() - w_dict.fallback = None - restore_builtins = gateway.interp2app(restore_builtins) - cls.w_restore_builtins = cls.space.wrap(restore_builtins) - - def test_same_code_in_different_modules(self): - import sys - mod1 = type(sys)("abc") - self.impl_used(mod1.__dict__) - glob1 = mod1.__dict__ - mod2 = type(sys)("abc") - self.impl_used(mod2.__dict__) - glob2 = mod2.__dict__ - def f(): - return x + 1 - code = f.func_code - f1 = type(f)(code, glob1) - mod1.x = 1 - assert not self.is_in_cache(code, glob1, "x") - assert f1() == 2 - assert self.is_in_cache(code, glob1, "x") - assert f1() == 2 - assert self.is_in_cache(code, glob1, "x") - mod1.x = 2 - assert f1() == 3 - assert self.is_in_cache(code, glob1, "x") - assert f1() == 3 - assert self.is_in_cache(code, glob1, "x") - f2 = type(f)(code, glob2) - mod2.x = 5 - assert not self.is_in_cache(code, glob2, "x") - assert f2() == 6 - assert self.is_in_cache(code, glob2, "x") - assert f2() == 6 - assert self.is_in_cache(code, glob2, "x") - mod2.x = 7 - assert f2() == 8 - assert self.is_in_cache(code, glob2, "x") - assert f2() == 8 - assert self.is_in_cache(code, glob2, "x") - - def test_override_builtins(self): - import sys, __builtin__ - mod1 = type(sys)("abc") - glob1 = mod1.__dict__ - self.impl_used(mod1.__dict__) - def f(): - return len(x) - code = f.func_code - f1 = type(f)(f.func_code, glob1) - mod1.x = [] - assert not self.is_in_cache(code, glob1, "len") - assert not self.is_in_cache(code, glob1, "x") - assert f1() == 0 - assert self.is_in_cache(code, glob1, "len") - assert self.is_in_cache(code, glob1, "x") - assert f1() == 0 - mod1.x.append(1) - assert f1() == 1 - assert self.is_in_cache(code, glob1, "len") - assert self.is_in_cache(code, glob1, "x") - mod1.len = lambda x: 15 - assert not self.is_in_cache(code, glob1, "len") - mod1.x.append(1) - assert f1() == 15 - assert self.is_in_cache(code, glob1, "len") - assert f1() == 15 - assert self.is_in_cache(code, glob1, "len") - del mod1.len - mod1.x.append(1) - assert not self.is_in_cache(code, glob1, "len") - assert f1() == 3 - assert self.is_in_cache(code, glob1, "len") - assert f1() == 3 - assert self.is_in_cache(code, glob1, "len") - orig_len = __builtins__.len - try: - __builtins__.len = lambda x: 12 - mod1.x.append(1) - assert self.is_in_cache(code, glob1, "len") - assert f1() == 12 - assert self.is_in_cache(code, glob1, "len") - assert f1() == 12 - assert self.is_in_cache(code, glob1, "len") - finally: - __builtins__.len = orig_len - - def test_override_builtins2(self): - import sys, __builtin__ - mod1 = type(sys)("abc") - glob1 = mod1.__dict__ - self.impl_used(mod1.__dict__) - def f(): - return l(x) - code = f.func_code - f1 = type(f)(f.func_code, glob1) - mod1.x = [] - __builtin__.l = len - try: - assert not self.is_in_cache(code, glob1, "l") - assert not self.is_in_cache(code, glob1, "x") - assert f1() == 0 - assert self.is_in_cache(code, glob1, "l") - assert self.is_in_cache(code, glob1, "x") - assert f1() == 0 - mod1.x.append(1) - assert f1() == 1 - assert self.is_in_cache(code, glob1, "l") - assert self.is_in_cache(code, glob1, "x") - del __builtin__.l - mod1.l = len - mod1.x.append(1) - assert not self.is_in_cache(code, glob1, "l") - assert f1() == 2 - assert self.is_in_cache(code, glob1, "l") - assert self.is_in_cache(code, glob1, "x") - finally: - if hasattr(__builtins__, "l"): - del __builtins__.l - - def test_generator(self): - import sys, __builtin__ - mod1 = type(sys)("abc") - glob1 = mod1.__dict__ - self.impl_used(mod1.__dict__) - def f(): - yield 1 - yield x - yield len(x) - code = f.func_code - f1 = type(f)(f.func_code, glob1) - mod1.x = [] - gen = f1() - assert not self.is_in_cache(code, glob1, "len") - assert not self.is_in_cache(code, glob1, "x") - v = gen.next() - assert v == 1 - assert not self.is_in_cache(code, glob1, "len") - assert not self.is_in_cache(code, glob1, "x") - v = gen.next() - assert v is mod1.x - assert not self.is_in_cache(code, glob1, "len") - assert self.is_in_cache(code, glob1, "x") - v = gen.next() - assert v == 0 - assert self.is_in_cache(code, glob1, "len") - assert self.is_in_cache(code, glob1, "x") - - def test_degenerate_to_rdict(self): - import sys - mod1 = type(sys)("abc") - self.impl_used(mod1.__dict__) - glob1 = mod1.__dict__ - def f(): - return x + 1 - code = f.func_code - f1 = type(f)(code, glob1) - mod1.x = 1 - assert not self.is_in_cache(code, glob1, "x") - assert f1() == 2 - assert self.is_in_cache(code, glob1, "x") - glob1[1] = 2 - assert not self.is_in_cache(code, glob1, "x") - assert f1() == 2 - assert not self.is_in_cache(code, glob1, "x") - - def test_degenerate_builtin_to_rdict(self): - import sys, __builtin__ - mod1 = type(sys)("abc") - self.impl_used(mod1.__dict__) - glob1 = mod1.__dict__ - def f(): - return len(x) - code = f.func_code - f1 = type(f)(code, glob1) - mod1.x = [1, 2] - assert not self.is_in_cache(code, glob1, "x") - assert not self.is_in_cache(code, glob1, "len") - assert f1() == 2 - assert self.is_in_cache(code, glob1, "x") - assert self.is_in_cache(code, glob1, "len") - self.rescue_builtins() - try: - __builtin__.__dict__[1] = 2 - assert not self.is_in_cache(code, glob1, "len") - assert f1() == 2 - assert not self.is_in_cache(code, glob1, "len") - finally: - self.restore_builtins() - - def test_mapping_as_locals(self): - import sys - if sys.version_info < (2,5) or not hasattr(sys, 'pypy_objspaceclass'): - skip("need CPython 2.5 or PyPy for non-dictionaries in exec statements") - class M(object): - def __getitem__(self, key): - return key - def __setitem__(self, key, value): - self.result[key] = value - m = M() - m.result = {} - exec "x=m" in {}, m - assert m.result == {'x': 'm'} - exec "y=n" in m # NOTE: this doesn't work in CPython 2.4 - assert m.result == {'x': 'm', 'y': 'n'} - - def test_subclass_of_dict_as_locals(self): - import sys - if sys.version_info < (2,5) or not hasattr(sys, 'pypy_objspaceclass'): - skip("need CPython 2.5 or PyPy for non-dictionaries in exec statements") - class M(dict): - def __getitem__(self, key): - return key - def __setitem__(self, key, value): - dict.__setitem__(self, key, value) - m = M() - exec "x=m" in {}, m - assert m == {'x': 'm'} - exec "y=n" in m # NOTE: this doesn't work in CPython 2.4 - assert m == {'x': 'm', 'y': 'n'} +space = FakeSpace() +class TestCellDict(object): + def test_basic_property(self): + d = ModuleDictImplementation(space) + d.setitem("a", 1) + assert d.getcell("a", False) is d.getcell("a", False) + acell = d.getcell("a", False) + d.setitem("b", 2) + assert d.getcell("b", False) is d.getcell("b", False) + assert d.getcell("c", True) is d.getcell("c", True) + + assert d.getitem("a") == 1 + assert d.getitem("b") == 2 + + d.delitem("a") + py.test.raises(KeyError, d.delitem, "a") + assert d.getitem("a") is None + assert d.getcell("a", False) is acell + assert d.length() == 1 + + d.clear() + assert d.getitem("a") is None + assert d.getcell("a", False) is acell + assert d.length() == 1 From cfbolz at codespeak.net Mon Dec 14 18:21:55 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 14 Dec 2009 18:21:55 +0100 (CET) Subject: [pypy-svn] r70111 - pypy/extradoc/planning Message-ID: <20091214172155.9596216802A@codespeak.net> Author: cfbolz Date: Mon Dec 14 18:21:55 2009 New Revision: 70111 Modified: pypy/extradoc/planning/jit.txt Log: add a point about threads Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Mon Dec 14 18:21:55 2009 @@ -9,6 +9,9 @@ - jit/asmgcc + threads? [DONE probably. Needs a bit more testing.] +- since threads are enabled with the JIT, getexecutioncontext cannot be folded + by the JIT anymore. we need to do something in that direction + - think about code memory management - forcing virtualizables should only force fields affected, not everything @@ -24,6 +27,7 @@ _trace_and_drag_out_of_nursery + Python interpreter: - goal: on average <=5 guards per original bytecode From afa at codespeak.net Tue Dec 15 10:20:24 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 15 Dec 2009 10:20:24 +0100 (CET) Subject: [pypy-svn] r70119 - in pypy/branch/import-builtin/pypy/module/imp: . test Message-ID: <20091215092024.A5146168015@codespeak.net> Author: afa Date: Tue Dec 15 10:20:24 2009 New Revision: 70119 Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Log: Test and fix for reload() Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Tue Dec 15 10:20:24 2009 @@ -428,7 +428,7 @@ space.w_ImportError, space.wrap("reload(): parent %s not in sys.modules" % ( parent_name,))) - w_path = space.getitem(w_parent, space.wrap("__path__")) + w_path = space.getattr(w_parent, space.wrap("__path__")) else: w_path = None Modified: pypy/branch/import-builtin/pypy/module/imp/test/test_import.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/test/test_import.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Tue Dec 15 10:20:24 2009 @@ -412,6 +412,10 @@ import os os.unlink(test_reload.__file__) + def test_reload_submodule(self): + import pkg.a + reload(pkg.a) + def _getlong(data): x = marshal.dumps(data) return x[-4:] From arigo at codespeak.net Tue Dec 15 10:31:14 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 15 Dec 2009 10:31:14 +0100 (CET) Subject: [pypy-svn] r70120 - pypy/trunk/pypy/translator/benchmark Message-ID: <20091215093114.31AB4168015@codespeak.net> Author: arigo Date: Tue Dec 15 10:31:13 2009 New Revision: 70120 Modified: pypy/trunk/pypy/translator/benchmark/bench-custom.py pypy/trunk/pypy/translator/benchmark/benchmarks.py pypy/trunk/pypy/translator/benchmark/result.py Log: * Add the --size-factor=N option, which runs the benchmarks for N times longer. Should let us see the effect of JIT compilation on the total runtime. * Record and display (if --verbose) the output of the process. This should be convenient to grab and parse the JIT summary information. Modified: pypy/trunk/pypy/translator/benchmark/bench-custom.py ============================================================================== --- pypy/trunk/pypy/translator/benchmark/bench-custom.py (original) +++ pypy/trunk/pypy/translator/benchmark/bench-custom.py Tue Dec 15 10:31:13 2009 @@ -29,6 +29,8 @@ if not b.check(): print "can't run %s benchmark for some reason"%(b.name,) else: + if int(options.sizefactor) > 1: + b = b * int(options.sizefactor) benchmarks.append(b) exes = get_executables(args) @@ -46,11 +48,11 @@ if not options.nocpython: exes = full_pythons + exes - for i in range(int(options.runcount)) + [None]: + for i in range(int(options.runcount)) or [None]: if i is not None: for exe in exes: for b in benchmarks: - benchmark_result.result(exe, allowcreate=True).run_benchmark(b, verbose=True) + benchmark_result.result(exe, allowcreate=True).run_benchmark(b, verbose=options.verbose) pickle.dump(benchmark_result, open(options.picklefile, 'wb')) @@ -106,5 +108,9 @@ '--no-cpython', action='store_true', dest='nocpython', default=None, ) + parser.add_option( + '--size-factor', dest='sizefactor', + default='1', + ) options, args = parser.parse_args(sys.argv[1:]) main(options, args) Modified: pypy/trunk/pypy/translator/benchmark/benchmarks.py ============================================================================== --- pypy/trunk/pypy/translator/benchmark/benchmarks.py (original) +++ pypy/trunk/pypy/translator/benchmark/benchmarks.py Tue Dec 15 10:31:13 2009 @@ -20,17 +20,30 @@ return float(line.split()[len(pattern.split())]) class Benchmark(object): - def __init__(self, name, runner, asc_good, units, check=lambda:True): - self.name = name + def __init__(self, name, runner, asc_good, units, + check=lambda:True, sizefactor=1): + if sizefactor > 1: + self.name = name + '*%d' % sizefactor + else: + self.name = name + self._basename = name self._run = runner self.asc_good = asc_good self.units = units self.check = check + self.sizefactor = sizefactor + def __mul__(self, n): + return Benchmark(self._basename, self._run, self.asc_good, self.units, + self.check, self.sizefactor * n) def run(self, exe): + global latest_output + latest_output = '' try: - return self._run(exe) - except BenchmarkFailed: - return '-FAILED-' + result = self._run(exe, self.sizefactor) + except BenchmarkFailed, e: + result = '-FAILED-' + self.latest_output = latest_output + return result def external_dependency(dirname, svnurl, revision): """Check out (if necessary) a given fixed revision of a svn url.""" @@ -54,24 +67,26 @@ return True def run_cmd(cmd): + global latest_output #print "running", cmd pipe = os.popen(cmd + ' 2>&1') r = pipe.read() status = pipe.close() + latest_output = r if status: raise BenchmarkFailed(status) return r -def run_pystone(executable='/usr/local/bin/python', n=''): +def run_pystone(executable='/usr/local/bin/python', sizefactor=1): from pypy.tool import autopath distdir = py.path.local(autopath.pypydir).dirpath() pystone = py.path.local(autopath.libpythondir).join('test', 'pystone.py') - txt = run_cmd('"%s" "%s" %s' % (executable, pystone, n)) + txt = run_cmd('"%s" "%s" %d' % (executable, pystone, 50000 * sizefactor)) return get_result(txt, PYSTONE_PATTERN) -def run_richards(executable='/usr/local/bin/python', n=5): +def run_richards(executable='/usr/local/bin/python', sizefactor=1): richards = py.path.local(__file__).dirpath().dirpath().join('goal').join('richards.py') - txt = run_cmd('"%s" %s %s' % (executable, richards, n)) + txt = run_cmd('"%s" %s %d' % (executable, richards, 5 * sizefactor)) return get_result(txt, RICHARDS_PATTERN) def run_translate(executable='/usr/local/bin/python'): @@ -117,7 +132,7 @@ # 'svn://svn.berlios.de/docutils/trunk/docutils', # 4821) -def run_templess(executable='/usr/local/bin/python'): +def run_templess(executable='/usr/local/bin/python', sizefactor=1): """ run some script in the templess package templess is some simple templating language, to check out use @@ -127,66 +142,54 @@ pypath = os.path.dirname(os.path.dirname(py.__file__)) templessdir = here.join('templess') testscript = templessdir.join('test/oneshot.py') - command = 'PYTHONPATH="%s:%s" "%s" "%s" 100' % (here, pypath, - executable, testscript) + command = 'PYTHONPATH="%s:%s" "%s" "%s" %d' % (here, pypath, + executable, testscript, + 100 * sizefactor) txt = run_cmd(command) - try: - result = float([line for line in txt.split('\n') if line.strip()][-1]) - except ValueError: + for line in txt.split('\n'): + if '.' in line: + try: + return float(line) / sizefactor + except ValueError: + pass + else: raise BenchmarkFailed - return result def check_templess(): return external_dependency('templess', 'http://johnnydebris.net/templess/trunk', 100) -def run_gadfly(executable='/usr/local/bin/python'): +def run_gadfly(executable='/usr/local/bin/python', sizefactor=1): """ run some tests in the gadfly pure Python database """ here = py.path.local(__file__).dirpath() gadfly = here.join('gadfly') testscript = gadfly.join('test', 'testsubset.py') - command = 'PYTHONPATH="%s" "%s" "%s"' % (gadfly, executable, testscript) + command = 'PYTHONPATH="%s" "%s" "%s" %d' % (gadfly, executable, testscript, + sizefactor) txt = run_cmd(command) - lines = [line for line in txt.split('\n') if line.strip()] - if lines[-1].strip() != 'OK': - raise BenchmarkFailed - lastword = lines[-2].split()[-1] - if not lastword.endswith('s'): - raise BenchmarkFailed - try: - result = float(lastword[:-1]) - except ValueError: - raise BenchmarkFailed - return result + return get_result(txt, 'Total running time:') / sizefactor def check_gadfly(): return external_dependency('gadfly', 'http://codespeak.net/svn/user/arigo/hack/pypy-hack/gadflyZip', - 54470) + 70117) -def run_mako(executable='/usr/local/bin/python'): +def run_mako(executable='/usr/local/bin/python', sizefactor=1): """ run some tests in the mako templating system """ here = py.path.local(__file__).dirpath() mako = here.join('mako') testscript = mako.join('examples', 'bench', 'basic.py') - command = 'PYTHONPATH="%s" "%s" "%s" mako' % (mako.join('lib'), - executable, testscript) + command = 'PYTHONPATH="%s" "%s" "%s" -n%d mako' % (mako.join('lib'), + executable, testscript, + 2000 * sizefactor) txt = run_cmd(command) - lines = [line for line in txt.split('\n') if line.strip()] - words = lines[-1].split() - if words[0] != 'Mako:': - raise BenchmarkFailed - try: - result = float(words[1]) - except ValueError: - raise BenchmarkFailed - return result + return get_result(txt, 'Mako:') def check_mako(): return external_dependency('mako', 'http://codespeak.net/svn/user/arigo/hack/pypy-hack/mako', - 40235) + 70118) def check_translate(): return False # XXX what should we do about the dependency on ctypes? Modified: pypy/trunk/pypy/translator/benchmark/result.py ============================================================================== --- pypy/trunk/pypy/translator/benchmark/result.py (original) +++ pypy/trunk/pypy/translator/benchmark/result.py Tue Dec 15 10:31:13 2009 @@ -89,12 +89,15 @@ self.asc_goods[benchmark.name] = benchmark.asc_good if self.run_counts.get(benchmark.name, 0) > self.max_results: return - if verbose: - print 'running', benchmark.name, 'for', self.exe_name, - sys.stdout.flush() + print 'running', benchmark.name, 'for', self.exe_name, + sys.stdout.flush() new_result = benchmark.run(self.exe_name) + print new_result if verbose: - print new_result + print '{' + for line in benchmark.latest_output.splitlines(False): + print '\t' + line + print '}' self.run_counts[benchmark.name] = self.run_counts.get(benchmark.name, 0) + 1 if new_result == '-FAILED-': return @@ -142,6 +145,8 @@ elif stat.startswith('bench:'): from pypy.translator.benchmark import benchmarks statkind, statdetail = stat.split(':', 1) + if '*' in statdetail: + statdetail = statdetail.split('*')[0] b = benchmarks.BENCHMARKS_BY_NAME[statdetail] return "%8.2f%s"%(statvalue, b.units), 1 elif stat == 'pypy_rev': From afa at codespeak.net Tue Dec 15 10:39:31 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 15 Dec 2009 10:39:31 +0100 (CET) Subject: [pypy-svn] r70121 - in pypy/branch/import-builtin/pypy/module: imp zipimport Message-ID: <20091215093931.AA681168015@codespeak.net> Author: afa Date: Tue Dec 15 10:39:31 2009 New Revision: 70121 Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py pypy/branch/import-builtin/pypy/module/zipimport/interp_zipimport.py Log: Fix tests in module/zipimport Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Tue Dec 15 10:39:31 2009 @@ -574,6 +574,14 @@ pycode = ec.compiler.compile(source, pathname, 'exec', 0) return pycode +def exec_code_module(space, w_mod, code_w): + w_dict = space.getattr(w_mod, space.wrap('__dict__')) + space.call_method(w_dict, 'setdefault', + space.wrap('__builtins__'), + space.wrap(space.builtin)) + code_w.exec_code(space, w_dict, w_dict) + + def load_source_module(space, w_modulename, w_mod, pathname, source, write_pyc=True): """ @@ -604,11 +612,7 @@ if space.config.objspace.usepycfiles and write_pyc: write_compiled_module(space, code_w, cpathname, mtime) - w_dict = space.getattr(w_mod, w('__dict__')) - space.call_method(w_dict, 'setdefault', - w('__builtins__'), - w(space.builtin)) - code_w.exec_code(space, w_dict, w_dict) + exec_code_module(space, w_mod, code_w) return w_mod @@ -689,12 +693,8 @@ "Bad magic number in %s" % cpathname)) #print "loading pyc file:", cpathname code_w = read_compiled_module(space, cpathname, source) + exec_code_module(space, w_mod, code_w) - w_dict = space.getattr(w_mod, w('__dict__')) - space.call_method(w_dict, 'setdefault', - w('__builtins__'), - w(space.builtin)) - code_w.exec_code(space, w_dict, w_dict) return w_mod Modified: pypy/branch/import-builtin/pypy/module/zipimport/interp_zipimport.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/zipimport/interp_zipimport.py (original) +++ pypy/branch/import-builtin/pypy/module/zipimport/interp_zipimport.py Tue Dec 15 10:39:31 2009 @@ -149,9 +149,9 @@ real_name = self.name + os.path.sep + self.corr_zname(filename) space.setattr(w_mod, w('__loader__'), space.wrap(self)) importing._prepare_module(space, w_mod, real_name, pkgpath) - result = importing.load_source_module(space, w(modname), w_mod, - filename, buf, write_pyc=False) - return result + code_w = importing.parse_source_module(space, filename, buf) + importing.exec_code_module(space, w_mod, code_w) + return w_mod def _parse_mtime(self, space, filename): w = space.wrap From arigo at codespeak.net Tue Dec 15 10:44:53 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 15 Dec 2009 10:44:53 +0100 (CET) Subject: [pypy-svn] r70122 - pypy/trunk/pypy/translator/benchmark Message-ID: <20091215094453.71703168015@codespeak.net> Author: arigo Date: Tue Dec 15 10:44:52 2009 New Revision: 70122 Modified: pypy/trunk/pypy/translator/benchmark/bench-custom.py Log: If running --no-cpython, don't crash if CPython is not recorded in the pickle file. Instead, don't create the pickle and don't produce the summary at all. Modified: pypy/trunk/pypy/translator/benchmark/bench-custom.py ============================================================================== --- pypy/trunk/pypy/translator/benchmark/bench-custom.py (original) +++ pypy/trunk/pypy/translator/benchmark/bench-custom.py Tue Dec 15 10:44:52 2009 @@ -54,6 +54,13 @@ for b in benchmarks: benchmark_result.result(exe, allowcreate=True).run_benchmark(b, verbose=options.verbose) + if options.relto: + relto = options.relto + else: + relto = full_pythons[0] + if relto not in benchmark_result.benchmarks: + continue + pickle.dump(benchmark_result, open(options.picklefile, 'wb')) exe_stats = ['stat:st_mtime', 'exe_name', 'pypy_rev'] @@ -63,10 +70,6 @@ stats = ['exe'] for b in benchmarks: stats.append('bench:'+b.name) - if options.relto: - relto = options.relto - else: - relto = full_pythons[0] kwds = {'relto': relto, 'filteron' :lambda r: r.exe_name in exes, } From afa at codespeak.net Tue Dec 15 10:47:29 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 15 Dec 2009 10:47:29 +0100 (CET) Subject: [pypy-svn] r70123 - pypy/branch/import-builtin/pypy/module/imp/test Message-ID: <20091215094729.3531A168015@codespeak.net> Author: afa Date: Tue Dec 15 10:47:28 2009 New Revision: 70123 Modified: pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Log: Try to make this test more robust against already imported modules Modified: pypy/branch/import-builtin/pypy/module/imp/test/test_import.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/test/test_import.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Tue Dec 15 10:47:28 2009 @@ -743,7 +743,9 @@ def find_module(self, fullname, path=None): tried_imports.append((fullname, path)) - import sys, math + import sys, datetime + del sys.modules["datetime"] + sys.meta_path.append(Importer()) try: import datetime @@ -762,7 +764,6 @@ def __init__(self, *namestoblock): self.namestoblock = dict.fromkeys(namestoblock) def find_module(self, fullname, path=None): - print "AFA FIND_MODULE", fullname if fullname in self.namestoblock: return self def load_module(self, fullname): From arigo at codespeak.net Tue Dec 15 11:13:20 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 15 Dec 2009 11:13:20 +0100 (CET) Subject: [pypy-svn] r70124 - pypy/trunk/pypy/translator/benchmark Message-ID: <20091215101320.817F5168015@codespeak.net> Author: arigo Date: Tue Dec 15 11:13:20 2009 New Revision: 70124 Added: pypy/trunk/pypy/translator/benchmark/jitbench.py Log: Replace jitbench.py with one that runs all of bench-custom, with default factors of 1, 2 and 5. The result is not uploaded anywhere; it's printed to stdout, where it's possibly captured by buildbot. Added: pypy/trunk/pypy/translator/benchmark/jitbench.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/translator/benchmark/jitbench.py Tue Dec 15 11:13:20 2009 @@ -0,0 +1,21 @@ +import sys, os +from optparse import OptionParser + +parser = OptionParser() +parser.add_option( + '--size-factor-list', dest='sizefactorlist', + default='1,2,5,1,2,5,1,2,5', + ) +options, args = parser.parse_args(sys.argv[1:]) +args = args or [sys.executable] +executables = [os.path.abspath(executable) for executable in args] +sizefactors = [int(s) for s in options.sizefactorlist.split(',')] + +os.chdir(os.path.dirname(sys.argv[0]) or '.') + +for sizefactor in sizefactors: + for executable in executables: + sys.argv[1:] = [executable, '--pickle=jitbench.benchmark_result', + '-v', '--no-cpython', + '--size-factor=%d' % sizefactor] + execfile('bench-custom.py') From arigo at codespeak.net Tue Dec 15 11:15:31 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 15 Dec 2009 11:15:31 +0100 (CET) Subject: [pypy-svn] r70125 - pypy/build/bot2/pypybuildbot Message-ID: <20091215101531.304B9168015@codespeak.net> Author: arigo Date: Tue Dec 15 11:15:30 2009 New Revision: 70125 Modified: pypy/build/bot2/pypybuildbot/builds.py Log: Remove command-line arguments in order to get a result that is as close as what users get by default. Modified: pypy/build/bot2/pypybuildbot/builds.py ============================================================================== --- pypy/build/bot2/pypybuildbot/builds.py (original) +++ pypy/build/bot2/pypybuildbot/builds.py Tue Dec 15 11:15:30 2009 @@ -159,9 +159,7 @@ setup_steps(platform, self) - self.addStep(Translate(['-Ojit', '--gc=hybrid', - '--gcrootfinder=asmgcc'], - ['--withoutmod-thread'])) + self.addStep(Translate(['-Ojit'], [])) self.addStep(ShellCmd( descritpion="run richards & upload results", command=["python", "pypy/translator/benchmark/jitbench.py", From arigo at codespeak.net Tue Dec 15 13:33:26 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 15 Dec 2009 13:33:26 +0100 (CET) Subject: [pypy-svn] r70127 - pypy/trunk/pypy/translator/benchmark Message-ID: <20091215123326.AEE8E168012@codespeak.net> Author: arigo Date: Tue Dec 15 13:33:26 2009 New Revision: 70127 Modified: pypy/trunk/pypy/translator/benchmark/result.py Log: Add the revision explicitly. Modified: pypy/trunk/pypy/translator/benchmark/result.py ============================================================================== --- pypy/trunk/pypy/translator/benchmark/result.py (original) +++ pypy/trunk/pypy/translator/benchmark/result.py Tue Dec 15 13:33:26 2009 @@ -90,6 +90,8 @@ if self.run_counts.get(benchmark.name, 0) > self.max_results: return print 'running', benchmark.name, 'for', self.exe_name, + if verbose and self.pypy_rev > 0: + print '[rev %d]' % self.pypy_rev, sys.stdout.flush() new_result = benchmark.run(self.exe_name) print new_result From afa at codespeak.net Tue Dec 15 13:38:17 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 15 Dec 2009 13:38:17 +0100 (CET) Subject: [pypy-svn] r70128 - pypy/branch/import-builtin/pypy/module/imp/test Message-ID: <20091215123817.0AE1D168012@codespeak.net> Author: afa Date: Tue Dec 15 13:38:17 2009 New Revision: 70128 Modified: pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Log: Use another package name for this test, so that it doest not clash with other tests. Modified: pypy/branch/import-builtin/pypy/module/imp/test/test_import.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/test/test_import.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Tue Dec 15 13:38:17 2009 @@ -368,9 +368,9 @@ def test_relative_import_pkg(self): import sys import imp - pkg = imp.new_module('pkg') - sys.modules['pkg'] = pkg - mydict = {'__name__': 'pkg.foo', '__path__': '/some/path'} + pkg = imp.new_module('newpkg') + sys.modules['newpkg'] = pkg + mydict = {'__name__': 'newpkg.foo', '__path__': '/some/path'} res = __import__('', mydict, None, ['bar'], 2) assert res is pkg From afa at codespeak.net Tue Dec 15 19:18:24 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 15 Dec 2009 19:18:24 +0100 (CET) Subject: [pypy-svn] r70138 - in pypy/branch/import-builtin/pypy/module/imp: . test Message-ID: <20091215181824.2790D168015@codespeak.net> Author: afa Date: Tue Dec 15 19:18:24 2009 New Revision: 70138 Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py pypy/branch/import-builtin/pypy/module/imp/interp_imp.py pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Log: - change imp.find_module, so that it finds lone .pyc files even when the normal import does not. - A failing reload() should keep the old module in sys.modules Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Tue Dec 15 19:18:24 2009 @@ -25,7 +25,7 @@ # PY_CODERESOURCE = 8 IMP_HOOK = 9 -def find_modtype(space, filepart): +def find_modtype(space, filepart, force_lonepycfiles=False): """Check which kind of module to import for the given filepart, which is a path without extension. Returns PY_SOURCE, PY_COMPILED or SEARCH_ERROR. @@ -38,7 +38,9 @@ # The .py file does not exist. By default on PyPy, lonepycfiles # is False: if a .py file does not exist, we don't even try to # look for a lone .pyc file. - if not space.config.objspace.lonepycfiles: + # The "imp" module does not respect this, and is allowed to find + # lone .pyc files. + if not space.config.objspace.lonepycfiles and not force_lonepycfiles: return SEARCH_ERROR, None, None # check the .pyc file @@ -258,7 +260,8 @@ def fromLoader(w_loader): return FindInfo(IMP_HOOK, '', None, w_loader=w_loader) -def find_module(space, modulename, w_modulename, partname, w_path, use_loader=True): +def find_module(space, modulename, w_modulename, partname, w_path, + use_loader=True, force_lonepycfiles=False): # Examin importhooks (PEP302) before doing the import if use_loader: w_loader = find_in_meta_path(space, w_modulename, w_path) @@ -289,14 +292,16 @@ filepart = os.path.join(path, partname) if os.path.isdir(filepart) and case_ok(filepart): initfile = os.path.join(filepart, '__init__') - modtype, _, _ = find_modtype(space, initfile) + modtype, _, _ = find_modtype(space, initfile, + force_lonepycfiles) if modtype in (PY_SOURCE, PY_COMPILED): return FindInfo(PKG_DIRECTORY, filepart, None) else: msg = "Not importing directory " +\ "'%s' missing __init__.py" % (filepart,) space.warn(msg, space.w_ImportWarning) - modtype, suffix, filemode = find_modtype(space, filepart) + modtype, suffix, filemode = find_modtype(space, filepart, + force_lonepycfiles) try: if modtype in (PY_SOURCE, PY_COMPILED): filename = filepart + suffix @@ -442,6 +447,11 @@ try: return load_module(space, w_modulename, find_info, reuse=True) + except: + # load_module probably removed name from modules because of + # the error. Put back the original module object. + space.sys.setmodule(w_module) + raise finally: if find_info.stream: find_info.stream.close() Modified: pypy/branch/import-builtin/pypy/module/imp/interp_imp.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/interp_imp.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/interp_imp.py Tue Dec 15 19:18:24 2009 @@ -36,7 +36,9 @@ w_path = None find_info = importing.find_module( - space, name, w_name, name, w_path, use_loader=False) + space, name, w_name, name, w_path, + use_loader=False, + force_lonepycfiles=True) if not find_info: raise OperationError( space.w_ImportError, Modified: pypy/branch/import-builtin/pypy/module/imp/test/test_import.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/test/test_import.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Tue Dec 15 19:18:24 2009 @@ -412,6 +412,21 @@ import os os.unlink(test_reload.__file__) + def test_reload_failing(self): + import test_reload + import time + time.sleep(1) + f = open(test_reload.__file__, "w") + f.write("a = 10 // 0\n") + f.close() + + # A failing reload should leave the previous module in sys.modules + raises(ZeroDivisionError, reload, test_reload) + import os, sys + assert 'test_reload' in sys.modules + assert test_reload.test + os.unlink(test_reload.__file__) + def test_reload_submodule(self): import pkg.a reload(pkg.a) @@ -857,22 +872,19 @@ sys.meta_path.append(i) sys.path_hooks.append(ImpWrapper) sys.path_importer_cache.clear() - mnames = ("colorsys", "urlparse", "distutils.core", "compiler.misc") - for mname in mnames: - parent = mname.split(".")[0] - for n in sys.modules.keys(): - if n.startswith(parent): - del sys.modules[n] - for mname in mnames: - m = __import__(mname, globals(), locals(), ["__dummy__"]) - m.__loader__ # to make sure we actually handled the import - # Delete urllib from modules because urlparse was imported above. - # Without this hack, test_socket_ssl fails if run in this order: - # regrtest.py test_codecmaps_tw test_importhooks test_socket_ssl try: - del sys.modules['urllib'] - except KeyError: - pass + mnames = ("colorsys", "urlparse", "distutils.core", "compiler.misc") + for mname in mnames: + parent = mname.split(".")[0] + for n in sys.modules.keys(): + if n.startswith(parent): + del sys.modules[n] + for mname in mnames: + m = __import__(mname, globals(), locals(), ["__dummy__"]) + m.__loader__ # to make sure we actually handled the import + finally: + sys.meta_path.pop(0) + sys.path_hooks.pop(0) class AppTestNoPycFile(object): usepycfiles = False @@ -891,6 +903,9 @@ _teardown(cls.space, cls.saved_modules) def test_import_possibly_from_pyc(self): + import sys + assert sys.meta_path == [] + assert sys.path_hooks == [] from compiled import x if self.usepycfiles: assert x.__file__.endswith('x.pyc') From fijal at codespeak.net Wed Dec 16 12:43:57 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 16 Dec 2009 12:43:57 +0100 (CET) Subject: [pypy-svn] r70144 - pypy/trunk/pypy/lang Message-ID: <20091216114357.8C90B16801B@codespeak.net> Author: fijal Date: Wed Dec 16 12:43:56 2009 New Revision: 70144 Removed: pypy/trunk/pypy/lang/ Log: Simply remove the lang directory, it contains only one test From pedronis at codespeak.net Wed Dec 16 14:22:44 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 16 Dec 2009 14:22:44 +0100 (CET) Subject: [pypy-svn] r70147 - in pypy/extradoc/talk/jitdec09: . ui Message-ID: <20091216132244.1E69716801B@codespeak.net> Author: pedronis Date: Wed Dec 16 14:22:43 2009 New Revision: 70147 Added: pypy/extradoc/talk/jitdec09/ pypy/extradoc/talk/jitdec09/result.png (contents, props changed) pypy/extradoc/talk/jitdec09/talk.html pypy/extradoc/talk/jitdec09/talk.txt (contents, props changed) pypy/extradoc/talk/jitdec09/ui/ (props changed) - copied from r70142, pypy/extradoc/talk/roadshow-ibm/ui/ Log: brief jit and jit status talk Added: pypy/extradoc/talk/jitdec09/result.png ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/jitdec09/talk.html ============================================================================== --- (empty file) +++ pypy/extradoc/talk/jitdec09/talk.html Wed Dec 16 14:22:43 2009 @@ -0,0 +1,414 @@ + + + + + + + +A generated JIT compiler for Python + + + + + + + + + + + + + + + + +
+
+
+ + +
+
+
+

A generated JIT compiler for Python

+ +++ + + + + + +
Author:Samuele Pedroni, Open End AB
Date:December '09
+ + + + + + + + + + +
+
+

Idea

+

Derive a dynamic compiler from a sufficiently rich intermediate representation +of the interpreter while it is turned into a VM

+
    +
  • all features supported by construction
    • +
  • semantics are more easily encoded as interpreting code
    • +
+
+
+

Tracing JIT compiler

+

Template: use the streamlined approach introduced by tracing JITs

+
    +
  • identify hot loops
    • +
  • produce linear traces with stay-on-trace checks (guards), follow calls!
    • +
  • compile the traces to assembler which run the loop
    • +
+
+
+

Details

+
    +
  • Identifying loops: hints for the JIT runtime to identify the +interpreter loop, and back edges of user loops
    • +
  • Tracing: build encoding such that interpreter functions can be +executed and their operation recorded in the traces
    • +
  • Exploit when heap allocation can be forgone, mirror Python frames on +the CPU stack
    • +
+
+
+

Example

+
+
+

Status

+
    +
  • Intel x86-32
    • +
  • Small, medium examples run
    • +
  • Algorithmic code is sped up, see benchmarks
    • +
  • Reasonable heuristic when traces get too long
    • +
+
+
+

Benchmarks

+
result.png
+
+
+

Work Left

+
    +
  • Further work on memory footprint
    • +
  • Correct and improved behavior in the presence of exceptions: work in progress
    • +
  • Better results for recursion, asm to asm calls
    • +
  • Bugs, correctness (sys.setrace for example)
    • +
+
+
+

Work Left (ii)

+
    +
  • Machine code management, discard no longer valid traces
    • +
  • Make the interpreter more JIT friendly
    • +
  • Interpreter level algorithmic improvements
    • +
  • Intel x86-64
    • +
+
+
+ + Added: pypy/extradoc/talk/jitdec09/talk.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/jitdec09/talk.txt Wed Dec 16 14:22:43 2009 @@ -0,0 +1,74 @@ +.. include:: + +========================================================= +A generated JIT compiler for Python +========================================================= + +:Author: Samuele Pedroni, Open End AB +:Date: December '09 + + +Idea +========== + +Derive a dynamic compiler from a sufficiently rich intermediate representation +of the interpreter while it is turned into a VM + +- all features supported by construction + +- semantics are more easily encoded as interpreting code + +Tracing JIT compiler +======================= + +Template: use the streamlined approach introduced by tracing JITs + +- identify hot loops +- produce linear traces with stay-on-trace checks (guards), follow calls! +- compile the traces to assembler which run the loop + +Details +========= + +- Identifying loops: hints for the JIT runtime to identify the + interpreter loop, and back edges of user loops + +- Tracing: build encoding such that interpreter functions can be + executed and their operation recorded in the traces + +- Exploit when heap allocation can be forgone, mirror Python frames on + the CPU stack + +Example +========= + + +Status +======= + +- Intel x86-32 +- Small, medium examples run +- Algorithmic code is sped up, see benchmarks +- Reasonable heuristic when traces get too long + +Benchmarks +============ + +.. image:: result.png + :align: center + +Work Left +========== + +- Further work on memory footprint +- Correct and improved behavior in the presence of exceptions: work in progress +- Better results for recursion, asm to asm calls +- Bugs, correctness (sys.setrace for example) + +Work Left (ii) +=============== + +- Machine code management, discard no longer valid traces +- Make the interpreter more JIT friendly +- Interpreter level algorithmic improvements +- Intel x86-64 From afa at codespeak.net Wed Dec 16 16:27:33 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 16 Dec 2009 16:27:33 +0100 (CET) Subject: [pypy-svn] r70148 - in pypy/branch/import-builtin: lib-python/modified-2.5.2/test pypy/module/imp pypy/module/imp/test Message-ID: <20091216152733.A391616801B@codespeak.net> Author: afa Date: Wed Dec 16 16:27:31 2009 New Revision: 70148 Added: pypy/branch/import-builtin/lib-python/modified-2.5.2/test/test_runpy.py - copied, changed from r70029, pypy/branch/import-builtin/lib-python/2.5.2/test/test_runpy.py Modified: pypy/branch/import-builtin/lib-python/modified-2.5.2/test/test_import.py pypy/branch/import-builtin/pypy/module/imp/importing.py pypy/branch/import-builtin/pypy/module/imp/interp_imp.py pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Log: Do it the other way and have imp.find_module consistent with __import__() and reload(); Skip the relevant tests in Python test suite. Modified: pypy/branch/import-builtin/lib-python/modified-2.5.2/test/test_import.py ============================================================================== --- pypy/branch/import-builtin/lib-python/modified-2.5.2/test/test_import.py (original) +++ pypy/branch/import-builtin/lib-python/modified-2.5.2/test/test_import.py Wed Dec 16 16:27:31 2009 @@ -56,6 +56,11 @@ os.unlink(source) try: + #--- the block below is to check that "reload" manages to import + #--- the .pyc file alone. We don't support it in PyPy in the default + #--- configuration. + return + try: reload(mod) except ImportError, err: Copied: pypy/branch/import-builtin/lib-python/modified-2.5.2/test/test_runpy.py (from r70029, pypy/branch/import-builtin/lib-python/2.5.2/test/test_runpy.py) ============================================================================== --- pypy/branch/import-builtin/lib-python/2.5.2/test/test_runpy.py (original) +++ pypy/branch/import-builtin/lib-python/modified-2.5.2/test/test_runpy.py Wed Dec 16 16:27:31 2009 @@ -150,6 +150,12 @@ del d1 # Ensure __loader__ entry doesn't keep file open __import__(mod_name) os.remove(mod_fname) + + #--- the block below is to check that "imp.find_module" + #--- manages to import the .pyc file alone. We don't + #--- support it in PyPy in the default configuration. + return + if verbose: print "Running from compiled:", mod_name d2 = run_module(mod_name) # Read from bytecode self.failUnless(d2["x"] == 1) Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Wed Dec 16 16:27:31 2009 @@ -25,7 +25,7 @@ # PY_CODERESOURCE = 8 IMP_HOOK = 9 -def find_modtype(space, filepart, force_lonepycfiles=False): +def find_modtype(space, filepart): """Check which kind of module to import for the given filepart, which is a path without extension. Returns PY_SOURCE, PY_COMPILED or SEARCH_ERROR. @@ -40,7 +40,7 @@ # look for a lone .pyc file. # The "imp" module does not respect this, and is allowed to find # lone .pyc files. - if not space.config.objspace.lonepycfiles and not force_lonepycfiles: + if not space.config.objspace.lonepycfiles: return SEARCH_ERROR, None, None # check the .pyc file @@ -261,7 +261,7 @@ return FindInfo(IMP_HOOK, '', None, w_loader=w_loader) def find_module(space, modulename, w_modulename, partname, w_path, - use_loader=True, force_lonepycfiles=False): + use_loader=True): # Examin importhooks (PEP302) before doing the import if use_loader: w_loader = find_in_meta_path(space, w_modulename, w_path) @@ -292,16 +292,14 @@ filepart = os.path.join(path, partname) if os.path.isdir(filepart) and case_ok(filepart): initfile = os.path.join(filepart, '__init__') - modtype, _, _ = find_modtype(space, initfile, - force_lonepycfiles) + modtype, _, _ = find_modtype(space, initfile) if modtype in (PY_SOURCE, PY_COMPILED): return FindInfo(PKG_DIRECTORY, filepart, None) else: msg = "Not importing directory " +\ "'%s' missing __init__.py" % (filepart,) space.warn(msg, space.w_ImportWarning) - modtype, suffix, filemode = find_modtype(space, filepart, - force_lonepycfiles) + modtype, suffix, filemode = find_modtype(space, filepart) try: if modtype in (PY_SOURCE, PY_COMPILED): filename = filepart + suffix @@ -449,7 +447,7 @@ return load_module(space, w_modulename, find_info, reuse=True) except: # load_module probably removed name from modules because of - # the error. Put back the original module object. + # the error. Put back the original module object. space.sys.setmodule(w_module) raise finally: Modified: pypy/branch/import-builtin/pypy/module/imp/interp_imp.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/interp_imp.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/interp_imp.py Wed Dec 16 16:27:31 2009 @@ -36,9 +36,7 @@ w_path = None find_info = importing.find_module( - space, name, w_name, name, w_path, - use_loader=False, - force_lonepycfiles=True) + space, name, w_name, name, w_path, use_loader=False) if not find_info: raise OperationError( space.w_ImportError, Modified: pypy/branch/import-builtin/pypy/module/imp/test/test_import.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/test/test_import.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Wed Dec 16 16:27:31 2009 @@ -883,8 +883,8 @@ m = __import__(mname, globals(), locals(), ["__dummy__"]) m.__loader__ # to make sure we actually handled the import finally: - sys.meta_path.pop(0) - sys.path_hooks.pop(0) + sys.meta_path.pop() + sys.path_hooks.pop() class AppTestNoPycFile(object): usepycfiles = False @@ -903,9 +903,6 @@ _teardown(cls.space, cls.saved_modules) def test_import_possibly_from_pyc(self): - import sys - assert sys.meta_path == [] - assert sys.path_hooks == [] from compiled import x if self.usepycfiles: assert x.__file__.endswith('x.pyc') From fijal at codespeak.net Wed Dec 16 17:34:36 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 16 Dec 2009 17:34:36 +0100 (CET) Subject: [pypy-svn] r70149 - pypy/extradoc/talk/jitdec09 Message-ID: <20091216163436.689B116801B@codespeak.net> Author: fijal Date: Wed Dec 16 17:34:34 2009 New Revision: 70149 Modified: pypy/extradoc/talk/jitdec09/talk.txt Log: typo Modified: pypy/extradoc/talk/jitdec09/talk.txt ============================================================================== --- pypy/extradoc/talk/jitdec09/talk.txt (original) +++ pypy/extradoc/talk/jitdec09/talk.txt Wed Dec 16 17:34:34 2009 @@ -63,7 +63,7 @@ - Further work on memory footprint - Correct and improved behavior in the presence of exceptions: work in progress - Better results for recursion, asm to asm calls -- Bugs, correctness (sys.setrace for example) +- Bugs, correctness (sys.settrace for example) Work Left (ii) =============== From afa at codespeak.net Wed Dec 16 17:39:18 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 16 Dec 2009 17:39:18 +0100 (CET) Subject: [pypy-svn] r70150 - pypy/branch/import-builtin/pypy/module/imp Message-ID: <20091216163918.672ED16801B@codespeak.net> Author: afa Date: Wed Dec 16 17:39:17 2009 New Revision: 70150 Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py Log: python2.4 does not parse try...except...finally Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Wed Dec 16 17:39:17 2009 @@ -444,15 +444,16 @@ raise OperationError(space.w_ImportError, space.wrap(msg)) try: - return load_module(space, w_modulename, find_info, reuse=True) + try: + return load_module(space, w_modulename, find_info, reuse=True) + finally: + if find_info.stream: + find_info.stream.close() except: # load_module probably removed name from modules because of # the error. Put back the original module object. space.sys.setmodule(w_module) raise - finally: - if find_info.stream: - find_info.stream.close() # __________________________________________________________________ From fijal at codespeak.net Wed Dec 16 19:04:29 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 16 Dec 2009 19:04:29 +0100 (CET) Subject: [pypy-svn] r70153 - pypy/branch/stringbuilder2 Message-ID: <20091216180429.A4ACE168015@codespeak.net> Author: fijal Date: Wed Dec 16 19:04:28 2009 New Revision: 70153 Added: pypy/branch/stringbuilder2/ - copied from r70152, pypy/trunk/ Log: Create a branch to deal a bit more comprehensibly with stringbuilder From fijal at codespeak.net Wed Dec 16 19:26:04 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 16 Dec 2009 19:26:04 +0100 (CET) Subject: [pypy-svn] r70154 - pypy/branch/stringbuilder2/pypy/rpython/lltypesystem Message-ID: <20091216182604.3D95616801B@codespeak.net> Author: fijal Date: Wed Dec 16 19:26:03 2009 New Revision: 70154 Modified: pypy/branch/stringbuilder2/pypy/rpython/lltypesystem/rbuilder.py Log: Kill the "allocate a lot" part. We never measured whether it makes sense or what's the number. to be decided Modified: pypy/branch/stringbuilder2/pypy/rpython/lltypesystem/rbuilder.py ============================================================================== --- pypy/branch/stringbuilder2/pypy/rpython/lltypesystem/rbuilder.py (original) +++ pypy/branch/stringbuilder2/pypy/rpython/lltypesystem/rbuilder.py Wed Dec 16 19:26:03 2009 @@ -14,14 +14,14 @@ def new_grow_func(name): def stringbuilder_grow(ll_builder, needed): allocated = ll_builder.allocated - if allocated < GROW_FAST_UNTIL: - new_allocated = allocated << 1 - else: - extra_size = allocated >> 2 - try: - new_allocated = ovfcheck(allocated + extra_size) - except OverflowError: - raise MemoryError + #if allocated < GROW_FAST_UNTIL: + # new_allocated = allocated << 1 + #else: + extra_size = allocated >> 2 + try: + new_allocated = ovfcheck(allocated + extra_size) + except OverflowError: + raise MemoryError try: new_allocated = ovfcheck(new_allocated + needed) except OverflowError: From hpk at codespeak.net Wed Dec 16 20:37:32 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 16 Dec 2009 20:37:32 +0100 (CET) Subject: [pypy-svn] r70157 - pypy/trunk/pypy/doc Message-ID: <20091216193732.74661168011@codespeak.net> Author: hpk Date: Wed Dec 16 20:37:31 2009 New Revision: 70157 Modified: pypy/trunk/pypy/doc/confrest.py Log: add google analytics tracking to pypy web page - this way we can probably get a clue what pages people are currently looking at/bailing out. Modified: pypy/trunk/pypy/doc/confrest.py ============================================================================== --- pypy/trunk/pypy/doc/confrest.py (original) +++ pypy/trunk/pypy/doc/confrest.py Wed Dec 16 20:37:31 2009 @@ -5,6 +5,17 @@ html = py.xml.html class PyPyPage(Page): + googlefragment = """ + + +""" def fill_menubar(self): self.menubar = html.div( html.a("home", @@ -31,6 +42,14 @@ def get_doclink(self, target): return relpath(self.targetpath.strpath, self.project.docpath.join(target).strpath) + + def unicode(self, doctype=True): + page = self._root.unicode() + page = page.replace("", self.googlefragment + "") + if doctype: + return self.doctype + page + else: + return page class Project(Project): From pedronis at codespeak.net Wed Dec 16 21:04:37 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 16 Dec 2009 21:04:37 +0100 (CET) Subject: [pypy-svn] r70158 - pypy/extradoc/talk/jitdec09 Message-ID: <20091216200437.BB38A168015@codespeak.net> Author: pedronis Date: Wed Dec 16 21:04:32 2009 New Revision: 70158 Modified: pypy/extradoc/talk/jitdec09/talk.txt Log: rethink this slide tomorrow morning Modified: pypy/extradoc/talk/jitdec09/talk.txt ============================================================================== --- pypy/extradoc/talk/jitdec09/talk.txt (original) +++ pypy/extradoc/talk/jitdec09/talk.txt Wed Dec 16 21:04:32 2009 @@ -46,6 +46,8 @@ Status ======= +xxx weak + - Intel x86-32 - Small, medium examples run - Algorithmic code is sped up, see benchmarks From dan at codespeak.net Wed Dec 16 21:41:01 2009 From: dan at codespeak.net (dan at codespeak.net) Date: Wed, 16 Dec 2009 21:41:01 +0100 (CET) Subject: [pypy-svn] r70159 - in pypy/branch/micronumpy/pypy/module/micronumpy: . test Message-ID: <20091216204101.DC55E168015@codespeak.net> Author: dan Date: Wed Dec 16 21:41:00 2009 New Revision: 70159 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/app_numarray.py pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Log: Fixed tests so that they fail as they should. Fixed applevel array() so that test_int_array test passes as does test_iterable_construction. Modified: pypy/branch/micronumpy/pypy/module/micronumpy/app_numarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/app_numarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/app_numarray.py Wed Dec 16 21:41:00 2009 @@ -1,3 +1,4 @@ +# NOT_RPYTHON types_list = [object, complex, float, int] def lowest_type(x): @@ -24,4 +25,8 @@ } #type = lowest_common_type(xs) #return arrays[type](xs) - return numpy.zeros(len(xs), dtype=int) #FIXME + result = numpy.zeros(len(xs), dtype=int) #FIXME: dtype=dtype ! + for i, x in enumerate(xs): + result[i] = x + return result + Modified: pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Wed Dec 16 21:41:00 2009 @@ -9,21 +9,22 @@ def compare(a, b): for x, y in zip(a, b): if x != y: return False + assert type(x) == type(y) return True return compare""") - - def create_type_test(type): - def test_type_array(self): - compare = self.compare + cls.w_array_type_test = cls.space.appexec([cls.w_compare], + """(compare): + def array_type_test(data_type): from numpy import array - data = [type(x) for x in xrange(4)] + data = [data_type(x) for x in xrange(4)] ar = array(data) assert compare(ar, data) - return test_type_array + return array_type_test + """) - test_int_array = create_type_test(int) - test_float_array = create_type_test(float) + def test_int_array(self): self.array_type_test(int) + def test_float_array(self): self.array_type_test(float) def test_iterable_construction(self): compare = self.compare From afa at codespeak.net Thu Dec 17 00:23:51 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 17 Dec 2009 00:23:51 +0100 (CET) Subject: [pypy-svn] r70161 - pypy/branch/import-builtin/pypy/lib/app_test Message-ID: <20091216232351.B5876168011@codespeak.net> Author: afa Date: Thu Dec 17 00:23:51 2009 New Revision: 70161 Modified: pypy/branch/import-builtin/pypy/lib/app_test/test_runpy.py Log: test_runpy is duplicated here (why?) Modified: pypy/branch/import-builtin/pypy/lib/app_test/test_runpy.py ============================================================================== --- pypy/branch/import-builtin/pypy/lib/app_test/test_runpy.py (original) +++ pypy/branch/import-builtin/pypy/lib/app_test/test_runpy.py Thu Dec 17 00:23:51 2009 @@ -137,6 +137,12 @@ d1 = run_module(mod_name) # Read from source __import__(mod_name) os.remove(mod_fname) + + #--- the block below is to check that "imp.find_module" + #--- manages to import the .pyc file alone. We don't + #--- support it in PyPy in the default configuration. + return + if verbose: print "Running from compiled:", mod_name d2 = run_module(mod_name) # Read from bytecode finally: From pedronis at codespeak.net Thu Dec 17 09:47:31 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 17 Dec 2009 09:47:31 +0100 (CET) Subject: [pypy-svn] r70164 - pypy/extradoc/talk/jitdec09 Message-ID: <20091217084731.BDF5E168012@codespeak.net> Author: pedronis Date: Thu Dec 17 09:47:30 2009 New Revision: 70164 Modified: pypy/extradoc/talk/jitdec09/talk.html pypy/extradoc/talk/jitdec09/talk.txt Log: tweaks Modified: pypy/extradoc/talk/jitdec09/talk.html ============================================================================== --- pypy/extradoc/talk/jitdec09/talk.html (original) +++ pypy/extradoc/talk/jitdec09/talk.html Thu Dec 17 09:47:30 2009 @@ -382,8 +382,8 @@

Status

  • Intel x86-32
    • -
  • Small, medium examples run
    • -
  • Algorithmic code is sped up, see benchmarks
    • +
  • Small and some medium examples run
    • +
  • Good results for algorithmic code, see benchmarks
  • Reasonable heuristic when traces get too long
@@ -397,15 +397,15 @@
  • Further work on memory footprint
  • Correct and improved behavior in the presence of exceptions: work in progress
  • Better results for recursion, asm to asm calls
    • -
  • Bugs, correctness (sys.setrace for example)
    • +
  • Bugs, correctness (sys.settrace for example)

    • Work Left (ii)

      -
    • Machine code management, discard no longer valid traces
      • -
    • Make the interpreter more JIT friendly
      • -
    • Interpreter level algorithmic improvements
      • +
    • Machine code management, discard no longer valid traces
      • +
    • Make the interpreter more JIT-friendly
      • +
    • Interpreter-level algorithmic improvements
    • Intel x86-64
    Modified: pypy/extradoc/talk/jitdec09/talk.txt ============================================================================== --- pypy/extradoc/talk/jitdec09/talk.txt (original) +++ pypy/extradoc/talk/jitdec09/talk.txt Thu Dec 17 09:47:30 2009 @@ -46,11 +46,9 @@ Status ======= -xxx weak - - Intel x86-32 -- Small, medium examples run -- Algorithmic code is sped up, see benchmarks +- Small and some medium examples run +- Good results for algorithmic code, see benchmarks - Reasonable heuristic when traces get too long Benchmarks @@ -70,7 +68,7 @@ Work Left (ii) =============== -- Machine code management, discard no longer valid traces -- Make the interpreter more JIT friendly -- Interpreter level algorithmic improvements +- Machine code management, discard no longer valid traces +- Make the interpreter more JIT-friendly +- Interpreter-level algorithmic improvements - Intel x86-64 From pedronis at codespeak.net Thu Dec 17 09:59:13 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 17 Dec 2009 09:59:13 +0100 (CET) Subject: [pypy-svn] r70165 - pypy/extradoc/planning Message-ID: <20091217085913.E29F9168012@codespeak.net> Author: pedronis Date: Thu Dec 17 09:59:13 2009 New Revision: 70165 Modified: pypy/extradoc/planning/jit.txt Log: another task to consider Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Thu Dec 17 09:59:13 2009 @@ -26,6 +26,7 @@ engine would do it) and not spend all the time in _trace_and_drag_out_of_nursery +- improve tracing/blackholing speed (?) Python interpreter: From pedronis at codespeak.net Thu Dec 17 10:08:45 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 17 Dec 2009 10:08:45 +0100 (CET) Subject: [pypy-svn] r70166 - pypy/extradoc/talk/jitdec09 Message-ID: <20091217090845.15E4416800D@codespeak.net> Author: pedronis Date: Thu Dec 17 10:08:44 2009 New Revision: 70166 Added: pypy/extradoc/talk/jitdec09/graph.dot Modified: pypy/extradoc/talk/jitdec09/talk.html pypy/extradoc/talk/jitdec09/talk.txt Log: links, example dot file Added: pypy/extradoc/talk/jitdec09/graph.dot ============================================================================== --- (empty file) +++ pypy/extradoc/talk/jitdec09/graph.dot Thu Dec 17 10:08:44 2009 @@ -0,0 +1,32 @@ +digraph _resop { +clusterrank="local" +subgraph cluster0 { +_graph0 [shape="octagon", label="Loop #0\n[p0, p1, p2, p3, i4, p5, p6, p7, p8, p9, p10, p11, p12, i13, p14]", color="black", fillcolor="#f084c2", style="filled", width="0.75"]; +_g0op0 [shape="box", label="debug_merge_point(Const(\" #3 LOAD_FAST\"))\ldebug_merge_point(Const(\" #6 LOAD_FAST\"))\lLOAD_FAST__AccessDirect_star_1:4 guard_nonnull_class(p12, Const(253), descr=)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +_g0op3 [shape="box", label="debug_merge_point(Const(\" #9 COMPARE_OP\"))\llt__Int_Int:6 i15 = getfield_gc_pure(p12, descr=)\llt__Int_Int:9 i16 = int_lt(i13, i15)\lObjSpace.newbool:0 guard_true(i16, descr=)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +_g0op39 [shape="box", label="debug_merge_point(Const(\" #0 LOAD_FAST\"))\ldebug_merge_point(Const(\" #3 LOAD_FAST\"))\ldebug_merge_point(Const(\" #6 BINARY_ADD\"))\ladd__Int_Int:16 i17 = int_add_ovf(i13, Const(1))\ladd__Int_Int:16 guard_no_overflow(, descr=)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +_g0op7 [shape="box", label="debug_merge_point(Const(\" #12 JUMP_IF_FALSE\"))\ldebug_merge_point(Const(\" #15 POP_TOP\"))\ldebug_merge_point(Const(\" #16 LOAD_GLOBAL\"))\lLOAD_GLOBAL__AccessDirect_star_1:3 p18 = getfield_gc(p0, descr=)\lLOAD_GLOBAL__AccessDirect_star_1:11 p19 = getarrayitem_gc(p18, Const(0), descr=)\lLOAD_GLOBAL__AccessDirect_star_1:14 p20 = getfield_gc(p19, descr=)\lLOAD_GLOBAL__AccessDirect_star_1:14 guard_nonnull_class(p20, Const(42), descr=)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +_g0op44 [shape="box", label="debug_merge_point(Const(\" #7 RETURN_VALUE\"))\lExecutionContext.leave:3 i21 = getfield_gc(p22, descr=)\lExecutionContext.leave:3 guard_isnull(i21, descr=)\l_extract_back_from_frame:33 i23 = oois(p24, p0)\l_extract_back_from_frame:30 guard_true(i23, descr=)\lExecutionContext._unchain:29 i25 = int_sub(i26, Const(1))\lExecutionContext._unchain:33 setfield_gc(p22, i25, descr=)\lExecutionContext.leave:29 p27 = getfield_gc(p22, descr=)\lExecutionContext.leave:29 guard_isnull(p27, descr=)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +_g0op14 [shape="box", label="debug_merge_point(Const(\" #19 LOAD_FAST\"))\ldebug_merge_point(Const(\" #22 LOAD_CONST\"))\ldebug_merge_point(Const(\" #25 LOAD_CONST\"))\ldebug_merge_point(Const(\" #28 CALL_FUNCTION\"))\lFunction.getcode:24 p28 = getfield_gc(p20, descr=)\lFunction.getcode:26 guard_value(p28, Const(*pypy.interpreter.pycode.PyCode), descr=)\lPyCode.funcrun:3 p29 = getfield_gc(p20, descr=)\lPyCode.funcrun:6 p30 = getfield_gc(p20, descr=)\lPyCode.funcrun:21 p31 = getfield_gc(p20, descr=)\lPyCode.funcrun:24 p32 = getfield_gc(p20, descr=)\lArguments._match_signature:620 i33 = arraylen_gc(p32, descr=)\lArguments._match_signature:623 i34 = int_sub(Const(2), i33)\lThreadLocals.getvalue:3 p22 = getfield_gc(Const(*pypy.interpreter.miscutils.ThreadLocals), descr=)\lObjSpace.getexecutioncontext:4 guard_nonnull(p22, descr=)\lExecutionContext.enter:3 i35 = getfield_gc(p22, descr=)\lExecutionContext.enter:6 i36 = getfield_gc(Const(*pypy.module.sys.Module), descr=)\lExecutionContext.enter:9 i37 = int_gt(i35, i36)\lExecutionContext.enter:9 guard_false(i37, descr=)\lExecutionContext._chain:6 i26 = int_add(i35, Const(1))\lExecutionContext._chain:10 setfield_gc(p22, i26, descr=)\lExecutionContext._chain:13 p24 = getfield_gc(p22, descr=)\lExecutionContext.gettopframe:3 guard_nonnull(p24, descr=)\lExecutionContext.gettopframe:16 i38 = oois(p24, p0)\lExecutionContext.gettopframe:13 guard_true(i38, descr=)\lExecutionContext.gettopframe:16 guard_isnull(p2, descr=)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +_g0op53 [shape="box", label="debug_merge_point(Const(\" #31 STORE_FAST\"))\ldebug_merge_point(Const(\" #34 JUMP_ABSOLUTE\"))\lActionFlag.get:3 i39 = getfield_gc(Const(*pypy.interpreter.executioncontext.ActionFlag), descr=)\lbytecode_trace__AccessDirect_None:7 i40 = int_and(i39, Const(6291456))\lbytecode_trace__AccessDirect_None:9 i41 = int_is_true(i40)\lbytecode_trace__AccessDirect_None:9 guard_false(i41, descr=)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +_g0op59 [shape="box", label="debug_merge_point(Const(\" #3 LOAD_FAST\"))\ljump(p0, p1, Const(* None), p3, Const(3), Const(* None), Const(* None), Const(* None), Const(* None), p9, p10, p11, p12, i17, p14, descr=)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +} +edge [label="", style="dashed", color="black", dir="forward", weight="5"]; +_graph0 -> _g0op0 +edge [label="", style="dashed", color="black", dir="forward", weight="5"]; +_g0op0 -> _g0op3 +edge [label="", style="dashed", color="black", dir="forward", weight="5"]; +_g0op3 -> _g0op7 +edge [label="", style="dashed", color="black", dir="forward", weight="5"]; +_g0op39 -> _g0op44 +edge [label="", style="dashed", color="black", dir="forward", weight="5"]; +_g0op7 -> _g0op14 +edge [label="", style="dashed", color="black", dir="forward", weight="5"]; +_g0op44 -> _g0op53 +edge [label="", style="dashed", color="black", dir="forward", weight="5"]; +_g0op14 -> _g0op39 +edge [label="", style="dashed", color="black", dir="forward", weight="5"]; +_g0op53 -> _g0op59 +edge [label="", style="dashed", color="black", dir="forward", weight="0"]; +_g0op59 -> _g0op0 +} \ No newline at end of file Modified: pypy/extradoc/talk/jitdec09/talk.html ============================================================================== --- pypy/extradoc/talk/jitdec09/talk.html (original) +++ pypy/extradoc/talk/jitdec09/talk.html Thu Dec 17 10:08:44 2009 @@ -409,6 +409,13 @@
  • Intel x86-64
    • + Modified: pypy/extradoc/talk/jitdec09/talk.txt ============================================================================== --- pypy/extradoc/talk/jitdec09/talk.txt (original) +++ pypy/extradoc/talk/jitdec09/talk.txt Thu Dec 17 10:08:44 2009 @@ -72,3 +72,12 @@ - Make the interpreter more JIT-friendly - Interpreter-level algorithmic improvements - Intel x86-64 + +Links +============ + +Status blog +http://morepypy.blogspot.com + +Website +http://codespeak.net/pypy From afa at codespeak.net Thu Dec 17 11:28:04 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 17 Dec 2009 11:28:04 +0100 (CET) Subject: [pypy-svn] r70167 - in pypy/branch/import-builtin/pypy/module: __builtin__ zipimport zipimport/test Message-ID: <20091217102804.33E5D168012@codespeak.net> Author: afa Date: Thu Dec 17 11:28:02 2009 New Revision: 70167 Modified: pypy/branch/import-builtin/pypy/module/__builtin__/__init__.py pypy/branch/import-builtin/pypy/module/zipimport/__init__.py pypy/branch/import-builtin/pypy/module/zipimport/test/test_zipimport.py Log: Have zipimport install itselfs in sys.path_hooks. Ensure that this is done only once Modified: pypy/branch/import-builtin/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/__builtin__/__init__.py (original) +++ pypy/branch/import-builtin/pypy/module/__builtin__/__init__.py Thu Dec 17 11:28:02 2009 @@ -151,16 +151,3 @@ space.exception_is_valid_obj_as_class_w = ab.exception_is_valid_obj_as_class_w.__get__(space) space.exception_getclass = ab.exception_getclass.__get__(space) space.exception_issubclass_w = ab.exception_issubclass_w.__get__(space) - - def startup(self, space): - # install zipimport hook if --withmod-zipimport is used - if space.config.objspace.usemodules.zipimport: - w_import = space.builtin.get('__import__') - w_zipimport = space.call(w_import, space.newlist( - [space.wrap('zipimport')])) - w_sys = space.getbuiltinmodule('sys') - w_path_hooks = space.getattr(w_sys, space.wrap('path_hooks')) - w_append = space.getattr(w_path_hooks, space.wrap('append')) - w_zipimporter = space.getattr(w_zipimport, - space.wrap('zipimporter')) - space.call(w_append, space.newlist([w_zipimporter])) Modified: pypy/branch/import-builtin/pypy/module/zipimport/__init__.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/zipimport/__init__.py (original) +++ pypy/branch/import-builtin/pypy/module/zipimport/__init__.py Thu Dec 17 11:28:02 2009 @@ -14,4 +14,12 @@ appleveldefs = { 'ZipImportError' : 'app_zipimport.ZipImportError', } - + + def setup_after_space_initialization(self): + """NOT_RPYTHON""" + space = self.space + # install zipimport hook + w_path_hooks = space.sys.get('path_hooks') + from pypy.module.zipimport.interp_zipimport import W_ZipImporter + w_zipimporter = space.gettypefor(W_ZipImporter) + space.call_method(w_path_hooks, 'append', w_zipimporter) Modified: pypy/branch/import-builtin/pypy/module/zipimport/test/test_zipimport.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/zipimport/test/test_zipimport.py (original) +++ pypy/branch/import-builtin/pypy/module/zipimport/test/test_zipimport.py Thu Dec 17 11:28:02 2009 @@ -255,6 +255,11 @@ l = [i for i in zipimport._zip_directory_cache] assert len(l) + def test_path_hooks(self): + import sys + import zipimport + assert sys.path_hooks.count(zipimport.zipimporter) == 1 + class AppTestZipimportDeflated(AppTestZipimport): compression = ZIP_DEFLATED From afa at codespeak.net Thu Dec 17 15:47:14 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 17 Dec 2009 15:47:14 +0100 (CET) Subject: [pypy-svn] r70171 - pypy/branch/import-builtin/lib-python/modified-2.5.2/test Message-ID: <20091217144714.0BD4E16800D@codespeak.net> Author: afa Date: Thu Dec 17 15:47:14 2009 New Revision: 70171 Removed: pypy/branch/import-builtin/lib-python/modified-2.5.2/test/test___all__.py pypy/branch/import-builtin/lib-python/modified-2.5.2/test/test_importhooks.py Log: Built-in modules may be now deleted from sys.modules, and will be re-imported if needed. Remove some modified tests. From fijal at codespeak.net Thu Dec 17 16:18:27 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 17 Dec 2009 16:18:27 +0100 (CET) Subject: [pypy-svn] r70172 - pypy/trunk/pypy/tool Message-ID: <20091217151827.0EC9816800D@codespeak.net> Author: fijal Date: Thu Dec 17 16:18:26 2009 New Revision: 70172 Modified: pypy/trunk/pypy/tool/udir.py Log: Accept an env variable how many usessions to keep Modified: pypy/trunk/pypy/tool/udir.py ============================================================================== --- pypy/trunk/pypy/tool/udir.py (original) +++ pypy/trunk/pypy/tool/udir.py Thu Dec 17 16:18:26 2009 @@ -30,6 +30,8 @@ else: return basename.split('/')[-2] +PYPY_KEEP = int(os.environ.get('PYPY_USESSION_KEEP', '3')) + def make_udir(dir=None, basename=None): if dir is not None: dir = local(dir) @@ -45,7 +47,7 @@ basename = basename + '-' return local.make_numbered_dir(rootdir = dir, prefix = 'usession' + basename, - keep = 3) + keep = PYPY_KEEP) udir = make_udir(dir = os.environ.get('PYPY_USESSION_DIR'), basename = os.environ.get('PYPY_USESSION_BASENAME')) From cfbolz at codespeak.net Thu Dec 17 16:46:26 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 17 Dec 2009 16:46:26 +0100 (CET) Subject: [pypy-svn] r70173 - pypy/trunk/pypy/config Message-ID: <20091217154626.810A816800D@codespeak.net> Author: cfbolz Date: Thu Dec 17 16:46:25 2009 New Revision: 70173 Modified: pypy/trunk/pypy/config/pypyoption.py Log: use inlinedict with -Omem Modified: pypy/trunk/pypy/config/pypyoption.py ============================================================================== --- pypy/trunk/pypy/config/pypyoption.py (original) +++ pypy/trunk/pypy/config/pypyoption.py Thu Dec 17 16:46:25 2009 @@ -345,7 +345,7 @@ config.objspace.std.suggest(withprebuiltint=True) config.objspace.std.suggest(withrangelist=True) config.objspace.std.suggest(withprebuiltchar=True) - config.objspace.std.suggest(withsharingdict=True) + config.objspace.std.suggest(withinlineddict=True) config.objspace.std.suggest(withstrslice=True) config.objspace.std.suggest(withstrjoin=True) # xxx other options? ropes maybe? From arigo at codespeak.net Thu Dec 17 17:26:36 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 17 Dec 2009 17:26:36 +0100 (CET) Subject: [pypy-svn] r70174 - pypy/trunk/pypy/translator/benchmark Message-ID: <20091217162636.8538516800B@codespeak.net> Author: arigo Date: Thu Dec 17 17:26:35 2009 New Revision: 70174 Modified: pypy/trunk/pypy/translator/benchmark/jitbench.py Log: Also show a 20x run in the benchmarks. Modified: pypy/trunk/pypy/translator/benchmark/jitbench.py ============================================================================== --- pypy/trunk/pypy/translator/benchmark/jitbench.py (original) +++ pypy/trunk/pypy/translator/benchmark/jitbench.py Thu Dec 17 17:26:35 2009 @@ -4,7 +4,7 @@ parser = OptionParser() parser.add_option( '--size-factor-list', dest='sizefactorlist', - default='1,2,5,1,2,5,1,2,5', + default='1,2,5,20,1,2,5,20,1,2,5,20', ) options, args = parser.parse_args(sys.argv[1:]) args = args or [sys.executable] From arigo at codespeak.net Thu Dec 17 17:35:01 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 17 Dec 2009 17:35:01 +0100 (CET) Subject: [pypy-svn] r70175 - pypy/extradoc/planning Message-ID: <20091217163501.4D71916800B@codespeak.net> Author: arigo Date: Thu Dec 17 17:35:00 2009 New Revision: 70175 Modified: pypy/extradoc/planning/jit.txt Log: Add an entry. Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Thu Dec 17 17:35:00 2009 @@ -28,6 +28,9 @@ - improve tracing/blackholing speed (?) +- some guards will always fail if they ever start failing + (e.g. the class version tag). Do something more clever about it. + Python interpreter: From fijal at codespeak.net Thu Dec 17 17:42:21 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 17 Dec 2009 17:42:21 +0100 (CET) Subject: [pypy-svn] r70176 - in pypy/trunk/pypy/tool: . test Message-ID: <20091217164221.49BB216800D@codespeak.net> Author: fijal Date: Thu Dec 17 17:42:20 2009 New Revision: 70176 Added: pypy/trunk/pypy/tool/package.py (contents, props changed) pypy/trunk/pypy/tool/test/test_package.py (contents, props changed) Log: A simple tool for packaging pypy, with a test Added: pypy/trunk/pypy/tool/package.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/tool/package.py Thu Dec 17 17:42:20 2009 @@ -0,0 +1,42 @@ +#!/usr/bin/env python +""" A sample script that packages PyPy, provided that it's already built +""" + +import autopath +import shutil +import sys +import py +import os +import tarfile +from pypy.tool.udir import udir + +class PyPyCNotFound(Exception): + pass + +def main(basedir, name='pypy-nightly'): + basedir = py.path.local(basedir) + pypy_c = basedir.join('pypy', 'translator', 'goal', 'pypy-c') + if not pypy_c.check(): + raise PyPyCNotFound('Please compile pypy first, using translate.py') + builddir = udir.ensure("build", dir=True) + pypydir = builddir.ensure("pypy", dir=True) + shutil.copytree(str(basedir.join('lib-python')), + str(pypydir.join('lib-python')), + ignore=shutil.ignore_patterns('.svn')) + pypydir.ensure('pypy', dir=True) + shutil.copytree(str(basedir.join('pypy', 'lib')), + str(pypydir.join('pypy', 'lib')), + ignore=shutil.ignore_patterns('.svn')) + pypydir.ensure('bin', dir=True) + shutil.copy(str(pypy_c), str(pypydir.join('bin', 'pypy-c'))) + old_dir = os.getcwd() + try: + os.chdir(str(builddir)) + os.system('tar cvjf ' + str(builddir.join(name + '.tar.bz2')) + + " pypy") + finally: + os.chdir(old_dir) + return builddir # for tests + +if __name__ == '__main__': + main(sys.argv[1]) Added: pypy/trunk/pypy/tool/test/test_package.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/tool/test/test_package.py Thu Dec 17 17:42:20 2009 @@ -0,0 +1,24 @@ + +import py +from pypy.tool.autopath import pypydir +from pypy.tool.package import main +import tarfile + +def test_dir_structure(): + # make sure we have sort of pypy-c + pypy_c = py.path.local(pypydir).join('translator', 'goal', 'pypy-c') + if not pypy_c.check(): + pypy_c.write("xxx") + fake_pypy_c = True + else: + fake_pypy_c = False + try: + builddir = main(py.path.local(pypydir).dirpath(), 'test') + assert builddir.join('pypy', 'lib-python', '2.5.2', 'test').check() + assert builddir.join('pypy', 'bin', 'pypy-c').check() + assert builddir.join('pypy', 'pypy', 'lib', 'syslog.py').check() + th = tarfile.open(str(builddir.join('test.tar.bz2'))) + assert th.getmember('pypy/pypy/lib/syslog.py') + finally: + if fake_pypy_c: + pypy_c.remove() From cfbolz at codespeak.net Thu Dec 17 18:02:11 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 17 Dec 2009 18:02:11 +0100 (CET) Subject: [pypy-svn] r70177 - pypy/extradoc/planning Message-ID: <20091217170211.966FD168012@codespeak.net> Author: cfbolz Date: Thu Dec 17 18:02:11 2009 New Revision: 70177 Modified: pypy/extradoc/planning/jit.txt Log: Result of todays sync meeting: list a few jit-related things that we think should be in the release, or where it would be cool if they were in the release. Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Thu Dec 17 18:02:11 2009 @@ -3,15 +3,8 @@ - sort out a benchmark infrastructure. graphs! -- improve on predictability: don't trace into signals ... but produce just a conditional call - -- directly call assembler for residual portal calls - - jit/asmgcc + threads? [DONE probably. Needs a bit more testing.] -- since threads are enabled with the JIT, getexecutioncontext cannot be folded - by the JIT anymore. we need to do something in that direction - - think about code memory management - forcing virtualizables should only force fields affected, not everything @@ -43,7 +36,6 @@ - look into failing pypy-c-jit apptests, pypy-c-jit translate.py -- we would like probably enabling sys.settrace() to leave the jit instead of just being ignored - list copy as ll operations, to avoid calling write barriers again and again while growing lists (lists of pointers are extra slow on pypy) @@ -51,6 +43,33 @@ - improve ''.join and u''.join by using stringbuilder, enable realloc for hybrid GC (on stringbuilder branch so far). + + +JIT-related Release Tasks +--------------------------- + +(there are other release tasks, specifically about packaging, documentation, +website and stability that need sorting out too. Whoever, they are beyond the +scope of this section) + +required: +- merge the virtual-forcing branch (this might need implementing store sinking + to make the performance not suck) + +wishlist: +- improve on predictability: don't trace into signals ... but produce just a + conditional call (or just abort the trace) +- directly call assembler for residual portal calls +- we would like probably enabling sys.settrace() to leave the jit instead of + just being ignored +- maybe think again about the recursion limit checking, which slows down calls + quite a bit +- since threads are enabled with the JIT, getexecutioncontext cannot be folded + by the JIT anymore. we need to do something in that direction + + + + META ----- From afa at codespeak.net Thu Dec 17 18:02:48 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 17 Dec 2009 18:02:48 +0100 (CET) Subject: [pypy-svn] r70178 - pypy/branch/import-builtin/pypy/interpreter Message-ID: <20091217170248.C6E9A168012@codespeak.net> Author: afa Date: Thu Dec 17 18:02:48 2009 New Revision: 70178 Modified: pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py pypy/branch/import-builtin/pypy/interpreter/mixedmodule.py pypy/branch/import-builtin/pypy/interpreter/module.py Log: Attempt to support the sys.setdefaultencoding hack: keep a copy of the dict module when freezing, and use it when reloading the module. Modified: pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py Thu Dec 17 18:02:48 2009 @@ -281,10 +281,9 @@ raise modname = self.str_w(w_modname) mod = self.interpclass_w(w_mod) - if isinstance(mod, Module) and not mod.startup_called: + if isinstance(mod, Module): self.timer.start("startup " + modname) - mod.startup(self) - mod.startup_called = True + mod.init(self) self.timer.stop("startup " + modname) @@ -372,10 +371,9 @@ # And initialize it from pypy.interpreter.module import Module mod = self.interpclass_w(w_mod) - if isinstance(mod, Module) and not mod.startup_called: + if isinstance(mod, Module): self.timer.start("startup " + name) - mod.startup(self) - mod.startup_called = True + mod.init(self) self.timer.stop("startup " + name) return w_mod Modified: pypy/branch/import-builtin/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/branch/import-builtin/pypy/interpreter/mixedmodule.py (original) +++ pypy/branch/import-builtin/pypy/interpreter/mixedmodule.py Thu Dec 17 18:02:48 2009 @@ -13,7 +13,8 @@ applevel_name = None expose__file__attribute = True - + w_initialdict = None + def __init__(self, space, w_name): """ NOT_RPYTHON """ Module.__init__(self, space, w_name) @@ -21,6 +22,13 @@ self.__class__.buildloaders() self.loaders = self.loaders.copy() # copy from the class to the inst + def init(self, space): + """This is called each time the module is imported or reloaded + """ + if self.w_initialdict is not None: + space.call_method(self.w_dict, 'update', self.w_initialdict) + Module.init(self, space) + def get_applevel_name(cls): """ NOT_RPYTHON """ if cls.applevel_name is not None: @@ -86,7 +94,7 @@ return self.w_dict def _freeze_(self): - self.getdict() + self.w_initialdict = self.space.call_method(self.getdict(), 'copy') self.startup_called = False # hint for the annotator: Modules can hold state, so they are # not constant Modified: pypy/branch/import-builtin/pypy/interpreter/module.py ============================================================================== --- pypy/branch/import-builtin/pypy/interpreter/module.py (original) +++ pypy/branch/import-builtin/pypy/interpreter/module.py Thu Dec 17 18:02:48 2009 @@ -22,6 +22,13 @@ """NOT_RPYTHON: to allow built-in modules to do some more setup after the space is fully initialized.""" + def init(self, space): + """This is called each time the module is imported or reloaded + """ + if not self.startup_called: + self.startup_called = True + self.startup(space) + def startup(self, space): """This is called at runtime on import to allow the module to do initialization when it is imported for the first time. From arigo at codespeak.net Thu Dec 17 18:17:16 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 17 Dec 2009 18:17:16 +0100 (CET) Subject: [pypy-svn] r70179 - pypy/extradoc/planning Message-ID: <20091217171716.78766168012@codespeak.net> Author: arigo Date: Thu Dec 17 18:17:16 2009 New Revision: 70179 Modified: pypy/extradoc/planning/jit.txt Log: "Whoever"? Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Thu Dec 17 18:17:16 2009 @@ -49,7 +49,7 @@ --------------------------- (there are other release tasks, specifically about packaging, documentation, -website and stability that need sorting out too. Whoever, they are beyond the +website and stability that need sorting out too. However, they are beyond the scope of this section) required: From cfbolz at codespeak.net Thu Dec 17 18:31:19 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 17 Dec 2009 18:31:19 +0100 (CET) Subject: [pypy-svn] r70180 - pypy/branch/change-celldict/pypy/module/pypyjit/test Message-ID: <20091217173119.BB214168012@codespeak.net> Author: cfbolz Date: Thu Dec 17 18:31:19 2009 New Revision: 70180 Modified: pypy/branch/change-celldict/pypy/module/pypyjit/test/test_pypy_c.py Log: Update those tests to match what you get for global lookups. Modified: pypy/branch/change-celldict/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/change-celldict/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/change-celldict/pypy/module/pypyjit/test/test_pypy_c.py Thu Dec 17 18:31:19 2009 @@ -210,21 +210,31 @@ def test_simple_call(self): self.run_source(''' + OFFSET = 0 def f(i): - return i + 1 + return i + 1 + OFFSET def main(n): i = 0 - while i < n: + while i < n+OFFSET: i = f(f(i)) return i - ''', 76, + ''', 96, ([20], 20), ([31], 32)) ops = self.get_by_bytecode("LOAD_GLOBAL") - assert len(ops) == 2 - assert ops[0].get_opnames() == ["getfield_gc", "getarrayitem_gc", + assert len(ops) == 5 + assert ops[0].get_opnames() == ["getfield_gc", "guard_value", + "getfield_gc", "guard_isnull", "getfield_gc", "guard_nonnull_class"] - assert not ops[1] # second LOAD_GLOBAL folded away + # the second getfield on the same globals is quicker + assert ops[1].get_opnames() == ["getfield_gc", "guard_nonnull_class"] + assert not ops[2] # second LOAD_GLOBAL of the same name folded away + # LOAD_GLOBAL of the same name but in different function partially + # folded away + # XXX could be improved + assert ops[3].get_opnames() == ["guard_value", + "getfield_gc", "guard_isnull"] + assert not ops[4] ops = self.get_by_bytecode("CALL_FUNCTION") assert len(ops) == 2 for bytecode in ops: @@ -279,7 +289,7 @@ while i < n: i = f(f(i), j=1) return i - ''', 98, + ''', 100, ([20], 20), ([31], 32)) ops = self.get_by_bytecode("CALL_FUNCTION") @@ -303,7 +313,7 @@ a.x = 2 i = i + a.x return i - ''', 63, + ''', 65, ([20], 20), ([31], 32)) From fijal at codespeak.net Thu Dec 17 20:20:36 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 17 Dec 2009 20:20:36 +0100 (CET) Subject: [pypy-svn] r70183 - pypy/branch/micronumpy/pypy/module/micronumpy Message-ID: <20091217192036.8D601168007@codespeak.net> Author: fijal Date: Thu Dec 17 20:20:35 2009 New Revision: 70183 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/numarray.py Log: kill dead code Modified: pypy/branch/micronumpy/pypy/module/micronumpy/numarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/numarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/numarray.py Thu Dec 17 20:20:35 2009 @@ -124,11 +124,6 @@ NumArray = IntArray # FIXME: compatibility for now FloatArray = create_numarray(float, float_unwrapper, 'FloatArray') -arrays = { - int: IntArray, - float: FloatArray - } - #def array(space, w_xs): # w_length = space.len(w_xs) # length = space.int_w(w_length) From fijal at codespeak.net Thu Dec 17 22:55:18 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 17 Dec 2009 22:55:18 +0100 (CET) Subject: [pypy-svn] r70185 - in pypy/branch/stringbuilder2/pypy/rpython/memory: gc gctransform Message-ID: <20091217215518.55AF6168012@codespeak.net> Author: fijal Date: Thu Dec 17 22:55:16 2009 New Revision: 70185 Modified: pypy/branch/stringbuilder2/pypy/rpython/memory/gc/base.py pypy/branch/stringbuilder2/pypy/rpython/memory/gc/hybrid.py pypy/branch/stringbuilder2/pypy/rpython/memory/gctransform/framework.py Log: Kill realloc attribute on a gc and assume framework gcs can always realloc. Ideally, we should kill the realloc implementation in transform.py, later. This breaks more or less everything Modified: pypy/branch/stringbuilder2/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/stringbuilder2/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/stringbuilder2/pypy/rpython/memory/gc/base.py Thu Dec 17 22:55:16 2009 @@ -17,7 +17,6 @@ needs_write_barrier = False malloc_zero_filled = False prebuilt_gc_objects_are_static_roots = True - can_realloc = False object_minimal_size = 0 def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE): Modified: pypy/branch/stringbuilder2/pypy/rpython/memory/gc/hybrid.py ============================================================================== --- pypy/branch/stringbuilder2/pypy/rpython/memory/gc/hybrid.py (original) +++ pypy/branch/stringbuilder2/pypy/rpython/memory/gc/hybrid.py Thu Dec 17 22:55:16 2009 @@ -82,7 +82,6 @@ """ first_unused_gcflag = _gcflag_next_bit prebuilt_gc_objects_are_static_roots = True - can_realloc = False # the following values override the default arguments of __init__ when # translating to a real backend. Modified: pypy/branch/stringbuilder2/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/stringbuilder2/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/stringbuilder2/pypy/rpython/memory/gctransform/framework.py Thu Dec 17 22:55:16 2009 @@ -711,7 +711,7 @@ hop.genop('adr_add', [v_gc_adr, c_ofs], resultvar=op.result) def _can_realloc(self): - return self.gcdata.gc.can_realloc + return True def perform_realloc(self, hop, v_ptr, v_newsize, c_const_size, c_itemsize, c_lengthofs, c_grow): From afa at codespeak.net Fri Dec 18 00:12:51 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 18 Dec 2009 00:12:51 +0100 (CET) Subject: [pypy-svn] r70186 - in pypy/branch/import-builtin/pypy: interpreter module/imp module/imp/test Message-ID: <20091217231251.EF03F16800B@codespeak.net> Author: afa Date: Fri Dec 18 00:12:50 2009 New Revision: 70186 Modified: pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py pypy/branch/import-builtin/pypy/interpreter/mixedmodule.py pypy/branch/import-builtin/pypy/module/imp/importing.py pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Log: Test and fix for the previous checkin Modified: pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py Fri Dec 18 00:12:50 2009 @@ -349,14 +349,17 @@ self.builtin_modules[name] = w_mod return name - def getbuiltinmodule(self, name): + def getbuiltinmodule(self, name, force_init=False): w_name = self.wrap(name) w_modules = self.sys.get('modules') try: - return self.getitem(w_modules, w_name) + w_mod = self.getitem(w_modules, w_name) except OperationError, e: if not e.match(self, self.w_KeyError): raise + else: + if not force_init: + return w_mod # If the module is a builtin but not yet imported, # retrieve it and initialize it Modified: pypy/branch/import-builtin/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/branch/import-builtin/pypy/interpreter/mixedmodule.py (original) +++ pypy/branch/import-builtin/pypy/interpreter/mixedmodule.py Fri Dec 18 00:12:50 2009 @@ -90,11 +90,12 @@ for name in self.loaders: w_value = self.get(name) space.setitem(self.w_dict, space.new_interned_str(name), w_value) - self.lazy = False + self.lazy = False + self.w_initialdict = space.call_method(self.w_dict, 'items') return self.w_dict def _freeze_(self): - self.w_initialdict = self.space.call_method(self.getdict(), 'copy') + self.getdict() self.startup_called = False # hint for the annotator: Modules can hold state, so they are # not constant Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Fri Dec 18 00:12:50 2009 @@ -330,7 +330,7 @@ return space.call_method(find_info.w_loader, "load_module", w_modulename) if find_info.modtype == C_BUILTIN: - return space.getbuiltinmodule(find_info.filename) + return space.getbuiltinmodule(find_info.filename, force_init=True) if find_info.modtype in (PY_SOURCE, PY_COMPILED, PKG_DIRECTORY): if reuse: Modified: pypy/branch/import-builtin/pypy/module/imp/test/test_import.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/test/test_import.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Fri Dec 18 00:12:50 2009 @@ -431,6 +431,15 @@ import pkg.a reload(pkg.a) + def test_reload_builtin(self): + import sys + try: + del sys.setdefaultencoding + except AttributeError: + pass + reload(sys) + assert 'setdefaultencoding' in dir(sys) + def _getlong(data): x = marshal.dumps(data) return x[-4:] From afa at codespeak.net Fri Dec 18 00:35:17 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 18 Dec 2009 00:35:17 +0100 (CET) Subject: [pypy-svn] r70187 - pypy/branch/import-builtin/pypy/interpreter Message-ID: <20091217233517.CE0C316800B@codespeak.net> Author: afa Date: Fri Dec 18 00:35:16 2009 New Revision: 70187 Modified: pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py Log: Translation fix Modified: pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py Fri Dec 18 00:35:16 2009 @@ -366,7 +366,10 @@ try: w_mod = self.builtin_modules[name] except KeyError: - raise e + raise OperationError( + space.w_SystemError, + space.wrap("getbuiltinmodule() called " + "with non-builtin module %s" % name)) else: # Add the module to sys.modules self.setitem(w_modules, w_name, w_mod) From afa at codespeak.net Fri Dec 18 00:43:41 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 18 Dec 2009 00:43:41 +0100 (CET) Subject: [pypy-svn] r70188 - pypy/branch/import-builtin/pypy/interpreter Message-ID: <20091217234341.E132416800B@codespeak.net> Author: afa Date: Fri Dec 18 00:43:40 2009 New Revision: 70188 Modified: pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py Log: Grrr Modified: pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py Fri Dec 18 00:43:40 2009 @@ -367,9 +367,9 @@ w_mod = self.builtin_modules[name] except KeyError: raise OperationError( - space.w_SystemError, - space.wrap("getbuiltinmodule() called " - "with non-builtin module %s" % name)) + self.w_SystemError, + self.wrap("getbuiltinmodule() called " + "with non-builtin module %s" % name)) else: # Add the module to sys.modules self.setitem(w_modules, w_name, w_mod) From afa at codespeak.net Fri Dec 18 14:42:00 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 18 Dec 2009 14:42:00 +0100 (CET) Subject: [pypy-svn] r70193 - pypy/branch/import-builtin/lib-python/modified-2.5.2/test Message-ID: <20091218134200.9410A168015@codespeak.net> Author: afa Date: Fri Dec 18 14:42:00 2009 New Revision: 70193 Modified: pypy/branch/import-builtin/lib-python/modified-2.5.2/test/test_import.py Log: test_infinite_reload() works now Modified: pypy/branch/import-builtin/lib-python/modified-2.5.2/test/test_import.py ============================================================================== --- pypy/branch/import-builtin/lib-python/modified-2.5.2/test/test_import.py (original) +++ pypy/branch/import-builtin/lib-python/modified-2.5.2/test/test_import.py Fri Dec 18 14:42:00 2009 @@ -1,4 +1,4 @@ -from test.test_support import TESTFN, TestFailed, check_impl_detail +from test.test_support import TESTFN, TestFailed import os import random @@ -243,5 +243,4 @@ finally: sys.path.pop(0) -if check_impl_detail(): - test_infinite_reload() +test_infinite_reload() From arigo at codespeak.net Fri Dec 18 15:10:54 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 18 Dec 2009 15:10:54 +0100 (CET) Subject: [pypy-svn] r70194 - in pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86: . test Message-ID: <20091218141054.045F9168011@codespeak.net> Author: arigo Date: Fri Dec 18 15:10:53 2009 New Revision: 70194 Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/assembler.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/codebuf.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/jump.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/regalloc.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py Log: Check-in old local changes (in-progress): adding MOV_rj and MOV_jr, and starting to tweak assembler.py (but that part is probably a bit useless now). Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/assembler.py Fri Dec 18 15:10:53 2009 @@ -7,11 +7,12 @@ from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.lltypesystem.lloperation import llop from pypy.tool.uid import fixid -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, lower_byte,\ - X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs +from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, \ + X86RegisterManager, X86XMMRegisterManager, is_stack, is_reg from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.jit.backend.x86 import codebuf -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86 import rx86 +from pypy.jit.backend.x86.rx86 import eax, ecx, edx, ebx, esp, ebp, esi, edi from pypy.jit.metainterp.resoperation import rop @@ -59,9 +60,8 @@ method.func_name = name return method -for name in dir(codebuf.MachineCodeBlock): - if name.upper() == name: - setattr(MachineCodeBlockWrapper, name, _new_method(name)) +for name in rx86.all_instructions: + setattr(MachineCodeBlockWrapper, name, _new_method(name)) class Assembler386(object): mc = None @@ -179,7 +179,7 @@ def _patchable_stackadjust(self): # stack adjustment LEA - self.mc.LEA(esp, fixedsize_ebp_ofs(0)) + self.mc.LEA32_rs(esp, 0) return self.mc.tell() - 4 def _patch_stackadjust(self, adr_lea, stack_depth): @@ -191,45 +191,49 @@ def _assemble_bootstrap_code(self, inputargs, arglocs): nonfloatlocs, floatlocs = arglocs - self.mc.PUSH(ebp) - self.mc.MOV(ebp, esp) - self.mc.PUSH(ebx) - self.mc.PUSH(esi) - self.mc.PUSH(edi) + self.mc.PUSH_r(ebp) + self.mc.MOV_rr(ebp, esp) + self.mc.PUSH_r(ebx) + self.mc.PUSH_r(esi) + self.mc.PUSH_r(edi) # NB. exactly 4 pushes above; if this changes, fix stack_pos(). # You must also keep _get_callshape() in sync. adr_stackadjust = self._patchable_stackadjust() tmp = X86RegisterManager.all_regs[0] xmmtmp = X86XMMRegisterManager.all_regs[0] + # + # 1. read from fail_boxes_ptr/int for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] - if loc is None: + if loc == -1: continue - if isinstance(loc, REG): - target = loc - else: - target = tmp if inputargs[i].type == REF: - # This uses XCHG to put zeroes in fail_boxes_ptr after - # reading them - self.mc.XOR(target, target) - self.mc.XCHG(target, addr_add(imm(self.fail_box_ptr_addr), - imm(i*WORD))) + base = self.fail_box_ptr_addr else: - self.mc.MOV(target, addr_add(imm(self.fail_box_int_addr), - imm(i*WORD))) - self.mc.MOV(loc, target) + base = self.fail_box_int_addr + addr = base + i*WORD + if is_stack(loc): + self.mc.MOV_rj(tmp, addr) + self.mc.MOV_sr(loc, tmp) + else: + self.mc.MOV_rj(loc, addr) + # + # 2. put zeroes in fail_boxes_ptr to clear them for the GC + self.mc.XOR_rr(tmp, tmp) + for i in range(len(nonfloatlocs)): + loc = nonfloatlocs[i] + if loc == -1: + continue + if inputargs[i].type == REF: + self.mc.MOV_jr(self.fail_box_ptr_addr + i*WORD, tmp) + # + # 3. read from fail_boxes_float for i in range(len(floatlocs)): loc = floatlocs[i] - if loc is None: + if loc == -1: continue - if isinstance(loc, REG): - self.mc.MOVSD(loc, addr64_add(imm(self.fail_box_float_addr), - imm(i*WORD*2))) - else: - self.mc.MOVSD(xmmtmp, addr64_add(imm(self.fail_box_float_addr), - imm(i*WORD*2))) - self.mc.MOVSD(loc, xmmtmp) + xxx + # return adr_stackadjust def dump(self, text): @@ -920,30 +924,30 @@ num = getattr(rop, opname.upper()) genop_list[num] = value -def new_addr_add(heap, mem, memsib): - def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): - if isinstance(reg_or_imm1, IMM32): - if isinstance(reg_or_imm2, IMM32): - return heap(reg_or_imm1.value + offset + - (reg_or_imm2.value << scale)) - else: - return memsib(None, reg_or_imm2, scale, reg_or_imm1.value + offset) - else: - if isinstance(reg_or_imm2, IMM32): - return mem(reg_or_imm1, offset + (reg_or_imm2.value << scale)) - else: - return memsib(reg_or_imm1, reg_or_imm2, scale, offset) - return addr_add - -addr8_add = new_addr_add(heap8, mem8, memSIB8) -addr_add = new_addr_add(heap, mem, memSIB) -addr64_add = new_addr_add(heap64, mem64, memSIB64) - -def addr_add_const(reg_or_imm1, offset): - if isinstance(reg_or_imm1, IMM32): - return heap(reg_or_imm1.value + offset) - else: - return mem(reg_or_imm1, offset) +##def new_addr_add(heap, mem, memsib): +## def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): +## if isinstance(reg_or_imm1, IMM32): +## if isinstance(reg_or_imm2, IMM32): +## return heap(reg_or_imm1.value + offset + +## (reg_or_imm2.value << scale)) +## else: +## return memsib(None, reg_or_imm2, scale, reg_or_imm1.value + offset) +## else: +## if isinstance(reg_or_imm2, IMM32): +## return mem(reg_or_imm1, offset + (reg_or_imm2.value << scale)) +## else: +## return memsib(reg_or_imm1, reg_or_imm2, scale, offset) +## return addr_add + +##addr8_add = new_addr_add(heap8, mem8, memSIB8) +##addr_add = new_addr_add(heap, mem, memSIB) +##addr64_add = new_addr_add(heap64, mem64, memSIB64) + +##def addr_add_const(reg_or_imm1, offset): +## if isinstance(reg_or_imm1, IMM32): +## return heap(reg_or_imm1.value + offset) +## else: +## return mem(reg_or_imm1, offset) def round_up_to_4(size): if size < 4: Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/codebuf.py Fri Dec 18 15:10:53 2009 @@ -1,11 +1,11 @@ import os from pypy.rpython.lltypesystem import lltype, rffi -from pypy.jit.backend.x86.ri386 import I386CodeBuilder +from pypy.jit.backend.x86.rx86 import X86_32_CodeBuilder from pypy.rlib.rmmap import PTR, alloc, free -class InMemoryCodeBuilder(I386CodeBuilder): +class InMemoryCodeBuilder(X86_32_CodeBuilder): _last_dump_start = 0 def __init__(self, start, end): @@ -18,21 +18,10 @@ self._size = map_size self._pos = 0 - def overwrite(self, pos, data): - assert pos + len(data) <= self._size - for c in data: - self._data[pos] = c - pos += 1 - return pos - - def write(self, data): - self._pos = self.overwrite(self._pos, data) - - def writechr(self, n): - # purely for performance: don't convert chr(n) to a str + def writechar(self, char): pos = self._pos assert pos + 1 <= self._size - self._data[pos] = chr(n) + self._data[pos] = char self._pos = pos + 1 def get_relative_pos(self): Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/jump.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/jump.py Fri Dec 18 15:10:53 2009 @@ -1,22 +1,6 @@ import sys from pypy.tool.pairtype import extendabletype -from pypy.jit.backend.x86.ri386 import * - -class __extend__(OPERAND): - __metaclass__ = extendabletype - def _getregkey(self): - raise AssertionError("should only happen to registers and stack " - "positions") - -class __extend__(REG): - __metaclass__ = extendabletype - def _getregkey(self): - return ~self.op - -class __extend__(MODRM): - __metaclass__ = extendabletype - def _getregkey(self): - return self.position +#from pypy.jit.backend.x86.rx86 import ... def remap_stack_layout(assembler, src_locations, dst_locations, tmpreg): Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/regalloc.py Fri Dec 18 15:10:53 2009 @@ -5,7 +5,9 @@ from pypy.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr, ResOperation, ConstAddr, BoxPtr, LoopToken, INT, REF, FLOAT) -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.rx86 import eax, ecx, edx, ebx, esi, edi +from pypy.jit.backend.x86.rx86 import xmm0, xmm1, xmm2, xmm3 +from pypy.jit.backend.x86.rx86 import xmm4, xmm5, xmm6, xmm7 from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable @@ -36,20 +38,6 @@ def call_result_location(self, v): return eax - def convert_to_imm(self, c): - if isinstance(c, ConstInt): - return imm(c.value) - elif isinstance(c, ConstPtr): - if we_are_translated() and c.value and rgc.can_move(c.value): - print "convert_to_imm: ConstPtr needs special care" - raise AssertionError - return imm(rffi.cast(lltype.Signed, c.value)) - elif isinstance(c, ConstAddr): - return imm(ll2ctypes.cast_adr_to_int(c.value)) - else: - print "convert_to_imm: got a %s" % c - raise AssertionError - BASE_CONSTANT_SIZE = 1000 # cheat cheat cheat.... @@ -82,15 +70,15 @@ self.constant_arrays[-1][1] = NAN self.constant_array_counter = 2 - def convert_to_imm(self, c): - if self.constant_array_counter >= BASE_CONSTANT_SIZE: - self.constant_arrays.append(self.new_const_array()) - self.constant_array_counter = 0 - res = self.constant_array_counter - self.constant_array_counter += 1 - arr = self.constant_arrays[-1] - arr[res] = c.getfloat() - return self.get_addr_of_const_float(-1, res) +## def convert_to_imm(self, c): +## if self.constant_array_counter >= BASE_CONSTANT_SIZE: +## self.constant_arrays.append(self.new_const_array()) +## self.constant_array_counter = 0 +## res = self.constant_array_counter +## self.constant_array_counter += 1 +## arr = self.constant_arrays[-1] +## arr[res] = c.getfloat() +## return self.get_addr_of_const_float(-1, res) def get_addr_of_const_float(self, num_arr, num_pos): arr = self.constant_arrays[num_arr] @@ -157,8 +145,8 @@ def _process_inputargs(self, inputargs): # XXX we can sort out here by longevity if we need something # more optimal - floatlocs = [None] * len(inputargs) - nonfloatlocs = [None] * len(inputargs) + floatlocs = [-1] * len(inputargs) + nonfloatlocs = [-1] * len(inputargs) # Don't use all_regs[0] for passing arguments around a loop. # Must be kept in sync with consider_jump(). # XXX this should probably go to llsupport/regalloc.py @@ -913,3 +901,9 @@ return dl else: raise NotImplementedError() + +def is_stack(loc): + return loc < 0 + +def is_reg(loc): + return loc >= 0 Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py Fri Dec 18 15:10:53 2009 @@ -1,3 +1,4 @@ +import py from pypy.rlib.rarithmetic import intmask, r_ulonglong from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated from pypy.rlib.objectmodel import specialize @@ -14,15 +15,12 @@ esi = 6 edi = 7 +# xmm registers +xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 = range(8) + # the following are extra registers available only on 64 bits -r8 = 8 -r9 = 9 -r10 = 10 -r11 = 11 -r12 = 12 -r13 = 13 -r14 = 14 -r15 = 15 +r8, r9, r10, r11, r12, r13, r14, r15 = range(8, 16) +xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15 = range(8, 16) def single_byte(value): return -128 <= value < 128 @@ -64,6 +62,7 @@ return 0 def register(argnum, factor=1): + assert factor in (1, 8) return encode_register, argnum, factor, rex_register # ____________________________________________________________ @@ -125,6 +124,8 @@ def encode_mem_reg_plus_const(mc, reg1_offset, _, orbyte): reg1 = reg_number_3bits(mc, intmask(reg1_offset >> 32)) offset = intmask(reg1_offset) + if mc.WORD == 8: + offset = offset & 0xFFFFFFFF no_offset = offset == 0 SIB = -1 # 64-bits special cases for reg1 == r12 or r13 @@ -189,12 +190,12 @@ SIB = chr(encoding & 0xFF) offset = intmask(reg1_reg2_scaleshift_offset) no_offset = offset == 0 - # 64-bits special cases for reg1 == r13 + # 64-bits special case for reg1 == r13 # (which look like ebp after being truncated to 3 bits) if mc.WORD == 8: if (encoding & 7) == ebp: no_offset = False - # end of 64-bits special cases + # end of 64-bits special case if no_offset: mc.writechar(chr(0x04 | orbyte)) mc.writechar(SIB) @@ -315,6 +316,7 @@ # "MOV reg1, [reg2+offset]" and the opposite direction MOV_rm = insn(rex_w, '\x8B', register(1,8), mem_reg_plus_const(2)) MOV_mr = insn(rex_w, '\x89', register(2,8), mem_reg_plus_const(1)) + #MOV_mi = insn(rex_w, '\xC7', mem_reg_plus_const(1), immediate(2)) # "MOV reg1, [reg2+reg3*scale+offset]" and the opposite direction MOV_ra = insn(rex_w, '\x8B', register(1,8), @@ -322,6 +324,10 @@ MOV_ar = insn(rex_w, '\x89', register(2,8), mem_reg_plus_scaled_reg_plus_const(1)) + # "MOV reg1, [immediate2]" and the opposite direction + MOV_rj = insn(rex_w, '\x8B', register(1,8), '\x05', immediate(2)) + MOV_jr = insn(rex_w, '\x89', register(2,8), '\x05', immediate(1)) + ADD_ri, ADD_rr, ADD_rs = common_modes(0) OR_ri, OR_rr, OR_rs = common_modes(1) AND_ri, AND_rr, AND_rs = common_modes(4) @@ -359,6 +365,12 @@ else: AbstractX86CodeBuilder.MOV_ri(self, reg, immed) + # unsupported yet + def MOV_rj(self, reg, mem_immed): + py.test.skip("MOV_rj unsupported yet") + def MOV_jr(self, mem_immed, reg): + py.test.skip("MOV_jr unsupported yet") + # ____________________________________________________________ all_instructions = [name for name in AbstractX86CodeBuilder.__dict__ Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86.py Fri Dec 18 15:10:53 2009 @@ -93,3 +93,10 @@ s.MOV_ri(r12, 0x80000042) assert s.getvalue() == ('\x48\xC7\xC1\xFE\xFF\xFF\xFF' + '\x49\xBC\x42\x00\x00\x80\x00\x00\x00\x00') + +def test_mov_rm_64(): + s = CodeBuilder64() + s.MOV_rm(edx, reg_offset(edi, 0)) + s.MOV_rm(edx, reg_offset(r12, 0)) + s.MOV_rm(edx, reg_offset(r13, 0)) + assert s.getvalue() == '\x48\x8B\x17\x49\x8b\x14\x24\x49\x8b\x55\x00' Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py Fri Dec 18 15:10:53 2009 @@ -22,7 +22,7 @@ if char != self.expected[self.index:self.index+1]: print self.op print "\x09from rx86.py:", hexdump(self.expected[self.instrindex:self.index] + char)+"..." - print "\x09from 'as': ", hexdump(self.expected[self.instrindex:self.index+1])+"..." + print "\x09from 'as': ", hexdump(self.expected[self.instrindex:self.index+15])+"..." raise Exception("Differs") self.index += 1 @@ -87,6 +87,7 @@ 'm': self.memory_tests, 'a': self.array_tests, 'i': self.imm32_tests, + 'j': self.imm32_tests, } def assembler_operand_reg(self, regnum): @@ -120,6 +121,9 @@ def assembler_operand_imm(self, value): return '$%d' % value + def assembler_operand_imm_addr(self, value): + return '%d' % value + def get_all_assembler_operands(self): return { 'r': self.assembler_operand_reg, @@ -127,6 +131,7 @@ 'm': self.assembler_operand_memory, 'a': self.assembler_operand_array, 'i': self.assembler_operand_imm, + 'j': self.assembler_operand_imm_addr, } def run_test(self, methname, instrname, argmodes, args_lists): @@ -204,37 +209,10 @@ 'SUB_ri', 'XOR_ri'): if args[0] == rx86.eax: return [] # ADD EAX, constant: there is a special encoding - ## if methname == "MOV_": - #### if args[0] == args[1]: - #### return [] # MOV reg, same reg - ## if ((args[0][1] in (i386.eax, i386.al)) - ## and args[1][1].assembler().lstrip('-').isdigit()): - ## return [] # MOV accum, [constant-address] - ## if ((args[1][1] in (i386.eax, i386.al)) - ## and args[0][1].assembler().lstrip('-').isdigit()): - ## return [] # MOV [constant-address], accum - ## if instrname == "MOV16": - ## return [] # skipped - ## if instrname == "LEA": - ## if (args[1][1].__class__ != i386.MODRM or - ## args[1][1].is_register()): - ## return [] - ## if instrname == "INT": - ## if args[0][1].value == 3: - ## return [] - ## if instrname in ('SHL', 'SHR', 'SAR'): - ## if args[1][1].assembler() == '$1': - ## return [] - ## if instrname in ('MOVZX', 'MOVSX'): - ## if args[1][1].width == 4: - ## return [] - ## if instrname == "TEST": - ## if (args[0] != args[1] and - ## isinstance(args[0][1], i386.REG) and - ## isinstance(args[1][1], i386.REG)): - ## return [] # TEST reg1, reg2 <=> TEST reg2, reg1 - ## if instrname.endswith('cond'): - ## return [] + if methname == 'MOV_rj' and args[0] == rx86.eax: + return [] # MOV EAX, [immediate]: there is a special encoding + if methname == 'MOV_jr' and args[1] == rx86.eax: + return [] # MOV [immediate], EAX: there is a special encoding return [args] def get_code_checker_class(self): Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py Fri Dec 18 15:10:53 2009 @@ -14,6 +14,12 @@ rx86.r8, rx86.r9, rx86.r10, rx86.r11, rx86.r12, rx86.r13, rx86.r14, rx86.r15] + def array_tests(self): + # reduce a little bit -- we spend too long in these tests + lst = super(TestRx86_64, self).array_tests() + random.shuffle(lst) + return lst[:int(len(lst) * 0.2)] + def imm64_tests(self): v = [-0x80000001, 0x80000000, -0x8000000000000000, 0x7FFFFFFFFFFFFFFF] From arigo at codespeak.net Fri Dec 18 15:15:14 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 18 Dec 2009 15:15:14 +0100 (CET) Subject: [pypy-svn] r70196 - pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86 Message-ID: <20091218141514.7B641168015@codespeak.net> Author: arigo Date: Fri Dec 18 15:15:14 2009 New Revision: 70196 Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py Log: Fix for 64-bit platforms. Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py Fri Dec 18 15:15:14 2009 @@ -28,6 +28,9 @@ def fits_in_32bits(value): return -2147483648 <= value <= 2147483647 +def intmask32(value): + return intmask(rffi.cast(rffi.INT, value)) + # ____________________________________________________________ # Emit a single char @@ -123,9 +126,7 @@ def encode_mem_reg_plus_const(mc, reg1_offset, _, orbyte): reg1 = reg_number_3bits(mc, intmask(reg1_offset >> 32)) - offset = intmask(reg1_offset) - if mc.WORD == 8: - offset = offset & 0xFFFFFFFF + offset = intmask32(reg1_offset) no_offset = offset == 0 SIB = -1 # 64-bits special cases for reg1 == r12 or r13 @@ -188,7 +189,7 @@ SIB = chr(encoding) else: SIB = chr(encoding & 0xFF) - offset = intmask(reg1_reg2_scaleshift_offset) + offset = intmask32(reg1_reg2_scaleshift_offset) no_offset = offset == 0 # 64-bits special case for reg1 == r13 # (which look like ebp after being truncated to 3 bits) @@ -352,7 +353,7 @@ WORD = 8 def writeimm64(self, imm): - self.writeimm32(intmask(rffi.cast(rffi.INT, imm))) + self.writeimm32(intmask32(imm)) self.writeimm32(imm >> 32) # MOV_ri from the parent class is not wrong, but here is a better encoding From arigo at codespeak.net Fri Dec 18 15:44:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 18 Dec 2009 15:44:40 +0100 (CET) Subject: [pypy-svn] r70197 - in pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86: . test Message-ID: <20091218144440.B119116801F@codespeak.net> Author: arigo Date: Fri Dec 18 15:44:40 2009 New Revision: 70197 Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py Log: This is probably definitely unsupported, not just "unsupported yet". Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py Fri Dec 18 15:44:40 2009 @@ -366,11 +366,11 @@ else: AbstractX86CodeBuilder.MOV_ri(self, reg, immed) - # unsupported yet + # unsupported -- must use e.g. MOV tmpreg, immed64; MOV reg, [tmpreg] def MOV_rj(self, reg, mem_immed): - py.test.skip("MOV_rj unsupported yet") + py.test.skip("MOV_rj unsupported") def MOV_jr(self, mem_immed, reg): - py.test.skip("MOV_jr unsupported yet") + py.test.skip("MOV_jr unsupported") # ____________________________________________________________ Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py Fri Dec 18 15:44:40 2009 @@ -29,9 +29,8 @@ (random.randrange(0,65536)<<16) | (random.randrange(0,65536)<<0)) v.append(x) - return v + self._old_imm32_tests() + return v + super(TestRx86_64, self).imm32_tests() def test_extra_MOV_ri64(self): - self._old_imm32_tests = self.imm32_tests self.imm32_tests = self.imm64_tests # patch on 'self' self.complete_test('MOV_ri') From arigo at codespeak.net Fri Dec 18 15:48:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 18 Dec 2009 15:48:25 +0100 (CET) Subject: [pypy-svn] r70198 - pypy/branch/remove-ri386-multimethod-2 Message-ID: <20091218144825.A7CCE16801F@codespeak.net> Author: arigo Date: Fri Dec 18 15:48:25 2009 New Revision: 70198 Added: pypy/branch/remove-ri386-multimethod-2/ - copied from r70197, pypy/trunk/ Log: Make a new branch, in order to keep branch/remove-ri386-multimethod relatively up-to-date. From arigo at codespeak.net Fri Dec 18 15:53:27 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 18 Dec 2009 15:53:27 +0100 (CET) Subject: [pypy-svn] r70199 - in pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86: . test Message-ID: <20091218145327.0B922168015@codespeak.net> Author: arigo Date: Fri Dec 18 15:53:26 2009 New Revision: 70199 Added: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/rx86.py - copied unchanged from r70198, pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86.py - copied unchanged from r70198, pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py - copied unchanged from r70198, pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py - copied unchanged from r70198, pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py Removed: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/ri386.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/ri386setup.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_ri386.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/codebuf.py Log: Merge the branch/remove-ri386-multimethod, ignoring experimental changes done in regalloc, assembler and jump.py. Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/codebuf.py Fri Dec 18 15:53:26 2009 @@ -2,12 +2,12 @@ import os, sys from pypy.rpython.lltypesystem import lltype, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.jit.backend.x86.ri386 import I386CodeBuilder +from pypy.jit.backend.x86.rx86 import X86_32_CodeBuilder from pypy.rlib.rmmap import PTR, alloc, free from pypy.rlib.debug import make_sure_not_resized -class InMemoryCodeBuilder(I386CodeBuilder): +class InMemoryCodeBuilder(X86_32_CodeBuilder): _last_dump_start = 0 def __init__(self, start, end): @@ -20,22 +20,10 @@ self._size = map_size self._pos = 0 - def overwrite(self, pos, listofchars): - make_sure_not_resized(listofchars) - assert pos + len(listofchars) <= self._size - for c in listofchars: - self._data[pos] = c - pos += 1 - return pos - - def write(self, listofchars): - self._pos = self.overwrite(self._pos, listofchars) - - def writechr(self, n): - # purely for performance: don't make the one-element list [chr(n)] + def writechar(self, char): pos = self._pos assert pos + 1 <= self._size - self._data[pos] = chr(n) + self._data[pos] = char self._pos = pos + 1 def get_relative_pos(self): From arigo at codespeak.net Fri Dec 18 18:50:15 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 18 Dec 2009 18:50:15 +0100 (CET) Subject: [pypy-svn] r70202 - in pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86: . test Message-ID: <20091218175015.5A0CE168007@codespeak.net> Author: arigo Date: Fri Dec 18 18:50:14 2009 New Revision: 70202 Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/rx86.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86.py Log: Test and fix and document. Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/rx86.py Fri Dec 18 18:50:14 2009 @@ -29,8 +29,15 @@ return -2147483648 <= value <= 2147483647 def intmask32(value): + # extract the 32 lower bits of 'value', returning a regular signed int + # (it is negative if 'value' is negative according to the 32-bits repr) return intmask(rffi.cast(rffi.INT, value)) +def cast32to64(value): + # returns 'value' in the 32 lower bits of a 64-bit integer, + # with the remaining bits set to 0 (even if value is negative). + return r_ulonglong(rffi.cast(rffi.UINT, value)) + # ____________________________________________________________ # Emit a single char @@ -122,7 +129,7 @@ # * 'reg1' is stored as byte 5 of the result. assert reg != esp and reg != ebp assert fits_in_32bits(offset) - return (r_ulonglong(reg) << 32) | r_ulonglong(rffi.r_uint(offset)) + return (r_ulonglong(reg) << 32) | cast32to64(offset) def encode_mem_reg_plus_const(mc, reg1_offset, _, orbyte): reg1 = reg_number_3bits(mc, intmask(reg1_offset >> 32)) @@ -179,7 +186,7 @@ encoding |= REX_X << 8 reg2 &= 7 encoding |= (scaleshift<<6) | (reg2<<3) | reg1 - return (r_ulonglong(encoding) << 32) | r_ulonglong(rffi.r_uint(offset)) + return (r_ulonglong(encoding) << 32) | cast32to64(offset) def encode_mem_reg_plus_scaled_reg_plus_const(mc, reg1_reg2_scaleshift_offset, _, orbyte): Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86.py Fri Dec 18 18:50:14 2009 @@ -100,3 +100,8 @@ s.MOV_rm(edx, reg_offset(r12, 0)) s.MOV_rm(edx, reg_offset(r13, 0)) assert s.getvalue() == '\x48\x8B\x17\x49\x8b\x14\x24\x49\x8b\x55\x00' + +def test_mov_rm_negative_64(): + s = CodeBuilder64() + s.MOV_rm(edx, reg_offset(edi, -1)) + assert s.getvalue() == '\x48\x8B\x57\xFF' From arigo at codespeak.net Fri Dec 18 19:14:07 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 18 Dec 2009 19:14:07 +0100 (CET) Subject: [pypy-svn] r70203 - in pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86: . test Message-ID: <20091218181407.6583B16800D@codespeak.net> Author: arigo Date: Fri Dec 18 19:14:07 2009 New Revision: 70203 Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/rx86.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py Log: Group the register names in a class R. Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/rx86.py Fri Dec 18 19:14:07 2009 @@ -5,22 +5,21 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rpython.lltypesystem import rffi -# the following are synonyms for rax, rcx, etc. on 64 bits -eax = 0 -ecx = 1 -edx = 2 -ebx = 3 -esp = 4 -ebp = 5 -esi = 6 -edi = 7 - -# xmm registers -xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 = range(8) - -# the following are extra registers available only on 64 bits -r8, r9, r10, r11, r12, r13, r14, r15 = range(8, 16) -xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15 = range(8, 16) +class R(object): + # the following are synonyms for rax, rcx, etc. on 64 bits + eax, ecx, edx, ebx, esp, ebp, esi, edi = range(8) + + # xmm registers + xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 = range(8) + + # the following are extra registers available only on 64 bits + r8, r9, r10, r11, r12, r13, r14, r15 = range(8, 16) + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15 = range(8, 16) + + names = ['eax', 'ecx', 'edx', 'ebx', 'esp', 'ebp', 'esi', 'edi', + 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15'] + xmmnames = ['xmm%d' % i for i in range(16)] + def single_byte(value): return -128 <= value < 128 @@ -109,11 +108,11 @@ @specialize.arg(2) def encode_stack(mc, offset, force_32bits, orbyte): if not force_32bits and single_byte(offset): - mc.writechar(chr(0x40 | orbyte | ebp)) + mc.writechar(chr(0x40 | orbyte | R.ebp)) mc.writeimm8(offset) else: assert fits_in_32bits(offset) - mc.writechar(chr(0x80 | orbyte | ebp)) + mc.writechar(chr(0x80 | orbyte | R.ebp)) mc.writeimm32(offset) return 0 @@ -127,7 +126,7 @@ # returns a 64-bits integer encoding "reg1+offset". # * 'offset' is stored as bytes 1-4 of the result; # * 'reg1' is stored as byte 5 of the result. - assert reg != esp and reg != ebp + assert reg != R.esp and reg != R.ebp assert fits_in_32bits(offset) return (r_ulonglong(reg) << 32) | cast32to64(offset) @@ -139,9 +138,9 @@ # 64-bits special cases for reg1 == r12 or r13 # (which look like esp or ebp after being truncated to 3 bits) if mc.WORD == 8: - if reg1 == esp: # forces an SIB byte: - SIB = (esp<<3) | esp # use [r12+(no index)+offset] - elif reg1 == ebp: + if reg1 == R.esp: # forces an SIB byte: + SIB = (R.esp<<3) | R.esp # use [r12+(no index)+offset] + elif reg1 == R.ebp: no_offset = False # end of 64-bits special cases if no_offset: @@ -174,8 +173,8 @@ # * 'offset' is stored as bytes 1-4 of the result; # * the SIB byte is computed and stored as byte 5 of the result; # * for 64-bits mode, the optional REX.B and REX.X flags go to byte 6. - assert 0 <= reg1 < 16 and reg1 != ebp - assert 0 <= reg2 < 16 and reg2 != esp + assert 0 <= reg1 < 16 and reg1 != R.ebp + assert 0 <= reg2 < 16 and reg2 != R.esp assert 0 <= scaleshift < 4 assert fits_in_32bits(offset) encoding = 0 @@ -201,7 +200,7 @@ # 64-bits special case for reg1 == r13 # (which look like ebp after being truncated to 3 bits) if mc.WORD == 8: - if (encoding & 7) == ebp: + if (encoding & 7) == R.ebp: no_offset = False # end of 64-bits special case if no_offset: Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86.py Fri Dec 18 19:14:07 2009 @@ -1,5 +1,6 @@ import py from pypy.jit.backend.x86.rx86 import * +globals().update(R.__dict__) class CodeBuilderMixin(object): def __init__(self): Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py Fri Dec 18 19:14:07 2009 @@ -44,7 +44,8 @@ X86_CodeBuilder = rx86.X86_32_CodeBuilder REGNAMES = ['%eax', '%ecx', '%edx', '%ebx', '%esp', '%ebp', '%esi', '%edi'] REGS = range(8) - NONSPECREGS = [rx86.eax, rx86.ecx, rx86.edx, rx86.ebx, rx86.esi, rx86.edi] + NONSPECREGS = [rx86.R.eax, rx86.R.ecx, rx86.R.edx, rx86.R.ebx, + rx86.R.esi, rx86.R.edi] def reg_tests(self): return self.REGS @@ -207,11 +208,11 @@ # special cases if methname in ('ADD_ri', 'AND_ri', 'CMP_ri', 'OR_ri', 'SUB_ri', 'XOR_ri'): - if args[0] == rx86.eax: + if args[0] == rx86.R.eax: return [] # ADD EAX, constant: there is a special encoding - if methname == 'MOV_rj' and args[0] == rx86.eax: + if methname == 'MOV_rj' and args[0] == rx86.R.eax: return [] # MOV EAX, [immediate]: there is a special encoding - if methname == 'MOV_jr' and args[1] == rx86.eax: + if methname == 'MOV_jr' and args[1] == rx86.R.eax: return [] # MOV [immediate], EAX: there is a special encoding return [args] Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py Fri Dec 18 19:14:07 2009 @@ -10,9 +10,10 @@ REGNAMES = ['%rax', '%rcx', '%rdx', '%rbx', '%rsp', '%rbp', '%rsi', '%rdi', '%r8', '%r9', '%r10', '%r11', '%r12', '%r13', '%r14', '%r15'] REGS = range(16) - NONSPECREGS = [rx86.eax, rx86.ecx, rx86.edx, rx86.ebx, rx86.esi, rx86.edi, - rx86.r8, rx86.r9, rx86.r10, rx86.r11, - rx86.r12, rx86.r13, rx86.r14, rx86.r15] + NONSPECREGS = [rx86.R.eax, rx86.R.ecx, rx86.R.edx, rx86.R.ebx, + rx86.R.esi, rx86.R.edi, + rx86.R.r8, rx86.R.r9, rx86.R.r10, rx86.R.r11, + rx86.R.r12, rx86.R.r13, rx86.R.r14, rx86.R.r15] def array_tests(self): # reduce a little bit -- we spend too long in these tests From arigo at codespeak.net Fri Dec 18 19:14:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 18 Dec 2009 19:14:25 +0100 (CET) Subject: [pypy-svn] r70204 - in pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86: . test Message-ID: <20091218181425.37F91168012@codespeak.net> Author: arigo Date: Fri Dec 18 19:14:24 2009 New Revision: 70204 Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/jump.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/regalloc.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_jump.py Log: Make test_jump.py pass again. Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/jump.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/jump.py Fri Dec 18 19:14:24 2009 @@ -1,22 +1,6 @@ import sys from pypy.tool.pairtype import extendabletype -from pypy.jit.backend.x86.ri386 import * - -class __extend__(OPERAND): - __metaclass__ = extendabletype - def _getregkey(self): - raise AssertionError("should only happen to registers and frame " - "positions") - -class __extend__(REG): - __metaclass__ = extendabletype - def _getregkey(self): - return ~self.op - -class __extend__(MODRM): - __metaclass__ = extendabletype - def _getregkey(self): - return self.position +from pypy.jit.backend.x86.regalloc import AssemblerLocation, StackLoc, RegLoc def remap_frame_layout(assembler, src_locations, dst_locations, tmpreg): @@ -27,7 +11,7 @@ srccount[dst._getregkey()] = 0 for i in range(len(dst_locations)): src = src_locations[i] - if isinstance(src, IMM32): + if not isinstance(src, AssemblerLocation): # if it's a constant continue key = src._getregkey() if key in srccount: @@ -46,7 +30,7 @@ srccount[key] = -1 # means "it's done" pending_dests -= 1 src = src_locations[i] - if not isinstance(src, IMM32): + if isinstance(src, AssemblerLocation): key = src._getregkey() if key in srccount: srccount[key] -= 1 @@ -80,7 +64,11 @@ assert pending_dests == 0 def _move(assembler, src, dst, tmpreg): - if isinstance(dst, MODRM) and isinstance(src, MODRM): - assembler.regalloc_mov(src, tmpreg) - src = tmpreg - assembler.regalloc_mov(src, dst) + if isinstance(dst, StackLoc): + if isinstance(src, StackLoc): + assembler.regalloc_load(src, tmpreg) + src = tmpreg + assembler.regalloc_store(src, dst) + else: + assert isinstance(dst, RegLoc) + assembler.regalloc_load(src, dst) Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/regalloc.py Fri Dec 18 19:14:24 2009 @@ -5,14 +5,14 @@ from pypy.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr, ResOperation, ConstAddr, BoxPtr, LoopToken, INT, REF, FLOAT) -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86 import rx86 from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable from pypy.rlib import rgc from pypy.jit.backend.llsupport import symbolic -from pypy.jit.backend.x86.jump import remap_frame_layout from pypy.jit.metainterp.resoperation import rop +from pypy.jit.metainterp.history import AbstractValue from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\ @@ -22,6 +22,37 @@ FRAME_FIXED_SIZE = 5 # ebp + ebx + esi + edi + force_index = 5 words FORCE_INDEX_OFS = -4*WORD + +class AssemblerLocation(AbstractValue): + __slots__ = 'value' + def _getregkey(self): + return self.value + +class StackLoc(AssemblerLocation): + def __init__(self, position, ebp_offset): + assert ebp_offset < 0 # so no confusion with RegLoc._loc + self.position = position + self.value = ebp_offset + def __repr__(self): + return '%d(%%ebp)' % (self.value,) + +class RegLoc(AssemblerLocation): + def __init__(self, regnum): + assert regnum >= 0 + self.value = regnum + def __repr__(self): + return rx86.R.names[self.value] + +class XmmRegLoc(RegLoc): + def __repr__(self): + return rx86.R.xmmnames[self.value] + +REGLOCS = [RegLoc(i) for i in range(8)] +XMMREGLOCS = [XmmRegLoc(i) for i in range(8)] +eax, ecx, edx, ebx, esp, ebp, esi, edi = REGLOCS +xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 = XMMREGLOCS + + width_of_type = { INT : 1, REF : 1, @@ -38,20 +69,6 @@ def call_result_location(self, v): return eax - def convert_to_imm(self, c): - if isinstance(c, ConstInt): - return imm(c.value) - elif isinstance(c, ConstPtr): - if we_are_translated() and c.value and rgc.can_move(c.value): - print "convert_to_imm: ConstPtr needs special care" - raise AssertionError - return imm(rffi.cast(lltype.Signed, c.value)) - elif isinstance(c, ConstAddr): - return imm(ll2ctypes.cast_adr_to_int(c.value)) - else: - print "convert_to_imm: got a %s" % c - raise AssertionError - BASE_CONSTANT_SIZE = 1000 @@ -73,15 +90,15 @@ self.constant_arrays = [self.new_const_array()] self.constant_array_counter = 0 - def convert_to_imm(self, c): - if self.constant_array_counter >= BASE_CONSTANT_SIZE: - self.constant_arrays.append(self.new_const_array()) - self.constant_array_counter = 0 - res = self.constant_array_counter - self.constant_array_counter += 1 - arr = self.constant_arrays[-1] - arr[res] = c.getfloat() - return self.get_addr_of_const_float(-1, res) +## def convert_to_imm(self, c): +## if self.constant_array_counter >= BASE_CONSTANT_SIZE: +## self.constant_arrays.append(self.new_const_array()) +## self.constant_array_counter = 0 +## res = self.constant_array_counter +## self.constant_array_counter += 1 +## arr = self.constant_arrays[-1] +## arr[res] = c.getfloat() +## return self.get_addr_of_const_float(-1, res) def get_addr_of_const_float(self, num_arr, num_pos): arr = self.constant_arrays[num_arr] @@ -98,14 +115,12 @@ @staticmethod def frame_pos(i, size): if size == 1: - res = mem(ebp, get_ebp_ofs(i)) + return StackLoc(i, get_ebp_ofs(i)) elif size == 2: - res = mem64(ebp, get_ebp_ofs(i + 1)) + return StackLoc(i, get_ebp_ofs(i + 1)) else: print "Unimplemented size %d" % i raise NotImplementedError("unimplemented size %d" % i) - res.position = i - return res class RegAlloc(object): exc = False @@ -152,8 +167,8 @@ def _process_inputargs(self, inputargs): # XXX we can sort out here by longevity if we need something # more optimal - floatlocs = [None] * len(inputargs) - nonfloatlocs = [None] * len(inputargs) + floatlocs = [-1] * len(inputargs) + nonfloatlocs = [-1] * len(inputargs) # Don't use all_regs[0] for passing arguments around a loop. # Must be kept in sync with consider_jump(). # XXX this should probably go to llsupport/regalloc.py @@ -878,6 +893,7 @@ consider_unicodegetitem = consider_strgetitem def consider_jump(self, op, ignored): + from pypy.jit.backend.x86.jump import remap_frame_layout assembler = self.assembler assert self.jump_target_descr is None descr = op.descr Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_jump.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_jump.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_jump.py Fri Dec 18 19:14:24 2009 @@ -1,5 +1,7 @@ -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.metainterp.history import ConstInt from pypy.jit.backend.x86.regalloc import X86FrameManager +from pypy.jit.backend.x86.regalloc import StackLoc, RegLoc +from pypy.jit.backend.x86.regalloc import eax, ebx, ecx, edx, esi, edi from pypy.jit.backend.x86.jump import remap_frame_layout frame_pos = X86FrameManager.frame_pos @@ -8,8 +10,15 @@ def __init__(self): self.ops = [] - def regalloc_mov(self, from_loc, to_loc): - self.ops.append(('mov', from_loc, to_loc)) + def regalloc_load(self, from_loc, to_loc): + assert isinstance(from_loc, (RegLoc, StackLoc, ConstInt)) + assert isinstance(to_loc, RegLoc) + self.ops.append(('load', from_loc, to_loc)) + + def regalloc_store(self, from_loc, to_loc): + assert isinstance(from_loc, (RegLoc, ConstInt)) + assert isinstance(to_loc, StackLoc) + self.ops.append(('store', from_loc, to_loc)) def regalloc_push(self, loc): self.ops.append(('push', loc)) @@ -25,11 +34,7 @@ continue assert len(op1) == len(op2) for x, y in zip(op1, op2): - if isinstance(x, MODRM) and isinstance(y, MODRM): - assert x.byte == y.byte - assert x.extradata == y.extradata - else: - assert x == y + assert x == y assert len(self.ops) == len(expected) return True @@ -52,9 +57,9 @@ def test_simple_registers(): assembler = MockAssembler() remap_frame_layout(assembler, [eax, ebx, ecx], [edx, esi, edi], '?') - assert assembler.ops == [('mov', eax, edx), - ('mov', ebx, esi), - ('mov', ecx, edi)] + assert assembler.ops == [('load', eax, edx), + ('load', ebx, esi), + ('load', ecx, edi)] def test_simple_framelocs(): assembler = MockAssembler() @@ -63,10 +68,10 @@ s20 = frame_pos(20, 1) s24 = frame_pos(221, 1) remap_frame_layout(assembler, [s8, eax, s12], [s20, s24, edi], edx) - assert assembler.ops == [('mov', s8, edx), - ('mov', edx, s20), - ('mov', eax, s24), - ('mov', s12, edi)] + assert assembler.ops == [('load', s8, edx), + ('store', edx, s20), + ('store', eax, s24), + ('load', s12, edi)] def test_reordering(): assembler = MockAssembler() @@ -76,10 +81,10 @@ s24 = frame_pos(1, 1) remap_frame_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, edi], '?') - assert assembler.got([('mov', ebx, edi), - ('mov', s8, ebx), - ('mov', eax, s8), - ('mov', s20, eax)]) + assert assembler.got([('load', ebx, edi), + ('load', s8, ebx), + ('store', eax, s8), + ('load', s20, eax)]) def test_cycle(): assembler = MockAssembler() @@ -90,9 +95,9 @@ remap_frame_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, s20], '?') assert assembler.got([('push', s8), - ('mov', eax, s8), - ('mov', s20, eax), - ('mov', ebx, s20), + ('store', eax, s8), + ('load', s20, eax), + ('store', ebx, s20), ('pop', ebx)]) def test_cycle_2(): @@ -107,37 +112,37 @@ [eax, s8, edi, s20, eax, s20, s24, esi, s2, s3], [s8, s20, edi, eax, edx, s24, ebx, s12, s3, s2], ecx) - assert assembler.got([('mov', eax, edx), - ('mov', s24, ebx), - ('mov', esi, s12), - ('mov', s20, ecx), - ('mov', ecx, s24), + assert assembler.got([('load', eax, edx), + ('load', s24, ebx), + ('store', esi, s12), + ('load', s20, ecx), + ('store', ecx, s24), ('push', s8), - ('mov', eax, s8), - ('mov', s20, eax), + ('store', eax, s8), + ('load', s20, eax), ('pop', s20), ('push', s3), - ('mov', s2, ecx), - ('mov', ecx, s3), + ('load', s2, ecx), + ('store', ecx, s3), ('pop', s2)]) def test_constants(): assembler = MockAssembler() - c3 = imm(3) + c3 = ConstInt(3) remap_frame_layout(assembler, [c3], [eax], '?') - assert assembler.ops == [('mov', c3, eax)] + assert assembler.ops == [('load', c3, eax)] assembler = MockAssembler() s12 = frame_pos(12, 1) remap_frame_layout(assembler, [c3], [s12], '?') - assert assembler.ops == [('mov', c3, s12)] + assert assembler.ops == [('store', c3, s12)] def test_constants_and_cycle(): assembler = MockAssembler() - c3 = imm(3) + c3 = ConstInt(3) s12 = frame_pos(13, 1) remap_frame_layout(assembler, [ebx, c3, s12], [s12, eax, ebx], edi) - assert assembler.ops == [('mov', c3, eax), + assert assembler.ops == [('load', c3, eax), ('push', s12), - ('mov', ebx, s12), + ('store', ebx, s12), ('pop', ebx)] From arigo at codespeak.net Fri Dec 18 19:18:12 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 18 Dec 2009 19:18:12 +0100 (CET) Subject: [pypy-svn] r70205 - pypy/branch/remove-ri386-multimethod/pypy/jit/backend/llsupport Message-ID: <20091218181812.5E11616800D@codespeak.net> Author: arigo Date: Fri Dec 18 19:18:11 2009 New Revision: 70205 Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/llsupport/regalloc.py Log: Old code. Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/llsupport/regalloc.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/llsupport/regalloc.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/llsupport/regalloc.py Fri Dec 18 19:18:11 2009 @@ -1,5 +1,5 @@ -from pypy.jit.metainterp.history import Const, Box +from pypy.jit.metainterp.history import Box from pypy.rlib.objectmodel import we_are_translated class TempBox(Box): @@ -60,6 +60,7 @@ self.position += incr def _check_type(self, v): + assert isinstance(v, Box) if not we_are_translated() and self.box_types is not None: assert isinstance(v, TempBox) or v.type in self.box_types @@ -69,7 +70,7 @@ point for all variables that might be in registers. """ self._check_type(v) - if isinstance(v, Const) or v not in self.reg_bindings: + if v not in self.reg_bindings: return if v not in self.longevity or self.longevity[v][1] <= self.position: self.free_regs.append(self.reg_bindings[v]) @@ -104,7 +105,6 @@ returns allocated register or None, if not possible. """ self._check_type(v) - assert not isinstance(v, Const) if selected_reg is not None: res = self.reg_bindings.get(v, None) if res is not None: @@ -199,8 +199,6 @@ """ Return the location of 'box'. """ self._check_type(box) - if isinstance(box, Const): - return self.convert_to_imm(box) try: return self.reg_bindings[box] except KeyError: @@ -237,10 +235,6 @@ 'force_allocate_reg' for the meaning of the optional arguments. """ self._check_type(v) - if isinstance(v, Const): - return self.return_constant(v, forbidden_vars, selected_reg, - imm_fine) - prev_loc = self.loc(v) loc = self.force_allocate_reg(v, forbidden_vars, selected_reg, need_lower_byte=need_lower_byte) @@ -270,15 +264,6 @@ """ self._check_type(result_v) self._check_type(v) - if isinstance(v, Const): - loc = self.make_sure_var_in_reg(v, forbidden_vars, - imm_fine=False) - # note that calling make_sure_var_in_reg with imm_fine=False - # will not allocate place in reg_bindings, we need to do it - # on our own - self.reg_bindings[result_v] = loc - self.free_regs = [reg for reg in self.free_regs if reg is not loc] - return loc if v not in self.reg_bindings: prev_loc = self.stack_manager.loc(v, self.reg_width) loc = self.force_allocate_reg(v, forbidden_vars) @@ -335,11 +320,6 @@ # abstract methods, override - def convert_to_imm(self, c): - """ Platform specific - convert a constant to imm - """ - raise NotImplementedError("Abstract") - def call_result_location(self, v): """ Platform specific - tell where the result of a call will be stored by the cpu, according to the variable type From arigo at codespeak.net Fri Dec 18 19:18:32 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 18 Dec 2009 19:18:32 +0100 (CET) Subject: [pypy-svn] r70206 - pypy/branch/remove-ri386-multimethod Message-ID: <20091218181832.705B316800D@codespeak.net> Author: arigo Date: Fri Dec 18 19:18:31 2009 New Revision: 70206 Removed: pypy/branch/remove-ri386-multimethod/ Log: Remove the branch, new code is in branch/remove-ri386-multimethod-2. From arigo at codespeak.net Sun Dec 20 11:35:01 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 20 Dec 2009 11:35:01 +0100 (CET) Subject: [pypy-svn] r70214 - in pypy/trunk/pypy/jit/backend/x86: . test Message-ID: <20091220103501.42295168015@codespeak.net> Author: arigo Date: Sun Dec 20 11:34:59 2009 New Revision: 70214 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py Log: Argh. We allocate 8KB of space for float constants, and we forget about it between each block of code. This means that most blocks (loops and branches) consumed 8KB of extra space for nothing. Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Sun Dec 20 11:34:59 2009 @@ -75,6 +75,7 @@ mc = None mc2 = None mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE + _float_constants = None def __init__(self, cpu, translate_support_code=False, failargs_limit=1000): Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Sun Dec 20 11:34:59 2009 @@ -52,7 +52,28 @@ print "convert_to_imm: got a %s" % c raise AssertionError -BASE_CONSTANT_SIZE = 1000 + +class FloatConstants(object): + BASE_CONSTANT_SIZE = 1000 + + def __init__(self): + self.cur_array_free = 0 + + def _get_new_array(self): + n = self.BASE_CONSTANT_SIZE + self.cur_array = lltype.malloc(rffi.CArray(lltype.Float), n, + flavor='raw') + self.cur_array_free = n + _get_new_array._dont_inline_ = True + + def record_float(self, floatval): + if self.cur_array_free == 0: + self._get_new_array() + arr = self.cur_array + n = self.cur_array_free - 1 + arr[n] = floatval + self.cur_array_free = n + return rffi.cast(lltype.Signed, arr) + n * 8 class X86XMMRegisterManager(RegisterManager): @@ -63,29 +84,19 @@ save_around_call_regs = all_regs reg_width = 2 - def new_const_array(self): - return lltype.malloc(rffi.CArray(lltype.Float), BASE_CONSTANT_SIZE, - flavor='raw') - def __init__(self, longevity, frame_manager=None, assembler=None): RegisterManager.__init__(self, longevity, frame_manager=frame_manager, assembler=assembler) - self.constant_arrays = [self.new_const_array()] - self.constant_array_counter = 0 + if assembler is None: + self.float_constants = FloatConstants() + else: + if assembler._float_constants is None: + assembler._float_constants = FloatConstants() + self.float_constants = assembler._float_constants def convert_to_imm(self, c): - if self.constant_array_counter >= BASE_CONSTANT_SIZE: - self.constant_arrays.append(self.new_const_array()) - self.constant_array_counter = 0 - res = self.constant_array_counter - self.constant_array_counter += 1 - arr = self.constant_arrays[-1] - arr[res] = c.getfloat() - return self.get_addr_of_const_float(-1, res) - - def get_addr_of_const_float(self, num_arr, num_pos): - arr = self.constant_arrays[num_arr] - return heap64(rffi.cast(lltype.Signed, arr) + num_pos * WORD * 2) + adr = self.float_constants.record_float(c.getfloat()) + return heap64(adr) def after_call(self, v): # the result is stored in st0, but we don't have this around, Modified: pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py Sun Dec 20 11:34:59 2009 @@ -9,7 +9,7 @@ from pypy.jit.backend.llsupport.descr import GcCache from pypy.jit.backend.x86.runner import CPU from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, X86RegisterManager,\ - BASE_CONSTANT_SIZE + FloatConstants from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper @@ -503,6 +503,7 @@ def test_float_overflow_const_list(self): ops = ['[f0]'] + BASE_CONSTANT_SIZE = FloatConstants.BASE_CONSTANT_SIZE for i in range(BASE_CONSTANT_SIZE * 2): ops.append('f%d = float_add(f%d, 3.5)' % (i + 1, i)) ops.append('finish(f%d)' % (BASE_CONSTANT_SIZE * 2)) From arigo at codespeak.net Sun Dec 20 11:56:16 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 20 Dec 2009 11:56:16 +0100 (CET) Subject: [pypy-svn] r70215 - in pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86: . test Message-ID: <20091220105616.C5E78168015@codespeak.net> Author: arigo Date: Sun Dec 20 11:56:16 2009 New Revision: 70215 Added: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/regloc.py (contents, props changed) Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/assembler.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/jump.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/regalloc.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/rx86.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/support.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_jump.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py Log: Simplify rx86 a bit, add SSE2 and CALL instructions. Start to port the rest of the code. Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/assembler.py Sun Dec 20 11:56:16 2009 @@ -11,9 +11,14 @@ from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, lower_byte,\ X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs, FRAME_FIXED_SIZE,\ FORCE_INDEX_OFS +from pypy.jit.backend.x86.regloc import (eax, ecx, edx, ebx, + esp, ebp, esi, edi, + xmm0, xmm1, xmm2, xmm3, + xmm4, xmm5, xmm6, xmm7, + RegLoc, StackLoc) from pypy.rlib.objectmodel import we_are_translated, specialize +from pypy.jit.backend.x86 import rx86 from pypy.jit.backend.x86 import codebuf -from pypy.jit.backend.x86.ri386 import * from pypy.jit.metainterp.resoperation import rop from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print @@ -67,9 +72,8 @@ method.func_name = name return method -for name in dir(codebuf.MachineCodeBlock): - if name.upper() == name: - setattr(MachineCodeBlockWrapper, name, _new_method(name)) +for _name in rx86.all_instructions: + setattr(MachineCodeBlockWrapper, _name, _new_method(_name)) class Assembler386(object): mc = None @@ -146,8 +150,8 @@ addr[5] = 2147483647 # / for abs addr[6] = 0 # addr[7] = 0 # - self.loc_float_const_neg = heap64(float_constants) - self.loc_float_const_abs = heap64(float_constants + 16) + self.loc_float_const_neg = float_constants + self.loc_float_const_abs = float_constants + 16 def assemble_loop(self, inputargs, operations, looptoken): """adds the following attributes to looptoken: @@ -219,7 +223,7 @@ def _patchable_stackadjust(self): # stack adjustment LEA - self.mc.LEA(esp, fixedsize_ebp_ofs(0)) + self.mc.LEA32_rb(esp.value, 0) return self.mc.tell() - 4 def _patch_stackadjust(self, adr_lea, reserved_depth): @@ -230,51 +234,92 @@ # Given that [EBP] is where we saved EBP, i.e. in the last word # of our fixed frame, then the 'words' value is: words = (FRAME_FIXED_SIZE - 1) + reserved_depth - mc.write(packimm32(-WORD * words)) + mc.writeimm32(-WORD * words) mc.done() def _assemble_bootstrap_code(self, inputargs, arglocs): nonfloatlocs, floatlocs = arglocs - self.mc.PUSH(ebp) - self.mc.MOV(ebp, esp) - self.mc.PUSH(ebx) - self.mc.PUSH(esi) - self.mc.PUSH(edi) + self.mc.PUSH_r(ebp.value) + self.mc.MOV_rr(ebp.value, esp.value) + self.mc.PUSH_r(ebx.value) + self.mc.PUSH_r(esi.value) + self.mc.PUSH_r(edi.value) # NB. the shape of the frame is hard-coded in get_basic_shape() too. # Also, make sure this is consistent with FRAME_FIXED_SIZE. adr_stackadjust = self._patchable_stackadjust() - tmp = X86RegisterManager.all_regs[0] - xmmtmp = X86XMMRegisterManager.all_regs[0] + # + self._bootstrap_load_floats(floatlocs) + # + # A bit of messy code to ensure that we load the value of 'esi' + # last, to let it be available as a temporary register in the + # preceeding parts. for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] + if loc is esi: + esi_type = inputargs[i].type + break + else: + i = 0 + esi_type = INT + if esi_type == REF: + self._bootstrap_load(inputargs, nonfloatlocs, + INT, self.fail_boxes_int, 0) + self._bootstrap_load(inputargs, nonfloatlocs, + REF, self.fail_boxes_ptr, i) + else: + self._bootstrap_load(inputargs, nonfloatlocs, + REF, self.fail_boxes_ptr, 0) + self._bootstrap_load(inputargs, nonfloatlocs, + INT, self.fail_boxes_int, i) + # + return adr_stackadjust + + def _bootstrap_load(self, inputargs, nonfloatlocs, type, fail_boxes, i): + # load all input arguments from 'nonfloatlocs' that are of the + # given type (INT or REF), loading the i'th argument last. + tmp = X86RegisterManager.all_regs[0] + adr = fail_boxes.get_base_addr() + self.mc.MOV_ri(tmp.value, adr) + for j in range(len(nonfloatlocs)): + i -= 1 + if i < 0: + i = len(nonfloatlocs) - 1 + loc = nonfloatlocs[i] if loc is None: continue - if isinstance(loc, REG): + if inputargs[i].type != type: + continue + if isinstance(loc, RegLoc): target = loc else: - target = tmp - if inputargs[i].type == REF: + target = esi + if type == REF: # This uses XCHG to put zeroes in fail_boxes_ptr after # reading them - self.mc.XOR(target, target) - adr = self.fail_boxes_ptr.get_addr_for_num(i) - self.mc.XCHG(target, heap(adr)) + self.mc.XOR_rr(target.value, target.value) + self.mc.XCHG_rm(target.value, (tmp.value, WORD*i)) else: - adr = self.fail_boxes_int.get_addr_for_num(i) - self.mc.MOV(target, heap(adr)) - if target is not loc: - self.mc.MOV(loc, target) + self.mc.MOV_rm(target.value, (tmp.value, WORD*i)) + if isinstance(loc, StackLoc): + self.mc.MOV_br(loc.value, target.value) + + def _bootstrap_load_floats(self, floatlocs): + tmp = X86RegisterManager.all_regs[0] + loaded = False for i in range(len(floatlocs)): loc = floatlocs[i] if loc is None: continue - adr = self.fail_boxes_float.get_addr_for_num(i) - if isinstance(loc, REG): - self.mc.MOVSD(loc, heap64(adr)) + if not loaded: + adr = self.fail_boxes_float.get_base_addr() + self.mc.MOV_ri(tmp.value, adr) + loaded = True + if isinstance(loc, RegLoc): + self.mc.MOVSD_rm(loc.value, (tmp.value, 8*i)) else: - self.mc.MOVSD(xmmtmp, heap64(adr)) - self.mc.MOVSD(loc, xmmtmp) - return adr_stackadjust + xmmtmp = X86XMMRegisterManager.all_regs[0] + self.mc.MOVSD_rm(xmmtmp.value, (tmp.value, 8*i)) + self.mc.MOVSD_sr(loc.value, xmmtmp.value) def dump(self, text): if not self.verbose: @@ -359,7 +404,13 @@ def _binaryop(asmop, can_swap=False): def genop_binary(self, op, arglocs, result_loc): - getattr(self.mc, asmop)(arglocs[0], arglocs[1]) + loc0, loc1 = arglocs + if isinstance(loc1, RegLoc): + getattr(self.mc, asmop + '_rr')(loc0.value, loc1.value) + elif isinstance(loc1, StackLoc): + getattr(self.mc, asmop + '_rb')(loc0.value, loc1.value) + else: + getattr(self.mc, asmop + '_ri')(loc0.value, loc1.getint()) return genop_binary def _cmpop(cond, rev_cond): @@ -825,15 +876,15 @@ if box is not None and box.type == FLOAT: withfloats = True break - mc.CALL(rel32(self.failure_recovery_code[exc + 2 * withfloats])) + mc.CALL_l(self.failure_recovery_code[exc + 2 * withfloats]) # write tight data that describes the failure recovery faildescr._x86_failure_recovery_bytecode = mc.tell() self.write_failure_recovery_description(mc, failargs, fail_locs) # write the fail_index too - mc.write(packimm32(fail_index)) + mc.writeimm32(fail_index) # for testing the decoding, write a final byte 0xCC if not we_are_translated(): - mc.writechr(0xCC) + mc.writechar('\xCC') faildescr._x86_debug_faillocs = [loc for loc in fail_locs if loc is not None] return addr @@ -859,26 +910,19 @@ else: raise AssertionError("bogus kind") loc = locs[i] - if isinstance(loc, MODRM): + if isinstance(loc, StackLoc): n = self.CODE_FROMSTACK//4 + loc.position else: - assert isinstance(loc, REG) - n = loc.op + assert isinstance(loc, RegLoc) + n = loc.value n = kind + 4*n while n > 0x7F: - mc.writechr((n & 0x7F) | 0x80) + mc.writechar(chr((n & 0x7F) | 0x80)) n >>= 7 else: n = self.CODE_HOLE - mc.writechr(n) - mc.writechr(self.CODE_STOP) - # preallocate the fail_boxes - i = len(failargs) - 1 - if i >= 0: - self.fail_boxes_int.get_addr_for_num(i) - self.fail_boxes_ptr.get_addr_for_num(i) - if self.cpu.supports_floats: - self.fail_boxes_float.get_addr_for_num(i) + mc.writechar(chr(n)) + mc.writechar(chr(self.CODE_STOP)) def rebuild_faillocs_from_descr(self, bytecode): from pypy.jit.backend.x86.regalloc import X86FrameManager @@ -1044,19 +1088,19 @@ # Assume that we are called at the beginning, when there is no risk # that 'mc' runs out of space. Checked by asserts in mc.write(). recovery_addr = mc.tell() - mc.PUSH(edi) - mc.PUSH(esi) - mc.PUSH(ebp) - mc.PUSH(esp) # <-- not really used, but needed to take up the space - mc.PUSH(ebx) - mc.PUSH(edx) - mc.PUSH(ecx) - mc.PUSH(eax) - mc.MOV(esi, esp) + mc.PUSH_r(edi.value) + mc.PUSH_r(esi.value) + mc.PUSH_r(ebp.value) + mc.PUSH_r(esp.value) # <-- not really used, but needed to take + mc.PUSH_r(ebx.value) # up the space + mc.PUSH_r(edx.value) + mc.PUSH_r(ecx.value) + mc.PUSH_r(eax.value) + mc.MOV_rr(esi.value, esp.value) if withfloats: - mc.SUB(esp, imm(8*8)) + mc.SUB_ri(esp.value, 8*8) for i in range(8): - mc.MOVSD(mem64(esp, 8*i), xmm_registers[i]) + mc.MOVSD_sr(8*i, i) # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1064,7 +1108,7 @@ # avoid unwarranted freeing # - optionally save exception depending on the flag addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) - mc.CALL(rel32(addr)) + mc.CALL_l(addr) # the following call saves all values from the stack and from # registers to the right 'fail_boxes_' location. @@ -1073,66 +1117,63 @@ # the XMM registers. Moreover, esi[8] is a pointer to the recovery # bytecode, pushed just before by the CALL instruction written by # generate_quick_failure(). - mc.PUSH(esi) - mc.CALL(rel32(failure_recovery_func)) + mc.PUSH_r(esi.value) + mc.CALL_l(failure_recovery_func) # returns in eax the fail_index # now we return from the complete frame, which starts from # _assemble_bootstrap_code(). The LEA below throws away most # of the frame, including all the PUSHes that we did just above. - mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) - mc.POP(edi) # [ebp-12] - mc.POP(esi) # [ebp-8] - mc.POP(ebx) # [ebp-4] - mc.POP(ebp) # [ebp] + mc.LEA_rb(esp.value, -3 * WORD) + mc.POP_r(edi.value) # [ebp-12] + mc.POP_r(esi.value) # [ebp-8] + mc.POP_r(ebx.value) # [ebp-4] + mc.POP_r(ebp.value) # [ebp] mc.RET() self.mc2.done() self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr - def generate_failure(self, mc, fail_index, locs, exc, locs_are_ref): - for i in range(len(locs)): - loc = locs[i] - if isinstance(loc, REG): - if loc.width == 8: - adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD(heap64(adr), loc) - else: - if locs_are_ref[i]: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - else: - adr = self.fail_boxes_int.get_addr_for_num(i) - mc.MOV(heap(adr), loc) - for i in range(len(locs)): - loc = locs[i] - if not isinstance(loc, REG): - if loc.width == 8: - mc.MOVSD(xmm0, loc) - adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD(heap64(adr), xmm0) - else: - if locs_are_ref[i]: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - else: - adr = self.fail_boxes_int.get_addr_for_num(i) - mc.MOV(eax, loc) - mc.MOV(heap(adr), eax) + def generate_failure_save(self, v, loc, i, tmpreg): + if v.type == FLOAT: + destadr = self.fail_boxes_float.get_addr_for_num(i) + self.mc.MOV_ri(tmpreg.value, destadr) + if isinstance(loc, RegLoc): + self.mc.MOVSD_mr((tmpreg.value, 0), loc.value) + else: + assert isinstance(loc, ConstInt) + value = rffi.cast(rffi.INTP, loc.value)[0] + self.mc.MOV_mi((tmpreg.value, 0), value) + value = rffi.cast(rffi.INTP, loc.value)[1] + self.mc.MOV_mi((tmpreg.value, 4), value) + else: + if v.type == REF: + destadr = self.fail_boxes_ref.get_addr_for_num(i) + else: + destadr = self.fail_boxes_int.get_addr_for_num(i) + self.mc.MOV_ri(tmpreg.value, destadr) + if isinstance(loc, RegLoc): + self.mc.MOV_mr((tmpreg.value, 0), loc.value) + else: + assert isinstance(loc, ConstInt) + self.mc.MOV_mi((tmpreg.value, 0), loc.value) + def generate_failure(self, fail_index, exc): + # avoid breaking the following code sequence, as we are almost done + mc = self.mc._mc # we call a provided function that will # - call our on_leave_jitted_hook which will mark # the fail_boxes_ptr array as pointing to young objects to # avoid unwarranted freeing # - optionally save exception depending on the flag addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) - mc.CALL(rel32(addr)) - - # don't break the following code sequence! xxx no reason any more? - mc = mc._mc - mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) - mc.MOV(eax, imm(fail_index)) - mc.POP(edi) # [ebp-12] - mc.POP(esi) # [ebp-8] - mc.POP(ebx) # [ebp-4] - mc.POP(ebp) # [ebp] + mc.CALL_l(addr) + # + mc.LEA_rb(esp.value, -3 * WORD) + mc.MOV_ri(eax.value, fail_index) + mc.POP_r(edi.value) # [ebp-12] + mc.POP_r(esi.value) # [ebp-8] + mc.POP_r(ebx.value) # [ebp-4] + mc.POP_r(ebp.value) # [ebp] mc.RET() @specialize.arg(2) @@ -1289,15 +1330,15 @@ return memsib(reg_or_imm1, reg_or_imm2, scale, offset) return addr_add -addr8_add = new_addr_add(heap8, mem8, memSIB8) -addr_add = new_addr_add(heap, mem, memSIB) -addr64_add = new_addr_add(heap64, mem64, memSIB64) - -def addr_add_const(reg_or_imm1, offset): - if isinstance(reg_or_imm1, IMM32): - return heap(reg_or_imm1.value + offset) - else: - return mem(reg_or_imm1, offset) +#addr8_add = new_addr_add(heap8, mem8, memSIB8) +#addr_add = new_addr_add(heap, mem, memSIB) +#addr64_add = new_addr_add(heap64, mem64, memSIB64) +# +#def addr_add_const(reg_or_imm1, offset): +# if isinstance(reg_or_imm1, IMM32): +# return heap(reg_or_imm1.value + offset) +# else: +# return mem(reg_or_imm1, offset) def round_up_to_4(size): if size < 4: Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/jump.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/jump.py Sun Dec 20 11:56:16 2009 @@ -1,6 +1,6 @@ import sys from pypy.tool.pairtype import extendabletype -from pypy.jit.backend.x86.regalloc import AssemblerLocation, StackLoc, RegLoc +from pypy.jit.backend.x86.regloc import AssemblerLocation, StackLoc, RegLoc def remap_frame_layout(assembler, src_locations, dst_locations, tmpreg): Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/regalloc.py Sun Dec 20 11:56:16 2009 @@ -5,14 +5,16 @@ from pypy.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr, ResOperation, ConstAddr, BoxPtr, LoopToken, INT, REF, FLOAT) -from pypy.jit.backend.x86 import rx86 +from pypy.jit.backend.x86.regloc import (StackLoc, eax, ecx, edx, ebx, + esp, ebp, esi, edi, + xmm0, xmm1, xmm2, xmm3, + xmm4, xmm5, xmm6, xmm7) from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable from pypy.rlib import rgc from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.resoperation import rop -from pypy.jit.metainterp.history import AbstractValue from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\ @@ -22,37 +24,6 @@ FRAME_FIXED_SIZE = 5 # ebp + ebx + esi + edi + force_index = 5 words FORCE_INDEX_OFS = -4*WORD - -class AssemblerLocation(AbstractValue): - __slots__ = 'value' - def _getregkey(self): - return self.value - -class StackLoc(AssemblerLocation): - def __init__(self, position, ebp_offset): - assert ebp_offset < 0 # so no confusion with RegLoc._loc - self.position = position - self.value = ebp_offset - def __repr__(self): - return '%d(%%ebp)' % (self.value,) - -class RegLoc(AssemblerLocation): - def __init__(self, regnum): - assert regnum >= 0 - self.value = regnum - def __repr__(self): - return rx86.R.names[self.value] - -class XmmRegLoc(RegLoc): - def __repr__(self): - return rx86.R.xmmnames[self.value] - -REGLOCS = [RegLoc(i) for i in range(8)] -XMMREGLOCS = [XmmRegLoc(i) for i in range(8)] -eax, ecx, edx, ebx, esp, ebp, esi, edi = REGLOCS -xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 = XMMREGLOCS - - width_of_type = { INT : 1, REF : 1, @@ -66,6 +37,9 @@ no_lower_byte_regs = [esi, edi] save_around_call_regs = [eax, edx, ecx] + def convert_to_imm(self, c): + return c + def call_result_location(self, v): return eax @@ -87,23 +61,22 @@ def __init__(self, longevity, frame_manager=None, assembler=None): RegisterManager.__init__(self, longevity, frame_manager=frame_manager, assembler=assembler) - self.constant_arrays = [self.new_const_array()] - self.constant_array_counter = 0 + self.past_constant_arrays = [] + self.cur_constant_array = None + self.constant_array_counter = BASE_CONSTANT_SIZE + + def convert_to_imm(self, c): + if self.constant_array_counter >= BASE_CONSTANT_SIZE: + self.cur_constant_array = self.new_const_array() + self.past_constant_arrays.append(self.cur_constant_array) + self.constant_array_counter = 0 + res = self.constant_array_counter + self.constant_array_counter += 1 + arr = self.cur_constant_array + arr[res] = c.getfloat() + addr = rffi.cast(lltype.Signed, arr) + res * 8 + return ConstInt(addr) -## def convert_to_imm(self, c): -## if self.constant_array_counter >= BASE_CONSTANT_SIZE: -## self.constant_arrays.append(self.new_const_array()) -## self.constant_array_counter = 0 -## res = self.constant_array_counter -## self.constant_array_counter += 1 -## arr = self.constant_arrays[-1] -## arr[res] = c.getfloat() -## return self.get_addr_of_const_float(-1, res) - - def get_addr_of_const_float(self, num_arr, num_pos): - arr = self.constant_arrays[num_arr] - return heap64(rffi.cast(lltype.Signed, arr) + num_pos * WORD * 2) - def after_call(self, v): # the result is stored in st0, but we don't have this around, # so genop_call will move it to some frame location immediately @@ -167,8 +140,8 @@ def _process_inputargs(self, inputargs): # XXX we can sort out here by longevity if we need something # more optimal - floatlocs = [-1] * len(inputargs) - nonfloatlocs = [-1] * len(inputargs) + floatlocs = [None] * len(inputargs) + nonfloatlocs = [None] * len(inputargs) # Don't use all_regs[0] for passing arguments around a loop. # Must be kept in sync with consider_jump(). # XXX this should probably go to llsupport/regalloc.py @@ -412,12 +385,17 @@ consider_guard_isnull = _consider_guard def consider_finish(self, op, ignored): - locs = [self.loc(v) for v in op.args] - locs_are_ref = [v.type == REF for v in op.args] + # xxx assumes no duplicate Boxes in op.args + tmpvar = TempBox() + tmpreg = self.rm.force_allocate_reg(tmpvar) + for i in range(len(op.args)): + v = op.args[i] + loc = self.make_sure_var_in_reg(v) + self.assembler.generate_failure_save(v, loc, i, tmpreg) + self.possibly_free_var(v) fail_index = self.assembler.cpu.get_fail_descr_number(op.descr) - self.assembler.generate_failure(self.assembler.mc, fail_index, locs, - self.exc, locs_are_ref) - self.possibly_free_vars(op.args) + self.assembler.generate_failure(fail_index, self.exc) + self.rm.possibly_free_var(tmpvar) def consider_guard_no_exception(self, op, ignored): self.perform_guard(op, [], None) Added: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/regloc.py ============================================================================== --- (empty file) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/regloc.py Sun Dec 20 11:56:16 2009 @@ -0,0 +1,35 @@ +from pypy.jit.metainterp.history import AbstractValue +from pypy.jit.backend.x86 import rx86 + + +class AssemblerLocation(AbstractValue): + __slots__ = 'value' + _immutable_ = True + def _getregkey(self): + return self.value + +class StackLoc(AssemblerLocation): + _immutable_ = True + def __init__(self, position, ebp_offset): + assert ebp_offset < 0 # so no confusion with RegLoc.value + self.position = position + self.value = ebp_offset + def __repr__(self): + return '%d(%%ebp)' % (self.value,) + +class RegLoc(AssemblerLocation): + _immutable_ = True + def __init__(self, regnum, is_xmm): + assert regnum >= 0 + self.value = regnum + self.is_xmm = is_xmm + def __repr__(self): + if self.is_xmm: + return rx86.R.xmmnames[self.value] + else: + return rx86.R.names[self.value] + +REGLOCS = [RegLoc(i, is_xmm=False) for i in range(8)] +XMMREGLOCS = [RegLoc(i, is_xmm=True) for i in range(8)] +eax, ecx, edx, ebx, esp, ebp, esi, edi = REGLOCS +xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 = XMMREGLOCS Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/rx86.py Sun Dec 20 11:56:16 2009 @@ -1,5 +1,5 @@ import py -from pypy.rlib.rarithmetic import intmask, r_ulonglong +from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated from pypy.rlib.objectmodel import specialize from pypy.rlib.unroll import unrolling_iterable @@ -27,16 +27,6 @@ def fits_in_32bits(value): return -2147483648 <= value <= 2147483647 -def intmask32(value): - # extract the 32 lower bits of 'value', returning a regular signed int - # (it is negative if 'value' is negative according to the 32-bits repr) - return intmask(rffi.cast(rffi.INT, value)) - -def cast32to64(value): - # returns 'value' in the 32 lower bits of a 64-bit integer, - # with the remaining bits set to 0 (even if value is negative). - return r_ulonglong(rffi.cast(rffi.UINT, value)) - # ____________________________________________________________ # Emit a single char @@ -103,10 +93,22 @@ return encode_immediate, argnum, width, None # ____________________________________________________________ +# Emit an immediate displacement (relative to the cur insn) + +def encode_relative(mc, target, _, orbyte): + assert orbyte == 0 + offset = target - (mc.tell() + 4) + mc.writeimm32(offset) + return 0 + +def relative(argnum): + return encode_relative, argnum, None, None + +# ____________________________________________________________ # Emit a mod/rm referencing a stack location [EBP+offset] @specialize.arg(2) -def encode_stack(mc, offset, force_32bits, orbyte): +def encode_stack_bp(mc, offset, force_32bits, orbyte): if not force_32bits and single_byte(offset): mc.writechar(chr(0x40 | orbyte | R.ebp)) mc.writeimm8(offset) @@ -116,23 +118,39 @@ mc.writeimm32(offset) return 0 -def stack(argnum, force_32bits=False): - return encode_stack, argnum, force_32bits, None +def stack_bp(argnum, force_32bits=False): + return encode_stack_bp, argnum, force_32bits, None + +# ____________________________________________________________ +# Emit a mod/rm referencing a stack location [ESP+offset] + +def encode_stack_sp(mc, offset, _, orbyte): + SIB = chr((R.esp<<3) | R.esp) # use [esp+(no index)+offset] + if offset == 0: + mc.writechar(chr(0x04 | orbyte)) + mc.writechar(SIB) + elif single_byte(offset): + mc.writechar(chr(0x44 | orbyte)) + mc.writechar(SIB) + mc.writeimm8(offset) + else: + assert fits_in_32bits(offset) + mc.writechar(chr(0x84 | orbyte)) + mc.writechar(SIB) + mc.writeimm32(offset) + return 0 + +def stack_sp(argnum): + return encode_stack_sp, argnum, None, None # ____________________________________________________________ # Emit a mod/rm referencing a memory location [reg1+offset] -def reg_offset(reg, offset): - # returns a 64-bits integer encoding "reg1+offset". - # * 'offset' is stored as bytes 1-4 of the result; - # * 'reg1' is stored as byte 5 of the result. +def encode_mem_reg_plus_const(mc, (reg, offset), _, orbyte): assert reg != R.esp and reg != R.ebp assert fits_in_32bits(offset) - return (r_ulonglong(reg) << 32) | cast32to64(offset) - -def encode_mem_reg_plus_const(mc, reg1_offset, _, orbyte): - reg1 = reg_number_3bits(mc, intmask(reg1_offset >> 32)) - offset = intmask32(reg1_offset) + # + reg1 = reg_number_3bits(mc, reg) no_offset = offset == 0 SIB = -1 # 64-bits special cases for reg1 == r12 or r13 @@ -156,9 +174,8 @@ mc.writeimm32(offset) return 0 -def rex_mem_reg_plus_const(mc, reg1_offset, _): - reg1 = intmask(reg1_offset >> 32) - if reg1 >= 8: +def rex_mem_reg_plus_const(mc, (reg, offset), _): + if reg >= 8: return REX_B return 0 @@ -168,39 +185,22 @@ # ____________________________________________________________ # Emit a mod/rm referencing an array memory location [reg1+reg2*scale+offset] -def reg_reg_scaleshift_offset(reg1, reg2, scaleshift, offset): - # returns a 64-bits integer encoding "reg1+reg2<= 8: - encoding |= REX_B << 8 - reg1 &= 7 - if reg2 >= 8: - encoding |= REX_X << 8 - reg2 &= 7 - encoding |= (scaleshift<<6) | (reg2<<3) | reg1 - return (r_ulonglong(encoding) << 32) | cast32to64(offset) - -def encode_mem_reg_plus_scaled_reg_plus_const(mc, reg1_reg2_scaleshift_offset, - _, orbyte): - encoding = intmask(reg1_reg2_scaleshift_offset >> 32) - if mc.WORD == 4: - assert encoding <= 0xFF # else registers r8..r15 have been used - SIB = chr(encoding) - else: - SIB = chr(encoding & 0xFF) - offset = intmask32(reg1_reg2_scaleshift_offset) + reg1 = reg_number_3bits(mc, reg1) + reg2 = reg_number_3bits(mc, reg2) + SIB = chr((scaleshift<<6) | (reg2<<3) | reg1) + # no_offset = offset == 0 # 64-bits special case for reg1 == r13 # (which look like ebp after being truncated to 3 bits) if mc.WORD == 8: - if (encoding & 7) == R.ebp: + if reg1 == R.ebp: no_offset = False # end of 64-bits special case if no_offset: @@ -216,8 +216,13 @@ mc.writeimm32(offset) return 0 -def rex_mem_reg_plus_scaled_reg_plus_const(mc, reg1_reg2_scaleshift_offset, _): - return intmask(reg1_reg2_scaleshift_offset >> (32+8)) +def rex_mem_reg_plus_scaled_reg_plus_const(mc, + (reg1, reg2, scaleshift, offset), + _): + rex = 0 + if reg1 >= 8: rex |= REX_B + if reg2 >= 8: rex |= REX_X + return rex def mem_reg_plus_scaled_reg_plus_const(argnum): return (encode_mem_reg_plus_scaled_reg_plus_const, argnum, None, @@ -232,34 +237,36 @@ REX_B = 1 @specialize.arg(2) -def encode_rex(mc, _, basevalue, orbyte): +def encode_rex(mc, rexbyte, basevalue, orbyte): if mc.WORD == 8: - assert 0 <= orbyte < 8 - if basevalue != 0x40 or orbyte != 0: - mc.writechar(chr(basevalue | orbyte)) + assert 0 <= rexbyte < 8 + if basevalue != 0x40 or rexbyte != 0: + mc.writechar(chr(basevalue | rexbyte)) else: - assert orbyte == 0 + assert rexbyte == 0 return 0 -rex_w = encode_rex, None, (0x40 | REX_W), None -rex_nw = encode_rex, None, 0x40, None +rex_w = encode_rex, 0, (0x40 | REX_W), None +rex_nw = encode_rex, 0, 0x40, None # ____________________________________________________________ def insn(*encoding): def encode(mc, *args): - orbyte = 0 + rexbyte = 0 if mc.WORD == 8: # compute the REX byte, if any for encode_step, arg, extra, rex_step in encoding_steps: if rex_step: if arg is not None: arg = args[arg-1] - orbyte |= rex_step(mc, arg, extra) + rexbyte |= rex_step(mc, arg, extra) + args = (rexbyte,) + args # emit the bytes of the instruction + orbyte = 0 for encode_step, arg, extra, rex_step in encoding_steps: if arg is not None: - arg = args[arg-1] + arg = args[arg] orbyte = encode_step(mc, arg, extra, orbyte) assert orbyte == 0 # @@ -274,22 +281,27 @@ encoding_steps = unrolling_iterable(encoding_steps) return encode +def xmminsn(*encoding): + encode = insn(*encoding) + encode.is_xmm_insn = True + return encode + def common_modes(group): base = group * 8 - INSN_ri8 = insn(rex_w, '\x83', orbyte(group<<3), register(1), '\xC0', - immediate(2,'b')) - INSN_ri32 = insn(rex_w, '\x81', orbyte(group<<3), register(1), '\xC0', - immediate(2)) + char = chr(0xC0 | base) + INSN_ri8 = insn(rex_w, '\x83', register(1), char, immediate(2,'b')) + INSN_ri32 = insn(rex_w, '\x81', register(1), char, immediate(2)) INSN_rr = insn(rex_w, chr(base+1), register(2,8), register(1,1), '\xC0') - INSN_rs = insn(rex_w, chr(base+3), register(1,8), stack(2)) + INSN_rb = insn(rex_w, chr(base+3), register(1,8), stack_bp(2)) def INSN_ri(mc, reg, immed): if single_byte(immed): INSN_ri8(mc, reg, immed) else: INSN_ri32(mc, reg, immed) + INSN_ri._always_inline_ = True # try to constant-fold single_byte() - return INSN_ri, INSN_rr, INSN_rs + return INSN_ri, INSN_rr, INSN_rb # ____________________________________________________________ @@ -314,16 +326,18 @@ self.writechar(chr((imm >> 16) & 0xFF)) self.writechar(chr((imm >> 24) & 0xFF)) + # ------------------------------ MOV ------------------------------ + MOV_ri = insn(rex_w, register(1), '\xB8', immediate(2, 'q')) - #MOV_si = insn(rex_w, '\xC7', orbyte(0<<3), stack(1), immediate(2)) MOV_rr = insn(rex_w, '\x89', register(2,8), register(1), '\xC0') - MOV_sr = insn(rex_w, '\x89', register(2,8), stack(1)) - MOV_rs = insn(rex_w, '\x8B', register(1,8), stack(2)) + MOV_br = insn(rex_w, '\x89', register(2,8), stack_bp(1)) + MOV_rb = insn(rex_w, '\x8B', register(1,8), stack_bp(2)) # "MOV reg1, [reg2+offset]" and the opposite direction MOV_rm = insn(rex_w, '\x8B', register(1,8), mem_reg_plus_const(2)) MOV_mr = insn(rex_w, '\x89', register(2,8), mem_reg_plus_const(1)) - #MOV_mi = insn(rex_w, '\xC7', mem_reg_plus_const(1), immediate(2)) + MOV_mi = insn(rex_w, '\xC7', orbyte(0<<3), mem_reg_plus_const(1), + immediate(2, 'i')) # "MOV reg1, [reg2+reg3*scale+offset]" and the opposite direction MOV_ra = insn(rex_w, '\x8B', register(1,8), @@ -335,20 +349,46 @@ MOV_rj = insn(rex_w, '\x8B', register(1,8), '\x05', immediate(2)) MOV_jr = insn(rex_w, '\x89', register(2,8), '\x05', immediate(1)) - ADD_ri, ADD_rr, ADD_rs = common_modes(0) - OR_ri, OR_rr, OR_rs = common_modes(1) - AND_ri, AND_rr, AND_rs = common_modes(4) - SUB_ri, SUB_rr, SUB_rs = common_modes(5) - XOR_ri, XOR_rr, XOR_rs = common_modes(6) - CMP_ri, CMP_rr, CMP_rs = common_modes(7) + # ------------------------------ Arithmetic ------------------------------ + + ADD_ri, ADD_rr, ADD_rb = common_modes(0) + OR_ri, OR_rr, OR_rb = common_modes(1) + AND_ri, AND_rr, AND_rb = common_modes(4) + SUB_ri, SUB_rr, SUB_rb = common_modes(5) + XOR_ri, XOR_rr, XOR_rb = common_modes(6) + CMP_ri, CMP_rr, CMP_rb = common_modes(7) + + # ------------------------------ Misc stuff ------------------------------ NOP = insn('\x90') RET = insn('\xC3') PUSH_r = insn(rex_nw, register(1), '\x50') + POP_r = insn(rex_nw, register(1), '\x58') + + LEA_rb = insn(rex_w, '\x8D', register(1,8), stack_bp(2)) + LEA32_rb = insn(rex_w, '\x8D', register(1,8),stack_bp(2,force_32bits=True)) + + CALL_l = insn('\xE8', relative(1)) + CALL_r = insn(rex_nw, '\xFF', register(1), chr(0xC0 | (2<<3))) + CALL_b = insn('\xFF', orbyte(2<<3), stack_bp(1)) + + XCHG_rm = insn(rex_w, '\x87', register(1,8), mem_reg_plus_const(2)) + + # ------------------------------ SSE2 ------------------------------ + + MOVSD_rr = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), register(2), + '\xC0') + MOVSD_rb = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), stack_bp(2)) + MOVSD_br = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2,8), stack_bp(1)) + MOVSD_rs = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), stack_sp(2)) + MOVSD_sr = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2,8), stack_sp(1)) + MOVSD_rm = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), + mem_reg_plus_const(2)) + MOVSD_mr = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2,8), + mem_reg_plus_const(1)) - LEA_rs = insn(rex_w, '\x8D', register(1,8), stack(2)) - LEA32_rs = insn(rex_w, '\x8D', register(1,8), stack(2, force_32bits=True)) + # ------------------------------------------------------------ class X86_32_CodeBuilder(AbstractX86CodeBuilder): @@ -359,7 +399,8 @@ WORD = 8 def writeimm64(self, imm): - self.writeimm32(intmask32(imm)) + imm32 = intmask(rffi.cast(rffi.INT, imm)) + self.writeimm32(imm32) self.writeimm32(imm >> 32) # MOV_ri from the parent class is not wrong, but here is a better encoding @@ -372,6 +413,16 @@ else: AbstractX86CodeBuilder.MOV_ri(self, reg, immed) + # case of a 64-bit immediate: encode via RAX (assuming it's ok to + # randomly change this register at that point in time) + def CALL_l(self, target): + offset = target - (self.tell() + 5) + if fits_in_32bits(offset): + AbstractX86CodeBuilder.CALL_l(self, target) + else: + AbstractX86CodeBuilder.MOV_ri(self, R.eax, target) + AbstractX86CodeBuilder.CALL_r(self, R.eax) + # unsupported -- must use e.g. MOV tmpreg, immed64; MOV reg, [tmpreg] def MOV_rj(self, reg, mem_immed): py.test.skip("MOV_rj unsupported") Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/support.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/support.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/support.py Sun Dec 20 11:56:16 2009 @@ -1,4 +1,4 @@ -from pypy.rpython.lltypesystem import lltype, rffi, llmemory +from pypy.rpython.lltypesystem import lltype, rffi def values_array(TP, size): ATP = lltype.GcArray(TP) @@ -11,6 +11,9 @@ return rffi.cast(lltype.Signed, lltype.direct_ptradd( lltype.direct_arrayitems(self.ar), i)) + def get_base_addr(self): + return rffi.cast(lltype.Signed, lltype.direct_arrayitems(self.ar)) + def setitem(self, i, v): self.ar[i] = v Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_jump.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_jump.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_jump.py Sun Dec 20 11:56:16 2009 @@ -1,7 +1,7 @@ from pypy.jit.metainterp.history import ConstInt from pypy.jit.backend.x86.regalloc import X86FrameManager -from pypy.jit.backend.x86.regalloc import StackLoc, RegLoc -from pypy.jit.backend.x86.regalloc import eax, ebx, ecx, edx, esi, edi +from pypy.jit.backend.x86.regloc import StackLoc, RegLoc +from pypy.jit.backend.x86.regloc import eax, ebx, ecx, edx, esi, edi from pypy.jit.backend.x86.jump import remap_frame_layout frame_pos = X86FrameManager.frame_pos Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86.py Sun Dec 20 11:56:16 2009 @@ -1,4 +1,4 @@ -import py +import py, struct from pypy.jit.backend.x86.rx86 import * globals().update(R.__dict__) @@ -7,11 +7,15 @@ self.buffer = [] def writechar(self, c): + assert isinstance(c, str) and len(c) == 1 self.buffer.append(c) # append a character def getvalue(self): return ''.join(self.buffer) + def tell(self): + return 0x76543210 + len(self.buffer) + class CodeBuilder32(CodeBuilderMixin, X86_32_CodeBuilder): pass @@ -26,44 +30,44 @@ s.MOV_rr(ebx, ebp) assert s.getvalue() == '\x89\xEB' -def test_mov_sr(): +def test_mov_br(): s = CodeBuilder32() - s.MOV_sr(-36, edx) + s.MOV_br(-36, edx) assert s.getvalue() == '\x89\x55\xDC' -def test_mov_rs(): +def test_mov_rb(): s = CodeBuilder32() - s.MOV_rs(edx, -36) + s.MOV_rb(edx, -36) assert s.getvalue() == '\x8B\x55\xDC' def test_mov_rm(): s = CodeBuilder32() - s.MOV_rm(edx, reg_offset(edi, 0)) - s.MOV_rm(edx, reg_offset(edi, -128)) - s.MOV_rm(edx, reg_offset(edi, 128)) + s.MOV_rm(edx, (edi, 0)) + s.MOV_rm(edx, (edi, -128)) + s.MOV_rm(edx, (edi, 128)) assert s.getvalue() == '\x8B\x17\x8B\x57\x80\x8B\x97\x80\x00\x00\x00' def test_mov_mr(): s = CodeBuilder32() - s.MOV_mr(reg_offset(edi, 0), edx) - s.MOV_mr(reg_offset(edi, -128), edx) - s.MOV_mr(reg_offset(edi, 128), edx) + s.MOV_mr((edi, 0), edx) + s.MOV_mr((edi, -128), edx) + s.MOV_mr((edi, 128), edx) assert s.getvalue() == '\x89\x17\x89\x57\x80\x89\x97\x80\x00\x00\x00' def test_mov_ra(): s = CodeBuilder32() - s.MOV_ra(edx, reg_reg_scaleshift_offset(esi, edi, 2, 0)) - s.MOV_ra(edx, reg_reg_scaleshift_offset(esi, edi, 2, -128)) - s.MOV_ra(edx, reg_reg_scaleshift_offset(esi, edi, 2, 128)) + s.MOV_ra(edx, (esi, edi, 2, 0)) + s.MOV_ra(edx, (esi, edi, 2, -128)) + s.MOV_ra(edx, (esi, edi, 2, 128)) assert s.getvalue() == ('\x8B\x14\xBE' + '\x8B\x54\xBE\x80' + '\x8B\x94\xBE\x80\x00\x00\x00') def test_mov_ar(): s = CodeBuilder32() - s.MOV_ar(reg_reg_scaleshift_offset(esi, edi, 2, 0), edx) - s.MOV_ar(reg_reg_scaleshift_offset(esi, edi, 2, -128), edx) - s.MOV_ar(reg_reg_scaleshift_offset(esi, edi, 2, 128), edx) + s.MOV_ar((esi, edi, 2, 0), edx) + s.MOV_ar((esi, edi, 2, -128), edx) + s.MOV_ar((esi, edi, 2, 128), edx) assert s.getvalue() == ('\x89\x14\xBE' + '\x89\x54\xBE\x80' + '\x89\x94\xBE\x80\x00\x00\x00') @@ -74,16 +78,22 @@ s.ADD_rr(eax, eax) assert s.getvalue() == '\x90\x01\xC0' -def test_lea_rs(): +def test_lea_rb(): s = CodeBuilder32() - s.LEA_rs(ecx, -36) + s.LEA_rb(ecx, -36) assert s.getvalue() == '\x8D\x4D\xDC' -def test_lea32_rs(): +def test_lea32_rb(): s = CodeBuilder32() - s.LEA32_rs(ecx, -36) + s.LEA32_rb(ecx, -36) assert s.getvalue() == '\x8D\x8D\xDC\xFF\xFF\xFF' +def test_call_l(s=None): + s = s or CodeBuilder32() + s.CALL_l(0x01234567) + ofs = 0x01234567 - (0x76543210+5) + assert s.getvalue() == '\xE8' + struct.pack("> 32) - offset = intmask(reg1_offset) + def assembler_operand_stack_sp(self, position): + return '%d(%s)' % (position, self.REGNAMES[4]) + + def assembler_operand_memory(self, (reg1, offset)): if not offset: offset = '' return '%s(%s)' % (offset, self.REGNAMES[reg1]) - def assembler_operand_array(self, reg1_reg2_scaleshift_offset): - SIB = intmask(reg1_reg2_scaleshift_offset >> 32) - rex = SIB >> 8 - SIB = SIB & 0xFF - offset = intmask(reg1_reg2_scaleshift_offset) + def assembler_operand_array(self, (reg1, reg2, scaleshift, offset)): if not offset: offset = '' - reg1 = SIB & 7 - reg2 = (SIB >> 3) & 7 - scaleshift = SIB >> 6 - if rex & rx86.REX_B: - reg1 |= 8 - if rex & rx86.REX_X: - reg2 |= 8 return '%s(%s,%s,%d)' % (offset, self.REGNAMES[reg1], self.REGNAMES[reg2], 1< Author: arigo Date: Sun Dec 20 14:19:50 2009 New Revision: 70216 Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/assembler.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/codebuf.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/regloc.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/rx86.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py Log: Progress. Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/assembler.py Sun Dec 20 14:19:50 2009 @@ -17,8 +17,7 @@ xmm4, xmm5, xmm6, xmm7, RegLoc, StackLoc) from pypy.rlib.objectmodel import we_are_translated, specialize -from pypy.jit.backend.x86 import rx86 -from pypy.jit.backend.x86 import codebuf +from pypy.jit.backend.x86 import rx86, regloc, codebuf from pypy.jit.metainterp.resoperation import rop from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print @@ -72,7 +71,7 @@ method.func_name = name return method -for _name in rx86.all_instructions: +for _name in rx86.all_instructions + regloc.all_extra_instructions: setattr(MachineCodeBlockWrapper, _name, _new_method(_name)) class Assembler386(object): @@ -180,8 +179,8 @@ arglocs = self.rebuild_faillocs_from_descr( faildescr._x86_failure_recovery_bytecode) if not we_are_translated(): - assert ([loc.assembler() for loc in arglocs] == - [loc.assembler() for loc in faildescr._x86_debug_faillocs]) + assert ([repr(loc) for loc in arglocs] == + [repr(loc) for loc in faildescr._x86_debug_faillocs]) regalloc = RegAlloc(self, self.cpu.translate_support_code) fail_depths = faildescr._x86_current_depths regalloc.prepare_bridge(fail_depths, inputargs, arglocs, @@ -239,11 +238,11 @@ def _assemble_bootstrap_code(self, inputargs, arglocs): nonfloatlocs, floatlocs = arglocs - self.mc.PUSH_r(ebp.value) + self.mc.PUSH(ebp) self.mc.MOV_rr(ebp.value, esp.value) - self.mc.PUSH_r(ebx.value) - self.mc.PUSH_r(esi.value) - self.mc.PUSH_r(edi.value) + self.mc.PUSH(ebx) + self.mc.PUSH(esi) + self.mc.PUSH(edi) # NB. the shape of the frame is hard-coded in get_basic_shape() too. # Also, make sure this is consistent with FRAME_FIXED_SIZE. adr_stackadjust = self._patchable_stackadjust() @@ -432,25 +431,29 @@ getattr(self.mc, 'SET' + cond)(lower_byte(result_loc)) return genop_cmp - def _cmpop_guard(cond, rev_cond, false_cond, false_rev_cond): + def _cmpop_guard(cond, rev_cond): + cond = rx86.Conditions[cond] + rev_cond = rx86.Conditions[rev_cond] + false_cond = cond ^ 1 + false_rev_cond = rev_cond ^ 1 + # def genop_cmp_guard(self, op, guard_op, addr, arglocs, result_loc): guard_opnum = guard_op.opnum - if isinstance(op.args[0], Const): - self.mc.CMP(arglocs[1], arglocs[0]) + loc0 = arglocs[0] + loc1 = arglocs[1] + if isinstance(loc0, Const): + self.mc.CMP_IMM(loc1, loc0) if guard_opnum == rop.GUARD_FALSE: - name = 'J' + rev_cond - return self.implement_guard(addr, getattr(self.mc, name)) + c = rev_cond else: - name = 'J' + false_rev_cond - return self.implement_guard(addr, getattr(self.mc, name)) + c = false_rev_cond else: - self.mc.CMP(arglocs[0], arglocs[1]) + self.mc.CMP(loc0, loc1) if guard_opnum == rop.GUARD_FALSE: - name = 'J' + cond - return self.implement_guard(addr, getattr(self.mc, name)) + c = cond else: - name = 'J' + false_cond - return self.implement_guard(addr, getattr(self.mc, name)) + c = false_cond + return self.implement_guard(addr, c) return genop_cmp_guard ## XXX redo me @@ -529,17 +532,17 @@ genop_uint_le = _cmpop("BE", "AE") genop_uint_ge = _cmpop("AE", "BE") - genop_guard_int_lt = _cmpop_guard("L", "G", "GE", "LE") - genop_guard_int_le = _cmpop_guard("LE", "GE", "G", "L") - genop_guard_int_eq = _cmpop_guard("E", "E", "NE", "NE") - genop_guard_int_ne = _cmpop_guard("NE", "NE", "E", "E") - genop_guard_int_gt = _cmpop_guard("G", "L", "LE", "GE") - genop_guard_int_ge = _cmpop_guard("GE", "LE", "L", "G") - - genop_guard_uint_gt = _cmpop_guard("A", "B", "BE", "AE") - genop_guard_uint_lt = _cmpop_guard("B", "A", "AE", "BE") - genop_guard_uint_le = _cmpop_guard("BE", "AE", "A", "B") - genop_guard_uint_ge = _cmpop_guard("AE", "BE", "B", "A") + genop_guard_int_lt = _cmpop_guard("L", "G") + genop_guard_int_le = _cmpop_guard("LE", "GE") + genop_guard_int_eq = _cmpop_guard("E", "E") + genop_guard_int_ne = _cmpop_guard("NE", "NE") + genop_guard_int_gt = _cmpop_guard("G", "L") + genop_guard_int_ge = _cmpop_guard("GE", "LE") + + genop_guard_uint_gt = _cmpop_guard("A", "B") + genop_guard_uint_lt = _cmpop_guard("B", "A") + genop_guard_uint_le = _cmpop_guard("BE", "AE") + genop_guard_uint_ge = _cmpop_guard("AE", "BE") def genop_float_neg(self, op, arglocs, resloc): # Following what gcc does: res = x ^ 0x8000000000000000 @@ -960,9 +963,9 @@ kind = code & 3 code >>= 2 if kind == self.DESCR_FLOAT: - loc = xmm_registers[code] + loc = regloc.XMMREGLOCS[code] else: - loc = registers[code] + loc = regloc.REGLOCS[code] arglocs.append(loc) return arglocs[:] @@ -1001,7 +1004,7 @@ def grab_frame_values(self, bytecode, frame_addr, allregisters): # no malloc allowed here!! - self.fail_ebp = allregisters[16 + ebp.op] + self.fail_ebp = allregisters[16 + ebp.value] num = 0 value_hi = 0 while 1: @@ -1068,7 +1071,7 @@ # original value of the registers, optionally the original # value of XMM registers, and finally a reference to the # recovery bytecode. See _build_failure_recovery() for details. - stack_at_ebp = registers[ebp.op] + stack_at_ebp = registers[ebp.value] bytecode = rffi.cast(rffi.UCHARP, registers[8]) allregisters = rffi.ptradd(registers, -16) return self.grab_frame_values(bytecode, stack_at_ebp, allregisters) @@ -1088,14 +1091,14 @@ # Assume that we are called at the beginning, when there is no risk # that 'mc' runs out of space. Checked by asserts in mc.write(). recovery_addr = mc.tell() - mc.PUSH_r(edi.value) - mc.PUSH_r(esi.value) - mc.PUSH_r(ebp.value) - mc.PUSH_r(esp.value) # <-- not really used, but needed to take - mc.PUSH_r(ebx.value) # up the space - mc.PUSH_r(edx.value) - mc.PUSH_r(ecx.value) - mc.PUSH_r(eax.value) + mc.PUSH(edi) + mc.PUSH(esi) + mc.PUSH(ebp) + mc.PUSH(esp) # <-- not really used, but needed to take up the space + mc.PUSH(ebx) + mc.PUSH(edx) + mc.PUSH(ecx) + mc.PUSH(eax) mc.MOV_rr(esi.value, esp.value) if withfloats: mc.SUB_ri(esp.value, 8*8) @@ -1117,7 +1120,7 @@ # the XMM registers. Moreover, esi[8] is a pointer to the recovery # bytecode, pushed just before by the CALL instruction written by # generate_quick_failure(). - mc.PUSH_r(esi.value) + mc.PUSH(esi) mc.CALL_l(failure_recovery_func) # returns in eax the fail_index @@ -1125,10 +1128,10 @@ # _assemble_bootstrap_code(). The LEA below throws away most # of the frame, including all the PUSHes that we did just above. mc.LEA_rb(esp.value, -3 * WORD) - mc.POP_r(edi.value) # [ebp-12] - mc.POP_r(esi.value) # [ebp-8] - mc.POP_r(ebx.value) # [ebp-4] - mc.POP_r(ebp.value) # [ebp] + mc.POP(edi) # [ebp-12] + mc.POP(esi) # [ebp-8] + mc.POP(ebx) # [ebp-4] + mc.POP(ebp) # [ebp] mc.RET() self.mc2.done() self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr @@ -1140,11 +1143,10 @@ if isinstance(loc, RegLoc): self.mc.MOVSD_mr((tmpreg.value, 0), loc.value) else: - assert isinstance(loc, ConstInt) - value = rffi.cast(rffi.INTP, loc.value)[0] - self.mc.MOV_mi((tmpreg.value, 0), value) - value = rffi.cast(rffi.INTP, loc.value)[1] - self.mc.MOV_mi((tmpreg.value, 4), value) + assert isinstance(loc, Const) + src = rffi.cast(rffi.INTP, loc.getint()) + self.mc.MOV_mi((tmpreg.value, 0), src[0]) + self.mc.MOV_mi((tmpreg.value, 4), src[1]) else: if v.type == REF: destadr = self.fail_boxes_ref.get_addr_for_num(i) @@ -1154,8 +1156,8 @@ if isinstance(loc, RegLoc): self.mc.MOV_mr((tmpreg.value, 0), loc.value) else: - assert isinstance(loc, ConstInt) - self.mc.MOV_mi((tmpreg.value, 0), loc.value) + assert isinstance(loc, Const) + self.mc.MOV_mi((tmpreg.value, 0), loc.getint()) def generate_failure(self, fail_index, exc): # avoid breaking the following code sequence, as we are almost done @@ -1170,15 +1172,14 @@ # mc.LEA_rb(esp.value, -3 * WORD) mc.MOV_ri(eax.value, fail_index) - mc.POP_r(edi.value) # [ebp-12] - mc.POP_r(esi.value) # [ebp-8] - mc.POP_r(ebx.value) # [ebp-4] - mc.POP_r(ebp.value) # [ebp] + mc.POP(edi) # [ebp-12] + mc.POP(esi) # [ebp-8] + mc.POP(ebx) # [ebp-4] + mc.POP(ebp) # [ebp] mc.RET() - @specialize.arg(2) - def implement_guard(self, addr, emit_jump): - emit_jump(rel32(addr)) + def implement_guard(self, addr, emit_jump_index): + self.mc.J_il(emit_jump_index, addr) return self.mc.tell() - 4 def genop_call(self, op, arglocs, resloc): @@ -1273,7 +1274,7 @@ return loop_token._x86_arglocs def closing_jump(self, loop_token): - self.mc.JMP(rel32(loop_token._x86_loop_code)) + self.mc.JMP_l(loop_token._x86_loop_code) def malloc_cond_fixedsize(self, nursery_free_adr, nursery_top_adr, size, tid, slowpath_addr): Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/codebuf.py Sun Dec 20 14:19:50 2009 @@ -3,11 +3,12 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.jit.backend.x86.rx86 import X86_32_CodeBuilder +from pypy.jit.backend.x86.regloc import LocationCodeBuilder from pypy.rlib.rmmap import PTR, alloc, free from pypy.rlib.debug import make_sure_not_resized -class InMemoryCodeBuilder(X86_32_CodeBuilder): +class InMemoryCodeBuilder(X86_32_CodeBuilder, LocationCodeBuilder): _last_dump_start = 0 def __init__(self, start, end): Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/regloc.py Sun Dec 20 14:19:50 2009 @@ -1,6 +1,12 @@ from pypy.jit.metainterp.history import AbstractValue from pypy.jit.backend.x86 import rx86 +# +# This module adds support for "locations", which can be either in a Const, +# or a RegLoc or a StackLoc. It also adds operations like mc.ADD(), which +# take two locations as arguments, decode them, and calls the right +# mc.ADD_rr()/ADD_rb()/ADD_ri(). +# class AssemblerLocation(AbstractValue): __slots__ = 'value' @@ -33,3 +39,64 @@ XMMREGLOCS = [RegLoc(i, is_xmm=True) for i in range(8)] eax, ecx, edx, ebx, esp, ebp, esi, edi = REGLOCS xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 = XMMREGLOCS + + +class LocationCodeBuilder(object): + _mixin_ = True + + def _binaryop(name): + def INSN(self, loc1, loc2): + assert isinstance(loc1, RegLoc) + val1 = loc1.value + if isinstance(loc2, RegLoc): + getattr(self, name + '_rr')(val1, loc2.value) + elif isinstance(loc2, StackLoc): + getattr(self, name + '_rb')(val1, loc2.value) + else: + getattr(self, name + '_ri')(val1, loc2.getint()) + return INSN + + ADD = _binaryop('ADD') + OR = _binaryop('OR') + AND = _binaryop('AND') + SUB = _binaryop('SUB') + XOR = _binaryop('XOR') + + def PUSH(self, loc): + assert isinstance(loc, RegLoc) + self.PUSH_r(loc.value) + + def POP(self, loc): + assert isinstance(loc, RegLoc) + self.POP_r(loc.value) + + def CMP(self, loc0, loc1): + if isinstance(loc0, RegLoc): + val0 = loc0.value + if isinstance(loc1, RegLoc): + self.CMP_rr(val0, loc1.value) + elif isinstance(loc1, StackLoc): + self.CMP_rb(val0, loc1.value) + else: + self.CMP_ri(val0, loc1.getint()) + else: + assert isinstance(loc0, StackLoc) + val0 = loc0.value + if isinstance(loc1, RegLoc): + self.CMP_br(val0, loc1.value) + else: + self.CMP_bi(val0, loc1.getint()) + + def CMPi(self, loc0, loc1): + # like CMP, but optimized for the case of loc1 being a Const + assert isinstance(loc1, Const) + if isinstance(loc0, RegLoc): + self.CMP_ri(loc0.value, loc1.getint()) + else: + assert isinstance(loc0, StackLoc) + self.CMP_bi(loc0.value, loc1.getint()) + + +all_extra_instructions = [name for name in LocationCodeBuilder.__dict__ + if name[0].isupper()] +all_extra_instructions.sort() Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/rx86.py Sun Dec 20 14:19:50 2009 @@ -1,5 +1,4 @@ import py -from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated from pypy.rlib.objectmodel import specialize from pypy.rlib.unroll import unrolling_iterable @@ -83,6 +82,8 @@ mc.writeimm8(immediate) elif width == 'h': mc.writeimm16(immediate) + elif width == 'o': + return immediate # in the 'orbyte' for the next command elif width == 'q' and mc.WORD == 8: mc.writeimm64(immediate) else: @@ -113,7 +114,6 @@ mc.writechar(chr(0x40 | orbyte | R.ebp)) mc.writeimm8(offset) else: - assert fits_in_32bits(offset) mc.writechar(chr(0x80 | orbyte | R.ebp)) mc.writeimm32(offset) return 0 @@ -134,7 +134,6 @@ mc.writechar(SIB) mc.writeimm8(offset) else: - assert fits_in_32bits(offset) mc.writechar(chr(0x84 | orbyte)) mc.writechar(SIB) mc.writeimm32(offset) @@ -148,7 +147,6 @@ def encode_mem_reg_plus_const(mc, (reg, offset), _, orbyte): assert reg != R.esp and reg != R.ebp - assert fits_in_32bits(offset) # reg1 = reg_number_3bits(mc, reg) no_offset = offset == 0 @@ -191,7 +189,6 @@ # emit "reg1 + (reg2 << scaleshift) + offset" assert reg1 != R.ebp and reg2 != R.esp assert 0 <= scaleshift < 4 - assert fits_in_32bits(offset) reg1 = reg_number_3bits(mc, reg1) reg2 = reg_number_3bits(mc, reg2) SIB = chr((scaleshift<<6) | (reg2<<3) | reg1) @@ -290,9 +287,12 @@ base = group * 8 char = chr(0xC0 | base) INSN_ri8 = insn(rex_w, '\x83', register(1), char, immediate(2,'b')) - INSN_ri32 = insn(rex_w, '\x81', register(1), char, immediate(2)) + INSN_ri32= insn(rex_w, '\x81', register(1), char, immediate(2)) INSN_rr = insn(rex_w, chr(base+1), register(2,8), register(1,1), '\xC0') + INSN_br = insn(rex_w, chr(base+1), register(2,8), stack_bp(1)) INSN_rb = insn(rex_w, chr(base+3), register(1,8), stack_bp(2)) + INSN_bi8 = insn(rex_w, '\x83', orbyte(base), stack_bp(1), immediate(2,'b')) + INSN_bi32= insn(rex_w, '\x81', orbyte(base), stack_bp(1), immediate(2)) def INSN_ri(mc, reg, immed): if single_byte(immed): @@ -301,7 +301,14 @@ INSN_ri32(mc, reg, immed) INSN_ri._always_inline_ = True # try to constant-fold single_byte() - return INSN_ri, INSN_rr, INSN_rb + def INSN_bi(mc, offset, immed): + if single_byte(immed): + INSN_bi8(mc, offset, immed) + else: + INSN_bi32(mc, offset, immed) + INSN_bi._always_inline_ = True # try to constant-fold single_byte() + + return INSN_ri, INSN_rr, INSN_rb, INSN_bi, INSN_br # ____________________________________________________________ @@ -351,12 +358,12 @@ # ------------------------------ Arithmetic ------------------------------ - ADD_ri, ADD_rr, ADD_rb = common_modes(0) - OR_ri, OR_rr, OR_rb = common_modes(1) - AND_ri, AND_rr, AND_rb = common_modes(4) - SUB_ri, SUB_rr, SUB_rb = common_modes(5) - XOR_ri, XOR_rr, XOR_rb = common_modes(6) - CMP_ri, CMP_rr, CMP_rb = common_modes(7) + ADD_ri, ADD_rr, ADD_rb, _, _ = common_modes(0) + OR_ri, OR_rr, OR_rb, _, _ = common_modes(1) + AND_ri, AND_rr, AND_rb, _, _ = common_modes(4) + SUB_ri, SUB_rr, SUB_rb, _, _ = common_modes(5) + XOR_ri, XOR_rr, XOR_rb, _, _ = common_modes(6) + CMP_ri, CMP_rr, CMP_rb, CMP_bi, CMP_br = common_modes(7) # ------------------------------ Misc stuff ------------------------------ @@ -375,6 +382,10 @@ XCHG_rm = insn(rex_w, '\x87', register(1,8), mem_reg_plus_const(2)) + JMP_l = insn('\xE9', relative(1)) + J_il = insn('\x0F', immediate(1,'o'), '\x80', relative(2)) + SET_ir = insn('\x0F', immediate(1,'o'),'\x90', register(2), '\xC0') + # ------------------------------ SSE2 ------------------------------ MOVSD_rr = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), register(2), @@ -390,6 +401,25 @@ # ------------------------------------------------------------ +Conditions = { + 'O': 0, + 'NO': 1, + 'C': 2, 'B': 2, 'NAE': 2, + 'NC': 3, 'NB': 3, 'AE': 3, + 'Z': 4, 'E': 4, + 'NZ': 5, 'NE': 5, + 'BE': 6, 'NA': 6, + 'NBE': 7, 'A': 7, + 'S': 8, + 'NS': 9, + 'P': 10, 'PE': 10, + 'NP': 11, 'PO': 11, + 'L': 12, 'NGE': 12, + 'NL': 13, 'GE': 13, + 'LE': 14, 'NG': 14, + 'NLE': 15, 'G': 15, +} + class X86_32_CodeBuilder(AbstractX86CodeBuilder): WORD = 4 @@ -399,9 +429,14 @@ WORD = 8 def writeimm64(self, imm): - imm32 = intmask(rffi.cast(rffi.INT, imm)) - self.writeimm32(imm32) - self.writeimm32(imm >> 32) + self.writechar(chr(imm & 0xFF)) + self.writechar(chr((imm >> 8) & 0xFF)) + self.writechar(chr((imm >> 16) & 0xFF)) + self.writechar(chr((imm >> 24) & 0xFF)) + self.writechar(chr((imm >> 32) & 0xFF)) + self.writechar(chr((imm >> 40) & 0xFF)) + self.writechar(chr((imm >> 48) & 0xFF)) + self.writechar(chr((imm >> 56) & 0xFF)) # MOV_ri from the parent class is not wrong, but here is a better encoding # for the common case where the immediate fits in 32 bits Modified: pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86.py Sun Dec 20 14:19:50 2009 @@ -94,6 +94,23 @@ ofs = 0x01234567 - (0x76543210+5) assert s.getvalue() == '\xE8' + struct.pack(" Author: fijal Date: Sun Dec 20 21:44:32 2009 New Revision: 70218 Modified: pypy/trunk/pypy/tool/package.py Log: Make this 2.4 and 2.5 compatible Modified: pypy/trunk/pypy/tool/package.py ============================================================================== --- pypy/trunk/pypy/tool/package.py (original) +++ pypy/trunk/pypy/tool/package.py Sun Dec 20 21:44:32 2009 @@ -1,5 +1,8 @@ #!/usr/bin/env python -""" A sample script that packages PyPy, provided that it's already built +""" A sample script that packages PyPy, provided that it's already built. +Usage: + +package.py pypydir [name-of-archive] """ import autopath @@ -7,9 +10,22 @@ import sys import py import os +import fnmatch import tarfile from pypy.tool.udir import udir +def ignore_patterns(*patterns): + """Function that can be used as copytree() ignore parameter. + + Patterns is a sequence of glob-style patterns + that are used to exclude files""" + def _ignore_patterns(path, names): + ignored_names = [] + for pattern in patterns: + ignored_names.extend(fnmatch.filter(names, pattern)) + return set(ignored_names) + return _ignore_patterns + class PyPyCNotFound(Exception): pass @@ -22,11 +38,11 @@ pypydir = builddir.ensure("pypy", dir=True) shutil.copytree(str(basedir.join('lib-python')), str(pypydir.join('lib-python')), - ignore=shutil.ignore_patterns('.svn')) + ignore=ignore_patterns('.svn')) pypydir.ensure('pypy', dir=True) shutil.copytree(str(basedir.join('pypy', 'lib')), str(pypydir.join('pypy', 'lib')), - ignore=shutil.ignore_patterns('.svn')) + ignore=ignore_patterns('.svn')) pypydir.ensure('bin', dir=True) shutil.copy(str(pypy_c), str(pypydir.join('bin', 'pypy-c'))) old_dir = os.getcwd() @@ -39,4 +55,10 @@ return builddir # for tests if __name__ == '__main__': - main(sys.argv[1]) + if len(sys.argv) == 1 or len(sys.argv) > 3: + print >>sys.stderr, __doc__ + sys.exit(1) + elif len(sys.argv) == 2: + main(sys.argv[1]) + else: + main(sys.argv[1], sys.argv[2]) From arigo at codespeak.net Mon Dec 21 15:36:14 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 21 Dec 2009 15:36:14 +0100 (CET) Subject: [pypy-svn] r70227 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091221143614.8F02716801F@codespeak.net> Author: arigo Date: Mon Dec 21 15:36:13 2009 New Revision: 70227 Modified: pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py Log: Remove the field 'level', which is not used anywhere. Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Mon Dec 21 15:36:13 2009 @@ -21,15 +21,10 @@ self.boxes = boxes class FrameInfo(object): - __slots__ = ('prev', 'jitcode', 'pc', 'exception_target', 'level') + __slots__ = ('prev', 'jitcode', 'pc', 'exception_target') def __init__(self, prev, frame): self.prev = prev - if prev is None: - level = 1 - else: - level = prev.level + 1 - self.level = level self.jitcode = frame.jitcode self.pc = frame.pc self.exception_target = frame.exception_target Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Mon Dec 21 15:36:13 2009 @@ -175,7 +175,6 @@ assert fi.jitcode is jitcode assert fi.pc == 1 assert fi.exception_target == 2 - assert fi.level == 1 jitcode1 = "JITCODE1" frame1 = FakeFrame(jitcode, 3, 4) @@ -184,7 +183,6 @@ assert fi1.jitcode is jitcode assert fi1.pc == 3 assert fi1.exception_target == 4 - assert fi1.level == 2 def test_Numbering_create(): l = [1, 2] From arigo at codespeak.net Mon Dec 21 15:54:32 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 21 Dec 2009 15:54:32 +0100 (CET) Subject: [pypy-svn] r70228 - pypy/trunk/pypy/jit/backend/x86/test Message-ID: <20091221145432.7BFFE16801F@codespeak.net> Author: arigo Date: Mon Dec 21 15:54:31 2009 New Revision: 70228 Modified: pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py Log: Fix test. Modified: pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py Mon Dec 21 15:54:31 2009 @@ -28,6 +28,7 @@ class MockAssembler(object): gcrefs = None + _float_constants = None def __init__(self, cpu=None, gc_ll_descr=None): self.movs = [] From arigo at codespeak.net Mon Dec 21 15:59:23 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 21 Dec 2009 15:59:23 +0100 (CET) Subject: [pypy-svn] r70229 - pypy/trunk/pypy/tool Message-ID: <20091221145923.D758F16801F@codespeak.net> Author: arigo Date: Mon Dec 21 15:59:22 2009 New Revision: 70229 Modified: pypy/trunk/pypy/tool/package.py Log: Skip the tests corresponding to this file as long as they fail on Python 2.4 and 2.5. Modified: pypy/trunk/pypy/tool/package.py ============================================================================== --- pypy/trunk/pypy/tool/package.py (original) +++ pypy/trunk/pypy/tool/package.py Mon Dec 21 15:59:22 2009 @@ -14,6 +14,8 @@ import tarfile from pypy.tool.udir import udir +if sys.version_info < (2,6): py.test.skip("requires 2.6 so far") + def ignore_patterns(*patterns): """Function that can be used as copytree() ignore parameter. From arigo at codespeak.net Mon Dec 21 16:10:01 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 21 Dec 2009 16:10:01 +0100 (CET) Subject: [pypy-svn] r70230 - pypy/branch/virtual-forcing Message-ID: <20091221151001.43FCD16801F@codespeak.net> Author: arigo Date: Mon Dec 21 16:10:00 2009 New Revision: 70230 Removed: pypy/branch/virtual-forcing/ Log: Remove this branch, to be updated with a copy of trunk. From arigo at codespeak.net Mon Dec 21 16:11:49 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 21 Dec 2009 16:11:49 +0100 (CET) Subject: [pypy-svn] r70231 - pypy/branch/virtual-forcing Message-ID: <20091221151149.4DCA7168020@codespeak.net> Author: arigo Date: Mon Dec 21 16:11:48 2009 New Revision: 70231 Added: pypy/branch/virtual-forcing/ - copied from r70230, pypy/trunk/ Log: Make again a branch. From arigo at codespeak.net Mon Dec 21 16:13:05 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 21 Dec 2009 16:13:05 +0100 (CET) Subject: [pypy-svn] r70232 - in pypy/branch/virtual-forcing/pypy: interpreter interpreter/test jit/backend jit/backend/llgraph jit/backend/test jit/backend/x86 jit/backend/x86/test jit/metainterp jit/metainterp/test module/_stackless module/pypyjit module/sys objspace/flow rlib rlib/test rpython rpython/lltypesystem rpython/lltypesystem/test rpython/test translator/c translator/cli translator/jvm Message-ID: <20091221151305.448AB168020@codespeak.net> Author: arigo Date: Mon Dec 21 16:13:02 2009 New Revision: 70232 Added: pypy/branch/virtual-forcing/pypy/jit/backend/x86/test/test_virtualref.py - copied unchanged from r70229, pypy/branch/virtual-forcing/pypy/jit/backend/x86/test/test_virtualref.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py - copied unchanged from r70229, pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py - copied unchanged from r70229, pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualref.py pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py - copied unchanged from r70229, pypy/branch/virtual-forcing/pypy/rlib/_jit_vref.py pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py - copied unchanged from r70229, pypy/branch/virtual-forcing/pypy/rlib/test/test__jit_vref.py Modified: pypy/branch/virtual-forcing/pypy/interpreter/executioncontext.py pypy/branch/virtual-forcing/pypy/interpreter/generator.py pypy/branch/virtual-forcing/pypy/interpreter/pyframe.py pypy/branch/virtual-forcing/pypy/interpreter/pytraceback.py pypy/branch/virtual-forcing/pypy/interpreter/test/test_executioncontext.py pypy/branch/virtual-forcing/pypy/interpreter/test/test_zzpickle_and_slow.py pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py pypy/branch/virtual-forcing/pypy/jit/backend/model.py pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py pypy/branch/virtual-forcing/pypy/jit/backend/test/support.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/test/test_tlc.py pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py pypy/branch/virtual-forcing/pypy/jit/metainterp/jitprof.py pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py pypy/branch/virtual-forcing/pypy/jit/metainterp/support.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_basic.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_jitprof.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_resume.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py pypy/branch/virtual-forcing/pypy/module/_stackless/interp_coroutine.py pypy/branch/virtual-forcing/pypy/module/pypyjit/interp_jit.py pypy/branch/virtual-forcing/pypy/module/sys/vm.py pypy/branch/virtual-forcing/pypy/objspace/flow/flowcontext.py pypy/branch/virtual-forcing/pypy/rlib/jit.py pypy/branch/virtual-forcing/pypy/rpython/llinterp.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/lloperation.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/opimpl.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rclass.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py pypy/branch/virtual-forcing/pypy/rpython/rtyper.py pypy/branch/virtual-forcing/pypy/rpython/rvirtualizable2.py pypy/branch/virtual-forcing/pypy/rpython/test/test_rvirtualizable2.py pypy/branch/virtual-forcing/pypy/translator/c/funcgen.py pypy/branch/virtual-forcing/pypy/translator/cli/opcodes.py pypy/branch/virtual-forcing/pypy/translator/jvm/opcodes.py Log: Check in the changes of the previous branch of the same name. Modified: pypy/branch/virtual-forcing/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/interpreter/executioncontext.py (original) +++ pypy/branch/virtual-forcing/pypy/interpreter/executioncontext.py Mon Dec 21 16:13:02 2009 @@ -17,7 +17,8 @@ def __init__(self, space): self.space = space - self._init_frame_chain() + self.topframeref = jit.vref_None + self.framestackdepth = 0 # tracing: space.frame_trace_action.fire() must be called to ensure # that tracing occurs whenever self.w_tracefunc or self.is_tracing # is modified. @@ -27,169 +28,43 @@ self.profilefunc = None self.w_profilefuncarg = None + def gettopframe(self): + return self.topframeref() + def gettopframe_nohidden(self): - frame = self.gettopframe() - # I guess this should just use getnextframe_nohidden XXX + frame = self.topframeref() while frame and frame.hide(): - frame = frame.f_back() + frame = frame.f_backref() return frame @staticmethod def getnextframe_nohidden(frame): - frame = frame.f_back() + frame = frame.f_backref() while frame and frame.hide(): - frame = frame.f_back() + frame = frame.f_backref() return frame def enter(self, frame): if self.framestackdepth > self.space.sys.recursionlimit: raise OperationError(self.space.w_RuntimeError, self.space.wrap("maximum recursion depth exceeded")) - self._chain(frame) + self.framestackdepth += 1 + frame.f_backref = self.topframeref + self.topframeref = jit.virtual_ref(frame) def leave(self, frame): - if self.profilefunc: - self._trace(frame, 'leaveframe', self.space.w_None) + try: + if self.profilefunc: + self._trace(frame, 'leaveframe', self.space.w_None) + finally: + self.topframeref = frame.f_backref + self.framestackdepth -= 1 + jit.virtual_ref_finish(frame) - self._unchain(frame) - if self.w_tracefunc is not None and not frame.hide(): self.space.frame_trace_action.fire() # ________________________________________________________________ - # the methods below are used for chaining frames in JIT-friendly way - # part of that stuff is obscure - - @jit.unroll_safe - def gettopframe(self): - frame = self.some_frame - if frame is not None: - while frame.f_forward is not None: - frame = frame.f_forward - return frame - - def _init_frame_chain(self): - # 'some_frame' points to any frame from this thread's frame stack - # (although in general it should point to the top one). - # XXX not true: some_frame must point to a frame from which we can - # reach the top frame by following the chain of f_forward - self.some_frame = None - self.framestackdepth = 0 - - @staticmethod - def _init_chaining_attributes(frame): - """ - explanation of the f_back handling: - ----------------------------------- - - in the non-JIT case, the frames simply form a doubly linked list via the - attributes f_back_some and f_forward. - - When the JIT is used, things become more complex, as functions can be - inlined into each other. In this case a frame chain can look like this: - - +---------------+ - | real_frame | - +---------------+ - | - | f_back_some - | - | - | +--------------+ - | | virtual frame| - | +--------------+ - | ^ - | | f_forward - | +--------------+ - | | virtual frame| - | +--------------+ - | ^ - | | - v | f_forward - +---------------+ - | real_frame | - +---------------+ - | - | - v - ... - - This ensures that the virtual frames don't escape via the f_back of the - real frames. For the same reason, the executioncontext's some_frame - attribute should only point to real frames. - - All places where a frame can become accessed from applevel-code (like - sys._getframe and traceback catching) need to call force_f_back to ensure - that the intermediate virtual frames are forced to be real ones. - - """ - frame.f_back_some = None - frame.f_forward = None - frame.f_back_forced = False - - def _chain(self, frame): - self.framestackdepth += 1 - # - frame.f_back_some = self.some_frame - if self._we_are_jitted(): - curtopframe = self.gettopframe() - assert curtopframe is not None - curtopframe.f_forward = frame - else: - self.some_frame = frame - - def _unchain(self, frame): - #assert frame is self.gettopframe() --- slowish - if self.some_frame is frame: - self.some_frame = frame.f_back_some - else: - f_back = frame.f_back() - if f_back is not None: - f_back.f_forward = None - - self.framestackdepth -= 1 - - @staticmethod - def _jit_rechain_frame(ec, frame): - # this method is called after the jit has seen enter (and thus _chain) - # of a frame, but then does not actually inline it. This method thus - # needs to make sure that the state is as if the _chain method had been - # executed outside of the jit. Note that this makes it important that - # _unchain does not call we_are_jitted - frame.f_back().f_forward = None - ec.some_frame = frame - - @staticmethod - @jit.unroll_safe - def _extract_back_from_frame(frame): - back_some = frame.f_back_some - if frame.f_back_forced: - # don't check back_some.f_forward in this case - return back_some - if back_some is None: - return None - while True: - f_forward = back_some.f_forward - if f_forward is frame or f_forward is None: - return back_some - back_some = f_forward - - @staticmethod - def _force_back_of_frame(frame): - orig_frame = frame - while frame is not None and not frame.f_back_forced: - frame.f_back_some = f_back = ExecutionContext._extract_back_from_frame(frame) - frame.f_back_forced = True - # now that we force the whole chain, we also have to set the - # forward links to None - frame.f_forward = None - frame = f_back - return orig_frame.f_back_some - - _we_are_jitted = staticmethod(we_are_jitted) # indirection for testing - - # the methods above are used for chaining frames in JIT-friendly way - # ________________________________________________________________ class Subcontext(object): @@ -204,7 +79,7 @@ self.is_tracing = 0 def enter(self, ec): - ec.some_frame = self.topframe + ec.topframeref = jit.non_virtual_ref(self.topframe) ec.framestackdepth = self.framestackdepth ec.w_tracefunc = self.w_tracefunc ec.profilefunc = self.profilefunc @@ -247,7 +122,7 @@ while index > 0: index -= 1 lst[index] = f - f = f.f_back() + f = f.f_backref() assert f is None return lst # coroutine: I think this is all, folks! Modified: pypy/branch/virtual-forcing/pypy/interpreter/generator.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/interpreter/generator.py (original) +++ pypy/branch/virtual-forcing/pypy/interpreter/generator.py Mon Dec 21 16:13:02 2009 @@ -2,6 +2,7 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import NoneNotWrapped from pypy.rlib.rarithmetic import intmask +from pypy.rlib import jit from pypy.interpreter.pyopcode import LoopBlock @@ -64,7 +65,7 @@ else: return w_result # YIELDed finally: - self.frame.f_back_some = None + self.frame.f_backref = jit.vref_None self.running = False def descr_throw(self, w_type, w_val=None, w_tb=None): Modified: pypy/branch/virtual-forcing/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/interpreter/pyframe.py (original) +++ pypy/branch/virtual-forcing/pypy/interpreter/pyframe.py Mon Dec 21 16:13:02 2009 @@ -34,13 +34,13 @@ * 'builtin' is the attached built-in module * 'valuestack_w', 'blockstack', control the interpretation """ - __metaclass__ = extendabletype frame_finished_execution = False last_instr = -1 last_exception = None + f_backref = jit.vref_None w_f_trace = None # For tracing instr_lb = 0 @@ -48,6 +48,9 @@ instr_prev = -1 is_being_profiled = False + # XXX temporary, kill me + f_back = property(lambda: None, lambda: None, lambda: None) # f_backref! + def __init__(self, space, code, w_globals, closure): self = hint(self, access_directly=True, fresh_virtualizable=True) assert isinstance(code, pycode.PyCode) @@ -64,7 +67,6 @@ self.fastlocals_w = [None]*self.numlocals make_sure_not_resized(self.fastlocals_w) self.f_lineno = code.co_firstlineno - ExecutionContext._init_chaining_attributes(self) def append_block(self, block): block.previous = self.lastblock @@ -308,7 +310,7 @@ w_tb = w(self.last_exception.application_traceback) tup_state = [ - w(self.f_back()), + w(self.f_backref()), w(self.get_builtin()), w(self.pycode), w_valuestack, @@ -360,8 +362,8 @@ # do not use the instance's __init__ but the base's, because we set # everything like cells from here PyFrame.__init__(self, space, pycode, w_globals, closure) - new_frame.f_back_some = space.interp_w(PyFrame, w_f_back, can_be_None=True) - new_frame.f_back_forced = True + f_back = space.interp_w(PyFrame, w_f_back, can_be_None=True) + new_frame.f_backref = jit.non_virtual_ref(f_back) new_frame.builtin = space.interp_w(Module, w_builtin) new_frame.set_blocklist([unpickle_block(space, w_blk) @@ -431,12 +433,6 @@ def _setcellvars(self, cellvars): pass - def f_back(self): - return ExecutionContext._extract_back_from_frame(self) - - def force_f_back(self): - return ExecutionContext._force_back_of_frame(self) - ### line numbers ### # for f*_f_* unwrapping through unwrap_spec in typedef.py @@ -584,7 +580,7 @@ return self.get_builtin().getdict() def fget_f_back(space, self): - return self.space.wrap(self.f_back()) + return self.space.wrap(self.f_backref()) def fget_f_lasti(space, self): return self.space.wrap(self.last_instr) @@ -605,27 +601,27 @@ def fget_f_exc_type(space, self): if self.last_exception is not None: - f = self.f_back() + f = self.f_backref() while f is not None and f.last_exception is None: - f = f.f_back() + f = f.f_backref() if f is not None: return f.last_exception.w_type return space.w_None def fget_f_exc_value(space, self): if self.last_exception is not None: - f = self.f_back() + f = self.f_backref() while f is not None and f.last_exception is None: - f = f.f_back() + f = f.f_backref() if f is not None: return f.last_exception.w_value return space.w_None def fget_f_exc_traceback(space, self): if self.last_exception is not None: - f = self.f_back() + f = self.f_backref() while f is not None and f.last_exception is None: - f = f.f_back() + f = f.f_backref() if f is not None: return space.wrap(f.last_exception.application_traceback) return space.w_None Modified: pypy/branch/virtual-forcing/pypy/interpreter/pytraceback.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/interpreter/pytraceback.py (original) +++ pypy/branch/virtual-forcing/pypy/interpreter/pytraceback.py Mon Dec 21 16:13:02 2009 @@ -49,7 +49,6 @@ self.next = space.interp_w(PyTraceback, w_next, can_be_None=True) def record_application_traceback(space, operror, frame, last_instruction): - frame.force_f_back() if frame.pycode.hidden_applevel: return tb = operror.application_traceback Modified: pypy/branch/virtual-forcing/pypy/interpreter/test/test_executioncontext.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/interpreter/test/test_executioncontext.py (original) +++ pypy/branch/virtual-forcing/pypy/interpreter/test/test_executioncontext.py Mon Dec 21 16:13:02 2009 @@ -1,6 +1,5 @@ import py from pypy.interpreter import executioncontext -from pypy.interpreter.executioncontext import ExecutionContext from pypy.conftest import gettestobjspace class Finished(Exception): @@ -220,542 +219,3 @@ """) events = space.unwrap(w_events) assert [i[0] for i in events] == ['c_call', 'c_return', 'c_call'] - - - -class TestFrameChaining(object): - class EC(ExecutionContext): - _some_frame = None - def __init__(self, jitted=False): - self.jitted = jitted - self.virtualizable = None - self.framestackdepth = 0 - self._init_frame_chain() - - def _we_are_jitted(self): - return self.jitted - - def _get_some_frame(self): - if self._some_frame: - self._some_frame.look_at() - return self._some_frame - def _set_some_frame(self, frame): - if frame is not None: - frame.force() - self._some_frame = frame - some_frame = property(_get_some_frame, _set_some_frame) - - class Frame(object): - _f_back_some = None - _f_forward = None - - def __init__(self, ec, virtual_with_base_frame=None): - self.ec = ec - self.virtual_with_base_frame = virtual_with_base_frame - self.escaped = not virtual_with_base_frame - ExecutionContext._init_chaining_attributes(self) - - def f_back(self): - return ExecutionContext._extract_back_from_frame(self) - - def force_f_back(self): - return ExecutionContext._force_back_of_frame(self) - - def force(self): - if not self.escaped: - self.virtual_with_base_frame = None - self.escaped = True - if self._f_back_some: - self._f_back_some.force() - if self._f_forward: - self._f_back_some.force() - - def look_at(self): - if (not self.ec.jitted or - self.ec.virtualizable is not self.virtual_with_base_frame): - self.force() - - def store_ref_to(self, other): - if (other.virtual_with_base_frame is not self and - other.virtual_with_base_frame is not self.virtual_with_base_frame): - other.force() - - def _get_f_back_some(self): - self.look_at() - return self._f_back_some - def _set_f_back_some(self, frame): - self.look_at() - if frame: - frame.look_at() - self.store_ref_to(frame) - self._f_back_some = frame - f_back_some = property(_get_f_back_some, _set_f_back_some) - - def _get_f_forward(self): - self.look_at() - return self._f_forward - def _set_f_forward(self, frame): - self.look_at() - if frame: - frame.look_at() - self.store_ref_to(frame) - self._f_forward = frame - f_forward = property(_get_f_forward, _set_f_forward) - - def test_f_back_no_jit(self): - ec = self.EC() - frame = self.Frame(ec) - frame2 = self.Frame(ec) - frame2.f_back_some = frame - - frame3 = self.Frame(ec) - frame3.f_back_some = frame2 - - assert frame3.f_back() is frame2 - assert frame2.f_back() is frame - assert frame.f_back() is None - - def test_f_back_jit(self): - ec = self.EC() - frame = self.Frame(ec) # real frame - frame2 = self.Frame(ec) # virtual frame - frame2.f_back_some = frame - frame.f_forward = frame2 - - frame3 = self.Frame(ec) # virtual frame - frame3.f_back_some = frame - frame2.f_forward = frame3 - - assert frame3.f_back() is frame2 - assert frame2.f_back() is frame - assert frame.f_back() is None - - frame4 = self.Frame(ec) # real frame again - frame4.f_back_some = frame - assert frame4.f_back() is frame3 - - def test_gettopframe_no_jit(self): - ec = self.EC() - frame = self.Frame(ec) - ec.some_frame = frame - assert ec.gettopframe() is frame - - def test_gettopframe_jit(self): - ec = self.EC() - frame = self.Frame(ec) # real frame - ec.some_frame = frame - assert ec.gettopframe() is frame - - frame2 = self.Frame(ec) # virtual frame - frame2.f_back_some = frame - frame.f_forward = frame2 - assert ec.gettopframe() is frame2 - - frame3 = self.Frame(ec) # virtual frame - frame3.f_back_some = frame - frame2.f_forward = frame3 - assert ec.gettopframe() is frame3 - - frame4 = self.Frame(ec) # real frame again - frame4.f_back_some = frame - ec.some_frame = frame4 - assert ec.gettopframe() is frame4 - - def test_frame_chain(self): - - ec = self.EC() - - assert ec.some_frame is None - assert ec.framestackdepth == 0 - - frame = self.Frame(ec) - ec._chain(frame) - assert ec.some_frame is frame - assert ec.framestackdepth == 1 - assert frame.f_back_some is None - assert frame.f_forward is None - assert ec.gettopframe() is frame - assert ec._extract_back_from_frame(frame) is None - - frame2 = self.Frame(ec) - ec._chain(frame2) - assert ec.some_frame is frame2 - assert ec.framestackdepth == 2 - assert frame2.f_back_some is frame - assert frame.f_forward is None - assert frame2.f_forward is None - assert ec.gettopframe() is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - frame3 = self.Frame(ec) - ec._chain(frame3) - assert ec.some_frame is frame3 - assert frame3.f_back_some is frame2 - assert frame2.f_forward is None - assert ec.gettopframe() is frame3 - assert ec._extract_back_from_frame(frame3) is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - assert frame3.f_back() is frame2 - ec._unchain(frame3) - assert ec.some_frame is frame2 - assert ec.framestackdepth == 2 - assert frame2.f_forward is None - assert frame3.f_back_some is frame2 - assert ec.gettopframe() is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - assert frame2.f_back() is frame - ec._unchain(frame2) - assert ec.some_frame is frame - assert ec.framestackdepth == 1 - assert frame.f_forward is None - assert frame2.f_back_some is frame - assert ec.gettopframe() is frame - assert ec._extract_back_from_frame(frame) is None - - assert frame.f_back() is None - ec._unchain(frame) - assert ec.some_frame is None - assert ec.framestackdepth == 0 - assert frame.f_back_some is None - assert ec.gettopframe() is None - - def test_frame_chain_forced(self): - - ec = self.EC() - - frame = self.Frame(ec) - ec._chain(frame) - assert ec.gettopframe() is frame - assert ec._extract_back_from_frame(frame) is None - - frame2 = self.Frame(ec) - ec._chain(frame2) - assert ec.some_frame is frame2 - assert ec.framestackdepth == 2 - assert frame2.f_back_some is frame - assert frame.f_forward is None - assert frame2.f_forward is None - res = frame2.force_f_back() - assert res is frame - assert frame.f_back_forced - assert ec.gettopframe() is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - frame3 = self.Frame(ec) - ec._chain(frame3) - assert ec.gettopframe() is frame3 - assert ec._extract_back_from_frame(frame3) is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - assert frame3.f_back() is frame2 - ec._unchain(frame3) - assert ec.some_frame is frame2 - assert frame3.f_back_some is frame2 - assert ec.gettopframe() is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - assert frame2.f_back() is frame - ec._unchain(frame2) - assert frame2.f_back_some is frame - assert ec.gettopframe() is frame - assert ec._extract_back_from_frame(frame) is None - - assert frame.f_back() is None - ec._unchain(frame) - assert ec.some_frame is None - assert frame.f_back_some is None - - assert frame2.f_back() is frame - assert frame.f_back() is None - assert ec.gettopframe() is None - - - def test_frame_chain_jitted(self): - - ec = self.EC() - - assert ec.some_frame is None - assert ec.framestackdepth == 0 - assert ec.gettopframe() is None - - frame = self.Frame(ec) - ec._chain(frame) - assert ec.some_frame is frame - assert ec.framestackdepth == 1 - assert frame.f_back_some is None - assert frame.f_forward is None - assert ec.gettopframe() is frame - assert ec._extract_back_from_frame(frame) is None - - ec.jitted = True - ec.virtualizable = frame - frame2 = self.Frame(ec, frame) - ec._chain(frame2) - assert ec.some_frame is frame - assert ec.framestackdepth == 2 - assert frame2.f_back_some is frame - assert frame.f_forward is frame2 - assert frame2.f_forward is None - assert ec.gettopframe() is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - # recursive enter/leave seen by the jit - frame3 = self.Frame(ec, frame) - ec._chain(frame3) - assert ec.some_frame is frame - assert frame3.f_back_some is frame - assert frame2.f_forward is frame3 - assert ec.gettopframe() is frame3 - assert ec._extract_back_from_frame(frame3) is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - assert frame3.f_back() is frame2 - ec._unchain(frame3) - assert ec.some_frame is frame - assert ec.framestackdepth == 2 - assert frame2.f_forward is None - assert frame3.f_back_some is frame - assert not frame3.escaped - assert not frame2.escaped - assert ec.gettopframe() is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - # recursive enter/leave not seen by the jit - ec.jitted = False - ec.virtualizable = None - ec._chain(frame3) - assert not frame2.escaped - assert ec.some_frame is frame3 - assert frame3.f_back_some is frame - assert frame2.f_forward is None - assert frame3.escaped - assert ec.gettopframe() is frame3 - assert ec._extract_back_from_frame(frame3) is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - assert frame3.f_back() is frame2 - ec._unchain(frame3) - assert ec.some_frame is frame - assert ec.framestackdepth == 2 - assert frame2.f_forward is None - assert frame3.f_back_some is frame - assert ec.gettopframe() is frame2 - assert ec._extract_back_from_frame(frame2) is frame - assert ec._extract_back_from_frame(frame) is None - - ec.jitted = True - ec.virtualizable = frame - - assert frame2.f_back() is frame - ec._unchain(frame2) - assert ec.some_frame is frame - assert ec.framestackdepth == 1 - assert frame.f_forward is None - assert frame2.f_back_some is frame - assert ec.gettopframe() is frame - assert ec._extract_back_from_frame(frame) is None - - ec.jitted = False - assert frame.f_back() is None - ec._unchain(frame) - assert ec.some_frame is None - assert ec.framestackdepth == 0 - assert frame.f_back_some is None - assert ec.gettopframe() is None - - @py.test.mark.xfail - def test_frame_chain_jitted_forced(self): - - ec = self.EC() - - assert ec.some_frame is None - assert ec.framestackdepth == 0 - - frame = self.Frame(ec) - ec._chain(frame) - - ec.jitted = True - frame2 = self.Frame(ec) - ec._chain(frame2) - - # recursive enter/leave seen by the jit - frame3 = self.Frame(ec) - ec._chain(frame3) - assert ec.gettopframe() is frame3 - res = frame3.force_f_back() - assert res is frame2 - assert ec.gettopframe() is frame3 - - assert frame3.f_back() is frame2 - ec._unchain(frame3) - - assert frame2.f_back() is frame - ec._unchain(frame2) - ec.jitted = False - assert frame.f_back() is None - ec._unchain(frame) - - assert frame3.f_back() is frame2 - assert frame2.f_back() is frame - assert frame.f_back() is None - - def enter_two_jitted_levels(self): - ec = self.EC() - - assert ec.some_frame is None - assert ec.framestackdepth == 0 - - frame = self.Frame(ec) - ec._chain(frame) - - ec.jitted = True - ec.virtualizable = frame - frame2 = self.Frame(ec, frame) - ec._chain(frame2) - assert not frame2.escaped - return ec, frame, frame2 - - def leave_two_jitted_levels(self, ec, frame, frame2): - assert frame2.f_back() is frame - ec._unchain(frame2) - ec.jitted = False - assert frame.f_back() is None - ec._unchain(frame) - - - def test_check_escaping_all_inlined(self): - ec, frame, frame2 = self.enter_two_jitted_levels() - - # recursive enter/leave seen by the jit - frame3 = self.Frame(ec, frame) - ec._chain(frame3) - assert not frame2.escaped - assert not frame3.escaped - - assert frame3.f_back() is frame2 - ec._unchain(frame3) - assert not frame2.escaped - self.leave_two_jitted_levels(ec, frame, frame2) - - - def test_check_escaping_not_all_inlined_enter_leave_not_seen(self): - ec, frame, frame2 = self.enter_two_jitted_levels() - - ec.jitted = False - # recursive enter/leave not seen by the jit - frame3 = self.Frame(ec) - ec._chain(frame3) - - assert not frame2.escaped - assert frame3.escaped - - ec._unchain(frame3) - ec.jitted = True - assert not frame2.escaped - - self.leave_two_jitted_levels(ec, frame, frame2) - - def test_check_escaping_not_all_inlined_enter_leave_seen(self): - ec, frame, frame2 = self.enter_two_jitted_levels() - - # recursive enter/leave seen by the jit - frame3 = self.Frame(ec, frame) - ec._chain(frame3) - ExecutionContext._jit_rechain_frame(ec, frame3) - ec.jitted = False - frame3.look_at() - assert not frame2.escaped - assert frame3.escaped - - ec.jitted = True - assert frame3.f_back() is frame2 - ec._unchain(frame3) - assert not frame2.escaped - - self.leave_two_jitted_levels(ec, frame, frame2) - - - def test_check_escaping_multi_non_jitted_levels(self): - ec, frame, frame2 = self.enter_two_jitted_levels() - - # recursive enter/leave seen by the jit - frame3 = self.Frame(ec, frame) - ec._chain(frame3) - ExecutionContext._jit_rechain_frame(ec, frame3) - ec.jitted = False - - assert frame3.escaped - assert not frame2.escaped - assert frame3.escaped - - frame4 = self.Frame(ec) - ec._chain(frame4) - assert ec.framestackdepth == 4 - - ec._unchain(frame4) - assert frame3.escaped - assert not frame2.escaped - - ec.jitted = True - assert frame3.f_back() is frame2 - ec._unchain(frame3) - assert not frame2.escaped - - self.leave_two_jitted_levels(ec, frame, frame2) - - def test_check_escaping_jitted_with_two_different_virtualizables(self): - ec, frame, frame2 = self.enter_two_jitted_levels() - - frame3 = self.Frame(ec, frame) - ec._chain(frame3) - # frame3 is not inlined, but contains a loop itself, for which code has - # been generated - ExecutionContext._jit_rechain_frame(ec, frame3) - ec.virtualizable = frame3 - - frame3.look_at() - assert not frame2.escaped - assert frame3.escaped - - frame4 = self.Frame(ec, frame3) - ec._chain(frame4) - assert ec.framestackdepth == 4 - assert not frame4.escaped - - ec._unchain(frame4) - assert frame3.escaped - assert not frame2.escaped - - ec.virtualizable = frame - - ec._unchain(frame3) - assert not frame2.escaped - - def test_frame_top_is_virtualizable(self): - ec, frame, frame2 = self.enter_two_jitted_levels() - frame3 = self.Frame(ec, frame2) - ec.jitted = False - ec._chain(frame3) - ec.gettopframe() - frame3.force_f_back() - ec._unchain(frame3) - assert not frame2.f_forward - assert ec.gettopframe() is frame2 - ec.jitted = True - ec._unchain(frame2) - assert not frame.f_forward - assert ec.gettopframe() is frame - ec._unchain(frame) - assert ec.gettopframe() is None Modified: pypy/branch/virtual-forcing/pypy/interpreter/test/test_zzpickle_and_slow.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/interpreter/test/test_zzpickle_and_slow.py (original) +++ pypy/branch/virtual-forcing/pypy/interpreter/test/test_zzpickle_and_slow.py Mon Dec 21 16:13:02 2009 @@ -2,6 +2,7 @@ from pypy import conftest from pypy.conftest import gettestobjspace from pypy.interpreter import gateway +from pypy.rlib.jit import non_virtual_ref, vref_None class AppTestSlow: def setup_class(cls): @@ -30,21 +31,18 @@ from pypy.interpreter import pytraceback def hide_top_frame(space, w_frame): w_last = None - while w_frame.f_back(): - # should have been forced by traceback capturing - assert w_frame.f_back_forced + while w_frame.f_backref(): w_last = w_frame - w_frame = w_frame.f_back() + w_frame = w_frame.f_backref() assert w_last - w_saved = w_last.f_back() - w_last.f_back_some = None + w_saved = w_last.f_backref() + w_last.f_backref = vref_None return w_saved def restore_top_frame(space, w_frame, w_saved): - while w_frame.f_back(): - assert w_frame.f_back_forced - w_frame = w_frame.f_back() - w_frame.f_back_some = w_saved + while w_frame.f_backref(): + w_frame = w_frame.f_backref() + w_frame.f_backref = non_virtual_ref(w_saved) def read_exc_type(space, w_frame): if w_frame.last_exception is None: Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py Mon Dec 21 16:13:02 2009 @@ -152,7 +152,9 @@ 'debug_merge_point': (('ref',), None), 'force_token' : ((), 'int'), 'call_may_force' : (('int', 'varargs'), 'intorptr'), - 'guard_not_forced': ((), None) + 'guard_not_forced': ((), None), + 'virtual_ref' : (('ref', 'int'), 'ref'), + 'virtual_ref_finish': (('ref', 'ref'), None), #'getitem' : (('void', 'ref', 'int'), 'int'), #'setitem' : (('void', 'ref', 'int', 'int'), None), #'newlist' : (('void', 'varargs'), 'ref'), @@ -813,6 +815,12 @@ if forced: raise GuardFailed + def op_virtual_ref(self, _, virtual, index): + return virtual + + def op_virtual_ref_finish(self, _, vref, virtual): + pass + class OOFrame(Frame): Modified: pypy/branch/virtual-forcing/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/model.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/model.py Mon Dec 21 16:13:02 2009 @@ -219,10 +219,6 @@ def do_cast_ptr_to_int(self, ptrbox): raise NotImplementedError - def do_force_token(self): - # this should not be implemented at all by the backends - raise NotImplementedError - def do_call_may_force(self, args, calldescr): return self.do_call(args, calldescr) Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Mon Dec 21 16:13:02 2009 @@ -779,6 +779,19 @@ r = self.execute_operation(rop.SAME_AS, [BoxFloat(5.5)], 'float') assert r.value == 5.5 + def test_virtual_ref(self): + # if VIRTUAL_REF reaches the backend, it should just be a SAME_AS + u_box = self.alloc_unicode(u"hello\u1234") + r = self.execute_operation(rop.VIRTUAL_REF, [u_box, ConstInt(2)], + 'ref') + assert r.value == u_box.value + + def test_virtual_ref_finish(self): + # if VIRTUAL_REF_FINISH reaches the backend, it is a no-op + self.execute_operation(rop.VIRTUAL_REF_FINISH, + [BoxInt(123), BoxInt(234)], + 'void') + def test_jump(self): # this test generates small loops where the JUMP passes many # arguments of various types, shuffling them around. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/support.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/test/support.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/test/support.py Mon Dec 21 16:13:02 2009 @@ -100,6 +100,9 @@ def check_aborted_count(self, *args, **kwds): pass + def check_aborted_count_at_least(self, *args, **kwds): + pass + def interp_operations(self, *args, **kwds): py.test.skip("interp_operations test skipped") Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py Mon Dec 21 16:13:02 2009 @@ -550,6 +550,7 @@ def genop_same_as(self, op, arglocs, resloc): self.mov(arglocs[0], resloc) genop_cast_ptr_to_int = genop_same_as + genop_virtual_ref = genop_same_as def genop_int_mod(self, op, arglocs, resloc): self.mc.CDQ() Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py Mon Dec 21 16:13:02 2009 @@ -861,6 +861,7 @@ resloc = self.force_allocate_reg(op.result) self.Perform(op, [argloc], resloc) consider_cast_ptr_to_int = consider_same_as + consider_virtual_ref = consider_same_as def consider_strlen(self, op, ignored): base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) @@ -919,6 +920,9 @@ def consider_debug_merge_point(self, op, ignored): pass + def consider_virtual_ref_finish(self, op, ignored): + self.possibly_free_vars(op.args) + def get_mark_gc_roots(self, gcrootmap): shape = gcrootmap.get_basic_shape() for v, val in self.fm.frame_bindings.items(): Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/test/test_tlc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/test/test_tlc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/test/test_tlc.py Mon Dec 21 16:13:02 2009 @@ -1,6 +1,5 @@ import py -py.test.skip("Widening to trash error") from pypy.jit.backend.x86.test.test_basic import Jit386Mixin from pypy.jit.metainterp.test.test_tlc import TLCTests from pypy.jit.tl import tlc @@ -10,6 +9,7 @@ # ====> ../../test/test_tlc.py def test_accumulator(self): + py.test.skip("investigate, maybe") path = py.path.local(tlc.__file__).dirpath('accumulator.tlc.src') code = path.read() res = self.exec_code(code, 20) @@ -18,6 +18,7 @@ assert res == 10 def test_fib(self): + py.test.skip("investigate, maybe") path = py.path.local(tlc.__file__).dirpath('fibo.tlc.src') code = path.read() res = self.exec_code(code, 7) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py Mon Dec 21 16:13:02 2009 @@ -1158,16 +1158,17 @@ self.var_position(op.args[3])) def serialize_op_jit_marker(self, op): - if op.args[0].value == 'jit_merge_point': - assert self.portal, "jit_merge_point in non-main graph!" - self.emit('jit_merge_point') - assert ([self.var_position(i) for i in op.args[2:]] == - range(0, 2*(len(op.args) - 2), 2)) - #for i in range(2, len(op.args)): - # arg = op.args[i] - # self._eventualy_builtin(arg) - elif op.args[0].value == 'can_enter_jit': - self.emit('can_enter_jit') + key = op.args[0].value + getattr(self, 'handle_jit_marker__%s' % key)(op) + + def handle_jit_marker__jit_merge_point(self, op): + assert self.portal, "jit_merge_point in non-main graph!" + self.emit('jit_merge_point') + assert ([self.var_position(i) for i in op.args[2:]] == + range(0, 2*(len(op.args) - 2), 2)) + + def handle_jit_marker__can_enter_jit(self, op): + self.emit('can_enter_jit') def serialize_op_direct_call(self, op): kind = self.codewriter.guess_call_kind(op) @@ -1213,7 +1214,7 @@ if pure and not all_promoted_args: effectinfo = calldescr.get_extra_info() assert (effectinfo is not None and - not effectinfo.promotes_virtualizables) + not effectinfo.forces_virtual_or_virtualizable) try: canraise = self.codewriter.raise_analyzer.can_raise(op) except lltype.DelayedPointer: @@ -1279,6 +1280,9 @@ return self._do_builtin_call(op, oopspec_name, args) def _do_builtin_call(self, op, oopspec_name, args): + if oopspec_name.startswith('virtual_ref'): + self.handle_virtual_ref_call(op, oopspec_name, args) + return argtypes = [v.concretetype for v in args] resulttype = op.result.concretetype c_func, TP = support.builtin_func_for_spec(self.codewriter.rtyper, @@ -1299,6 +1303,16 @@ self.emit_varargs([c_func] + non_void_args) self.register_var(op.result) + def handle_virtual_ref_call(self, op, oopspec_name, args): + self.emit(oopspec_name) # 'virtual_ref' or 'virtual_ref_finish' + self.emit(self.var_position(args[0])) + self.register_var(op.result) + # + from pypy.jit.metainterp.virtualref import jit_virtual_ref_vtable + from pypy.jit.metainterp.virtualref import JIT_VIRTUAL_REF + self.codewriter.register_known_gctype(jit_virtual_ref_vtable, + JIT_VIRTUAL_REF) + def _array_of_voids(self, ARRAY): if isinstance(ARRAY, ootype.Array): return ARRAY.ITEM == ootype.Void @@ -1557,12 +1571,15 @@ log.WARNING("found debug_assert in %r; should have be removed" % (self.graph,)) - def serialize_op_promote_virtualizable(self, op): + def serialize_op_jit_force_virtualizable(self, op): vinfo = self.codewriter.metainterp_sd.virtualizable_info assert vinfo is not None assert vinfo.is_vtypeptr(op.args[0].concretetype) self.vable_flags[op.args[0]] = op.args[2].value + def serialize_op_jit_force_virtual(self, op): + self._do_builtin_call(op, 'jit_force_virtual', op.args) + serialize_op_oostring = handle_builtin_call serialize_op_oounicode = handle_builtin_call serialize_op_gc_identityhash = handle_builtin_call @@ -1688,3 +1705,6 @@ def __str__(self): return "using virtualizable array in illegal way in %r" % ( self.args[0],) + +class ForcingVirtualRef(Exception): + pass Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py Mon Dec 21 16:13:02 2009 @@ -237,23 +237,49 @@ from pypy.jit.metainterp.pyjitpl import MetaInterp metainterp = MetaInterp(metainterp_sd) token = metainterp_sd.cpu.get_latest_force_token() - data = self.fetch_data(token) - if data is None: - data = [] - metainterp._already_allocated_resume_virtuals = data + all_virtuals = self.fetch_data(token) + if all_virtuals is None: + all_virtuals = [] + metainterp._already_allocated_resume_virtuals = all_virtuals self.counter = -2 # never compile return metainterp.handle_guard_failure(self) - def force_virtualizable(self, vinfo, virtualizable, force_token): + @staticmethod + def force_now(cpu, token): + # Called during a residual call from the assembler, if the code + # actually needs to force one of the virtualrefs or the virtualizable. + # Implemented by forcing *all* virtualrefs and the virtualizable. + faildescr = cpu.force(token) + assert isinstance(faildescr, ResumeGuardForcedDescr) + faildescr.handle_async_forcing(token) + + def handle_async_forcing(self, force_token): from pypy.jit.metainterp.pyjitpl import MetaInterp from pypy.jit.metainterp.resume import force_from_resumedata - metainterp = MetaInterp(self.metainterp_sd) + from pypy.jit.metainterp.virtualref import forced_single_vref + # To handle the forcing itself, we create a temporary MetaInterp + # as a convenience to move the various data to its proper place. + metainterp_sd = self.metainterp_sd + metainterp = MetaInterp(metainterp_sd) metainterp.history = None # blackholing - liveboxes = metainterp.cpu.make_boxes_from_latest_values(self) - virtualizable_boxes, data = force_from_resumedata(metainterp, - liveboxes, self) - vinfo.write_boxes(virtualizable, virtualizable_boxes) - self.save_data(force_token, data) + liveboxes = metainterp_sd.cpu.make_boxes_from_latest_values(self) + # + expect_virtualizable = metainterp_sd.virtualizable_info is not None + forced_data = force_from_resumedata(metainterp, liveboxes, self, + expect_virtualizable) + virtualizable_boxes, virtualref_boxes, all_virtuals = forced_data + # + # Handle virtualref_boxes: mark each JIT_VIRTUAL_REF as forced + for i in range(0, len(virtualref_boxes), 2): + virtualbox = virtualref_boxes[i] + vrefbox = virtualref_boxes[i+1] + forced_single_vref(vrefbox.getref_base(), virtualbox.getref_base()) + # Handle virtualizable_boxes: store them on the real virtualizable now + if expect_virtualizable: + metainterp_sd.virtualizable_info.forced_vable(virtualizable_boxes) + # Handle all_virtuals: keep them for later blackholing from the + # future failure of the GUARD_NOT_FORCED + self.save_data(force_token, all_virtuals) def save_data(self, key, value): globaldata = self.metainterp_sd.globaldata Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py Mon Dec 21 16:13:02 2009 @@ -8,19 +8,20 @@ _cache = {} def __new__(cls, write_descrs_fields, write_descrs_arrays, - promotes_virtualizables=False): + forces_virtual_or_virtualizable=False): key = (frozenset(write_descrs_fields), frozenset(write_descrs_arrays), - promotes_virtualizables) + forces_virtual_or_virtualizable) if key in cls._cache: return cls._cache[key] result = object.__new__(cls) result.write_descrs_fields = write_descrs_fields result.write_descrs_arrays = write_descrs_arrays - result.promotes_virtualizables = promotes_virtualizables + result.forces_virtual_or_virtualizable= forces_virtual_or_virtualizable cls._cache[key] = result return result -def effectinfo_from_writeanalyze(effects, cpu, promotes_virtualizables=False): +def effectinfo_from_writeanalyze(effects, cpu, + forces_virtual_or_virtualizable=False): from pypy.translator.backendopt.writeanalyze import top_set if effects is top_set: return None @@ -44,7 +45,7 @@ else: assert 0 return EffectInfo(write_descrs_fields, write_descrs_arrays, - promotes_virtualizables) + forces_virtual_or_virtualizable) def consider_struct(TYPE, fieldname): if fieldType(TYPE, fieldname) is lltype.Void: @@ -73,4 +74,5 @@ class VirtualizableAnalyzer(BoolGraphAnalyzer): def analyze_simple_operation(self, op): - return op.opname == 'promote_virtualizable' + return op.opname in ('jit_force_virtualizable', + 'jit_force_virtual') Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/executor.py Mon Dec 21 16:13:02 2009 @@ -6,7 +6,7 @@ from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask -from pypy.jit.metainterp.history import BoxInt, ConstInt, check_descr +from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt, check_descr from pypy.jit.metainterp.history import INT, REF, ConstFloat from pypy.jit.metainterp import resoperation from pypy.jit.metainterp.resoperation import rop @@ -220,6 +220,15 @@ # ____________________________________________________________ +def do_force_token(cpu): + raise NotImplementedError + +def do_virtual_ref(cpu, box1, box2): + raise NotImplementedError + +def do_virtual_ref_finish(cpu, box1, box2): + raise NotImplementedError + def do_debug_merge_point(cpu, box1): from pypy.jit.metainterp.warmspot import get_stats loc = box1._get_str() Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/jitprof.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/jitprof.py Mon Dec 21 16:13:02 2009 @@ -20,6 +20,7 @@ OPT_FORCINGS ABORT_TOO_LONG ABORT_BRIDGE +ABORT_ESCAPE NVIRTUALS NVHOLES NVREUSED @@ -176,8 +177,9 @@ self._print_intline("opt ops", cnt[OPT_OPS]) self._print_intline("opt guards", cnt[OPT_GUARDS]) self._print_intline("forcings", cnt[OPT_FORCINGS]) - self._print_intline("trace too long", cnt[ABORT_TOO_LONG]) - self._print_intline("bridge abort", cnt[ABORT_BRIDGE]) + self._print_intline("abort: trace too long", cnt[ABORT_TOO_LONG]) + self._print_intline("abort: compiling", cnt[ABORT_BRIDGE]) + self._print_intline("abort: vable escape", cnt[ABORT_ESCAPE]) self._print_intline("nvirtuals", cnt[NVIRTUALS]) self._print_intline("nvholes", cnt[NVHOLES]) self._print_intline("nvreused", cnt[NVREUSED]) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py Mon Dec 21 16:13:02 2009 @@ -736,6 +736,50 @@ def optimize_OOIS(self, op): self._optimize_oois_ooisnot(op, False) + def optimize_VIRTUAL_REF(self, op): + indexbox = op.args[1] + # + # get some constants (these calls are all 'memo') + from pypy.jit.metainterp import virtualref + cpu = self.cpu + c_cls = virtualref.get_jit_virtual_ref_const_class(cpu) + descr_virtual_token = virtualref.get_descr_virtual_token(cpu) + descr_virtualref_index = virtualref.get_descr_virtualref_index(cpu) + # + # Replace the VIRTUAL_REF operation with a virtual structure of type + # 'jit_virtual_ref'. The jit_virtual_ref structure may be forced soon, + # but the point is that doing so does not force the original structure. + op = ResOperation(rop.NEW_WITH_VTABLE, [c_cls], op.result) + vrefvalue = self.make_virtual(c_cls, op.result, op) + tokenbox = BoxInt() + self.emit_operation(ResOperation(rop.FORCE_TOKEN, [], tokenbox)) + vrefvalue.setfield(descr_virtual_token, self.getvalue(tokenbox)) + vrefvalue.setfield(descr_virtualref_index, self.getvalue(indexbox)) + + def optimize_VIRTUAL_REF_FINISH(self, op): + # Set the 'forced' field of the virtual_ref. + # In good cases, this is all virtual, so has no effect. + # Otherwise, this forces the real object -- but only now, as + # opposed to much earlier. This is important because the object is + # typically a PyPy PyFrame, and now is the end of its execution, so + # forcing it now does not have catastrophic effects. + from pypy.jit.metainterp import virtualref + # - set 'forced' to point to the real object + op1 = ResOperation(rop.SETFIELD_GC, op.args, None, + descr = virtualref.get_descr_forced(self.cpu)) + self.optimize_SETFIELD_GC(op1) + # - set 'virtual_token' to TOKEN_NONE + args = [op.args[0], ConstInt(0)] + op1 = ResOperation(rop.SETFIELD_GC, args, None, + descr = virtualref.get_descr_virtual_token(self.cpu)) + self.optimize_SETFIELD_GC(op1) + # Note that in some cases the virtual in op.args[1] has been forced + # already. This is fine. In that case, and *if* a residual + # CALL_MAY_FORCE suddenly turns out to access it, then it will + # trigger a ResumeGuardForcedDescr.handle_async_forcing() which + # will work too (but just be a little pointless, as the structure + # was already forced). + def optimize_GETFIELD_GC(self, op): value = self.getvalue(op.args[0]) if value.is_virtual(): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Mon Dec 21 16:13:02 2009 @@ -5,13 +5,13 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_start, debug_stop, debug_print -from pypy.jit.metainterp import history, compile, resume +from pypy.jit.metainterp import history, compile, resume, virtualref from pypy.jit.metainterp.history import Const, ConstInt, Box from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp import codewriter, executor from pypy.jit.metainterp.logger import Logger from pypy.jit.metainterp.jitprof import BLACKHOLED_OPS, EmptyProfiler -from pypy.jit.metainterp.jitprof import GUARDS, RECORDED_OPS +from pypy.jit.metainterp.jitprof import GUARDS, RECORDED_OPS, ABORT_ESCAPE from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG, ABORT_BRIDGE from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import specialize @@ -834,8 +834,7 @@ try: self.metainterp.reached_can_enter_jit(self.env) except GiveUp: - self.metainterp.staticdata.profiler.count(ABORT_BRIDGE) - self.metainterp.switch_to_blackhole() + self.metainterp.switch_to_blackhole(ABORT_BRIDGE) if self.metainterp.is_blackholing(): self.blackhole_reached_merge_point(self.env) return True @@ -846,6 +845,7 @@ greenkey = self.env[:num_green_args] sd = self.metainterp.staticdata loc = sd.state.get_location_str(greenkey) + debug_print(loc) constloc = self.metainterp.cpu.ts.conststr(loc) self.metainterp.history.record(rop.DEBUG_MERGE_POINT, [constloc], None) @@ -886,6 +886,53 @@ return self.metainterp.finishframe_exception(self.exception_box, self.exc_value_box) + @arguments("box") + def opimpl_virtual_ref(self, box): + # Details on the content of metainterp.virtualref_boxes: + # + # * it's a list whose items go two by two, containing first the + # virtual box (e.g. the PyFrame) and then the vref box (e.g. + # the 'virtual_ref(frame)'). + # + # * if we detect that the virtual box escapes during tracing + # already (by generating a CALl_MAY_FORCE that marks the flags + # in the vref), then we replace the vref in the list with + # ConstPtr(NULL). + # + metainterp = self.metainterp + if metainterp.is_blackholing(): + resbox = box # good enough when blackholing + else: + obj = box.getref_base() + vref = virtualref.virtual_ref_during_tracing(obj) + resbox = history.BoxPtr(vref) + cindex = history.ConstInt(len(metainterp.virtualref_boxes) // 2) + metainterp.history.record(rop.VIRTUAL_REF, [box, cindex], resbox) + # Note: we allocate a JIT_VIRTUAL_REF here + # (in virtual_ref_during_tracing()), in order to detect when + # the virtual escapes during tracing already. We record it as a + # VIRTUAL_REF operation, although the backend sees this operation + # as a no-op. The point is that the backend should not really see + # it in practice, as optimizeopt.py should either kill it or + # replace it with a NEW_WITH_VTABLE followed by SETFIELD_GCs. + metainterp.virtualref_boxes.append(box) + metainterp.virtualref_boxes.append(resbox) + self.make_result_box(resbox) + + @arguments("box") + def opimpl_virtual_ref_finish(self, box): + # virtual_ref_finish() assumes that we have a stack-like, last-in + # first-out order. + metainterp = self.metainterp + vrefbox = metainterp.virtualref_boxes.pop() + lastbox = metainterp.virtualref_boxes.pop() + assert box.getref_base() == lastbox.getref_base() + if not metainterp.is_blackholing(): + vref = vrefbox.getref_base() + if virtualref.is_virtual_ref(vref): + metainterp.history.record(rop.VIRTUAL_REF_FINISH, + [vrefbox, lastbox], None) + # ------------------------------ def setup_call(self, argboxes): @@ -947,7 +994,7 @@ if metainterp.staticdata.virtualizable_info is not None: virtualizable_boxes = metainterp.virtualizable_boxes resume.capture_resumedata(metainterp.framestack, virtualizable_boxes, - resumedescr) + metainterp.virtualref_boxes, resumedescr) self.metainterp.staticdata.profiler.count_ops(opnum, GUARDS) # count metainterp.attach_debug_info(guard_op) @@ -988,13 +1035,13 @@ def do_residual_call(self, argboxes, descr, exc): effectinfo = descr.get_extra_info() - if effectinfo is None or effectinfo.promotes_virtualizables: + if effectinfo is None or effectinfo.forces_virtual_or_virtualizable: # residual calls require attention to keep virtualizables in-sync - self.metainterp.vable_before_residual_call() + self.metainterp.vable_and_vrefs_before_residual_call() # xxx do something about code duplication resbox = self.metainterp.execute_and_record_varargs( rop.CALL_MAY_FORCE, argboxes, descr=descr) - self.metainterp.vable_after_residual_call() + self.metainterp.vable_and_vrefs_after_residual_call() if resbox is not None: self.make_result_box(resbox) self.generate_guard(self.pc, rop.GUARD_NOT_FORCED, None, []) @@ -1234,8 +1281,7 @@ try: self.compile_done_with_this_frame(resultbox) except GiveUp: - self.staticdata.profiler.count(ABORT_BRIDGE) - self.switch_to_blackhole() + self.switch_to_blackhole(ABORT_BRIDGE) sd = self.staticdata if sd.result_type == 'void': assert resultbox is None @@ -1272,8 +1318,7 @@ try: self.compile_exit_frame_with_exception(excvaluebox) except GiveUp: - self.staticdata.profiler.count(ABORT_BRIDGE) - self.switch_to_blackhole() + self.switch_to_blackhole(ABORT_BRIDGE) raise self.staticdata.ExitFrameWithExceptionRef(self.cpu, excvaluebox.getref_base()) def check_recursion_invariant(self): @@ -1391,7 +1436,8 @@ op.pc = self.framestack[-1].pc op.name = self.framestack[-1].jitcode.name - def switch_to_blackhole(self): + def switch_to_blackhole(self, reason): + self.staticdata.profiler.count(reason) debug_print('~~~ ABORTING TRACING') debug_stop('jit-tracing') debug_start('jit-blackhole') @@ -1399,15 +1445,15 @@ self.staticdata.stats.aborted() self.staticdata.profiler.end_tracing() self.staticdata.profiler.start_blackhole() + switch_to_blackhole._dont_inline_ = True def switch_to_blackhole_if_trace_too_long(self): if not self.is_blackholing(): warmrunnerstate = self.staticdata.state if len(self.history.operations) > warmrunnerstate.trace_limit: - self.staticdata.profiler.count(ABORT_TOO_LONG) self.greenkey_of_huge_function = self.find_biggest_function() self.portal_trace_positions = None - self.switch_to_blackhole() + self.switch_to_blackhole(ABORT_TOO_LONG) def _interpret(self): # Execute the frames forward until we raise a DoneWithThisFrame, @@ -1531,6 +1577,7 @@ len(self.virtualizable_boxes)-1, duplicates) live_arg_boxes += self.virtualizable_boxes[:-1] + assert len(self.virtualref_boxes) == 0, "missing virtual_ref_finish()?" # Called whenever we reach the 'can_enter_jit' hint. # First, attempt to make a bridge: # - if self.resumekey is a ResumeGuardDescr, it starts from a guard @@ -1686,6 +1733,7 @@ f = self.newframe(self.staticdata.portal_code) f.pc = 0 f.env = original_boxes[:] + self.virtualref_boxes = [] self.initialize_virtualizable(original_boxes) return original_boxes @@ -1723,9 +1771,17 @@ virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) vinfo.clear_vable_token(virtualizable) - def vable_before_residual_call(self): + def vable_and_vrefs_before_residual_call(self): if self.is_blackholing(): return + # + for i in range(1, len(self.virtualref_boxes), 2): + vrefbox = self.virtualref_boxes[i] + vref = vrefbox.getref_base() + virtualref.tracing_before_residual_call(vref) + # the FORCE_TOKEN is already set at runtime in each vref when + # it is created, by optimizeopt.py. + # vinfo = self.staticdata.virtualizable_info if vinfo is not None: virtualizable_box = self.virtualizable_boxes[-1] @@ -1738,23 +1794,49 @@ force_token_box], None, descr=vinfo.vable_token_descr) - def vable_after_residual_call(self): + def vable_and_vrefs_after_residual_call(self): if self.is_blackholing(): - vable_escapes = True + escapes = True else: - vable_escapes = False + escapes = False + # + for i in range(0, len(self.virtualref_boxes), 2): + virtualbox = self.virtualref_boxes[i] + vrefbox = self.virtualref_boxes[i+1] + vref = vrefbox.getref_base() + if virtualref.tracing_after_residual_call(vref): + # this vref was really a virtual_ref, but it escaped + # during this CALL_MAY_FORCE. Mark this fact by + # generating a VIRTUAL_REF_FINISH on it and replacing + # it by ConstPtr(NULL). + self.stop_tracking_virtualref(i) + # vinfo = self.staticdata.virtualizable_info if vinfo is not None: virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) if vinfo.tracing_after_residual_call(virtualizable): - # We just did the residual call, and it shows that the - # virtualizable escapes. - self.switch_to_blackhole() - vable_escapes = True - if vable_escapes: + # the virtualizable escaped during CALL_MAY_FORCE. + escapes = True + # + if escapes: + self.switch_to_blackhole(ABORT_ESCAPE) + # + if escapes: self.load_fields_from_virtualizable() + def stop_tracking_virtualref(self, i): + virtualbox = self.virtualref_boxes[i] + vrefbox = self.virtualref_boxes[i+1] + # record VIRTUAL_REF_FINISH just before the current CALL_MAY_FORCE + call_may_force_op = self.history.operations.pop() + assert call_may_force_op.opnum == rop.CALL_MAY_FORCE + self.history.record(rop.VIRTUAL_REF_FINISH, + [vrefbox, virtualbox], None) + self.history.operations.append(call_may_force_op) + # mark by replacing it with ConstPtr(NULL) + self.virtualref_boxes[i+1] = self.cpu.ts.CONST_NULL + def handle_exception(self): etype = self.cpu.get_exception() evalue = self.cpu.get_exc_value() @@ -1787,7 +1869,19 @@ vinfo = self.staticdata.virtualizable_info self.framestack = [] expect_virtualizable = vinfo is not None - virtualizable_boxes = resume.rebuild_from_resumedata(self, newboxes, resumedescr, expect_virtualizable) + virtualizable_boxes, virtualref_boxes = resume.rebuild_from_resumedata( + self, newboxes, resumedescr, expect_virtualizable) + # + # virtual refs: make the vrefs point to the freshly allocated virtuals + self.virtualref_boxes = virtualref_boxes + for i in range(0, len(virtualref_boxes), 2): + virtualbox = virtualref_boxes[i] + vrefbox = virtualref_boxes[i+1] + virtualref.continue_tracing(vrefbox.getref_base(), + virtualbox.getref_base()) + # + # virtualizable: synchronize the real virtualizable and the local + # boxes, in whichever direction is appropriate if expect_virtualizable: self.virtualizable_boxes = virtualizable_boxes if self._already_allocated_resume_virtuals is not None: @@ -1796,12 +1890,19 @@ self.load_fields_from_virtualizable() return # just jumped away from assembler (case 4 in the comment in - # virtualizable.py) into tracing (case 2); check that vable_rti - # is and stays NULL. + # virtualizable.py) into tracing (case 2); check that vable_token + # is and stays 0. Note the call to reset_vable_token() in + # warmstate.py. virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) assert not virtualizable.vable_token - self.synchronize_virtualizable() + if self._already_allocated_resume_virtuals is not None: + # resuming from a ResumeGuardForcedDescr: load the new values + # currently stored on the virtualizable fields + self.load_fields_from_virtualizable() + else: + # normal case: fill the virtualizable with the local boxes + self.synchronize_virtualizable() def check_synchronized_virtualizable(self): if not we_are_translated(): @@ -1856,6 +1957,10 @@ for i in range(len(boxes)): if boxes[i] is oldbox: boxes[i] = newbox + boxes = self.virtualref_boxes + for i in range(len(boxes)): + if boxes[i] is oldbox: + boxes[i] = newbox if self.staticdata.virtualizable_info is not None: boxes = self.virtualizable_boxes for i in range(len(boxes)): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py Mon Dec 21 16:13:02 2009 @@ -205,6 +205,8 @@ 'NEW/0d', 'NEW_WITH_VTABLE/1', 'NEW_ARRAY/1d', + 'FORCE_TOKEN/0', + 'VIRTUAL_REF/2', '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- 'SETARRAYITEM_GC/3d', @@ -221,7 +223,7 @@ 'COND_CALL_GC_MALLOC', # [a, b, if_(a<=b)_result, if_(a>b)_call, args...] # => result (for mallocs) 'DEBUG_MERGE_POINT/1', # debugging only - 'FORCE_TOKEN/0', + 'VIRTUAL_REF_FINISH/2', '_CANRAISE_FIRST', # ----- start of can_raise operations ----- 'CALL', Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py Mon Dec 21 16:13:02 2009 @@ -42,7 +42,8 @@ back.parent_resumedata_snapshot, back.env[:]) -def capture_resumedata(framestack, virtualizable_boxes, storage): +def capture_resumedata(framestack, virtualizable_boxes, virtualref_boxes, + storage): n = len(framestack)-1 top = framestack[n] _ensure_parent_resumedata(framestack, n) @@ -50,6 +51,7 @@ top) storage.rd_frame_info_list = frame_info_list snapshot = Snapshot(top.parent_resumedata_snapshot, top.env[:]) + snapshot = Snapshot(snapshot, virtualref_boxes[:]) # xxx for now if virtualizable_boxes is not None: snapshot = Snapshot(snapshot, virtualizable_boxes[:]) # xxx for now storage.rd_snapshot = snapshot @@ -434,11 +436,13 @@ debug_print("\t\t", str(untag(i))) -def rebuild_from_resumedata(metainterp, newboxes, storage, expects_virtualizables): +def rebuild_from_resumedata(metainterp, newboxes, storage, + expects_virtualizables): resumereader = ResumeDataReader(storage, newboxes, metainterp) virtualizable_boxes = None if expects_virtualizables: virtualizable_boxes = resumereader.consume_boxes() + virtualref_boxes = resumereader.consume_boxes() frameinfo = storage.rd_frame_info_list while True: env = resumereader.consume_boxes() @@ -448,11 +452,16 @@ if frameinfo is None: break metainterp.framestack.reverse() - return virtualizable_boxes + return virtualizable_boxes, virtualref_boxes -def force_from_resumedata(metainterp, newboxes, storage): +def force_from_resumedata(metainterp, newboxes, storage, + expects_virtualizables): resumereader = ResumeDataReader(storage, newboxes, metainterp) - return resumereader.consume_boxes(), resumereader.virtuals + virtualizable_boxes = None + if expects_virtualizables: + virtualizable_boxes = resumereader.consume_boxes() + virtualref_boxes = resumereader.consume_boxes() + return virtualizable_boxes, virtualref_boxes, resumereader.virtuals class ResumeDataReader(object): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/support.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/support.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/support.py Mon Dec 21 16:13:02 2009 @@ -3,6 +3,7 @@ from pypy.rpython import rlist from pypy.rpython.lltypesystem import rstr as ll_rstr, rdict as ll_rdict from pypy.rpython.lltypesystem import rlist as lltypesystem_rlist +from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.ootypesystem import rdict as oo_rdict from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.extregistry import ExtRegistryEntry @@ -136,6 +137,9 @@ def _ll_1_gc_identityhash(x): return lltype.identityhash(x) +def _ll_1_jit_force_virtual(inst): + return llop.jit_force_virtual(lltype.typeOf(inst), inst) + class LLtypeHelpers: Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_basic.py Mon Dec 21 16:13:02 2009 @@ -52,6 +52,8 @@ assert get_stats().exec_jumps <= maxcount def check_aborted_count(self, count): assert get_stats().aborted_count == count + def check_aborted_count_at_least(self, count): + assert get_stats().aborted_count >= count def meta_interp(self, *args, **kwds): kwds['CPUClass'] = self.CPUClass Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py Mon Dec 21 16:13:02 2009 @@ -2,7 +2,7 @@ from pypy.rlib import jit from pypy.jit.metainterp import support, typesystem from pypy.jit.metainterp.policy import JitPolicy -from pypy.jit.metainterp.codewriter import CodeWriter +from pypy.jit.metainterp.codewriter import CodeWriter, ForcingVirtualRef from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.translator.translator import graphof from pypy.rpython.lltypesystem.rbuiltin import ll_instantiate @@ -283,7 +283,7 @@ assert calldescrs[0][4] is not None assert not calldescrs[0][4].write_descrs_fields assert not calldescrs[0][4].write_descrs_arrays - assert not calldescrs[0][4].promotes_virtualizables + assert not calldescrs[0][4].forces_virtual_or_virtualizable def test_oosend_look_inside_only_one(self): class A: @@ -394,7 +394,7 @@ assert cw.list_of_addr2name[0][1].endswith('.A1') assert cw.list_of_addr2name[1][1] == 'A1.g' - def test_promote_virtualizable_effectinfo(self): + def test_jit_force_virtualizable_effectinfo(self): class Frame(object): _virtualizable2_ = ['x'] @@ -430,9 +430,38 @@ effectinfo_g1 = calldescrs[1][4] effectinfo_g2 = calldescrs[2][4] effectinfo_h = calldescrs[3][4] - assert effectinfo_g1.promotes_virtualizables - assert effectinfo_g2.promotes_virtualizables - assert not effectinfo_h.promotes_virtualizables + assert effectinfo_g1.forces_virtual_or_virtualizable + assert effectinfo_g2.forces_virtual_or_virtualizable + assert not effectinfo_h.forces_virtual_or_virtualizable + + def test_vref_simple(self): + class X: + pass + def f(): + return jit.virtual_ref(X()) + graphs = self.make_graphs(f, []) + assert graphs[0].func is f + assert graphs[1].func is jit.virtual_ref + cw = CodeWriter(self.rtyper) + cw.candidate_graphs = [graphs[0]] + cw._start(self.metainterp_sd, None) + jitcode = cw.make_one_bytecode((graphs[0], None), False) + assert 'virtual_ref' in jitcode._source + + def test_vref_forced(self): + class X: + pass + def f(): + vref = jit.virtual_ref(X()) + return vref() + graphs = self.make_graphs(f, []) + assert graphs[0].func is f + assert graphs[1].func is jit.virtual_ref + cw = CodeWriter(self.rtyper) + cw.candidate_graphs = [graphs[0]] + cw._start(self.metainterp_sd, None) + py.test.raises(ForcingVirtualRef, cw.make_one_bytecode, + (graphs[0], None), False) # assert it does not work class ImmutableFieldsTests: Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_jitprof.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_jitprof.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_jitprof.py Mon Dec 21 16:13:02 2009 @@ -64,7 +64,7 @@ assert profiler.events == expected assert profiler.times == [2, 1, 1, 1] assert profiler.counters == [1, 1, 1, 1, 4, 3, 1, 1, 7, 1, 0, 0, 0, - 0, 0, 0] + 0, 0, 0, 0] def test_simple_loop_with_call(self): @dont_look_inside Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py Mon Dec 21 16:13:02 2009 @@ -1,6 +1,6 @@ import py, random -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE @@ -38,12 +38,13 @@ type_system = 'lltype' def get_class_of_box(self, box): - from pypy.rpython.lltypesystem import rclass return box.getref(rclass.OBJECTPTR).typeptr node_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True) + node_vtable.name = rclass.alloc_array_name('node') node_vtable_adr = llmemory.cast_ptr_to_adr(node_vtable) node_vtable2 = lltype.malloc(OBJECT_VTABLE, immortal=True) + node_vtable2.name = rclass.alloc_array_name('node2') node_vtable_adr2 = llmemory.cast_ptr_to_adr(node_vtable2) cpu = runner.LLtypeCPU(None) @@ -98,6 +99,13 @@ nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([], [])) writeadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([adescr], [])) writearraydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([adescr], [arraydescr])) + mayforcevirtdescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo([], [], forces_virtual_or_virtualizable=True)) + + from pypy.jit.metainterp.virtualref import jit_virtual_ref_vtable + from pypy.jit.metainterp.virtualref import JIT_VIRTUAL_REF + virtualtokendescr = cpu.fielddescrof(JIT_VIRTUAL_REF, 'virtual_token') + virtualrefindexdescr = cpu.fielddescrof(JIT_VIRTUAL_REF,'virtualref_index') cpu.class_sizes = {cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), cpu.cast_adr_to_int(node_vtable_adr2): cpu.sizeof(NODE2), Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py Mon Dec 21 16:13:02 2009 @@ -2031,6 +2031,81 @@ """ self.optimize_loop(ops, 'Not, Not, Not', expected) + def test_vref_nonvirtual(self): + ops = """ + [p1] + p2 = virtual_ref(p1, 5) + jump(p1) + """ + py.test.raises(compile.GiveUp, self.optimize_loop, ops, 'Not', ops) + + def test_vref_virtual_1(self): + ops = """ + [p0, i1] + # + p1 = new_with_vtable(ConstClass(node_vtable)) + p1b = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1b, 252, descr=valuedescr) + setfield_gc(p1, p1b, descr=nextdescr) + # + p2 = virtual_ref(p1, 3) + setfield_gc(p0, p2, descr=nextdescr) + call_may_force(i1, descr=mayforcevirtdescr) + guard_not_forced() [i1] + setfield_gc(p0, NULL, descr=nextdescr) + jump(p0, i1) + """ + expected = """ + [p0, i1] + i3 = force_token() + p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, 3, descr=virtualrefindexdescr) + setfield_gc(p0, p2, descr=nextdescr) + call_may_force(i1, descr=mayforcevirtdescr) + guard_not_forced() [i1] + setfield_gc(p0, NULL, descr=nextdescr) + jump(p0, i1) + """ + self.optimize_loop(ops, 'Not, Not', expected) + + def test_vref_virtual_2(self): + self.make_fail_descr() + ops = """ + [p0, i1] + # + p1 = new_with_vtable(ConstClass(node_vtable)) + p1b = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1b, i1, descr=valuedescr) + setfield_gc(p1, p1b, descr=nextdescr) + # + p2 = virtual_ref(p1, 2) + setfield_gc(p0, p2, descr=nextdescr) + call_may_force(i1, descr=mayforcevirtdescr) + guard_not_forced(descr=fdescr) [p1] + setfield_gc(p0, NULL, descr=nextdescr) + jump(p0, i1) + """ + expected = """ + [p0, i1] + i3 = force_token() + p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, 2, descr=virtualrefindexdescr) + setfield_gc(p0, p2, descr=nextdescr) + call_may_force(i1, descr=mayforcevirtdescr) + guard_not_forced(descr=fdescr) [i1] + setfield_gc(p0, NULL, descr=nextdescr) + jump(p0, i1) + """ + # the point of this test is that 'i1' should show up in the fail_args + # of 'guard_not_forced', because it was stored in the virtual 'p1b'. + self.optimize_loop(ops, 'Not, Not', expected) + self.check_expanded_fail_descr('''p1 + where p1 is a node_vtable, nextdescr=p1b + where p1b is a node_vtable, valuedescr=i1 + ''') + class TestOOtype(BaseTestOptimizeOpt, OOtypeMixin): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_pyjitpl.py Mon Dec 21 16:13:02 2009 @@ -33,7 +33,8 @@ def test_simple_opimpl_exist(): rop = resoperation.rop for opnum, opname in resoperation.opname.items(): - if opnum in (rop.SAME_AS, rop.CALL_PURE, rop.OOSEND_PURE): + if opnum in (rop.SAME_AS, rop.CALL_PURE, rop.OOSEND_PURE, + rop.FORCE_TOKEN): continue if rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST: assert hasattr(pyjitpl.MIFrame, 'opimpl_' + opname.lower()), opname Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_resume.py Mon Dec 21 16:13:02 2009 @@ -201,29 +201,32 @@ fs = [FakeFrame("code0", 0, -1, b1, c1, b2)] storage = Storage() - capture_resumedata(fs, None, storage) + capture_resumedata(fs, None, [], storage) assert fs[0].parent_resumedata_snapshot is None assert fs[0].parent_resumedata_frame_info_list is None assert storage.rd_frame_info_list.prev is None assert storage.rd_frame_info_list.jitcode == 'code0' - assert storage.rd_snapshot.prev is None - assert storage.rd_snapshot.boxes == fs[0].env - assert storage.rd_snapshot.boxes is not fs[0].env + assert storage.rd_snapshot.boxes == [] # for virtualrefs + snapshot = storage.rd_snapshot.prev + assert snapshot.prev is None + assert snapshot.boxes == fs[0].env + assert snapshot.boxes is not fs[0].env storage = Storage() fs = [FakeFrame("code0", 0, -1, b1, c1, b2), FakeFrame("code1", 3, 7, b3, c2, b1), FakeFrame("code2", 9, -1, c3, b2)] - capture_resumedata(fs, None, storage) + capture_resumedata(fs, None, [], storage) frame_info_list = storage.rd_frame_info_list assert frame_info_list.prev is fs[2].parent_resumedata_frame_info_list assert frame_info_list.jitcode == 'code2' assert frame_info_list.pc == 9 - snapshot = storage.rd_snapshot + assert storage.rd_snapshot.boxes == [] # for virtualrefs + snapshot = storage.rd_snapshot.prev assert snapshot.prev is fs[2].parent_resumedata_snapshot assert snapshot.boxes == fs[2].env assert snapshot.boxes is not fs[2].env @@ -247,8 +250,9 @@ fs[2].env = [b2, b3] fs[2].pc = 15 vbs = [b1, b2] - capture_resumedata(fs, vbs, storage) - + vrs = [b3] + capture_resumedata(fs, vbs, vrs, storage) + frame_info_list = storage.rd_frame_info_list assert frame_info_list.prev is fs[2].parent_resumedata_frame_info_list assert frame_info_list.jitcode == 'code2' @@ -259,6 +263,10 @@ assert snapshot.boxes is not vbs snapshot = snapshot.prev + assert snapshot.boxes == vrs + assert snapshot.boxes is not vrs + + snapshot = snapshot.prev assert snapshot.prev is fs[2].parent_resumedata_snapshot assert snapshot.boxes == fs[2].env @@ -276,7 +284,7 @@ fs = [FakeFrame("code0", 0, -1, b1, c1, b2), FakeFrame("code1", 3, 7, b3, c2, b1), FakeFrame("code2", 9, -1, c3, b2)] - capture_resumedata(fs, None, storage) + capture_resumedata(fs, None, [], storage) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish({}) @@ -287,7 +295,7 @@ result = rebuild_from_resumedata(metainterp, newboxes, storage, False) - assert result is None + assert result == (None, []) fs2 = [FakeFrame("code0", 0, -1, b1t, c1, b2t), FakeFrame("code1", 3, 7, b3t, c2, b1t), FakeFrame("code2", 9, -1, c3, b2t)] @@ -300,7 +308,7 @@ fs = [FakeFrame("code0", 0, -1, b1, c1, b2), FakeFrame("code1", 3, 7, b3, c2, b1), FakeFrame("code2", 9, -1, c3, b2)] - capture_resumedata(fs, [b4], storage) + capture_resumedata(fs, [b4], [], storage) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish({}) @@ -311,7 +319,7 @@ result = rebuild_from_resumedata(metainterp, newboxes, storage, True) - assert result == [b4t] + assert result == ([b4t], []) fs2 = [FakeFrame("code0", 0, -1, b1t, c1, b2t), FakeFrame("code1", 3, 7, b3t, c2, b1t), FakeFrame("code2", 9, -1, c3, b2t)] @@ -324,10 +332,10 @@ fs = [FakeFrame("code0", 0, -1, b1, c1, b2), FakeFrame("code1", 3, 7, b3, c2, b1), FakeFrame("code2", 9, -1, c3, b2)] - capture_resumedata(fs, None, storage) + capture_resumedata(fs, None, [], storage) storage2 = Storage() fs = fs[:-1] + [FakeFrame("code2", 10, -1, c3, b2, b4)] - capture_resumedata(fs, None, storage2) + capture_resumedata(fs, None, [], storage2) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) @@ -343,7 +351,7 @@ result = rebuild_from_resumedata(metainterp, newboxes, storage, False) - assert result is None + assert result == (None, []) fs2 = [FakeFrame("code0", 0, -1, b1t, c1, b2t), FakeFrame("code1", 3, 7, b3t, c2, b1t), FakeFrame("code2", 9, -1, c3, b2t)] @@ -354,7 +362,7 @@ metainterp.framestack = [] result = rebuild_from_resumedata(metainterp, newboxes, storage2, False) - assert result is None + assert result == (None, []) fs2 = fs2[:-1] + [FakeFrame("code2", 10, -1, c3, b2t, b4t)] assert metainterp.framestack == fs2 @@ -382,10 +390,10 @@ fs = [FakeFrame("code0", 0, -1, b1, c1, b2), FakeFrame("code1", 3, 7, b3, c2, b1), FakeFrame("code2", 9, -1, c3, b2)] - capture_resumedata(fs, None, storage) + capture_resumedata(fs, None, [], storage) storage2 = Storage() fs = fs[:-1] + [FakeFrame("code2", 10, -1, c3, b2, b4)] - capture_resumedata(fs, None, storage2) + capture_resumedata(fs, None, [], storage2) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) values = {b2: virtual_value(b2, b5, c4)} @@ -441,7 +449,7 @@ LLtypeMixin.nodebox.constbox()] storage = Storage() fs = [FakeFrame("code0", 0, -1, c1, b2, b3)] - capture_resumedata(fs, None, storage) + capture_resumedata(fs, None, [], storage) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) values = {b2: virtual_value(b2, b5, c4)} @@ -453,7 +461,7 @@ storage2 = Storage() fs = [FakeFrame("code0", 0, -1, b1, b4, b2)] - capture_resumedata(fs, None, storage2) + capture_resumedata(fs, None, [], storage2) values[b4] = virtual_value(b4, b6, c4) modifier = ResumeDataVirtualAdder(storage2, memo) liveboxes = modifier.finish(values) @@ -466,7 +474,7 @@ b1, b2, b3 = [BoxPtr(), BoxPtr(), BoxInt()] storage = Storage() fs = [FakeFrame("code0", 0, -1, b1, b2)] - capture_resumedata(fs, None, storage) + capture_resumedata(fs, None, [], storage) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) v1 = virtual_value(b1, b3, None) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Mon Dec 21 16:13:02 2009 @@ -28,7 +28,7 @@ hop.inputconst(lltype.Void, hop.args_v[1].value), hop.inputconst(lltype.Void, {})] hop.exception_cannot_occur() - return hop.genop('promote_virtualizable', + return hop.genop('jit_force_virtualizable', args_v, resulttype=lltype.Void) debug_print = lloperation.llop.debug_print @@ -907,6 +907,39 @@ res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) assert res == f(123) + def test_bridge_forces(self): + jitdriver = JitDriver(greens = [], reds = ['frame'], + virtualizables = ['frame']) + + class Frame(object): + _virtualizable2_ = ['x', 'y'] + class SomewhereElse: + pass + somewhere_else = SomewhereElse() + + def g(): + n = somewhere_else.top_frame.y + 700 + debug_print(lltype.Void, '-+-+-+-+- external write:', n) + somewhere_else.top_frame.y = n + + def f(n): + frame = Frame() + frame.x = n + frame.y = 10 + somewhere_else.counter = 0 + somewhere_else.top_frame = frame + while frame.x > 0: + jitdriver.can_enter_jit(frame=frame) + jitdriver.jit_merge_point(frame=frame) + if frame.y > 17: + g() + frame.x -= 5 + frame.y += 1 + return frame.y + + res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) + assert res == f(123) + def test_promote_index_in_virtualizable_list(self): jitdriver = JitDriver(greens = [], reds = ['frame', 'n'], virtualizables = ['frame']) @@ -980,9 +1013,11 @@ for block, op in graph.iterblockops() if op.opname == 'direct_call'] - assert direct_calls(f_graph) == ['__init__', 'force_if_necessary', 'll_portal_runner'] - assert direct_calls(portal_graph) == ['force_if_necessary', 'maybe_enter_jit'] - + assert direct_calls(f_graph) == ['__init__', + 'force_virtualizable_if_necessary', + 'll_portal_runner'] + assert direct_calls(portal_graph)==['force_virtualizable_if_necessary', + 'maybe_enter_jit'] assert direct_calls(init_graph) == [] def test_virtual_child_frame(self): @@ -1158,8 +1193,7 @@ self.check_loops(getfield_gc=0, setfield_gc=0) def test_blackhole_should_not_reenter(self): - from pypy.jit.backend.test.support import BaseCompiledMixin - if isinstance(self, BaseCompiledMixin): + if not self.basic: py.test.skip("purely frontend test") myjitdriver = JitDriver(greens = [], reds = ['frame', 'fail'], Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py Mon Dec 21 16:13:02 2009 @@ -11,8 +11,8 @@ class VirtualizableInfo: - token_none = 0 - token_tracing = -1 + TOKEN_NONE = 0 + TOKEN_TRACING_RESCALL = -1 def __init__(self, warmrunnerdesc): self.warmrunnerdesc = warmrunnerdesc @@ -153,16 +153,17 @@ def finish(self): # - def force_if_necessary(virtualizable): + def force_virtualizable_if_necessary(virtualizable): if virtualizable.vable_token: self.force_now(virtualizable) - force_if_necessary._always_inline_ = True + force_virtualizable_if_necessary._always_inline_ = True # all_graphs = self.warmrunnerdesc.translator.graphs ts = self.warmrunnerdesc.cpu.ts (_, FUNCPTR) = ts.get_FuncType([self.VTYPEPTR], lltype.Void) - funcptr = self.warmrunnerdesc.helper_func(FUNCPTR, force_if_necessary) - rvirtualizable2.replace_promote_virtualizable_with_call( + funcptr = self.warmrunnerdesc.helper_func( + FUNCPTR, force_virtualizable_if_necessary) + rvirtualizable2.replace_force_virtualizable_with_call( all_graphs, self.VTYPEPTR, funcptr) def unwrap_virtualizable_box(self, virtualizable_box): @@ -176,7 +177,7 @@ return rvirtualizable2.match_virtualizable_type(TYPE, self.VTYPEPTR) def reset_vable_token(self, virtualizable): - virtualizable.vable_token = self.token_none + virtualizable.vable_token = self.TOKEN_NONE def clear_vable_token(self, virtualizable): if virtualizable.vable_token: @@ -185,14 +186,14 @@ def tracing_before_residual_call(self, virtualizable): assert not virtualizable.vable_token - virtualizable.vable_token = self.token_tracing + virtualizable.vable_token = self.TOKEN_TRACING_RESCALL def tracing_after_residual_call(self, virtualizable): if virtualizable.vable_token: # not modified by the residual call; assert that it is still - # set to 'tracing_vable_rti' and clear it. - assert virtualizable.vable_token == self.token_tracing - virtualizable.vable_token = self.token_none + # set to TOKEN_TRACING_RESCALL and clear it. + assert virtualizable.vable_token == self.TOKEN_TRACING_RESCALL + virtualizable.vable_token = self.TOKEN_NONE return False else: # marker "modified during residual call" set. @@ -200,32 +201,36 @@ def force_now(self, virtualizable): token = virtualizable.vable_token - virtualizable.vable_token = self.token_none - if token == self.token_tracing: + if token == self.TOKEN_TRACING_RESCALL: # The values in the virtualizable are always correct during - # tracing. We only need to reset vable_token to token_none + # tracing. We only need to reset vable_token to TOKEN_NONE # as a marker for the tracing, to tell it that this # virtualizable escapes. - pass + virtualizable.vable_token = self.TOKEN_NONE else: from pypy.jit.metainterp.compile import ResumeGuardForcedDescr - faildescr = self.cpu.force(token) - assert isinstance(faildescr, ResumeGuardForcedDescr) - faildescr.force_virtualizable(self, virtualizable, token) + ResumeGuardForcedDescr.force_now(self.cpu, token) + assert virtualizable.vable_token == self.TOKEN_NONE force_now._dont_inline_ = True + def forced_vable(self, virtualizable_boxes): + virtualizable_box = virtualizable_boxes[-1] + virtualizable = self.unwrap_virtualizable_box(virtualizable_box) + self.write_boxes(virtualizable, virtualizable_boxes) + virtualizable.vable_token = self.TOKEN_NONE + # ____________________________________________________________ # # The 'vable_token' field of a virtualizable is either 0, -1, or points # into the CPU stack to a particular field in the current frame. It is: # -# 1. 0 (token_none) if not in the JIT at all, except as described below. +# 1. 0 (TOKEN_NONE) if not in the JIT at all, except as described below. # # 2. equal to 0 when tracing is in progress; except: # -# 3. equal to -1 (token_tracing) during tracing when we do a residual call, -# calling random unknown other parts of the interpreter; it is -# reset to 0 as soon as something occurs to the virtualizable. +# 3. equal to -1 (TOKEN_TRACING_RESCALL) during tracing when we do a +# residual call, calling random unknown other parts of the interpreter; +# it is reset to 0 as soon as something occurs to the virtualizable. # # 4. when running the machine code with a virtualizable, it is set # to the address in the CPU stack by the FORCE_TOKEN operation. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/warmspot.py Mon Dec 21 16:13:02 2009 @@ -1,4 +1,4 @@ -import sys +import sys, py from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr from pypy.rpython.ootypesystem import ootype from pypy.rpython.annlowlevel import llhelper, MixLevelHelperAnnotator,\ @@ -17,7 +17,7 @@ from pypy.translator.unsimplify import call_final_function from pypy.jit.metainterp import codewriter -from pypy.jit.metainterp import support, history, pyjitpl, gc +from pypy.jit.metainterp import support, history, pyjitpl, gc, virtualref from pypy.jit.metainterp.pyjitpl import MetaInterpStaticData, MetaInterp from pypy.jit.metainterp.policy import JitPolicy from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper @@ -177,6 +177,7 @@ ) self.rewrite_can_enter_jit() self.rewrite_set_param() + self.rewrite_force_virtual() self.add_profiler_finish() self.metainterp_sd.finish_setup(optimizer=optimizer) @@ -599,6 +600,12 @@ op.opname = 'direct_call' op.args[:3] = [closures[funcname]] + def rewrite_force_virtual(self): + if self.cpu.ts.name != 'lltype': + py.test.skip("rewrite_force_virtual: port it to ootype") + all_graphs = self.translator.graphs + virtualref.replace_force_virtual_with_call(self, all_graphs) + def decode_hp_hint_args(op): # Returns (list-of-green-vars, list-of-red-vars) without Voids. Modified: pypy/branch/virtual-forcing/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/branch/virtual-forcing/pypy/module/_stackless/interp_coroutine.py Mon Dec 21 16:13:02 2009 @@ -275,11 +275,10 @@ return space.newtuple([]) items = [None] * index f = self.subctx.topframe - f.force_f_back() while index > 0: index -= 1 items[index] = space.wrap(f) - f = f.f_back() + f = f.f_backref() assert f is None return space.newtuple(items) Modified: pypy/branch/virtual-forcing/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/virtual-forcing/pypy/module/pypyjit/interp_jit.py Mon Dec 21 16:13:02 2009 @@ -21,7 +21,7 @@ PyFrame._virtualizable2_ = ['last_instr', 'pycode', 'valuestackdepth', 'valuestack_w[*]', - 'fastlocals_w[*]', 'f_forward', + 'fastlocals_w[*]', 'last_exception', ] @@ -35,12 +35,6 @@ name = opcode_method_names[ord(bytecode.co_code[next_instr])] return '%s #%d %s' % (bytecode.get_repr(), next_instr, name) -def leave(next_instr, pycode, frame, ec): - from pypy.interpreter.executioncontext import ExecutionContext - # can't use a method here, since this function is seen later than the main - # annotation XXX no longer true, could be fixed - ExecutionContext._jit_rechain_frame(ec, frame) - def get_jitcell_at(next_instr, bytecode): return bytecode.jit_cells.get(next_instr, None) @@ -63,7 +57,6 @@ pypyjitdriver = PyPyJitDriver(can_inline = can_inline, get_printable_location = get_printable_location, - leave = leave, get_jitcell_at = get_jitcell_at, set_jitcell_at = set_jitcell_at) Modified: pypy/branch/virtual-forcing/pypy/module/sys/vm.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/sys/vm.py (original) +++ pypy/branch/virtual-forcing/pypy/module/sys/vm.py Mon Dec 21 16:13:02 2009 @@ -31,7 +31,6 @@ space.wrap("frame index must not be negative")) ec = space.getexecutioncontext() f = ec.gettopframe_nohidden() - f.force_f_back() while True: if f is None: raise OperationError(space.w_ValueError, Modified: pypy/branch/virtual-forcing/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/objspace/flow/flowcontext.py (original) +++ pypy/branch/virtual-forcing/pypy/objspace/flow/flowcontext.py Mon Dec 21 16:13:02 2009 @@ -5,6 +5,7 @@ from pypy.interpreter.argument import ArgumentsForTranslation from pypy.objspace.flow.model import * from pypy.objspace.flow.framestate import FrameState +from pypy.rlib import jit class OperationThatShouldNotBePropagatedError(OperationError): @@ -260,8 +261,8 @@ except StopFlowing: continue # restarting a dead SpamBlock try: - old_frame = self.some_frame - self.some_frame = frame + old_frameref = self.topframeref + self.topframeref = jit.non_virtual_ref(frame) self.crnt_frame = frame try: w_result = frame.dispatch(frame.pycode, @@ -269,7 +270,7 @@ self) finally: self.crnt_frame = None - self.some_frame = old_frame + self.topframeref = old_frameref except OperationThatShouldNotBePropagatedError, e: raise Exception( Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/jit.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/jit.py Mon Dec 21 16:13:02 2009 @@ -2,6 +2,7 @@ import sys from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rlib.objectmodel import CDefinedIntSymbolic +from pypy.rlib.objectmodel import keepalive_until_here from pypy.rlib.unroll import unrolling_iterable def purefunction(func): @@ -99,6 +100,66 @@ return hop.inputconst(lltype.Signed, _we_are_jitted) # ____________________________________________________________ +# VRefs + +def virtual_ref(x): + + """Creates a 'vref' object that contains a reference to 'x'. Calls + to virtual_ref/virtual_ref_finish must be properly nested. The idea + is that the object 'x' is supposed to be JITted as a virtual between + the calls to virtual_ref and virtual_ref_finish, but the 'vref' + object can escape at any point in time. If at runtime it is + dereferenced (by the call syntax 'vref()'), it returns 'x', which is + then forced.""" + return DirectJitVRef(x) +virtual_ref.oopspec = 'virtual_ref(x)' + +def virtual_ref_finish(x): + """See docstring in virtual_ref(x). Note that virtual_ref_finish + takes as argument the real object, not the vref.""" + keepalive_until_here(x) # otherwise the whole function call is removed +virtual_ref_finish.oopspec = 'virtual_ref_finish(x)' + +def non_virtual_ref(x): + """Creates a 'vref' that just returns x when called; nothing more special. + Used for None or for frames outside JIT scope.""" + return DirectVRef(x) + +# ---------- implementation-specific ---------- + +class DirectVRef(object): + def __init__(self, x): + self._x = x + def __call__(self): + return self._x + +class DirectJitVRef(DirectVRef): + def __init__(self, x): + assert x is not None, "virtual_ref(None) is not allowed" + DirectVRef.__init__(self, x) + +class Entry(ExtRegistryEntry): + _about_ = (non_virtual_ref, DirectJitVRef) + + def compute_result_annotation(self, s_obj): + from pypy.rlib import _jit_vref + return _jit_vref.SomeVRef(s_obj) + + def specialize_call(self, hop): + return hop.r_result.specialize_call(hop) + +class Entry(ExtRegistryEntry): + _type_ = DirectVRef + + def compute_annotation(self): + from pypy.rlib import _jit_vref + assert isinstance(self.instance, DirectVRef) + s_obj = self.bookkeeper.immutablevalue(self.instance()) + return _jit_vref.SomeVRef(s_obj) + +vref_None = non_virtual_ref(None) + +# ____________________________________________________________ # User interface for the hotpath JIT policy class JitHintError(Exception): Modified: pypy/branch/virtual-forcing/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/llinterp.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/llinterp.py Mon Dec 21 16:13:02 2009 @@ -546,9 +546,6 @@ def op_jit_marker(self, *args): pass - def op_promote_virtualizable(self, *args): - pass - def op_get_exception_addr(self, *args): raise NotImplementedError Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py Mon Dec 21 16:13:02 2009 @@ -21,8 +21,7 @@ from pypy.rlib.rarithmetic import r_uint, r_singlefloat, intmask from pypy.annotation import model as annmodel from pypy.rpython.llinterp import LLInterpreter, LLException -from pypy.rpython.lltypesystem.rclass import OBJECT -from pypy.rpython.annlowlevel import base_ptr_lltype +from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE from pypy.rpython import raddress from pypy.translator.platform import platform @@ -31,7 +30,6 @@ _ctypes_cache = {} _eci_cache = {} -_parent_cache = {} def _setup_ctypes_cache(): from pypy.rpython.lltypesystem import rffi @@ -255,7 +253,6 @@ convert_struct(field_value, csubstruct) subcontainer = getattr(container, field_name) substorage = subcontainer._storage - update_parent_cache(substorage, subcontainer) elif field_name == STRUCT._arrayfld: # inlined var-sized part csubarray = getattr(cstruct, field_name) convert_array(field_value, csubarray) @@ -314,7 +311,6 @@ struct_storage = getattr(ctypes_storage, field_name) struct_use_ctypes_storage(struct_container, struct_storage) struct_container._setparentstructure(container, field_name) - update_parent_cache(ctypes_storage, struct_container) elif isinstance(FIELDTYPE, lltype.Array): assert FIELDTYPE._hints.get('nolength', False) == False arraycontainer = _array_of_known_length(FIELDTYPE) @@ -500,25 +496,6 @@ _callback2obj = {} _callback_exc_info = None -# this is just another hack that passes around references to applevel types -# disguised as base_ptr_lltype -class Dummy(object): - pass - -_opaque_cache = {Dummy():0} -_opaque_list = [Dummy()] - -def new_opaque_object(llobj): - try: - return _opaque_cache[llobj] - except KeyError: - assert len(_opaque_cache) == len(_opaque_list) - ctypes_type = get_ctypes_type(base_ptr_lltype()) - val = ctypes.cast(len(_opaque_cache), ctypes_type) - _opaque_list.append(llobj) - _opaque_cache[llobj] = val - return val - def get_rtyper(): llinterp = LLInterpreter.current_interpreter if llinterp is not None: @@ -542,8 +519,6 @@ return ctypes.c_void_p(0) return get_ctypes_type(T)() - if T is base_ptr_lltype(): - return new_opaque_object(llobj) if T == llmemory.GCREF: if isinstance(llobj._obj, _llgcopaque): return ctypes.c_void_p(llobj._obj.intval) @@ -655,8 +630,6 @@ raise NotImplementedError(T) container._ctypes_storage_was_allocated() storage = container._storage - if lltype.parentlink(container)[0] is not None: - update_parent_cache(storage, container) p = ctypes.pointer(storage) if index: p = ctypes.cast(p, ctypes.c_void_p) @@ -694,29 +667,37 @@ if isinstance(T, lltype.Ptr): if not cobj: # NULL pointer return lltype.nullptr(T.TO) - if T is base_ptr_lltype(): - return _opaque_list[ctypes.cast(cobj, ctypes.c_void_p).value] if isinstance(T.TO, lltype.Struct): + REAL_TYPE = T.TO if T.TO._arrayfld is not None: carray = getattr(cobj.contents, T.TO._arrayfld) container = lltype._struct(T.TO, carray.length) else: # special treatment of 'OBJECT' subclasses - if get_rtyper() and lltype._castdepth(T.TO, OBJECT) > 0: - ctypes_object = get_ctypes_type(lltype.Ptr(OBJECT)) - as_obj = ctypes2lltype(lltype.Ptr(OBJECT), - ctypes.cast(cobj, ctypes_object)) - TObj = get_rtyper().get_type_for_typeptr(as_obj.typeptr) - if TObj != T.TO: - ctypes_instance = get_ctypes_type(lltype.Ptr(TObj)) - return lltype.cast_pointer(T, - ctypes2lltype(lltype.Ptr(TObj), - ctypes.cast(cobj, ctypes_instance))) - container = lltype._struct(T.TO) + if get_rtyper() and lltype._castdepth(REAL_TYPE, OBJECT) >= 0: + # figure out the real type of the object + containerheader = lltype._struct(OBJECT) + cobjheader = ctypes.cast(cobj, + get_ctypes_type(lltype.Ptr(OBJECT))) + struct_use_ctypes_storage(containerheader, + cobjheader.contents) + REAL_TYPE = get_rtyper().get_type_for_typeptr( + containerheader.typeptr) + REAL_T = lltype.Ptr(REAL_TYPE) + cobj = ctypes.cast(cobj, get_ctypes_type(REAL_T)) + container = lltype._struct(REAL_TYPE) struct_use_ctypes_storage(container, cobj.contents) - addr = ctypes.addressof(cobj.contents) - if addr in _parent_cache: - setparentstructure(container, _parent_cache[addr]) + if REAL_TYPE != T.TO: + p = container._as_ptr() + container = lltype.cast_pointer(T, p)._as_obj() + # special treatment of 'OBJECT_VTABLE' subclasses + if get_rtyper() and lltype._castdepth(REAL_TYPE, + OBJECT_VTABLE) >= 0: + # figure out the real object that this vtable points to, + # and just return that + p = get_rtyper().get_real_typeptr_for_typeptr( + container._as_ptr()) + container = lltype.cast_pointer(T, p)._as_obj() elif isinstance(T.TO, lltype.Array): if T.TO._hints.get('nolength', False): container = _array_of_unknown_length(T.TO) @@ -1163,46 +1144,6 @@ return hop.genop('cast_adr_to_int', [adr], resulttype = lltype.Signed) -# ------------------------------------------------------------ - -def parentchain(container): - current = container - links = [] - while True: - link = lltype.parentlink(current) - if link[0] is None: - try: - addr = ctypes.addressof(container._storage) - actual = _parent_cache[addr] - if len(links) < len(actual): - return actual - except KeyError: - pass - return links - links.append(link) - current = link[0] - -def update_parent_cache(storage, container): - chain = parentchain(container) - addr = ctypes.addressof(storage) - try: - current = _parent_cache[addr] - if len(chain) > len(current): - _parent_cache[addr] = chain - except KeyError: - _parent_cache[addr] = chain - -def setparentstructure(container, chain): - TP = lltype.typeOf(container) - current = container - for i, elem in enumerate(chain): - if lltype.typeOf(elem[0]) == TP: - chain = chain[i + 1:] - break - for elem in chain: - current._setparentstructure(*elem) - current = elem[0] - # ____________________________________________________________ # errno Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/lloperation.py Mon Dec 21 16:13:02 2009 @@ -427,7 +427,8 @@ # __________ used by the JIT ________ 'jit_marker': LLOp(), - 'promote_virtualizable':LLOp(canrun=True), + 'jit_force_virtualizable':LLOp(canrun=True), + 'jit_force_virtual': LLOp(canrun=True), 'get_exception_addr': LLOp(), 'get_exc_value_addr': LLOp(), 'do_malloc_fixedsize_clear': LLOp(canunwindgc=True), Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/opimpl.py Mon Dec 21 16:13:02 2009 @@ -444,8 +444,11 @@ def op_gc_stack_bottom(): pass # marker for trackgcroot.py -def op_promote_virtualizable(object, fieldname, flags): - pass # XXX should do something +def op_jit_force_virtualizable(*args): + pass + +def op_jit_force_virtual(x): + return x def op_get_group_member(TYPE, grpptr, memberoffset): from pypy.rpython.lltypesystem import llgroup Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rclass.py Mon Dec 21 16:13:02 2009 @@ -86,6 +86,13 @@ vtable = vtable.super return vtable +def alloc_array_name(name): + p = malloc(Array(Char), len(name)+1, immortal=True) + for i in range(len(name)): + p[i] = name[i] + p[len(name)] = '\x00' + return p + class ClassRepr(AbstractClassRepr): def __init__(self, rtyper, classdef): @@ -192,10 +199,7 @@ name = 'object' else: name = rsubcls.classdef.shortname - vtable.name = malloc(Array(Char), len(name)+1, immortal=True) - for i in range(len(name)): - vtable.name[i] = name[i] - vtable.name[len(name)] = '\x00' + vtable.name = alloc_array_name(name) if hasattr(rsubcls.classdef, 'my_instantiate_graph'): graph = rsubcls.classdef.my_instantiate_graph vtable.instantiate = self.rtyper.getcallable(graph) @@ -379,7 +383,7 @@ ll_runtime_type_info, OBJECT, destrptr) vtable = self.rclass.getvtable() - self.rtyper.type_for_typeptr[vtable._obj] = self.lowleveltype.TO + self.rtyper.set_type_for_typeptr(vtable, self.lowleveltype.TO) self.rtyper.lltype2vtable[self.lowleveltype.TO] = vtable def common_repr(self): # -> object or nongcobject reprs @@ -689,3 +693,23 @@ break raise AttributeError("%s has no field %s" % (lltype.typeOf(widest), name)) + +def declare_type_for_typeptr(vtable, TYPE): + """Hack for custom low-level-only 'subclasses' of OBJECT: + call this somewhere annotated, in order to declare that it is + of the given TYPE and has got the corresponding vtable.""" + +class Entry(ExtRegistryEntry): + _about_ = declare_type_for_typeptr + def compute_result_annotation(self, s_vtable, s_TYPE): + assert s_vtable.is_constant() + assert s_TYPE.is_constant() + return annmodel.s_None + def specialize_call(self, hop): + vtable = hop.args_v[0].value + TYPE = hop.args_v[1].value + assert lltype.typeOf(vtable) == CLASSTYPE + assert isinstance(TYPE, GcStruct) + assert lltype._castdepth(TYPE, OBJECT) > 0 + hop.rtyper.set_type_for_typeptr(vtable, TYPE) + return hop.inputconst(lltype.Void, None) Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Mon Dec 21 16:13:02 2009 @@ -928,34 +928,6 @@ assert op.args[1].value == pypy.rpython.lltypesystem.rstr.LLHelpers assert op.args[3].value == -2 - def test_pass_around_t_object(self): - from pypy.rpython.annlowlevel import base_ptr_lltype - T = base_ptr_lltype() - - class X(object): - _TYPE = T - x = 10 - - def callback(x): - return x.x - - c_source = py.code.Source(""" - long eating_callback(void *arg, long(*call)(void*)) - { - return call(arg); - } - """) - - eci = ExternalCompilationInfo(separate_module_sources=[c_source], - export_symbols=['eating_callback']) - - args = [T, rffi.CCallback([T], rffi.LONG)] - eating_callback = rffi.llexternal('eating_callback', args, rffi.LONG, - compilation_info=eci) - - res = eating_callback(X(), callback) - assert res == 10 - def test_recursive_struct_more(self): NODE = lltype.ForwardReference() NODE.become(lltype.Struct('NODE', ('value', lltype.Signed), @@ -1134,7 +1106,104 @@ #import pdb; pdb.set_trace() assert adr1_2 == adr1 assert adr1 == adr1_2 - + + def test_object_subclass(self): + from pypy.rpython.lltypesystem import rclass + from pypy.rpython.annlowlevel import cast_instance_to_base_ptr + from pypy.rpython.annlowlevel import cast_base_ptr_to_instance + class S: + pass + def f(n): + s = S() + s.x = n + ls = cast_instance_to_base_ptr(s) + as_num = rffi.cast(lltype.Signed, ls) + # --- around this point, only 'as_num' is passed + t = rffi.cast(rclass.OBJECTPTR, as_num) + u = cast_base_ptr_to_instance(S, t) + return u.x + res = interpret(f, [123]) + assert res == 123 + + def test_object_subclass_2(self): + from pypy.rpython.lltypesystem import rclass + SCLASS = lltype.GcStruct('SCLASS', + ('parent', rclass.OBJECT), + ('n', lltype.Signed)) + sclass_vtable = lltype.malloc(rclass.OBJECT_VTABLE, zero=True, + immortal=True) + sclass_vtable.name = rclass.alloc_array_name('SClass') + def f(n): + rclass.declare_type_for_typeptr(sclass_vtable, SCLASS) + s = lltype.malloc(SCLASS) + s.parent.typeptr = sclass_vtable + s.n = n + as_num = rffi.cast(lltype.Signed, s) + # --- around this point, only 'as_num' is passed + t = rffi.cast(lltype.Ptr(SCLASS), as_num) + return t.n + res = interpret(f, [123]) + assert res == 123 + + def test_object_subclass_3(self): + from pypy.rpython.lltypesystem import rclass + from pypy.rpython.annlowlevel import cast_instance_to_base_ptr + from pypy.rpython.annlowlevel import cast_base_ptr_to_instance + class S: + pass + def f(n): + s = S() + s.x = n + ls = cast_instance_to_base_ptr(s) + as_num = rffi.cast(lltype.Signed, ls) + # --- around this point, only 'as_num' is passed + r = rffi.cast(llmemory.GCREF, as_num) + t = lltype.cast_opaque_ptr(rclass.OBJECTPTR, r) + u = cast_base_ptr_to_instance(S, t) + return u.x + res = interpret(f, [123]) + assert res == 123 + + def test_object_subclass_4(self): + from pypy.rpython.lltypesystem import rclass + SCLASS = lltype.GcStruct('SCLASS', + ('parent', rclass.OBJECT), + ('n', lltype.Signed)) + sclass_vtable = lltype.malloc(rclass.OBJECT_VTABLE, zero=True, + immortal=True) + sclass_vtable.name = rclass.alloc_array_name('SClass') + def f(n): + rclass.declare_type_for_typeptr(sclass_vtable, SCLASS) + s = lltype.malloc(SCLASS) + s.parent.typeptr = sclass_vtable + s.n = n + as_num = rffi.cast(lltype.Signed, s) + # --- around this point, only 'as_num' is passed + r = rffi.cast(llmemory.GCREF, as_num) + t = lltype.cast_opaque_ptr(lltype.Ptr(SCLASS), r) + return t.n + res = interpret(f, [123]) + assert res == 123 + + def test_object_subclass_5(self): + from pypy.rpython.lltypesystem import rclass + from pypy.rpython.annlowlevel import cast_instance_to_base_ptr + from pypy.rpython.annlowlevel import cast_base_ptr_to_instance + class S: + x = 5 # entry in the vtable + class T(S): + x = 6 + def f(): + s = T() + ls = cast_instance_to_base_ptr(s) + as_num = rffi.cast(lltype.Signed, ls) + # --- around this point, only 'as_num' is passed + t = rffi.cast(rclass.OBJECTPTR, as_num) + u = cast_base_ptr_to_instance(S, t) + return u.x + res = interpret(f, []) + assert res == 6 + class TestPlatform(object): def test_lib_on_libpaths(self): from pypy.translator.platform import platform Modified: pypy/branch/virtual-forcing/pypy/rpython/rtyper.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/rtyper.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/rtyper.py Mon Dec 21 16:13:02 2009 @@ -133,15 +133,33 @@ return result def get_type_for_typeptr(self, typeptr): + search = typeptr._obj try: - return self.type_for_typeptr[typeptr._obj] + return self.type_for_typeptr[search] except KeyError: - # rehash the dictionary + # rehash the dictionary, and perform a linear scan + # for the case of ll2ctypes typeptr + found = None type_for_typeptr = {} for key, value in self.type_for_typeptr.items(): type_for_typeptr[key] = value + if key == search: + found = value self.type_for_typeptr = type_for_typeptr - return self.type_for_typeptr[typeptr._obj] + if found is None: + raise KeyError(search) + return found + + def set_type_for_typeptr(self, typeptr, TYPE): + self.type_for_typeptr[typeptr._obj] = TYPE + + def get_real_typeptr_for_typeptr(self, typeptr): + # perform a linear scan for the case of ll2ctypes typeptr + search = typeptr._obj + for key, value in self.type_for_typeptr.items(): + if key == search: + return key._as_ptr() + raise KeyError(search) def makekey(self, s_obj): return pair(self.type_system, s_obj).rtyper_makekey(self) Modified: pypy/branch/virtual-forcing/pypy/rpython/rvirtualizable2.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/rvirtualizable2.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/rvirtualizable2.py Mon Dec 21 16:13:02 2009 @@ -52,10 +52,10 @@ #if not flags.get('access_directly'): if cname.value in self.my_redirected_fields: cflags = inputconst(lltype.Void, flags) - llops.genop('promote_virtualizable', [vinst, cname, cflags]) + llops.genop('jit_force_virtualizable', [vinst, cname, cflags]) -def replace_promote_virtualizable_with_call(graphs, VTYPEPTR, funcptr): +def replace_force_virtualizable_with_call(graphs, VTYPEPTR, funcptr): # funcptr should be an ll or oo function pointer with a VTYPEPTR argument c_funcptr = inputconst(lltype.typeOf(funcptr), funcptr) count = 0 @@ -65,7 +65,7 @@ continue newoplist = [] for i, op in enumerate(block.operations): - if (op.opname == 'promote_virtualizable' and + if (op.opname == 'jit_force_virtualizable' and match_virtualizable_type(op.args[0].concretetype, VTYPEPTR)): if op.args[-1].value.get('access_directly', False): @@ -75,7 +75,7 @@ count += 1 newoplist.append(op) block.operations = newoplist - log("replaced %d 'promote_virtualizable' with %r" % (count, funcptr)) + log("replaced %d 'jit_force_virtualizable' with %r" % (count, funcptr)) def match_virtualizable_type(TYPE, VTYPEPTR): if isinstance(TYPE, ootype.Instance): Modified: pypy/branch/virtual-forcing/pypy/rpython/test/test_rvirtualizable2.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/test/test_rvirtualizable2.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/test/test_rvirtualizable2.py Mon Dec 21 16:13:02 2009 @@ -1,7 +1,7 @@ import py from pypy.rpython.lltypesystem import lltype from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin -from pypy.rpython.rvirtualizable2 import replace_promote_virtualizable_with_call +from pypy.rpython.rvirtualizable2 import replace_force_virtualizable_with_call from pypy.rlib.jit import hint from pypy.objspace.flow.model import summary from pypy.rpython.llinterp import LLInterpreter @@ -33,15 +33,15 @@ def __init__(self, v0): self.v0 = v0 -def get_promote_virtualizable_flags(graph): +def get_force_virtualizable_flags(graph): res = [] for block, op in graph.iterblockops(): - if op.opname == 'promote_virtualizable': + if op.opname == 'jit_force_virtualizable': res.append(op.args[-1].value) return res class BaseTest(BaseRtypingTest): - def test_generate_promote_virtualizable(self): + def test_generate_force_virtualizable(self): def fn(n): vinst = V(n) return vinst.v @@ -51,11 +51,11 @@ op_getfield = block.operations[-1] assert op_getfield.opname in ('getfield', 'oogetfield') v_inst = op_getfield.args[0] - assert op_promote.opname == 'promote_virtualizable' + assert op_promote.opname == 'jit_force_virtualizable' assert op_promote.args[0] is v_inst assert op_promote.args[-1].value == {} - def test_generate_promote_virtualizable_subclass(self): + def test_generate_force_virtualizable_subclass(self): def fn(n): V(n) # to attach v to V vinst = SubclassV(n) @@ -66,11 +66,11 @@ op_getfield = block.operations[-1] assert op_getfield.opname in ('getfield', 'oogetfield') v_inst = op_getfield.args[0] - assert op_promote.opname == 'promote_virtualizable' + assert op_promote.opname == 'jit_force_virtualizable' assert op_promote.args[0] is v_inst assert op_promote.args[-1].value == {} - def test_no_promote_virtualizable_for_other_fields(self): + def test_no_force_virtualizable_for_other_fields(self): def fn(n): vinst = V(n) return vinst.w @@ -81,7 +81,7 @@ assert op_getfield.opname in ('getfield', 'oogetfield') assert op_call.opname == 'direct_call' # to V.__init__ - def test_generate_promote_virtualizable_array(self): + def test_generate_force_virtualizable_array(self): def fn(n): vinst = VArray([n, n+1]) return vinst.lst[1] @@ -93,7 +93,7 @@ assert op_getarrayitem.opname == 'direct_call' # to ll_getitem_xxx assert op_getfield.opname in ('getfield', 'oogetfield') v_inst = op_getfield.args[0] - assert op_promote.opname == 'promote_virtualizable' + assert op_promote.opname == 'jit_force_virtualizable' assert op_promote.args[0] is v_inst assert op_promote.args[-1].value == {} @@ -130,13 +130,13 @@ TYPE = self.gettype(w_inst) assert 'virtualizable2_accessor' not in TYPE._hints - def replace_promote_virtualizable(self, rtyper, graphs): + def replace_force_virtualizable(self, rtyper, graphs): from pypy.annotation import model as annmodel from pypy.rpython.annlowlevel import MixLevelHelperAnnotator graph = graphs[0] for block, op in graph.iterblockops(): - if op.opname == 'promote_virtualizable': + if op.opname == 'jit_force_virtualizable': v_inst_ll_type = op.args[0].concretetype break @@ -150,11 +150,10 @@ s_vinst = annmodel.SomeOOInstance(v_inst_ll_type) funcptr = annhelper.delayedfunction(mycall, [s_vinst], annmodel.s_None) annhelper.finish() - replace_promote_virtualizable_with_call(graphs, v_inst_ll_type, - funcptr) + replace_force_virtualizable_with_call(graphs, v_inst_ll_type, funcptr) return funcptr - def test_replace_promote_virtualizable_with_call(self): + def test_replace_force_virtualizable_with_call(self): def fn(n): vinst = V(n) return vinst.v @@ -162,7 +161,7 @@ block = graph.startblock op_getfield = block.operations[-1] assert op_getfield.opname in ('getfield', 'oogetfield') - funcptr = self.replace_promote_virtualizable(rtyper, [graph]) + funcptr = self.replace_force_virtualizable(rtyper, [graph]) if getattr(conftest.option, 'view', False): graph.show() op_promote = block.operations[-2] @@ -190,9 +189,9 @@ g_graph = t._graphof(g) expected = [{'access_directly': True}] * 3 - assert get_promote_virtualizable_flags(g_graph) == expected + assert get_force_virtualizable_flags(g_graph) == expected - self.replace_promote_virtualizable(typer, [g_graph]) + self.replace_force_virtualizable(typer, [g_graph]) assert summary(g_graph) == {self.GETFIELD: 2, self.SETFIELD: 1, 'int_add': 1} res = self.interpret(f, [23]) @@ -213,7 +212,7 @@ f_graph = t._graphof(f) g_graph = t._graphof(g) - self.replace_promote_virtualizable(typer, [f_graph, g_graph]) + self.replace_force_virtualizable(typer, [f_graph, g_graph]) t.checkgraphs() res = self.interpret(f, [23]) @@ -236,12 +235,12 @@ g_graphs.sort() assert g_graphs[0][0] is None - assert get_promote_virtualizable_flags(g_graphs[0][1]) == [{}] + assert get_force_virtualizable_flags(g_graphs[0][1]) == [{}] expected = [{'access_directly': True}] - assert get_promote_virtualizable_flags(g_graphs[1][1]) == expected + assert get_force_virtualizable_flags(g_graphs[1][1]) == expected - self.replace_promote_virtualizable(typer, [g_graphs[0][1], - g_graphs[1][1]]) + self.replace_force_virtualizable(typer, [g_graphs[0][1], + g_graphs[1][1]]) assert summary(g_graphs[0][1]) == {'direct_call': 1, self.GETFIELD: 1} assert summary(g_graphs[1][1]) == {self.GETFIELD: 1} @@ -276,8 +275,9 @@ assert summary(g_graphs[1][1]) == {self.SETFIELD: 1} h_graph = t._graphof(h) - assert summary(h_graph) == {'promote_virtualizable': 1, self.GETFIELD: 1} - assert get_promote_virtualizable_flags(h_graph) == [{}] + assert summary(h_graph) == {'jit_force_virtualizable': 1, + self.GETFIELD: 1} + assert get_force_virtualizable_flags(h_graph) == [{}] res = self.interpret(f, [23]) assert res == 23 @@ -303,7 +303,7 @@ t, typer, graph = self.gengraph(f, [int]) g_graph = t._graphof(A.g.im_func) - self.replace_promote_virtualizable(typer, [g_graph]) + self.replace_force_virtualizable(typer, [g_graph]) assert summary(g_graph) == {self.GETFIELD: 1, 'int_mul': 1} Modified: pypy/branch/virtual-forcing/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/c/funcgen.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/c/funcgen.py Mon Dec 21 16:13:02 2009 @@ -793,8 +793,12 @@ def OP_JIT_MARKER(self, op): return '/* JIT_MARKER %s */' % op - def OP_PROMOTE_VIRTUALIZABLE(self, op): - return '/* PROMOTE_VIRTUALIZABLE %s */' % op + def OP_JIT_FORCE_VIRTUALIZABLE(self, op): + return '/* JIT_FORCE_VIRTUALIZABLE %s */' % op + + def OP_JIT_FORCE_VIRTUAL(self, op): + return '%s = %s; /* JIT_FORCE_VIRTUAL */' % (self.expr(op.result), + self.expr(op.args[0])) def OP_GET_GROUP_MEMBER(self, op): typename = self.db.gettype(op.result.concretetype) Modified: pypy/branch/virtual-forcing/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/cli/opcodes.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/cli/opcodes.py Mon Dec 21 16:13:02 2009 @@ -84,7 +84,8 @@ 'debug_fatalerror': [PushAllArgs, 'call void [pypylib]pypy.runtime.Debug::DEBUG_FATALERROR(string)'], 'keepalive': Ignore, 'jit_marker': Ignore, - 'promote_virtualizable': Ignore, + 'jit_force_virtualizable': Ignore, + 'jit_force_virtual': DoNothing, } # __________ numeric operations __________ Modified: pypy/branch/virtual-forcing/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/jvm/opcodes.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/jvm/opcodes.py Mon Dec 21 16:13:02 2009 @@ -97,7 +97,8 @@ 'gc_set_max_heap_size': Ignore, 'resume_point': Ignore, 'jit_marker': Ignore, - 'promote_virtualizable': Ignore, + 'jit_force_virtualizable': Ignore, + 'jit_force_virtual': DoNothing, 'debug_assert': [], # TODO: implement? From benjamin at codespeak.net Mon Dec 21 17:36:22 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 21 Dec 2009 17:36:22 +0100 (CET) Subject: [pypy-svn] r70233 - pypy/trunk Message-ID: <20091221163622.0C838168020@codespeak.net> Author: benjamin Date: Mon Dec 21 17:36:21 2009 New Revision: 70233 Modified: pypy/trunk/LICENSE Log: add myself Modified: pypy/trunk/LICENSE ============================================================================== --- pypy/trunk/LICENSE (original) +++ pypy/trunk/LICENSE Mon Dec 21 17:36:21 2009 @@ -86,6 +86,7 @@ Guenter Jantzen Dinu Gherman Georg Brandl + Benjamin Peterson Ben Young Nicolas Chauvat Jean-Paul Calderone From arigo at codespeak.net Tue Dec 22 13:47:03 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 22 Dec 2009 13:47:03 +0100 (CET) Subject: [pypy-svn] r70246 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091222124703.8FDEE168021@codespeak.net> Author: arigo Date: Tue Dec 22 13:47:02 2009 New Revision: 70246 Modified: pypy/trunk/pypy/jit/metainterp/resume.py Log: Use the most negative value for UNASSIGNED values, instead of a random negative value. Add a longish comment and an assert to explain why any negative value would actually be fine. Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Tue Dec 22 13:47:02 2009 @@ -93,8 +93,8 @@ TAGBOX = 2 TAGVIRTUAL = 3 -UNASSIGNED = tag(-2 ** 12 - 1, TAGBOX) -UNASSIGNEDVIRTUAL = tag(-2 ** 12 - 1, TAGVIRTUAL) +UNASSIGNED = tag(-1<<13, TAGBOX) +UNASSIGNEDVIRTUAL = tag(-1<<13, TAGVIRTUAL) NULLREF = tag(-1, TAGCONST) @@ -301,6 +301,11 @@ memo = self.memo new_liveboxes = [None] * memo.num_cached_boxes() count = 0 + # So far, self.liveboxes should contain 'tagged' values that are + # either UNASSIGNED, UNASSIGNEDVIRTUAL, or a *non-negative* value + # with the TAGVIRTUAL. The following loop removes the UNASSIGNED + # and UNASSIGNEDVIRTUAL entries, and replaces them with real + # negative values. for box, tagged in self.liveboxes.iteritems(): i, tagbits = untag(tagged) if tagbits == TAGBOX: @@ -315,6 +320,8 @@ assert box not in self.liveboxes_from_env index = memo.assign_number_to_virtual(box) self.liveboxes[box] = tag(index, TAGVIRTUAL) + else: + assert i >= 0 new_liveboxes.reverse() liveboxes.extend(new_liveboxes) nholes = len(new_liveboxes) - count From pedronis at codespeak.net Tue Dec 22 14:32:54 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 22 Dec 2009 14:32:54 +0100 (CET) Subject: [pypy-svn] r70247 - in pypy/trunk/pypy: jit/backend/x86 rlib rpython/memory/gctransform rpython/memory/gctransform/test Message-ID: <20091222133254.66D69168021@codespeak.net> Author: pedronis Date: Tue Dec 22 14:32:54 2009 New Revision: 70247 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/rlib/rgc.py pypy/trunk/pypy/rpython/memory/gctransform/framework.py pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py Log: support marking functions to be checked as not triggering collects in the case of framework usage, mark some crucial failure handling funcs in the jit that need this property Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Tue Dec 22 14:32:54 2009 @@ -17,6 +17,7 @@ from pypy.jit.metainterp.resoperation import rop from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print +from pypy.rlib import rgc # our calling convention - we pass first 6 args in registers # and the rest stays on the stack @@ -956,6 +957,7 @@ boxes.append(box) return boxes + @rgc.no_collect def grab_frame_values(self, bytecode, frame_addr, allregisters): # no malloc allowed here!! self.fail_ebp = allregisters[16 + ebp.op] @@ -1020,6 +1022,7 @@ def setup_failure_recovery(self): + @rgc.no_collect def failure_recovery_func(registers): # 'registers' is a pointer to a structure containing the # original value of the registers, optionally the original Modified: pypy/trunk/pypy/rlib/rgc.py ============================================================================== --- pypy/trunk/pypy/rlib/rgc.py (original) +++ pypy/trunk/pypy/rlib/rgc.py Tue Dec 22 14:32:54 2009 @@ -357,3 +357,8 @@ keepalive_until_here(source) keepalive_until_here(dest) ll_arraycopy._annspecialcase_ = 'specialize:ll' + +def no_collect(func): + func._dont_inline_ = True + func._gc_no_collect_ = True + return func Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/framework.py Tue Dec 22 14:32:54 2009 @@ -567,6 +567,11 @@ f.close() def transform_graph(self, graph): + func = getattr(graph, 'func', None) + if func and getattr(func, '_gc_no_collect_', False): + if self.collect_analyzer.analyze_direct_call(graph): + raise Exception("no_collect function can trigger collection") + if self.write_barrier_ptr: self.clean_sets = ( find_clean_setarrayitems(self.collect_analyzer, graph).union( Modified: pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py Tue Dec 22 14:32:54 2009 @@ -1,5 +1,6 @@ from pypy.objspace.flow.model import Constant, SpaceOperation from pypy.annotation.model import SomeInteger +from pypy.annotation.listdef import s_list_of_strings from pypy.rpython.memory.gc.marksweep import MarkSweepGC from pypy.rpython.memory.gctransform.test.test_transform import rtype, \ rtype_and_transform @@ -34,7 +35,6 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.translator.c.genc import CStandaloneBuilder from pypy.translator.c import gc - from pypy.annotation.listdef import s_list_of_strings t = rtype(entrypoint, [s_list_of_strings]) cbuild = CStandaloneBuilder(t, entrypoint, t.config, @@ -113,6 +113,50 @@ gg = graphof(t, g) assert CollectAnalyzer(t).analyze_direct_call(gg) +def test_no_collect(): + from pypy.rlib import rgc + from pypy.translator.c.genc import CStandaloneBuilder + from pypy.translator.c import gc + + @rgc.no_collect + def g(): + return 1 + + assert g._dont_inline_ + assert g._gc_no_collect_ + + def entrypoint(argv): + return g() + 2 + + t = rtype(entrypoint, [s_list_of_strings]) + cbuild = CStandaloneBuilder(t, entrypoint, t.config, + gcpolicy=FrameworkGcPolicy2) + db = cbuild.generate_graphs_for_llinterp() + +def test_no_collect_detection(): + from pypy.rlib import rgc + from pypy.translator.c.genc import CStandaloneBuilder + from pypy.translator.c import gc + + class A(object): + def __init__(self, x): + self.x = x + + @rgc.no_collect + def g(): + return A(1).x + + assert g._dont_inline_ + assert g._gc_no_collect_ + + def entrypoint(argv): + return g() + 2 + + t = rtype(entrypoint, [s_list_of_strings]) + cbuild = CStandaloneBuilder(t, entrypoint, t.config, + gcpolicy=FrameworkGcPolicy2) + f = py.test.raises(Exception, cbuild.generate_graphs_for_llinterp) + assert str(f.value) == 'no_collect function can trigger collection' class WriteBarrierTransformer(FrameworkGCTransformer): clean_sets = {} From arigo at codespeak.net Tue Dec 22 14:46:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 22 Dec 2009 14:46:25 +0100 (CET) Subject: [pypy-svn] r70248 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091222134625.9296E168021@codespeak.net> Author: arigo Date: Tue Dec 22 14:46:25 2009 New Revision: 70248 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/resume.py Log: Comments. Kill unused line. Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Tue Dec 22 14:46:25 2009 @@ -234,7 +234,6 @@ def get_args_for_fail(self, modifier): if self.box is None and not modifier.already_seen_virtual(self.keybox): - # modifier.already_seen_virtual() # checks for recursion: it is False unless # we have already seen the very same keybox lst = self._get_field_descr_list() @@ -302,11 +301,9 @@ def get_args_for_fail(self, modifier): if self.box is None and not modifier.already_seen_virtual(self.keybox): - # modifier.already_seen_virtual() # checks for recursion: it is False unless # we have already seen the very same keybox itemboxes = [] - const = self.optimizer.new_const_item(self.arraydescr) for itemvalue in self._items: itemboxes.append(itemvalue.get_key_box()) modifier.register_virtual_fields(self.keybox, itemboxes) Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Tue Dec 22 14:46:25 2009 @@ -195,6 +195,7 @@ return len(self.cached_boxes) def assign_number_to_box(self, box, boxes): + # returns a negative number if box in self.cached_boxes: num = self.cached_boxes[box] boxes[-num-1] = box @@ -208,6 +209,7 @@ return len(self.cached_virtuals) def assign_number_to_virtual(self, box): + # returns a negative number if box in self.cached_virtuals: num = self.cached_virtuals[box] else: @@ -230,8 +232,6 @@ def __init__(self, storage, memo): self.storage = storage self.memo = memo - #self.virtuals = [] - #self.vfieldboxes = [] def make_virtual(self, known_class, fielddescrs): return VirtualInfo(known_class, fielddescrs) From pedronis at codespeak.net Tue Dec 22 14:46:44 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 22 Dec 2009 14:46:44 +0100 (CET) Subject: [pypy-svn] r70249 - in pypy/trunk/pypy/rpython/memory/gctransform: . test Message-ID: <20091222134644.71947168021@codespeak.net> Author: pedronis Date: Tue Dec 22 14:46:44 2009 New Revision: 70249 Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py Log: give the function name in the exception Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/framework.py Tue Dec 22 14:46:44 2009 @@ -570,7 +570,8 @@ func = getattr(graph, 'func', None) if func and getattr(func, '_gc_no_collect_', False): if self.collect_analyzer.analyze_direct_call(graph): - raise Exception("no_collect function can trigger collection") + raise Exception("no_collect function can trigger collection: %s" + % func.__name__) if self.write_barrier_ptr: self.clean_sets = ( Modified: pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py Tue Dec 22 14:46:44 2009 @@ -156,7 +156,7 @@ cbuild = CStandaloneBuilder(t, entrypoint, t.config, gcpolicy=FrameworkGcPolicy2) f = py.test.raises(Exception, cbuild.generate_graphs_for_llinterp) - assert str(f.value) == 'no_collect function can trigger collection' + assert str(f.value) == 'no_collect function can trigger collection: g' class WriteBarrierTransformer(FrameworkGCTransformer): clean_sets = {} From arigo at codespeak.net Tue Dec 22 14:59:53 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 22 Dec 2009 14:59:53 +0100 (CET) Subject: [pypy-svn] r70250 - pypy/branch/jit-delayed-write Message-ID: <20091222135953.D62BA168021@codespeak.net> Author: arigo Date: Tue Dec 22 14:59:53 2009 New Revision: 70250 Added: pypy/branch/jit-delayed-write/ - copied from r70249, pypy/trunk/ Log: A branch in which to implement delayed writes in optimizeopt.py. From arigo at codespeak.net Tue Dec 22 15:16:37 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 22 Dec 2009 15:16:37 +0100 (CET) Subject: [pypy-svn] r70251 - pypy/branch/jit-delayed-write/pypy/translator/backendopt/test Message-ID: <20091222141637.EBFA8168021@codespeak.net> Author: arigo Date: Tue Dec 22 15:16:37 2009 New Revision: 70251 Modified: pypy/branch/jit-delayed-write/pypy/translator/backendopt/test/test_writeanalyze.py Log: First, rename this class to a reasonable name. Modified: pypy/branch/jit-delayed-write/pypy/translator/backendopt/test/test_writeanalyze.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/translator/backendopt/test/test_writeanalyze.py (original) +++ pypy/branch/jit-delayed-write/pypy/translator/backendopt/test/test_writeanalyze.py Tue Dec 22 15:16:37 2009 @@ -8,7 +8,7 @@ from pypy.conftest import option -class BaseTestCanRaise(object): +class BaseTestWriteAnalyze(object): type_system = None @@ -146,7 +146,7 @@ assert not result -class TestLLtype(BaseTestCanRaise): +class TestLLtype(BaseTestWriteAnalyze): type_system = 'lltype' def test_list(self): @@ -205,7 +205,7 @@ assert name.endswith("foobar") -class TestOOtype(BaseTestCanRaise): +class TestOOtype(BaseTestWriteAnalyze): type_system = 'ootype' def test_array(self): From arigo at codespeak.net Wed Dec 23 12:22:11 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 Dec 2009 12:22:11 +0100 (CET) Subject: [pypy-svn] r70257 - in pypy/branch/jit-delayed-write/pypy/translator/backendopt: . test Message-ID: <20091223112211.7F7CE168027@codespeak.net> Author: arigo Date: Wed Dec 23 12:22:10 2009 New Revision: 70257 Modified: pypy/branch/jit-delayed-write/pypy/translator/backendopt/test/test_writeanalyze.py pypy/branch/jit-delayed-write/pypy/translator/backendopt/writeanalyze.py Log: Add a ReadWriteAnalyzer to the module 'writeanalyze'. Modified: pypy/branch/jit-delayed-write/pypy/translator/backendopt/test/test_writeanalyze.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/translator/backendopt/test/test_writeanalyze.py (original) +++ pypy/branch/jit-delayed-write/pypy/translator/backendopt/test/test_writeanalyze.py Wed Dec 23 12:22:10 2009 @@ -4,14 +4,15 @@ from pypy.translator.translator import TranslationContext, graphof from pypy.translator.simplify import get_funcobj from pypy.translator.backendopt.writeanalyze import WriteAnalyzer, top_set +from pypy.translator.backendopt.writeanalyze import ReadWriteAnalyzer from pypy.translator.backendopt.all import backend_optimizations from pypy.conftest import option -class BaseTestWriteAnalyze(object): +class BaseTest(object): type_system = None - + Analyzer = WriteAnalyzer def translate(self, func, sig): t = TranslationContext() @@ -19,7 +20,10 @@ t.buildrtyper(type_system=self.type_system).specialize() if option.view: t.view() - return t, WriteAnalyzer(t) + return t, self.Analyzer(t) + + +class BaseTestWriteAnalyze(BaseTest): def test_writes_simple(self): def g(x): @@ -240,3 +244,65 @@ result = wa.analyze(ggraph.startblock.operations[0]) assert result is top_set + + +class TestLLtypeReadWriteAnalyze(BaseTest): + Analyzer = ReadWriteAnalyzer + type_system = 'lltype' + + def test_read_simple(self): + def g(x): + return True + + def f(x): + return g(x - 1) + t, wa = self.translate(f, [int]) + fgraph = graphof(t, f) + result = wa.analyze(fgraph.startblock.operations[0]) + assert not result + + def test_read_really(self): + class A(object): + def __init__(self, y): + self.y = y + def f(self): + self.x = 1 + return self.y + def h(flag): + obj = A(flag) + return obj.f() + + t, wa = self.translate(h, [int]) + hgraph = graphof(t, h) + op_call_f = hgraph.startblock.operations[-1] + + # check that we fished the expected ops + assert op_call_f.opname == "direct_call" + assert get_funcobj(op_call_f.args[0].value)._name == 'A.f' + + result = wa.analyze(op_call_f) + assert len(result) == 2 + result = list(result) + result.sort() + [(struct1, T1, name1), (struct2, T2, name2)] = result + assert struct1 == "readstruct" + assert name1.endswith("y") + assert struct2 == "struct" + assert name2.endswith("x") + assert T1 == T2 + + def test_contains(self): + def g(x, y, z): + l = [x] + return f(l, y, z) + def f(x, y, z): + return y in x + + t, wa = self.translate(g, [int, int, int]) + ggraph = graphof(t, g) + assert ggraph.startblock.operations[-1].opname == 'direct_call' + + result = wa.analyze(ggraph.startblock.operations[-1]) + ARRAYPTR = list(result)[0][1] + assert list(result) == [("readarray", ARRAYPTR)] + assert isinstance(ARRAYPTR.TO, lltype.GcArray) Modified: pypy/branch/jit-delayed-write/pypy/translator/backendopt/writeanalyze.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/translator/backendopt/writeanalyze.py (original) +++ pypy/branch/jit-delayed-write/pypy/translator/backendopt/writeanalyze.py Wed Dec 23 12:22:10 2009 @@ -45,3 +45,15 @@ elif methname in ('ll_getitem_fast', 'll_length'): return self.bottom_result() return graphanalyze.GraphAnalyzer.analyze_external_method(self, op, TYPE, meth) + + +class ReadWriteAnalyzer(WriteAnalyzer): + + def analyze_simple_operation(self, op): + if op.opname == "getfield": + return frozenset([ + ("readstruct", op.args[0].concretetype, op.args[1].value)]) + elif op.opname == "getarrayitem": + return frozenset([ + ("readarray", op.args[0].concretetype)]) + return WriteAnalyzer.analyze_simple_operation(self, op) From arigo at codespeak.net Wed Dec 23 12:48:08 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 Dec 2009 12:48:08 +0100 (CET) Subject: [pypy-svn] r70258 - in pypy/branch/jit-delayed-write/pypy/jit/metainterp: . test Message-ID: <20091223114808.DC6E8168026@codespeak.net> Author: arigo Date: Wed Dec 23 12:48:08 2009 New Revision: 70258 Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/codewriter.py pypy/branch/jit-delayed-write/pypy/jit/metainterp/effectinfo.py pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_effectinfo.py Log: Bring the info about which fields are read into the EffectInfo class in the jit. Improve tests a little bit. Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/codewriter.py Wed Dec 23 12:48:08 2009 @@ -11,7 +11,7 @@ from pypy.tool.udir import udir from pypy.translator.simplify import get_funcobj, get_functype from pypy.translator.backendopt.canraise import RaiseAnalyzer -from pypy.translator.backendopt.writeanalyze import WriteAnalyzer +from pypy.translator.backendopt.writeanalyze import ReadWriteAnalyzer from pypy.jit.metainterp.typesystem import deref, arrayItem, fieldType from pypy.jit.metainterp.effectinfo import effectinfo_from_writeanalyze from pypy.jit.metainterp.effectinfo import VirtualizableAnalyzer @@ -185,7 +185,7 @@ self.portal_runner_ptr = portal_runner_ptr translator = self.rtyper.annotator.translator self.raise_analyzer = RaiseAnalyzer(translator) - self.write_analyzer = WriteAnalyzer(translator) + self.readwrite_analyzer = ReadWriteAnalyzer(translator) self.virtualizable_analyzer = VirtualizableAnalyzer(translator) def make_portal_bytecode(self, graph): @@ -326,7 +326,7 @@ # ok if consider_effects_of is not None: effectinfo = effectinfo_from_writeanalyze( - self.write_analyzer.analyze(consider_effects_of), + self.readwrite_analyzer.analyze(consider_effects_of), self.cpu, self.virtualizable_analyzer.analyze(consider_effects_of)) calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT, effectinfo) Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/effectinfo.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/effectinfo.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/effectinfo.py Wed Dec 23 12:48:08 2009 @@ -7,13 +7,16 @@ class EffectInfo(object): _cache = {} - def __new__(cls, write_descrs_fields, write_descrs_arrays, - promotes_virtualizables=False): - key = (frozenset(write_descrs_fields), frozenset(write_descrs_arrays), + def __new__(cls, read_descrs_fields, write_descrs_fields, + write_descrs_arrays, promotes_virtualizables=False): + key = (frozenset(read_descrs_fields), + frozenset(write_descrs_fields), + frozenset(write_descrs_arrays), promotes_virtualizables) if key in cls._cache: return cls._cache[key] result = object.__new__(cls) + result.read_descrs_fields = read_descrs_fields result.write_descrs_fields = write_descrs_fields result.write_descrs_arrays = write_descrs_arrays result.promotes_virtualizables = promotes_virtualizables @@ -24,26 +27,37 @@ from pypy.translator.backendopt.writeanalyze import top_set if effects is top_set: return None + read_descrs_fields = [] + # read_descrs_arrays = [] --- not enabled for now write_descrs_fields = [] write_descrs_arrays = [] + + def add_struct(descrs_fields, (_, T, fieldname)): + T = deref(T) + if consider_struct(T, fieldname): + descr = cpu.fielddescrof(T, fieldname) + descrs_fields.append(descr) + + def add_array(descrs_arrays, (_, T)): + ARRAY = deref(T) + if consider_array(ARRAY): + descr = cpu.arraydescrof(ARRAY) + descrs_arrays.append(descr) + for tup in effects: if tup[0] == "struct": - _, T, fieldname = tup - T = deref(T) - if not consider_struct(T, fieldname): - continue - descr = cpu.fielddescrof(T, fieldname) - write_descrs_fields.append(descr) + add_struct(write_descrs_fields, tup) + elif tup[0] == "readstruct": + add_struct(read_descrs_fields, tup) elif tup[0] == "array": - _, T = tup - ARRAY = deref(T) - if not consider_array(ARRAY): - continue - descr = cpu.arraydescrof(ARRAY) - write_descrs_arrays.append(descr) + add_array(write_descrs_arrays, tup) + elif tup[0] == "readarray": + pass else: assert 0 - return EffectInfo(write_descrs_fields, write_descrs_arrays, + return EffectInfo(read_descrs_fields, + write_descrs_fields, + write_descrs_arrays, promotes_virtualizables) def consider_struct(TYPE, fieldname): Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_effectinfo.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_effectinfo.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_effectinfo.py Wed Dec 23 12:48:08 2009 @@ -3,32 +3,68 @@ from pypy.rpython.ootypesystem import ootype from pypy.jit.metainterp.effectinfo import effectinfo_from_writeanalyze +class FakeCPU: + def fielddescrof(self, T, fieldname): + return ('fielddescr', T, fieldname) + def arraydescrof(self, A): + return ('arraydescr', A) + +def test_include_read_field(): + S = lltype.GcStruct("S", ("a", lltype.Signed)) + effects = frozenset([("readstruct", lltype.Ptr(S), "a")]) + effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU()) + assert list(effectinfo.read_descrs_fields) == [('fielddescr', S, "a")] + assert not effectinfo.write_descrs_fields + assert not effectinfo.write_descrs_arrays + +def test_include_write_field(): + S = lltype.GcStruct("S", ("a", lltype.Signed)) + effects = frozenset([("struct", lltype.Ptr(S), "a")]) + effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU()) + assert list(effectinfo.write_descrs_fields) == [('fielddescr', S, "a")] + assert not effectinfo.read_descrs_fields + assert not effectinfo.write_descrs_arrays + +def test_include_write_array(): + A = lltype.GcArray(lltype.Signed) + effects = frozenset([("array", lltype.Ptr(A))]) + effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU()) + assert not effectinfo.read_descrs_fields + assert not effectinfo.write_descrs_fields + assert list(effectinfo.write_descrs_arrays) == [('arraydescr', A)] + + def test_filter_out_typeptr(): effects = frozenset([("struct", lltype.Ptr(OBJECT), "typeptr")]) effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.read_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays def test_filter_out_array_of_void(): effects = frozenset([("array", lltype.Ptr(lltype.GcArray(lltype.Void)))]) effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.read_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays def test_filter_out_struct_with_void(): effects = frozenset([("struct", lltype.Ptr(lltype.GcStruct("x", ("a", lltype.Void))), "a")]) effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.read_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays def test_filter_out_ooarray_of_void(): effects = frozenset([("array", ootype.Array(ootype.Void))]) effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.read_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays def test_filter_out_instance_with_void(): effects = frozenset([("struct", ootype.Instance("x", ootype.ROOT, {"a": ootype.Void}), "a")]) effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.read_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays From arigo at codespeak.net Wed Dec 23 12:52:35 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 Dec 2009 12:52:35 +0100 (CET) Subject: [pypy-svn] r70259 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091223115235.CBDA7168026@codespeak.net> Author: arigo Date: Wed Dec 23 12:52:35 2009 New Revision: 70259 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py Log: Oups? Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Wed Dec 23 12:52:35 2009 @@ -918,7 +918,8 @@ opnum == rop.SETARRAYITEM_GC or opnum == rop.DEBUG_MERGE_POINT): return - if opnum == rop.CALL: + if (opnum == rop.CALL or + opnum == rop.CALL_MAY_FORCE): effectinfo = op.descr.get_extra_info() if effectinfo is not None: for fielddescr in effectinfo.write_descrs_fields: From arigo at codespeak.net Wed Dec 23 13:58:13 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 Dec 2009 13:58:13 +0100 (CET) Subject: [pypy-svn] r70260 - pypy/branch/jit-delayed-write/pypy/jit/metainterp/test Message-ID: <20091223125813.5EC59168026@codespeak.net> Author: arigo Date: Wed Dec 23 13:58:12 2009 New Revision: 70260 Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py Log: Write a target test. (Does not pass yet.) Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizefindnode.py Wed Dec 23 13:58:12 2009 @@ -95,9 +95,14 @@ onedescr = cpu.fielddescrof(U, 'one') FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) - nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([], [])) - writeadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([adescr], [])) - writearraydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([adescr], [arraydescr])) + nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo([], [], [])) + writeadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo([], [adescr], [])) + writearraydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo([], [adescr], [arraydescr])) + readadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo([adescr], [], [])) cpu.class_sizes = {cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), cpu.cast_adr_to_int(node_vtable_adr2): cpu.sizeof(NODE2), Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py Wed Dec 23 13:58:12 2009 @@ -2031,6 +2031,26 @@ """ self.optimize_loop(ops, 'Not, Not, Not', expected) + def test_residual_call_invalidates_some_read_caches(self): + ops = """ + [p1, i1, p2, i2] + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(p2, i2, descr=adescr) + i3 = call(i1, descr=readadescr) + setfield_gc(p1, i3, descr=valuedescr) + setfield_gc(p2, i3, descr=adescr) + jump(p1, i1, p2, i2) + """ + expected = """ + [p1, i1, p2, i2] + setfield_gc(p2, i2, descr=adescr) + i3 = call(i1, descr=readadescr) + setfield_gc(p1, i3, descr=valuedescr) + setfield_gc(p2, i3, descr=adescr) + jump(p1, i1, p2, i2) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', expected) + class TestOOtype(BaseTestOptimizeOpt, OOtypeMixin): From arigo at codespeak.net Wed Dec 23 14:05:48 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 Dec 2009 14:05:48 +0100 (CET) Subject: [pypy-svn] r70261 - pypy/branch/jit-delayed-write/pypy/jit/metainterp/test Message-ID: <20091223130548.0CEA516802D@codespeak.net> Author: arigo Date: Wed Dec 23 14:05:47 2009 New Revision: 70261 Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py Log: Add a passing test. Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py Wed Dec 23 14:05:47 2009 @@ -2031,6 +2031,27 @@ """ self.optimize_loop(ops, 'Not, Not, Not', expected) + def test_setfield_might_alias(self): + # the point is that p1 and p2 might be equal or not + ops = """ + [p1, p2, i1, i1a] + setfield_gc(p1, i1, descr=adescr) + setfield_gc(p2, i1a, descr=adescr) + i2 = call(i1, descr=readadescr) + i3a = getfield_gc(p2, descr=adescr) # can go away + i3 = getfield_gc(p1, descr=adescr) # cannot go away + jump(p1, p2, i3, i1a) + """ + expected = """ + [p1, p2, i1, i1a] + setfield_gc(p1, i1, descr=adescr) + setfield_gc(p2, i1a, descr=adescr) + i2 = call(i1, descr=readadescr) + i3 = getfield_gc(p1, descr=adescr) + jump(p1, p2, i3, i1a) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', expected) + def test_residual_call_invalidates_some_read_caches(self): ops = """ [p1, i1, p2, i2] From arigo at codespeak.net Wed Dec 23 14:11:52 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 Dec 2009 14:11:52 +0100 (CET) Subject: [pypy-svn] r70262 - in pypy/branch/jit-delayed-write/pypy/jit/metainterp: . test Message-ID: <20091223131152.7B05216802C@codespeak.net> Author: arigo Date: Wed Dec 23 14:11:51 2009 New Revision: 70262 Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py Log: Remove again that test, as I found out that it's already tested by test_setfield_of_same_type_clears. Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py Wed Dec 23 14:11:51 2009 @@ -848,6 +848,9 @@ def cache_field_value(self, descr, value, fieldvalue, write=False): if write: + # when seeing a setfield, we have to clear the cache for the same + # field on any other structure, just in case they are aliasing + # each other d = self.cached_fields[descr] = {} else: d = self.cached_fields.setdefault(descr, {}) @@ -921,6 +924,8 @@ if opnum == rop.CALL: effectinfo = op.descr.get_extra_info() if effectinfo is not None: + # XXX we can get the wrong complexity here, if the lists + # XXX stored on effectinfo are large for fielddescr in effectinfo.write_descrs_fields: try: del self.cached_fields[fielddescr] Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py Wed Dec 23 14:11:51 2009 @@ -2031,27 +2031,6 @@ """ self.optimize_loop(ops, 'Not, Not, Not', expected) - def test_setfield_might_alias(self): - # the point is that p1 and p2 might be equal or not - ops = """ - [p1, p2, i1, i1a] - setfield_gc(p1, i1, descr=adescr) - setfield_gc(p2, i1a, descr=adescr) - i2 = call(i1, descr=readadescr) - i3a = getfield_gc(p2, descr=adescr) # can go away - i3 = getfield_gc(p1, descr=adescr) # cannot go away - jump(p1, p2, i3, i1a) - """ - expected = """ - [p1, p2, i1, i1a] - setfield_gc(p1, i1, descr=adescr) - setfield_gc(p2, i1a, descr=adescr) - i2 = call(i1, descr=readadescr) - i3 = getfield_gc(p1, descr=adescr) - jump(p1, p2, i3, i1a) - """ - self.optimize_loop(ops, 'Not, Not, Not, Not', expected) - def test_residual_call_invalidates_some_read_caches(self): ops = """ [p1, i1, p2, i2] From arigo at codespeak.net Wed Dec 23 14:27:49 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 Dec 2009 14:27:49 +0100 (CET) Subject: [pypy-svn] r70263 - pypy/branch/jit-delayed-write/pypy/jit/metainterp/test Message-ID: <20091223132749.CC744168030@codespeak.net> Author: arigo Date: Wed Dec 23 14:27:49 2009 New Revision: 70263 Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py Log: More non-passing-yet tests. Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py Wed Dec 23 14:27:49 2009 @@ -1293,6 +1293,57 @@ """ self.optimize_loop(ops, 'Not, Not', ops) + def test_duplicate_setfield_1(self): + ops = """ + [p1, i1, i2] + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(p1, i2, descr=valuedescr) + jump(p1, i1, i2) + """ + expected = """ + [p1, i1, i2] + setfield_gc(p1, i2, descr=valuedescr) + jump(p1, i1, i2) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + + def test_duplicate_setfield_sideeffects_1(self): + ops = """ + [p1, i1, i2] + setfield_gc(p1, i1, descr=valuedescr) + escape() + setfield_gc(p1, i2, descr=valuedescr) + jump(p1, i1, i2) + """ + self.optimize_loop(ops, 'Not, Not, Not', ops) + + def test_duplicate_setfield_aliasing(self): + # a case where aliasing issues (and not enough cleverness) mean + # that we fail to remove any setfield_gc + ops = """ + [p1, p2, i1, i2, i3] + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(p2, i2, descr=valuedescr) + setfield_gc(p1, i3, descr=valuedescr) + jump(p1, p2, i1, i2, i3) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not, Not', ops) + + def test_duplicate_setfield_guard_value_const(self): + ops = """ + [p1, i1, i2] + guard_value(p1, ConstPtr(myptr)) [] + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(ConstPtr(myptr), i2, descr=valuedescr) + jump(p1, i1, i2) + """ + expected = """ + [i1, i2] + setfield_gc(ConstPtr(myptr), i2, descr=valuedescr) + jump(i1, i2) + """ + self.optimize_loop(ops, 'Constant(myptr)', expected) + def test_duplicate_getarrayitem_1(self): ops = """ [p1] From arigo at codespeak.net Fri Dec 25 18:46:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 25 Dec 2009 18:46:40 +0100 (CET) Subject: [pypy-svn] r70271 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091225174640.1FB0B168025@codespeak.net> Author: arigo Date: Fri Dec 25 18:46:40 2009 New Revision: 70271 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py Log: is_always_pure() is a subset of has_no_side_effect(). Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Fri Dec 25 18:46:40 2009 @@ -905,8 +905,6 @@ return None def emitting_operation(self, op): - if op.is_always_pure(): - return if op.has_no_side_effect(): return if op.is_ovf(): From electronicru at codespeak.net Fri Dec 25 20:17:30 2009 From: electronicru at codespeak.net (electronicru at codespeak.net) Date: Fri, 25 Dec 2009 20:17:30 +0100 (CET) Subject: [pypy-svn] r70272 - pypy/branch/2.6ish Message-ID: <20091225191730.C12DA16801F@codespeak.net> Author: electronicru Date: Fri Dec 25 20:17:30 2009 New Revision: 70272 Added: pypy/branch/2.6ish/ (props changed) - copied from r70270, pypy/trunk/ Log: Adding branch for Python 2.6 support. From arigo at codespeak.net Fri Dec 25 20:29:17 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 25 Dec 2009 20:29:17 +0100 (CET) Subject: [pypy-svn] r70273 - in pypy/branch/jit-delayed-write/pypy/jit/metainterp: . test Message-ID: <20091225192917.9156F16801F@codespeak.net> Author: arigo Date: Fri Dec 25 20:29:16 2009 New Revision: 70273 Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/effectinfo.py pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_effectinfo.py pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py Log: Get optimizeopt.py to delay some writes. All tests pass so far but might be slightly wrong. Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/effectinfo.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/effectinfo.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/effectinfo.py Fri Dec 25 20:29:16 2009 @@ -7,16 +7,16 @@ class EffectInfo(object): _cache = {} - def __new__(cls, read_descrs_fields, write_descrs_fields, + def __new__(cls, readonly_descrs_fields, write_descrs_fields, write_descrs_arrays, promotes_virtualizables=False): - key = (frozenset(read_descrs_fields), + key = (frozenset(readonly_descrs_fields), frozenset(write_descrs_fields), frozenset(write_descrs_arrays), promotes_virtualizables) if key in cls._cache: return cls._cache[key] result = object.__new__(cls) - result.read_descrs_fields = read_descrs_fields + result.readonly_descrs_fields = readonly_descrs_fields result.write_descrs_fields = write_descrs_fields result.write_descrs_arrays = write_descrs_arrays result.promotes_virtualizables = promotes_virtualizables @@ -27,8 +27,8 @@ from pypy.translator.backendopt.writeanalyze import top_set if effects is top_set: return None - read_descrs_fields = [] - # read_descrs_arrays = [] --- not enabled for now + readonly_descrs_fields = [] + # readonly_descrs_arrays = [] --- not enabled for now write_descrs_fields = [] write_descrs_arrays = [] @@ -48,14 +48,16 @@ if tup[0] == "struct": add_struct(write_descrs_fields, tup) elif tup[0] == "readstruct": - add_struct(read_descrs_fields, tup) + tupw = ("struct",) + tup[1:] + if tupw not in effects: + add_struct(readonly_descrs_fields, tup) elif tup[0] == "array": add_array(write_descrs_arrays, tup) elif tup[0] == "readarray": pass else: assert 0 - return EffectInfo(read_descrs_fields, + return EffectInfo(readonly_descrs_fields, write_descrs_fields, write_descrs_arrays, promotes_virtualizables) Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py Fri Dec 25 20:29:16 2009 @@ -512,6 +512,9 @@ def emit_operation(self, op, must_clone=True): self.heap_op_optimizer.emitting_operation(op) + self._emit_operation(op, must_clone) + + def _emit_operation(self, op, must_clone=True): for i in range(len(op.args)): arg = op.args[i] if arg in self.values: @@ -836,11 +839,13 @@ class HeapOpOptimizer(object): def __init__(self, optimizer): self.optimizer = optimizer - # cached OptValues for each field descr + # cached fields: {descr: {OptValue_instance: OptValue_fieldvalue}} self.cached_fields = {} - - # cached OptValues for each field descr + # cached array items: {descr: CachedArrayItems} self.cached_arrayitems = {} + # lazily written setfields (at most one per descr): {descr: op} + self.lazy_setfields = {} + self.lazy_setfields_descrs = [] # keys (at least) of previous dict def clean_caches(self): self.cached_fields.clear() @@ -908,8 +913,6 @@ return None def emitting_operation(self, op): - if op.is_always_pure(): - return if op.has_no_side_effect(): return if op.is_ovf(): @@ -921,12 +924,16 @@ opnum == rop.SETARRAYITEM_GC or opnum == rop.DEBUG_MERGE_POINT): return - if opnum == rop.CALL: + if (opnum == rop.CALL or + opnum == rop.CALL_MAY_FORCE): effectinfo = op.descr.get_extra_info() if effectinfo is not None: # XXX we can get the wrong complexity here, if the lists # XXX stored on effectinfo are large + for fielddescr in effectinfo.readonly_descrs_fields: + self.force_lazy_setfield(fielddescr) for fielddescr in effectinfo.write_descrs_fields: + self.force_lazy_setfield(fielddescr) try: del self.cached_fields[fielddescr] except KeyError: @@ -937,9 +944,38 @@ except KeyError: pass return + self.force_all_lazy_setfields() + elif op.is_final() or (not we_are_translated() and + op.opnum < 0): # escape() operations + self.force_all_lazy_setfields() self.clean_caches() + def force_lazy_setfield(self, descr): + try: + op = self.lazy_setfields[descr] + except KeyError: + return + del self.lazy_setfields[descr] + self.optimizer._emit_operation(op) + + def force_all_lazy_setfields(self): + if len(self.lazy_setfields_descrs) > 0: + for descr in self.lazy_setfields_descrs: + self.force_lazy_setfield(descr) + del self.lazy_setfields_descrs[:] + + def force_lazy_setfield_if_necessary(self, op, value, write=False): + try: + op1 = self.lazy_setfields[op.descr] + except KeyError: + if write: + self.lazy_setfields_descrs.append(op.descr) + else: + if self.optimizer.getvalue(op1.args[0]) is not value: + self.force_lazy_setfield(op.descr) + def optimize_GETFIELD_GC(self, op, value): + self.force_lazy_setfield_if_necessary(op, value) # check if the field was read from another getfield_gc just before # or has been written to recently fieldvalue = self.read_cached_field(op.descr, value) @@ -954,7 +990,8 @@ self.cache_field_value(op.descr, value, fieldvalue) def optimize_SETFIELD_GC(self, op, value, fieldvalue): - self.optimizer.emit_operation(op) + self.force_lazy_setfield_if_necessary(op, value, write=True) + self.lazy_setfields[op.descr] = op # remember the result of future reads of the field self.cache_field_value(op.descr, value, fieldvalue, write=True) Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_effectinfo.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_effectinfo.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_effectinfo.py Fri Dec 25 20:29:16 2009 @@ -13,7 +13,7 @@ S = lltype.GcStruct("S", ("a", lltype.Signed)) effects = frozenset([("readstruct", lltype.Ptr(S), "a")]) effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU()) - assert list(effectinfo.read_descrs_fields) == [('fielddescr', S, "a")] + assert list(effectinfo.readonly_descrs_fields) == [('fielddescr', S, "a")] assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays @@ -22,49 +22,58 @@ effects = frozenset([("struct", lltype.Ptr(S), "a")]) effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU()) assert list(effectinfo.write_descrs_fields) == [('fielddescr', S, "a")] - assert not effectinfo.read_descrs_fields + assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_arrays def test_include_write_array(): A = lltype.GcArray(lltype.Signed) effects = frozenset([("array", lltype.Ptr(A))]) effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU()) - assert not effectinfo.read_descrs_fields + assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert list(effectinfo.write_descrs_arrays) == [('arraydescr', A)] +def test_dont_include_read_and_write_field(): + S = lltype.GcStruct("S", ("a", lltype.Signed)) + effects = frozenset([("readstruct", lltype.Ptr(S), "a"), + ("struct", lltype.Ptr(S), "a")]) + effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU()) + assert not effectinfo.readonly_descrs_fields + assert list(effectinfo.write_descrs_fields) == [('fielddescr', S, "a")] + assert not effectinfo.write_descrs_arrays + def test_filter_out_typeptr(): effects = frozenset([("struct", lltype.Ptr(OBJECT), "typeptr")]) effectinfo = effectinfo_from_writeanalyze(effects, None) - assert not effectinfo.read_descrs_fields + assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays def test_filter_out_array_of_void(): effects = frozenset([("array", lltype.Ptr(lltype.GcArray(lltype.Void)))]) effectinfo = effectinfo_from_writeanalyze(effects, None) - assert not effectinfo.read_descrs_fields + assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays def test_filter_out_struct_with_void(): effects = frozenset([("struct", lltype.Ptr(lltype.GcStruct("x", ("a", lltype.Void))), "a")]) effectinfo = effectinfo_from_writeanalyze(effects, None) - assert not effectinfo.read_descrs_fields + assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays def test_filter_out_ooarray_of_void(): effects = frozenset([("array", ootype.Array(ootype.Void))]) effectinfo = effectinfo_from_writeanalyze(effects, None) - assert not effectinfo.read_descrs_fields + assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays def test_filter_out_instance_with_void(): effects = frozenset([("struct", ootype.Instance("x", ootype.ROOT, {"a": ootype.Void}), "a")]) effectinfo = effectinfo_from_writeanalyze(effects, None) - assert not effectinfo.read_descrs_fields + assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizefindnode.py Fri Dec 25 20:29:16 2009 @@ -95,6 +95,7 @@ onedescr = cpu.fielddescrof(U, 'one') FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) + plaincalldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([], [], [])) writeadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py Fri Dec 25 20:29:16 2009 @@ -606,10 +606,10 @@ p3sub = getfield_gc(p3, descr=nextdescr) i3 = getfield_gc(p3sub, descr=valuedescr) escape(i3) + p1 = new_with_vtable(ConstClass(node_vtable)) p2sub = new_with_vtable(ConstClass(node_vtable2)) setfield_gc(p2sub, i1, descr=valuedescr) setfield_gc(p2, p2sub, descr=nextdescr) - p1 = new_with_vtable(ConstClass(node_vtable)) jump(i1, p1, p2) """ # The same as test_p123_simple, but in the end the "old" p2 contains @@ -1307,6 +1307,65 @@ """ self.optimize_loop(ops, 'Not, Not, Not', expected) + def test_duplicate_setfield_2(self): + ops = """ + [p1, i1, i3] + setfield_gc(p1, i1, descr=valuedescr) + i2 = getfield_gc(p1, descr=valuedescr) + setfield_gc(p1, i3, descr=valuedescr) + escape(i2) + jump(p1, i1, i3) + """ + expected = """ + [p1, i1, i3] + setfield_gc(p1, i3, descr=valuedescr) + escape(i1) + jump(p1, i1, i3) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + + def test_duplicate_setfield_3(self): + ops = """ + [p1, p2, i1, i3] + setfield_gc(p1, i1, descr=valuedescr) + i2 = getfield_gc(p2, descr=valuedescr) + setfield_gc(p1, i3, descr=valuedescr) + escape(i2) + jump(p1, p2, i1, i3) + """ + # potential aliasing of p1 and p2 means that we cannot kill the + # the setfield_gc + self.optimize_loop(ops, 'Not, Not, Not, Not', ops) + + def test_duplicate_setfield_4(self): + ops = """ + [p1, i1, i2, p3] + setfield_gc(p1, i1, descr=valuedescr) + # + # some operations on which the above setfield_gc cannot have effect + i3 = getarrayitem_gc_pure(p3, 1, descr=arraydescr) + i4 = getarrayitem_gc(p3, i3, descr=arraydescr) + i5 = int_add(i3, i4) + setarrayitem_gc(p3, 0, i5, descr=arraydescr) + setfield_gc(p1, i4, descr=nextdescr) + # + setfield_gc(p1, i2, descr=valuedescr) + jump(p1, i1, i2, p3) + """ + expected = """ + [p1, i1, i2, p3] + # + i3 = getarrayitem_gc_pure(p3, 1, descr=arraydescr) + i4 = getarrayitem_gc(p3, i3, descr=arraydescr) + i5 = int_add(i3, i4) + setarrayitem_gc(p3, 0, i5, descr=arraydescr) + # + setfield_gc(p1, i2, descr=valuedescr) + setfield_gc(p1, i4, descr=nextdescr) + jump(p1, i1, i2, p3) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', expected) + def test_duplicate_setfield_sideeffects_1(self): ops = """ [p1, i1, i2] @@ -1342,7 +1401,7 @@ setfield_gc(ConstPtr(myptr), i2, descr=valuedescr) jump(i1, i2) """ - self.optimize_loop(ops, 'Constant(myptr)', expected) + self.optimize_loop(ops, 'Constant(myptr), Not, Not', expected) def test_duplicate_getarrayitem_1(self): ops = """ @@ -2082,7 +2141,7 @@ """ self.optimize_loop(ops, 'Not, Not, Not', expected) - def test_residual_call_invalidates_some_read_caches(self): + def test_residual_call_invalidates_some_read_caches_1(self): ops = """ [p1, i1, p2, i2] setfield_gc(p1, i1, descr=valuedescr) @@ -2102,6 +2161,38 @@ """ self.optimize_loop(ops, 'Not, Not, Not, Not', expected) + def test_residual_call_invalidates_some_read_caches_2(self): + ops = """ + [p1, i1, p2, i2] + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(p2, i2, descr=adescr) + i3 = call(i1, descr=writeadescr) + setfield_gc(p1, i3, descr=valuedescr) + setfield_gc(p2, i3, descr=adescr) + jump(p1, i1, p2, i2) + """ + expected = """ + [p1, i1, p2, i2] + setfield_gc(p2, i2, descr=adescr) + i3 = call(i1, descr=writeadescr) + setfield_gc(p1, i3, descr=valuedescr) + setfield_gc(p2, i3, descr=adescr) + jump(p1, i1, p2, i2) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', expected) + + def test_residual_call_invalidates_some_read_caches_3(self): + ops = """ + [p1, i1, p2, i2] + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(p2, i2, descr=adescr) + i3 = call(i1, descr=plaincalldescr) + setfield_gc(p1, i3, descr=valuedescr) + setfield_gc(p2, i3, descr=adescr) + jump(p1, i1, p2, i2) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', ops) + class TestOOtype(BaseTestOptimizeOpt, OOtypeMixin): From arigo at codespeak.net Fri Dec 25 22:09:41 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 25 Dec 2009 22:09:41 +0100 (CET) Subject: [pypy-svn] r70274 - in pypy/branch/jit-delayed-write/pypy/jit/metainterp: . test Message-ID: <20091225210941.E983416801F@codespeak.net> Author: arigo Date: Fri Dec 25 22:09:39 2009 New Revision: 70274 Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py Log: Step 1: force lazy fields on all guards. Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py Fri Dec 25 22:09:39 2009 @@ -918,6 +918,7 @@ if op.is_ovf(): return if op.is_guard(): + self.force_all_lazy_setfields() return opnum = op.opnum if (opnum == rop.SETFIELD_GC or Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py Fri Dec 25 22:09:39 2009 @@ -1376,6 +1376,17 @@ """ self.optimize_loop(ops, 'Not, Not, Not', ops) + def test_duplicate_setfield_residual_guard(self): + ops = """ + [p1, i1, i2, i3] + setfield_gc(p1, i1, descr=valuedescr) + guard_true(i3) [] + i4 = int_neg(i2) + setfield_gc(p1, i2, descr=valuedescr) + jump(p1, i1, i2, i4) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', ops) + def test_duplicate_setfield_aliasing(self): # a case where aliasing issues (and not enough cleverness) mean # that we fail to remove any setfield_gc From arigo at codespeak.net Fri Dec 25 22:15:50 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 25 Dec 2009 22:15:50 +0100 (CET) Subject: [pypy-svn] r70275 - in pypy/branch/jit-delayed-write/pypy/jit/metainterp: . test Message-ID: <20091225211550.87BFB16801F@codespeak.net> Author: arigo Date: Fri Dec 25 22:15:50 2009 New Revision: 70275 Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py Log: force_all_lazy_setfields_of_nonvirtuals(). Missing: job in resume.py to handle the lazy setfields Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py Fri Dec 25 22:15:50 2009 @@ -918,7 +918,7 @@ if op.is_ovf(): return if op.is_guard(): - self.force_all_lazy_setfields() + self.force_all_lazy_setfields_of_nonvirtuals() return opnum = op.opnum if (opnum == rop.SETFIELD_GC or @@ -965,6 +965,16 @@ self.force_lazy_setfield(descr) del self.lazy_setfields_descrs[:] + def force_all_lazy_setfields_of_nonvirtuals(self): + for descr in self.lazy_setfields_descrs: + try: + op = self.lazy_setfields[descr] + except KeyError: + continue + fieldvalue = self.optimizer.getvalue(op.args[1]) + if not fieldvalue.is_virtual(): + self.force_lazy_setfield(descr) + def force_lazy_setfield_if_necessary(self, op, value, write=False): try: op1 = self.lazy_setfields[op.descr] Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py Fri Dec 25 22:15:50 2009 @@ -1376,7 +1376,7 @@ """ self.optimize_loop(ops, 'Not, Not, Not', ops) - def test_duplicate_setfield_residual_guard(self): + def test_duplicate_setfield_residual_guard_1(self): ops = """ [p1, i1, i2, i3] setfield_gc(p1, i1, descr=valuedescr) @@ -1387,6 +1387,27 @@ """ self.optimize_loop(ops, 'Not, Not, Not, Not', ops) + def test_duplicate_setfield_residual_guard_2(self): + # the difference with the previous test is that the field value is + # a virtual, which we try hard to keep virtual + ops = """ + [p1, i2, i3] + p2 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, p2, descr=nextdescr) + guard_true(i3) [] + i4 = int_neg(i2) + setfield_gc(p1, NULL, descr=nextdescr) + jump(p1, i2, i4) + """ + expected = """ + [p1, i2, i3] + guard_true(i3) [] + i4 = int_neg(i2) + setfield_gc(p1, NULL, descr=nextdescr) + jump(p1, i2, i4) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + def test_duplicate_setfield_aliasing(self): # a case where aliasing issues (and not enough cleverness) mean # that we fail to remove any setfield_gc From arigo at codespeak.net Sat Dec 26 15:08:03 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 26 Dec 2009 15:08:03 +0100 (CET) Subject: [pypy-svn] r70276 - in pypy/branch/jit-delayed-write/pypy/jit/metainterp: . test Message-ID: <20091226140803.6163F16801D@codespeak.net> Author: arigo Date: Sat Dec 26 15:08:01 2009 New Revision: 70276 Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py pypy/branch/jit-delayed-write/pypy/jit/metainterp/resume.py pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_resume.py Log: In-progress. Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py Sat Dec 26 15:08:01 2009 @@ -63,7 +63,11 @@ def get_args_for_fail(self, modifier): pass - def make_virtual_info(self, modifier, fieldnums): + def get_backstore(self): + return (None, None) + + def make_virtual_info(self, modifier, fieldnums, + backstore_num, backstore_descr): raise NotImplementedError # should not be called on this level def is_constant(self): @@ -138,10 +142,12 @@ class AbstractVirtualValue(OptValue): - _attrs_ = ('optimizer', 'keybox', 'source_op', '_cached_vinfo') + _attrs_ = ('optimizer', 'keybox', 'source_op', '_cached_vinfo', + 'backstore_field') box = None level = LEVEL_NONNULL _cached_vinfo = None + backstore_field = None # the fielddescr from lazy_setfields def __init__(self, optimizer, keybox, source_op=None): self.optimizer = optimizer @@ -160,19 +166,40 @@ self._really_force() return self.box - def make_virtual_info(self, modifier, fieldnums): - vinfo = self._cached_vinfo - if vinfo is not None and resume.tagged_list_eq( - vinfo.fieldnums, fieldnums): + def get_backstore(self): + if self.backstore_field is None: + return (None, None) + heapopt = self.optimizer.heap_op_optimizer + try: + op = heapopt.lazy_setfields[self.backstore_field] + except KeyError: + self.backstore_field = None + return (None, None) + if self.optimizer.getvalue(op.args[1]) is not self: + self.backstore_field = None + return (None, None) + return (op.args[0], self.backstore_field) + + def make_virtual_info(self, modifier, fieldnums, + backstore_num, backstore_descr): + vinfo = self._cached_vinfo + if vinfo is not None and vinfo.equals(fieldnums, backstore_num, + backstore_descr): return vinfo vinfo = self._make_virtual(modifier) - vinfo.fieldnums = fieldnums + vinfo.set_content(fieldnums, backstore_num, backstore_descr) self._cached_vinfo = vinfo return vinfo def _make_virtual(self, modifier): raise NotImplementedError("abstract base") + def register_virtual_fields(self, modifier, fieldboxes): + modifier.register_virtual_fields(self.keybox, fieldboxes) + parentbox, parentdescr = self.get_backstore() + if parentdescr is not None: + modifier.register_box(parentbox) + def get_fielddescrlist_cache(cpu): if not hasattr(cpu, '_optimizeopt_fielddescrlist_cache'): result = descrlist_dict() @@ -238,7 +265,7 @@ # we have already seen the very same keybox lst = self._get_field_descr_list() fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst] - modifier.register_virtual_fields(self.keybox, fieldboxes) + self.register_virtual_fields(modifier, fieldboxes) for ofs in lst: fieldvalue = self._fields[ofs] fieldvalue.get_args_for_fail(modifier) @@ -306,7 +333,7 @@ itemboxes = [] for itemvalue in self._items: itemboxes.append(itemvalue.get_key_box()) - modifier.register_virtual_fields(self.keybox, itemboxes) + self.register_virtual_fields(modifier, itemboxes) for itemvalue in self._items: if itemvalue is not self.constvalue: itemvalue.get_args_for_fail(modifier) @@ -535,6 +562,7 @@ self.newoperations.append(op) def store_final_boxes_in_guard(self, op): + self.heap_op_optimizer.force_lazy_setfields_for_guard() descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo) @@ -918,7 +946,6 @@ if op.is_ovf(): return if op.is_guard(): - self.force_all_lazy_setfields_of_nonvirtuals() return opnum = op.opnum if (opnum == rop.SETFIELD_GC or @@ -957,6 +984,9 @@ except KeyError: return del self.lazy_setfields[descr] + fieldvalue = self.optimizer.getvalue(op.args[1]) + if isinstance(fieldvalue, AbstractVirtualValue): + fieldvalue.backstore_field = None self.optimizer._emit_operation(op) def force_all_lazy_setfields(self): @@ -965,15 +995,26 @@ self.force_lazy_setfield(descr) del self.lazy_setfields_descrs[:] - def force_all_lazy_setfields_of_nonvirtuals(self): + def force_lazy_setfields_for_guard(self): for descr in self.lazy_setfields_descrs: try: op = self.lazy_setfields[descr] except KeyError: continue + # the only really interesting case that we need to handle in the + # guards' resume data is that of a virtual object that is stored + # into a field of a non-virtual object. The later object cannot + # actually be virtual here (verified by an assert), but the + # former object 'fieldvalue' can be. + value = self.optimizer.getvalue(op.args[0]) + assert not value.is_virtual() fieldvalue = self.optimizer.getvalue(op.args[1]) - if not fieldvalue.is_virtual(): - self.force_lazy_setfield(descr) + if fieldvalue.is_virtual(): + assert isinstance(fieldvalue, AbstractVirtualValue) + if fieldvalue.backstore_field is descr: + # this is the case that can be handled by resume data + continue + self.force_lazy_setfield(descr) def force_lazy_setfield_if_necessary(self, op, value, write=False): try: @@ -1003,6 +1044,8 @@ def optimize_SETFIELD_GC(self, op, value, fieldvalue): self.force_lazy_setfield_if_necessary(op, value, write=True) self.lazy_setfields[op.descr] = op + if isinstance(fieldvalue, AbstractVirtualValue): + fieldvalue.backstore_field = op.descr # remember the result of future reads of the field self.cache_field_value(op.descr, value, fieldvalue, write=True) Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/resume.py Sat Dec 26 15:08:01 2009 @@ -252,8 +252,6 @@ if (isinstance(box, Box) and box not in self.liveboxes_from_env and box not in self.liveboxes): self.liveboxes[box] = UNASSIGNED - return True - return False def _register_boxes(self, boxes): for box in boxes: @@ -298,6 +296,7 @@ return liveboxes[:] def _number_virtuals(self, liveboxes, values, num_env_virtuals): + # !! 'liveboxes' is a list that is extend()ed in-place !! memo = self.memo new_liveboxes = [None] * memo.num_cached_boxes() count = 0 @@ -339,7 +338,13 @@ value = values[virtualbox] fieldnums = [self._gettagged(box) for box in fieldboxes] - vinfo = value.make_virtual_info(self, fieldnums) + backstore_box, backstore_descr = value.get_backstore() + if backstore_descr is not None: + backstore_num = self._gettagged(backstore_box) + else: + backstore_num = NULLREF + vinfo = value.make_virtual_info(self, fieldnums, + backstore_num, backstore_descr) # if a new vinfo instance is made, we get the fieldnums list we # pass in as an attribute. hackish. if vinfo.fieldnums is not fieldnums: @@ -366,11 +371,47 @@ return self.liveboxes_from_env[box] return self.liveboxes[box] + +class BackstoreRef(object): + # Used to say that a virtual object must, after being created because + # of a guard failure, be stored back on the given field of the given + # non-virtual object. For lazy setfields. Limited to one place per + # virtual for now. + def __init__(self, parentdescr, parentnum): + self.parentdescr = parentdescr + self.parentnum = parentnum + + def setfields(self, metainterp, box, fn_decode_box): + parentbox = fn_decode_box(self.parentnum) + metainterp.execute_and_record(rop.SETFIELD_GC, self.parentdescr, + parentbox, box) + class AbstractVirtualInfo(object): + backstore_ref = None + def allocate(self, metainterp): raise NotImplementedError + def setfields(self, metainterp, box, fn_decode_box): - raise NotImplementedError + if self.backstore_ref is not None: + self.backstore_ref.setfields(metainterp, box, fn_decode_box) + + def equals(self, fieldnums, backstore_num, backstore_descr): + return (tagged_list_eq(self.fieldnums, fieldnums) and + self.backstore_equals(backstore_num, backstore_descr)) + + def backstore_equals(self, backstore_num, backstore_descr): + if backstore_descr is None: + return self.backstore_ref is None + else: + return (self.backstore_ref is not None and + self.backstore_ref.parentdescr == backstore_descr and + self.backstore_ref.parentnum == backstore_num) + + def set_content(self, fieldnums, backstore_num, backstore_descr): + self.fieldnums = fieldnums + if backstore_descr is not None: + self.backstore_ref = BackstoreRef(backstore_descr, backstore_num) class AbstractVirtualStructInfo(AbstractVirtualInfo): @@ -384,6 +425,7 @@ metainterp.execute_and_record(rop.SETFIELD_GC, self.fielddescrs[i], box, fieldbox) + AbstractVirtualInfo.setfields(self, metainterp, box, fn_decode_box) def debug_prints(self): assert len(self.fielddescrs) == len(self.fieldnums) @@ -434,6 +476,7 @@ metainterp.execute_and_record(rop.SETARRAYITEM_GC, self.arraydescr, box, ConstInt(i), itembox) + AbstractVirtualInfo.setfields(self, metainterp, box, fn_decode_box) def debug_prints(self): debug_print("\tvarrayinfo", self.arraydescr) Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py Sat Dec 26 15:08:01 2009 @@ -87,18 +87,39 @@ def test_reuse_vinfo(): class FakeVInfo(object): - pass + def set_content(self, fieldnums, backstore_num, backstore_descr): + self.fieldnums = fieldnums + self.backstore_num = backstore_num + self.backstore_descr = backstore_descr + def equals(self, fieldnums, backstore_num, backstore_descr): + return (self.fieldnums == fieldnums and + self.backstore_num == backstore_num and + self.backstore_descr == backstore_descr) class FakeVirtualValue(optimizeopt.AbstractVirtualValue): def _make_virtual(self, *args): return FakeVInfo() v1 = FakeVirtualValue(None, None, None) - vinfo1 = v1.make_virtual_info(None, [1, 2, 4]) - vinfo2 = v1.make_virtual_info(None, [1, 2, 4]) + vinfo1 = v1.make_virtual_info(None, [1, 2, 4], None, None) + vinfo2 = v1.make_virtual_info(None, [1, 2, 4], None, None) assert vinfo1 is vinfo2 - vinfo3 = v1.make_virtual_info(None, [1, 2, 6]) + vinfo3 = v1.make_virtual_info(None, [1, 2, 6], None, None) assert vinfo3 is not vinfo2 - vinfo4 = v1.make_virtual_info(None, [1, 2, 6]) + vinfo4 = v1.make_virtual_info(None, [1, 2, 6], None, None) assert vinfo3 is vinfo4 + # + descr = object() + descr2 = object() + vinfo1 = v1.make_virtual_info(None, [1, 2, 4], None, None) + vinfo2 = v1.make_virtual_info(None, [1, 2, 4], 6, descr) + assert vinfo1 is not vinfo2 + vinfo3 = v1.make_virtual_info(None, [1, 2, 4], 6, descr) + assert vinfo3 is vinfo2 + vinfo4 = v1.make_virtual_info(None, [1, 2, 4], 5, descr) + assert vinfo4 is not vinfo3 + vinfo5 = v1.make_virtual_info(None, [1, 2, 4], 5, descr2) + assert vinfo5 is not vinfo4 + vinfo6 = v1.make_virtual_info(None, [1, 2, 4], 5, descr2) + assert vinfo6 is vinfo5 def test_descrlist_dict(): from pypy.jit.metainterp import optimizeutil @@ -2060,6 +2081,54 @@ where p7v is a node_vtable, valuedescr=iv ''') + def test_expand_fail_lazy_setfield_1(self): + self.make_fail_descr() + ops = """ + [p1, i2, i3] + p2 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p2, i2, descr=valuedescr) + setfield_gc(p1, p2, descr=nextdescr) + guard_true(i3, descr=fdescr) [p1] + i4 = int_neg(i2) + setfield_gc(p1, NULL, descr=nextdescr) + jump(p1, i2, i4) + """ + expected = """ + [p1, i2, i3] + guard_true(i3, descr=fdescr) [p1, i2] + i4 = int_neg(i2) + setfield_gc(p1, NULL, descr=nextdescr) + jump(p1, i2, i4) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + self.check_expanded_fail_descr('''p1 + where p2 is a node_vtable, valuedescr=i2 --> p1.nextdescr + ''') + + def test_expand_fail_lazy_setfield_2(self): + self.make_fail_descr() + ops = """ + [i2, i3] + p2 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p2, i2, descr=valuedescr) + setfield_gc(ConstPtr(myptr), p2, descr=nextdescr) + guard_true(i3, descr=fdescr) [] + i4 = int_neg(i2) + setfield_gc(ConstPtr(myptr), NULL, descr=nextdescr) + jump(i2, i4) + """ + expected = """ + [i2, i3] + guard_true(i3, descr=fdescr) [i2] + i4 = int_neg(i2) + setfield_gc(ConstPtr(myptr), NULL, descr=nextdescr) + jump(i2, i4) + """ + self.optimize_loop(ops, 'Not, Not', expected) + self.check_expanded_fail_descr(''' + where p2 is a node_vtable, valuedescr=i2 --> myptr.nextdescr + ''') + class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_resume.py Sat Dec 26 15:08:01 2009 @@ -40,6 +40,24 @@ assert not tagged_list_eq([tag(1, TAGBOX)], [tag(-2, TAGBOX)]) assert not tagged_list_eq([tag(1, TAGBOX), tag(-2, TAGBOX)], [tag(1, TAGBOX)]) +def test_vinfo(): + v1 = AbstractVirtualInfo() + v1.set_content([1, 2, 4], -1, None) + assert v1.backstore_ref is None + assert v1.equals([1, 2, 4], -1, None) + assert not v1.equals([1, 2, 6], -1, None) + assert not v1.equals([1, 2, 4], 3, object()) + # + v2 = AbstractVirtualInfo() + descr = object() + v2.set_content([1, 2, 4], 3, descr) + assert v2.backstore_ref is not None + assert v2.backstore_ref.parentdescr is descr + assert v2.backstore_ref.parentnum == 3 + assert v2.equals([1, 2, 4], 3, descr) + assert not v2.equals([1, 2, 4], 2, descr) + assert not v2.equals([1, 2, 4], 3, object()) + class MyMetaInterp: _already_allocated_resume_virtuals = None @@ -960,6 +978,73 @@ assert ptr.a == 111 assert ptr.b == lltype.nullptr(LLtypeMixin.NODE) + +def test_virtual_adder_make_virtual_backstore(): + b2s, b4s, b5s = [BoxPtr(), BoxPtr(), BoxPtr()] + c1s = ConstInt(111) + storage = Storage() + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) + modifier = ResumeDataVirtualAdder(storage, memo) + modifier.liveboxes_from_env = {} + modifier.liveboxes = {} + modifier.vfieldboxes = {} + + class FakeOptimizer(object): + class cpu: + pass + def getvalue(self, box): + return {b2s: v2, b4s: v4}[box] + fakeoptimizer = FakeOptimizer() + class HeapOpOptimizer(object): + pass + fakeoptimizer.heap_op_optimizer = HeapOpOptimizer() + class FakeSetFieldOperation(object): + args = [b2s, b4s] + fakeoptimizer.heap_op_optimizer.lazy_setfields = { + LLtypeMixin.nextdescr: FakeSetFieldOperation()} + + v4 = VirtualValue(fakeoptimizer, ConstAddr(LLtypeMixin.node_vtable_adr2, + LLtypeMixin.cpu), b4s) + v4.backstore_field = LLtypeMixin.nextdescr + v4.setfield(LLtypeMixin.valuedescr, OptValue(c1s)) + v4.setfield(LLtypeMixin.otherdescr, OptValue(b5s)) + v4._cached_sorted_fields = [LLtypeMixin.valuedescr, LLtypeMixin.otherdescr] + + v2 = OptValue(b2s) + + v4.register_virtual_fields(modifier, [c1s, b5s]) + liveboxes = [] + modifier._number_virtuals(liveboxes, {b4s: v4}, 0) + assert liveboxes == [b2s, b5s] + storage.rd_consts = memo.consts[:] + storage.rd_numb = None + # resume + demo55.next = lltype.nullptr(LLtypeMixin.NODE) + b2t = BoxPtr(demo55o) + b5t = BoxPtr(demo66o) + newboxes = _resume_remap(liveboxes, [b2s, b5s], b2t, b5t) + + metainterp = MyMetaInterp() + reader = ResumeDataReader(storage, newboxes, metainterp) + assert len(reader.virtuals) == 1 + b4t = reader._decode_box(modifier._gettagged(b4s)) + trace = metainterp.trace + b4new = (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr2, + LLtypeMixin.cpu)], + b4t, None) + b4set1 = (rop.SETFIELD_GC, [b4t, c1s], None, LLtypeMixin.valuedescr) + b4set2 = (rop.SETFIELD_GC, [b4t, b5t], None, LLtypeMixin.otherdescr) + b2set = (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.nextdescr) + expected = [b4new, b4set1, b4set2, b2set] + + for x, y in zip(expected, trace): + assert x == y + ptr = demo55.next + assert ptr.value == 111 + ptr = lltype.cast_pointer(lltype.Ptr(LLtypeMixin.NODE2), ptr) + assert ptr.other == demo66 + + def test_invalidation_needed(): class options: failargs_limit = 10 From afa at codespeak.net Mon Dec 28 00:29:42 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 28 Dec 2009 00:29:42 +0100 (CET) Subject: [pypy-svn] r70292 - pypy/trunk/pypy/tool Message-ID: <20091227232942.58040168020@codespeak.net> Author: afa Date: Mon Dec 28 00:29:41 2009 New Revision: 70292 Modified: pypy/trunk/pypy/tool/udir.py Log: Ensure that the prefix used to name the usession-xxx directory is not a unicode string. Otherwise a unicode filename is built and compared to the content of os.listdir('/tmp'), which fails if it contains some file with non-ascii-7 letters. Modified: pypy/trunk/pypy/tool/udir.py ============================================================================== --- pypy/trunk/pypy/tool/udir.py (original) +++ pypy/trunk/pypy/tool/udir.py Mon Dec 28 00:29:41 2009 @@ -18,7 +18,7 @@ # import autopath -import os +import os, sys import py from py.path import local @@ -39,6 +39,8 @@ try: p = py.path.local(__file__).dirpath() basename = svn_info(py.path.svnwc(p).info().url) + if isinstance(basename, unicode): + basename = basename.encode(sys.getdefaultencoding()) except: basename = '' if not basename.startswith('-'): From dan at codespeak.net Mon Dec 28 09:29:22 2009 From: dan at codespeak.net (dan at codespeak.net) Date: Mon, 28 Dec 2009 09:29:22 +0100 (CET) Subject: [pypy-svn] r70294 - pypy/branch/micronumpy/pypy/module/micronumpy Message-ID: <20091228082922.91DBB168015@codespeak.net> Author: dan Date: Mon Dec 28 09:29:21 2009 New Revision: 70294 Added: pypy/branch/micronumpy/pypy/module/micronumpy/array.py pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py pypy/branch/micronumpy/pypy/module/micronumpy/mdarray.py pypy/branch/micronumpy/pypy/module/micronumpy/ndarray.py pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py - copied, changed from r70213, pypy/branch/micronumpy/pypy/module/micronumpy/numarray.py Removed: pypy/branch/micronumpy/pypy/module/micronumpy/numarray.py Modified: pypy/branch/micronumpy/pypy/module/micronumpy/__init__.py pypy/branch/micronumpy/pypy/module/micronumpy/app_numarray.py pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py Log: Many changes. 7/9 tests passing. Organized source code. Parameterized single dimensional array such that it can be applied to different types. Added applevel visible ndarray type, like NumPy supports. Lots of little growing pain issues abound, committing just to make my changes publicly visible. Modified: pypy/branch/micronumpy/pypy/module/micronumpy/__init__.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/__init__.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/__init__.py Mon Dec 28 09:29:21 2009 @@ -5,13 +5,12 @@ applevel_name = 'numpy' appleveldefs = { - 'array' : 'app_numarray.array', + #'array' : 'app_numarray.array', } interpleveldefs = { - 'zeros' : 'numarray.zeros', + 'array' : 'ndarray.array', + 'zeros' : 'ndarray.zeros', 'minimum' : 'ufunc.minimum', - 'IntArray' : 'numarray.IntArray', - 'FloatArray' : 'numarray.FloatArray', } Modified: pypy/branch/micronumpy/pypy/module/micronumpy/app_numarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/app_numarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/app_numarray.py Mon Dec 28 09:29:21 2009 @@ -16,16 +16,10 @@ result = t return result +#FIXME: move me to interplevel def array(xs, dtype=None): import numpy - arrays = { - int: numpy.IntArray, - float: numpy.FloatArray, - #complex: ComplexNumArray, - } - #type = lowest_common_type(xs) - #return arrays[type](xs) - result = numpy.zeros(len(xs), dtype=int) #FIXME: dtype=dtype ! + result = numpy.zeros(len(xs), dtype=dtype if dtype else lowest_common_type(xs)) for i, x in enumerate(xs): result[i] = x return result Added: pypy/branch/micronumpy/pypy/module/micronumpy/array.py ============================================================================== --- (empty file) +++ pypy/branch/micronumpy/pypy/module/micronumpy/array.py Mon Dec 28 09:29:21 2009 @@ -0,0 +1,49 @@ +class BaseNumArray(object): + pass + +def iterable_type(space, w_xs): + xs = space.fixedview(w_xs) + type = int + for i in range(len(xs)): + type = result_types[type, xs[i]] + return type + +def mul_operation(): + def mul(x, y): return x * y + return mul + +def div_operation(): + def div(x, y): return x / y + return div + +def add_operation(): + def add(x, y): return x * y + return add + +def sub_operation(): + def sub(x, y): return x - y + return sub + +def copy_operation(): + def copy(x, y): return x #XXX: I sure hope GCC can optimize this + return copy + +def app_mul_operation(): + def mul(space, x, y): + return space.mul(x, y) + return mul + +def app_div_operation(): + def div(space, x, y): + return space.div(x, y) + return div + +def app_add_operation(): + def add(space, x, y): + return space.add(x, y) + return add + +def app_sub_operation(): + def sub(space, x, y): + return space.sub(x, y) + return sub Added: pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py ============================================================================== --- (empty file) +++ pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py Mon Dec 28 09:29:21 2009 @@ -0,0 +1,15 @@ +def unwrap_int(space, w_x): + return space.int_w(w_x) +def coerce_int(space, w_x): + return unwrap_int(space, space.int(w_x)) + +def unwrap_float(space, w_x): + return space.float_w(w_x) +def coerce_float(space, w_x): + return unwrap_float(space, space.float(w_x)) + +from rlib.rarithmetic import r_singlefloat as float32 +def unwrap_float32(space, w_x): + return float32(space.float_w(w_x)) +def coerce_float32(space, w_x): + return unwrap_float32(space, space.float(w_x)) Added: pypy/branch/micronumpy/pypy/module/micronumpy/mdarray.py ============================================================================== --- (empty file) +++ pypy/branch/micronumpy/pypy/module/micronumpy/mdarray.py Mon Dec 28 09:29:21 2009 @@ -0,0 +1,63 @@ +from pypy.interpreter.error import OperationError +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import interp2app, NoneNotWrapped +from pypy.interpreter.baseobjspace import ObjSpace, W_Root, Wrappable +from pypy.rlib.debug import make_sure_not_resized + +from pypy.module.micronumpy.array import BaseNumArray + +def compute_pos(space, indexes, dim): + current = 1 + pos = 0 + for i in range(len(indexes)): + index = indexes[i] + d = dim[i] + if index >= d or index <= -d - 1: + raise OperationError(space.w_IndexError, + space.wrap("invalid index")) + if index < 0: + index = d + index + pos += index * current + current *= d + return pos + +class MultiDimArray(BaseNumArray): + def __init__(self, space, shape, dtype): + self.shape = shape + self.space = space + # ignore dtype for now + size = 1 + for dimension in shape: + size *= dimension + self.storage = [0] * size + make_sure_not_resized(self.storage) + + def _unpack_indexes(self, space, w_index): + indexes = [space.int_w(w_i) for w_i in space.fixedview(w_index)] + if len(indexes) != len(self.shape): + raise OperationError(space.w_IndexError, space.wrap( + 'Wrong index')) + return indexes + + def getitem(self, w_index): + space = self.space + indexes = self._unpack_indexes(space, w_index) + pos = compute_pos(space, indexes, self.shape) + return space.wrap(self.storage[pos]) + + def setitem(self, w_index, w_value): + space = self.space + indexes = self._unpack_indexes(space, w_index) + pos = compute_pos(space, indexes, self.shape) + self.storage[pos] = space.int_w(w_value) #FIXME: lets get this thing generalized! + return space.w_None #XXX: necessary? + + def len(self): + space = self.space + return space.wrap(self.shape[0]) + +def unpack_dtype(space, w_dtype): + if space.is_w(w_dtype, space.w_int): + return 'i' + else: + raise NotImplementedError Added: pypy/branch/micronumpy/pypy/module/micronumpy/ndarray.py ============================================================================== --- (empty file) +++ pypy/branch/micronumpy/pypy/module/micronumpy/ndarray.py Mon Dec 28 09:29:21 2009 @@ -0,0 +1,131 @@ +from pypy.interpreter.baseobjspace import ObjSpace, W_Root, Wrappable +from pypy.interpreter.error import OperationError +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import interp2app, NoneNotWrapped +from pypy.interpreter.baseobjspace import ObjSpace, W_Root, Wrappable +from pypy.rlib.debug import make_sure_not_resized + +from pypy.module.micronumpy.sdarray import sdresult +from pypy.module.micronumpy.sdarray import GenericArray + +from pypy.module.micronumpy.mdarray import MultiDimArray + +def unpack_shape(space, w_shape): + if space.is_true(space.isinstance(w_shape, space.w_int)): + return [space.int_w(w_shape)] + shape_w = space.fixedview(w_shape) + return [space.int_w(w_i) for w_i in shape_w] + +class ArrayIter(Wrappable): #FIXME: 1d only! + def __init__(self, space, array, i): + self.space = space + self.array = array + self.i = i + + def descr_iter(self): + self.space = space + return space.wrap(self) + descr_iter.unwrap_spec = ['self'] + + def descr_next(self): + space = self.space + try: + result = self.array.array.storage[self.i] + self.i += 1 + return space.wrap(result) + except IndexError, e: + raise OperationError(space.w_StopIteration, space.wrap("")) + descr_iter.unwrap_spec = ['self'] + +ArrayIter.typedef = TypeDef('ArrayIter', + __iter__ = interp2app(ArrayIter.descr_iter), + next = interp2app(ArrayIter.descr_next)) + +def infer_shape(space, w_values): + return space.len(w_values) #TODO: handle multi-dimensional arrays... + +class ndarray(Wrappable): + def __init__(self, space, w_values, w_shape, w_dtype): + self.array = None + self.space = space + if w_dtype == space.w_None: + #TODO: infer type from w_values (better than this) + w_dtype = space.type(space.fixedview(w_values)[0]) #FIXME: Allocates an entire array and throws it away! + + self.dtype = w_dtype + if w_shape == space.w_None: + values_shape = infer_shape(space, w_values) + shape_w = unpack_shape(space, values_shape) + else: + shape_w = unpack_shape(space, w_shape) + if len(shape_w) == 1: + length = shape_w[0] + try: + self.array = sdresult(space, w_dtype)(space, length) + except KeyError, e: + raise OperationError(space.w_NotImplementedError, + space.wrap("Haven't implemented generic array yet!")) + + if not w_values == space.w_None: #space.is_true() ? + self.array.load_iterable(space, w_values) + else: + self.array = MultiDimArray(space, unpack_shape(space, w_shape), int) #FIXME: multi-dimensional arrays not *really* implemented + + def validate_index(self, space, w_i): + try: + index_dimensionality = space.int_w(space.len(w_i)) + array_dimensionality = len(self.array.shape) + if index_dimensionality > array_dimensionality: + raise OperationError(space.w_IndexError, + space.wrap("Index dimensionality (%d) greater than array dimensionality (%d)." % (index_dimensionality, array_dimensionality))) + except OperationError, e: + if e.match(space, space.w_TypeError): pass + else: raise + + def descr_mul(self, w_x): + space = self.space + if space.type(w_x) in (W_ListType, W_TupleType): #TODO: fooo + #xs = space.fixedview(w_x) + pass + else: + result_array = sdresult(space, space.type(w_x))(space, self.array.length) + result_array.mul_scalar(self.array, w_x) + result = ndarray(space, space.w_None, space.wrap(result_array.length), space.w_None) #FIXME: make ndarray.__init__ understand null args + result.array = result_array + descr_mul.unwrap_spec = ['self', W_Root] + + def descr_iter(self): + space = self.space + return space.wrap(ArrayIter(space, self, 0)) + descr_iter.unwrap_spec = ['self'] + + def descr_getitem(self, space, w_i): + self.validate_index(space, w_i) + return self.array.getitem(w_i) + descr_getitem.unwrap_spec = ['self', ObjSpace, W_Root] + + def descr_setitem(self, space, w_i, w_x): + self.validate_index(space, w_i) + self.array.setitem(w_i, w_x) + descr_setitem.unwrap_spec = ['self', ObjSpace, W_Root, W_Root] + + def descr_len(self, space): + return self.array.len() + descr_len.unwrap_spec = ['self', ObjSpace] + +ndarray.typedef = TypeDef( + 'ndarray', + #__init__ = interp2app(descr_init), #FIXME + __iter__ = interp2app(ndarray.descr_iter), + __getitem__ = interp2app(ndarray.descr_getitem), + __setitem__ = interp2app(ndarray.descr_setitem), + __len__ = interp2app(ndarray.descr_len), +) + +def array(space, w_values, w_shape=None, w_dtype=None): + return ndarray(space, w_values, w_shape, w_dtype) +array.unwrap_spec = [ObjSpace, W_Root, W_Root, W_Root] + +def zeros(space, w_shape, w_dtype): + return array(space, space.w_None, w_shape, w_dtype) +zeros.unwrap_spec = [ObjSpace, W_Root, W_Root] Copied: pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py (from r70213, pypy/branch/micronumpy/pypy/module/micronumpy/numarray.py) ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/numarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py Mon Dec 28 09:29:21 2009 @@ -3,59 +3,46 @@ from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app, NoneNotWrapped from pypy.rlib.debug import make_sure_not_resized +from pypy.objspace.std.typeobject import W_TypeObject -from pypy.rlib.objectmodel import specialize +from pypy.module.micronumpy.array import BaseNumArray +from pypy.module.micronumpy.array import mul_operation, div_operation, add_operation, sub_operation +from pypy.module.micronumpy.array import copy_operation +from pypy.module.micronumpy.dtype import unwrap_int, coerce_int +from pypy.module.micronumpy.dtype import unwrap_float, coerce_float +from pypy.module.micronumpy.dtype import unwrap_float32, coerce_float32, float32 -result_types = { - (int, int): int, - (int, float): float, - (float, int): float, - (float, float): float - } - -class BaseNumArray(Wrappable): - pass - -def iterable_type(space, w_xs): - xs = space.fixedview(w_xs) - type = int - for i in range(len(xs)): - type = result_types[type, xs[i]] - return type - -def int_unwrapper(space, w_x): - return space.int_w(space.int(w_x)) - -def float_unwrapper(space, w_x): - return space.float_w(space.float(w_x)) - -def create_numarray(type, unwrapper, name): +def create_numarray(data_type, unwrap, coerce): class NumArray(BaseNumArray): - def __init__(self, space, length, dtype): - #ignore dtype, irrelevant to optimized numarray implementations too + def __init__(self, space, length): + self.shape = (1,) self.length = length self.space = space - self.storage = [type(0.0)] * length + self.storage = [data_type(0.0)] * length make_sure_not_resized(self.storage) + mul = mul_operation() + div = div_operation() + add = add_operation() + sub = sub_operation() + copy = copy_operation() - def _dup_size(self, type): - return self.__class__(space, result_types[self.type, type], self.length) - - def create_scalar_op(unwrap, f): - def scalar_operation(self, w_x): + def create_scalar_op(f): + def scalar_operation(self, space, source, w_x): space = self.space - x = unwrap(w_x) - result = self._dup_size(type(x)) - for i in range(self.length): - result[i] = f(self.storage[i], x) - return space.wrap(result) + x = self.coerce(space, w_x) + for i in range(source.length): + self.storage[i] = f(source.storage[i], x) return scalar_operation - def create_fixedview_op(unwrap, f): + mul_scalar = create_scalar_op(mul) +# div_scalar = create_scalar_op(div) +# add_scalar = create_scalar_op(add) +# sub_scalar = create_scalar_op(sub) + + def create_fixedview_op(f): def fixedview_operation(self, w_xs): space = self.space - try: xs = space.fixedview(w_xs, len(self.storage)) except UnpackValueError, e: @@ -63,150 +50,68 @@ raise OperationError(space.w_ValueError, space.wrap("shape mismatch: objects cannot be broadcast to the same shape")) - result = self._dup_size(iterable_type(space, xs)) - i = 0 for w_x in xs: - result[i] = f(self.storage[i], unwrap(w_x)) + self.storage[i] = f(source.storage[i], self.coerce(w_x)) #TODO: probably shouldn't coerce i += 1 return result return fixedview_operation - #def mul_iterable(self, w_xs): - #return self.fixedview_operation(w_xs, mul) - -# def descr_mul(self, w_x): -# space = self.space -# if space.type(w_x) in [W_Int, W_Float]: #complex, long -# try: -# return self.mul_scalar(space.int_w(w_x)) -# except TypeError: -# return self.mul_scalar(space.float_w(w_x)) -# else: -# return self.mul_iterable(w_x) -# descr_mul.unwrap_spec = ['self', W_Root] + copy_iterable = create_fixedview_op(copy) + + def load_iterable(self, space, w_values): #FIXME: less than ideal + i = 0 + for x in space.fixedview(w_values, self.length): + self.storage[i] = unwrap(space, x) + i += 1 - def descr_getitem(self, index): + def getitem(self, w_index): space = self.space + index = space.int_w(w_index) try: return space.wrap(self.storage[index]) except IndexError: raise OperationError(space.w_IndexError, space.wrap("list index out of range")) - descr_getitem.unwrap_spec = ['self', int] - def descr_setitem(self, index, w_value): + def setitem(self, w_index, w_value): space = self.space + index = space.int_w(w_index) try: - self.storage[index] = unwrapper(space, w_value) + self.storage[index] = coerce(space, w_value) except IndexError: raise OperationError(space.w_IndexError, space.wrap("list index out of range")) return space.w_None - descr_setitem.unwrap_spec = ['self', int, W_Root] - def descr_len(self): - return self.space.wrap(len(self.storage)) - descr_len.unwrap_spec = ['self'] - - def descr_init(xs): pass - - NumArray.typedef = TypeDef( - name, - #__init__ = interp2app(descr_init), #FIXME - __getitem__ = interp2app(NumArray.descr_getitem), - __setitem__ = interp2app(NumArray.descr_setitem), - __len__ = interp2app(NumArray.descr_len), - ) + def len(self): + space = self.space + return space.wrap(len(self.storage)) + return NumArray -IntArray = create_numarray(int, int_unwrapper, 'IntArray') +IntArray = create_numarray(int, unwrap_int, coerce_int) NumArray = IntArray # FIXME: compatibility for now -FloatArray = create_numarray(float, float_unwrapper, 'FloatArray') +DoubleArray = create_numarray(float, unwrap_float, coerce_float) +FloatArray = create_numarray(float32, unwrap_float32, coerce_float32) +GenericArray = None -#def array(space, w_xs): -# w_length = space.len(w_xs) -# length = space.int_w(w_length) -# #TODO: discover type -# result = NumArray(space, type, length) -#array.unwrap_spec = [ObjSpace, W_Root] - -def compute_pos(space, indexes, dim): - current = 1 - pos = 0 - for i in range(len(indexes)): - index = indexes[i] - d = dim[i] - if index >= d or index <= -d - 1: - raise OperationError(space.w_IndexError, - space.wrap("invalid index")) - if index < 0: - index = d + index - pos += index * current - current *= d - return pos - -class MultiDimArray(BaseNumArray): - def __init__(self, space, dim, dtype): - self.dim = dim +class ResultFactory(object): + def __init__(self, space): self.space = space - # ignore dtype for now - size = 1 - for el in dim: - size *= el - self.storage = [0] * size - make_sure_not_resized(self.storage) - - def _unpack_indexes(self, space, w_index): - indexes = [space.int_w(w_i) for w_i in space.fixedview(w_index)] - if len(indexes) != len(self.dim): - raise OperationError(space.w_IndexError, space.wrap( - 'Wrong index')) - return indexes - - def descr_getitem(self, w_index): - space = self.space - indexes = self._unpack_indexes(space, w_index) - pos = compute_pos(space, indexes, self.dim) - return space.wrap(self.storage[pos]) - descr_getitem.unwrap_spec = ['self', W_Root] - - def descr_setitem(self, w_index, value): - space = self.space - indexes = self._unpack_indexes(space, w_index) - pos = compute_pos(space, indexes, self.dim) - self.storage[pos] = value - return space.w_None - descr_setitem.unwrap_spec = ['self', W_Root, int] - - def descr_len(self): - return self.space.wrap(self.dim[0]) - descr_len.unwrap_spec = ['self'] - -MultiDimArray.typedef = TypeDef( - 'NumArray', - __getitem__ = interp2app(MultiDimArray.descr_getitem), - __setitem__ = interp2app(MultiDimArray.descr_setitem), - __len__ = interp2app(MultiDimArray.descr_len), -) - -def unpack_dim(space, w_dim): - if space.is_true(space.isinstance(w_dim, space.w_int)): - return [space.int_w(w_dim)] - dim_w = space.fixedview(w_dim) - return [space.int_w(w_i) for w_i in dim_w] - -def unpack_dtype(space, w_dtype): - if space.is_w(w_dtype, space.w_int): - return 'i' - else: - raise NotImplementedError - -def zeros(space, w_dim, w_dtype): - dim = unpack_dim(space, w_dim) - dtype = unpack_dtype(space, w_dtype) - if len(dim) == 1: - return space.wrap(NumArray(space, dim[0], dtype)) - else: - return space.wrap(MultiDimArray(space, dim, dtype)) -zeros.unwrap_spec = [ObjSpace, W_Root, W_Root] + from pypy.objspace.std.inttype import int_typedef + int_type = space.gettypeobject(int_typedef) + from pypy.objspace.std.floattype import float_typedef + float_type = space.gettypeobject(float_typedef) + + self.types = { + int_type: IntArray, + float_type: FloatArray, + } + +result_factory = None +def sdresult(space, t): + global result_factory + if result_factory is None: + result_factory = ResultFactory(space) + return result_factory.types[t] Modified: pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py Mon Dec 28 09:29:21 2009 @@ -1,22 +1,21 @@ - -from numarray import NumArray +from pypy.module.micronumpy.ndarray import array, zeros, ndarray from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.interpreter.error import OperationError def minimum(space, w_a, w_b): - if not isinstance(w_a, NumArray) or not isinstance(w_b, NumArray): + if not isinstance(w_a, ndarray) or not isinstance(w_b, ndarray): raise OperationError(space.w_TypeError, - space.wrap("expecting NumArray object")) - if w_a.length != w_b.length: + space.wrap("expecting ndarray object")) + if w_a.array.length != w_b.array.length: raise OperationError(space.w_ValueError, space.wrap("minimum of arrays of different length")) - res = NumArray(space, w_a.length, 'i') - for i in range(len(w_a.storage)): - one = w_a.storage[i] - two = w_b.storage[i] + res = zeros(space, space.wrap(w_a.array.length), w_a.dtype) + for i in range(len(w_a.array.storage)): + one = w_a.array.storage[i] + two = w_b.array.storage[i] if one < two: - res.storage[i] = one + res.array.storage[i] = one else: - res.storage[i] = two + res.array.storage[i] = two return space.wrap(res) minimum.unwrap_spec = [ObjSpace, W_Root, W_Root] From dan at codespeak.net Mon Dec 28 10:50:53 2009 From: dan at codespeak.net (dan at codespeak.net) Date: Mon, 28 Dec 2009 10:50:53 +0100 (CET) Subject: [pypy-svn] r70295 - pypy/branch/micronumpy/pypy/module/micronumpy Message-ID: <20091228095053.E7D9B168015@codespeak.net> Author: dan Date: Mon Dec 28 10:50:53 2009 New Revision: 70295 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/mdarray.py pypy/branch/micronumpy/pypy/module/micronumpy/ndarray.py pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py Log: Made MultiDimArray aware of multiple types, although it's not yet tested. Modified: pypy/branch/micronumpy/pypy/module/micronumpy/mdarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/mdarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/mdarray.py Mon Dec 28 10:50:53 2009 @@ -6,6 +6,10 @@ from pypy.module.micronumpy.array import BaseNumArray +from pypy.module.micronumpy.dtype import unwrap_int, coerce_int +from pypy.module.micronumpy.dtype import unwrap_float, coerce_float +from pypy.module.micronumpy.dtype import unwrap_float32, coerce_float32, float32 + def compute_pos(space, indexes, dim): current = 1 pos = 0 @@ -21,43 +25,58 @@ current *= d return pos -class MultiDimArray(BaseNumArray): - def __init__(self, space, shape, dtype): - self.shape = shape +def create_mdarray(data_type, unwrap, coerce): + class MultiDimArray(BaseNumArray): + def __init__(self, space, shape): + self.shape = shape + self.space = space + size = 1 + for dimension in shape: + size *= dimension + self.storage = [data_type(0.0)] * size + make_sure_not_resized(self.storage) + + def _unpack_indexes(self, space, w_index): + indexes = [space.int_w(w_i) for w_i in space.fixedview(w_index)] + if len(indexes) != len(self.shape): + raise OperationError(space.w_IndexError, space.wrap( + 'Wrong index')) + return indexes + + def getitem(self, w_index): + space = self.space + indexes = self._unpack_indexes(space, w_index) + pos = compute_pos(space, indexes, self.shape) + return space.wrap(self.storage[pos]) + + def setitem(self, w_index, w_value): + space = self.space + indexes = self._unpack_indexes(space, w_index) + pos = compute_pos(space, indexes, self.shape) + self.storage[pos] = coerce(space, w_value) + return space.w_None #XXX: necessary? + + def len(self): + space = self.space + return space.wrap(self.shape[0]) + return MultiDimArray + +MultiDimIntArray = create_mdarray(int, unwrap_int, coerce_int) +MultiDimArray = MultiDimIntArray #XXX: compatibility +MultiDimFloatArray = create_mdarray(float, unwrap_float, coerce_float) + +class ResultFactory(object): + def __init__(self, space): self.space = space - # ignore dtype for now - size = 1 - for dimension in shape: - size *= dimension - self.storage = [0] * size - make_sure_not_resized(self.storage) - - def _unpack_indexes(self, space, w_index): - indexes = [space.int_w(w_i) for w_i in space.fixedview(w_index)] - if len(indexes) != len(self.shape): - raise OperationError(space.w_IndexError, space.wrap( - 'Wrong index')) - return indexes - - def getitem(self, w_index): - space = self.space - indexes = self._unpack_indexes(space, w_index) - pos = compute_pos(space, indexes, self.shape) - return space.wrap(self.storage[pos]) - - def setitem(self, w_index, w_value): - space = self.space - indexes = self._unpack_indexes(space, w_index) - pos = compute_pos(space, indexes, self.shape) - self.storage[pos] = space.int_w(w_value) #FIXME: lets get this thing generalized! - return space.w_None #XXX: necessary? - - def len(self): - space = self.space - return space.wrap(self.shape[0]) - -def unpack_dtype(space, w_dtype): - if space.is_w(w_dtype, space.w_int): - return 'i' - else: - raise NotImplementedError + + self.types = { + space.w_int: MultiDimIntArray, + space.w_float: MultiDimFloatArray, + } + +result_factory = None +def mdresult(space, t): + global result_factory + if result_factory is None: + result_factory = ResultFactory(space) + return result_factory.types[t] Modified: pypy/branch/micronumpy/pypy/module/micronumpy/ndarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/ndarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/ndarray.py Mon Dec 28 10:50:53 2009 @@ -8,7 +8,7 @@ from pypy.module.micronumpy.sdarray import sdresult from pypy.module.micronumpy.sdarray import GenericArray -from pypy.module.micronumpy.mdarray import MultiDimArray +from pypy.module.micronumpy.mdarray import mdresult def unpack_shape(space, w_shape): if space.is_true(space.isinstance(w_shape, space.w_int)): @@ -58,20 +58,27 @@ shape_w = unpack_shape(space, values_shape) else: shape_w = unpack_shape(space, w_shape) - if len(shape_w) == 1: - length = shape_w[0] - try: + + try: + if len(shape_w) == 1: + length = shape_w[0] self.array = sdresult(space, w_dtype)(space, length) - except KeyError, e: - raise OperationError(space.w_NotImplementedError, - space.wrap("Haven't implemented generic array yet!")) + else: + self.array = mdresult(space, w_dtype)(space, unpack_shape(space, w_shape)) + except KeyError, e: + raise OperationError(space.w_NotImplementedError, + space.wrap("Haven't implemented generic array yet!")) - if not w_values == space.w_None: #space.is_true() ? - self.array.load_iterable(space, w_values) - else: - self.array = MultiDimArray(space, unpack_shape(space, w_shape), int) #FIXME: multi-dimensional arrays not *really* implemented + if not w_values == space.w_None: + self.array.load_iterable(space, w_values) #TODO: implement loading for multi-dimensional arrays def validate_index(self, space, w_i): + if space.type(w_i) == space.w_int: return + + if space.type(w_i) == space.w_str: + raise OperationError(space.w_NotImplementedError, + space.wrap("Haven't implemented field access yet!")) + try: index_dimensionality = space.int_w(space.len(w_i)) array_dimensionality = len(self.array.shape) Modified: pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py Mon Dec 28 10:50:53 2009 @@ -12,7 +12,7 @@ from pypy.module.micronumpy.dtype import unwrap_float, coerce_float from pypy.module.micronumpy.dtype import unwrap_float32, coerce_float32, float32 -def create_numarray(data_type, unwrap, coerce): +def create_sdarray(data_type, unwrap, coerce): class NumArray(BaseNumArray): def __init__(self, space, length): self.shape = (1,) @@ -90,23 +90,19 @@ return NumArray -IntArray = create_numarray(int, unwrap_int, coerce_int) +IntArray = create_sdarray(int, unwrap_int, coerce_int) NumArray = IntArray # FIXME: compatibility for now -DoubleArray = create_numarray(float, unwrap_float, coerce_float) -FloatArray = create_numarray(float32, unwrap_float32, coerce_float32) +FloatArray = create_sdarray(float, unwrap_float, coerce_float) +Float32Array = create_sdarray(float32, unwrap_float32, coerce_float32) GenericArray = None class ResultFactory(object): def __init__(self, space): self.space = space - from pypy.objspace.std.inttype import int_typedef - int_type = space.gettypeobject(int_typedef) - from pypy.objspace.std.floattype import float_typedef - float_type = space.gettypeobject(float_typedef) self.types = { - int_type: IntArray, - float_type: FloatArray, + space.w_int: IntArray, + space.w_float: FloatArray, } result_factory = None From afa at codespeak.net Mon Dec 28 12:02:31 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 28 Dec 2009 12:02:31 +0100 (CET) Subject: [pypy-svn] r70296 - in pypy/branch/import-builtin/pypy/module/imp: . test Message-ID: <20091228110231.C159D168015@codespeak.net> Author: afa Date: Mon Dec 28 12:02:31 2009 New Revision: 70296 Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Log: Fix the recursive reload case Modified: pypy/branch/import-builtin/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/importing.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/importing.py Mon Dec 28 12:02:31 2009 @@ -420,40 +420,51 @@ space.w_ImportError, space.wrap("reload(): module %s not in sys.modules" % (modulename,))) - namepath = modulename.split('.') - subname = namepath[-1] - parent_name = '.'.join(namepath[:-1]) - parent = None - if parent_name: - w_parent = check_sys_modules(space, space.wrap(parent_name)) - if w_parent is None: - raise OperationError( - space.w_ImportError, - space.wrap("reload(): parent %s not in sys.modules" % ( - parent_name,))) - w_path = space.getattr(w_parent, space.wrap("__path__")) - else: - w_path = None + try: + w_mod = space.reloading_modules[modulename] + # Due to a recursive reload, this module is already being reloaded. + return w_mod + except KeyError: + pass + + space.reloading_modules[modulename] = w_module + try: + namepath = modulename.split('.') + subname = namepath[-1] + parent_name = '.'.join(namepath[:-1]) + parent = None + if parent_name: + w_parent = check_sys_modules(space, space.wrap(parent_name)) + if w_parent is None: + raise OperationError( + space.w_ImportError, + space.wrap("reload(): parent %s not in sys.modules" % ( + parent_name,))) + w_path = space.getattr(w_parent, space.wrap("__path__")) + else: + w_path = None - find_info = find_module( - space, modulename, w_modulename, subname, w_path) + find_info = find_module( + space, modulename, w_modulename, subname, w_path) - if not find_info: - # ImportError - msg = "No module named %s" % modulename - raise OperationError(space.w_ImportError, space.wrap(msg)) + if not find_info: + # ImportError + msg = "No module named %s" % modulename + raise OperationError(space.w_ImportError, space.wrap(msg)) - try: try: - return load_module(space, w_modulename, find_info, reuse=True) - finally: - if find_info.stream: - find_info.stream.close() - except: - # load_module probably removed name from modules because of - # the error. Put back the original module object. - space.sys.setmodule(w_module) - raise + try: + return load_module(space, w_modulename, find_info, reuse=True) + finally: + if find_info.stream: + find_info.stream.close() + except: + # load_module probably removed name from modules because of + # the error. Put back the original module object. + space.sys.setmodule(w_module) + raise + finally: + space.reloading_modules.clear() # __________________________________________________________________ Modified: pypy/branch/import-builtin/pypy/module/imp/test/test_import.py ============================================================================== --- pypy/branch/import-builtin/pypy/module/imp/test/test_import.py (original) +++ pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Mon Dec 28 12:02:31 2009 @@ -35,6 +35,7 @@ b = "imamodule = 1\ninpackage = 0", ambig = "imamodule = 1", test_reload = "def test():\n raise ValueError\n", + infinite_reload = "import infinite_reload; reload(infinite_reload)", ) root.ensure("notapackage", dir=1) # empty, no __init__.py setuppkg("pkg", @@ -440,6 +441,9 @@ reload(sys) assert 'setdefaultencoding' in dir(sys) + def test_reload_infinite(self): + import infinite_reload + def _getlong(data): x = marshal.dumps(data) return x[-4:] From afa at codespeak.net Mon Dec 28 13:21:11 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 28 Dec 2009 13:21:11 +0100 (CET) Subject: [pypy-svn] r70297 - pypy/branch/import-builtin/pypy/interpreter Message-ID: <20091228122111.A920E168015@codespeak.net> Author: afa Date: Mon Dec 28 13:21:10 2009 New Revision: 70297 Modified: pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py Log: This belongs to the previous checkin Modified: pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/import-builtin/pypy/interpreter/baseobjspace.py Mon Dec 28 13:21:10 2009 @@ -240,6 +240,7 @@ self.config = config self.builtin_modules = {} + self.reloading_modules = {} # import extra modules for side-effects import pypy.interpreter.nestedscope # register *_DEREF bytecodes From afa at codespeak.net Mon Dec 28 13:42:25 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 28 Dec 2009 13:42:25 +0100 (CET) Subject: [pypy-svn] r70298 - pypy/branch/import-builtin/lib-python/modified-2.5.2/test Message-ID: <20091228124225.58C6F168015@codespeak.net> Author: afa Date: Mon Dec 28 13:42:24 2009 New Revision: 70298 Modified: pypy/branch/import-builtin/lib-python/modified-2.5.2/test/infinite_reload.py Log: Fix test output, and make this file identical to the original one Modified: pypy/branch/import-builtin/lib-python/modified-2.5.2/test/infinite_reload.py ============================================================================== --- pypy/branch/import-builtin/lib-python/modified-2.5.2/test/infinite_reload.py (original) +++ pypy/branch/import-builtin/lib-python/modified-2.5.2/test/infinite_reload.py Mon Dec 28 13:42:24 2009 @@ -3,8 +3,5 @@ # reload()ing. This module is imported by test_import.py:test_infinite_reload # to make sure this doesn't happen any more. -print 1 import infinite_reload -print 2 reload(infinite_reload) -print 3 From arigo at codespeak.net Mon Dec 28 14:47:16 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 28 Dec 2009 14:47:16 +0100 (CET) Subject: [pypy-svn] r70300 - pypy/trunk/pypy/jit/backend Message-ID: <20091228134716.A1807168015@codespeak.net> Author: arigo Date: Mon Dec 28 14:47:15 2009 New Revision: 70300 Modified: pypy/trunk/pypy/jit/backend/detect_cpu.py Log: Instead of giving segfaults at runtime, fail cleanly when building the JIT on Mac OS/X. Modified: pypy/trunk/pypy/jit/backend/detect_cpu.py ============================================================================== --- pypy/trunk/pypy/jit/backend/detect_cpu.py (original) +++ pypy/trunk/pypy/jit/backend/detect_cpu.py Mon Dec 28 14:47:15 2009 @@ -9,6 +9,8 @@ pass def autodetect_main_model(): + if sys.platform == 'darwin': + raise Exception("JIT not supported on Mac OS/X right now") mach = None try: import platform From arigo at codespeak.net Mon Dec 28 15:28:28 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 28 Dec 2009 15:28:28 +0100 (CET) Subject: [pypy-svn] r70302 - in pypy/trunk/pypy: jit/backend/llsupport rpython/tool translator translator/c translator/c/test Message-ID: <20091228142828.35662168015@codespeak.net> Author: arigo Date: Mon Dec 28 15:28:27 2009 New Revision: 70302 Modified: pypy/trunk/pypy/jit/backend/llsupport/gc.py pypy/trunk/pypy/rpython/tool/rffi_platform.py pypy/trunk/pypy/translator/c/gc.py pypy/trunk/pypy/translator/c/test/test_boehm.py pypy/trunk/pypy/translator/c/test/test_stackless.py pypy/trunk/pypy/translator/driver.py Log: check_boehm() returned either an eci or None, but None was not checked everywhere. Fix it by renaming it to configure_boehm() and letting the CompilationError go through. Modified: pypy/trunk/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/gc.py Mon Dec 28 15:28:27 2009 @@ -41,7 +41,7 @@ GcLLDescription.__init__(self, gcdescr, translator) # grab a pointer to the Boehm 'malloc' function from pypy.rpython.tool import rffi_platform - compilation_info = rffi_platform.check_boehm() + compilation_info = rffi_platform.configure_boehm() # Versions 6.x of libgc needs to use GC_local_malloc(). # Versions 7.x of libgc removed this function; GC_malloc() has Modified: pypy/trunk/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/trunk/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/trunk/pypy/rpython/tool/rffi_platform.py Mon Dec 28 15:28:27 2009 @@ -644,7 +644,7 @@ else: raise CompilationError("Library %s is not installed" % (name,)) -def check_boehm(platform=None): +def configure_boehm(platform=None): if platform is None: from pypy.translator.platform import platform if sys.platform == 'win32': @@ -658,13 +658,10 @@ includes=includes, libraries=['gc'], ) - try: - return configure_external_library( - 'gc', eci, - [dict(prefix='gc-', include_dir='include', library_dir=library_dir)], - symbol='GC_init') - except CompilationError: - return None + return configure_external_library( + 'gc', eci, + [dict(prefix='gc-', include_dir='include', library_dir=library_dir)], + symbol='GC_init') if __name__ == '__main__': doc = """Example: Modified: pypy/trunk/pypy/translator/c/gc.py ============================================================================== --- pypy/trunk/pypy/translator/c/gc.py (original) +++ pypy/trunk/pypy/translator/c/gc.py Mon Dec 28 15:28:27 2009 @@ -215,8 +215,8 @@ def compilation_info(self): eci = BasicGcPolicy.compilation_info(self) - from pypy.rpython.tool.rffi_platform import check_boehm - eci = eci.merge(check_boehm()) + from pypy.rpython.tool.rffi_platform import configure_boehm + eci = eci.merge(configure_boehm()) pre_include_bits = [] if sys.platform == "linux2": Modified: pypy/trunk/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_boehm.py (original) +++ pypy/trunk/pypy/translator/c/test/test_boehm.py Mon Dec 28 15:28:27 2009 @@ -3,12 +3,15 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.memory.test import snippet -from pypy.rpython.tool.rffi_platform import check_boehm from pypy.translator.c.genc import CExtModuleBuilder from pypy import conftest def setup_module(mod): - if not check_boehm(): + from pypy.rpython.tool.rffi_platform import configure_boehm + from pypy.translator.platform import CompilationError + try: + configure_boehm() + except CompilationError: py.test.skip("Boehm GC not present") class AbstractGCTestClass(object): Modified: pypy/trunk/pypy/translator/c/test/test_stackless.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_stackless.py (original) +++ pypy/trunk/pypy/translator/c/test/test_stackless.py Mon Dec 28 15:28:27 2009 @@ -20,8 +20,11 @@ import py py.test.skip("stackless + refcounting doesn't work any more for now") elif cls.gcpolicy == "boehm": - from pypy.rpython.tool.rffi_platform import check_boehm - if not check_boehm(): + from pypy.rpython.tool.rffi_platform import configure_boehm + from pypy.translator.platform import CompilationError + try: + configure_boehm() + except CompilationError: py.test.skip("Boehm GC not present") def wrap_stackless_function(self, fn): Modified: pypy/trunk/pypy/translator/driver.py ============================================================================== --- pypy/trunk/pypy/translator/driver.py (original) +++ pypy/trunk/pypy/translator/driver.py Mon Dec 28 15:28:27 2009 @@ -428,10 +428,13 @@ def possibly_check_for_boehm(self): if self.config.translation.gc == "boehm": - from pypy.rpython.tool.rffi_platform import check_boehm - if not check_boehm(self.translator.platform): + from pypy.rpython.tool.rffi_platform import configure_boehm + from pypy.translator.platform import CompilationError + try: + configure_boehm(self.translator.platform) + except CompilationError, e: i = 'Boehm GC not installed. Try e.g. "translate.py --gc=hybrid"' - raise Exception(i) + raise Exception(str(e) + '\n' + i) def task_database_c(self): translator = self.translator From arigo at codespeak.net Mon Dec 28 15:55:16 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 28 Dec 2009 15:55:16 +0100 (CET) Subject: [pypy-svn] r70303 - pypy/trunk/pypy/jit/backend/test Message-ID: <20091228145516.C2C96168015@codespeak.net> Author: arigo Date: Mon Dec 28 15:55:15 2009 New Revision: 70303 Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py Log: Add a test that seems to crash consistently if the stack is misaligned. Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Mon Dec 28 15:55:15 2009 @@ -1,5 +1,5 @@ -import py, sys, random +import py, sys, random, os from pypy.jit.metainterp.history import (AbstractFailDescr, BasicFailDescr, BoxInt, Box, BoxPtr, @@ -465,6 +465,29 @@ 'float', descr=calldescr) assert abs(res.value - 4.6) < 0.0001 + def test_call_stack_alignment(self): + # test stack alignment issues, notably for Mac OS/X + + def func_ints(*ints): + s = str(ints) + '\n' + os.write(1, s) # don't remove -- crash if the stack is misaligned + return sum(ints) + + for nb_args in range(0, 35): + cpu = self.cpu + TP = lltype.Signed + # + FPTR = self.Ptr(self.FuncType([TP] * nb_args, TP)) + func_ptr = llhelper(FPTR, func_ints) + FUNC = deref(FPTR) + calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + funcbox = self.get_funcbox(cpu, func_ptr) + args = [280-24*i for i in range(nb_args)] + res = self.execute_operation(rop.CALL, + [funcbox] + map(BoxInt, args), + 'int', descr=calldescr) + assert res.value == sum(args) + def test_field_basic(self): t_box, T_box = self.alloc_instance(self.T) fielddescr = self.cpu.fielddescrof(self.S, 'value') From afa at codespeak.net Mon Dec 28 16:02:55 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 28 Dec 2009 16:02:55 +0100 (CET) Subject: [pypy-svn] r70304 - in pypy/trunk: lib-python lib-python/modified-2.5.2/test pypy pypy/config pypy/interpreter pypy/lib pypy/lib/app_test pypy/module/__builtin__ pypy/module/__builtin__/test pypy/module/__pypy__ pypy/module/_demo pypy/module/_demo/test pypy/module/imp pypy/module/imp/test pypy/module/oracle pypy/module/thread pypy/module/zipimport pypy/module/zipimport/test Message-ID: <20091228150255.286C5168011@codespeak.net> Author: afa Date: Mon Dec 28 16:02:53 2009 New Revision: 70304 Added: pypy/trunk/lib-python/modified-2.5.2/test/test_runpy.py - copied unchanged from r70301, pypy/branch/import-builtin/lib-python/modified-2.5.2/test/test_runpy.py pypy/trunk/pypy/module/_demo/test/ (props changed) - copied from r70025, pypy/trunk/pypy/module/_demo/test/ pypy/trunk/pypy/module/_demo/test/test_import.py - copied unchanged from r70025, pypy/trunk/pypy/module/_demo/test/test_import.py pypy/trunk/pypy/module/imp/ (props changed) - copied from r70299, pypy/branch/import-builtin/pypy/module/imp/ pypy/trunk/pypy/module/imp/__init__.py - copied unchanged from r70299, pypy/branch/import-builtin/pypy/module/imp/__init__.py pypy/trunk/pypy/module/imp/importing.py - copied unchanged from r70299, pypy/branch/import-builtin/pypy/module/imp/importing.py pypy/trunk/pypy/module/imp/interp_imp.py - copied unchanged from r70299, pypy/branch/import-builtin/pypy/module/imp/interp_imp.py pypy/trunk/pypy/module/imp/test/ (props changed) - copied from r70299, pypy/branch/import-builtin/pypy/module/imp/test/ pypy/trunk/pypy/module/imp/test/test_app.py - copied unchanged from r70299, pypy/branch/import-builtin/pypy/module/imp/test/test_app.py pypy/trunk/pypy/module/imp/test/test_import.py - copied unchanged from r70299, pypy/branch/import-builtin/pypy/module/imp/test/test_import.py Removed: pypy/trunk/lib-python/modified-2.5.2/test/test___all__.py pypy/trunk/lib-python/modified-2.5.2/test/test_importhooks.py pypy/trunk/pypy/lib/app_test/test_imp_extra.py pypy/trunk/pypy/lib/imp.py pypy/trunk/pypy/module/__builtin__/app_misc.py pypy/trunk/pypy/module/__builtin__/importing.py pypy/trunk/pypy/module/__builtin__/test/test_import.py pypy/trunk/pypy/module/thread/importlock.py Modified: pypy/trunk/lib-python/ (props changed) pypy/trunk/lib-python/modified-2.5.2/test/infinite_reload.py pypy/trunk/lib-python/modified-2.5.2/test/test_import.py pypy/trunk/pypy/ (props changed) pypy/trunk/pypy/config/pypyoption.py pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/interpreter/mixedmodule.py pypy/trunk/pypy/interpreter/module.py pypy/trunk/pypy/lib/app_test/test_runpy.py pypy/trunk/pypy/module/__builtin__/__init__.py pypy/trunk/pypy/module/__pypy__/__init__.py pypy/trunk/pypy/module/_demo/__init__.py pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_error.py pypy/trunk/pypy/module/thread/__init__.py pypy/trunk/pypy/module/zipimport/__init__.py pypy/trunk/pypy/module/zipimport/interp_zipimport.py pypy/trunk/pypy/module/zipimport/test/test_zipimport.py Log: Merge the import-builtin branch: Builtin modules are now translated and compiled, but not necessarily imported by the interpreter. This may speed up startup times! space.getbuiltinmodule() is still the way to import/retrieve these modules. It is now possible to reload builtin modules. Like CPython, the module init function (MixedModule.startup) is called only once, the module dictionary is captured on first import and restored when reloading the module; this is different from .py modules, which execute the code again. The imp module has been rewritten at interplevel, and now shares its implementation with the __import__ function. --Cette ligne, et les suivantes ci-dessous, seront ignor?es-- _M pypy M pypy/module/__pypy__/__init__.py M pypy/module/thread/__init__.py D pypy/module/thread/importlock.py AM + pypy/module/_demo/test A + pypy/module/_demo/test/test_import.py M pypy/module/_demo/__init__.py AM + pypy/module/imp A + pypy/module/imp/importing.py AM + pypy/module/imp/test A + pypy/module/imp/test/test_import.py A + pypy/module/imp/test/test_app.py A + pypy/module/imp/__init__.py A + pypy/module/imp/interp_imp.py M pypy/module/zipimport/test/test_zipimport.py M pypy/module/zipimport/interp_zipimport.py M pypy/module/zipimport/__init__.py D pypy/module/__builtin__/test/test_import.py D pypy/module/__builtin__/app_misc.py M pypy/module/__builtin__/__init__.py D pypy/module/__builtin__/importing.py M pypy/module/oracle/__init__.py M pypy/module/oracle/interp_error.py M pypy/interpreter/mixedmodule.py M pypy/interpreter/baseobjspace.py M pypy/interpreter/module.py M pypy/config/pypyoption.py M pypy/lib/app_test/test_runpy.py D pypy/lib/app_test/test_imp_extra.py D pypy/lib/imp.py _M lib-python M lib-python/modified-2.5.2/test/infinite_reload.py M lib-python/modified-2.5.2/test/test_import.py D lib-python/modified-2.5.2/test/test___all__.py A + lib-python/modified-2.5.2/test/test_runpy.py D lib-python/modified-2.5.2/test/test_importhooks.py Modified: pypy/trunk/lib-python/modified-2.5.2/test/infinite_reload.py ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/test/infinite_reload.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/test/infinite_reload.py Mon Dec 28 16:02:53 2009 @@ -3,8 +3,5 @@ # reload()ing. This module is imported by test_import.py:test_infinite_reload # to make sure this doesn't happen any more. -print 1 import infinite_reload -print 2 reload(infinite_reload) -print 3 Modified: pypy/trunk/lib-python/modified-2.5.2/test/test_import.py ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/test/test_import.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/test/test_import.py Mon Dec 28 16:02:53 2009 @@ -1,4 +1,4 @@ -from test.test_support import TESTFN, TestFailed, check_impl_detail +from test.test_support import TESTFN, TestFailed import os import random @@ -56,6 +56,11 @@ os.unlink(source) try: + #--- the block below is to check that "reload" manages to import + #--- the .pyc file alone. We don't support it in PyPy in the default + #--- configuration. + return + try: reload(mod) except ImportError, err: @@ -238,5 +243,4 @@ finally: sys.path.pop(0) -if check_impl_detail(): - test_infinite_reload() +test_infinite_reload() Modified: pypy/trunk/pypy/config/pypyoption.py ============================================================================== --- pypy/trunk/pypy/config/pypyoption.py (original) +++ pypy/trunk/pypy/config/pypyoption.py Mon Dec 28 16:02:53 2009 @@ -16,7 +16,7 @@ default_modules = essential_modules.copy() default_modules.update(dict.fromkeys( - ["_codecs", "gc", "_weakref", "marshal", "errno", + ["_codecs", "gc", "_weakref", "marshal", "errno", "imp", "math", "_sre", "_pickle_support", "operator", "parser", "symbol", "token", "_ast", "_random", "__pypy__", "_testing"])) Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Mon Dec 28 16:02:53 2009 @@ -239,6 +239,9 @@ config = get_pypy_config(translating=False) self.config = config + self.builtin_modules = {} + self.reloading_modules = {} + # import extra modules for side-effects import pypy.interpreter.nestedscope # register *_DEREF bytecodes @@ -266,16 +269,22 @@ def startup(self): # To be called before using the space - # Initialize all builtin modules + # Initialize already imported builtin modules from pypy.interpreter.module import Module + w_modules = self.sys.get('modules') for w_modname in self.unpackiterable( self.sys.get('builtin_module_names')): + try: + w_mod = self.getitem(w_modules, w_modname) + except OperationError, e: + if e.match(self, self.w_KeyError): + continue + raise modname = self.str_w(w_modname) - mod = self.interpclass_w(self.getbuiltinmodule(modname)) + mod = self.interpclass_w(w_mod) if isinstance(mod, Module): - import time self.timer.start("startup " + modname) - mod.startup(self) + mod.init(self) self.timer.stop("startup " + modname) def finish(self): @@ -339,14 +348,42 @@ w_name = self.wrap(name) w_mod = self.wrap(Module(self, w_name)) - w_modules = self.sys.get('modules') - self.setitem(w_modules, w_name, w_mod) + self.builtin_modules[name] = w_mod return name - def getbuiltinmodule(self, name): + def getbuiltinmodule(self, name, force_init=False): w_name = self.wrap(name) w_modules = self.sys.get('modules') - return self.getitem(w_modules, w_name) + try: + w_mod = self.getitem(w_modules, w_name) + except OperationError, e: + if not e.match(self, self.w_KeyError): + raise + else: + if not force_init: + return w_mod + + # If the module is a builtin but not yet imported, + # retrieve it and initialize it + try: + w_mod = self.builtin_modules[name] + except KeyError: + raise OperationError( + self.w_SystemError, + self.wrap("getbuiltinmodule() called " + "with non-builtin module %s" % name)) + else: + # Add the module to sys.modules + self.setitem(w_modules, w_name, w_mod) + + # And initialize it + from pypy.interpreter.module import Module + mod = self.interpclass_w(w_mod) + if isinstance(mod, Module): + self.timer.start("startup " + name) + mod.init(self) + self.timer.stop("startup " + name) + return w_mod def get_builtinmodule_to_install(self): """NOT_RPYTHON""" @@ -390,26 +427,27 @@ "NOT_RPYTHON: only for initializing the space." from pypy.module.exceptions import Module - w_name_exceptions = self.wrap('exceptions') - self.exceptions_module = Module(self, w_name_exceptions) + w_name = self.wrap('exceptions') + self.exceptions_module = Module(self, w_name) + self.builtin_modules['exceptions'] = self.wrap(self.exceptions_module) from pypy.module.sys import Module w_name = self.wrap('sys') self.sys = Module(self, w_name) - w_modules = self.sys.get('modules') - self.setitem(w_modules, w_name, self.wrap(self.sys)) + self.builtin_modules['sys'] = self.wrap(self.sys) - self.setitem(w_modules, w_name_exceptions, - self.wrap(self.exceptions_module)) + from pypy.module.imp import Module + w_name = self.wrap('imp') + self.builtin_modules['imp'] = self.wrap(Module(self, w_name)) from pypy.module.__builtin__ import Module w_name = self.wrap('__builtin__') self.builtin = Module(self, w_name) w_builtin = self.wrap(self.builtin) - self.setitem(w_modules, w_name, w_builtin) + self.builtin_modules['__builtin__'] = self.wrap(w_builtin) self.setitem(self.builtin.w_dict, self.wrap('__builtins__'), w_builtin) - bootstrap_modules = ['sys', '__builtin__', 'exceptions'] + bootstrap_modules = ['sys', 'imp', '__builtin__', 'exceptions'] installed_builtin_modules = bootstrap_modules[:] self.export_builtin_exceptions() @@ -480,12 +518,11 @@ def setup_builtin_modules(self): "NOT_RPYTHON: only for initializing the space." - from pypy.interpreter.module import Module - for w_modname in self.unpackiterable(self.sys.get('builtin_module_names')): - modname = self.unwrap(w_modname) - mod = self.getbuiltinmodule(modname) - if isinstance(mod, Module): - mod.setup_after_space_initialization() + self.getbuiltinmodule('sys') + self.getbuiltinmodule('imp') + self.getbuiltinmodule('__builtin__') + for mod in self.builtin_modules.values(): + mod.setup_after_space_initialization() def initialize(self): """NOT_RPYTHON: Abstract method that should put some minimal Modified: pypy/trunk/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/trunk/pypy/interpreter/mixedmodule.py (original) +++ pypy/trunk/pypy/interpreter/mixedmodule.py Mon Dec 28 16:02:53 2009 @@ -13,7 +13,8 @@ applevel_name = None expose__file__attribute = True - + w_initialdict = None + def __init__(self, space, w_name): """ NOT_RPYTHON """ Module.__init__(self, space, w_name) @@ -21,6 +22,13 @@ self.__class__.buildloaders() self.loaders = self.loaders.copy() # copy from the class to the inst + def init(self, space): + """This is called each time the module is imported or reloaded + """ + if self.w_initialdict is not None: + space.call_method(self.w_dict, 'update', self.w_initialdict) + Module.init(self, space) + def get_applevel_name(cls): """ NOT_RPYTHON """ if cls.applevel_name is not None: @@ -82,7 +90,8 @@ for name in self.loaders: w_value = self.get(name) space.setitem(self.w_dict, space.new_interned_str(name), w_value) - self.lazy = False + self.lazy = False + self.w_initialdict = space.call_method(self.w_dict, 'items') return self.w_dict def _freeze_(self): Modified: pypy/trunk/pypy/interpreter/module.py ============================================================================== --- pypy/trunk/pypy/interpreter/module.py (original) +++ pypy/trunk/pypy/interpreter/module.py Mon Dec 28 16:02:53 2009 @@ -15,12 +15,20 @@ self.w_dict = w_dict self.w_name = w_name if w_name is not None: - space.setitem(w_dict, space.new_interned_str('__name__'), w_name) + space.setitem(w_dict, space.new_interned_str('__name__'), w_name) + self.startup_called = False def setup_after_space_initialization(self): """NOT_RPYTHON: to allow built-in modules to do some more setup after the space is fully initialized.""" + def init(self, space): + """This is called each time the module is imported or reloaded + """ + if not self.startup_called: + self.startup_called = True + self.startup(space) + def startup(self, space): """This is called at runtime before the space gets uses to allow the module to do initialization at runtime. Modified: pypy/trunk/pypy/lib/app_test/test_runpy.py ============================================================================== --- pypy/trunk/pypy/lib/app_test/test_runpy.py (original) +++ pypy/trunk/pypy/lib/app_test/test_runpy.py Mon Dec 28 16:02:53 2009 @@ -137,6 +137,12 @@ d1 = run_module(mod_name) # Read from source __import__(mod_name) os.remove(mod_fname) + + #--- the block below is to check that "imp.find_module" + #--- manages to import the .pyc file alone. We don't + #--- support it in PyPy in the default configuration. + return + if verbose: print "Running from compiled:", mod_name d2 = run_module(mod_name) # Read from bytecode finally: Modified: pypy/trunk/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/__init__.py (original) +++ pypy/trunk/pypy/module/__builtin__/__init__.py Mon Dec 28 16:02:53 2009 @@ -1,6 +1,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter import module from pypy.interpreter.mixedmodule import MixedModule +import pypy.module.imp.importing # put builtins here that should be optimized somehow @@ -35,9 +36,6 @@ 'vars' : 'app_inspect.vars', 'dir' : 'app_inspect.dir', - '_find_module' : 'app_misc.find_module', - 'reload' : 'app_misc.reload', - '__filestub' : 'app_file_stub.file', } @@ -88,7 +86,8 @@ 'compile' : 'compiling.compile', 'eval' : 'compiling.eval', - '__import__' : 'importing.importhook', + '__import__' : 'pypy.module.imp.importing.importhook', + 'reload' : 'pypy.module.imp.importing.reload', 'range' : 'functional.range_int', 'xrange' : 'functional.W_XRange', @@ -152,16 +151,3 @@ space.exception_is_valid_obj_as_class_w = ab.exception_is_valid_obj_as_class_w.__get__(space) space.exception_getclass = ab.exception_getclass.__get__(space) space.exception_issubclass_w = ab.exception_issubclass_w.__get__(space) - - def startup(self, space): - # install zipimport hook if --withmod-zipimport is used - if space.config.objspace.usemodules.zipimport: - w_import = space.builtin.get('__import__') - w_zipimport = space.call(w_import, space.newlist( - [space.wrap('zipimport')])) - w_sys = space.getbuiltinmodule('sys') - w_path_hooks = space.getattr(w_sys, space.wrap('path_hooks')) - w_append = space.getattr(w_path_hooks, space.wrap('append')) - w_zipimporter = space.getattr(w_zipimport, - space.wrap('zipimporter')) - space.call(w_append, space.newlist([w_zipimporter])) Modified: pypy/trunk/pypy/module/__pypy__/__init__.py ============================================================================== --- pypy/trunk/pypy/module/__pypy__/__init__.py (original) +++ pypy/trunk/pypy/module/__pypy__/__init__.py Mon Dec 28 16:02:53 2009 @@ -1,7 +1,7 @@ # Package initialisation from pypy.interpreter.mixedmodule import MixedModule -from pypy.module.__builtin__.importing import get_pyc_magic +from pypy.module.imp.importing import get_pyc_magic class Module(MixedModule): appleveldefs = { Modified: pypy/trunk/pypy/module/_demo/__init__.py ============================================================================== --- pypy/trunk/pypy/module/_demo/__init__.py (original) +++ pypy/trunk/pypy/module/_demo/__init__.py Mon Dec 28 16:02:53 2009 @@ -12,3 +12,13 @@ appleveldefs = { 'DemoError' : 'app_demo.DemoError', } + + # Used in tests + demo_events = [] + def setup_after_space_initialization(self): + Module.demo_events.append('setup') + def startup(self, space): + Module.demo_events.append('startup') + def shutdown(self, space): + Module.demo_events.append('shutdown') + Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Mon Dec 28 16:02:53 2009 @@ -39,6 +39,7 @@ def startup(self, space): from pypy.module.oracle.interp_error import get state = get(space) + state.startup(space) (state.w_DecimalType, state.w_DateTimeType, state.w_DateType, state.w_TimedeltaType, ) = space.fixedview(space.appexec([], """(): Modified: pypy/trunk/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_error.py (original) +++ pypy/trunk/pypy/module/oracle/interp_error.py Mon Dec 28 16:02:53 2009 @@ -4,32 +4,47 @@ from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError + from pypy.module.oracle import roci, config +from pypy.rlib.unroll import unrolling_iterable + +exported_names = unrolling_iterable(""" + DatabaseError OperationalError InterfaceError ProgrammingError + NotSupportedError IntegrityError InternalError DataError + Variable Connection""".split()) class State: # XXX move to another file + def __init__(self, space): - w_module = space.getbuiltinmodule('cx_Oracle') - def get(name): - return space.getattr(w_module, space.wrap(name)) + "NOT_RPYTHON" + self.variableTypeByPythonType = {} + self.w_DecimalType = None + self.w_DateTimeType = None + self.w_DateType = None + self.w_TimedeltaType = None - self.w_DatabaseError = get('DatabaseError') - self.w_OperationalError = get('OperationalError') - self.w_InterfaceError = get('InterfaceError') - self.w_ProgrammingError = get('ProgrammingError') - self.w_NotSupportedError = get('NotSupportedError') - self.w_IntegrityError = get('IntegrityError') - self.w_InternalError = get('InternalError') - self.w_DataError = get('DataError') - self.w_Variable = get('Variable') - self.w_Connection = get('Connection') + for name in exported_names: + setattr(self, 'w_' + name, None) + + def startup(self, space): + w_module = space.getbuiltinmodule('cx_Oracle') + for name in exported_names: + setattr(self, 'w_' + name, space.getattr(w_module, space.wrap(name))) from pypy.module.oracle.interp_variable import all_variable_types - self.variableTypeByPythonType = {} for varType in all_variable_types: w_type = space.gettypeobject(varType.typedef) self.variableTypeByPythonType[w_type] = varType + (self.w_DecimalType, + self.w_DateTimeType, self.w_DateType, self.w_TimedeltaType, + ) = space.fixedview(space.appexec([], """(): + import decimal, datetime + return (decimal.Decimal, + datetime.datetime, datetime.date, datetime.timedelta) + """)) + def get(space): return space.fromcache(State) Modified: pypy/trunk/pypy/module/thread/__init__.py ============================================================================== --- pypy/trunk/pypy/module/thread/__init__.py (original) +++ pypy/trunk/pypy/module/thread/__init__.py Mon Dec 28 16:02:53 2009 @@ -18,11 +18,6 @@ 'allocate': 'os_lock.allocate_lock', # obsolete synonym 'LockType': 'os_lock.getlocktype(space)', '_local': 'os_local.getlocaltype(space)', - - # custom interface for the 'imp' module - '_importlock_held': 'importlock.held', - '_importlock_acquire': 'importlock.acquire', - '_importlock_release': 'importlock.release', } def __init__(self, space, *args): Modified: pypy/trunk/pypy/module/zipimport/__init__.py ============================================================================== --- pypy/trunk/pypy/module/zipimport/__init__.py (original) +++ pypy/trunk/pypy/module/zipimport/__init__.py Mon Dec 28 16:02:53 2009 @@ -14,4 +14,12 @@ appleveldefs = { 'ZipImportError' : 'app_zipimport.ZipImportError', } - + + def setup_after_space_initialization(self): + """NOT_RPYTHON""" + space = self.space + # install zipimport hook + w_path_hooks = space.sys.get('path_hooks') + from pypy.module.zipimport.interp_zipimport import W_ZipImporter + w_zipimporter = space.gettypefor(W_ZipImporter) + space.call_method(w_path_hooks, 'append', w_zipimporter) Modified: pypy/trunk/pypy/module/zipimport/interp_zipimport.py ============================================================================== --- pypy/trunk/pypy/module/zipimport/interp_zipimport.py (original) +++ pypy/trunk/pypy/module/zipimport/interp_zipimport.py Mon Dec 28 16:02:53 2009 @@ -5,7 +5,7 @@ from pypy.interpreter.gateway import interp2app from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.module import Module -from pypy.module.__builtin__ import importing +from pypy.module.imp import importing from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rzipfile import RZipFile, BadZipfile import os @@ -149,9 +149,9 @@ real_name = self.name + os.path.sep + self.corr_zname(filename) space.setattr(w_mod, w('__loader__'), space.wrap(self)) importing._prepare_module(space, w_mod, real_name, pkgpath) - result = importing.load_source_module(space, w(modname), w_mod, - filename, buf, write_pyc=False) - return result + code_w = importing.parse_source_module(space, filename, buf) + importing.exec_code_module(space, w_mod, code_w) + return w_mod def _parse_mtime(self, space, filename): w = space.wrap Modified: pypy/trunk/pypy/module/zipimport/test/test_zipimport.py ============================================================================== --- pypy/trunk/pypy/module/zipimport/test/test_zipimport.py (original) +++ pypy/trunk/pypy/module/zipimport/test/test_zipimport.py Mon Dec 28 16:02:53 2009 @@ -4,7 +4,7 @@ import py import time import struct -from pypy.module.__builtin__.importing import get_pyc_magic, _w_long +from pypy.module.imp.importing import get_pyc_magic, _w_long from StringIO import StringIO from pypy.tool.udir import udir @@ -255,6 +255,11 @@ l = [i for i in zipimport._zip_directory_cache] assert len(l) + def test_path_hooks(self): + import sys + import zipimport + assert sys.path_hooks.count(zipimport.zipimporter) == 1 + class AppTestZipimportDeflated(AppTestZipimport): compression = ZIP_DEFLATED From pedronis at codespeak.net Mon Dec 28 17:08:07 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 28 Dec 2009 17:08:07 +0100 (CET) Subject: [pypy-svn] r70305 - in pypy/trunk/pypy: . jit/backend/llvm/test jit/backend/x86/test jit/metainterp module/exceptions objspace/std/test tool translator/c/test translator/platform translator/platform/test Message-ID: <20091228160807.BCEA6168015@codespeak.net> Author: pedronis Date: Mon Dec 28 17:08:06 2009 New Revision: 70305 Modified: pypy/trunk/pypy/ (props changed) pypy/trunk/pypy/jit/backend/llvm/test/conftest.py (props changed) pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py (props changed) pypy/trunk/pypy/jit/metainterp/logger.py (props changed) pypy/trunk/pypy/module/exceptions/ (props changed) pypy/trunk/pypy/objspace/std/test/test_setobject.py (props changed) pypy/trunk/pypy/tool/gcc_cache.py pypy/trunk/pypy/translator/c/test/test_refcount.py (props changed) pypy/trunk/pypy/translator/platform/__init__.py pypy/trunk/pypy/translator/platform/darwin.py pypy/trunk/pypy/translator/platform/posix.py pypy/trunk/pypy/translator/platform/test/test_darwin.py pypy/trunk/pypy/translator/platform/test/test_platform.py Log: - partial merge from Leonardo's force-arch-darwin branch, force 32 vs 64 bits builds based on the hosting python - make sure the key used for the gcc cache also uses information from the platform (name, 64/32 bits, relevant environment values) Modified: pypy/trunk/pypy/tool/gcc_cache.py ============================================================================== --- pypy/trunk/pypy/tool/gcc_cache.py (original) +++ pypy/trunk/pypy/tool/gcc_cache.py Mon Dec 28 17:08:06 2009 @@ -10,7 +10,7 @@ def cache_file_path(c_files, eci, cachename): cache_dir = cache_dir_root.join(cachename).ensure(dir=1) filecontents = [c_file.read() for c_file in c_files] - key = repr((filecontents, eci)) + key = repr((filecontents, eci, platform.key())) hash = md5(key).hexdigest() return cache_dir.join(hash) Modified: pypy/trunk/pypy/translator/platform/__init__.py ============================================================================== --- pypy/trunk/pypy/translator/platform/__init__.py (original) +++ pypy/trunk/pypy/translator/platform/__init__.py Mon Dec 28 17:08:06 2009 @@ -52,6 +52,8 @@ name = "abstract platform" c_environ = None + relevant_environ = [] + so_prefixes = [''] def __init__(self, cc): @@ -98,6 +100,12 @@ return (self.__class__ is other.__class__ and self.__dict__ == other.__dict__) + def key(self): + bits = [self.__class__.__name__, 'cc=%s' % self.cc] + for varname in self.relevant_environ: + bits.append('%s=%s' % (varname, os.environ.get(varname))) + return ' '.join(bits) + # some helpers which seem to be cross-platform enough def _execute_c_compiler(self, cc, args, outname): @@ -171,8 +179,15 @@ else: host_factory = Linux64 elif sys.platform == 'darwin': - from pypy.translator.platform.darwin import Darwin - host_factory = Darwin + from pypy.translator.platform.darwin import Darwin_i386, Darwin_x86_64 + import platform + if platform.machine() == 'i386': + if sys.maxint <= 2147483647: + host_factory = Darwin_i386 + else: + host_factory = Darwin_x86_64 + else: + host_factory = Darwin elif sys.platform == 'freebsd7': from pypy.translator.platform.freebsd7 import Freebsd7, Freebsd7_64 import platform Modified: pypy/trunk/pypy/translator/platform/darwin.py ============================================================================== --- pypy/trunk/pypy/translator/platform/darwin.py (original) +++ pypy/trunk/pypy/translator/platform/darwin.py Mon Dec 28 17:08:06 2009 @@ -4,7 +4,7 @@ class Darwin(posix.BasePosix): name = "darwin" - + link_flags = ['-mmacosx-version-min=10.4'] cflags = ['-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4'] standalone_only = ['-mdynamic-no-pic'] @@ -44,3 +44,13 @@ include_dirs = self._includedirs(eci.include_dirs) return (args + frameworks + include_dirs) +class Darwin_i386(Darwin): + name = "darwin_i386" + link_flags = ['-arch', 'i386', '-mmacosx-version-min=10.4'] + cflags = ['-arch', 'i386', '-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4'] + +class Darwin_x86_64(Darwin): + name = "darwin_x86_64" + link_flags = ['-arch', 'x86_64', '-mmacosx-version-min=10.4'] + cflags = ['-arch', 'x86_64', '-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4'] + Modified: pypy/trunk/pypy/translator/platform/posix.py ============================================================================== --- pypy/trunk/pypy/translator/platform/posix.py (original) +++ pypy/trunk/pypy/translator/platform/posix.py Mon Dec 28 17:08:06 2009 @@ -10,6 +10,8 @@ exe_ext = '' make_cmd = 'make' + relevant_environ=['CPATH', 'LIBRARY_PATH', 'C_INCLUDE_PATH'] + def __init__(self, cc=None): if cc is None: cc = 'gcc' Modified: pypy/trunk/pypy/translator/platform/test/test_darwin.py ============================================================================== --- pypy/trunk/pypy/translator/platform/test/test_darwin.py (original) +++ pypy/trunk/pypy/translator/platform/test/test_darwin.py Mon Dec 28 17:08:06 2009 @@ -2,17 +2,25 @@ """ File containing darwin platform tests """ -import py, sys +import py, sys, platform if sys.platform != 'darwin': py.test.skip("Darwin only") from pypy.tool.udir import udir -from pypy.translator.platform.darwin import Darwin +from pypy.translator.platform.darwin import Darwin_i386, Darwin_x86_64 from pypy.translator.platform.test.test_platform import TestPlatform as BasicTest from pypy.translator.tool.cbuild import ExternalCompilationInfo +if platform.machine() == 'i386': + if sys.maxint <= 2147483647: + host_factory = Darwin_i386 + else: + host_factory = Darwin_x86_64 +else: + host_factory = Darwin + class TestDarwin(BasicTest): - platform = Darwin() + platform = host_factory() def test_frameworks(self): objcfile = udir.join('test_simple.m') @@ -39,3 +47,81 @@ res = self.platform.execute(executable) self.check_res(res) + def test_64_32_results(self): + if platform.machine() != 'i386': + py.test.skip("i386 only") + plat32 = Darwin_i386() + plat64 = Darwin_x86_64() + cfile = udir.join('test_int_size.c') + cfile.write(r''' + #include + #include + + int main() { + printf("%d\n", INT_MAX < LONG_MAX); + return 0; + } + ''') + eci = ExternalCompilationInfo() + executable = plat32.compile([cfile], eci) + res = plat32.execute(executable) + self.check_res(res, '0\n') + if host_factory == Darwin_x86_64: + executable = plat64.compile([cfile], eci) + res = plat64.execute(executable) + self.check_res(res, '1\n') + + def test_longsize(self): + if platform.machine() != 'i386': + py.test.skip("i386 only") + cfile = udir.join('test_int_size.c') + cfile.write(r''' + #include + #include + + int main() { + printf("%ld\n", LONG_MAX); + return 0; + } + ''') + eci = ExternalCompilationInfo() + executable = self.platform.compile([cfile], eci) + res = self.platform.execute(executable) + self.check_res(res, str(sys.maxint) + '\n') + + def test_32bit_makefile(self): + if platform.machine() != 'i386': + py.test.skip("i386 only") + plat32 = Darwin_i386() + plat64 = Darwin_x86_64() + eci = ExternalCompilationInfo() + cfile_content =r''' + #include + #include + + int main() { + printf("%d\n", INT_MAX < LONG_MAX); + return 0; + } + ''' + + tmpdir = udir.join('32_makefile' + self.__class__.__name__).ensure(dir=1) + cfile = tmpdir.join('test_int_size.c') + cfile.write(cfile_content) + mk = plat32.gen_makefile([cfile], ExternalCompilationInfo(), + path=tmpdir) + mk.write() + plat32.execute_makefile(mk) + res = plat32.execute(tmpdir.join('test_int_size')) + self.check_res(res, '0\n') + if host_factory == Darwin_x86_64: + tmpdir = udir.join('64_makefile' + self.__class__.__name__).ensure(dir=1) + cfile = tmpdir.join('test_int_size.c') + cfile.write(cfile_content) + mk = plat64.gen_makefile([cfile], ExternalCompilationInfo(), + path=tmpdir) + mk.write() + plat64.execute_makefile(mk) + res = plat64.execute(tmpdir.join('test_int_size')) + self.check_res(res, '1\n') + Modified: pypy/trunk/pypy/translator/platform/test/test_platform.py ============================================================================== --- pypy/trunk/pypy/translator/platform/test/test_platform.py (original) +++ pypy/trunk/pypy/translator/platform/test/test_platform.py Mon Dec 28 17:08:06 2009 @@ -115,6 +115,16 @@ finally: del os.environ['_SOME_VARIABLE_2'] + def test_key(self): + class XPlatform(Platform): + relevant_environ = ['CPATH'] + + def __init__(self): + self.cc = 'xcc' + x = XPlatform() + res = x.key() + assert res.startswith('XPlatform cc=xcc CPATH=') + def test_equality(): class X(Platform): def __init__(self): From arigo at codespeak.net Mon Dec 28 17:34:51 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 28 Dec 2009 17:34:51 +0100 (CET) Subject: [pypy-svn] r70306 - in pypy/branch/jit-delayed-write/pypy/jit/metainterp: . test Message-ID: <20091228163451.F0822168020@codespeak.net> Author: arigo Date: Mon Dec 28 17:34:49 2009 New Revision: 70306 Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py Log: hackish: reverse the order of the last two operations if it makes sense to avoid a situation like "int_eq/setfield_gc/guard_true" which the backend (at least the x86 backend) is not good at handling. Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py Mon Dec 28 17:34:49 2009 @@ -978,7 +978,7 @@ self.force_all_lazy_setfields() self.clean_caches() - def force_lazy_setfield(self, descr): + def force_lazy_setfield(self, descr, before_guard=False): try: op = self.lazy_setfields[descr] except KeyError: @@ -988,6 +988,18 @@ if isinstance(fieldvalue, AbstractVirtualValue): fieldvalue.backstore_field = None self.optimizer._emit_operation(op) + # + # hackish: reverse the order of the last two operations if it makes + # sense to avoid the situation "int_eq/setfield_gc/guard_true" + newoperations = self.optimizer.newoperations + if before_guard and len(newoperations) >= 2: + lastop = newoperations[-1] + prevop = newoperations[-2] + if prevop.is_always_pure() and prevop.result not in lastop.args: + del newoperations[-1] + del newoperations[-1] + newoperations.append(lastop) + newoperations.append(prevop) def force_all_lazy_setfields(self): if len(self.lazy_setfields_descrs) > 0: @@ -1014,7 +1026,7 @@ if fieldvalue.backstore_field is descr: # this is the case that can be handled by resume data continue - self.force_lazy_setfield(descr) + self.force_lazy_setfield(descr, before_guard=True) def force_lazy_setfield_if_necessary(self, op, value, write=False): try: Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py Mon Dec 28 17:34:49 2009 @@ -1429,6 +1429,20 @@ """ self.optimize_loop(ops, 'Not, Not, Not', expected) + def test_duplicate_setfield_residual_guard_3(self): + # test that the setfield_gc does not end up between int_eq and + # the following guard_true + ops = """ + [p1, i1, i2, i3] + setfield_gc(p1, i1, descr=valuedescr) + i5 = int_eq(i3, 5) + guard_true(i5) [] + i4 = int_neg(i2) + setfield_gc(p1, i2, descr=valuedescr) + jump(p1, i1, i2, i4) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', ops) + def test_duplicate_setfield_aliasing(self): # a case where aliasing issues (and not enough cleverness) mean # that we fail to remove any setfield_gc @@ -2088,20 +2102,20 @@ p2 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p2, i2, descr=valuedescr) setfield_gc(p1, p2, descr=nextdescr) - guard_true(i3, descr=fdescr) [p1] + guard_true(i3, descr=fdescr) [] i4 = int_neg(i2) setfield_gc(p1, NULL, descr=nextdescr) jump(p1, i2, i4) """ expected = """ [p1, i2, i3] - guard_true(i3, descr=fdescr) [p1, i2] + guard_true(i3, descr=fdescr) [i2] i4 = int_neg(i2) setfield_gc(p1, NULL, descr=nextdescr) jump(p1, i2, i4) """ self.optimize_loop(ops, 'Not, Not, Not', expected) - self.check_expanded_fail_descr('''p1 + self.check_expanded_fail_descr(''' where p2 is a node_vtable, valuedescr=i2 --> p1.nextdescr ''') From pedronis at codespeak.net Mon Dec 28 17:43:26 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 28 Dec 2009 17:43:26 +0100 (CET) Subject: [pypy-svn] r70307 - in pypy/trunk/pypy/jit/backend: . x86 Message-ID: <20091228164326.83319168020@codespeak.net> Author: pedronis Date: Mon Dec 28 17:43:25 2009 New Revision: 70307 Modified: pypy/trunk/pypy/jit/backend/detect_cpu.py pypy/trunk/pypy/jit/backend/x86/assembler.py Log: fix call stack-aligment issues on Mac OS X Modified: pypy/trunk/pypy/jit/backend/detect_cpu.py ============================================================================== --- pypy/trunk/pypy/jit/backend/detect_cpu.py (original) +++ pypy/trunk/pypy/jit/backend/detect_cpu.py Mon Dec 28 17:43:25 2009 @@ -9,8 +9,6 @@ pass def autodetect_main_model(): - if sys.platform == 'darwin': - raise Exception("JIT not supported on Mac OS/X right now") mach = None try: import platform Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Mon Dec 28 17:43:25 2009 @@ -28,7 +28,6 @@ else: CALL_ALIGN = 1 - def align_stack_words(words): return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) @@ -226,13 +225,14 @@ def _patch_stackadjust(self, adr_lea, reserved_depth): # patch stack adjustment LEA - # possibly align, e.g. for Mac OS X mc = codebuf.InMemoryCodeBuilder(adr_lea, adr_lea + 4) # Compute the correct offset for the instruction LEA ESP, [EBP-4*words]. # Given that [EBP] is where we saved EBP, i.e. in the last word # of our fixed frame, then the 'words' value is: words = (FRAME_FIXED_SIZE - 1) + reserved_depth - mc.write(packimm32(-WORD * words)) + # align, e.g. for Mac OS X + aligned_words = align_stack_words(words+2)-2 # 2 = EIP+EBP + mc.write(packimm32(-WORD * aligned_words)) mc.done() def _assemble_bootstrap_code(self, inputargs, arglocs): @@ -404,14 +404,6 @@ return self.implement_guard(addr, getattr(self.mc, name)) return genop_cmp_guard -## XXX redo me -## def align_stack_for_call(self, nargs): -## # xxx do something when we don't use push anymore for calls -## extra_on_stack = align_stack_words(nargs) -## for i in range(extra_on_stack-nargs): -## self.mc.PUSH(imm(0)) --- or just use a single SUB(esp, imm) -## return extra_on_stack - def _emit_call(self, x, arglocs, start=0, tmp=eax): p = 0 n = len(arglocs) From afa at codespeak.net Mon Dec 28 17:48:48 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 28 Dec 2009 17:48:48 +0100 (CET) Subject: [pypy-svn] r70308 - pypy/trunk/pypy/doc/config Message-ID: <20091228164848.844F0168020@codespeak.net> Author: afa Date: Mon Dec 28 17:48:48 2009 New Revision: 70308 Added: pypy/trunk/pypy/doc/config/objspace.usemodules.imp.txt Log: This file was somehow lost by the merge Added: pypy/trunk/pypy/doc/config/objspace.usemodules.imp.txt ============================================================================== --- (empty file) +++ pypy/trunk/pypy/doc/config/objspace.usemodules.imp.txt Mon Dec 28 17:48:48 2009 @@ -0,0 +1,2 @@ +Use the 'imp' module. +This module is included by default. From afa at codespeak.net Mon Dec 28 18:01:36 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 28 Dec 2009 18:01:36 +0100 (CET) Subject: [pypy-svn] r70309 - pypy/trunk/pypy/interpreter Message-ID: <20091228170136.D4810168015@codespeak.net> Author: afa Date: Mon Dec 28 18:01:36 2009 New Revision: 70309 Modified: pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/interpreter/mixedmodule.py pypy/trunk/pypy/interpreter/module.py Log: "svn merge" lost some changes in these files, just copy them from the branch. Resetting startup_called to False is specially important, otherwise after reload(sys) we retrieve the state captured during translation... Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Mon Dec 28 18:01:36 2009 @@ -292,11 +292,9 @@ if w_exitfunc is not None: self.call_function(w_exitfunc) from pypy.interpreter.module import Module - for w_modname in self.unpackiterable( - self.sys.get('builtin_module_names')): - modname = self.str_w(w_modname) - mod = self.interpclass_w(self.getbuiltinmodule(modname)) - if isinstance(mod, Module): + for w_mod in self.builtin_modules.values(): + mod = self.interpclass_w(w_mod) + if isinstance(mod, Module) and mod.startup_called: mod.shutdown(self) if self.config.objspace.std.withdictmeasurement: from pypy.objspace.std.dictmultiobject import report Modified: pypy/trunk/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/trunk/pypy/interpreter/mixedmodule.py (original) +++ pypy/trunk/pypy/interpreter/mixedmodule.py Mon Dec 28 18:01:36 2009 @@ -96,6 +96,7 @@ def _freeze_(self): self.getdict() + self.startup_called = False # hint for the annotator: Modules can hold state, so they are # not constant return False Modified: pypy/trunk/pypy/interpreter/module.py ============================================================================== --- pypy/trunk/pypy/interpreter/module.py (original) +++ pypy/trunk/pypy/interpreter/module.py Mon Dec 28 18:01:36 2009 @@ -30,15 +30,15 @@ self.startup(space) def startup(self, space): - """This is called at runtime before the space gets uses to allow - the module to do initialization at runtime. + """This is called at runtime on import to allow the module to + do initialization when it is imported for the first time. """ def shutdown(self, space): """This is called when the space is shut down, just after - sys.exitfunc(). + sys.exitfunc(), if the module has been imported. """ - + def getdict(self): return self.w_dict From benjamin at codespeak.net Mon Dec 28 21:18:15 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 28 Dec 2009 21:18:15 +0100 (CET) Subject: [pypy-svn] r70312 - in pypy/trunk/pypy/interpreter: . test Message-ID: <20091228201815.CA5ED168022@codespeak.net> Author: benjamin Date: Mon Dec 28 21:18:13 2009 New Revision: 70312 Modified: pypy/trunk/pypy/interpreter/gateway.py pypy/trunk/pypy/interpreter/test/test_gateway.py Log: add an unwrap_spec decorator for attaching unwrap_spec to functions Modified: pypy/trunk/pypy/interpreter/gateway.py ============================================================================== --- pypy/trunk/pypy/interpreter/gateway.py (original) +++ pypy/trunk/pypy/interpreter/gateway.py Mon Dec 28 21:18:13 2009 @@ -386,6 +386,15 @@ else: return typ.__name__ + '_w' + +def unwrap_spec(*spec): + """A decorator which attaches the unwrap_spec attribute.""" + def decorator(func): + func.unwrap_spec = spec + return func + return decorator + + class BuiltinCode(eval.Code): "The code object implementing a built-in (interpreter-level) hook." _immutable_ = True Modified: pypy/trunk/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_gateway.py (original) +++ pypy/trunk/pypy/interpreter/test/test_gateway.py Mon Dec 28 21:18:13 2009 @@ -520,6 +520,16 @@ w_res = space.call_obj_args(w_g, w_self, args3) assert space.is_true(space.eq(w_res, space.wrap(('g', 'self', 3)))) + def test_unwrap_spec_decorator(self): + space = self.space + @gateway.unwrap_spec(gateway.ObjSpace, gateway.W_Root, int) + def g(space, w_thing, i): + return space.newtuple([w_thing, space.wrap(i)]) + w_g = space.wrap(gateway.interp2app_temp(g)) + args = argument.Arguments(space, [space.wrap(-1), space.wrap(0)]) + w_res = space.call_args(w_g, args) + assert space.eq_w(w_res, space.wrap((-1, 0))) + class TestPassThroughArguments: From arigo at codespeak.net Mon Dec 28 21:27:06 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 28 Dec 2009 21:27:06 +0100 (CET) Subject: [pypy-svn] r70313 - in pypy/branch/jit-delayed-write/pypy/jit/metainterp: . test Message-ID: <20091228202706.CB382168022@codespeak.net> Author: arigo Date: Mon Dec 28 21:27:04 2009 New Revision: 70313 Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py pypy/branch/jit-delayed-write/pypy/jit/metainterp/resume.py pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_resume.py Log: Fix and mostly finish it. The previous attempt was a dead-end; instead just pass a list of pending setfields to the ResumeDataVirtualAdder which gets stored to 'rd_pendingfields'. To do later maybe: make the rd_pendingfields list even more compact. Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py Mon Dec 28 21:27:04 2009 @@ -66,8 +66,7 @@ def get_backstore(self): return (None, None) - def make_virtual_info(self, modifier, fieldnums, - backstore_num, backstore_descr): + def make_virtual_info(self, modifier, fieldnums): raise NotImplementedError # should not be called on this level def is_constant(self): @@ -142,12 +141,10 @@ class AbstractVirtualValue(OptValue): - _attrs_ = ('optimizer', 'keybox', 'source_op', '_cached_vinfo', - 'backstore_field') + _attrs_ = ('optimizer', 'keybox', 'source_op', '_cached_vinfo') box = None level = LEVEL_NONNULL _cached_vinfo = None - backstore_field = None # the fielddescr from lazy_setfields def __init__(self, optimizer, keybox, source_op=None): self.optimizer = optimizer @@ -166,28 +163,12 @@ self._really_force() return self.box - def get_backstore(self): - if self.backstore_field is None: - return (None, None) - heapopt = self.optimizer.heap_op_optimizer - try: - op = heapopt.lazy_setfields[self.backstore_field] - except KeyError: - self.backstore_field = None - return (None, None) - if self.optimizer.getvalue(op.args[1]) is not self: - self.backstore_field = None - return (None, None) - return (op.args[0], self.backstore_field) - - def make_virtual_info(self, modifier, fieldnums, - backstore_num, backstore_descr): + def make_virtual_info(self, modifier, fieldnums): vinfo = self._cached_vinfo - if vinfo is not None and vinfo.equals(fieldnums, backstore_num, - backstore_descr): + if vinfo is not None and vinfo.equals(fieldnums): return vinfo vinfo = self._make_virtual(modifier) - vinfo.set_content(fieldnums, backstore_num, backstore_descr) + vinfo.set_content(fieldnums) self._cached_vinfo = vinfo return vinfo @@ -562,11 +543,11 @@ self.newoperations.append(op) def store_final_boxes_in_guard(self, op): - self.heap_op_optimizer.force_lazy_setfields_for_guard() + pendingfields = self.heap_op_optimizer.force_lazy_setfields_for_guard() descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo) - newboxes = modifier.finish(self.values) + newboxes = modifier.finish(self.values, pendingfields) if len(newboxes) > self.metainterp_sd.options.failargs_limit: # XXX be careful here raise compile.GiveUp descr.store_final_boxes(op, newboxes) @@ -984,22 +965,18 @@ except KeyError: return del self.lazy_setfields[descr] - fieldvalue = self.optimizer.getvalue(op.args[1]) - if isinstance(fieldvalue, AbstractVirtualValue): - fieldvalue.backstore_field = None self.optimizer._emit_operation(op) # # hackish: reverse the order of the last two operations if it makes - # sense to avoid the situation "int_eq/setfield_gc/guard_true" + # sense to avoid a situation like "int_eq/setfield_gc/guard_true", + # which the backend (at least the x86 backend) does not handle well. newoperations = self.optimizer.newoperations if before_guard and len(newoperations) >= 2: lastop = newoperations[-1] prevop = newoperations[-2] if prevop.is_always_pure() and prevop.result not in lastop.args: - del newoperations[-1] - del newoperations[-1] - newoperations.append(lastop) - newoperations.append(prevop) + newoperations[-2] = lastop + newoperations[-1] = prevop def force_all_lazy_setfields(self): if len(self.lazy_setfields_descrs) > 0: @@ -1008,6 +985,7 @@ del self.lazy_setfields_descrs[:] def force_lazy_setfields_for_guard(self): + pendingfields = [] for descr in self.lazy_setfields_descrs: try: op = self.lazy_setfields[descr] @@ -1015,18 +993,17 @@ continue # the only really interesting case that we need to handle in the # guards' resume data is that of a virtual object that is stored - # into a field of a non-virtual object. The later object cannot - # actually be virtual here (verified by an assert), but the - # former object 'fieldvalue' can be. + # into a field of a non-virtual object. value = self.optimizer.getvalue(op.args[0]) - assert not value.is_virtual() + assert not value.is_virtual() # it must be a non-virtual fieldvalue = self.optimizer.getvalue(op.args[1]) if fieldvalue.is_virtual(): - assert isinstance(fieldvalue, AbstractVirtualValue) - if fieldvalue.backstore_field is descr: - # this is the case that can be handled by resume data - continue - self.force_lazy_setfield(descr, before_guard=True) + # this is the case that we leave to resume.py + pendingfields.append((descr, value.box, + fieldvalue.get_key_box())) + else: + self.force_lazy_setfield(descr, before_guard=True) + return pendingfields def force_lazy_setfield_if_necessary(self, op, value, write=False): try: @@ -1056,8 +1033,6 @@ def optimize_SETFIELD_GC(self, op, value, fieldvalue): self.force_lazy_setfield_if_necessary(op, value, write=True) self.lazy_setfields[op.descr] = op - if isinstance(fieldvalue, AbstractVirtualValue): - fieldvalue.backstore_field = op.descr # remember the result of future reads of the field self.cache_field_value(op.descr, value, fieldvalue, write=True) Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/resume.py Mon Dec 28 21:27:04 2009 @@ -266,7 +266,7 @@ _, tagbits = untag(tagged) return tagbits == TAGVIRTUAL - def finish(self, values): + def finish(self, values, pending_setfields=[]): # compute the numbering storage = self.storage numb, liveboxes_from_env, v = self.memo.number(values, @@ -289,7 +289,14 @@ value = values[box] value.get_args_for_fail(self) + for _, box, fieldbox in pending_setfields: + self.register_box(box) + self.register_box(fieldbox) + value = values[fieldbox] + value.get_args_for_fail(self) + self._number_virtuals(liveboxes, values, v) + self._add_pending_fields(pending_setfields) storage.rd_consts = self.memo.consts dump_storage(storage, liveboxes) @@ -338,13 +345,7 @@ value = values[virtualbox] fieldnums = [self._gettagged(box) for box in fieldboxes] - backstore_box, backstore_descr = value.get_backstore() - if backstore_descr is not None: - backstore_num = self._gettagged(backstore_box) - else: - backstore_num = NULLREF - vinfo = value.make_virtual_info(self, fieldnums, - backstore_num, backstore_descr) + vinfo = value.make_virtual_info(self, fieldnums) # if a new vinfo instance is made, we get the fieldnums list we # pass in as an attribute. hackish. if vinfo.fieldnums is not fieldnums: @@ -363,6 +364,16 @@ return True return False + def _add_pending_fields(self, pending_setfields): + rd_pendingfields = None + if pending_setfields: + rd_pendingfields = [] + for descr, box, fieldbox in pending_setfields: + num = self._gettagged(box) + fieldnum = self._gettagged(fieldbox) + rd_pendingfields.append((descr, num, fieldnum)) + self.storage.rd_pendingfields = rd_pendingfields + def _gettagged(self, box): if isinstance(box, Const): return self.memo.getconst(box) @@ -372,46 +383,15 @@ return self.liveboxes[box] -class BackstoreRef(object): - # Used to say that a virtual object must, after being created because - # of a guard failure, be stored back on the given field of the given - # non-virtual object. For lazy setfields. Limited to one place per - # virtual for now. - def __init__(self, parentdescr, parentnum): - self.parentdescr = parentdescr - self.parentnum = parentnum - - def setfields(self, metainterp, box, fn_decode_box): - parentbox = fn_decode_box(self.parentnum) - metainterp.execute_and_record(rop.SETFIELD_GC, self.parentdescr, - parentbox, box) - class AbstractVirtualInfo(object): - backstore_ref = None - def allocate(self, metainterp): raise NotImplementedError - def setfields(self, metainterp, box, fn_decode_box): - if self.backstore_ref is not None: - self.backstore_ref.setfields(metainterp, box, fn_decode_box) - - def equals(self, fieldnums, backstore_num, backstore_descr): - return (tagged_list_eq(self.fieldnums, fieldnums) and - self.backstore_equals(backstore_num, backstore_descr)) - - def backstore_equals(self, backstore_num, backstore_descr): - if backstore_descr is None: - return self.backstore_ref is None - else: - return (self.backstore_ref is not None and - self.backstore_ref.parentdescr == backstore_descr and - self.backstore_ref.parentnum == backstore_num) - - def set_content(self, fieldnums, backstore_num, backstore_descr): + raise NotImplementedError + def equals(self, fieldnums): + return tagged_list_eq(self.fieldnums, fieldnums) + def set_content(self, fieldnums): self.fieldnums = fieldnums - if backstore_descr is not None: - self.backstore_ref = BackstoreRef(backstore_descr, backstore_num) class AbstractVirtualStructInfo(AbstractVirtualInfo): @@ -425,7 +405,6 @@ metainterp.execute_and_record(rop.SETFIELD_GC, self.fielddescrs[i], box, fieldbox) - AbstractVirtualInfo.setfields(self, metainterp, box, fn_decode_box) def debug_prints(self): assert len(self.fielddescrs) == len(self.fieldnums) @@ -476,7 +455,6 @@ metainterp.execute_and_record(rop.SETARRAYITEM_GC, self.arraydescr, box, ConstInt(i), itembox) - AbstractVirtualInfo.setfields(self, metainterp, box, fn_decode_box) def debug_prints(self): debug_print("\tvarrayinfo", self.arraydescr) @@ -514,6 +492,7 @@ self.liveboxes = liveboxes self.cpu = metainterp.cpu self._prepare_virtuals(metainterp, storage.rd_virtuals) + self._prepare_pendingfields(metainterp, storage.rd_pendingfields) def _prepare_virtuals(self, metainterp, virtuals): if virtuals: @@ -532,6 +511,16 @@ vinfo.setfields(metainterp, self.virtuals[i], self._decode_box) + def _prepare_pendingfields(self, metainterp, pendingfields): + if pendingfields: + if metainterp._already_allocated_resume_virtuals is not None: + return + for descr, num, fieldnum in pendingfields: + box = self._decode_box(num) + fieldbox = self._decode_box(fieldnum) + metainterp.execute_and_record(rop.SETFIELD_GC, + descr, box, fieldbox) + def consume_boxes(self): numb = self.cur_numb assert numb is not None Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_optimizeopt.py Mon Dec 28 21:27:04 2009 @@ -87,39 +87,21 @@ def test_reuse_vinfo(): class FakeVInfo(object): - def set_content(self, fieldnums, backstore_num, backstore_descr): + def set_content(self, fieldnums): self.fieldnums = fieldnums - self.backstore_num = backstore_num - self.backstore_descr = backstore_descr - def equals(self, fieldnums, backstore_num, backstore_descr): - return (self.fieldnums == fieldnums and - self.backstore_num == backstore_num and - self.backstore_descr == backstore_descr) + def equals(self, fieldnums): + return self.fieldnums == fieldnums class FakeVirtualValue(optimizeopt.AbstractVirtualValue): def _make_virtual(self, *args): return FakeVInfo() v1 = FakeVirtualValue(None, None, None) - vinfo1 = v1.make_virtual_info(None, [1, 2, 4], None, None) - vinfo2 = v1.make_virtual_info(None, [1, 2, 4], None, None) + vinfo1 = v1.make_virtual_info(None, [1, 2, 4]) + vinfo2 = v1.make_virtual_info(None, [1, 2, 4]) assert vinfo1 is vinfo2 - vinfo3 = v1.make_virtual_info(None, [1, 2, 6], None, None) + vinfo3 = v1.make_virtual_info(None, [1, 2, 6]) assert vinfo3 is not vinfo2 - vinfo4 = v1.make_virtual_info(None, [1, 2, 6], None, None) + vinfo4 = v1.make_virtual_info(None, [1, 2, 6]) assert vinfo3 is vinfo4 - # - descr = object() - descr2 = object() - vinfo1 = v1.make_virtual_info(None, [1, 2, 4], None, None) - vinfo2 = v1.make_virtual_info(None, [1, 2, 4], 6, descr) - assert vinfo1 is not vinfo2 - vinfo3 = v1.make_virtual_info(None, [1, 2, 4], 6, descr) - assert vinfo3 is vinfo2 - vinfo4 = v1.make_virtual_info(None, [1, 2, 4], 5, descr) - assert vinfo4 is not vinfo3 - vinfo5 = v1.make_virtual_info(None, [1, 2, 4], 5, descr2) - assert vinfo5 is not vinfo4 - vinfo6 = v1.make_virtual_info(None, [1, 2, 4], 5, descr2) - assert vinfo6 is vinfo5 def test_descrlist_dict(): from pypy.jit.metainterp import optimizeutil @@ -1422,7 +1404,7 @@ """ expected = """ [p1, i2, i3] - guard_true(i3) [] + guard_true(i3) [p1] i4 = int_neg(i2) setfield_gc(p1, NULL, descr=nextdescr) jump(p1, i2, i4) @@ -1430,6 +1412,26 @@ self.optimize_loop(ops, 'Not, Not, Not', expected) def test_duplicate_setfield_residual_guard_3(self): + ops = """ + [p1, i2, i3] + p2 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p2, i2, descr=valuedescr) + setfield_gc(p1, p2, descr=nextdescr) + guard_true(i3) [] + i4 = int_neg(i2) + setfield_gc(p1, NULL, descr=nextdescr) + jump(p1, i2, i4) + """ + expected = """ + [p1, i2, i3] + guard_true(i3) [p1, i2] + i4 = int_neg(i2) + setfield_gc(p1, NULL, descr=nextdescr) + jump(p1, i2, i4) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + + def test_duplicate_setfield_residual_guard_4(self): # test that the setfield_gc does not end up between int_eq and # the following guard_true ops = """ @@ -1811,6 +1813,14 @@ tag = ('virtual', self.namespace[match.group(2)]) virtuals[pvar] = (tag, None, fieldstext) # + r2 = re.compile(r"([\w\d()]+)[.](\w+)\s*=\s*([\w\d()]+)") + pendingfields = [] + for match in r2.finditer(text): + pvar = match.group(1) + pfieldname = match.group(2) + pfieldvar = match.group(3) + pendingfields.append((pvar, pfieldname, pfieldvar)) + # def _variables_equal(box, varname, strict): if varname not in virtuals: if strict: @@ -1832,11 +1842,21 @@ else: virtuals[varname] = tag, box, fieldstext # - basetext = text[:ends[0]] + basetext = text.splitlines()[0] varnames = [s.strip() for s in basetext.split(',')] + if varnames == ['']: + varnames = [] assert len(boxes) == len(varnames) for box, varname in zip(boxes, varnames): _variables_equal(box, varname, strict=True) + for pvar, pfieldname, pfieldvar in pendingfields: + box = oparse.getvar(pvar) + fielddescr = self.namespace[pfieldname.strip()] + fieldbox = executor.execute(self.cpu, + rop.GETFIELD_GC, + fielddescr, + box) + _variables_equal(fieldbox, pfieldvar, strict=True) # for match in parts: pvar = match.group(1) @@ -2109,14 +2129,16 @@ """ expected = """ [p1, i2, i3] - guard_true(i3, descr=fdescr) [i2] + guard_true(i3, descr=fdescr) [p1, i2] i4 = int_neg(i2) setfield_gc(p1, NULL, descr=nextdescr) jump(p1, i2, i4) """ self.optimize_loop(ops, 'Not, Not, Not', expected) + self.loop.inputargs[0].value = self.nodebox.value self.check_expanded_fail_descr(''' - where p2 is a node_vtable, valuedescr=i2 --> p1.nextdescr + p1.nextdescr = p2 + where p2 is a node_vtable, valuedescr=i2 ''') def test_expand_fail_lazy_setfield_2(self): @@ -2140,7 +2162,8 @@ """ self.optimize_loop(ops, 'Not, Not', expected) self.check_expanded_fail_descr(''' - where p2 is a node_vtable, valuedescr=i2 --> myptr.nextdescr + ConstPtr(myptr).nextdescr = p2 + where p2 is a node_vtable, valuedescr=i2 ''') Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/test/test_resume.py Mon Dec 28 21:27:04 2009 @@ -12,6 +12,8 @@ rd_frame_info_list = None rd_numb = None rd_consts = [] + rd_virtuals = None + rd_pendingfields = None def test_tag(): assert tag(3, 1) == rffi.r_short(3<<2|1) @@ -42,21 +44,9 @@ def test_vinfo(): v1 = AbstractVirtualInfo() - v1.set_content([1, 2, 4], -1, None) - assert v1.backstore_ref is None - assert v1.equals([1, 2, 4], -1, None) - assert not v1.equals([1, 2, 6], -1, None) - assert not v1.equals([1, 2, 4], 3, object()) - # - v2 = AbstractVirtualInfo() - descr = object() - v2.set_content([1, 2, 4], 3, descr) - assert v2.backstore_ref is not None - assert v2.backstore_ref.parentdescr is descr - assert v2.backstore_ref.parentnum == 3 - assert v2.equals([1, 2, 4], 3, descr) - assert not v2.equals([1, 2, 4], 2, descr) - assert not v2.equals([1, 2, 4], 3, object()) + v1.set_content([1, 2, 4]) + assert v1.equals([1, 2, 4]) + assert not v1.equals([1, 2, 6]) class MyMetaInterp: _already_allocated_resume_virtuals = None @@ -98,7 +88,6 @@ tag(0, TAGBOX), tag(1, TAGBOX)]) storage.rd_numb = numb - storage.rd_virtuals = None b1s, b2s, b3s = [BoxInt(), BoxPtr(), BoxInt()] assert b1s != b3s @@ -121,7 +110,6 @@ tag(0, TAGBOX), tag(1, TAGBOX)]) storage.rd_numb = numb - storage.rd_virtuals = None b1s, b2s, b3s = [BoxInt(), BoxPtr(), BoxInt()] assert b1s != b3s reader = ResumeDataReader(storage, [b1s, b2s, b3s], MyMetaInterp()) @@ -143,6 +131,7 @@ rd_virtuals = [FakeVinfo(), None] rd_numb = [] rd_consts = [] + rd_pendingfields = None class FakeMetainterp(object): _already_allocated_resume_virtuals = None cpu = None @@ -979,9 +968,8 @@ assert ptr.b == lltype.nullptr(LLtypeMixin.NODE) -def test_virtual_adder_make_virtual_backstore(): - b2s, b4s, b5s = [BoxPtr(), BoxPtr(), BoxPtr()] - c1s = ConstInt(111) +def test_virtual_adder_pending_fields(): + b2s, b4s = [BoxPtr(), BoxPtr()] storage = Storage() memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) @@ -989,60 +977,34 @@ modifier.liveboxes = {} modifier.vfieldboxes = {} - class FakeOptimizer(object): - class cpu: - pass - def getvalue(self, box): - return {b2s: v2, b4s: v4}[box] - fakeoptimizer = FakeOptimizer() - class HeapOpOptimizer(object): - pass - fakeoptimizer.heap_op_optimizer = HeapOpOptimizer() - class FakeSetFieldOperation(object): - args = [b2s, b4s] - fakeoptimizer.heap_op_optimizer.lazy_setfields = { - LLtypeMixin.nextdescr: FakeSetFieldOperation()} - - v4 = VirtualValue(fakeoptimizer, ConstAddr(LLtypeMixin.node_vtable_adr2, - LLtypeMixin.cpu), b4s) - v4.backstore_field = LLtypeMixin.nextdescr - v4.setfield(LLtypeMixin.valuedescr, OptValue(c1s)) - v4.setfield(LLtypeMixin.otherdescr, OptValue(b5s)) - v4._cached_sorted_fields = [LLtypeMixin.valuedescr, LLtypeMixin.otherdescr] - v2 = OptValue(b2s) + v4 = OptValue(b4s) + modifier.register_box(b2s) + modifier.register_box(b4s) - v4.register_virtual_fields(modifier, [c1s, b5s]) + values = {b4s: v4, b2s: v2} liveboxes = [] - modifier._number_virtuals(liveboxes, {b4s: v4}, 0) - assert liveboxes == [b2s, b5s] + modifier._number_virtuals(liveboxes, values, 0) + assert liveboxes == [b2s, b4s] + modifier._add_pending_fields([(LLtypeMixin.nextdescr, b2s, b4s)]) storage.rd_consts = memo.consts[:] storage.rd_numb = None # resume demo55.next = lltype.nullptr(LLtypeMixin.NODE) b2t = BoxPtr(demo55o) - b5t = BoxPtr(demo66o) - newboxes = _resume_remap(liveboxes, [b2s, b5s], b2t, b5t) + b4t = BoxPtr(demo66o) + newboxes = _resume_remap(liveboxes, [b2s, b4s], b2t, b4t) metainterp = MyMetaInterp() reader = ResumeDataReader(storage, newboxes, metainterp) - assert len(reader.virtuals) == 1 - b4t = reader._decode_box(modifier._gettagged(b4s)) + assert reader.virtuals is None trace = metainterp.trace - b4new = (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr2, - LLtypeMixin.cpu)], - b4t, None) - b4set1 = (rop.SETFIELD_GC, [b4t, c1s], None, LLtypeMixin.valuedescr) - b4set2 = (rop.SETFIELD_GC, [b4t, b5t], None, LLtypeMixin.otherdescr) b2set = (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.nextdescr) - expected = [b4new, b4set1, b4set2, b2set] + expected = [b2set] for x, y in zip(expected, trace): assert x == y - ptr = demo55.next - assert ptr.value == 111 - ptr = lltype.cast_pointer(lltype.Ptr(LLtypeMixin.NODE2), ptr) - assert ptr.other == demo66 + assert demo55.next == demo66 def test_invalidation_needed(): From arigo at codespeak.net Mon Dec 28 22:03:09 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 28 Dec 2009 22:03:09 +0100 (CET) Subject: [pypy-svn] r70314 - in pypy/branch/jit-delayed-write/pypy: rpython rpython/test translator/backendopt/test Message-ID: <20091228210309.8111D168015@codespeak.net> Author: arigo Date: Mon Dec 28 22:03:08 2009 New Revision: 70314 Modified: pypy/branch/jit-delayed-write/pypy/rpython/rptr.py pypy/branch/jit-delayed-write/pypy/rpython/test/test_rptr.py pypy/branch/jit-delayed-write/pypy/translator/backendopt/test/test_writeanalyze.py Log: Don't generate a dummy "Void=getfield(p,'name')" for all ADT fields 'name'. It's in principle wrong because 'name' is not the name of a field, although it did not show up so far because it's always a Void. Added a test to test_writeanalyze that shows in which situation the issue caused a problem. Modified: pypy/branch/jit-delayed-write/pypy/rpython/rptr.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/rpython/rptr.py (original) +++ pypy/branch/jit-delayed-write/pypy/rpython/rptr.py Mon Dec 28 22:03:08 2009 @@ -39,6 +39,14 @@ attr = hop.args_s[1].const if isinstance(hop.s_result, annmodel.SomeLLADTMeth): return hop.inputarg(hop.r_result, arg=0) + try: + self.lowleveltype._example()._lookup_adtmeth(attr) + except AttributeError: + pass + else: + assert hop.s_result.is_constant() + return hop.inputconst(hop.r_result, hop.s_result.const) + assert attr in self.lowleveltype.TO._flds # check that the field exists FIELD_TYPE = getattr(self.lowleveltype.TO, attr) if isinstance(FIELD_TYPE, lltype.ContainerType): if (attr, FIELD_TYPE) == self.lowleveltype.TO._first_struct(): Modified: pypy/branch/jit-delayed-write/pypy/rpython/test/test_rptr.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/rpython/test/test_rptr.py (original) +++ pypy/branch/jit-delayed-write/pypy/rpython/test/test_rptr.py Mon Dec 28 22:03:08 2009 @@ -337,3 +337,13 @@ return f([1]) s, t = ll_rtype(lltest, []) assert s.is_constant() == False + +def test_staticadtmeths(): + ll_func = staticAdtMethod(lambda x: x + 42) + S = GcStruct('S', adtmeths={'ll_func': ll_func}) + def f(): + return malloc(S).ll_func(5) + s, t = ll_rtype(f, []) + graphf = t.graphs[0] + for op in graphf.startblock.operations: + assert op.opname != 'getfield' Modified: pypy/branch/jit-delayed-write/pypy/translator/backendopt/test/test_writeanalyze.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/translator/backendopt/test/test_writeanalyze.py (original) +++ pypy/branch/jit-delayed-write/pypy/translator/backendopt/test/test_writeanalyze.py Mon Dec 28 22:03:08 2009 @@ -306,3 +306,26 @@ ARRAYPTR = list(result)[0][1] assert list(result) == [("readarray", ARRAYPTR)] assert isinstance(ARRAYPTR.TO, lltype.GcArray) + + def test_adt_method(self): + def ll_callme(n): + return n + ll_callme = lltype.staticAdtMethod(ll_callme) + S = lltype.GcStruct('S', ('x', lltype.Signed), + adtmeths = {'yep': True, + 'callme': ll_callme}) + def g(x, y, z): + p = lltype.malloc(S) + p.x = x + if p.yep: + z *= p.callme(y) + return z + def f(x, y, z): + return g(x, y, z) + + t, wa = self.translate(f, [int, int, int]) + fgraph = graphof(t, f) + assert fgraph.startblock.operations[-1].opname == 'direct_call' + + result = wa.analyze(fgraph.startblock.operations[-1]) + assert list(result) == [("struct", lltype.Ptr(S), "x")] From arigo at codespeak.net Mon Dec 28 22:17:59 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 28 Dec 2009 22:17:59 +0100 (CET) Subject: [pypy-svn] r70315 - pypy/branch/jit-delayed-write/pypy/jit/metainterp Message-ID: <20091228211759.0B630168022@codespeak.net> Author: arigo Date: Mon Dec 28 22:17:59 2009 New Revision: 70315 Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py Log: Fix a case shown by test_virtualizable. Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py Mon Dec 28 22:17:59 2009 @@ -974,7 +974,10 @@ if before_guard and len(newoperations) >= 2: lastop = newoperations[-1] prevop = newoperations[-2] - if prevop.is_always_pure() and prevop.result not in lastop.args: + # - is_always_pure() for cases like "int_eq/setfield_gc/guard_true" + # - CALL_MAY_FORCE: "call_may_force/setfield_gc/guard_not_forced" + if ((prevop.is_always_pure() or prevop.opnum == rop.CALL_MAY_FORCE) + and prevop.result not in lastop.args): newoperations[-2] = lastop newoperations[-1] = prevop From arigo at codespeak.net Tue Dec 29 09:27:15 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 09:27:15 +0100 (CET) Subject: [pypy-svn] r70320 - pypy/branch/jit-delayed-write/pypy/objspace/flow Message-ID: <20091229082715.54107168022@codespeak.net> Author: arigo Date: Tue Dec 29 09:27:14 2009 New Revision: 70320 Modified: pypy/branch/jit-delayed-write/pypy/objspace/flow/flowcontext.py Log: Small performance improvement. Modified: pypy/branch/jit-delayed-write/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/objspace/flow/flowcontext.py (original) +++ pypy/branch/jit-delayed-write/pypy/objspace/flow/flowcontext.py Tue Dec 29 09:27:14 2009 @@ -384,6 +384,9 @@ operr = OperationError(operr.w_type, operr.w_value) return operr + def exception_trace(self, frame, operationerr): + pass # overridden for performance only + # hack for unrolling iterables, don't use this def replace_in_stack(self, oldvalue, newvalue): w_new = Constant(newvalue) From arigo at codespeak.net Tue Dec 29 09:35:24 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 09:35:24 +0100 (CET) Subject: [pypy-svn] r70321 - pypy/branch/jit-delayed-write/pypy/jit/metainterp Message-ID: <20091229083524.8B89A168022@codespeak.net> Author: arigo Date: Tue Dec 29 09:35:23 2009 New Revision: 70321 Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py Log: Use is_comparison() instead of always_pure(), to follow precisely what the backend is doing. Not too important. Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py Tue Dec 29 09:35:23 2009 @@ -974,9 +974,9 @@ if before_guard and len(newoperations) >= 2: lastop = newoperations[-1] prevop = newoperations[-2] - # - is_always_pure() for cases like "int_eq/setfield_gc/guard_true" + # - is_comparison() for cases like "int_eq/setfield_gc/guard_true" # - CALL_MAY_FORCE: "call_may_force/setfield_gc/guard_not_forced" - if ((prevop.is_always_pure() or prevop.opnum == rop.CALL_MAY_FORCE) + if ((prevop.is_comparison() or prevop.opnum == rop.CALL_MAY_FORCE) and prevop.result not in lastop.args): newoperations[-2] = lastop newoperations[-1] = prevop From arigo at codespeak.net Tue Dec 29 09:44:34 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 09:44:34 +0100 (CET) Subject: [pypy-svn] r70322 - pypy/branch/jit-delayed-write/pypy/jit/metainterp Message-ID: <20091229084434.E8543168012@codespeak.net> Author: arigo Date: Tue Dec 29 09:44:32 2009 New Revision: 70322 Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py Log: Revert now-pointless changes left behind. Modified: pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-delayed-write/pypy/jit/metainterp/optimizeopt.py Tue Dec 29 09:44:32 2009 @@ -63,9 +63,6 @@ def get_args_for_fail(self, modifier): pass - def get_backstore(self): - return (None, None) - def make_virtual_info(self, modifier, fieldnums): raise NotImplementedError # should not be called on this level @@ -175,11 +172,6 @@ def _make_virtual(self, modifier): raise NotImplementedError("abstract base") - def register_virtual_fields(self, modifier, fieldboxes): - modifier.register_virtual_fields(self.keybox, fieldboxes) - parentbox, parentdescr = self.get_backstore() - if parentdescr is not None: - modifier.register_box(parentbox) def get_fielddescrlist_cache(cpu): if not hasattr(cpu, '_optimizeopt_fielddescrlist_cache'): @@ -246,7 +238,7 @@ # we have already seen the very same keybox lst = self._get_field_descr_list() fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst] - self.register_virtual_fields(modifier, fieldboxes) + modifier.register_virtual_fields(self.keybox, fieldboxes) for ofs in lst: fieldvalue = self._fields[ofs] fieldvalue.get_args_for_fail(modifier) @@ -314,7 +306,7 @@ itemboxes = [] for itemvalue in self._items: itemboxes.append(itemvalue.get_key_box()) - self.register_virtual_fields(modifier, itemboxes) + modifier.register_virtual_fields(self.keybox, itemboxes) for itemvalue in self._items: if itemvalue is not self.constvalue: itemvalue.get_args_for_fail(modifier) From arigo at codespeak.net Tue Dec 29 09:52:15 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 09:52:15 +0100 (CET) Subject: [pypy-svn] r70323 - pypy/trunk/pypy/objspace/flow Message-ID: <20091229085215.CF955168006@codespeak.net> Author: arigo Date: Tue Dec 29 09:52:15 2009 New Revision: 70323 Modified: pypy/trunk/pypy/objspace/flow/flowcontext.py Log: Small performance improvement. (first check-in was to the branch/jit-delayed-write, by mistake) Modified: pypy/trunk/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/flowcontext.py (original) +++ pypy/trunk/pypy/objspace/flow/flowcontext.py Tue Dec 29 09:52:15 2009 @@ -384,6 +384,9 @@ operr = OperationError(operr.w_type, operr.w_value) return operr + def exception_trace(self, frame, operationerr): + pass # overridden for performance only + # hack for unrolling iterables, don't use this def replace_in_stack(self, oldvalue, newvalue): w_new = Constant(newvalue) From arigo at codespeak.net Tue Dec 29 10:03:15 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 10:03:15 +0100 (CET) Subject: [pypy-svn] r70324 - in pypy/trunk/pypy: jit/metainterp jit/metainterp/test rpython rpython/test translator/backendopt translator/backendopt/test Message-ID: <20091229090315.C4207168012@codespeak.net> Author: arigo Date: Tue Dec 29 10:03:14 2009 New Revision: 70324 Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py pypy/trunk/pypy/jit/metainterp/effectinfo.py pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py pypy/trunk/pypy/rpython/rptr.py pypy/trunk/pypy/rpython/test/test_rptr.py pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py pypy/trunk/pypy/translator/backendopt/writeanalyze.py Log: Merge jit-delayed-write. Allows repeated setfield_gc's to get optimized out. The main result is that if a setfield_gc's would stick a virtual structure on a non-virtual object, and if that setfield_gc is later overridden, then optimizing it out allows the complete structure to remain virtual. Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/codewriter.py Tue Dec 29 10:03:14 2009 @@ -11,7 +11,7 @@ from pypy.tool.udir import udir from pypy.translator.simplify import get_funcobj, get_functype from pypy.translator.backendopt.canraise import RaiseAnalyzer -from pypy.translator.backendopt.writeanalyze import WriteAnalyzer +from pypy.translator.backendopt.writeanalyze import ReadWriteAnalyzer from pypy.jit.metainterp.typesystem import deref, arrayItem, fieldType from pypy.jit.metainterp.effectinfo import effectinfo_from_writeanalyze from pypy.jit.metainterp.effectinfo import VirtualizableAnalyzer @@ -185,7 +185,7 @@ self.portal_runner_ptr = portal_runner_ptr translator = self.rtyper.annotator.translator self.raise_analyzer = RaiseAnalyzer(translator) - self.write_analyzer = WriteAnalyzer(translator) + self.readwrite_analyzer = ReadWriteAnalyzer(translator) self.virtualizable_analyzer = VirtualizableAnalyzer(translator) def make_portal_bytecode(self, graph): @@ -326,7 +326,7 @@ # ok if consider_effects_of is not None: effectinfo = effectinfo_from_writeanalyze( - self.write_analyzer.analyze(consider_effects_of), + self.readwrite_analyzer.analyze(consider_effects_of), self.cpu, self.virtualizable_analyzer.analyze(consider_effects_of)) calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT, effectinfo) Modified: pypy/trunk/pypy/jit/metainterp/effectinfo.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/effectinfo.py (original) +++ pypy/trunk/pypy/jit/metainterp/effectinfo.py Tue Dec 29 10:03:14 2009 @@ -7,13 +7,16 @@ class EffectInfo(object): _cache = {} - def __new__(cls, write_descrs_fields, write_descrs_arrays, - promotes_virtualizables=False): - key = (frozenset(write_descrs_fields), frozenset(write_descrs_arrays), + def __new__(cls, readonly_descrs_fields, write_descrs_fields, + write_descrs_arrays, promotes_virtualizables=False): + key = (frozenset(readonly_descrs_fields), + frozenset(write_descrs_fields), + frozenset(write_descrs_arrays), promotes_virtualizables) if key in cls._cache: return cls._cache[key] result = object.__new__(cls) + result.readonly_descrs_fields = readonly_descrs_fields result.write_descrs_fields = write_descrs_fields result.write_descrs_arrays = write_descrs_arrays result.promotes_virtualizables = promotes_virtualizables @@ -24,26 +27,39 @@ from pypy.translator.backendopt.writeanalyze import top_set if effects is top_set: return None + readonly_descrs_fields = [] + # readonly_descrs_arrays = [] --- not enabled for now write_descrs_fields = [] write_descrs_arrays = [] + + def add_struct(descrs_fields, (_, T, fieldname)): + T = deref(T) + if consider_struct(T, fieldname): + descr = cpu.fielddescrof(T, fieldname) + descrs_fields.append(descr) + + def add_array(descrs_arrays, (_, T)): + ARRAY = deref(T) + if consider_array(ARRAY): + descr = cpu.arraydescrof(ARRAY) + descrs_arrays.append(descr) + for tup in effects: if tup[0] == "struct": - _, T, fieldname = tup - T = deref(T) - if not consider_struct(T, fieldname): - continue - descr = cpu.fielddescrof(T, fieldname) - write_descrs_fields.append(descr) + add_struct(write_descrs_fields, tup) + elif tup[0] == "readstruct": + tupw = ("struct",) + tup[1:] + if tupw not in effects: + add_struct(readonly_descrs_fields, tup) elif tup[0] == "array": - _, T = tup - ARRAY = deref(T) - if not consider_array(ARRAY): - continue - descr = cpu.arraydescrof(ARRAY) - write_descrs_arrays.append(descr) + add_array(write_descrs_arrays, tup) + elif tup[0] == "readarray": + pass else: assert 0 - return EffectInfo(write_descrs_fields, write_descrs_arrays, + return EffectInfo(readonly_descrs_fields, + write_descrs_fields, + write_descrs_arrays, promotes_virtualizables) def consider_struct(TYPE, fieldname): Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Tue Dec 29 10:03:14 2009 @@ -161,18 +161,18 @@ return self.box def make_virtual_info(self, modifier, fieldnums): - vinfo = self._cached_vinfo - if vinfo is not None and resume.tagged_list_eq( - vinfo.fieldnums, fieldnums): + vinfo = self._cached_vinfo + if vinfo is not None and vinfo.equals(fieldnums): return vinfo vinfo = self._make_virtual(modifier) - vinfo.fieldnums = fieldnums + vinfo.set_content(fieldnums) self._cached_vinfo = vinfo return vinfo def _make_virtual(self, modifier): raise NotImplementedError("abstract base") + def get_fielddescrlist_cache(cpu): if not hasattr(cpu, '_optimizeopt_fielddescrlist_cache'): result = descrlist_dict() @@ -512,6 +512,9 @@ def emit_operation(self, op, must_clone=True): self.heap_op_optimizer.emitting_operation(op) + self._emit_operation(op, must_clone) + + def _emit_operation(self, op, must_clone=True): for i in range(len(op.args)): arg = op.args[i] if arg in self.values: @@ -532,10 +535,11 @@ self.newoperations.append(op) def store_final_boxes_in_guard(self, op): + pendingfields = self.heap_op_optimizer.force_lazy_setfields_for_guard() descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo) - newboxes = modifier.finish(self.values) + newboxes = modifier.finish(self.values, pendingfields) if len(newboxes) > self.metainterp_sd.options.failargs_limit: # XXX be careful here raise compile.GiveUp descr.store_final_boxes(op, newboxes) @@ -836,11 +840,13 @@ class HeapOpOptimizer(object): def __init__(self, optimizer): self.optimizer = optimizer - # cached OptValues for each field descr + # cached fields: {descr: {OptValue_instance: OptValue_fieldvalue}} self.cached_fields = {} - - # cached OptValues for each field descr + # cached array items: {descr: CachedArrayItems} self.cached_arrayitems = {} + # lazily written setfields (at most one per descr): {descr: op} + self.lazy_setfields = {} + self.lazy_setfields_descrs = [] # keys (at least) of previous dict def clean_caches(self): self.cached_fields.clear() @@ -848,6 +854,9 @@ def cache_field_value(self, descr, value, fieldvalue, write=False): if write: + # when seeing a setfield, we have to clear the cache for the same + # field on any other structure, just in case they are aliasing + # each other d = self.cached_fields[descr] = {} else: d = self.cached_fields.setdefault(descr, {}) @@ -920,7 +929,12 @@ opnum == rop.CALL_MAY_FORCE): effectinfo = op.descr.get_extra_info() if effectinfo is not None: + # XXX we can get the wrong complexity here, if the lists + # XXX stored on effectinfo are large + for fielddescr in effectinfo.readonly_descrs_fields: + self.force_lazy_setfield(fielddescr) for fielddescr in effectinfo.write_descrs_fields: + self.force_lazy_setfield(fielddescr) try: del self.cached_fields[fielddescr] except KeyError: @@ -931,9 +945,73 @@ except KeyError: pass return + self.force_all_lazy_setfields() + elif op.is_final() or (not we_are_translated() and + op.opnum < 0): # escape() operations + self.force_all_lazy_setfields() self.clean_caches() + def force_lazy_setfield(self, descr, before_guard=False): + try: + op = self.lazy_setfields[descr] + except KeyError: + return + del self.lazy_setfields[descr] + self.optimizer._emit_operation(op) + # + # hackish: reverse the order of the last two operations if it makes + # sense to avoid a situation like "int_eq/setfield_gc/guard_true", + # which the backend (at least the x86 backend) does not handle well. + newoperations = self.optimizer.newoperations + if before_guard and len(newoperations) >= 2: + lastop = newoperations[-1] + prevop = newoperations[-2] + # - is_comparison() for cases like "int_eq/setfield_gc/guard_true" + # - CALL_MAY_FORCE: "call_may_force/setfield_gc/guard_not_forced" + if ((prevop.is_comparison() or prevop.opnum == rop.CALL_MAY_FORCE) + and prevop.result not in lastop.args): + newoperations[-2] = lastop + newoperations[-1] = prevop + + def force_all_lazy_setfields(self): + if len(self.lazy_setfields_descrs) > 0: + for descr in self.lazy_setfields_descrs: + self.force_lazy_setfield(descr) + del self.lazy_setfields_descrs[:] + + def force_lazy_setfields_for_guard(self): + pendingfields = [] + for descr in self.lazy_setfields_descrs: + try: + op = self.lazy_setfields[descr] + except KeyError: + continue + # the only really interesting case that we need to handle in the + # guards' resume data is that of a virtual object that is stored + # into a field of a non-virtual object. + value = self.optimizer.getvalue(op.args[0]) + assert not value.is_virtual() # it must be a non-virtual + fieldvalue = self.optimizer.getvalue(op.args[1]) + if fieldvalue.is_virtual(): + # this is the case that we leave to resume.py + pendingfields.append((descr, value.box, + fieldvalue.get_key_box())) + else: + self.force_lazy_setfield(descr, before_guard=True) + return pendingfields + + def force_lazy_setfield_if_necessary(self, op, value, write=False): + try: + op1 = self.lazy_setfields[op.descr] + except KeyError: + if write: + self.lazy_setfields_descrs.append(op.descr) + else: + if self.optimizer.getvalue(op1.args[0]) is not value: + self.force_lazy_setfield(op.descr) + def optimize_GETFIELD_GC(self, op, value): + self.force_lazy_setfield_if_necessary(op, value) # check if the field was read from another getfield_gc just before # or has been written to recently fieldvalue = self.read_cached_field(op.descr, value) @@ -948,7 +1026,8 @@ self.cache_field_value(op.descr, value, fieldvalue) def optimize_SETFIELD_GC(self, op, value, fieldvalue): - self.optimizer.emit_operation(op) + self.force_lazy_setfield_if_necessary(op, value, write=True) + self.lazy_setfields[op.descr] = op # remember the result of future reads of the field self.cache_field_value(op.descr, value, fieldvalue, write=True) Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Tue Dec 29 10:03:14 2009 @@ -252,8 +252,6 @@ if (isinstance(box, Box) and box not in self.liveboxes_from_env and box not in self.liveboxes): self.liveboxes[box] = UNASSIGNED - return True - return False def _register_boxes(self, boxes): for box in boxes: @@ -268,7 +266,7 @@ _, tagbits = untag(tagged) return tagbits == TAGVIRTUAL - def finish(self, values): + def finish(self, values, pending_setfields=[]): # compute the numbering storage = self.storage numb, liveboxes_from_env, v = self.memo.number(values, @@ -291,13 +289,21 @@ value = values[box] value.get_args_for_fail(self) + for _, box, fieldbox in pending_setfields: + self.register_box(box) + self.register_box(fieldbox) + value = values[fieldbox] + value.get_args_for_fail(self) + self._number_virtuals(liveboxes, values, v) + self._add_pending_fields(pending_setfields) storage.rd_consts = self.memo.consts dump_storage(storage, liveboxes) return liveboxes[:] def _number_virtuals(self, liveboxes, values, num_env_virtuals): + # !! 'liveboxes' is a list that is extend()ed in-place !! memo = self.memo new_liveboxes = [None] * memo.num_cached_boxes() count = 0 @@ -358,6 +364,16 @@ return True return False + def _add_pending_fields(self, pending_setfields): + rd_pendingfields = None + if pending_setfields: + rd_pendingfields = [] + for descr, box, fieldbox in pending_setfields: + num = self._gettagged(box) + fieldnum = self._gettagged(fieldbox) + rd_pendingfields.append((descr, num, fieldnum)) + self.storage.rd_pendingfields = rd_pendingfields + def _gettagged(self, box): if isinstance(box, Const): return self.memo.getconst(box) @@ -366,11 +382,16 @@ return self.liveboxes_from_env[box] return self.liveboxes[box] + class AbstractVirtualInfo(object): def allocate(self, metainterp): raise NotImplementedError def setfields(self, metainterp, box, fn_decode_box): raise NotImplementedError + def equals(self, fieldnums): + return tagged_list_eq(self.fieldnums, fieldnums) + def set_content(self, fieldnums): + self.fieldnums = fieldnums class AbstractVirtualStructInfo(AbstractVirtualInfo): @@ -471,6 +492,7 @@ self.liveboxes = liveboxes self.cpu = metainterp.cpu self._prepare_virtuals(metainterp, storage.rd_virtuals) + self._prepare_pendingfields(metainterp, storage.rd_pendingfields) def _prepare_virtuals(self, metainterp, virtuals): if virtuals: @@ -489,6 +511,16 @@ vinfo.setfields(metainterp, self.virtuals[i], self._decode_box) + def _prepare_pendingfields(self, metainterp, pendingfields): + if pendingfields: + if metainterp._already_allocated_resume_virtuals is not None: + return + for descr, num, fieldnum in pendingfields: + box = self._decode_box(num) + fieldbox = self._decode_box(fieldnum) + metainterp.execute_and_record(rop.SETFIELD_GC, + descr, box, fieldbox) + def consume_boxes(self): numb = self.cur_numb assert numb is not None Modified: pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py Tue Dec 29 10:03:14 2009 @@ -3,32 +3,77 @@ from pypy.rpython.ootypesystem import ootype from pypy.jit.metainterp.effectinfo import effectinfo_from_writeanalyze +class FakeCPU: + def fielddescrof(self, T, fieldname): + return ('fielddescr', T, fieldname) + def arraydescrof(self, A): + return ('arraydescr', A) + +def test_include_read_field(): + S = lltype.GcStruct("S", ("a", lltype.Signed)) + effects = frozenset([("readstruct", lltype.Ptr(S), "a")]) + effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU()) + assert list(effectinfo.readonly_descrs_fields) == [('fielddescr', S, "a")] + assert not effectinfo.write_descrs_fields + assert not effectinfo.write_descrs_arrays + +def test_include_write_field(): + S = lltype.GcStruct("S", ("a", lltype.Signed)) + effects = frozenset([("struct", lltype.Ptr(S), "a")]) + effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU()) + assert list(effectinfo.write_descrs_fields) == [('fielddescr', S, "a")] + assert not effectinfo.readonly_descrs_fields + assert not effectinfo.write_descrs_arrays + +def test_include_write_array(): + A = lltype.GcArray(lltype.Signed) + effects = frozenset([("array", lltype.Ptr(A))]) + effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU()) + assert not effectinfo.readonly_descrs_fields + assert not effectinfo.write_descrs_fields + assert list(effectinfo.write_descrs_arrays) == [('arraydescr', A)] + +def test_dont_include_read_and_write_field(): + S = lltype.GcStruct("S", ("a", lltype.Signed)) + effects = frozenset([("readstruct", lltype.Ptr(S), "a"), + ("struct", lltype.Ptr(S), "a")]) + effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU()) + assert not effectinfo.readonly_descrs_fields + assert list(effectinfo.write_descrs_fields) == [('fielddescr', S, "a")] + assert not effectinfo.write_descrs_arrays + + def test_filter_out_typeptr(): effects = frozenset([("struct", lltype.Ptr(OBJECT), "typeptr")]) effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays def test_filter_out_array_of_void(): effects = frozenset([("array", lltype.Ptr(lltype.GcArray(lltype.Void)))]) effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays def test_filter_out_struct_with_void(): effects = frozenset([("struct", lltype.Ptr(lltype.GcStruct("x", ("a", lltype.Void))), "a")]) effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays def test_filter_out_ooarray_of_void(): effects = frozenset([("array", ootype.Array(ootype.Void))]) effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays def test_filter_out_instance_with_void(): effects = frozenset([("struct", ootype.Instance("x", ootype.ROOT, {"a": ootype.Void}), "a")]) effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py Tue Dec 29 10:03:14 2009 @@ -95,9 +95,15 @@ onedescr = cpu.fielddescrof(U, 'one') FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) - nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([], [])) - writeadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([adescr], [])) - writearraydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([adescr], [arraydescr])) + plaincalldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo([], [], [])) + writeadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo([], [adescr], [])) + writearraydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo([], [adescr], [arraydescr])) + readadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo([adescr], [], [])) cpu.class_sizes = {cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), cpu.cast_adr_to_int(node_vtable_adr2): cpu.sizeof(NODE2), Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Tue Dec 29 10:03:14 2009 @@ -87,7 +87,10 @@ def test_reuse_vinfo(): class FakeVInfo(object): - pass + def set_content(self, fieldnums): + self.fieldnums = fieldnums + def equals(self, fieldnums): + return self.fieldnums == fieldnums class FakeVirtualValue(optimizeopt.AbstractVirtualValue): def _make_virtual(self, *args): return FakeVInfo() @@ -606,10 +609,10 @@ p3sub = getfield_gc(p3, descr=nextdescr) i3 = getfield_gc(p3sub, descr=valuedescr) escape(i3) + p1 = new_with_vtable(ConstClass(node_vtable)) p2sub = new_with_vtable(ConstClass(node_vtable2)) setfield_gc(p2sub, i1, descr=valuedescr) setfield_gc(p2, p2sub, descr=nextdescr) - p1 = new_with_vtable(ConstClass(node_vtable)) jump(i1, p1, p2) """ # The same as test_p123_simple, but in the end the "old" p2 contains @@ -1293,6 +1296,182 @@ """ self.optimize_loop(ops, 'Not, Not', ops) + def test_duplicate_setfield_1(self): + ops = """ + [p1, i1, i2] + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(p1, i2, descr=valuedescr) + jump(p1, i1, i2) + """ + expected = """ + [p1, i1, i2] + setfield_gc(p1, i2, descr=valuedescr) + jump(p1, i1, i2) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + + def test_duplicate_setfield_2(self): + ops = """ + [p1, i1, i3] + setfield_gc(p1, i1, descr=valuedescr) + i2 = getfield_gc(p1, descr=valuedescr) + setfield_gc(p1, i3, descr=valuedescr) + escape(i2) + jump(p1, i1, i3) + """ + expected = """ + [p1, i1, i3] + setfield_gc(p1, i3, descr=valuedescr) + escape(i1) + jump(p1, i1, i3) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + + def test_duplicate_setfield_3(self): + ops = """ + [p1, p2, i1, i3] + setfield_gc(p1, i1, descr=valuedescr) + i2 = getfield_gc(p2, descr=valuedescr) + setfield_gc(p1, i3, descr=valuedescr) + escape(i2) + jump(p1, p2, i1, i3) + """ + # potential aliasing of p1 and p2 means that we cannot kill the + # the setfield_gc + self.optimize_loop(ops, 'Not, Not, Not, Not', ops) + + def test_duplicate_setfield_4(self): + ops = """ + [p1, i1, i2, p3] + setfield_gc(p1, i1, descr=valuedescr) + # + # some operations on which the above setfield_gc cannot have effect + i3 = getarrayitem_gc_pure(p3, 1, descr=arraydescr) + i4 = getarrayitem_gc(p3, i3, descr=arraydescr) + i5 = int_add(i3, i4) + setarrayitem_gc(p3, 0, i5, descr=arraydescr) + setfield_gc(p1, i4, descr=nextdescr) + # + setfield_gc(p1, i2, descr=valuedescr) + jump(p1, i1, i2, p3) + """ + expected = """ + [p1, i1, i2, p3] + # + i3 = getarrayitem_gc_pure(p3, 1, descr=arraydescr) + i4 = getarrayitem_gc(p3, i3, descr=arraydescr) + i5 = int_add(i3, i4) + setarrayitem_gc(p3, 0, i5, descr=arraydescr) + # + setfield_gc(p1, i2, descr=valuedescr) + setfield_gc(p1, i4, descr=nextdescr) + jump(p1, i1, i2, p3) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', expected) + + def test_duplicate_setfield_sideeffects_1(self): + ops = """ + [p1, i1, i2] + setfield_gc(p1, i1, descr=valuedescr) + escape() + setfield_gc(p1, i2, descr=valuedescr) + jump(p1, i1, i2) + """ + self.optimize_loop(ops, 'Not, Not, Not', ops) + + def test_duplicate_setfield_residual_guard_1(self): + ops = """ + [p1, i1, i2, i3] + setfield_gc(p1, i1, descr=valuedescr) + guard_true(i3) [] + i4 = int_neg(i2) + setfield_gc(p1, i2, descr=valuedescr) + jump(p1, i1, i2, i4) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', ops) + + def test_duplicate_setfield_residual_guard_2(self): + # the difference with the previous test is that the field value is + # a virtual, which we try hard to keep virtual + ops = """ + [p1, i2, i3] + p2 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, p2, descr=nextdescr) + guard_true(i3) [] + i4 = int_neg(i2) + setfield_gc(p1, NULL, descr=nextdescr) + jump(p1, i2, i4) + """ + expected = """ + [p1, i2, i3] + guard_true(i3) [p1] + i4 = int_neg(i2) + setfield_gc(p1, NULL, descr=nextdescr) + jump(p1, i2, i4) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + + def test_duplicate_setfield_residual_guard_3(self): + ops = """ + [p1, i2, i3] + p2 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p2, i2, descr=valuedescr) + setfield_gc(p1, p2, descr=nextdescr) + guard_true(i3) [] + i4 = int_neg(i2) + setfield_gc(p1, NULL, descr=nextdescr) + jump(p1, i2, i4) + """ + expected = """ + [p1, i2, i3] + guard_true(i3) [p1, i2] + i4 = int_neg(i2) + setfield_gc(p1, NULL, descr=nextdescr) + jump(p1, i2, i4) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + + def test_duplicate_setfield_residual_guard_4(self): + # test that the setfield_gc does not end up between int_eq and + # the following guard_true + ops = """ + [p1, i1, i2, i3] + setfield_gc(p1, i1, descr=valuedescr) + i5 = int_eq(i3, 5) + guard_true(i5) [] + i4 = int_neg(i2) + setfield_gc(p1, i2, descr=valuedescr) + jump(p1, i1, i2, i4) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', ops) + + def test_duplicate_setfield_aliasing(self): + # a case where aliasing issues (and not enough cleverness) mean + # that we fail to remove any setfield_gc + ops = """ + [p1, p2, i1, i2, i3] + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(p2, i2, descr=valuedescr) + setfield_gc(p1, i3, descr=valuedescr) + jump(p1, p2, i1, i2, i3) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not, Not', ops) + + def test_duplicate_setfield_guard_value_const(self): + ops = """ + [p1, i1, i2] + guard_value(p1, ConstPtr(myptr)) [] + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(ConstPtr(myptr), i2, descr=valuedescr) + jump(p1, i1, i2) + """ + expected = """ + [i1, i2] + setfield_gc(ConstPtr(myptr), i2, descr=valuedescr) + jump(i1, i2) + """ + self.optimize_loop(ops, 'Constant(myptr), Not, Not', expected) + def test_duplicate_getarrayitem_1(self): ops = """ [p1] @@ -1634,6 +1813,14 @@ tag = ('virtual', self.namespace[match.group(2)]) virtuals[pvar] = (tag, None, fieldstext) # + r2 = re.compile(r"([\w\d()]+)[.](\w+)\s*=\s*([\w\d()]+)") + pendingfields = [] + for match in r2.finditer(text): + pvar = match.group(1) + pfieldname = match.group(2) + pfieldvar = match.group(3) + pendingfields.append((pvar, pfieldname, pfieldvar)) + # def _variables_equal(box, varname, strict): if varname not in virtuals: if strict: @@ -1655,11 +1842,21 @@ else: virtuals[varname] = tag, box, fieldstext # - basetext = text[:ends[0]] + basetext = text.splitlines()[0] varnames = [s.strip() for s in basetext.split(',')] + if varnames == ['']: + varnames = [] assert len(boxes) == len(varnames) for box, varname in zip(boxes, varnames): _variables_equal(box, varname, strict=True) + for pvar, pfieldname, pfieldvar in pendingfields: + box = oparse.getvar(pvar) + fielddescr = self.namespace[pfieldname.strip()] + fieldbox = executor.execute(self.cpu, + rop.GETFIELD_GC, + fielddescr, + box) + _variables_equal(fieldbox, pfieldvar, strict=True) # for match in parts: pvar = match.group(1) @@ -1918,6 +2115,57 @@ where p7v is a node_vtable, valuedescr=iv ''') + def test_expand_fail_lazy_setfield_1(self): + self.make_fail_descr() + ops = """ + [p1, i2, i3] + p2 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p2, i2, descr=valuedescr) + setfield_gc(p1, p2, descr=nextdescr) + guard_true(i3, descr=fdescr) [] + i4 = int_neg(i2) + setfield_gc(p1, NULL, descr=nextdescr) + jump(p1, i2, i4) + """ + expected = """ + [p1, i2, i3] + guard_true(i3, descr=fdescr) [p1, i2] + i4 = int_neg(i2) + setfield_gc(p1, NULL, descr=nextdescr) + jump(p1, i2, i4) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + self.loop.inputargs[0].value = self.nodebox.value + self.check_expanded_fail_descr(''' + p1.nextdescr = p2 + where p2 is a node_vtable, valuedescr=i2 + ''') + + def test_expand_fail_lazy_setfield_2(self): + self.make_fail_descr() + ops = """ + [i2, i3] + p2 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p2, i2, descr=valuedescr) + setfield_gc(ConstPtr(myptr), p2, descr=nextdescr) + guard_true(i3, descr=fdescr) [] + i4 = int_neg(i2) + setfield_gc(ConstPtr(myptr), NULL, descr=nextdescr) + jump(i2, i4) + """ + expected = """ + [i2, i3] + guard_true(i3, descr=fdescr) [i2] + i4 = int_neg(i2) + setfield_gc(ConstPtr(myptr), NULL, descr=nextdescr) + jump(i2, i4) + """ + self.optimize_loop(ops, 'Not, Not', expected) + self.check_expanded_fail_descr(''' + ConstPtr(myptr).nextdescr = p2 + where p2 is a node_vtable, valuedescr=i2 + ''') + class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): @@ -2031,6 +2279,58 @@ """ self.optimize_loop(ops, 'Not, Not, Not', expected) + def test_residual_call_invalidates_some_read_caches_1(self): + ops = """ + [p1, i1, p2, i2] + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(p2, i2, descr=adescr) + i3 = call(i1, descr=readadescr) + setfield_gc(p1, i3, descr=valuedescr) + setfield_gc(p2, i3, descr=adescr) + jump(p1, i1, p2, i2) + """ + expected = """ + [p1, i1, p2, i2] + setfield_gc(p2, i2, descr=adescr) + i3 = call(i1, descr=readadescr) + setfield_gc(p1, i3, descr=valuedescr) + setfield_gc(p2, i3, descr=adescr) + jump(p1, i1, p2, i2) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', expected) + + def test_residual_call_invalidates_some_read_caches_2(self): + ops = """ + [p1, i1, p2, i2] + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(p2, i2, descr=adescr) + i3 = call(i1, descr=writeadescr) + setfield_gc(p1, i3, descr=valuedescr) + setfield_gc(p2, i3, descr=adescr) + jump(p1, i1, p2, i2) + """ + expected = """ + [p1, i1, p2, i2] + setfield_gc(p2, i2, descr=adescr) + i3 = call(i1, descr=writeadescr) + setfield_gc(p1, i3, descr=valuedescr) + setfield_gc(p2, i3, descr=adescr) + jump(p1, i1, p2, i2) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', expected) + + def test_residual_call_invalidates_some_read_caches_3(self): + ops = """ + [p1, i1, p2, i2] + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(p2, i2, descr=adescr) + i3 = call(i1, descr=plaincalldescr) + setfield_gc(p1, i3, descr=valuedescr) + setfield_gc(p2, i3, descr=adescr) + jump(p1, i1, p2, i2) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', ops) + class TestOOtype(BaseTestOptimizeOpt, OOtypeMixin): Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Tue Dec 29 10:03:14 2009 @@ -12,6 +12,8 @@ rd_frame_info_list = None rd_numb = None rd_consts = [] + rd_virtuals = None + rd_pendingfields = None def test_tag(): assert tag(3, 1) == rffi.r_short(3<<2|1) @@ -40,6 +42,12 @@ assert not tagged_list_eq([tag(1, TAGBOX)], [tag(-2, TAGBOX)]) assert not tagged_list_eq([tag(1, TAGBOX), tag(-2, TAGBOX)], [tag(1, TAGBOX)]) +def test_vinfo(): + v1 = AbstractVirtualInfo() + v1.set_content([1, 2, 4]) + assert v1.equals([1, 2, 4]) + assert not v1.equals([1, 2, 6]) + class MyMetaInterp: _already_allocated_resume_virtuals = None @@ -80,7 +88,6 @@ tag(0, TAGBOX), tag(1, TAGBOX)]) storage.rd_numb = numb - storage.rd_virtuals = None b1s, b2s, b3s = [BoxInt(), BoxPtr(), BoxInt()] assert b1s != b3s @@ -103,7 +110,6 @@ tag(0, TAGBOX), tag(1, TAGBOX)]) storage.rd_numb = numb - storage.rd_virtuals = None b1s, b2s, b3s = [BoxInt(), BoxPtr(), BoxInt()] assert b1s != b3s reader = ResumeDataReader(storage, [b1s, b2s, b3s], MyMetaInterp()) @@ -125,6 +131,7 @@ rd_virtuals = [FakeVinfo(), None] rd_numb = [] rd_consts = [] + rd_pendingfields = None class FakeMetainterp(object): _already_allocated_resume_virtuals = None cpu = None @@ -960,6 +967,46 @@ assert ptr.a == 111 assert ptr.b == lltype.nullptr(LLtypeMixin.NODE) + +def test_virtual_adder_pending_fields(): + b2s, b4s = [BoxPtr(), BoxPtr()] + storage = Storage() + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) + modifier = ResumeDataVirtualAdder(storage, memo) + modifier.liveboxes_from_env = {} + modifier.liveboxes = {} + modifier.vfieldboxes = {} + + v2 = OptValue(b2s) + v4 = OptValue(b4s) + modifier.register_box(b2s) + modifier.register_box(b4s) + + values = {b4s: v4, b2s: v2} + liveboxes = [] + modifier._number_virtuals(liveboxes, values, 0) + assert liveboxes == [b2s, b4s] + modifier._add_pending_fields([(LLtypeMixin.nextdescr, b2s, b4s)]) + storage.rd_consts = memo.consts[:] + storage.rd_numb = None + # resume + demo55.next = lltype.nullptr(LLtypeMixin.NODE) + b2t = BoxPtr(demo55o) + b4t = BoxPtr(demo66o) + newboxes = _resume_remap(liveboxes, [b2s, b4s], b2t, b4t) + + metainterp = MyMetaInterp() + reader = ResumeDataReader(storage, newboxes, metainterp) + assert reader.virtuals is None + trace = metainterp.trace + b2set = (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.nextdescr) + expected = [b2set] + + for x, y in zip(expected, trace): + assert x == y + assert demo55.next == demo66 + + def test_invalidation_needed(): class options: failargs_limit = 10 Modified: pypy/trunk/pypy/rpython/rptr.py ============================================================================== --- pypy/trunk/pypy/rpython/rptr.py (original) +++ pypy/trunk/pypy/rpython/rptr.py Tue Dec 29 10:03:14 2009 @@ -39,6 +39,14 @@ attr = hop.args_s[1].const if isinstance(hop.s_result, annmodel.SomeLLADTMeth): return hop.inputarg(hop.r_result, arg=0) + try: + self.lowleveltype._example()._lookup_adtmeth(attr) + except AttributeError: + pass + else: + assert hop.s_result.is_constant() + return hop.inputconst(hop.r_result, hop.s_result.const) + assert attr in self.lowleveltype.TO._flds # check that the field exists FIELD_TYPE = getattr(self.lowleveltype.TO, attr) if isinstance(FIELD_TYPE, lltype.ContainerType): if (attr, FIELD_TYPE) == self.lowleveltype.TO._first_struct(): Modified: pypy/trunk/pypy/rpython/test/test_rptr.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rptr.py (original) +++ pypy/trunk/pypy/rpython/test/test_rptr.py Tue Dec 29 10:03:14 2009 @@ -337,3 +337,13 @@ return f([1]) s, t = ll_rtype(lltest, []) assert s.is_constant() == False + +def test_staticadtmeths(): + ll_func = staticAdtMethod(lambda x: x + 42) + S = GcStruct('S', adtmeths={'ll_func': ll_func}) + def f(): + return malloc(S).ll_func(5) + s, t = ll_rtype(f, []) + graphf = t.graphs[0] + for op in graphf.startblock.operations: + assert op.opname != 'getfield' Modified: pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py (original) +++ pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py Tue Dec 29 10:03:14 2009 @@ -4,14 +4,15 @@ from pypy.translator.translator import TranslationContext, graphof from pypy.translator.simplify import get_funcobj from pypy.translator.backendopt.writeanalyze import WriteAnalyzer, top_set +from pypy.translator.backendopt.writeanalyze import ReadWriteAnalyzer from pypy.translator.backendopt.all import backend_optimizations from pypy.conftest import option -class BaseTestCanRaise(object): +class BaseTest(object): type_system = None - + Analyzer = WriteAnalyzer def translate(self, func, sig): t = TranslationContext() @@ -19,7 +20,10 @@ t.buildrtyper(type_system=self.type_system).specialize() if option.view: t.view() - return t, WriteAnalyzer(t) + return t, self.Analyzer(t) + + +class BaseTestWriteAnalyze(BaseTest): def test_writes_simple(self): def g(x): @@ -146,7 +150,7 @@ assert not result -class TestLLtype(BaseTestCanRaise): +class TestLLtype(BaseTestWriteAnalyze): type_system = 'lltype' def test_list(self): @@ -205,7 +209,7 @@ assert name.endswith("foobar") -class TestOOtype(BaseTestCanRaise): +class TestOOtype(BaseTestWriteAnalyze): type_system = 'ootype' def test_array(self): @@ -240,3 +244,88 @@ result = wa.analyze(ggraph.startblock.operations[0]) assert result is top_set + + +class TestLLtypeReadWriteAnalyze(BaseTest): + Analyzer = ReadWriteAnalyzer + type_system = 'lltype' + + def test_read_simple(self): + def g(x): + return True + + def f(x): + return g(x - 1) + t, wa = self.translate(f, [int]) + fgraph = graphof(t, f) + result = wa.analyze(fgraph.startblock.operations[0]) + assert not result + + def test_read_really(self): + class A(object): + def __init__(self, y): + self.y = y + def f(self): + self.x = 1 + return self.y + def h(flag): + obj = A(flag) + return obj.f() + + t, wa = self.translate(h, [int]) + hgraph = graphof(t, h) + op_call_f = hgraph.startblock.operations[-1] + + # check that we fished the expected ops + assert op_call_f.opname == "direct_call" + assert get_funcobj(op_call_f.args[0].value)._name == 'A.f' + + result = wa.analyze(op_call_f) + assert len(result) == 2 + result = list(result) + result.sort() + [(struct1, T1, name1), (struct2, T2, name2)] = result + assert struct1 == "readstruct" + assert name1.endswith("y") + assert struct2 == "struct" + assert name2.endswith("x") + assert T1 == T2 + + def test_contains(self): + def g(x, y, z): + l = [x] + return f(l, y, z) + def f(x, y, z): + return y in x + + t, wa = self.translate(g, [int, int, int]) + ggraph = graphof(t, g) + assert ggraph.startblock.operations[-1].opname == 'direct_call' + + result = wa.analyze(ggraph.startblock.operations[-1]) + ARRAYPTR = list(result)[0][1] + assert list(result) == [("readarray", ARRAYPTR)] + assert isinstance(ARRAYPTR.TO, lltype.GcArray) + + def test_adt_method(self): + def ll_callme(n): + return n + ll_callme = lltype.staticAdtMethod(ll_callme) + S = lltype.GcStruct('S', ('x', lltype.Signed), + adtmeths = {'yep': True, + 'callme': ll_callme}) + def g(x, y, z): + p = lltype.malloc(S) + p.x = x + if p.yep: + z *= p.callme(y) + return z + def f(x, y, z): + return g(x, y, z) + + t, wa = self.translate(f, [int, int, int]) + fgraph = graphof(t, f) + assert fgraph.startblock.operations[-1].opname == 'direct_call' + + result = wa.analyze(fgraph.startblock.operations[-1]) + assert list(result) == [("struct", lltype.Ptr(S), "x")] Modified: pypy/trunk/pypy/translator/backendopt/writeanalyze.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/writeanalyze.py (original) +++ pypy/trunk/pypy/translator/backendopt/writeanalyze.py Tue Dec 29 10:03:14 2009 @@ -45,3 +45,15 @@ elif methname in ('ll_getitem_fast', 'll_length'): return self.bottom_result() return graphanalyze.GraphAnalyzer.analyze_external_method(self, op, TYPE, meth) + + +class ReadWriteAnalyzer(WriteAnalyzer): + + def analyze_simple_operation(self, op): + if op.opname == "getfield": + return frozenset([ + ("readstruct", op.args[0].concretetype, op.args[1].value)]) + elif op.opname == "getarrayitem": + return frozenset([ + ("readarray", op.args[0].concretetype)]) + return WriteAnalyzer.analyze_simple_operation(self, op) From arigo at codespeak.net Tue Dec 29 10:03:32 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 10:03:32 +0100 (CET) Subject: [pypy-svn] r70325 - pypy/branch/jit-delayed-write Message-ID: <20091229090332.344B3168012@codespeak.net> Author: arigo Date: Tue Dec 29 10:03:31 2009 New Revision: 70325 Removed: pypy/branch/jit-delayed-write/ Log: Merged branch. From arigo at codespeak.net Tue Dec 29 10:12:41 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 10:12:41 +0100 (CET) Subject: [pypy-svn] r70326 - in pypy/branch/virtual-forcing: . lib-python lib-python/modified-2.5.2/test pypy pypy/config pypy/doc/config pypy/interpreter pypy/interpreter/test pypy/jit/backend/llsupport pypy/jit/backend/llvm/test pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/metainterp pypy/jit/metainterp/test pypy/lib pypy/lib/app_test pypy/module/__builtin__ pypy/module/__builtin__/test pypy/module/__pypy__ pypy/module/_demo pypy/module/_demo/test pypy/module/exceptions pypy/module/imp pypy/module/imp/test pypy/module/oracle pypy/module/thread pypy/module/zipimport pypy/module/zipimport/test pypy/objspace/flow pypy/objspace/std/test pypy/rlib pypy/rpython pypy/rpython/memory/gctransform pypy/rpython/memory/gctransform/test pypy/rpython/test pypy/rpython/tool pypy/tool pypy/translator pypy/translator/backendopt pypy/translator/backendopt/test pypy/translator/c pypy/translator/c/test pypy/translator/platform pypy/translator/platform/test Message-ID: <20091229091241.DA650168021@codespeak.net> Author: arigo Date: Tue Dec 29 10:12:39 2009 New Revision: 70326 Added: pypy/branch/virtual-forcing/lib-python/modified-2.5.2/test/test_runpy.py - copied unchanged from r70325, pypy/trunk/lib-python/modified-2.5.2/test/test_runpy.py pypy/branch/virtual-forcing/pypy/doc/config/objspace.usemodules.imp.txt - copied unchanged from r70325, pypy/trunk/pypy/doc/config/objspace.usemodules.imp.txt pypy/branch/virtual-forcing/pypy/module/_demo/test/ (props changed) - copied from r70325, pypy/trunk/pypy/module/_demo/test/ pypy/branch/virtual-forcing/pypy/module/imp/ (props changed) - copied from r70325, pypy/trunk/pypy/module/imp/ Removed: pypy/branch/virtual-forcing/lib-python/modified-2.5.2/test/test___all__.py pypy/branch/virtual-forcing/lib-python/modified-2.5.2/test/test_importhooks.py pypy/branch/virtual-forcing/pypy/lib/app_test/test_imp_extra.py pypy/branch/virtual-forcing/pypy/lib/imp.py pypy/branch/virtual-forcing/pypy/module/__builtin__/app_misc.py pypy/branch/virtual-forcing/pypy/module/__builtin__/importing.py pypy/branch/virtual-forcing/pypy/module/__builtin__/test/test_import.py pypy/branch/virtual-forcing/pypy/module/thread/importlock.py Modified: pypy/branch/virtual-forcing/LICENSE pypy/branch/virtual-forcing/lib-python/ (props changed) pypy/branch/virtual-forcing/lib-python/modified-2.5.2/test/infinite_reload.py pypy/branch/virtual-forcing/lib-python/modified-2.5.2/test/test_import.py pypy/branch/virtual-forcing/pypy/ (props changed) pypy/branch/virtual-forcing/pypy/config/pypyoption.py pypy/branch/virtual-forcing/pypy/interpreter/baseobjspace.py pypy/branch/virtual-forcing/pypy/interpreter/gateway.py pypy/branch/virtual-forcing/pypy/interpreter/mixedmodule.py pypy/branch/virtual-forcing/pypy/interpreter/module.py pypy/branch/virtual-forcing/pypy/interpreter/test/test_gateway.py pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/gc.py pypy/branch/virtual-forcing/pypy/jit/backend/llvm/test/conftest.py (props changed) pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/test/test_gc_integration.py (props changed) pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py pypy/branch/virtual-forcing/pypy/jit/metainterp/logger.py (props changed) pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_effectinfo.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_resume.py pypy/branch/virtual-forcing/pypy/lib/app_test/test_runpy.py pypy/branch/virtual-forcing/pypy/module/__builtin__/__init__.py pypy/branch/virtual-forcing/pypy/module/__pypy__/__init__.py pypy/branch/virtual-forcing/pypy/module/_demo/__init__.py pypy/branch/virtual-forcing/pypy/module/exceptions/ (props changed) pypy/branch/virtual-forcing/pypy/module/imp/test/ (props changed) pypy/branch/virtual-forcing/pypy/module/oracle/__init__.py pypy/branch/virtual-forcing/pypy/module/oracle/interp_error.py pypy/branch/virtual-forcing/pypy/module/thread/__init__.py pypy/branch/virtual-forcing/pypy/module/zipimport/__init__.py pypy/branch/virtual-forcing/pypy/module/zipimport/interp_zipimport.py pypy/branch/virtual-forcing/pypy/module/zipimport/test/test_zipimport.py pypy/branch/virtual-forcing/pypy/objspace/flow/flowcontext.py pypy/branch/virtual-forcing/pypy/objspace/std/test/test_setobject.py (props changed) pypy/branch/virtual-forcing/pypy/rlib/rgc.py pypy/branch/virtual-forcing/pypy/rpython/memory/gctransform/framework.py pypy/branch/virtual-forcing/pypy/rpython/memory/gctransform/test/test_framework.py pypy/branch/virtual-forcing/pypy/rpython/rptr.py pypy/branch/virtual-forcing/pypy/rpython/test/test_rptr.py pypy/branch/virtual-forcing/pypy/rpython/tool/rffi_platform.py pypy/branch/virtual-forcing/pypy/tool/gcc_cache.py pypy/branch/virtual-forcing/pypy/tool/udir.py pypy/branch/virtual-forcing/pypy/translator/backendopt/test/test_writeanalyze.py pypy/branch/virtual-forcing/pypy/translator/backendopt/writeanalyze.py pypy/branch/virtual-forcing/pypy/translator/c/gc.py pypy/branch/virtual-forcing/pypy/translator/c/test/test_boehm.py pypy/branch/virtual-forcing/pypy/translator/c/test/test_refcount.py (props changed) pypy/branch/virtual-forcing/pypy/translator/c/test/test_stackless.py pypy/branch/virtual-forcing/pypy/translator/driver.py pypy/branch/virtual-forcing/pypy/translator/platform/__init__.py pypy/branch/virtual-forcing/pypy/translator/platform/darwin.py pypy/branch/virtual-forcing/pypy/translator/platform/posix.py pypy/branch/virtual-forcing/pypy/translator/platform/test/test_darwin.py pypy/branch/virtual-forcing/pypy/translator/platform/test/test_platform.py Log: Merge the changes on trunk, from -r70231:70325. Modified: pypy/branch/virtual-forcing/LICENSE ============================================================================== --- pypy/branch/virtual-forcing/LICENSE (original) +++ pypy/branch/virtual-forcing/LICENSE Tue Dec 29 10:12:39 2009 @@ -86,6 +86,7 @@ Guenter Jantzen Dinu Gherman Georg Brandl + Benjamin Peterson Ben Young Nicolas Chauvat Jean-Paul Calderone Modified: pypy/branch/virtual-forcing/lib-python/modified-2.5.2/test/infinite_reload.py ============================================================================== --- pypy/branch/virtual-forcing/lib-python/modified-2.5.2/test/infinite_reload.py (original) +++ pypy/branch/virtual-forcing/lib-python/modified-2.5.2/test/infinite_reload.py Tue Dec 29 10:12:39 2009 @@ -3,8 +3,5 @@ # reload()ing. This module is imported by test_import.py:test_infinite_reload # to make sure this doesn't happen any more. -print 1 import infinite_reload -print 2 reload(infinite_reload) -print 3 Modified: pypy/branch/virtual-forcing/lib-python/modified-2.5.2/test/test_import.py ============================================================================== --- pypy/branch/virtual-forcing/lib-python/modified-2.5.2/test/test_import.py (original) +++ pypy/branch/virtual-forcing/lib-python/modified-2.5.2/test/test_import.py Tue Dec 29 10:12:39 2009 @@ -1,4 +1,4 @@ -from test.test_support import TESTFN, TestFailed, check_impl_detail +from test.test_support import TESTFN, TestFailed import os import random @@ -56,6 +56,11 @@ os.unlink(source) try: + #--- the block below is to check that "reload" manages to import + #--- the .pyc file alone. We don't support it in PyPy in the default + #--- configuration. + return + try: reload(mod) except ImportError, err: @@ -238,5 +243,4 @@ finally: sys.path.pop(0) -if check_impl_detail(): - test_infinite_reload() +test_infinite_reload() Modified: pypy/branch/virtual-forcing/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/config/pypyoption.py (original) +++ pypy/branch/virtual-forcing/pypy/config/pypyoption.py Tue Dec 29 10:12:39 2009 @@ -16,7 +16,7 @@ default_modules = essential_modules.copy() default_modules.update(dict.fromkeys( - ["_codecs", "gc", "_weakref", "marshal", "errno", + ["_codecs", "gc", "_weakref", "marshal", "errno", "imp", "math", "_sre", "_pickle_support", "operator", "parser", "symbol", "token", "_ast", "_random", "__pypy__", "_testing"])) Modified: pypy/branch/virtual-forcing/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/virtual-forcing/pypy/interpreter/baseobjspace.py Tue Dec 29 10:12:39 2009 @@ -239,6 +239,9 @@ config = get_pypy_config(translating=False) self.config = config + self.builtin_modules = {} + self.reloading_modules = {} + # import extra modules for side-effects import pypy.interpreter.nestedscope # register *_DEREF bytecodes @@ -266,16 +269,22 @@ def startup(self): # To be called before using the space - # Initialize all builtin modules + # Initialize already imported builtin modules from pypy.interpreter.module import Module + w_modules = self.sys.get('modules') for w_modname in self.unpackiterable( self.sys.get('builtin_module_names')): + try: + w_mod = self.getitem(w_modules, w_modname) + except OperationError, e: + if e.match(self, self.w_KeyError): + continue + raise modname = self.str_w(w_modname) - mod = self.interpclass_w(self.getbuiltinmodule(modname)) + mod = self.interpclass_w(w_mod) if isinstance(mod, Module): - import time self.timer.start("startup " + modname) - mod.startup(self) + mod.init(self) self.timer.stop("startup " + modname) def finish(self): @@ -283,11 +292,9 @@ if w_exitfunc is not None: self.call_function(w_exitfunc) from pypy.interpreter.module import Module - for w_modname in self.unpackiterable( - self.sys.get('builtin_module_names')): - modname = self.str_w(w_modname) - mod = self.interpclass_w(self.getbuiltinmodule(modname)) - if isinstance(mod, Module): + for w_mod in self.builtin_modules.values(): + mod = self.interpclass_w(w_mod) + if isinstance(mod, Module) and mod.startup_called: mod.shutdown(self) if self.config.objspace.std.withdictmeasurement: from pypy.objspace.std.dictmultiobject import report @@ -339,14 +346,42 @@ w_name = self.wrap(name) w_mod = self.wrap(Module(self, w_name)) - w_modules = self.sys.get('modules') - self.setitem(w_modules, w_name, w_mod) + self.builtin_modules[name] = w_mod return name - def getbuiltinmodule(self, name): + def getbuiltinmodule(self, name, force_init=False): w_name = self.wrap(name) w_modules = self.sys.get('modules') - return self.getitem(w_modules, w_name) + try: + w_mod = self.getitem(w_modules, w_name) + except OperationError, e: + if not e.match(self, self.w_KeyError): + raise + else: + if not force_init: + return w_mod + + # If the module is a builtin but not yet imported, + # retrieve it and initialize it + try: + w_mod = self.builtin_modules[name] + except KeyError: + raise OperationError( + self.w_SystemError, + self.wrap("getbuiltinmodule() called " + "with non-builtin module %s" % name)) + else: + # Add the module to sys.modules + self.setitem(w_modules, w_name, w_mod) + + # And initialize it + from pypy.interpreter.module import Module + mod = self.interpclass_w(w_mod) + if isinstance(mod, Module): + self.timer.start("startup " + name) + mod.init(self) + self.timer.stop("startup " + name) + return w_mod def get_builtinmodule_to_install(self): """NOT_RPYTHON""" @@ -390,26 +425,27 @@ "NOT_RPYTHON: only for initializing the space." from pypy.module.exceptions import Module - w_name_exceptions = self.wrap('exceptions') - self.exceptions_module = Module(self, w_name_exceptions) + w_name = self.wrap('exceptions') + self.exceptions_module = Module(self, w_name) + self.builtin_modules['exceptions'] = self.wrap(self.exceptions_module) from pypy.module.sys import Module w_name = self.wrap('sys') self.sys = Module(self, w_name) - w_modules = self.sys.get('modules') - self.setitem(w_modules, w_name, self.wrap(self.sys)) + self.builtin_modules['sys'] = self.wrap(self.sys) - self.setitem(w_modules, w_name_exceptions, - self.wrap(self.exceptions_module)) + from pypy.module.imp import Module + w_name = self.wrap('imp') + self.builtin_modules['imp'] = self.wrap(Module(self, w_name)) from pypy.module.__builtin__ import Module w_name = self.wrap('__builtin__') self.builtin = Module(self, w_name) w_builtin = self.wrap(self.builtin) - self.setitem(w_modules, w_name, w_builtin) + self.builtin_modules['__builtin__'] = self.wrap(w_builtin) self.setitem(self.builtin.w_dict, self.wrap('__builtins__'), w_builtin) - bootstrap_modules = ['sys', '__builtin__', 'exceptions'] + bootstrap_modules = ['sys', 'imp', '__builtin__', 'exceptions'] installed_builtin_modules = bootstrap_modules[:] self.export_builtin_exceptions() @@ -480,12 +516,11 @@ def setup_builtin_modules(self): "NOT_RPYTHON: only for initializing the space." - from pypy.interpreter.module import Module - for w_modname in self.unpackiterable(self.sys.get('builtin_module_names')): - modname = self.unwrap(w_modname) - mod = self.getbuiltinmodule(modname) - if isinstance(mod, Module): - mod.setup_after_space_initialization() + self.getbuiltinmodule('sys') + self.getbuiltinmodule('imp') + self.getbuiltinmodule('__builtin__') + for mod in self.builtin_modules.values(): + mod.setup_after_space_initialization() def initialize(self): """NOT_RPYTHON: Abstract method that should put some minimal Modified: pypy/branch/virtual-forcing/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/interpreter/gateway.py (original) +++ pypy/branch/virtual-forcing/pypy/interpreter/gateway.py Tue Dec 29 10:12:39 2009 @@ -386,6 +386,15 @@ else: return typ.__name__ + '_w' + +def unwrap_spec(*spec): + """A decorator which attaches the unwrap_spec attribute.""" + def decorator(func): + func.unwrap_spec = spec + return func + return decorator + + class BuiltinCode(eval.Code): "The code object implementing a built-in (interpreter-level) hook." _immutable_ = True Modified: pypy/branch/virtual-forcing/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/interpreter/mixedmodule.py (original) +++ pypy/branch/virtual-forcing/pypy/interpreter/mixedmodule.py Tue Dec 29 10:12:39 2009 @@ -13,7 +13,8 @@ applevel_name = None expose__file__attribute = True - + w_initialdict = None + def __init__(self, space, w_name): """ NOT_RPYTHON """ Module.__init__(self, space, w_name) @@ -21,6 +22,13 @@ self.__class__.buildloaders() self.loaders = self.loaders.copy() # copy from the class to the inst + def init(self, space): + """This is called each time the module is imported or reloaded + """ + if self.w_initialdict is not None: + space.call_method(self.w_dict, 'update', self.w_initialdict) + Module.init(self, space) + def get_applevel_name(cls): """ NOT_RPYTHON """ if cls.applevel_name is not None: @@ -82,11 +90,13 @@ for name in self.loaders: w_value = self.get(name) space.setitem(self.w_dict, space.new_interned_str(name), w_value) - self.lazy = False + self.lazy = False + self.w_initialdict = space.call_method(self.w_dict, 'items') return self.w_dict def _freeze_(self): self.getdict() + self.startup_called = False # hint for the annotator: Modules can hold state, so they are # not constant return False Modified: pypy/branch/virtual-forcing/pypy/interpreter/module.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/interpreter/module.py (original) +++ pypy/branch/virtual-forcing/pypy/interpreter/module.py Tue Dec 29 10:12:39 2009 @@ -15,22 +15,30 @@ self.w_dict = w_dict self.w_name = w_name if w_name is not None: - space.setitem(w_dict, space.new_interned_str('__name__'), w_name) + space.setitem(w_dict, space.new_interned_str('__name__'), w_name) + self.startup_called = False def setup_after_space_initialization(self): """NOT_RPYTHON: to allow built-in modules to do some more setup after the space is fully initialized.""" + def init(self, space): + """This is called each time the module is imported or reloaded + """ + if not self.startup_called: + self.startup_called = True + self.startup(space) + def startup(self, space): - """This is called at runtime before the space gets uses to allow - the module to do initialization at runtime. + """This is called at runtime on import to allow the module to + do initialization when it is imported for the first time. """ def shutdown(self, space): """This is called when the space is shut down, just after - sys.exitfunc(). + sys.exitfunc(), if the module has been imported. """ - + def getdict(self): return self.w_dict Modified: pypy/branch/virtual-forcing/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/interpreter/test/test_gateway.py (original) +++ pypy/branch/virtual-forcing/pypy/interpreter/test/test_gateway.py Tue Dec 29 10:12:39 2009 @@ -520,6 +520,16 @@ w_res = space.call_obj_args(w_g, w_self, args3) assert space.is_true(space.eq(w_res, space.wrap(('g', 'self', 3)))) + def test_unwrap_spec_decorator(self): + space = self.space + @gateway.unwrap_spec(gateway.ObjSpace, gateway.W_Root, int) + def g(space, w_thing, i): + return space.newtuple([w_thing, space.wrap(i)]) + w_g = space.wrap(gateway.interp2app_temp(g)) + args = argument.Arguments(space, [space.wrap(-1), space.wrap(0)]) + w_res = space.call_args(w_g, args) + assert space.eq_w(w_res, space.wrap((-1, 0))) + class TestPassThroughArguments: Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/gc.py Tue Dec 29 10:12:39 2009 @@ -41,7 +41,7 @@ GcLLDescription.__init__(self, gcdescr, translator) # grab a pointer to the Boehm 'malloc' function from pypy.rpython.tool import rffi_platform - compilation_info = rffi_platform.check_boehm() + compilation_info = rffi_platform.configure_boehm() # Versions 6.x of libgc needs to use GC_local_malloc(). # Versions 7.x of libgc removed this function; GC_malloc() has Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Tue Dec 29 10:12:39 2009 @@ -1,5 +1,5 @@ -import py, sys, random +import py, sys, random, os from pypy.jit.metainterp.history import (AbstractFailDescr, BasicFailDescr, BoxInt, Box, BoxPtr, @@ -465,6 +465,29 @@ 'float', descr=calldescr) assert abs(res.value - 4.6) < 0.0001 + def test_call_stack_alignment(self): + # test stack alignment issues, notably for Mac OS/X + + def func_ints(*ints): + s = str(ints) + '\n' + os.write(1, s) # don't remove -- crash if the stack is misaligned + return sum(ints) + + for nb_args in range(0, 35): + cpu = self.cpu + TP = lltype.Signed + # + FPTR = self.Ptr(self.FuncType([TP] * nb_args, TP)) + func_ptr = llhelper(FPTR, func_ints) + FUNC = deref(FPTR) + calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + funcbox = self.get_funcbox(cpu, func_ptr) + args = [280-24*i for i in range(nb_args)] + res = self.execute_operation(rop.CALL, + [funcbox] + map(BoxInt, args), + 'int', descr=calldescr) + assert res.value == sum(args) + def test_field_basic(self): t_box, T_box = self.alloc_instance(self.T) fielddescr = self.cpu.fielddescrof(self.S, 'value') Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py Tue Dec 29 10:12:39 2009 @@ -17,6 +17,7 @@ from pypy.jit.metainterp.resoperation import rop from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print +from pypy.rlib import rgc # our calling convention - we pass first 6 args in registers # and the rest stays on the stack @@ -27,7 +28,6 @@ else: CALL_ALIGN = 1 - def align_stack_words(words): return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) @@ -225,13 +225,14 @@ def _patch_stackadjust(self, adr_lea, reserved_depth): # patch stack adjustment LEA - # possibly align, e.g. for Mac OS X mc = codebuf.InMemoryCodeBuilder(adr_lea, adr_lea + 4) # Compute the correct offset for the instruction LEA ESP, [EBP-4*words]. # Given that [EBP] is where we saved EBP, i.e. in the last word # of our fixed frame, then the 'words' value is: words = (FRAME_FIXED_SIZE - 1) + reserved_depth - mc.write(packimm32(-WORD * words)) + # align, e.g. for Mac OS X + aligned_words = align_stack_words(words+2)-2 # 2 = EIP+EBP + mc.write(packimm32(-WORD * aligned_words)) mc.done() def _assemble_bootstrap_code(self, inputargs, arglocs): @@ -403,14 +404,6 @@ return self.implement_guard(addr, getattr(self.mc, name)) return genop_cmp_guard -## XXX redo me -## def align_stack_for_call(self, nargs): -## # xxx do something when we don't use push anymore for calls -## extra_on_stack = align_stack_words(nargs) -## for i in range(extra_on_stack-nargs): -## self.mc.PUSH(imm(0)) --- or just use a single SUB(esp, imm) -## return extra_on_stack - def _emit_call(self, x, arglocs, start=0, tmp=eax): p = 0 n = len(arglocs) @@ -957,6 +950,7 @@ boxes.append(box) return boxes + @rgc.no_collect def grab_frame_values(self, bytecode, frame_addr, allregisters): # no malloc allowed here!! self.fail_ebp = allregisters[16 + ebp.op] @@ -1021,6 +1015,7 @@ def setup_failure_recovery(self): + @rgc.no_collect def failure_recovery_func(registers): # 'registers' is a pointer to a structure containing the # original value of the registers, optionally the original Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py Tue Dec 29 10:12:39 2009 @@ -11,7 +11,7 @@ from pypy.tool.udir import udir from pypy.translator.simplify import get_funcobj, get_functype from pypy.translator.backendopt.canraise import RaiseAnalyzer -from pypy.translator.backendopt.writeanalyze import WriteAnalyzer +from pypy.translator.backendopt.writeanalyze import ReadWriteAnalyzer from pypy.jit.metainterp.typesystem import deref, arrayItem, fieldType from pypy.jit.metainterp.effectinfo import effectinfo_from_writeanalyze from pypy.jit.metainterp.effectinfo import VirtualizableAnalyzer @@ -185,7 +185,7 @@ self.portal_runner_ptr = portal_runner_ptr translator = self.rtyper.annotator.translator self.raise_analyzer = RaiseAnalyzer(translator) - self.write_analyzer = WriteAnalyzer(translator) + self.readwrite_analyzer = ReadWriteAnalyzer(translator) self.virtualizable_analyzer = VirtualizableAnalyzer(translator) def make_portal_bytecode(self, graph): @@ -326,7 +326,7 @@ # ok if consider_effects_of is not None: effectinfo = effectinfo_from_writeanalyze( - self.write_analyzer.analyze(consider_effects_of), + self.readwrite_analyzer.analyze(consider_effects_of), self.cpu, self.virtualizable_analyzer.analyze(consider_effects_of)) calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT, effectinfo) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py Tue Dec 29 10:12:39 2009 @@ -7,13 +7,17 @@ class EffectInfo(object): _cache = {} - def __new__(cls, write_descrs_fields, write_descrs_arrays, + def __new__(cls, readonly_descrs_fields, + write_descrs_fields, write_descrs_arrays, forces_virtual_or_virtualizable=False): - key = (frozenset(write_descrs_fields), frozenset(write_descrs_arrays), + key = (frozenset(readonly_descrs_fields), + frozenset(write_descrs_fields), + frozenset(write_descrs_arrays), forces_virtual_or_virtualizable) if key in cls._cache: return cls._cache[key] result = object.__new__(cls) + result.readonly_descrs_fields = readonly_descrs_fields result.write_descrs_fields = write_descrs_fields result.write_descrs_arrays = write_descrs_arrays result.forces_virtual_or_virtualizable= forces_virtual_or_virtualizable @@ -25,26 +29,39 @@ from pypy.translator.backendopt.writeanalyze import top_set if effects is top_set: return None + readonly_descrs_fields = [] + # readonly_descrs_arrays = [] --- not enabled for now write_descrs_fields = [] write_descrs_arrays = [] + + def add_struct(descrs_fields, (_, T, fieldname)): + T = deref(T) + if consider_struct(T, fieldname): + descr = cpu.fielddescrof(T, fieldname) + descrs_fields.append(descr) + + def add_array(descrs_arrays, (_, T)): + ARRAY = deref(T) + if consider_array(ARRAY): + descr = cpu.arraydescrof(ARRAY) + descrs_arrays.append(descr) + for tup in effects: if tup[0] == "struct": - _, T, fieldname = tup - T = deref(T) - if not consider_struct(T, fieldname): - continue - descr = cpu.fielddescrof(T, fieldname) - write_descrs_fields.append(descr) + add_struct(write_descrs_fields, tup) + elif tup[0] == "readstruct": + tupw = ("struct",) + tup[1:] + if tupw not in effects: + add_struct(readonly_descrs_fields, tup) elif tup[0] == "array": - _, T = tup - ARRAY = deref(T) - if not consider_array(ARRAY): - continue - descr = cpu.arraydescrof(ARRAY) - write_descrs_arrays.append(descr) + add_array(write_descrs_arrays, tup) + elif tup[0] == "readarray": + pass else: assert 0 - return EffectInfo(write_descrs_fields, write_descrs_arrays, + return EffectInfo(readonly_descrs_fields, + write_descrs_fields, + write_descrs_arrays, forces_virtual_or_virtualizable) def consider_struct(TYPE, fieldname): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py Tue Dec 29 10:12:39 2009 @@ -161,18 +161,18 @@ return self.box def make_virtual_info(self, modifier, fieldnums): - vinfo = self._cached_vinfo - if vinfo is not None and resume.tagged_list_eq( - vinfo.fieldnums, fieldnums): + vinfo = self._cached_vinfo + if vinfo is not None and vinfo.equals(fieldnums): return vinfo vinfo = self._make_virtual(modifier) - vinfo.fieldnums = fieldnums + vinfo.set_content(fieldnums) self._cached_vinfo = vinfo return vinfo def _make_virtual(self, modifier): raise NotImplementedError("abstract base") + def get_fielddescrlist_cache(cpu): if not hasattr(cpu, '_optimizeopt_fielddescrlist_cache'): result = descrlist_dict() @@ -234,7 +234,6 @@ def get_args_for_fail(self, modifier): if self.box is None and not modifier.already_seen_virtual(self.keybox): - # modifier.already_seen_virtual() # checks for recursion: it is False unless # we have already seen the very same keybox lst = self._get_field_descr_list() @@ -302,11 +301,9 @@ def get_args_for_fail(self, modifier): if self.box is None and not modifier.already_seen_virtual(self.keybox): - # modifier.already_seen_virtual() # checks for recursion: it is False unless # we have already seen the very same keybox itemboxes = [] - const = self.optimizer.new_const_item(self.arraydescr) for itemvalue in self._items: itemboxes.append(itemvalue.get_key_box()) modifier.register_virtual_fields(self.keybox, itemboxes) @@ -515,6 +512,9 @@ def emit_operation(self, op, must_clone=True): self.heap_op_optimizer.emitting_operation(op) + self._emit_operation(op, must_clone) + + def _emit_operation(self, op, must_clone=True): for i in range(len(op.args)): arg = op.args[i] if arg in self.values: @@ -535,10 +535,11 @@ self.newoperations.append(op) def store_final_boxes_in_guard(self, op): + pendingfields = self.heap_op_optimizer.force_lazy_setfields_for_guard() descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo) - newboxes = modifier.finish(self.values) + newboxes = modifier.finish(self.values, pendingfields) if len(newboxes) > self.metainterp_sd.options.failargs_limit: # XXX be careful here raise compile.GiveUp descr.store_final_boxes(op, newboxes) @@ -883,11 +884,13 @@ class HeapOpOptimizer(object): def __init__(self, optimizer): self.optimizer = optimizer - # cached OptValues for each field descr + # cached fields: {descr: {OptValue_instance: OptValue_fieldvalue}} self.cached_fields = {} - - # cached OptValues for each field descr + # cached array items: {descr: CachedArrayItems} self.cached_arrayitems = {} + # lazily written setfields (at most one per descr): {descr: op} + self.lazy_setfields = {} + self.lazy_setfields_descrs = [] # keys (at least) of previous dict def clean_caches(self): self.cached_fields.clear() @@ -895,6 +898,9 @@ def cache_field_value(self, descr, value, fieldvalue, write=False): if write: + # when seeing a setfield, we have to clear the cache for the same + # field on any other structure, just in case they are aliasing + # each other d = self.cached_fields[descr] = {} else: d = self.cached_fields.setdefault(descr, {}) @@ -952,8 +958,6 @@ return None def emitting_operation(self, op): - if op.is_always_pure(): - return if op.has_no_side_effect(): return if op.is_ovf(): @@ -965,10 +969,16 @@ opnum == rop.SETARRAYITEM_GC or opnum == rop.DEBUG_MERGE_POINT): return - if opnum == rop.CALL: + if (opnum == rop.CALL or + opnum == rop.CALL_MAY_FORCE): effectinfo = op.descr.get_extra_info() if effectinfo is not None: + # XXX we can get the wrong complexity here, if the lists + # XXX stored on effectinfo are large + for fielddescr in effectinfo.readonly_descrs_fields: + self.force_lazy_setfield(fielddescr) for fielddescr in effectinfo.write_descrs_fields: + self.force_lazy_setfield(fielddescr) try: del self.cached_fields[fielddescr] except KeyError: @@ -979,9 +989,73 @@ except KeyError: pass return + self.force_all_lazy_setfields() + elif op.is_final() or (not we_are_translated() and + op.opnum < 0): # escape() operations + self.force_all_lazy_setfields() self.clean_caches() + def force_lazy_setfield(self, descr, before_guard=False): + try: + op = self.lazy_setfields[descr] + except KeyError: + return + del self.lazy_setfields[descr] + self.optimizer._emit_operation(op) + # + # hackish: reverse the order of the last two operations if it makes + # sense to avoid a situation like "int_eq/setfield_gc/guard_true", + # which the backend (at least the x86 backend) does not handle well. + newoperations = self.optimizer.newoperations + if before_guard and len(newoperations) >= 2: + lastop = newoperations[-1] + prevop = newoperations[-2] + # - is_comparison() for cases like "int_eq/setfield_gc/guard_true" + # - CALL_MAY_FORCE: "call_may_force/setfield_gc/guard_not_forced" + if ((prevop.is_comparison() or prevop.opnum == rop.CALL_MAY_FORCE) + and prevop.result not in lastop.args): + newoperations[-2] = lastop + newoperations[-1] = prevop + + def force_all_lazy_setfields(self): + if len(self.lazy_setfields_descrs) > 0: + for descr in self.lazy_setfields_descrs: + self.force_lazy_setfield(descr) + del self.lazy_setfields_descrs[:] + + def force_lazy_setfields_for_guard(self): + pendingfields = [] + for descr in self.lazy_setfields_descrs: + try: + op = self.lazy_setfields[descr] + except KeyError: + continue + # the only really interesting case that we need to handle in the + # guards' resume data is that of a virtual object that is stored + # into a field of a non-virtual object. + value = self.optimizer.getvalue(op.args[0]) + assert not value.is_virtual() # it must be a non-virtual + fieldvalue = self.optimizer.getvalue(op.args[1]) + if fieldvalue.is_virtual(): + # this is the case that we leave to resume.py + pendingfields.append((descr, value.box, + fieldvalue.get_key_box())) + else: + self.force_lazy_setfield(descr, before_guard=True) + return pendingfields + + def force_lazy_setfield_if_necessary(self, op, value, write=False): + try: + op1 = self.lazy_setfields[op.descr] + except KeyError: + if write: + self.lazy_setfields_descrs.append(op.descr) + else: + if self.optimizer.getvalue(op1.args[0]) is not value: + self.force_lazy_setfield(op.descr) + def optimize_GETFIELD_GC(self, op, value): + self.force_lazy_setfield_if_necessary(op, value) # check if the field was read from another getfield_gc just before # or has been written to recently fieldvalue = self.read_cached_field(op.descr, value) @@ -996,7 +1070,8 @@ self.cache_field_value(op.descr, value, fieldvalue) def optimize_SETFIELD_GC(self, op, value, fieldvalue): - self.optimizer.emit_operation(op) + self.force_lazy_setfield_if_necessary(op, value, write=True) + self.lazy_setfields[op.descr] = op # remember the result of future reads of the field self.cache_field_value(op.descr, value, fieldvalue, write=True) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py Tue Dec 29 10:12:39 2009 @@ -95,8 +95,8 @@ TAGBOX = 2 TAGVIRTUAL = 3 -UNASSIGNED = tag(-2 ** 12 - 1, TAGBOX) -UNASSIGNEDVIRTUAL = tag(-2 ** 12 - 1, TAGVIRTUAL) +UNASSIGNED = tag(-1<<13, TAGBOX) +UNASSIGNEDVIRTUAL = tag(-1<<13, TAGVIRTUAL) NULLREF = tag(-1, TAGCONST) @@ -197,6 +197,7 @@ return len(self.cached_boxes) def assign_number_to_box(self, box, boxes): + # returns a negative number if box in self.cached_boxes: num = self.cached_boxes[box] boxes[-num-1] = box @@ -210,6 +211,7 @@ return len(self.cached_virtuals) def assign_number_to_virtual(self, box): + # returns a negative number if box in self.cached_virtuals: num = self.cached_virtuals[box] else: @@ -232,8 +234,6 @@ def __init__(self, storage, memo): self.storage = storage self.memo = memo - #self.virtuals = [] - #self.vfieldboxes = [] def make_virtual(self, known_class, fielddescrs): return VirtualInfo(known_class, fielddescrs) @@ -254,8 +254,6 @@ if (isinstance(box, Box) and box not in self.liveboxes_from_env and box not in self.liveboxes): self.liveboxes[box] = UNASSIGNED - return True - return False def _register_boxes(self, boxes): for box in boxes: @@ -270,7 +268,7 @@ _, tagbits = untag(tagged) return tagbits == TAGVIRTUAL - def finish(self, values): + def finish(self, values, pending_setfields=[]): # compute the numbering storage = self.storage numb, liveboxes_from_env, v = self.memo.number(values, @@ -293,16 +291,29 @@ value = values[box] value.get_args_for_fail(self) + for _, box, fieldbox in pending_setfields: + self.register_box(box) + self.register_box(fieldbox) + value = values[fieldbox] + value.get_args_for_fail(self) + self._number_virtuals(liveboxes, values, v) + self._add_pending_fields(pending_setfields) storage.rd_consts = self.memo.consts dump_storage(storage, liveboxes) return liveboxes[:] def _number_virtuals(self, liveboxes, values, num_env_virtuals): + # !! 'liveboxes' is a list that is extend()ed in-place !! memo = self.memo new_liveboxes = [None] * memo.num_cached_boxes() count = 0 + # So far, self.liveboxes should contain 'tagged' values that are + # either UNASSIGNED, UNASSIGNEDVIRTUAL, or a *non-negative* value + # with the TAGVIRTUAL. The following loop removes the UNASSIGNED + # and UNASSIGNEDVIRTUAL entries, and replaces them with real + # negative values. for box, tagged in self.liveboxes.iteritems(): i, tagbits = untag(tagged) if tagbits == TAGBOX: @@ -317,6 +328,8 @@ assert box not in self.liveboxes_from_env index = memo.assign_number_to_virtual(box) self.liveboxes[box] = tag(index, TAGVIRTUAL) + else: + assert i >= 0 new_liveboxes.reverse() liveboxes.extend(new_liveboxes) nholes = len(new_liveboxes) - count @@ -353,6 +366,16 @@ return True return False + def _add_pending_fields(self, pending_setfields): + rd_pendingfields = None + if pending_setfields: + rd_pendingfields = [] + for descr, box, fieldbox in pending_setfields: + num = self._gettagged(box) + fieldnum = self._gettagged(fieldbox) + rd_pendingfields.append((descr, num, fieldnum)) + self.storage.rd_pendingfields = rd_pendingfields + def _gettagged(self, box): if isinstance(box, Const): return self.memo.getconst(box) @@ -361,11 +384,16 @@ return self.liveboxes_from_env[box] return self.liveboxes[box] + class AbstractVirtualInfo(object): def allocate(self, metainterp): raise NotImplementedError def setfields(self, metainterp, box, fn_decode_box): raise NotImplementedError + def equals(self, fieldnums): + return tagged_list_eq(self.fieldnums, fieldnums) + def set_content(self, fieldnums): + self.fieldnums = fieldnums class AbstractVirtualStructInfo(AbstractVirtualInfo): @@ -473,6 +501,7 @@ self.liveboxes = liveboxes self.cpu = metainterp.cpu self._prepare_virtuals(metainterp, storage.rd_virtuals) + self._prepare_pendingfields(metainterp, storage.rd_pendingfields) def _prepare_virtuals(self, metainterp, virtuals): if virtuals: @@ -491,6 +520,16 @@ vinfo.setfields(metainterp, self.virtuals[i], self._decode_box) + def _prepare_pendingfields(self, metainterp, pendingfields): + if pendingfields: + if metainterp._already_allocated_resume_virtuals is not None: + return + for descr, num, fieldnum in pendingfields: + box = self._decode_box(num) + fieldbox = self._decode_box(fieldnum) + metainterp.execute_and_record(rop.SETFIELD_GC, + descr, box, fieldbox) + def consume_boxes(self): numb = self.cur_numb assert numb is not None Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_effectinfo.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_effectinfo.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_effectinfo.py Tue Dec 29 10:12:39 2009 @@ -3,32 +3,77 @@ from pypy.rpython.ootypesystem import ootype from pypy.jit.metainterp.effectinfo import effectinfo_from_writeanalyze +class FakeCPU: + def fielddescrof(self, T, fieldname): + return ('fielddescr', T, fieldname) + def arraydescrof(self, A): + return ('arraydescr', A) + +def test_include_read_field(): + S = lltype.GcStruct("S", ("a", lltype.Signed)) + effects = frozenset([("readstruct", lltype.Ptr(S), "a")]) + effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU()) + assert list(effectinfo.readonly_descrs_fields) == [('fielddescr', S, "a")] + assert not effectinfo.write_descrs_fields + assert not effectinfo.write_descrs_arrays + +def test_include_write_field(): + S = lltype.GcStruct("S", ("a", lltype.Signed)) + effects = frozenset([("struct", lltype.Ptr(S), "a")]) + effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU()) + assert list(effectinfo.write_descrs_fields) == [('fielddescr', S, "a")] + assert not effectinfo.readonly_descrs_fields + assert not effectinfo.write_descrs_arrays + +def test_include_write_array(): + A = lltype.GcArray(lltype.Signed) + effects = frozenset([("array", lltype.Ptr(A))]) + effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU()) + assert not effectinfo.readonly_descrs_fields + assert not effectinfo.write_descrs_fields + assert list(effectinfo.write_descrs_arrays) == [('arraydescr', A)] + +def test_dont_include_read_and_write_field(): + S = lltype.GcStruct("S", ("a", lltype.Signed)) + effects = frozenset([("readstruct", lltype.Ptr(S), "a"), + ("struct", lltype.Ptr(S), "a")]) + effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU()) + assert not effectinfo.readonly_descrs_fields + assert list(effectinfo.write_descrs_fields) == [('fielddescr', S, "a")] + assert not effectinfo.write_descrs_arrays + + def test_filter_out_typeptr(): effects = frozenset([("struct", lltype.Ptr(OBJECT), "typeptr")]) effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays def test_filter_out_array_of_void(): effects = frozenset([("array", lltype.Ptr(lltype.GcArray(lltype.Void)))]) effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays def test_filter_out_struct_with_void(): effects = frozenset([("struct", lltype.Ptr(lltype.GcStruct("x", ("a", lltype.Void))), "a")]) effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays def test_filter_out_ooarray_of_void(): effects = frozenset([("array", ootype.Array(ootype.Void))]) effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays def test_filter_out_instance_with_void(): effects = frozenset([("struct", ootype.Instance("x", ootype.ROOT, {"a": ootype.Void}), "a")]) effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.readonly_descrs_fields assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py Tue Dec 29 10:12:39 2009 @@ -96,9 +96,15 @@ onedescr = cpu.fielddescrof(U, 'one') FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) - nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([], [])) - writeadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([adescr], [])) - writearraydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([adescr], [arraydescr])) + plaincalldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo([], [], [])) + writeadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo([], [adescr], [])) + writearraydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo([], [adescr], [arraydescr])) + readadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo([adescr], [], [])) mayforcevirtdescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([], [], forces_virtual_or_virtualizable=True)) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py Tue Dec 29 10:12:39 2009 @@ -87,7 +87,10 @@ def test_reuse_vinfo(): class FakeVInfo(object): - pass + def set_content(self, fieldnums): + self.fieldnums = fieldnums + def equals(self, fieldnums): + return self.fieldnums == fieldnums class FakeVirtualValue(optimizeopt.AbstractVirtualValue): def _make_virtual(self, *args): return FakeVInfo() @@ -606,10 +609,10 @@ p3sub = getfield_gc(p3, descr=nextdescr) i3 = getfield_gc(p3sub, descr=valuedescr) escape(i3) + p1 = new_with_vtable(ConstClass(node_vtable)) p2sub = new_with_vtable(ConstClass(node_vtable2)) setfield_gc(p2sub, i1, descr=valuedescr) setfield_gc(p2, p2sub, descr=nextdescr) - p1 = new_with_vtable(ConstClass(node_vtable)) jump(i1, p1, p2) """ # The same as test_p123_simple, but in the end the "old" p2 contains @@ -1293,6 +1296,182 @@ """ self.optimize_loop(ops, 'Not, Not', ops) + def test_duplicate_setfield_1(self): + ops = """ + [p1, i1, i2] + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(p1, i2, descr=valuedescr) + jump(p1, i1, i2) + """ + expected = """ + [p1, i1, i2] + setfield_gc(p1, i2, descr=valuedescr) + jump(p1, i1, i2) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + + def test_duplicate_setfield_2(self): + ops = """ + [p1, i1, i3] + setfield_gc(p1, i1, descr=valuedescr) + i2 = getfield_gc(p1, descr=valuedescr) + setfield_gc(p1, i3, descr=valuedescr) + escape(i2) + jump(p1, i1, i3) + """ + expected = """ + [p1, i1, i3] + setfield_gc(p1, i3, descr=valuedescr) + escape(i1) + jump(p1, i1, i3) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + + def test_duplicate_setfield_3(self): + ops = """ + [p1, p2, i1, i3] + setfield_gc(p1, i1, descr=valuedescr) + i2 = getfield_gc(p2, descr=valuedescr) + setfield_gc(p1, i3, descr=valuedescr) + escape(i2) + jump(p1, p2, i1, i3) + """ + # potential aliasing of p1 and p2 means that we cannot kill the + # the setfield_gc + self.optimize_loop(ops, 'Not, Not, Not, Not', ops) + + def test_duplicate_setfield_4(self): + ops = """ + [p1, i1, i2, p3] + setfield_gc(p1, i1, descr=valuedescr) + # + # some operations on which the above setfield_gc cannot have effect + i3 = getarrayitem_gc_pure(p3, 1, descr=arraydescr) + i4 = getarrayitem_gc(p3, i3, descr=arraydescr) + i5 = int_add(i3, i4) + setarrayitem_gc(p3, 0, i5, descr=arraydescr) + setfield_gc(p1, i4, descr=nextdescr) + # + setfield_gc(p1, i2, descr=valuedescr) + jump(p1, i1, i2, p3) + """ + expected = """ + [p1, i1, i2, p3] + # + i3 = getarrayitem_gc_pure(p3, 1, descr=arraydescr) + i4 = getarrayitem_gc(p3, i3, descr=arraydescr) + i5 = int_add(i3, i4) + setarrayitem_gc(p3, 0, i5, descr=arraydescr) + # + setfield_gc(p1, i2, descr=valuedescr) + setfield_gc(p1, i4, descr=nextdescr) + jump(p1, i1, i2, p3) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', expected) + + def test_duplicate_setfield_sideeffects_1(self): + ops = """ + [p1, i1, i2] + setfield_gc(p1, i1, descr=valuedescr) + escape() + setfield_gc(p1, i2, descr=valuedescr) + jump(p1, i1, i2) + """ + self.optimize_loop(ops, 'Not, Not, Not', ops) + + def test_duplicate_setfield_residual_guard_1(self): + ops = """ + [p1, i1, i2, i3] + setfield_gc(p1, i1, descr=valuedescr) + guard_true(i3) [] + i4 = int_neg(i2) + setfield_gc(p1, i2, descr=valuedescr) + jump(p1, i1, i2, i4) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', ops) + + def test_duplicate_setfield_residual_guard_2(self): + # the difference with the previous test is that the field value is + # a virtual, which we try hard to keep virtual + ops = """ + [p1, i2, i3] + p2 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, p2, descr=nextdescr) + guard_true(i3) [] + i4 = int_neg(i2) + setfield_gc(p1, NULL, descr=nextdescr) + jump(p1, i2, i4) + """ + expected = """ + [p1, i2, i3] + guard_true(i3) [p1] + i4 = int_neg(i2) + setfield_gc(p1, NULL, descr=nextdescr) + jump(p1, i2, i4) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + + def test_duplicate_setfield_residual_guard_3(self): + ops = """ + [p1, i2, i3] + p2 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p2, i2, descr=valuedescr) + setfield_gc(p1, p2, descr=nextdescr) + guard_true(i3) [] + i4 = int_neg(i2) + setfield_gc(p1, NULL, descr=nextdescr) + jump(p1, i2, i4) + """ + expected = """ + [p1, i2, i3] + guard_true(i3) [p1, i2] + i4 = int_neg(i2) + setfield_gc(p1, NULL, descr=nextdescr) + jump(p1, i2, i4) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + + def test_duplicate_setfield_residual_guard_4(self): + # test that the setfield_gc does not end up between int_eq and + # the following guard_true + ops = """ + [p1, i1, i2, i3] + setfield_gc(p1, i1, descr=valuedescr) + i5 = int_eq(i3, 5) + guard_true(i5) [] + i4 = int_neg(i2) + setfield_gc(p1, i2, descr=valuedescr) + jump(p1, i1, i2, i4) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', ops) + + def test_duplicate_setfield_aliasing(self): + # a case where aliasing issues (and not enough cleverness) mean + # that we fail to remove any setfield_gc + ops = """ + [p1, p2, i1, i2, i3] + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(p2, i2, descr=valuedescr) + setfield_gc(p1, i3, descr=valuedescr) + jump(p1, p2, i1, i2, i3) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not, Not', ops) + + def test_duplicate_setfield_guard_value_const(self): + ops = """ + [p1, i1, i2] + guard_value(p1, ConstPtr(myptr)) [] + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(ConstPtr(myptr), i2, descr=valuedescr) + jump(p1, i1, i2) + """ + expected = """ + [i1, i2] + setfield_gc(ConstPtr(myptr), i2, descr=valuedescr) + jump(i1, i2) + """ + self.optimize_loop(ops, 'Constant(myptr), Not, Not', expected) + def test_duplicate_getarrayitem_1(self): ops = """ [p1] @@ -1634,6 +1813,14 @@ tag = ('virtual', self.namespace[match.group(2)]) virtuals[pvar] = (tag, None, fieldstext) # + r2 = re.compile(r"([\w\d()]+)[.](\w+)\s*=\s*([\w\d()]+)") + pendingfields = [] + for match in r2.finditer(text): + pvar = match.group(1) + pfieldname = match.group(2) + pfieldvar = match.group(3) + pendingfields.append((pvar, pfieldname, pfieldvar)) + # def _variables_equal(box, varname, strict): if varname not in virtuals: if strict: @@ -1655,11 +1842,21 @@ else: virtuals[varname] = tag, box, fieldstext # - basetext = text[:ends[0]] + basetext = text.splitlines()[0] varnames = [s.strip() for s in basetext.split(',')] + if varnames == ['']: + varnames = [] assert len(boxes) == len(varnames) for box, varname in zip(boxes, varnames): _variables_equal(box, varname, strict=True) + for pvar, pfieldname, pfieldvar in pendingfields: + box = oparse.getvar(pvar) + fielddescr = self.namespace[pfieldname.strip()] + fieldbox = executor.execute(self.cpu, + rop.GETFIELD_GC, + fielddescr, + box) + _variables_equal(fieldbox, pfieldvar, strict=True) # for match in parts: pvar = match.group(1) @@ -1918,6 +2115,57 @@ where p7v is a node_vtable, valuedescr=iv ''') + def test_expand_fail_lazy_setfield_1(self): + self.make_fail_descr() + ops = """ + [p1, i2, i3] + p2 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p2, i2, descr=valuedescr) + setfield_gc(p1, p2, descr=nextdescr) + guard_true(i3, descr=fdescr) [] + i4 = int_neg(i2) + setfield_gc(p1, NULL, descr=nextdescr) + jump(p1, i2, i4) + """ + expected = """ + [p1, i2, i3] + guard_true(i3, descr=fdescr) [p1, i2] + i4 = int_neg(i2) + setfield_gc(p1, NULL, descr=nextdescr) + jump(p1, i2, i4) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + self.loop.inputargs[0].value = self.nodebox.value + self.check_expanded_fail_descr(''' + p1.nextdescr = p2 + where p2 is a node_vtable, valuedescr=i2 + ''') + + def test_expand_fail_lazy_setfield_2(self): + self.make_fail_descr() + ops = """ + [i2, i3] + p2 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p2, i2, descr=valuedescr) + setfield_gc(ConstPtr(myptr), p2, descr=nextdescr) + guard_true(i3, descr=fdescr) [] + i4 = int_neg(i2) + setfield_gc(ConstPtr(myptr), NULL, descr=nextdescr) + jump(i2, i4) + """ + expected = """ + [i2, i3] + guard_true(i3, descr=fdescr) [i2] + i4 = int_neg(i2) + setfield_gc(ConstPtr(myptr), NULL, descr=nextdescr) + jump(i2, i4) + """ + self.optimize_loop(ops, 'Not, Not', expected) + self.check_expanded_fail_descr(''' + ConstPtr(myptr).nextdescr = p2 + where p2 is a node_vtable, valuedescr=i2 + ''') + class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): @@ -2031,6 +2279,58 @@ """ self.optimize_loop(ops, 'Not, Not, Not', expected) + def test_residual_call_invalidates_some_read_caches_1(self): + ops = """ + [p1, i1, p2, i2] + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(p2, i2, descr=adescr) + i3 = call(i1, descr=readadescr) + setfield_gc(p1, i3, descr=valuedescr) + setfield_gc(p2, i3, descr=adescr) + jump(p1, i1, p2, i2) + """ + expected = """ + [p1, i1, p2, i2] + setfield_gc(p2, i2, descr=adescr) + i3 = call(i1, descr=readadescr) + setfield_gc(p1, i3, descr=valuedescr) + setfield_gc(p2, i3, descr=adescr) + jump(p1, i1, p2, i2) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', expected) + + def test_residual_call_invalidates_some_read_caches_2(self): + ops = """ + [p1, i1, p2, i2] + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(p2, i2, descr=adescr) + i3 = call(i1, descr=writeadescr) + setfield_gc(p1, i3, descr=valuedescr) + setfield_gc(p2, i3, descr=adescr) + jump(p1, i1, p2, i2) + """ + expected = """ + [p1, i1, p2, i2] + setfield_gc(p2, i2, descr=adescr) + i3 = call(i1, descr=writeadescr) + setfield_gc(p1, i3, descr=valuedescr) + setfield_gc(p2, i3, descr=adescr) + jump(p1, i1, p2, i2) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', expected) + + def test_residual_call_invalidates_some_read_caches_3(self): + ops = """ + [p1, i1, p2, i2] + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(p2, i2, descr=adescr) + i3 = call(i1, descr=plaincalldescr) + setfield_gc(p1, i3, descr=valuedescr) + setfield_gc(p2, i3, descr=adescr) + jump(p1, i1, p2, i2) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', ops) + def test_vref_nonvirtual(self): ops = """ [p1] Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_resume.py Tue Dec 29 10:12:39 2009 @@ -12,6 +12,8 @@ rd_frame_info_list = None rd_numb = None rd_consts = [] + rd_virtuals = None + rd_pendingfields = None def test_tag(): assert tag(3, 1) == rffi.r_short(3<<2|1) @@ -40,6 +42,12 @@ assert not tagged_list_eq([tag(1, TAGBOX)], [tag(-2, TAGBOX)]) assert not tagged_list_eq([tag(1, TAGBOX), tag(-2, TAGBOX)], [tag(1, TAGBOX)]) +def test_vinfo(): + v1 = AbstractVirtualInfo() + v1.set_content([1, 2, 4]) + assert v1.equals([1, 2, 4]) + assert not v1.equals([1, 2, 6]) + class MyMetaInterp: _already_allocated_resume_virtuals = None @@ -80,7 +88,6 @@ tag(0, TAGBOX), tag(1, TAGBOX)]) storage.rd_numb = numb - storage.rd_virtuals = None b1s, b2s, b3s = [BoxInt(), BoxPtr(), BoxInt()] assert b1s != b3s @@ -103,7 +110,6 @@ tag(0, TAGBOX), tag(1, TAGBOX)]) storage.rd_numb = numb - storage.rd_virtuals = None b1s, b2s, b3s = [BoxInt(), BoxPtr(), BoxInt()] assert b1s != b3s reader = ResumeDataReader(storage, [b1s, b2s, b3s], MyMetaInterp()) @@ -125,6 +131,7 @@ rd_virtuals = [FakeVinfo(), None] rd_numb = [] rd_consts = [] + rd_pendingfields = None class FakeMetainterp(object): _already_allocated_resume_virtuals = None cpu = None @@ -968,6 +975,46 @@ assert ptr.a == 111 assert ptr.b == lltype.nullptr(LLtypeMixin.NODE) + +def test_virtual_adder_pending_fields(): + b2s, b4s = [BoxPtr(), BoxPtr()] + storage = Storage() + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) + modifier = ResumeDataVirtualAdder(storage, memo) + modifier.liveboxes_from_env = {} + modifier.liveboxes = {} + modifier.vfieldboxes = {} + + v2 = OptValue(b2s) + v4 = OptValue(b4s) + modifier.register_box(b2s) + modifier.register_box(b4s) + + values = {b4s: v4, b2s: v2} + liveboxes = [] + modifier._number_virtuals(liveboxes, values, 0) + assert liveboxes == [b2s, b4s] + modifier._add_pending_fields([(LLtypeMixin.nextdescr, b2s, b4s)]) + storage.rd_consts = memo.consts[:] + storage.rd_numb = None + # resume + demo55.next = lltype.nullptr(LLtypeMixin.NODE) + b2t = BoxPtr(demo55o) + b4t = BoxPtr(demo66o) + newboxes = _resume_remap(liveboxes, [b2s, b4s], b2t, b4t) + + metainterp = MyMetaInterp() + reader = ResumeDataReader(storage, newboxes, metainterp) + assert reader.virtuals is None + trace = metainterp.trace + b2set = (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.nextdescr) + expected = [b2set] + + for x, y in zip(expected, trace): + assert x == y + assert demo55.next == demo66 + + def test_invalidation_needed(): class options: failargs_limit = 10 Modified: pypy/branch/virtual-forcing/pypy/lib/app_test/test_runpy.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/lib/app_test/test_runpy.py (original) +++ pypy/branch/virtual-forcing/pypy/lib/app_test/test_runpy.py Tue Dec 29 10:12:39 2009 @@ -137,6 +137,12 @@ d1 = run_module(mod_name) # Read from source __import__(mod_name) os.remove(mod_fname) + + #--- the block below is to check that "imp.find_module" + #--- manages to import the .pyc file alone. We don't + #--- support it in PyPy in the default configuration. + return + if verbose: print "Running from compiled:", mod_name d2 = run_module(mod_name) # Read from bytecode finally: Modified: pypy/branch/virtual-forcing/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/__builtin__/__init__.py (original) +++ pypy/branch/virtual-forcing/pypy/module/__builtin__/__init__.py Tue Dec 29 10:12:39 2009 @@ -1,6 +1,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter import module from pypy.interpreter.mixedmodule import MixedModule +import pypy.module.imp.importing # put builtins here that should be optimized somehow @@ -35,9 +36,6 @@ 'vars' : 'app_inspect.vars', 'dir' : 'app_inspect.dir', - '_find_module' : 'app_misc.find_module', - 'reload' : 'app_misc.reload', - '__filestub' : 'app_file_stub.file', } @@ -88,7 +86,8 @@ 'compile' : 'compiling.compile', 'eval' : 'compiling.eval', - '__import__' : 'importing.importhook', + '__import__' : 'pypy.module.imp.importing.importhook', + 'reload' : 'pypy.module.imp.importing.reload', 'range' : 'functional.range_int', 'xrange' : 'functional.W_XRange', @@ -152,16 +151,3 @@ space.exception_is_valid_obj_as_class_w = ab.exception_is_valid_obj_as_class_w.__get__(space) space.exception_getclass = ab.exception_getclass.__get__(space) space.exception_issubclass_w = ab.exception_issubclass_w.__get__(space) - - def startup(self, space): - # install zipimport hook if --withmod-zipimport is used - if space.config.objspace.usemodules.zipimport: - w_import = space.builtin.get('__import__') - w_zipimport = space.call(w_import, space.newlist( - [space.wrap('zipimport')])) - w_sys = space.getbuiltinmodule('sys') - w_path_hooks = space.getattr(w_sys, space.wrap('path_hooks')) - w_append = space.getattr(w_path_hooks, space.wrap('append')) - w_zipimporter = space.getattr(w_zipimport, - space.wrap('zipimporter')) - space.call(w_append, space.newlist([w_zipimporter])) Modified: pypy/branch/virtual-forcing/pypy/module/__pypy__/__init__.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/__pypy__/__init__.py (original) +++ pypy/branch/virtual-forcing/pypy/module/__pypy__/__init__.py Tue Dec 29 10:12:39 2009 @@ -1,7 +1,7 @@ # Package initialisation from pypy.interpreter.mixedmodule import MixedModule -from pypy.module.__builtin__.importing import get_pyc_magic +from pypy.module.imp.importing import get_pyc_magic class Module(MixedModule): appleveldefs = { Modified: pypy/branch/virtual-forcing/pypy/module/_demo/__init__.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/_demo/__init__.py (original) +++ pypy/branch/virtual-forcing/pypy/module/_demo/__init__.py Tue Dec 29 10:12:39 2009 @@ -12,3 +12,13 @@ appleveldefs = { 'DemoError' : 'app_demo.DemoError', } + + # Used in tests + demo_events = [] + def setup_after_space_initialization(self): + Module.demo_events.append('setup') + def startup(self, space): + Module.demo_events.append('startup') + def shutdown(self, space): + Module.demo_events.append('shutdown') + Modified: pypy/branch/virtual-forcing/pypy/module/oracle/__init__.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/oracle/__init__.py (original) +++ pypy/branch/virtual-forcing/pypy/module/oracle/__init__.py Tue Dec 29 10:12:39 2009 @@ -39,6 +39,7 @@ def startup(self, space): from pypy.module.oracle.interp_error import get state = get(space) + state.startup(space) (state.w_DecimalType, state.w_DateTimeType, state.w_DateType, state.w_TimedeltaType, ) = space.fixedview(space.appexec([], """(): Modified: pypy/branch/virtual-forcing/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/oracle/interp_error.py (original) +++ pypy/branch/virtual-forcing/pypy/module/oracle/interp_error.py Tue Dec 29 10:12:39 2009 @@ -4,32 +4,47 @@ from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError + from pypy.module.oracle import roci, config +from pypy.rlib.unroll import unrolling_iterable + +exported_names = unrolling_iterable(""" + DatabaseError OperationalError InterfaceError ProgrammingError + NotSupportedError IntegrityError InternalError DataError + Variable Connection""".split()) class State: # XXX move to another file + def __init__(self, space): - w_module = space.getbuiltinmodule('cx_Oracle') - def get(name): - return space.getattr(w_module, space.wrap(name)) + "NOT_RPYTHON" + self.variableTypeByPythonType = {} + self.w_DecimalType = None + self.w_DateTimeType = None + self.w_DateType = None + self.w_TimedeltaType = None - self.w_DatabaseError = get('DatabaseError') - self.w_OperationalError = get('OperationalError') - self.w_InterfaceError = get('InterfaceError') - self.w_ProgrammingError = get('ProgrammingError') - self.w_NotSupportedError = get('NotSupportedError') - self.w_IntegrityError = get('IntegrityError') - self.w_InternalError = get('InternalError') - self.w_DataError = get('DataError') - self.w_Variable = get('Variable') - self.w_Connection = get('Connection') + for name in exported_names: + setattr(self, 'w_' + name, None) + + def startup(self, space): + w_module = space.getbuiltinmodule('cx_Oracle') + for name in exported_names: + setattr(self, 'w_' + name, space.getattr(w_module, space.wrap(name))) from pypy.module.oracle.interp_variable import all_variable_types - self.variableTypeByPythonType = {} for varType in all_variable_types: w_type = space.gettypeobject(varType.typedef) self.variableTypeByPythonType[w_type] = varType + (self.w_DecimalType, + self.w_DateTimeType, self.w_DateType, self.w_TimedeltaType, + ) = space.fixedview(space.appexec([], """(): + import decimal, datetime + return (decimal.Decimal, + datetime.datetime, datetime.date, datetime.timedelta) + """)) + def get(space): return space.fromcache(State) Modified: pypy/branch/virtual-forcing/pypy/module/thread/__init__.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/thread/__init__.py (original) +++ pypy/branch/virtual-forcing/pypy/module/thread/__init__.py Tue Dec 29 10:12:39 2009 @@ -18,11 +18,6 @@ 'allocate': 'os_lock.allocate_lock', # obsolete synonym 'LockType': 'os_lock.getlocktype(space)', '_local': 'os_local.getlocaltype(space)', - - # custom interface for the 'imp' module - '_importlock_held': 'importlock.held', - '_importlock_acquire': 'importlock.acquire', - '_importlock_release': 'importlock.release', } def __init__(self, space, *args): Modified: pypy/branch/virtual-forcing/pypy/module/zipimport/__init__.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/zipimport/__init__.py (original) +++ pypy/branch/virtual-forcing/pypy/module/zipimport/__init__.py Tue Dec 29 10:12:39 2009 @@ -14,4 +14,12 @@ appleveldefs = { 'ZipImportError' : 'app_zipimport.ZipImportError', } - + + def setup_after_space_initialization(self): + """NOT_RPYTHON""" + space = self.space + # install zipimport hook + w_path_hooks = space.sys.get('path_hooks') + from pypy.module.zipimport.interp_zipimport import W_ZipImporter + w_zipimporter = space.gettypefor(W_ZipImporter) + space.call_method(w_path_hooks, 'append', w_zipimporter) Modified: pypy/branch/virtual-forcing/pypy/module/zipimport/interp_zipimport.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/zipimport/interp_zipimport.py (original) +++ pypy/branch/virtual-forcing/pypy/module/zipimport/interp_zipimport.py Tue Dec 29 10:12:39 2009 @@ -5,7 +5,7 @@ from pypy.interpreter.gateway import interp2app from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.module import Module -from pypy.module.__builtin__ import importing +from pypy.module.imp import importing from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rzipfile import RZipFile, BadZipfile import os @@ -149,9 +149,9 @@ real_name = self.name + os.path.sep + self.corr_zname(filename) space.setattr(w_mod, w('__loader__'), space.wrap(self)) importing._prepare_module(space, w_mod, real_name, pkgpath) - result = importing.load_source_module(space, w(modname), w_mod, - filename, buf, write_pyc=False) - return result + code_w = importing.parse_source_module(space, filename, buf) + importing.exec_code_module(space, w_mod, code_w) + return w_mod def _parse_mtime(self, space, filename): w = space.wrap Modified: pypy/branch/virtual-forcing/pypy/module/zipimport/test/test_zipimport.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/zipimport/test/test_zipimport.py (original) +++ pypy/branch/virtual-forcing/pypy/module/zipimport/test/test_zipimport.py Tue Dec 29 10:12:39 2009 @@ -4,7 +4,7 @@ import py import time import struct -from pypy.module.__builtin__.importing import get_pyc_magic, _w_long +from pypy.module.imp.importing import get_pyc_magic, _w_long from StringIO import StringIO from pypy.tool.udir import udir @@ -255,6 +255,11 @@ l = [i for i in zipimport._zip_directory_cache] assert len(l) + def test_path_hooks(self): + import sys + import zipimport + assert sys.path_hooks.count(zipimport.zipimporter) == 1 + class AppTestZipimportDeflated(AppTestZipimport): compression = ZIP_DEFLATED Modified: pypy/branch/virtual-forcing/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/objspace/flow/flowcontext.py (original) +++ pypy/branch/virtual-forcing/pypy/objspace/flow/flowcontext.py Tue Dec 29 10:12:39 2009 @@ -385,6 +385,9 @@ operr = OperationError(operr.w_type, operr.w_value) return operr + def exception_trace(self, frame, operationerr): + pass # overridden for performance only + # hack for unrolling iterables, don't use this def replace_in_stack(self, oldvalue, newvalue): w_new = Constant(newvalue) Modified: pypy/branch/virtual-forcing/pypy/rlib/rgc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/rgc.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/rgc.py Tue Dec 29 10:12:39 2009 @@ -357,3 +357,8 @@ keepalive_until_here(source) keepalive_until_here(dest) ll_arraycopy._annspecialcase_ = 'specialize:ll' + +def no_collect(func): + func._dont_inline_ = True + func._gc_no_collect_ = True + return func Modified: pypy/branch/virtual-forcing/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/memory/gctransform/framework.py Tue Dec 29 10:12:39 2009 @@ -567,6 +567,12 @@ f.close() def transform_graph(self, graph): + func = getattr(graph, 'func', None) + if func and getattr(func, '_gc_no_collect_', False): + if self.collect_analyzer.analyze_direct_call(graph): + raise Exception("no_collect function can trigger collection: %s" + % func.__name__) + if self.write_barrier_ptr: self.clean_sets = ( find_clean_setarrayitems(self.collect_analyzer, graph).union( Modified: pypy/branch/virtual-forcing/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/memory/gctransform/test/test_framework.py Tue Dec 29 10:12:39 2009 @@ -1,5 +1,6 @@ from pypy.objspace.flow.model import Constant, SpaceOperation from pypy.annotation.model import SomeInteger +from pypy.annotation.listdef import s_list_of_strings from pypy.rpython.memory.gc.marksweep import MarkSweepGC from pypy.rpython.memory.gctransform.test.test_transform import rtype, \ rtype_and_transform @@ -34,7 +35,6 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.translator.c.genc import CStandaloneBuilder from pypy.translator.c import gc - from pypy.annotation.listdef import s_list_of_strings t = rtype(entrypoint, [s_list_of_strings]) cbuild = CStandaloneBuilder(t, entrypoint, t.config, @@ -113,6 +113,50 @@ gg = graphof(t, g) assert CollectAnalyzer(t).analyze_direct_call(gg) +def test_no_collect(): + from pypy.rlib import rgc + from pypy.translator.c.genc import CStandaloneBuilder + from pypy.translator.c import gc + + @rgc.no_collect + def g(): + return 1 + + assert g._dont_inline_ + assert g._gc_no_collect_ + + def entrypoint(argv): + return g() + 2 + + t = rtype(entrypoint, [s_list_of_strings]) + cbuild = CStandaloneBuilder(t, entrypoint, t.config, + gcpolicy=FrameworkGcPolicy2) + db = cbuild.generate_graphs_for_llinterp() + +def test_no_collect_detection(): + from pypy.rlib import rgc + from pypy.translator.c.genc import CStandaloneBuilder + from pypy.translator.c import gc + + class A(object): + def __init__(self, x): + self.x = x + + @rgc.no_collect + def g(): + return A(1).x + + assert g._dont_inline_ + assert g._gc_no_collect_ + + def entrypoint(argv): + return g() + 2 + + t = rtype(entrypoint, [s_list_of_strings]) + cbuild = CStandaloneBuilder(t, entrypoint, t.config, + gcpolicy=FrameworkGcPolicy2) + f = py.test.raises(Exception, cbuild.generate_graphs_for_llinterp) + assert str(f.value) == 'no_collect function can trigger collection: g' class WriteBarrierTransformer(FrameworkGCTransformer): clean_sets = {} Modified: pypy/branch/virtual-forcing/pypy/rpython/rptr.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/rptr.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/rptr.py Tue Dec 29 10:12:39 2009 @@ -39,6 +39,14 @@ attr = hop.args_s[1].const if isinstance(hop.s_result, annmodel.SomeLLADTMeth): return hop.inputarg(hop.r_result, arg=0) + try: + self.lowleveltype._example()._lookup_adtmeth(attr) + except AttributeError: + pass + else: + assert hop.s_result.is_constant() + return hop.inputconst(hop.r_result, hop.s_result.const) + assert attr in self.lowleveltype.TO._flds # check that the field exists FIELD_TYPE = getattr(self.lowleveltype.TO, attr) if isinstance(FIELD_TYPE, lltype.ContainerType): if (attr, FIELD_TYPE) == self.lowleveltype.TO._first_struct(): Modified: pypy/branch/virtual-forcing/pypy/rpython/test/test_rptr.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/test/test_rptr.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/test/test_rptr.py Tue Dec 29 10:12:39 2009 @@ -337,3 +337,13 @@ return f([1]) s, t = ll_rtype(lltest, []) assert s.is_constant() == False + +def test_staticadtmeths(): + ll_func = staticAdtMethod(lambda x: x + 42) + S = GcStruct('S', adtmeths={'ll_func': ll_func}) + def f(): + return malloc(S).ll_func(5) + s, t = ll_rtype(f, []) + graphf = t.graphs[0] + for op in graphf.startblock.operations: + assert op.opname != 'getfield' Modified: pypy/branch/virtual-forcing/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/tool/rffi_platform.py Tue Dec 29 10:12:39 2009 @@ -644,7 +644,7 @@ else: raise CompilationError("Library %s is not installed" % (name,)) -def check_boehm(platform=None): +def configure_boehm(platform=None): if platform is None: from pypy.translator.platform import platform if sys.platform == 'win32': @@ -658,13 +658,10 @@ includes=includes, libraries=['gc'], ) - try: - return configure_external_library( - 'gc', eci, - [dict(prefix='gc-', include_dir='include', library_dir=library_dir)], - symbol='GC_init') - except CompilationError: - return None + return configure_external_library( + 'gc', eci, + [dict(prefix='gc-', include_dir='include', library_dir=library_dir)], + symbol='GC_init') if __name__ == '__main__': doc = """Example: Modified: pypy/branch/virtual-forcing/pypy/tool/gcc_cache.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/tool/gcc_cache.py (original) +++ pypy/branch/virtual-forcing/pypy/tool/gcc_cache.py Tue Dec 29 10:12:39 2009 @@ -10,7 +10,7 @@ def cache_file_path(c_files, eci, cachename): cache_dir = cache_dir_root.join(cachename).ensure(dir=1) filecontents = [c_file.read() for c_file in c_files] - key = repr((filecontents, eci)) + key = repr((filecontents, eci, platform.key())) hash = md5(key).hexdigest() return cache_dir.join(hash) Modified: pypy/branch/virtual-forcing/pypy/tool/udir.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/tool/udir.py (original) +++ pypy/branch/virtual-forcing/pypy/tool/udir.py Tue Dec 29 10:12:39 2009 @@ -18,7 +18,7 @@ # import autopath -import os +import os, sys import py from py.path import local @@ -39,6 +39,8 @@ try: p = py.path.local(__file__).dirpath() basename = svn_info(py.path.svnwc(p).info().url) + if isinstance(basename, unicode): + basename = basename.encode(sys.getdefaultencoding()) except: basename = '' if not basename.startswith('-'): Modified: pypy/branch/virtual-forcing/pypy/translator/backendopt/test/test_writeanalyze.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/backendopt/test/test_writeanalyze.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/backendopt/test/test_writeanalyze.py Tue Dec 29 10:12:39 2009 @@ -4,14 +4,15 @@ from pypy.translator.translator import TranslationContext, graphof from pypy.translator.simplify import get_funcobj from pypy.translator.backendopt.writeanalyze import WriteAnalyzer, top_set +from pypy.translator.backendopt.writeanalyze import ReadWriteAnalyzer from pypy.translator.backendopt.all import backend_optimizations from pypy.conftest import option -class BaseTestCanRaise(object): +class BaseTest(object): type_system = None - + Analyzer = WriteAnalyzer def translate(self, func, sig): t = TranslationContext() @@ -19,7 +20,10 @@ t.buildrtyper(type_system=self.type_system).specialize() if option.view: t.view() - return t, WriteAnalyzer(t) + return t, self.Analyzer(t) + + +class BaseTestWriteAnalyze(BaseTest): def test_writes_simple(self): def g(x): @@ -146,7 +150,7 @@ assert not result -class TestLLtype(BaseTestCanRaise): +class TestLLtype(BaseTestWriteAnalyze): type_system = 'lltype' def test_list(self): @@ -205,7 +209,7 @@ assert name.endswith("foobar") -class TestOOtype(BaseTestCanRaise): +class TestOOtype(BaseTestWriteAnalyze): type_system = 'ootype' def test_array(self): @@ -240,3 +244,88 @@ result = wa.analyze(ggraph.startblock.operations[0]) assert result is top_set + + +class TestLLtypeReadWriteAnalyze(BaseTest): + Analyzer = ReadWriteAnalyzer + type_system = 'lltype' + + def test_read_simple(self): + def g(x): + return True + + def f(x): + return g(x - 1) + t, wa = self.translate(f, [int]) + fgraph = graphof(t, f) + result = wa.analyze(fgraph.startblock.operations[0]) + assert not result + + def test_read_really(self): + class A(object): + def __init__(self, y): + self.y = y + def f(self): + self.x = 1 + return self.y + def h(flag): + obj = A(flag) + return obj.f() + + t, wa = self.translate(h, [int]) + hgraph = graphof(t, h) + op_call_f = hgraph.startblock.operations[-1] + + # check that we fished the expected ops + assert op_call_f.opname == "direct_call" + assert get_funcobj(op_call_f.args[0].value)._name == 'A.f' + + result = wa.analyze(op_call_f) + assert len(result) == 2 + result = list(result) + result.sort() + [(struct1, T1, name1), (struct2, T2, name2)] = result + assert struct1 == "readstruct" + assert name1.endswith("y") + assert struct2 == "struct" + assert name2.endswith("x") + assert T1 == T2 + + def test_contains(self): + def g(x, y, z): + l = [x] + return f(l, y, z) + def f(x, y, z): + return y in x + + t, wa = self.translate(g, [int, int, int]) + ggraph = graphof(t, g) + assert ggraph.startblock.operations[-1].opname == 'direct_call' + + result = wa.analyze(ggraph.startblock.operations[-1]) + ARRAYPTR = list(result)[0][1] + assert list(result) == [("readarray", ARRAYPTR)] + assert isinstance(ARRAYPTR.TO, lltype.GcArray) + + def test_adt_method(self): + def ll_callme(n): + return n + ll_callme = lltype.staticAdtMethod(ll_callme) + S = lltype.GcStruct('S', ('x', lltype.Signed), + adtmeths = {'yep': True, + 'callme': ll_callme}) + def g(x, y, z): + p = lltype.malloc(S) + p.x = x + if p.yep: + z *= p.callme(y) + return z + def f(x, y, z): + return g(x, y, z) + + t, wa = self.translate(f, [int, int, int]) + fgraph = graphof(t, f) + assert fgraph.startblock.operations[-1].opname == 'direct_call' + + result = wa.analyze(fgraph.startblock.operations[-1]) + assert list(result) == [("struct", lltype.Ptr(S), "x")] Modified: pypy/branch/virtual-forcing/pypy/translator/backendopt/writeanalyze.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/backendopt/writeanalyze.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/backendopt/writeanalyze.py Tue Dec 29 10:12:39 2009 @@ -45,3 +45,15 @@ elif methname in ('ll_getitem_fast', 'll_length'): return self.bottom_result() return graphanalyze.GraphAnalyzer.analyze_external_method(self, op, TYPE, meth) + + +class ReadWriteAnalyzer(WriteAnalyzer): + + def analyze_simple_operation(self, op): + if op.opname == "getfield": + return frozenset([ + ("readstruct", op.args[0].concretetype, op.args[1].value)]) + elif op.opname == "getarrayitem": + return frozenset([ + ("readarray", op.args[0].concretetype)]) + return WriteAnalyzer.analyze_simple_operation(self, op) Modified: pypy/branch/virtual-forcing/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/c/gc.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/c/gc.py Tue Dec 29 10:12:39 2009 @@ -215,8 +215,8 @@ def compilation_info(self): eci = BasicGcPolicy.compilation_info(self) - from pypy.rpython.tool.rffi_platform import check_boehm - eci = eci.merge(check_boehm()) + from pypy.rpython.tool.rffi_platform import configure_boehm + eci = eci.merge(configure_boehm()) pre_include_bits = [] if sys.platform == "linux2": Modified: pypy/branch/virtual-forcing/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/c/test/test_boehm.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/c/test/test_boehm.py Tue Dec 29 10:12:39 2009 @@ -3,12 +3,15 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.memory.test import snippet -from pypy.rpython.tool.rffi_platform import check_boehm from pypy.translator.c.genc import CExtModuleBuilder from pypy import conftest def setup_module(mod): - if not check_boehm(): + from pypy.rpython.tool.rffi_platform import configure_boehm + from pypy.translator.platform import CompilationError + try: + configure_boehm() + except CompilationError: py.test.skip("Boehm GC not present") class AbstractGCTestClass(object): Modified: pypy/branch/virtual-forcing/pypy/translator/c/test/test_stackless.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/c/test/test_stackless.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/c/test/test_stackless.py Tue Dec 29 10:12:39 2009 @@ -20,8 +20,11 @@ import py py.test.skip("stackless + refcounting doesn't work any more for now") elif cls.gcpolicy == "boehm": - from pypy.rpython.tool.rffi_platform import check_boehm - if not check_boehm(): + from pypy.rpython.tool.rffi_platform import configure_boehm + from pypy.translator.platform import CompilationError + try: + configure_boehm() + except CompilationError: py.test.skip("Boehm GC not present") def wrap_stackless_function(self, fn): Modified: pypy/branch/virtual-forcing/pypy/translator/driver.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/driver.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/driver.py Tue Dec 29 10:12:39 2009 @@ -428,10 +428,13 @@ def possibly_check_for_boehm(self): if self.config.translation.gc == "boehm": - from pypy.rpython.tool.rffi_platform import check_boehm - if not check_boehm(self.translator.platform): + from pypy.rpython.tool.rffi_platform import configure_boehm + from pypy.translator.platform import CompilationError + try: + configure_boehm(self.translator.platform) + except CompilationError, e: i = 'Boehm GC not installed. Try e.g. "translate.py --gc=hybrid"' - raise Exception(i) + raise Exception(str(e) + '\n' + i) def task_database_c(self): translator = self.translator Modified: pypy/branch/virtual-forcing/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/platform/__init__.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/platform/__init__.py Tue Dec 29 10:12:39 2009 @@ -52,6 +52,8 @@ name = "abstract platform" c_environ = None + relevant_environ = [] + so_prefixes = [''] def __init__(self, cc): @@ -98,6 +100,12 @@ return (self.__class__ is other.__class__ and self.__dict__ == other.__dict__) + def key(self): + bits = [self.__class__.__name__, 'cc=%s' % self.cc] + for varname in self.relevant_environ: + bits.append('%s=%s' % (varname, os.environ.get(varname))) + return ' '.join(bits) + # some helpers which seem to be cross-platform enough def _execute_c_compiler(self, cc, args, outname): @@ -171,8 +179,15 @@ else: host_factory = Linux64 elif sys.platform == 'darwin': - from pypy.translator.platform.darwin import Darwin - host_factory = Darwin + from pypy.translator.platform.darwin import Darwin_i386, Darwin_x86_64 + import platform + if platform.machine() == 'i386': + if sys.maxint <= 2147483647: + host_factory = Darwin_i386 + else: + host_factory = Darwin_x86_64 + else: + host_factory = Darwin elif sys.platform == 'freebsd7': from pypy.translator.platform.freebsd7 import Freebsd7, Freebsd7_64 import platform Modified: pypy/branch/virtual-forcing/pypy/translator/platform/darwin.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/platform/darwin.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/platform/darwin.py Tue Dec 29 10:12:39 2009 @@ -4,7 +4,7 @@ class Darwin(posix.BasePosix): name = "darwin" - + link_flags = ['-mmacosx-version-min=10.4'] cflags = ['-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4'] standalone_only = ['-mdynamic-no-pic'] @@ -44,3 +44,13 @@ include_dirs = self._includedirs(eci.include_dirs) return (args + frameworks + include_dirs) +class Darwin_i386(Darwin): + name = "darwin_i386" + link_flags = ['-arch', 'i386', '-mmacosx-version-min=10.4'] + cflags = ['-arch', 'i386', '-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4'] + +class Darwin_x86_64(Darwin): + name = "darwin_x86_64" + link_flags = ['-arch', 'x86_64', '-mmacosx-version-min=10.4'] + cflags = ['-arch', 'x86_64', '-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4'] + Modified: pypy/branch/virtual-forcing/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/platform/posix.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/platform/posix.py Tue Dec 29 10:12:39 2009 @@ -10,6 +10,8 @@ exe_ext = '' make_cmd = 'make' + relevant_environ=['CPATH', 'LIBRARY_PATH', 'C_INCLUDE_PATH'] + def __init__(self, cc=None): if cc is None: cc = 'gcc' Modified: pypy/branch/virtual-forcing/pypy/translator/platform/test/test_darwin.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/platform/test/test_darwin.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/platform/test/test_darwin.py Tue Dec 29 10:12:39 2009 @@ -2,17 +2,25 @@ """ File containing darwin platform tests """ -import py, sys +import py, sys, platform if sys.platform != 'darwin': py.test.skip("Darwin only") from pypy.tool.udir import udir -from pypy.translator.platform.darwin import Darwin +from pypy.translator.platform.darwin import Darwin_i386, Darwin_x86_64 from pypy.translator.platform.test.test_platform import TestPlatform as BasicTest from pypy.translator.tool.cbuild import ExternalCompilationInfo +if platform.machine() == 'i386': + if sys.maxint <= 2147483647: + host_factory = Darwin_i386 + else: + host_factory = Darwin_x86_64 +else: + host_factory = Darwin + class TestDarwin(BasicTest): - platform = Darwin() + platform = host_factory() def test_frameworks(self): objcfile = udir.join('test_simple.m') @@ -39,3 +47,81 @@ res = self.platform.execute(executable) self.check_res(res) + def test_64_32_results(self): + if platform.machine() != 'i386': + py.test.skip("i386 only") + plat32 = Darwin_i386() + plat64 = Darwin_x86_64() + cfile = udir.join('test_int_size.c') + cfile.write(r''' + #include + #include + + int main() { + printf("%d\n", INT_MAX < LONG_MAX); + return 0; + } + ''') + eci = ExternalCompilationInfo() + executable = plat32.compile([cfile], eci) + res = plat32.execute(executable) + self.check_res(res, '0\n') + if host_factory == Darwin_x86_64: + executable = plat64.compile([cfile], eci) + res = plat64.execute(executable) + self.check_res(res, '1\n') + + def test_longsize(self): + if platform.machine() != 'i386': + py.test.skip("i386 only") + cfile = udir.join('test_int_size.c') + cfile.write(r''' + #include + #include + + int main() { + printf("%ld\n", LONG_MAX); + return 0; + } + ''') + eci = ExternalCompilationInfo() + executable = self.platform.compile([cfile], eci) + res = self.platform.execute(executable) + self.check_res(res, str(sys.maxint) + '\n') + + def test_32bit_makefile(self): + if platform.machine() != 'i386': + py.test.skip("i386 only") + plat32 = Darwin_i386() + plat64 = Darwin_x86_64() + eci = ExternalCompilationInfo() + cfile_content =r''' + #include + #include + + int main() { + printf("%d\n", INT_MAX < LONG_MAX); + return 0; + } + ''' + + tmpdir = udir.join('32_makefile' + self.__class__.__name__).ensure(dir=1) + cfile = tmpdir.join('test_int_size.c') + cfile.write(cfile_content) + mk = plat32.gen_makefile([cfile], ExternalCompilationInfo(), + path=tmpdir) + mk.write() + plat32.execute_makefile(mk) + res = plat32.execute(tmpdir.join('test_int_size')) + self.check_res(res, '0\n') + if host_factory == Darwin_x86_64: + tmpdir = udir.join('64_makefile' + self.__class__.__name__).ensure(dir=1) + cfile = tmpdir.join('test_int_size.c') + cfile.write(cfile_content) + mk = plat64.gen_makefile([cfile], ExternalCompilationInfo(), + path=tmpdir) + mk.write() + plat64.execute_makefile(mk) + res = plat64.execute(tmpdir.join('test_int_size')) + self.check_res(res, '1\n') + Modified: pypy/branch/virtual-forcing/pypy/translator/platform/test/test_platform.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/platform/test/test_platform.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/platform/test/test_platform.py Tue Dec 29 10:12:39 2009 @@ -115,6 +115,16 @@ finally: del os.environ['_SOME_VARIABLE_2'] + def test_key(self): + class XPlatform(Platform): + relevant_environ = ['CPATH'] + + def __init__(self): + self.cc = 'xcc' + x = XPlatform() + res = x.key() + assert res.startswith('XPlatform cc=xcc CPATH=') + def test_equality(): class X(Platform): def __init__(self): From arigo at codespeak.net Tue Dec 29 10:24:17 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 10:24:17 +0100 (CET) Subject: [pypy-svn] r70327 - in pypy/branch/virtual-forcing/pypy/jit/metainterp: . test Message-ID: <20091229092417.1CDDA168021@codespeak.net> Author: arigo Date: Tue Dec 29 10:24:16 2009 New Revision: 70327 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py Log: Fix the test. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py Tue Dec 29 10:24:16 2009 @@ -1705,6 +1705,3 @@ def __str__(self): return "using virtualizable array in illegal way in %r" % ( self.args[0],) - -class ForcingVirtualRef(Exception): - pass Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py Tue Dec 29 10:24:16 2009 @@ -2,7 +2,7 @@ from pypy.rlib import jit from pypy.jit.metainterp import support, typesystem from pypy.jit.metainterp.policy import JitPolicy -from pypy.jit.metainterp.codewriter import CodeWriter, ForcingVirtualRef +from pypy.jit.metainterp.codewriter import CodeWriter from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.translator.translator import graphof from pypy.rpython.lltypesystem.rbuiltin import ll_instantiate @@ -101,6 +101,8 @@ def make_graphs(self, func, values, type_system='lltype'): class FakeMetaInterpSd: virtualizable_info = None + class options: + listops = True def find_opcode(self, name): default = len(self.opname_to_index) return self.opname_to_index.setdefault(name, default) @@ -460,8 +462,11 @@ cw = CodeWriter(self.rtyper) cw.candidate_graphs = [graphs[0]] cw._start(self.metainterp_sd, None) - py.test.raises(ForcingVirtualRef, cw.make_one_bytecode, - (graphs[0], None), False) # assert it does not work + jitcode = cw.make_one_bytecode((graphs[0], None), False) + assert 'virtual_ref' in jitcode._source + # the call vref() becomes a residual call to a helper that contains + # itself a copy of the call. + assert 'residual_call' in jitcode._source class ImmutableFieldsTests: From arigo at codespeak.net Tue Dec 29 10:29:38 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 10:29:38 +0100 (CET) Subject: [pypy-svn] r70328 - pypy/trunk/pypy/jit/backend/test Message-ID: <20091229092938.37315168021@codespeak.net> Author: arigo Date: Tue Dec 29 10:29:37 2009 New Revision: 70328 Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py Log: Improve the test to also check that the arguments arrive in the right order. Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Tue Dec 29 10:29:37 2009 @@ -466,12 +466,13 @@ assert abs(res.value - 4.6) < 0.0001 def test_call_stack_alignment(self): - # test stack alignment issues, notably for Mac OS/X + # test stack alignment issues, notably for Mac OS/X. + # also test the ordering of the arguments. def func_ints(*ints): s = str(ints) + '\n' os.write(1, s) # don't remove -- crash if the stack is misaligned - return sum(ints) + return sum([(10+i)*(5+j) for i, j in enumerate(ints)]) for nb_args in range(0, 35): cpu = self.cpu @@ -486,7 +487,7 @@ res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr) - assert res.value == sum(args) + assert res.value == func_ints(*args) def test_field_basic(self): t_box, T_box = self.alloc_instance(self.T) From arigo at codespeak.net Tue Dec 29 10:39:04 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 10:39:04 +0100 (CET) Subject: [pypy-svn] r70329 - pypy/branch/virtual-forcing/pypy/jit/metainterp/test Message-ID: <20091229093904.78ED1168021@codespeak.net> Author: arigo Date: Tue Dec 29 10:39:03 2009 New Revision: 70329 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py Log: Fix more tests. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py Tue Dec 29 10:39:03 2009 @@ -106,12 +106,13 @@ readadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([adescr], [], [])) mayforcevirtdescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], forces_virtual_or_virtualizable=True)) + EffectInfo([], [], [], forces_virtual_or_virtualizable=True)) from pypy.jit.metainterp.virtualref import jit_virtual_ref_vtable from pypy.jit.metainterp.virtualref import JIT_VIRTUAL_REF virtualtokendescr = cpu.fielddescrof(JIT_VIRTUAL_REF, 'virtual_token') virtualrefindexdescr = cpu.fielddescrof(JIT_VIRTUAL_REF,'virtualref_index') + virtualforceddescr = cpu.fielddescrof(JIT_VIRTUAL_REF, 'forced') cpu.class_sizes = {cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), cpu.cast_adr_to_int(node_vtable_adr2): cpu.sizeof(NODE2), Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py Tue Dec 29 10:39:03 2009 @@ -2331,13 +2331,42 @@ """ self.optimize_loop(ops, 'Not, Not, Not, Not', ops) - def test_vref_nonvirtual(self): + def test_vref_nonvirtual_nonescape(self): ops = """ [p1] p2 = virtual_ref(p1, 5) + virtual_ref_finish(p2, p1) jump(p1) """ - py.test.raises(compile.GiveUp, self.optimize_loop, ops, 'Not', ops) + expected = """ + [p1] + i0 = force_token() + jump(p1) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_vref_nonvirtual_escape(self): + ops = """ + [p1] + p2 = virtual_ref(p1, 5) + escape(p2) + virtual_ref_finish(p2, p1) + jump(p1) + """ + expected = """ + [p1] + i0 = force_token() + p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, i0, descr=virtualtokendescr) + setfield_gc(p2, 5, descr=virtualrefindexdescr) + escape(p2) + setfield_gc(p2, p1, descr=virtualforceddescr) + setfield_gc(p2, 0, descr=virtualtokendescr) + jump(p1) + """ + # XXX we should optimize a bit more the case of a nonvirtual. + # in theory it is enough to just do 'p2 = p1'. + self.optimize_loop(ops, 'Not', expected) def test_vref_virtual_1(self): ops = """ From arigo at codespeak.net Tue Dec 29 10:40:43 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 10:40:43 +0100 (CET) Subject: [pypy-svn] r70330 - pypy/branch/virtual-forcing/pypy/jit/metainterp/test Message-ID: <20091229094043.C0120168021@codespeak.net> Author: arigo Date: Tue Dec 29 10:40:43 2009 New Revision: 70330 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Fix. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py Tue Dec 29 10:40:43 2009 @@ -106,7 +106,8 @@ readadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([adescr], [], [])) mayforcevirtdescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], forces_virtual_or_virtualizable=True)) + EffectInfo([nextdescr], [], [], + forces_virtual_or_virtualizable=True)) from pypy.jit.metainterp.virtualref import jit_virtual_ref_vtable from pypy.jit.metainterp.virtualref import JIT_VIRTUAL_REF From arigo at codespeak.net Tue Dec 29 11:20:41 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 11:20:41 +0100 (CET) Subject: [pypy-svn] r70333 - pypy/branch/virtual-forcing/pypy/jit/metainterp/test Message-ID: <20091229102041.CEA67168021@codespeak.net> Author: arigo Date: Tue Dec 29 11:20:41 2009 New Revision: 70333 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py Log: Finish fixing the tests. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizefindnode.py Tue Dec 29 11:20:41 2009 @@ -68,6 +68,12 @@ nextdescr = cpu.fielddescrof(NODE, 'next') otherdescr = cpu.fielddescrof(NODE2, 'other') + NODEOBJ = lltype.GcStruct('NODEOBJ', ('parent', OBJECT), + ('ref', lltype.Ptr(OBJECT))) + nodeobj = lltype.malloc(NODEOBJ) + nodeobjvalue = lltype.cast_opaque_ptr(llmemory.GCREF, nodeobj) + refdescr = cpu.fielddescrof(NODEOBJ, 'ref') + arraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Signed)) floatarraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Float)) @@ -114,10 +120,14 @@ virtualtokendescr = cpu.fielddescrof(JIT_VIRTUAL_REF, 'virtual_token') virtualrefindexdescr = cpu.fielddescrof(JIT_VIRTUAL_REF,'virtualref_index') virtualforceddescr = cpu.fielddescrof(JIT_VIRTUAL_REF, 'forced') + jvr_vtable_adr = llmemory.cast_ptr_to_adr(jit_virtual_ref_vtable) - cpu.class_sizes = {cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), - cpu.cast_adr_to_int(node_vtable_adr2): cpu.sizeof(NODE2), - cpu.cast_adr_to_int(u_vtable_adr): cpu.sizeof(U)} + cpu.class_sizes = { + cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), + cpu.cast_adr_to_int(node_vtable_adr2): cpu.sizeof(NODE2), + cpu.cast_adr_to_int(u_vtable_adr): cpu.sizeof(U), + cpu.cast_adr_to_int(jvr_vtable_adr): cpu.sizeof(JIT_VIRTUAL_REF), + } namespace = locals() class OOtypeMixin(object): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py Tue Dec 29 11:20:41 2009 @@ -2381,19 +2381,30 @@ setfield_gc(p0, p2, descr=nextdescr) call_may_force(i1, descr=mayforcevirtdescr) guard_not_forced() [i1] + virtual_ref_finish(p2, p1) setfield_gc(p0, NULL, descr=nextdescr) jump(p0, i1) """ expected = """ [p0, i1] i3 = force_token() + # p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) setfield_gc(p2, i3, descr=virtualtokendescr) setfield_gc(p2, 3, descr=virtualrefindexdescr) setfield_gc(p0, p2, descr=nextdescr) + # call_may_force(i1, descr=mayforcevirtdescr) guard_not_forced() [i1] setfield_gc(p0, NULL, descr=nextdescr) + # + p1 = new_with_vtable(ConstClass(node_vtable)) + p1b = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1b, 252, descr=valuedescr) + setfield_gc(p1, p1b, descr=nextdescr) + setfield_gc(p2, p1, descr=virtualforceddescr) + setfield_gc(p2, 0, descr=virtualtokendescr) + # jump(p0, i1) """ self.optimize_loop(ops, 'Not, Not', expected) @@ -2411,26 +2422,76 @@ p2 = virtual_ref(p1, 2) setfield_gc(p0, p2, descr=nextdescr) call_may_force(i1, descr=mayforcevirtdescr) - guard_not_forced(descr=fdescr) [p1] + guard_not_forced(descr=fdescr) [p2, p1] + virtual_ref_finish(p2, p1) setfield_gc(p0, NULL, descr=nextdescr) jump(p0, i1) """ expected = """ [p0, i1] i3 = force_token() + # p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) setfield_gc(p2, i3, descr=virtualtokendescr) setfield_gc(p2, 2, descr=virtualrefindexdescr) setfield_gc(p0, p2, descr=nextdescr) + # call_may_force(i1, descr=mayforcevirtdescr) - guard_not_forced(descr=fdescr) [i1] + guard_not_forced(descr=fdescr) [p2, i1] setfield_gc(p0, NULL, descr=nextdescr) + # + p1 = new_with_vtable(ConstClass(node_vtable)) + p1b = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1b, i1, descr=valuedescr) + setfield_gc(p1, p1b, descr=nextdescr) + setfield_gc(p2, p1, descr=virtualforceddescr) + setfield_gc(p2, 0, descr=virtualtokendescr) + # jump(p0, i1) """ # the point of this test is that 'i1' should show up in the fail_args # of 'guard_not_forced', because it was stored in the virtual 'p1b'. self.optimize_loop(ops, 'Not, Not', expected) - self.check_expanded_fail_descr('''p1 + self.check_expanded_fail_descr('''p2, p1 + where p1 is a node_vtable, nextdescr=p1b + where p1b is a node_vtable, valuedescr=i1 + ''') + + def test_vref_virtual_and_lazy_setfield(self): + self.make_fail_descr() + ops = """ + [p0, i1] + # + p1 = new_with_vtable(ConstClass(node_vtable)) + p1b = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1b, i1, descr=valuedescr) + setfield_gc(p1, p1b, descr=nextdescr) + # + p2 = virtual_ref(p1, 2) + setfield_gc(p0, p2, descr=refdescr) + call(i1, descr=nonwritedescr) + guard_no_exception(descr=fdescr) [p2, p1] + virtual_ref_finish(p2, p1) + setfield_gc(p0, NULL, descr=refdescr) + jump(p0, i1) + """ + expected = """ + [p0, i1] + i3 = force_token() + call(i1, descr=nonwritedescr) + guard_no_exception(descr=fdescr) [i3, i1, p0] + setfield_gc(p0, NULL, descr=refdescr) + jump(p0, i1) + """ + self.optimize_loop(ops, 'Not, Not', expected) + # the fail_args contain [i3, i1, p0]: + # - i3 is from the virtual expansion of p2 + # - i1 is from the virtual expansion of p1 + # - p0 is from the extra pendingfields + self.loop.inputargs[0].value = self.nodeobjvalue + self.check_expanded_fail_descr('''p2, p1 + p0.refdescr = p2 + where p2 is a jit_virtual_ref_vtable, virtualtokendescr=i3, virtualrefindexdescr=2 where p1 is a node_vtable, nextdescr=p1b where p1b is a node_vtable, valuedescr=i1 ''') From arigo at codespeak.net Tue Dec 29 11:29:07 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 11:29:07 +0100 (CET) Subject: [pypy-svn] r70334 - pypy/branch/virtual-forcing/pypy/jit/metainterp/test Message-ID: <20091229102907.CC88A168021@codespeak.net> Author: arigo Date: Tue Dec 29 11:29:07 2009 New Revision: 70334 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Log: Fix these tests, mostly by expecting less new's now. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualref.py Tue Dec 29 11:29:07 2009 @@ -102,8 +102,7 @@ x = X() x.n = n exctx.topframeref = vref = virtual_ref(x) - # here, 'x' should be virtual. (This is ensured because - # we call virtual_ref(x).) + # here, 'x' should be virtual exctx.topframeref = vref_None virtual_ref_finish(x) # 'x' and 'vref' can randomly escape after the call to @@ -113,7 +112,42 @@ return 1 # self.meta_interp(f, [10]) - self.check_loops(new_with_vtable=2) # the vref, and later the X + self.check_loops(new_with_vtable=2) # the vref and the X + self.check_aborted_count(0) + + def test_simple_all_removed(self): + myjitdriver = JitDriver(greens = [], reds = ['n']) + # + A = lltype.GcArray(lltype.Signed) + class XY: + pass + class ExCtx: + pass + exctx = ExCtx() + # + @dont_look_inside + def externalfn(n): + return 1 + # + def f(n): + while n > 0: + myjitdriver.can_enter_jit(n=n) + myjitdriver.jit_merge_point(n=n) + xy = XY() + xy.next1 = lltype.malloc(A, 0) + xy.next2 = lltype.malloc(A, 0) + xy.next3 = lltype.malloc(A, 0) + exctx.topframeref = virtual_ref(xy) + n -= externalfn(n) + exctx.topframeref = vref_None + xy.next1 = lltype.nullptr(A) + xy.next2 = lltype.nullptr(A) + xy.next3 = lltype.nullptr(A) + virtual_ref_finish(xy) + # + self.meta_interp(f, [15]) + self.check_loops(new_with_vtable=0, # all virtualized + new_array=0) self.check_aborted_count(0) def test_simple_no_access(self): @@ -269,8 +303,8 @@ # res = self.meta_interp(f, [30]) assert res == 13 - self.check_loops(new_with_vtable=2, # the vref, XY() at the end - new_array=0) # but not next1/2/3 + self.check_loops(new_with_vtable=0, # all virtualized in the n!=13 loop + new_array=0) self.check_loop_count(1) self.check_aborted_count(0) @@ -319,7 +353,7 @@ res = self.meta_interp(f, [72]) assert res == 6 self.check_loop_count(2) # the loop and the bridge - self.check_loops(new_with_vtable=3, # loop: vref, xy; bridge: xy + self.check_loops(new_with_vtable=2, # loop: nothing; bridge: vref, xy new_array=2) # bridge: next4, next5 self.check_aborted_count(0) From electronicru at codespeak.net Tue Dec 29 11:33:09 2009 From: electronicru at codespeak.net (electronicru at codespeak.net) Date: Tue, 29 Dec 2009 11:33:09 +0100 (CET) Subject: [pypy-svn] r70335 - pypy/branch/micronumpy/pypy/module/micronumpy Message-ID: <20091229103309.EFCCC168021@codespeak.net> Author: electronicru Date: Tue Dec 29 11:33:09 2009 New Revision: 70335 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/mdarray.py pypy/branch/micronumpy/pypy/module/micronumpy/ndarray.py pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py Log: I have modified some issues, but I have no time to implement slices fully. Going out for 20 or more days. Happy New Year! Modified: pypy/branch/micronumpy/pypy/module/micronumpy/mdarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/mdarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/mdarray.py Tue Dec 29 11:33:09 2009 @@ -25,45 +25,123 @@ current *= d return pos -def create_mdarray(data_type, unwrap, coerce): - class MultiDimArray(BaseNumArray): - def __init__(self, space, shape): - self.shape = shape - self.space = space - size = 1 - for dimension in shape: - size *= dimension - self.storage = [data_type(0.0)] * size - make_sure_not_resized(self.storage) - - def _unpack_indexes(self, space, w_index): - indexes = [space.int_w(w_i) for w_i in space.fixedview(w_index)] - if len(indexes) != len(self.shape): - raise OperationError(space.w_IndexError, space.wrap( - 'Wrong index')) - return indexes - - def getitem(self, w_index): - space = self.space - indexes = self._unpack_indexes(space, w_index) - pos = compute_pos(space, indexes, self.shape) - return space.wrap(self.storage[pos]) - - def setitem(self, w_index, w_value): - space = self.space - indexes = self._unpack_indexes(space, w_index) - pos = compute_pos(space, indexes, self.shape) - self.storage[pos] = coerce(space, w_value) - return space.w_None #XXX: necessary? - - def len(self): - space = self.space - return space.wrap(self.shape[0]) - return MultiDimArray +def compute_slices(space, slices, dim): + strides = [] + i = 1 + for d in dim: + strides.append(i) + i *= d + strides.reverse() + length = i + shape = [] + sliceout = [] + for v in space.unpackiterable(slices): + sliceoutnew=[] #FIXME: Not RPYthon - there are no slices. + if space.is_true(space.isinstance(v, space.w_slice)): + sl=space.unwrap(v) + if sl.step<0: + reverse=True + sl=slice(sl.stop, sl.start, -sl.step) + else: + reverse=False + stride=strides.pop(0) + if sl.step == 1: + newsl = [slice(stride*sl.start, stride*sl.stop)] + else: + newsl = [] + for i in range(sl.start, sl.stop, sl.step): + newsl.append(slice(stride*i, stride*(i+1))) + + if reverse: + newsl.reverse() + + shape.append((sl.stop-sl.start)//sl.step) + + #here multiple old slices x new slices. + for sl in sliceout: + for sl2 in newsl: + pass #I have no time + + else: + #extract item from slices, without appending to shape + + sliceout = sliceoutnew + + return shape, sliceout + +#Was undetectable +class MultiDimArrayAbstract(BaseNumArray): + + def dtype(self, data): + return self.__class__.data_type(data) + + def unwrap(self, w_data): #XXX: NOT USED + return self.__class__.data_w(w_data) + + def coerce(self, data): + return self.__class__.data_coerce(data) + + def __init__(self, space, shape): + self.shape = shape + self.space = space + size = 1 + for dimension in shape: + size *= dimension + self.storage = [self.dtype(0.0)] * size + make_sure_not_resized(self.storage) + + def _unpack_indexes(self, space, w_index): + indexes = [space.int_w(w_i) for w_i in space.fixedview(w_index)] + if len(indexes) != len(self.shape): + raise OperationError(space.w_IndexError, space.wrap( + 'Wrong index')) + return indexes + + def getitem(self, w_index): + space = self.space + indexes = self._unpack_indexes(space, w_index) + pos = compute_pos(space, indexes, self.shape) + return space.wrap(self.storage[pos]) + + def setitem(self, w_index, w_value): + space = self.space + indexes = self._unpack_indexes(space, w_index) + pos = compute_pos(space, indexes, self.shape) + self.storage[pos] = self.coerce(space, w_value) + return space.w_None #XXX: necessary? + + def len(self): + space = self.space + return space.wrap(self.shape[0]) + + def load_iterable(self, space, w_values): + self._load_iter(space, w_values, 0) + + def _load_iter(self, space, w_values, start): #TODO: shape check + vals=space.unpackiterable(w_values) + if space.is_true(space.isinstance(vals[0], space.w_tuple) or space.is_true(space.isinstance(vals[0], space.w_list): + idx=start + for v in vals: + add=self._load_iter(space, v, idx) + idx+=add + return idx + else: + idx=start + for v in vals: + self.storage[idx]=self.unwrap(val) + idx+=1 + return idx + + +mdarraytype=MultiDimArrayAbstract + +class MultiDimIntArray(MultiDimArrayAbstact): + data_type, data_w, data_coerce = int, unwrap_int, coerce_int -MultiDimIntArray = create_mdarray(int, unwrap_int, coerce_int) MultiDimArray = MultiDimIntArray #XXX: compatibility -MultiDimFloatArray = create_mdarray(float, unwrap_float, coerce_float) + +class MultiDimFloatArray(MultiDimArrayAbstact): + data_type, data_w, data_coerce = float, unwrap_float, coerce_float class ResultFactory(object): def __init__(self, space): Modified: pypy/branch/micronumpy/pypy/module/micronumpy/ndarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/ndarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/ndarray.py Tue Dec 29 11:33:09 2009 @@ -50,7 +50,7 @@ self.space = space if w_dtype == space.w_None: #TODO: infer type from w_values (better than this) - w_dtype = space.type(space.fixedview(w_values)[0]) #FIXME: Allocates an entire array and throws it away! + w_dtype = space.type(space.getitem(w_values, space.wrap(0))) #FIXED self.dtype = w_dtype if w_shape == space.w_None: @@ -64,7 +64,7 @@ length = shape_w[0] self.array = sdresult(space, w_dtype)(space, length) else: - self.array = mdresult(space, w_dtype)(space, unpack_shape(space, w_shape)) + self.array = mdresult(space, w_dtype)(space, shape_w) #w_shape still may be w_None except KeyError, e: raise OperationError(space.w_NotImplementedError, space.wrap("Haven't implemented generic array yet!")) Modified: pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/sdarray.py Tue Dec 29 11:33:09 2009 @@ -13,87 +13,104 @@ from pypy.module.micronumpy.dtype import unwrap_float32, coerce_float32, float32 def create_sdarray(data_type, unwrap, coerce): - class NumArray(BaseNumArray): - def __init__(self, space, length): - self.shape = (1,) - self.length = length - self.space = space - self.storage = [data_type(0.0)] * length - make_sure_not_resized(self.storage) - - mul = mul_operation() - div = div_operation() - add = add_operation() - sub = sub_operation() - copy = copy_operation() - - def create_scalar_op(f): - def scalar_operation(self, space, source, w_x): - space = self.space - x = self.coerce(space, w_x) - for i in range(source.length): - self.storage[i] = f(source.storage[i], x) - return scalar_operation +class NumArrayAbstract(BaseNumArray): + + def dtype(self, data): + return self.__class__.data_type(data) + + def unwrap(self, w_data): + return self.__class__.data_w(w_data) + + def coerce(self, data): + return self.__class__.data_coerce(data) + + def __init__(self, space, length): + self.shape = (length,) #As in numpy + self.length = length + self.space = space + self.storage = [self.dtype(0.0)] * length + make_sure_not_resized(self.storage) + + mul = mul_operation() + div = div_operation() + add = add_operation() + sub = sub_operation() + copy = copy_operation() + + def create_scalar_op(f): + def scalar_operation(self, space, source, w_x): + space = self.space + x = self.coerce(space, w_x) + for i in range(source.length): + self.storage[i] = f(source.storage[i], x) + return scalar_operation - mul_scalar = create_scalar_op(mul) + mul_scalar = create_scalar_op(mul) # div_scalar = create_scalar_op(div) # add_scalar = create_scalar_op(add) # sub_scalar = create_scalar_op(sub) - def create_fixedview_op(f): - def fixedview_operation(self, w_xs): - space = self.space - try: - xs = space.fixedview(w_xs, len(self.storage)) - except UnpackValueError, e: - # w_xs is of the wrong size - raise OperationError(space.w_ValueError, - space.wrap("shape mismatch: objects cannot be broadcast to the same shape")) - - i = 0 - for w_x in xs: - self.storage[i] = f(source.storage[i], self.coerce(w_x)) #TODO: probably shouldn't coerce - i += 1 - return result - return fixedview_operation - - copy_iterable = create_fixedview_op(copy) + def create_fixedview_op(f): + def fixedview_operation(self, w_xs): + space = self.space + try: + xs = space.fixedview(w_xs, len(self.storage)) + except UnpackValueError, e: + # w_xs is of the wrong size + raise OperationError(space.w_ValueError, + space.wrap("shape mismatch: objects cannot be broadcast to the same shape")) - def load_iterable(self, space, w_values): #FIXME: less than ideal i = 0 - for x in space.fixedview(w_values, self.length): - self.storage[i] = unwrap(space, x) + for w_x in xs: + self.storage[i] = f(source.storage[i], self.coerce(w_x)) #TODO: probably shouldn't coerce i += 1 + return result + return fixedview_operation - def getitem(self, w_index): - space = self.space - index = space.int_w(w_index) - try: - return space.wrap(self.storage[index]) - except IndexError: - raise OperationError(space.w_IndexError, - space.wrap("list index out of range")) + copy_iterable = create_fixedview_op(copy) - def setitem(self, w_index, w_value): - space = self.space - index = space.int_w(w_index) - try: - self.storage[index] = coerce(space, w_value) - except IndexError: - raise OperationError(space.w_IndexError, - space.wrap("list index out of range")) - return space.w_None + def load_iterable(self, space, w_values): #FIXME: less than ideal + i = 0 + for x in space.fixedview(w_values, self.length): + self.storage[i] = self.unwrap(space, x) + i += 1 + + def getitem(self, w_index): + space = self.space + index = space.int_w(w_index) + try: + return space.wrap(self.storage[index]) + except IndexError: + raise OperationError(space.w_IndexError, + space.wrap("list index out of range")) + + def setitem(self, w_index, w_value): + space = self.space + index = space.int_w(w_index) + try: + self.storage[index] = self.coerce(space, w_value) + except IndexError: + raise OperationError(space.w_IndexError, + space.wrap("list index out of range")) + return space.w_None + + def len(self): + space = self.space + return space.wrap(len(self.storage)) + +sdarraytype=NumArrayAbstact + +class IntArray(NumArrayAbstact): + data_type, data_w, data_coerce = int, unwrap_int, coerce_int - def len(self): - space = self.space - return space.wrap(len(self.storage)) +NumArray = IntArray #XXX: compatibility + +class FloatArray(NumArrayAbstact): + data_type, data_w, data_coerce = float, unwrap_float, coerce_float - return NumArray +class Float32Array(NumArrayAbstract): + data_type, data_w, data_coerce = float32, unwrap_float32, coerce_float32 -IntArray = create_sdarray(int, unwrap_int, coerce_int) -NumArray = IntArray # FIXME: compatibility for now -FloatArray = create_sdarray(float, unwrap_float, coerce_float) -Float32Array = create_sdarray(float32, unwrap_float32, coerce_float32) GenericArray = None class ResultFactory(object): Modified: pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/ufunc.py Tue Dec 29 11:33:09 2009 @@ -1,11 +1,16 @@ from pypy.module.micronumpy.ndarray import array, zeros, ndarray +from pypy.module.micronumpy.sdarray import sdarrytype +from pypy.module.micronumpy.mdarray import mdarrytype from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.interpreter.error import OperationError -def minimum(space, w_a, w_b): +def _assert_both(space, w_a, w_b): if not isinstance(w_a, ndarray) or not isinstance(w_b, ndarray): raise OperationError(space.w_TypeError, space.wrap("expecting ndarray object")) + +def minimum(space, w_a, w_b): + _assert_both(w_a, w_b) if w_a.array.length != w_b.array.length: raise OperationError(space.w_ValueError, space.wrap("minimum of arrays of different length")) @@ -19,3 +24,15 @@ res.array.storage[i] = two return space.wrap(res) minimum.unwrap_spec = [ObjSpace, W_Root, W_Root] + +def dot(space, w_a, w_b): + _assert_both(w_a, w_b) + if len(w_b.array.shape)==1: + w_b_new=zeros(space, space.newtuple([space.wrap(1), space.wrap(w_b.array.shape[0])])) + for idx, value in enumerate(w_b.array.storage): + w_b_new.array.storage[idx]=value + w_b=w_b_new + + #waiting for slice. + + From arigo at codespeak.net Tue Dec 29 14:08:24 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 14:08:24 +0100 (CET) Subject: [pypy-svn] r70336 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091229130824.43BC816800D@codespeak.net> Author: arigo Date: Tue Dec 29 14:08:23 2009 New Revision: 70336 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: Bug test and fix, with an XXX. Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Tue Dec 29 14:08:23 2009 @@ -754,11 +754,11 @@ def optimize_SETFIELD_GC(self, op): value = self.getvalue(op.args[0]) + fieldvalue = self.getvalue(op.args[1]) if value.is_virtual(): - value.setfield(op.descr, self.getvalue(op.args[1])) + value.setfield(op.descr, fieldvalue) else: value.ensure_nonnull() - fieldvalue = self.getvalue(op.args[1]) self.heap_op_optimizer.optimize_SETFIELD_GC(op, value, fieldvalue) def optimize_NEW_WITH_VTABLE(self, op): @@ -863,9 +863,14 @@ d[value] = fieldvalue def read_cached_field(self, descr, value): + # XXX self.cached_fields and self.lazy_setfields should probably + # be merged somehow d = self.cached_fields.get(descr, None) if d is None: - return None + op = self.lazy_setfields.get(descr, None) + if op is None: + return None + return self.optimizer.getvalue(op.args[1]) return d.get(value, None) def cache_arrayitem_value(self, descr, value, indexvalue, fieldvalue, write=False): Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Tue Dec 29 14:08:23 2009 @@ -1369,6 +1369,28 @@ """ self.optimize_loop(ops, 'Not, Not, Not, Not', expected) + def test_duplicate_setfield_5(self): + ops = """ + [p0, i1] + p1 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(p0, p1, descr=nextdescr) + setfield_raw(i1, i1, descr=valuedescr) # random op with side-effects + p2 = getfield_gc(p0, descr=nextdescr) + i2 = getfield_gc(p2, descr=valuedescr) + setfield_gc(p0, NULL, descr=nextdescr) + escape(i2) + jump(p0, i1) + """ + expected = """ + [p0, i1] + setfield_raw(i1, i1, descr=valuedescr) + setfield_gc(p0, NULL, descr=nextdescr) + escape(i1) + jump(p0, i1) + """ + self.optimize_loop(ops, 'Not, Not', expected) + def test_duplicate_setfield_sideeffects_1(self): ops = """ [p1, i1, i2] From arigo at codespeak.net Tue Dec 29 15:19:49 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 15:19:49 +0100 (CET) Subject: [pypy-svn] r70337 - pypy/trunk/pypy/translator/benchmark Message-ID: <20091229141949.F2135168023@codespeak.net> Author: arigo Date: Tue Dec 29 15:19:49 2009 New Revision: 70337 Modified: pypy/trunk/pypy/translator/benchmark/bench-custom.py pypy/trunk/pypy/translator/benchmark/jitbench.py pypy/trunk/pypy/translator/benchmark/result.py Log: When at least one test fails, exit with a non-zero exit code. This should let buildbot report it as an error. Modified: pypy/trunk/pypy/translator/benchmark/bench-custom.py ============================================================================== --- pypy/trunk/pypy/translator/benchmark/bench-custom.py (original) +++ pypy/trunk/pypy/translator/benchmark/bench-custom.py Tue Dec 29 15:19:49 2009 @@ -44,6 +44,7 @@ sys.stdout.flush() refs = {} + final_error_count = 0 if not options.nocpython: exes = full_pythons + exes @@ -52,7 +53,10 @@ if i is not None: for exe in exes: for b in benchmarks: - benchmark_result.result(exe, allowcreate=True).run_benchmark(b, verbose=options.verbose) + br = benchmark_result.result(exe, allowcreate=True) + result = br.run_benchmark(b, verbose=options.verbose) + if not result: + final_error_count += 1 if options.relto: relto = options.relto @@ -83,6 +87,9 @@ print row print + if final_error_count: + raise SystemExit("%d benchmark run(s) failed" % final_error_count) + if __name__ == '__main__': from optparse import OptionParser parser = OptionParser() Modified: pypy/trunk/pypy/translator/benchmark/jitbench.py ============================================================================== --- pypy/trunk/pypy/translator/benchmark/jitbench.py (original) +++ pypy/trunk/pypy/translator/benchmark/jitbench.py Tue Dec 29 15:19:49 2009 @@ -13,9 +13,17 @@ os.chdir(os.path.dirname(sys.argv[0]) or '.') +errors = [] + for sizefactor in sizefactors: for executable in executables: sys.argv[1:] = [executable, '--pickle=jitbench.benchmark_result', '-v', '--no-cpython', '--size-factor=%d' % sizefactor] - execfile('bench-custom.py') + try: + execfile('bench-custom.py') + except SystemExit, e: + errors.append(str(e)) + +if errors: + raise SystemExit('\n'.join(errors)) Modified: pypy/trunk/pypy/translator/benchmark/result.py ============================================================================== --- pypy/trunk/pypy/translator/benchmark/result.py (original) +++ pypy/trunk/pypy/translator/benchmark/result.py Tue Dec 29 15:19:49 2009 @@ -88,7 +88,7 @@ def run_benchmark(self, benchmark, verbose=False): self.asc_goods[benchmark.name] = benchmark.asc_good if self.run_counts.get(benchmark.name, 0) > self.max_results: - return + return -1 print 'running', benchmark.name, 'for', self.exe_name, if verbose and self.pypy_rev > 0: print '[rev %d]' % self.pypy_rev, @@ -102,7 +102,7 @@ print '}' self.run_counts[benchmark.name] = self.run_counts.get(benchmark.name, 0) + 1 if new_result == '-FAILED-': - return + return 0 self.benchmarks.setdefault(benchmark.name, []).append(new_result) if benchmark.name in self.best_benchmarks: old_result = self.best_benchmarks[benchmark.name] @@ -111,6 +111,7 @@ else: new_result = min(new_result, old_result) self.best_benchmarks[benchmark.name] = new_result + return 1 def getstat(self, *args): # oh for supplied-p! From arigo at codespeak.net Tue Dec 29 16:28:43 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 16:28:43 +0100 (CET) Subject: [pypy-svn] r70338 - in pypy/branch/virtual-forcing/pypy: jit/backend/test jit/metainterp jit/metainterp/test translator/benchmark Message-ID: <20091229152843.31B6D168023@codespeak.net> Author: arigo Date: Tue Dec 29 16:28:42 2009 New Revision: 70338 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/virtual-forcing/pypy/translator/benchmark/bench-custom.py pypy/branch/virtual-forcing/pypy/translator/benchmark/jitbench.py pypy/branch/virtual-forcing/pypy/translator/benchmark/result.py Log: Merge trunk changes, -r70326:70337. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Tue Dec 29 16:28:42 2009 @@ -466,12 +466,13 @@ assert abs(res.value - 4.6) < 0.0001 def test_call_stack_alignment(self): - # test stack alignment issues, notably for Mac OS/X + # test stack alignment issues, notably for Mac OS/X. + # also test the ordering of the arguments. def func_ints(*ints): s = str(ints) + '\n' os.write(1, s) # don't remove -- crash if the stack is misaligned - return sum(ints) + return sum([(10+i)*(5+j) for i, j in enumerate(ints)]) for nb_args in range(0, 35): cpu = self.cpu @@ -486,7 +487,7 @@ res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr) - assert res.value == sum(args) + assert res.value == func_ints(*args) def test_field_basic(self): t_box, T_box = self.alloc_instance(self.T) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/optimizeopt.py Tue Dec 29 16:28:42 2009 @@ -798,11 +798,11 @@ def optimize_SETFIELD_GC(self, op): value = self.getvalue(op.args[0]) + fieldvalue = self.getvalue(op.args[1]) if value.is_virtual(): - value.setfield(op.descr, self.getvalue(op.args[1])) + value.setfield(op.descr, fieldvalue) else: value.ensure_nonnull() - fieldvalue = self.getvalue(op.args[1]) self.heap_op_optimizer.optimize_SETFIELD_GC(op, value, fieldvalue) def optimize_NEW_WITH_VTABLE(self, op): @@ -907,9 +907,14 @@ d[value] = fieldvalue def read_cached_field(self, descr, value): + # XXX self.cached_fields and self.lazy_setfields should probably + # be merged somehow d = self.cached_fields.get(descr, None) if d is None: - return None + op = self.lazy_setfields.get(descr, None) + if op is None: + return None + return self.optimizer.getvalue(op.args[1]) return d.get(value, None) def cache_arrayitem_value(self, descr, value, indexvalue, fieldvalue, write=False): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_optimizeopt.py Tue Dec 29 16:28:42 2009 @@ -1369,6 +1369,28 @@ """ self.optimize_loop(ops, 'Not, Not, Not, Not', expected) + def test_duplicate_setfield_5(self): + ops = """ + [p0, i1] + p1 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(p0, p1, descr=nextdescr) + setfield_raw(i1, i1, descr=valuedescr) # random op with side-effects + p2 = getfield_gc(p0, descr=nextdescr) + i2 = getfield_gc(p2, descr=valuedescr) + setfield_gc(p0, NULL, descr=nextdescr) + escape(i2) + jump(p0, i1) + """ + expected = """ + [p0, i1] + setfield_raw(i1, i1, descr=valuedescr) + setfield_gc(p0, NULL, descr=nextdescr) + escape(i1) + jump(p0, i1) + """ + self.optimize_loop(ops, 'Not, Not', expected) + def test_duplicate_setfield_sideeffects_1(self): ops = """ [p1, i1, i2] Modified: pypy/branch/virtual-forcing/pypy/translator/benchmark/bench-custom.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/benchmark/bench-custom.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/benchmark/bench-custom.py Tue Dec 29 16:28:42 2009 @@ -44,6 +44,7 @@ sys.stdout.flush() refs = {} + final_error_count = 0 if not options.nocpython: exes = full_pythons + exes @@ -52,7 +53,10 @@ if i is not None: for exe in exes: for b in benchmarks: - benchmark_result.result(exe, allowcreate=True).run_benchmark(b, verbose=options.verbose) + br = benchmark_result.result(exe, allowcreate=True) + result = br.run_benchmark(b, verbose=options.verbose) + if not result: + final_error_count += 1 if options.relto: relto = options.relto @@ -83,6 +87,9 @@ print row print + if final_error_count: + raise SystemExit("%d benchmark run(s) failed" % final_error_count) + if __name__ == '__main__': from optparse import OptionParser parser = OptionParser() Modified: pypy/branch/virtual-forcing/pypy/translator/benchmark/jitbench.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/benchmark/jitbench.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/benchmark/jitbench.py Tue Dec 29 16:28:42 2009 @@ -13,9 +13,17 @@ os.chdir(os.path.dirname(sys.argv[0]) or '.') +errors = [] + for sizefactor in sizefactors: for executable in executables: sys.argv[1:] = [executable, '--pickle=jitbench.benchmark_result', '-v', '--no-cpython', '--size-factor=%d' % sizefactor] - execfile('bench-custom.py') + try: + execfile('bench-custom.py') + except SystemExit, e: + errors.append(str(e)) + +if errors: + raise SystemExit('\n'.join(errors)) Modified: pypy/branch/virtual-forcing/pypy/translator/benchmark/result.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/benchmark/result.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/benchmark/result.py Tue Dec 29 16:28:42 2009 @@ -88,7 +88,7 @@ def run_benchmark(self, benchmark, verbose=False): self.asc_goods[benchmark.name] = benchmark.asc_good if self.run_counts.get(benchmark.name, 0) > self.max_results: - return + return -1 print 'running', benchmark.name, 'for', self.exe_name, if verbose and self.pypy_rev > 0: print '[rev %d]' % self.pypy_rev, @@ -102,7 +102,7 @@ print '}' self.run_counts[benchmark.name] = self.run_counts.get(benchmark.name, 0) + 1 if new_result == '-FAILED-': - return + return 0 self.benchmarks.setdefault(benchmark.name, []).append(new_result) if benchmark.name in self.best_benchmarks: old_result = self.best_benchmarks[benchmark.name] @@ -111,6 +111,7 @@ else: new_result = min(new_result, old_result) self.best_benchmarks[benchmark.name] = new_result + return 1 def getstat(self, *args): # oh for supplied-p! From arigo at codespeak.net Tue Dec 29 17:03:07 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 17:03:07 +0100 (CET) Subject: [pypy-svn] r70339 - in pypy/branch/virtual-forcing/pypy/jit/tool: . test Message-ID: <20091229160307.4B7E9168027@codespeak.net> Author: arigo Date: Tue Dec 29 17:03:06 2009 New Revision: 70339 Modified: pypy/branch/virtual-forcing/pypy/jit/tool/jitoutput.py pypy/branch/virtual-forcing/pypy/jit/tool/test/test_jitoutput.py Log: Fix the parsing tool, fix the test. Modified: pypy/branch/virtual-forcing/pypy/jit/tool/jitoutput.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/tool/jitoutput.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/tool/jitoutput.py Tue Dec 29 17:03:06 2009 @@ -26,8 +26,12 @@ (('opt_ops',), '^opt ops:\s+(\d+)$'), (('opt_guards',), '^opt guards:\s+(\d+)$'), (('forcings',), '^forcings:\s+(\d+)$'), - (('trace_too_long',), '^trace too long:\s+(\d+)$'), - (('bridge_abort',), '^bridge abort:\s+(\d+)$'), + (('abort.trace_too_long',), '^abort: trace too long:\s+(\d+)$'), + (('abort.compiling',), '^abort: compiling:\s+(\d+)$'), + (('abort.vable_escape',), '^abort: vable escape:\s+(\d+)$'), + (('nvirtuals',), '^nvirtuals:\s+(\d+)$'), + (('nvholes',), '^nvholes:\s+(\d+)$'), + (('nvreused',), '^nvreused:\s+(\d+)$'), ] class Ops(object): @@ -35,6 +39,11 @@ calls = 0 pure_calls = 0 +class Aborts(object): + trace_too_long = 0 + compiling = 0 + vable_escape = 0 + class OutputInfo(object): tracing_no = 0 tracing_time = 0.0 @@ -45,13 +54,16 @@ guards = 0 opt_ops = 0 opt_guards = 0 - trace_too_long = 0 - bridge_abort = 0 + forcings = 0 + nvirtuals = 0 + nvholes = 0 + nvreused = 0 def __init__(self): self.ops = Ops() self.recorded_ops = Ops() self.blackholed_ops = Ops() + self.abort = Aborts() def parse_prof(output): lines = output.splitlines() Modified: pypy/branch/virtual-forcing/pypy/jit/tool/test/test_jitoutput.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/tool/test/test_jitoutput.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/tool/test/test_jitoutput.py Tue Dec 29 17:03:06 2009 @@ -46,8 +46,6 @@ assert info.opt_ops == 6 assert info.opt_guards == 1 assert info.forcings == 0 - assert info.trace_too_long == 0 - assert info.bridge_abort == 0 DATA = '''Tracing: 1 0.006992 Backend: 1 0.000525 @@ -66,8 +64,12 @@ opt ops: 6 opt guards: 1 forcings: 1 -trace too long: 2 -bridge abort: 3 +abort: trace too long: 10 +abort: compiling: 11 +abort: vable escape: 12 +nvirtuals: 13 +nvholes: 14 +nvreused: 15 ''' def test_parse(): @@ -90,5 +92,9 @@ assert info.opt_ops == 6 assert info.opt_guards == 1 assert info.forcings == 1 - assert info.trace_too_long == 2 - assert info.bridge_abort == 3 + assert info.abort.trace_too_long == 10 + assert info.abort.compiling == 11 + assert info.abort.vable_escape == 12 + assert info.nvirtuals == 13 + assert info.nvholes == 14 + assert info.nvreused == 15 From arigo at codespeak.net Tue Dec 29 17:23:30 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 17:23:30 +0100 (CET) Subject: [pypy-svn] r70340 - pypy/trunk/pypy/doc Message-ID: <20091229162330.E91E7168025@codespeak.net> Author: arigo Date: Tue Dec 29 17:23:30 2009 New Revision: 70340 Modified: pypy/trunk/pypy/doc/coding-guide.txt Log: Use the common style instead of an absolute hard-coded path. Modified: pypy/trunk/pypy/doc/coding-guide.txt ============================================================================== --- pypy/trunk/pypy/doc/coding-guide.txt (original) +++ pypy/trunk/pypy/doc/coding-guide.txt Tue Dec 29 17:23:30 2009 @@ -160,7 +160,8 @@ An example can be found in the current implementation which is quite elegant: For the definition of all the opcodes of the Python interpreter, the module ``dis`` is imported and used to initialize our -bytecode interpreter. (See ``__initclass__`` in `pyopcode.py`_). This +bytecode interpreter. (See ``__initclass__`` in +`pypy/interpreter/pyopcode.py`_). This saves us from adding extra modules to PyPy. The import code is run at startup time, and we are allowed to use the CPython builtin import function. @@ -173,8 +174,6 @@ enables the code generator to emit efficient machine level replacements for pure integer objects, for instance. -.. _`pyopcode.py`: http://codespeak.net/svn/pypy/dist/pypy/interpreter/pyopcode.py - Restricted Python ================= From arigo at codespeak.net Tue Dec 29 17:33:22 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 17:33:22 +0100 (CET) Subject: [pypy-svn] r70341 - pypy/trunk/pypy/doc Message-ID: <20091229163322.49170168025@codespeak.net> Author: arigo Date: Tue Dec 29 17:33:21 2009 New Revision: 70341 Modified: pypy/trunk/pypy/doc/getting-started.txt Log: Document svn/pypy/trunk instead of svn/pypy/dist. Give the appropriate warnings about checking that out. Modified: pypy/trunk/pypy/doc/getting-started.txt ============================================================================== --- pypy/trunk/pypy/doc/getting-started.txt (original) +++ pypy/trunk/pypy/doc/getting-started.txt Tue Dec 29 17:33:21 2009 @@ -43,14 +43,23 @@ If you choose to use subversion, you must issue the following command on your command line, DOS box, or terminal:: - svn co http://codespeak.net/svn/pypy/dist pypy-dist + svn co http://codespeak.net/svn/pypy/trunk pypy-trunk + +This will check out the subversion head and place it into a directory +named ``pypy-trunk``, and will get you the PyPy source in +``pypy-trunk/pypy`` and documentation files in ``pypy-trunk/pypy/doc``. +We try to ensure that the head is always stable, but it might +occasionally be broken. You may want to check out `our nightly tests:`_ +find a revision (5-digit number) that passed at least the +``{own}`` and ``{applevel}`` tests (corresponding to a ``+`` sign on the +line ``success``) and then check out using:: + + svn co -rXXXXX http://codespeak.net/svn/pypy/trunk pypy-trunk + +where XXXXX is the revision number. + +.. _`our nightly tests:`: http://codespeak.net:8099/summary?branch= -This will check out the most recent stable release from subversion and -place it into a directory named ``pypy-dist``, and will get you the PyPy -source in ``pypy-dist/pypy`` and documentation files in -``pypy-dist/pypy/doc``. If you would prefer to check out the "cutting edge" -version of PyPy - which may not always be stable! - then check out -from ``http://codespeak.net/svn/pypy/trunk`` intead. Where to go from here ---------------------- From pedronis at codespeak.net Tue Dec 29 17:39:55 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 29 Dec 2009 17:39:55 +0100 (CET) Subject: [pypy-svn] r70342 - pypy/trunk/pypy/doc Message-ID: <20091229163955.ECAE9168022@codespeak.net> Author: pedronis Date: Tue Dec 29 17:39:54 2009 New Revision: 70342 Modified: pypy/trunk/pypy/doc/getting-started.txt Log: I would use this formulation really Modified: pypy/trunk/pypy/doc/getting-started.txt ============================================================================== --- pypy/trunk/pypy/doc/getting-started.txt (original) +++ pypy/trunk/pypy/doc/getting-started.txt Tue Dec 29 17:39:54 2009 @@ -35,8 +35,8 @@ Before you can play with PyPy, you will need to obtain a copy of the sources. This can be done either by `downloading them from the download page`_ or by checking them out from the -repository using subversion. We suggest using subversion as it -offers access to the most recent versions. +repository using subversion. We suggest using subversion if one +wants to access the current development. .. _`downloading them from the download page`: download.html From arigo at codespeak.net Tue Dec 29 17:49:06 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 17:49:06 +0100 (CET) Subject: [pypy-svn] r70343 - pypy/trunk/pypy/module/imp/test Message-ID: <20091229164906.9C1A4168022@codespeak.net> Author: arigo Date: Tue Dec 29 17:49:05 2009 New Revision: 70343 Modified: pypy/trunk/pypy/module/imp/test/test_import.py Log: Use rstrip() to avoid issues with white space at the end of the string in CPython. Modified: pypy/trunk/pypy/module/imp/test/test_import.py ============================================================================== --- pypy/trunk/pypy/module/imp/test/test_import.py (original) +++ pypy/trunk/pypy/module/imp/test/test_import.py Tue Dec 29 17:49:05 2009 @@ -348,7 +348,7 @@ def test_future_relative_import_error_when_in_non_package(self): exec """def imp(): from .string import inpackage - """ + """.rstrip() raises(ValueError, imp) def test_relative_import_with___name__(self): From afa at codespeak.net Tue Dec 29 18:00:33 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 29 Dec 2009 18:00:33 +0100 (CET) Subject: [pypy-svn] r70344 - pypy/trunk/pypy/module/imp/test Message-ID: <20091229170033.E9E91168023@codespeak.net> Author: afa Date: Tue Dec 29 18:00:31 2009 New Revision: 70344 Modified: pypy/trunk/pypy/module/imp/test/test_import.py Log: Skip this test for the moment: reload(sys) replaces sys.path with another object, identical to the initial version. I still can't see the place where sys.path is replaced instead of modifed in-place. Modified: pypy/trunk/pypy/module/imp/test/test_import.py ============================================================================== --- pypy/trunk/pypy/module/imp/test/test_import.py (original) +++ pypy/trunk/pypy/module/imp/test/test_import.py Tue Dec 29 18:00:31 2009 @@ -433,6 +433,7 @@ reload(pkg.a) def test_reload_builtin(self): + skip("XXX fixme") import sys try: del sys.setdefaultencoding From arigo at codespeak.net Tue Dec 29 21:53:04 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 Dec 2009 21:53:04 +0100 (CET) Subject: [pypy-svn] r70347 - pypy/branch/virtual-forcing/pypy/jit/tool/test Message-ID: <20091229205304.4130C168026@codespeak.net> Author: arigo Date: Tue Dec 29 21:53:02 2009 New Revision: 70347 Modified: pypy/branch/virtual-forcing/pypy/jit/tool/test/test_jitoutput.py Log: Tabs! Modified: pypy/branch/virtual-forcing/pypy/jit/tool/test/test_jitoutput.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/tool/test/test_jitoutput.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/tool/test/test_jitoutput.py Tue Dec 29 21:53:02 2009 @@ -64,12 +64,12 @@ opt ops: 6 opt guards: 1 forcings: 1 -abort: trace too long: 10 -abort: compiling: 11 -abort: vable escape: 12 -nvirtuals: 13 -nvholes: 14 -nvreused: 15 +abort: trace too long: 10 +abort: compiling: 11 +abort: vable escape: 12 +nvirtuals: 13 +nvholes: 14 +nvreused: 15 ''' def test_parse(): From dan at codespeak.net Wed Dec 30 10:21:05 2009 From: dan at codespeak.net (dan at codespeak.net) Date: Wed, 30 Dec 2009 10:21:05 +0100 (CET) Subject: [pypy-svn] r70348 - pypy/branch/micronumpy-ElectronicRU Message-ID: <20091230092105.15D93168021@codespeak.net> Author: dan Date: Wed Dec 30 10:21:02 2009 New Revision: 70348 Added: pypy/branch/micronumpy-ElectronicRU/ - copied from r70347, pypy/branch/micronumpy/ Log: This broke all tests, keeping the code alive in this branch though. From afa at codespeak.net Wed Dec 30 12:03:34 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 30 Dec 2009 12:03:34 +0100 (CET) Subject: [pypy-svn] r70350 - in pypy/trunk/pypy: module/imp/test translator/goal Message-ID: <20091230110334.333ED168022@codespeak.net> Author: afa Date: Wed Dec 30 12:03:33 2009 New Revision: 70350 Modified: pypy/trunk/pypy/module/imp/test/test_import.py pypy/trunk/pypy/translator/goal/app_main.py Log: Finally found where sys.path is replaced instead of modified in-place. Fixes reload(sys) Modified: pypy/trunk/pypy/module/imp/test/test_import.py ============================================================================== --- pypy/trunk/pypy/module/imp/test/test_import.py (original) +++ pypy/trunk/pypy/module/imp/test/test_import.py Wed Dec 30 12:03:33 2009 @@ -433,13 +433,16 @@ reload(pkg.a) def test_reload_builtin(self): - skip("XXX fixme") import sys + oldpath = sys.path try: del sys.setdefaultencoding except AttributeError: pass + reload(sys) + + assert sys.path is oldpath assert 'setdefaultencoding' in dir(sys) def test_reload_infinite(self): Modified: pypy/trunk/pypy/translator/goal/app_main.py ============================================================================== --- pypy/trunk/pypy/translator/goal/app_main.py (original) +++ pypy/trunk/pypy/translator/goal/app_main.py Wed Dec 30 12:03:33 2009 @@ -228,7 +228,7 @@ newpath.insert(0, '') # remove duplicates _seen = {} - sys.path = [] + del sys.path[:] for dir in newpath: if dir not in _seen: sys.path.append(dir) From arigo at codespeak.net Wed Dec 30 13:21:15 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 30 Dec 2009 13:21:15 +0100 (CET) Subject: [pypy-svn] r70352 - pypy/branch/virtual-forcing/pypy/module/pypyjit/test Message-ID: <20091230122115.38883168022@codespeak.net> Author: arigo Date: Wed Dec 30 13:21:14 2009 New Revision: 70352 Modified: pypy/branch/virtual-forcing/pypy/module/pypyjit/test/test_pypy_c.py Log: Allow up to 7200 operations for richards. There are a bit more now because of the creation of some frames. Modified: pypy/branch/virtual-forcing/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/virtual-forcing/pypy/module/pypyjit/test/test_pypy_c.py Wed Dec 30 13:21:14 2009 @@ -205,7 +205,7 @@ def main(): return richards.main(iterations = 1) - ''' % (sys.path,), 7000, + ''' % (sys.path,), 7200, ([], 42)) def test_simple_call(self): From arigo at codespeak.net Wed Dec 30 14:43:27 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 30 Dec 2009 14:43:27 +0100 (CET) Subject: [pypy-svn] r70355 - pypy/branch/virtual-forcing/pypy/module/pypyjit/test Message-ID: <20091230134327.820A9168021@codespeak.net> Author: arigo Date: Wed Dec 30 14:43:26 2009 New Revision: 70355 Modified: pypy/branch/virtual-forcing/pypy/module/pypyjit/test/test_pypy_c.py Log: Verify the pypy-c passed as option to "--pypy": it must have the jit, but not threads, because for now the tests get confused with the extra thread stuff. Modified: pypy/branch/virtual-forcing/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/virtual-forcing/pypy/module/pypyjit/test/test_pypy_c.py Wed Dec 30 14:43:26 2009 @@ -445,7 +445,27 @@ def setup_class(cls): if option.pypy_c is None: py.test.skip("pass --pypy!") + if not has_info(option.pypy_c, 'translation.jit'): + py.test.skip("must give a pypy-c with the jit enabled") + if has_info(option.pypy_c, 'translation.thread'): + py.test.skip("for now, must give a pypy-c-jit without threads") cls.tmpdir = udir.join('pypy-jit') cls.tmpdir.ensure(dir=1) cls.counter = 0 cls.pypy_c = option.pypy_c + +def has_info(pypy_c, option): + g = os.popen('"%s" --info' % pypy_c, 'r') + lines = g.readlines() + g.close() + for line in lines: + line = line.strip() + if line.startswith(option + ':'): + line = line[len(option)+1:].strip() + if line == 'True': + return True + elif line == 'False': + return False + else: + return line + raise ValueError(option + ' not found in ' + pypy_c) From arigo at codespeak.net Wed Dec 30 16:10:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 30 Dec 2009 16:10:29 +0100 (CET) Subject: [pypy-svn] r70359 - pypy/branch/virtual-forcing/pypy/module/pypyjit/test Message-ID: <20091230151029.6BBE5168021@codespeak.net> Author: arigo Date: Wed Dec 30 16:10:28 2009 New Revision: 70359 Modified: pypy/branch/virtual-forcing/pypy/module/pypyjit/test/test_pypy_c.py Log: Fix the test. A bit fragile, but a setfield_gc ends up here (it's delayed from a bit earlier). Modified: pypy/branch/virtual-forcing/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/virtual-forcing/pypy/module/pypyjit/test/test_pypy_c.py Wed Dec 30 16:10:28 2009 @@ -113,8 +113,8 @@ assert result assert result.splitlines()[-1].strip() == 'OK :-)' self.parse_loops(logfilepath) + self.print_loops() if self.total_ops > expected_max_ops: - self.print_loops() assert 0, "too many operations: got %d, expected maximum %d" % ( self.total_ops, expected_max_ops) @@ -267,7 +267,9 @@ ops = self.get_by_bytecode("LOAD_ATTR") assert len(ops) == 2 assert ops[0].get_opnames() == ["getfield_gc", "getarrayitem_gc", + "setfield_gc", # (*) "guard_nonnull_class"] + # (*) delayed write of the frames depth assert not ops[1] # second LOAD_ATTR folded away def test_default_and_kw(self):