From mwh at codespeak.net Mon Dec 1 09:21:35 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Mon, 1 Dec 2008 09:21:35 +0100 (CET) Subject: [pypy-svn] r60250 - pypy/extradoc/talk/osdc2008 Message-ID: <20081201082135.F09F11683F8@codespeak.net> Author: mwh Date: Mon Dec 1 09:21:34 2008 New Revision: 60250 Added: pypy/extradoc/talk/osdc2008/osdc08.pdf (contents, props changed) Log: first cut at updated slides from my pycon07 talk Added: pypy/extradoc/talk/osdc2008/osdc08.pdf ============================================================================== Binary file. No diff available. From arigo at codespeak.net Mon Dec 1 14:36:34 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 1 Dec 2008 14:36:34 +0100 (CET) Subject: [pypy-svn] r60259 - in pypy/branch/oo-jit/pypy: annotation rlib rpython rpython/ootypesystem rpython/test Message-ID: <20081201133634.3FFCC1683F7@codespeak.net> Author: arigo Date: Mon Dec 1 14:36:32 2008 New Revision: 60259 Modified: pypy/branch/oo-jit/pypy/annotation/dictdef.py pypy/branch/oo-jit/pypy/rlib/objectmodel.py pypy/branch/oo-jit/pypy/rpython/llinterp.py pypy/branch/oo-jit/pypy/rpython/ootypesystem/rdict.py pypy/branch/oo-jit/pypy/rpython/test/test_remptydict.py Log: Implement the "never-seeing-any-key" case for r_dict's. Modified: pypy/branch/oo-jit/pypy/annotation/dictdef.py ============================================================================== --- pypy/branch/oo-jit/pypy/annotation/dictdef.py (original) +++ pypy/branch/oo-jit/pypy/annotation/dictdef.py Mon Dec 1 14:36:32 2008 @@ -59,6 +59,8 @@ replace_othereq = replace_otherhash = () s_key = self.s_value + if s_key == s_ImpossibleValue: + return def check_eqfn(annotator, graph): s = annotator.binding(graph.getreturnvar()) Modified: pypy/branch/oo-jit/pypy/rlib/objectmodel.py ============================================================================== --- pypy/branch/oo-jit/pypy/rlib/objectmodel.py (original) +++ pypy/branch/oo-jit/pypy/rlib/objectmodel.py Mon Dec 1 14:36:32 2008 @@ -301,6 +301,9 @@ def get(self, key, default): return self._dict.get(_r_dictkey(self, key), default) + def setdefault(self, key, default): + return self._dict.setdefault(_r_dictkey(self, key), default) + def copy(self): result = r_dict(self.key_eq, self.key_hash) result.update(self) Modified: pypy/branch/oo-jit/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/llinterp.py (original) +++ pypy/branch/oo-jit/pypy/rpython/llinterp.py Mon Dec 1 14:36:32 2008 @@ -1135,6 +1135,12 @@ def wrap_callable(llinterpreter, fn, obj, method_name): if method_name is None: + if fn is None: + # case "fn is None and method_name is None": no key supplied at + # all, for empty r_dict's. + def not_callable(*args): + assert False, "I should not be called" + return 'not_callable', not_callable # fn is a StaticMethod if obj is not None: self_arg = [obj] Modified: pypy/branch/oo-jit/pypy/rpython/ootypesystem/rdict.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/ootypesystem/rdict.py (original) +++ pypy/branch/oo-jit/pypy/rpython/ootypesystem/rdict.py Mon Dec 1 14:36:32 2008 @@ -5,7 +5,7 @@ from pypy.rpython.rdict import AbstractDictRepr, AbstractDictIteratorRepr,\ rtype_newdict, dum_variant, dum_keys, dum_values, dum_items from pypy.rpython.rpbc import MethodOfFrozenPBCRepr,\ - AbstractFunctionsPBCRepr, AbstractMethodsPBCRepr + AbstractFunctionsPBCRepr, AbstractMethodsPBCRepr, SingleFrozenPBCRepr from pypy.rpython.ootypesystem import ootype from pypy.rpython.ootypesystem.rlist import ll_newlist from pypy.rlib.rarithmetic import r_uint @@ -181,7 +181,8 @@ except KeyError: self.setup() l_dict = ll_newdict(self.DICT) - if self.custom_eq_hash: + if (self.custom_eq_hash and + self.dictkey.s_value != annmodel.s_ImpossibleValue): interp = llinterp.LLInterpreter(self.rtyper) EQ_FUNC = ootype.StaticMethod([self.DICT._KEYTYPE, self.DICT._KEYTYPE], ootype.Bool) sm_eq = self.__get_func(interp, self.r_rdict_eqfn, dictobj.key_eq, EQ_FUNC) @@ -251,6 +252,8 @@ v_fn = r_impl.get_unique_llfn() v_obj = hop.inputarg(r_func, arg=arg) c_method_name = hop.inputconst(ootype.Void, None) + elif isinstance(r_func, SingleFrozenPBCRepr): + v_fn = v_obj = c_method_name = hop.inputconst(ootype.Void, None) return v_fn, v_obj, c_method_name @@ -262,7 +265,7 @@ hop.exception_cannot_occur() # the signature of oonewcustomdict is a bit complicated because we - # can have three different ways to pass the equal (and hash) + # can have four different ways to pass the equal (and hash) # callables: # 1. pass a plain function: v_eqfn is a StaticMethod, v_eqobj # and c_eq_method_name are None @@ -271,6 +274,8 @@ # 3. pass a method of a frozen PBC: v_eqfn is a StaticMethod, # v_eqobj is the PBC to be pushed in front of the StaticMethod, # c_eq_method_name is None + # 4. pass nothing (for always-empty r_dict's, whose keys and hash + # function are never called). s_key = r_dict.dictkey.s_value v_eqfn, v_eqobj, c_eq_method_name =\ Modified: pypy/branch/oo-jit/pypy/rpython/test/test_remptydict.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/test/test_remptydict.py (original) +++ pypy/branch/oo-jit/pypy/rpython/test/test_remptydict.py Mon Dec 1 14:36:32 2008 @@ -1,5 +1,6 @@ import py from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin +from pypy.rlib.objectmodel import r_dict class BaseTestRemptydict(BaseRtypingTest): def test_empty_dict(self): @@ -27,6 +28,21 @@ res = self.interpret(f, []) assert res == 0 + def test_empty_r_dict(self): + class A: + pass + def key_eq(a, b): + return len(a) == len(b) + def key_hash(a): + return len(a) + a = A() + a.d1 = r_dict(key_eq, key_hash) + def func(): + a.d2 = r_dict(key_eq, key_hash) + return bool(a.d1) or bool(a.d2) + res = self.interpret(func, []) + assert res is False + class TestLLtype(BaseTestRemptydict, LLRtypeMixin): pass From antocuni at codespeak.net Tue Dec 2 14:50:09 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 2 Dec 2008 14:50:09 +0100 (CET) Subject: [pypy-svn] r60279 - in pypy/branch/oo-jit/pypy: jit/hintannotator/test rpython/ootypesystem Message-ID: <20081202135009.D57CE1680A3@codespeak.net> Author: antocuni Date: Tue Dec 2 14:50:07 2008 New Revision: 60279 Modified: pypy/branch/oo-jit/pypy/jit/hintannotator/test/test_annotator.py pypy/branch/oo-jit/pypy/rpython/ootypesystem/rdict.py Log: make sure that getitem on a green dictionary returns a green on ootype. For now, it's enough to add oopspec to the right function, but in theory when this skipped test pass it will work also without oopspec. Modified: pypy/branch/oo-jit/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/branch/oo-jit/pypy/jit/hintannotator/test/test_annotator.py Tue Dec 2 14:50:07 2008 @@ -979,6 +979,31 @@ hs = self.hannotate(f, [int], policy=P) assert hs.is_green() + def test_getitem_deepfrozen_dict(self): + d = {1: 2, 2: 4, 3: 9} + def f(k): + d1 = hint(d, deepfreeze=True) + k1 = hint(k, concrete=True) + return d1[k1] + hs = self.hannotate(f, [int], policy=P_OOPSPEC_NOVIRTUAL) + assert hs.myorigin + assert hs.is_green() + + def test_missing_myorigin(self): + py.test.skip('do we want this to pass?') + def g(k): + if k: + return k + else: + raise KeyError + def f(k): + k1 = hint(k, concrete=True) + return g(k) + hs = self.hannotate(f, [int], policy=P_OOPSPEC_NOVIRTUAL) + assert hs.myorigin + assert hs.is_green() + + class TestLLType(BaseAnnotatorTest): type_system = 'lltype' @@ -1105,12 +1130,6 @@ class TestOOType(BaseAnnotatorTest): type_system = 'ootype' malloc = property(lambda self: ootype.new) - - def fixpolicy(self, policy): - import copy - newpolicy = copy.copy(policy) - newpolicy.oopspec = False - return newpolicy def make_struct(self, name, *fields, **kwds): fields = dict(fields) Modified: pypy/branch/oo-jit/pypy/rpython/ootypesystem/rdict.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/ootypesystem/rdict.py (original) +++ pypy/branch/oo-jit/pypy/rpython/ootypesystem/rdict.py Tue Dec 2 14:50:07 2008 @@ -307,12 +307,13 @@ value = it.ll_current_value() d1.ll_set(key, value) - def ll_dict_getitem(d, key): if d.ll_contains(key): return d.ll_get(key) else: raise KeyError +ll_dict_getitem.oopspec = 'dict.getitem(d, key)' +ll_dict_getitem.oopargcheck = lambda d, key: bool(d) def ll_dict_delitem(d, key): if not d.ll_remove(key): From antocuni at codespeak.net Tue Dec 2 16:01:18 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 2 Dec 2008 16:01:18 +0100 (CET) Subject: [pypy-svn] r60280 - in pypy/branch/oo-jit/pypy: jit/hintannotator jit/rainbow/test jit/timeshifter jit/tl jit/tl/test rpython/ootypesystem Message-ID: <20081202150118.226AF1683ED@codespeak.net> Author: antocuni Date: Tue Dec 2 16:01:17 2008 New Revision: 60280 Modified: pypy/branch/oo-jit/pypy/jit/hintannotator/policy.py pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py pypy/branch/oo-jit/pypy/jit/timeshifter/vdict.py pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py pypy/branch/oo-jit/pypy/rpython/ootypesystem/ootype.py Log: (antocuni, arigo around) implement some basic object-oriented features for tlc; whack around to fix various bugs until we eliminate all the overhead of dynamic dispatch Modified: pypy/branch/oo-jit/pypy/jit/hintannotator/policy.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/hintannotator/policy.py (original) +++ pypy/branch/oo-jit/pypy/jit/hintannotator/policy.py Tue Dec 2 16:01:17 2008 @@ -24,6 +24,13 @@ return new def look_inside_graph(self, graph): + try: + func = graph.func + except AttributeError: + return True + # explicitly pure functions are always opaque + if getattr(func, '_pure_function_', False): + return False return True Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py Tue Dec 2 16:01:17 2008 @@ -14,16 +14,30 @@ small = False def _get_interp(self): - def interp(llbytecode, pc, inputarg): + def decode_strlist(s): + lists = s.split('|') + return [lst.split(',') for lst in lists] + + def interp(llbytecode, pc, inputarg, llstrlists): from pypy.rpython.annlowlevel import hlstr bytecode = hlstr(llbytecode) - return tlc.interp_without_call(bytecode, pc, inputarg) + strlists = hlstr(llstrlists) + pool = tlc.ConstantPool() + pool.strlists = decode_strlist(strlists) + obj = tlc.interp_eval_without_call(bytecode, + pc, + tlc.IntObj(inputarg), + pool) + return obj.int_o() to_rstr = self.to_rstr def build_bytecode(s): result = ''.join([chr(int(t)) for t in s.split(',')]) return to_rstr(result) - interp.convert_arguments = [build_bytecode, int, int] + def build_strlist(items): + lists = [','.join(lst) for lst in items] + return to_rstr('|'.join(lists)) + interp.convert_arguments = [build_bytecode, int, int, build_strlist] return interp @@ -38,7 +52,7 @@ interp = self._get_interp() res = self.timeshift_from_portal(interp, tlc.interp_eval_without_call, - [bytecode, 0, n], + [bytecode, 0, n, []], policy=P_OOPSPEC)#, backendoptimize=True) assert res == expected self.check_insns(malloc=1) @@ -61,10 +75,30 @@ interp = self._get_interp() res = self.timeshift_from_portal(interp, tlc.interp_eval_without_call, - [bytecode, 0, 1], + [bytecode, 0, 1, []], policy=P_OOPSPEC)#, backendoptimize=True) assert res == 20 + def test_getattr(self): + from pypy.jit.tl.tlc import interp_eval, nil, ConstantPool + pool = ConstantPool() + code = tlc.compile(""" + NEW foo,bar + PICK 0 + PUSH 42 + SETATTR bar, + GETATTR bar, + """, pool) + bytecode = ','.join([str(ord(c)) for c in code]) + interp = self._get_interp() + res = self.timeshift_from_portal(interp, + tlc.interp_eval_without_call, + [bytecode, 0, 0, pool.strlists], + policy=P_OOPSPEC) + assert res == 42 + self.check_insns(malloc=1) + + class TestLLType(BaseTestTLC): type_system = "lltype" Modified: pypy/branch/oo-jit/pypy/jit/timeshifter/vdict.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/timeshifter/vdict.py (original) +++ pypy/branch/oo-jit/pypy/jit/timeshifter/vdict.py Tue Dec 2 16:01:17 2008 @@ -217,7 +217,8 @@ return eq, hash # work at least for primitive types def gen_newdict(self, builder, args_gv): - raise NotImplementedError + assert len(args_gv) == 0 + return builder.genop_new(self.alloctoken) def gen_insertclean(self, builder, args_gv): raise NotImplementedError @@ -387,3 +388,12 @@ oop_dict_method_set = oop_dict_setitem oop_dict_method_get = oop_dict_getitem oop_dict_method_contains = oop_dict_contains + +def oop_dict_method_length(jitstate, oopspecdesc, deepfrozen, selfbox): + content = selfbox.content + if isinstance(content, AbstractVirtualDict): + return rvalue.ll_fromvalue(jitstate, len(content.item_boxes)) + else: + return oopspecdesc.residual_call(jitstate, [selfbox], + deepfrozen=deepfrozen) +oop_dict_getitem.couldfold = True Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py Tue Dec 2 16:01:17 2008 @@ -1,7 +1,16 @@ import py -from pypy.jit.tl.tlopcode import compile +from pypy.jit.tl.tlopcode import compile, NEW from pypy.jit.tl.test import test_tl - +from pypy.jit.tl.tlc import ConstantPool + +def test_constant_pool(): + pool = ConstantPool() + bytecode = compile(""" + NEW foo,bar + """, pool) + expected = test_tl.list2bytecode([NEW, 0]) + assert expected == bytecode + assert pool.strlists == [['foo', 'bar']] class TestTLC(test_tl.TestTL): from pypy.jit.tl.tlc import interp @@ -110,3 +119,39 @@ """) py.test.raises(TypeError, self.interp, bytecode, 0, 0) + def test_new_obj(self): + from pypy.jit.tl.tlc import interp_eval, InstanceObj + pool = ConstantPool() + bytecode = compile(""" + NEW foo,bar + """, pool) + obj = interp_eval(bytecode, 0, None, pool) + assert isinstance(obj, InstanceObj) + assert len(obj.values) == 2 + assert sorted(obj.cls.attributes.keys()) == ['bar', 'foo'] + + def test_setattr(self): + from pypy.jit.tl.tlc import interp_eval, nil + pool = ConstantPool() + bytecode = compile(""" + NEW foo,bar + PICK 0 + PUSH 42 + SETATTR foo, + """, pool) + obj = interp_eval(bytecode, 0, None, pool) + assert obj.values[0].int_o() == 42 + assert obj.values[1] is nil + + def test_getattr(self): + from pypy.jit.tl.tlc import interp_eval, nil + pool = ConstantPool() + bytecode = compile(""" + NEW foo,bar + PICK 0 + PUSH 42 + SETATTR bar, + GETATTR bar, + """, pool) + res = interp_eval(bytecode, 0, nil, pool) + assert res.int_o() == 42 Modified: pypy/branch/oo-jit/pypy/jit/tl/tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlc.py Tue Dec 2 16:01:17 2008 @@ -5,6 +5,7 @@ from pypy.rlib.objectmodel import specialize from pypy.jit.tl.tlopcode import * from pypy.jit.tl import tlopcode +from pypy.rlib.jit import hint class Obj(object): @@ -25,6 +26,63 @@ def _concat(self, other): raise TypeError + # object oriented features + def getattr(self, name): raise TypeError + def setattr(self, name, value): raise TypeError + + +class ConstantPool(object): + + def __init__(self): + self.strlists = [] + + def add_strlist(self, items): + idx = len(self.strlists) + self.strlists.append(items) + return idx + +class Class(object): + + classes = [] # [(attributes, cls), ...] + + def get(key): + for attributes, cls in Class.classes: + if attributes == key: + return cls + result = Class(key) + Class.classes.append((key, result)) + return result + get._pure_function_ = True + get = staticmethod(get) + + def __init__(self, attrlist): + attributes = {} # attrname -> index + for name in attrlist: + attributes[name] = len(attributes) + self.attributes = hint(attributes, deepfreeze=True) + self.length = len(attributes) + +class InstanceObj(Obj): + + def __init__(self, cls): + self.cls = cls + cls2 = hint(cls, deepfreeze=True) + self.values = [nil] * cls2.length + + def getclass(self): + # promote and deepfreeze the class + cls = hint(self.cls, promote=True) + return hint(cls, deepfreeze=True) + + def getattr(self, name): + i = self.getclass().attributes[name] + return self.values[i] + + def setattr(self, name, value): + i = self.getclass().attributes[name] + self.values[i] = value + return value + class IntObj(Obj): def __init__(self, value): @@ -126,10 +184,12 @@ raise TypeError("code '%s' should be a string" % str(code)) return interp_eval(code, pc, IntObj(inputarg)).int_o() - - def interp_eval(code, pc, inputarg): + + def interp_eval(code, pc, inputarg, pool2=ConstantPool()): code_len = len(code) stack = [] + pool3 = hint(pool2, concrete=True) + pool = hint(pool3, deepfreeze=True) while pc < code_len: hint(None, global_merge_point=True) @@ -272,11 +332,37 @@ elif opcode == PUSHARG: stack.append(inputarg) + elif opcode == NEW: + idx = char2int(code[pc]) + pc += 1 + attributes = pool.strlists[idx] + cls = Class.get(attributes) + stack.append(InstanceObj(cls)) + + elif opcode == GETATTR: + idx = char2int(code[pc]) + pc += 1 + attributes = pool.strlists[idx] + name = attributes[0] + a = stack.pop() + hint(a, promote_class=True) + stack.append(a.getattr(name)) + + elif opcode == SETATTR: + idx = char2int(code[pc]) + pc += 1 + attributes = pool.strlists[idx] + name = attributes[0] + a, b = stack.pop(), stack.pop() + hint(a, promote_class=True) + hint(b, promote_class=True) + b.setattr(name, a) + else: raise RuntimeError("unknown opcode: " + str(opcode)) return stack[-1] - + return interp, interp_eval Modified: pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py Tue Dec 2 16:01:17 2008 @@ -44,11 +44,15 @@ opcode(26, "CAR") opcode(27, "CDR") +# object oriented features of tlc +opcode(28, "NEW") +opcode(29, "GETATTR") +opcode(30, "SETATTR") del opcode -def compile(code=''): +def compile(code='', pool=None): bytecode = [] labels = {} #[key] = pc label_usage = [] #(name, pc) @@ -60,15 +64,27 @@ continue t = s.split() if t[0].endswith(':'): + assert ',' not in t[0] labels[ t[0][:-1] ] = len(bytecode) continue bytecode.append(names[t[0]]) if len(t) > 1: + arg = t[1] try: - bytecode.append( int(t[1]) ) + bytecode.append( int(arg) ) except ValueError: - label_usage.append( (t[1], len(bytecode)) ) - bytecode.append( 0 ) + if ',' in arg: + # it's a list of strings + items = arg.split(',') + items = map(str.strip, items) + items = [x for x in items if x] + assert pool is not None + idx = pool.add_strlist(items) + bytecode.append(idx) + else: + # it's a label + label_usage.append( (arg, len(bytecode)) ) + bytecode.append( 0 ) for label, pc in label_usage: bytecode[pc] = labels[label] - pc - 1 return ''.join([chr(i & 0xff) for i in bytecode]) Modified: pypy/branch/oo-jit/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/branch/oo-jit/pypy/rpython/ootypesystem/ootype.py Tue Dec 2 16:01:17 2008 @@ -1526,6 +1526,8 @@ def ll_length(self): # NOT_RPYTHON return len(self._list) + ll_length.oopargcheck = lambda a: bool(a) + ll_length.foldable = True def _ll_resize_ge(self, length): # NOT_RPYTHON @@ -1553,6 +1555,8 @@ assert typeOf(index) == Signed assert index >= 0 return self._list[index] + ll_getitem_fast.oopargcheck = lambda l, k: bool(l) + ll_getitem_fast.couldfold = True # XXX? def ll_setitem_fast(self, index, item): # NOT_RPYTHON @@ -1607,6 +1611,8 @@ def ll_length(self): # NOT_RPYTHON return len(self._dict) + ll_length.oopargcheck = lambda a: bool(a) + ll_length.foldable = True def ll_get(self, key): # NOT_RPYTHON From antocuni at codespeak.net Tue Dec 2 16:24:27 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 2 Dec 2008 16:24:27 +0100 (CET) Subject: [pypy-svn] r60281 - in pypy/branch/oo-jit/pypy/jit: rainbow/test tl Message-ID: <20081202152427.DA14A1683BF@codespeak.net> Author: antocuni Date: Tue Dec 2 16:24:27 2008 New Revision: 60281 Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlc.py Log: simplify a bit the code, and make the test more robust (as there could be a malloc/new hidden inside a direct_call to a helper) Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py Tue Dec 2 16:24:27 2008 @@ -96,7 +96,7 @@ [bytecode, 0, 0, pool.strlists], policy=P_OOPSPEC) assert res == 42 - self.check_insns(malloc=1) + self.check_insns(malloc=1, direct_call=0) Modified: pypy/branch/oo-jit/pypy/jit/tl/tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlc.py Tue Dec 2 16:24:27 2008 @@ -59,15 +59,14 @@ attributes = {} # attrname -> index for name in attrlist: attributes[name] = len(attributes) - self.attributes = hint(attributes, deepfreeze=True) - self.length = len(attributes) + self.attributes = attributes class InstanceObj(Obj): def __init__(self, cls): self.cls = cls - cls2 = hint(cls, deepfreeze=True) - self.values = [nil] * cls2.length + frozenclass = hint(cls, deepfreeze=True) + self.values = [nil] * len(frozenclass.attributes) def getclass(self): # promote and deepfreeze the class @@ -188,8 +187,7 @@ def interp_eval(code, pc, inputarg, pool2=ConstantPool()): code_len = len(code) stack = [] - pool3 = hint(pool2, concrete=True) - pool = hint(pool3, deepfreeze=True) + pool = hint(hint(pool2, concrete=True), deepfreeze=True) while pc < code_len: hint(None, global_merge_point=True) From antocuni at codespeak.net Tue Dec 2 18:32:42 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 2 Dec 2008 18:32:42 +0100 (CET) Subject: [pypy-svn] r60282 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081202173242.399BF1683D9@codespeak.net> Author: antocuni Date: Tue Dec 2 18:32:40 2008 New Revision: 60282 Added: pypy/extradoc/talk/ecoop2009/ pypy/extradoc/talk/ecoop2009/abstract.tex pypy/extradoc/talk/ecoop2009/clibackend.tex pypy/extradoc/talk/ecoop2009/diagram0.odg (contents, props changed) pypy/extradoc/talk/ecoop2009/diagram0.pdf (contents, props changed) pypy/extradoc/talk/ecoop2009/diagram1.odg (contents, props changed) pypy/extradoc/talk/ecoop2009/diagram1.pdf (contents, props changed) pypy/extradoc/talk/ecoop2009/intro.tex pypy/extradoc/talk/ecoop2009/jitgen.tex pypy/extradoc/talk/ecoop2009/llncs.cls pypy/extradoc/talk/ecoop2009/main.bbl pypy/extradoc/talk/ecoop2009/main.tex pypy/extradoc/talk/ecoop2009/rainbow.tex Log: first draft of the ecoop paper Added: pypy/extradoc/talk/ecoop2009/abstract.tex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/abstract.tex Tue Dec 2 18:32:40 2008 @@ -0,0 +1,3 @@ +\begin{abstract} +Dynamic languages are increasingly popular etc. etc. +\end{abstract} Added: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Tue Dec 2 18:32:40 2008 @@ -0,0 +1,8 @@ +\section{CLI backend for Rainbow interpreter} + +\subsection{Promotion on CLI} + +Implementing promotion on top of CLI is not straightforward, as it needs to +patch and modify the already generated code, and this is not possible on .NET. + +To solve, we do xxx and yyy etc. etc. Added: pypy/extradoc/talk/ecoop2009/diagram0.odg ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/ecoop2009/diagram0.pdf ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/ecoop2009/diagram1.odg ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/ecoop2009/diagram1.pdf ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/ecoop2009/intro.tex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/intro.tex Tue Dec 2 18:32:40 2008 @@ -0,0 +1,70 @@ +\section{Introduction} + +Dynamic languages are increasingly popular etc. etc. + +The easiest way to implement a dynamic language such as Python is to write an +interpreter; however, interpreters are slow. + +The alternative is to write a compiler; writing a compiler that targets a high +level virtual machine like CLI or JVM is easier than targeting a real CPU, but +it still require a lot of work, as IronPython, Jython, JRuby demonstrate. + +Moreover, writing a static compiler is often not enough to get high +performances; IronPython and JRuby are going in the direction of JIT compiling +specialized versions of the code depending on the actual values/types seen at +runtime; this approach seems to work, but write it manually requires an +enormous effort. + +PyPy's idea is to automatize the generation of static/JIT compilers in order +to reduce at minimun the effort required to get a fast implementation of a +dynamic language; all you have to do is to write a high level specification of +the language (by writing an interpreter), and put it through PyPy's +translation toolchain. + +\subsection{PyPy and RPython} + +\begin{figure}[h] +\begin{center} +\includegraphics[width=.6\textwidth]{diagram0} +\caption{PyPy infrastracture for generating an interpreter of a + language $L$ for several platforms}\label{pypy-fig} +\end{center} +\end{figure} + +The \emph{PyPy} project\footnote{\texttt{http://codespeak.net/pypy/dist/pypy/doc/home.html}} +\cite{RigoPedroni06} was initially conceived to develop an implementation of Python which +could be easily portable and extensible without renouncing efficiency. +To achieve these aims, the PyPy implementation is based on a highly +modular design which allows high-level aspects +to be separated from lower-level implementation details. +The abstract semantics of Python is defined by an interpreter written +in a high-level language, called RPython \cite{AACM-DLS07}, which is in fact a subset of +Python where some dynamic features have been sacrificed to allow an +efficient translation of the interpreter to low-level code. + +Compilation of the interpreter is implemented as a stepwise +refinement by means of a translation toolchain which performs type +analysis, code optimizations and several transformations aiming at +incrementally providing implementation details as memory management or the threading model. +The different kinds of intermediate codes which are refined +during the translation process are all represented by a collection of control flow graphs, +at several levels of abstractions. + +Finally, the low-level control flow-graphs produced by the toolchain +can be translated to executable code for a specific platform by a +corresponding backend. +Currently, three fully developed backends are available to produce +executable C/POSIX code, Java and CLI/.NET bytecode. + +Despite the PyPy infrastructure was specifically developed +for Python, in fact it can be used for implementing +other languages. Indeed, PyPy has been successfully experimented with +several languages as Smalltalk \cite{BolzEtAl08}, JavaScript, Scheme and Prolog. +As suggested by Figure~\ref{pypy-fig}, a portable interpreter for a +generic language $L$ can be +easily developed once an abstract interpreter for $L$ is implemented in +RPython. + +Another interesting feature of PyPy +is that just-in-time compilers can be semi-automatically generated from the +interpreter source. Added: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Tue Dec 2 18:32:40 2008 @@ -0,0 +1,66 @@ +\section{Automatic generation of JIT compilers} + +Traditional JIT compilers are hard to write, time consuming, hard to evolve, +etc. etc. + +\begin{figure}[h] +\begin{center} +\includegraphics[width=.6\textwidth]{diagram1} +\caption{PyPy infrastructure for generating an interpreter of a + language $L$ with JIT compilation for the .NET platform} +\end{center} +\end{figure} + +The JIT generation framework uses partial evaluation techniques to generate a +dynamic compiler from an interpreter; the idea is inspired by Psyco, which +uses the same techniques but it's manually written instead of being +automatically generated. + +The original idea is by Futamura \cite{Futamura99}. He proposed to generate compilers +from interpreters with automatic specialization, but his work has had +relatively little practical impact so far. + +\subsection{Partial evaluation} + +Assume the Python bytecode to be constant, and constant-propagate it into the +Python interpreter. + +Example:: +\begin{lstlisting}[language=Python] + def f(x, y): + x2 = x * x + y2 = y * y + return x2 + y2 + +**case x=3** :: + + def f_3(y): + y2 = y * y + return 9 + y2 + +**case x=10** :: + + def f_10(y): + y2 = y * y + return 100 + y2 +\end{lstlisting} + +A shortcoming of PE is that in many cases not much can be really assumed +constant at compile-time, and this leads to poor results. Effective dynamic +compilation requires feedback of runtime information into compile-time; for a +dynamic language, types are a primary example. + + +\subsection{Execution steps} + +* Translation time + + - pypy-c-jit is translated into an executable + + - the JIT compiler is automatically generated + +* Compile-time: the JIT compiler runs + +* Runtime: the JIT compiled code runs + +* Compile-time and runtime are intermixed Added: pypy/extradoc/talk/ecoop2009/llncs.cls ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/llncs.cls Tue Dec 2 18:32:40 2008 @@ -0,0 +1,1190 @@ +% LLNCS DOCUMENT CLASS -- version 2.14 (17-Aug-2004) +% Springer Verlag LaTeX2e support for Lecture Notes in Computer Science +% +%% +%% \CharacterTable +%% {Upper-case \A\B\C\D\E\F\G\H\I\J\K\L\M\N\O\P\Q\R\S\T\U\V\W\X\Y\Z +%% Lower-case \a\b\c\d\e\f\g\h\i\j\k\l\m\n\o\p\q\r\s\t\u\v\w\x\y\z +%% Digits \0\1\2\3\4\5\6\7\8\9 +%% Exclamation \! Double quote \" Hash (number) \# +%% Dollar \$ Percent \% Ampersand \& +%% Acute accent \' Left paren \( Right paren \) +%% Asterisk \* Plus \+ Comma \, +%% Minus \- Point \. Solidus \/ +%% Colon \: Semicolon \; Less than \< +%% Equals \= Greater than \> Question mark \? +%% Commercial at \@ Left bracket \[ Backslash \\ +%% Right bracket \] Circumflex \^ Underscore \_ +%% Grave accent \` Left brace \{ Vertical bar \| +%% Right brace \} Tilde \~} +%% +\NeedsTeXFormat{LaTeX2e}[1995/12/01] +\ProvidesClass{llncs}[2004/08/17 v2.14 +^^J LaTeX document class for Lecture Notes in Computer Science] +% Options +\let\if at envcntreset\iffalse +\DeclareOption{envcountreset}{\let\if at envcntreset\iftrue} +\DeclareOption{citeauthoryear}{\let\citeauthoryear=Y} +\DeclareOption{oribibl}{\let\oribibl=Y} +\let\if at custvec\iftrue +\DeclareOption{orivec}{\let\if at custvec\iffalse} +\let\if at envcntsame\iffalse +\DeclareOption{envcountsame}{\let\if at envcntsame\iftrue} +\let\if at envcntsect\iffalse +\DeclareOption{envcountsect}{\let\if at envcntsect\iftrue} +\let\if at runhead\iffalse +\DeclareOption{runningheads}{\let\if at runhead\iftrue} + +\let\if at openbib\iffalse +\DeclareOption{openbib}{\let\if at openbib\iftrue} + +% languages +\let\switcht@@therlang\relax +\def\ds at deutsch{\def\switcht@@therlang{\switcht at deutsch}} +\def\ds at francais{\def\switcht@@therlang{\switcht at francais}} + +\DeclareOption*{\PassOptionsToClass{\CurrentOption}{article}} + +\ProcessOptions + +\LoadClass[twoside]{article} +\RequirePackage{multicol} % needed for the list of participants, index + +\setlength{\textwidth}{12.2cm} +\setlength{\textheight}{19.3cm} +\renewcommand\@pnumwidth{2em} +\renewcommand\@tocrmarg{3.5em} +% +\def\@dottedtocline#1#2#3#4#5{% + \ifnum #1>\c at tocdepth \else + \vskip \z@ \@plus.2\p@ + {\leftskip #2\relax \rightskip \@tocrmarg \advance\rightskip by 0pt plus 2cm + \parfillskip -\rightskip \pretolerance=10000 + \parindent #2\relax\@afterindenttrue + \interlinepenalty\@M + \leavevmode + \@tempdima #3\relax + \advance\leftskip \@tempdima \null\nobreak\hskip -\leftskip + {#4}\nobreak + \leaders\hbox{$\m at th + \mkern \@dotsep mu\hbox{.}\mkern \@dotsep + mu$}\hfill + \nobreak + \hb at xt@\@pnumwidth{\hfil\normalfont \normalcolor #5}% + \par}% + \fi} +% +\def\switcht at albion{% +\def\abstractname{Abstract.} +\def\ackname{Acknowledgement.} +\def\andname{and} +\def\lastandname{\unskip, and} +\def\appendixname{Appendix} +\def\chaptername{Chapter} +\def\claimname{Claim} +\def\conjecturename{Conjecture} +\def\contentsname{Table of Contents} +\def\corollaryname{Corollary} +\def\definitionname{Definition} +\def\examplename{Example} +\def\exercisename{Exercise} +\def\figurename{Fig.} +\def\keywordname{{\bf Key words:}} +\def\indexname{Index} +\def\lemmaname{Lemma} +\def\contriblistname{List of Contributors} +\def\listfigurename{List of Figures} +\def\listtablename{List of Tables} +\def\mailname{{\it Correspondence to\/}:} +\def\noteaddname{Note added in proof} +\def\notename{Note} +\def\partname{Part} +\def\problemname{Problem} +\def\proofname{Proof} +\def\propertyname{Property} +\def\propositionname{Proposition} +\def\questionname{Question} +\def\remarkname{Remark} +\def\seename{see} +\def\solutionname{Solution} +\def\subclassname{{\it Subject Classifications\/}:} +\def\tablename{Table} +\def\theoremname{Theorem}} +\switcht at albion +% Names of theorem like environments are already defined +% but must be translated if another language is chosen +% +% French section +\def\switcht at francais{%\typeout{On parle francais.}% + \def\abstractname{R\'esum\'e.}% + \def\ackname{Remerciements.}% + \def\andname{et}% + \def\lastandname{ et}% + \def\appendixname{Appendice} + \def\chaptername{Chapitre}% + \def\claimname{Pr\'etention}% + \def\conjecturename{Hypoth\`ese}% + \def\contentsname{Table des mati\`eres}% + \def\corollaryname{Corollaire}% + \def\definitionname{D\'efinition}% + \def\examplename{Exemple}% + \def\exercisename{Exercice}% + \def\figurename{Fig.}% + \def\keywordname{{\bf Mots-cl\'e:}} + \def\indexname{Index} + \def\lemmaname{Lemme}% + \def\contriblistname{Liste des contributeurs} + \def\listfigurename{Liste des figures}% + \def\listtablename{Liste des tables}% + \def\mailname{{\it Correspondence to\/}:} + \def\noteaddname{Note ajout\'ee \`a l'\'epreuve}% + \def\notename{Remarque}% + \def\partname{Partie}% + \def\problemname{Probl\`eme}% + \def\proofname{Preuve}% + \def\propertyname{Caract\'eristique}% +%\def\propositionname{Proposition}% + \def\questionname{Question}% + \def\remarkname{Remarque}% + \def\seename{voir} + \def\solutionname{Solution}% + \def\subclassname{{\it Subject Classifications\/}:} + \def\tablename{Tableau}% + \def\theoremname{Th\'eor\`eme}% +} +% +% German section +\def\switcht at deutsch{%\typeout{Man spricht deutsch.}% + \def\abstractname{Zusammenfassung.}% + \def\ackname{Danksagung.}% + \def\andname{und}% + \def\lastandname{ und}% + \def\appendixname{Anhang}% + \def\chaptername{Kapitel}% + \def\claimname{Behauptung}% + \def\conjecturename{Hypothese}% + \def\contentsname{Inhaltsverzeichnis}% + \def\corollaryname{Korollar}% +%\def\definitionname{Definition}% + \def\examplename{Beispiel}% + \def\exercisename{\"Ubung}% + \def\figurename{Abb.}% + \def\keywordname{{\bf Schl\"usselw\"orter:}} + \def\indexname{Index} +%\def\lemmaname{Lemma}% + \def\contriblistname{Mitarbeiter} + \def\listfigurename{Abbildungsverzeichnis}% + \def\listtablename{Tabellenverzeichnis}% + \def\mailname{{\it Correspondence to\/}:} + \def\noteaddname{Nachtrag}% + \def\notename{Anmerkung}% + \def\partname{Teil}% +%\def\problemname{Problem}% + \def\proofname{Beweis}% + \def\propertyname{Eigenschaft}% +%\def\propositionname{Proposition}% + \def\questionname{Frage}% + \def\remarkname{Anmerkung}% + \def\seename{siehe} + \def\solutionname{L\"osung}% + \def\subclassname{{\it Subject Classifications\/}:} + \def\tablename{Tabelle}% +%\def\theoremname{Theorem}% +} + +% Ragged bottom for the actual page +\def\thisbottomragged{\def\@textbottom{\vskip\z@ plus.0001fil +\global\let\@textbottom\relax}} + +\renewcommand\small{% + \@setfontsize\small\@ixpt{11}% + \abovedisplayskip 8.5\p@ \@plus3\p@ \@minus4\p@ + \abovedisplayshortskip \z@ \@plus2\p@ + \belowdisplayshortskip 4\p@ \@plus2\p@ \@minus2\p@ + \def\@listi{\leftmargin\leftmargini + \parsep 0\p@ \@plus1\p@ \@minus\p@ + \topsep 8\p@ \@plus2\p@ \@minus4\p@ + \itemsep0\p@}% + \belowdisplayskip \abovedisplayskip +} + +\frenchspacing +\widowpenalty=10000 +\clubpenalty=10000 + +\setlength\oddsidemargin {63\p@} +\setlength\evensidemargin {63\p@} +\setlength\marginparwidth {90\p@} + +\setlength\headsep {16\p@} + +\setlength\footnotesep{7.7\p@} +\setlength\textfloatsep{8mm\@plus 2\p@ \@minus 4\p@} +\setlength\intextsep {8mm\@plus 2\p@ \@minus 2\p@} + +\setcounter{secnumdepth}{2} + +\newcounter {chapter} +\renewcommand\thechapter {\@arabic\c at chapter} + +\newif\if at mainmatter \@mainmattertrue +\newcommand\frontmatter{\cleardoublepage + \@mainmatterfalse\pagenumbering{Roman}} +\newcommand\mainmatter{\cleardoublepage + \@mainmattertrue\pagenumbering{arabic}} +\newcommand\backmatter{\if at openright\cleardoublepage\else\clearpage\fi + \@mainmatterfalse} + +\renewcommand\part{\cleardoublepage + \thispagestyle{empty}% + \if at twocolumn + \onecolumn + \@tempswatrue + \else + \@tempswafalse + \fi + \null\vfil + \secdef\@part\@spart} + +\def\@part[#1]#2{% + \ifnum \c at secnumdepth >-2\relax + \refstepcounter{part}% + \addcontentsline{toc}{part}{\thepart\hspace{1em}#1}% + \else + \addcontentsline{toc}{part}{#1}% + \fi + \markboth{}{}% + {\centering + \interlinepenalty \@M + \normalfont + \ifnum \c at secnumdepth >-2\relax + \huge\bfseries \partname~\thepart + \par + \vskip 20\p@ + \fi + \Huge \bfseries #2\par}% + \@endpart} +\def\@spart#1{% + {\centering + \interlinepenalty \@M + \normalfont + \Huge \bfseries #1\par}% + \@endpart} +\def\@endpart{\vfil\newpage + \if at twoside + \null + \thispagestyle{empty}% + \newpage + \fi + \if at tempswa + \twocolumn + \fi} + +\newcommand\chapter{\clearpage + \thispagestyle{empty}% + \global\@topnum\z@ + \@afterindentfalse + \secdef\@chapter\@schapter} +\def\@chapter[#1]#2{\ifnum \c at secnumdepth >\m at ne + \if at mainmatter + \refstepcounter{chapter}% + \typeout{\@chapapp\space\thechapter.}% + \addcontentsline{toc}{chapter}% + {\protect\numberline{\thechapter}#1}% + \else + \addcontentsline{toc}{chapter}{#1}% + \fi + \else + \addcontentsline{toc}{chapter}{#1}% + \fi + \chaptermark{#1}% + \addtocontents{lof}{\protect\addvspace{10\p@}}% + \addtocontents{lot}{\protect\addvspace{10\p@}}% + \if at twocolumn + \@topnewpage[\@makechapterhead{#2}]% + \else + \@makechapterhead{#2}% + \@afterheading + \fi} +\def\@makechapterhead#1{% +% \vspace*{50\p@}% + {\centering + \ifnum \c at secnumdepth >\m at ne + \if at mainmatter + \large\bfseries \@chapapp{} \thechapter + \par\nobreak + \vskip 20\p@ + \fi + \fi + \interlinepenalty\@M + \Large \bfseries #1\par\nobreak + \vskip 40\p@ + }} +\def\@schapter#1{\if at twocolumn + \@topnewpage[\@makeschapterhead{#1}]% + \else + \@makeschapterhead{#1}% + \@afterheading + \fi} +\def\@makeschapterhead#1{% +% \vspace*{50\p@}% + {\centering + \normalfont + \interlinepenalty\@M + \Large \bfseries #1\par\nobreak + \vskip 40\p@ + }} + +\renewcommand\section{\@startsection{section}{1}{\z@}% + {-18\p@ \@plus -4\p@ \@minus -4\p@}% + {12\p@ \@plus 4\p@ \@minus 4\p@}% + {\normalfont\large\bfseries\boldmath + \rightskip=\z@ \@plus 8em\pretolerance=10000 }} +\renewcommand\subsection{\@startsection{subsection}{2}{\z@}% + {-18\p@ \@plus -4\p@ \@minus -4\p@}% + {8\p@ \@plus 4\p@ \@minus 4\p@}% + {\normalfont\normalsize\bfseries\boldmath + \rightskip=\z@ \@plus 8em\pretolerance=10000 }} +\renewcommand\subsubsection{\@startsection{subsubsection}{3}{\z@}% + {-18\p@ \@plus -4\p@ \@minus -4\p@}% + {-0.5em \@plus -0.22em \@minus -0.1em}% + {\normalfont\normalsize\bfseries\boldmath}} +\renewcommand\paragraph{\@startsection{paragraph}{4}{\z@}% + {-12\p@ \@plus -4\p@ \@minus -4\p@}% + {-0.5em \@plus -0.22em \@minus -0.1em}% + {\normalfont\normalsize\itshape}} +\renewcommand\subparagraph[1]{\typeout{LLNCS warning: You should not use + \string\subparagraph\space with this class}\vskip0.5cm +You should not use \verb|\subparagraph| with this class.\vskip0.5cm} + +\DeclareMathSymbol{\Gamma}{\mathalpha}{letters}{"00} +\DeclareMathSymbol{\Delta}{\mathalpha}{letters}{"01} +\DeclareMathSymbol{\Theta}{\mathalpha}{letters}{"02} +\DeclareMathSymbol{\Lambda}{\mathalpha}{letters}{"03} +\DeclareMathSymbol{\Xi}{\mathalpha}{letters}{"04} +\DeclareMathSymbol{\Pi}{\mathalpha}{letters}{"05} +\DeclareMathSymbol{\Sigma}{\mathalpha}{letters}{"06} +\DeclareMathSymbol{\Upsilon}{\mathalpha}{letters}{"07} +\DeclareMathSymbol{\Phi}{\mathalpha}{letters}{"08} +\DeclareMathSymbol{\Psi}{\mathalpha}{letters}{"09} +\DeclareMathSymbol{\Omega}{\mathalpha}{letters}{"0A} + +\let\footnotesize\small + +\if at custvec +\def\vec#1{\mathchoice{\mbox{\boldmath$\displaystyle#1$}} +{\mbox{\boldmath$\textstyle#1$}} +{\mbox{\boldmath$\scriptstyle#1$}} +{\mbox{\boldmath$\scriptscriptstyle#1$}}} +\fi + +\def\squareforqed{\hbox{\rlap{$\sqcap$}$\sqcup$}} +\def\qed{\ifmmode\squareforqed\else{\unskip\nobreak\hfil +\penalty50\hskip1em\null\nobreak\hfil\squareforqed +\parfillskip=0pt\finalhyphendemerits=0\endgraf}\fi} + +\def\getsto{\mathrel{\mathchoice {\vcenter{\offinterlineskip +\halign{\hfil +$\displaystyle##$\hfil\cr\gets\cr\to\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\textstyle##$\hfil\cr\gets +\cr\to\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\scriptstyle##$\hfil\cr\gets +\cr\to\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\scriptscriptstyle##$\hfil\cr +\gets\cr\to\cr}}}}} +\def\lid{\mathrel{\mathchoice {\vcenter{\offinterlineskip\halign{\hfil +$\displaystyle##$\hfil\cr<\cr\noalign{\vskip1.2pt}=\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\textstyle##$\hfil\cr<\cr +\noalign{\vskip1.2pt}=\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\scriptstyle##$\hfil\cr<\cr +\noalign{\vskip1pt}=\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\scriptscriptstyle##$\hfil\cr +<\cr +\noalign{\vskip0.9pt}=\cr}}}}} +\def\gid{\mathrel{\mathchoice {\vcenter{\offinterlineskip\halign{\hfil +$\displaystyle##$\hfil\cr>\cr\noalign{\vskip1.2pt}=\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\textstyle##$\hfil\cr>\cr +\noalign{\vskip1.2pt}=\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\scriptstyle##$\hfil\cr>\cr +\noalign{\vskip1pt}=\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\scriptscriptstyle##$\hfil\cr +>\cr +\noalign{\vskip0.9pt}=\cr}}}}} +\def\grole{\mathrel{\mathchoice {\vcenter{\offinterlineskip +\halign{\hfil +$\displaystyle##$\hfil\cr>\cr\noalign{\vskip-1pt}<\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\textstyle##$\hfil\cr +>\cr\noalign{\vskip-1pt}<\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\scriptstyle##$\hfil\cr +>\cr\noalign{\vskip-0.8pt}<\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\scriptscriptstyle##$\hfil\cr +>\cr\noalign{\vskip-0.3pt}<\cr}}}}} +\def\bbbr{{\rm I\!R}} %reelle Zahlen +\def\bbbm{{\rm I\!M}} +\def\bbbn{{\rm I\!N}} %natuerliche Zahlen +\def\bbbf{{\rm I\!F}} +\def\bbbh{{\rm I\!H}} +\def\bbbk{{\rm I\!K}} +\def\bbbp{{\rm I\!P}} +\def\bbbone{{\mathchoice {\rm 1\mskip-4mu l} {\rm 1\mskip-4mu l} +{\rm 1\mskip-4.5mu l} {\rm 1\mskip-5mu l}}} +\def\bbbc{{\mathchoice {\setbox0=\hbox{$\displaystyle\rm C$}\hbox{\hbox +to0pt{\kern0.4\wd0\vrule height0.9\ht0\hss}\box0}} +{\setbox0=\hbox{$\textstyle\rm C$}\hbox{\hbox +to0pt{\kern0.4\wd0\vrule height0.9\ht0\hss}\box0}} +{\setbox0=\hbox{$\scriptstyle\rm C$}\hbox{\hbox +to0pt{\kern0.4\wd0\vrule height0.9\ht0\hss}\box0}} +{\setbox0=\hbox{$\scriptscriptstyle\rm C$}\hbox{\hbox +to0pt{\kern0.4\wd0\vrule height0.9\ht0\hss}\box0}}}} +\def\bbbq{{\mathchoice {\setbox0=\hbox{$\displaystyle\rm +Q$}\hbox{\raise +0.15\ht0\hbox to0pt{\kern0.4\wd0\vrule height0.8\ht0\hss}\box0}} +{\setbox0=\hbox{$\textstyle\rm Q$}\hbox{\raise +0.15\ht0\hbox to0pt{\kern0.4\wd0\vrule height0.8\ht0\hss}\box0}} +{\setbox0=\hbox{$\scriptstyle\rm Q$}\hbox{\raise +0.15\ht0\hbox to0pt{\kern0.4\wd0\vrule height0.7\ht0\hss}\box0}} +{\setbox0=\hbox{$\scriptscriptstyle\rm Q$}\hbox{\raise +0.15\ht0\hbox to0pt{\kern0.4\wd0\vrule height0.7\ht0\hss}\box0}}}} +\def\bbbt{{\mathchoice {\setbox0=\hbox{$\displaystyle\rm +T$}\hbox{\hbox to0pt{\kern0.3\wd0\vrule height0.9\ht0\hss}\box0}} +{\setbox0=\hbox{$\textstyle\rm T$}\hbox{\hbox +to0pt{\kern0.3\wd0\vrule height0.9\ht0\hss}\box0}} +{\setbox0=\hbox{$\scriptstyle\rm T$}\hbox{\hbox +to0pt{\kern0.3\wd0\vrule height0.9\ht0\hss}\box0}} +{\setbox0=\hbox{$\scriptscriptstyle\rm T$}\hbox{\hbox +to0pt{\kern0.3\wd0\vrule height0.9\ht0\hss}\box0}}}} +\def\bbbs{{\mathchoice +{\setbox0=\hbox{$\displaystyle \rm S$}\hbox{\raise0.5\ht0\hbox +to0pt{\kern0.35\wd0\vrule height0.45\ht0\hss}\hbox +to0pt{\kern0.55\wd0\vrule height0.5\ht0\hss}\box0}} +{\setbox0=\hbox{$\textstyle \rm S$}\hbox{\raise0.5\ht0\hbox +to0pt{\kern0.35\wd0\vrule height0.45\ht0\hss}\hbox +to0pt{\kern0.55\wd0\vrule height0.5\ht0\hss}\box0}} +{\setbox0=\hbox{$\scriptstyle \rm S$}\hbox{\raise0.5\ht0\hbox +to0pt{\kern0.35\wd0\vrule height0.45\ht0\hss}\raise0.05\ht0\hbox +to0pt{\kern0.5\wd0\vrule height0.45\ht0\hss}\box0}} +{\setbox0=\hbox{$\scriptscriptstyle\rm S$}\hbox{\raise0.5\ht0\hbox +to0pt{\kern0.4\wd0\vrule height0.45\ht0\hss}\raise0.05\ht0\hbox +to0pt{\kern0.55\wd0\vrule height0.45\ht0\hss}\box0}}}} +\def\bbbz{{\mathchoice {\hbox{$\mathsf\textstyle Z\kern-0.4em Z$}} +{\hbox{$\mathsf\textstyle Z\kern-0.4em Z$}} +{\hbox{$\mathsf\scriptstyle Z\kern-0.3em Z$}} +{\hbox{$\mathsf\scriptscriptstyle Z\kern-0.2em Z$}}}} + +\let\ts\, + +\setlength\leftmargini {17\p@} +\setlength\leftmargin {\leftmargini} +\setlength\leftmarginii {\leftmargini} +\setlength\leftmarginiii {\leftmargini} +\setlength\leftmarginiv {\leftmargini} +\setlength \labelsep {.5em} +\setlength \labelwidth{\leftmargini} +\addtolength\labelwidth{-\labelsep} + +\def\@listI{\leftmargin\leftmargini + \parsep 0\p@ \@plus1\p@ \@minus\p@ + \topsep 8\p@ \@plus2\p@ \@minus4\p@ + \itemsep0\p@} +\let\@listi\@listI +\@listi +\def\@listii {\leftmargin\leftmarginii + \labelwidth\leftmarginii + \advance\labelwidth-\labelsep + \topsep 0\p@ \@plus2\p@ \@minus\p@} +\def\@listiii{\leftmargin\leftmarginiii + \labelwidth\leftmarginiii + \advance\labelwidth-\labelsep + \topsep 0\p@ \@plus\p@\@minus\p@ + \parsep \z@ + \partopsep \p@ \@plus\z@ \@minus\p@} + +\renewcommand\labelitemi{\normalfont\bfseries --} +\renewcommand\labelitemii{$\m at th\bullet$} + +\setlength\arraycolsep{1.4\p@} +\setlength\tabcolsep{1.4\p@} + +\def\tableofcontents{\chapter*{\contentsname\@mkboth{{\contentsname}}% + {{\contentsname}}} + \def\authcount##1{\setcounter{auco}{##1}\setcounter{@auth}{1}} + \def\lastand{\ifnum\value{auco}=2\relax + \unskip{} \andname\ + \else + \unskip \lastandname\ + \fi}% + \def\and{\stepcounter{@auth}\relax + \ifnum\value{@auth}=\value{auco}% + \lastand + \else + \unskip, + \fi}% + \@starttoc{toc}\if at restonecol\twocolumn\fi} + +\def\l at part#1#2{\addpenalty{\@secpenalty}% + \addvspace{2em plus\p@}% % space above part line + \begingroup + \parindent \z@ + \rightskip \z@ plus 5em + \hrule\vskip5pt + \large % same size as for a contribution heading + \bfseries\boldmath % set line in boldface + \leavevmode % TeX command to enter horizontal mode. + #1\par + \vskip5pt + \hrule + \vskip1pt + \nobreak % Never break after part entry + \endgroup} + +\def\@dotsep{2} + +\def\hyperhrefextend{\ifx\hyper at anchor\@undefined\else +{chapter.\thechapter}\fi} + +\def\addnumcontentsmark#1#2#3{% +\addtocontents{#1}{\protect\contentsline{#2}{\protect\numberline + {\thechapter}#3}{\thepage}\hyperhrefextend}} +\def\addcontentsmark#1#2#3{% +\addtocontents{#1}{\protect\contentsline{#2}{#3}{\thepage}\hyperhrefextend}} +\def\addcontentsmarkwop#1#2#3{% +\addtocontents{#1}{\protect\contentsline{#2}{#3}{0}\hyperhrefextend}} + +\def\@adcmk[#1]{\ifcase #1 \or +\def\@gtempa{\addnumcontentsmark}% + \or \def\@gtempa{\addcontentsmark}% + \or \def\@gtempa{\addcontentsmarkwop}% + \fi\@gtempa{toc}{chapter}} +\def\addtocmark{\@ifnextchar[{\@adcmk}{\@adcmk[3]}} + +\def\l at chapter#1#2{\addpenalty{-\@highpenalty} + \vskip 1.0em plus 1pt \@tempdima 1.5em \begingroup + \parindent \z@ \rightskip \@tocrmarg + \advance\rightskip by 0pt plus 2cm + \parfillskip -\rightskip \pretolerance=10000 + \leavevmode \advance\leftskip\@tempdima \hskip -\leftskip + {\large\bfseries\boldmath#1}\ifx0#2\hfil\null + \else + \nobreak + \leaders\hbox{$\m at th \mkern \@dotsep mu.\mkern + \@dotsep mu$}\hfill + \nobreak\hbox to\@pnumwidth{\hss #2}% + \fi\par + \penalty\@highpenalty \endgroup} + +\def\l at title#1#2{\addpenalty{-\@highpenalty} + \addvspace{8pt plus 1pt} + \@tempdima \z@ + \begingroup + \parindent \z@ \rightskip \@tocrmarg + \advance\rightskip by 0pt plus 2cm + \parfillskip -\rightskip \pretolerance=10000 + \leavevmode \advance\leftskip\@tempdima \hskip -\leftskip + #1\nobreak + \leaders\hbox{$\m at th \mkern \@dotsep mu.\mkern + \@dotsep mu$}\hfill + \nobreak\hbox to\@pnumwidth{\hss #2}\par + \penalty\@highpenalty \endgroup} + +\def\l at author#1#2{\addpenalty{\@highpenalty} + \@tempdima=15\p@ %\z@ + \begingroup + \parindent \z@ \rightskip \@tocrmarg + \advance\rightskip by 0pt plus 2cm + \pretolerance=10000 + \leavevmode \advance\leftskip\@tempdima %\hskip -\leftskip + \textit{#1}\par + \penalty\@highpenalty \endgroup} + +\setcounter{tocdepth}{0} +\newdimen\tocchpnum +\newdimen\tocsecnum +\newdimen\tocsectotal +\newdimen\tocsubsecnum +\newdimen\tocsubsectotal +\newdimen\tocsubsubsecnum +\newdimen\tocsubsubsectotal +\newdimen\tocparanum +\newdimen\tocparatotal +\newdimen\tocsubparanum +\tocchpnum=\z@ % no chapter numbers +\tocsecnum=15\p@ % section 88. plus 2.222pt +\tocsubsecnum=23\p@ % subsection 88.8 plus 2.222pt +\tocsubsubsecnum=27\p@ % subsubsection 88.8.8 plus 1.444pt +\tocparanum=35\p@ % paragraph 88.8.8.8 plus 1.666pt +\tocsubparanum=43\p@ % subparagraph 88.8.8.8.8 plus 1.888pt +\def\calctocindent{% +\tocsectotal=\tocchpnum +\advance\tocsectotal by\tocsecnum +\tocsubsectotal=\tocsectotal +\advance\tocsubsectotal by\tocsubsecnum +\tocsubsubsectotal=\tocsubsectotal +\advance\tocsubsubsectotal by\tocsubsubsecnum +\tocparatotal=\tocsubsubsectotal +\advance\tocparatotal by\tocparanum} +\calctocindent + +\def\l at section{\@dottedtocline{1}{\tocchpnum}{\tocsecnum}} +\def\l at subsection{\@dottedtocline{2}{\tocsectotal}{\tocsubsecnum}} +\def\l at subsubsection{\@dottedtocline{3}{\tocsubsectotal}{\tocsubsubsecnum}} +\def\l at paragraph{\@dottedtocline{4}{\tocsubsubsectotal}{\tocparanum}} +\def\l at subparagraph{\@dottedtocline{5}{\tocparatotal}{\tocsubparanum}} + +\def\listoffigures{\@restonecolfalse\if at twocolumn\@restonecoltrue\onecolumn + \fi\section*{\listfigurename\@mkboth{{\listfigurename}}{{\listfigurename}}} + \@starttoc{lof}\if at restonecol\twocolumn\fi} +\def\l at figure{\@dottedtocline{1}{0em}{1.5em}} + +\def\listoftables{\@restonecolfalse\if at twocolumn\@restonecoltrue\onecolumn + \fi\section*{\listtablename\@mkboth{{\listtablename}}{{\listtablename}}} + \@starttoc{lot}\if at restonecol\twocolumn\fi} +\let\l at table\l at figure + +\renewcommand\listoffigures{% + \section*{\listfigurename + \@mkboth{\listfigurename}{\listfigurename}}% + \@starttoc{lof}% + } + +\renewcommand\listoftables{% + \section*{\listtablename + \@mkboth{\listtablename}{\listtablename}}% + \@starttoc{lot}% + } + +\ifx\oribibl\undefined +\ifx\citeauthoryear\undefined +\renewenvironment{thebibliography}[1] + {\section*{\refname} + \def\@biblabel##1{##1.} + \small + \list{\@biblabel{\@arabic\c at enumiv}}% + {\settowidth\labelwidth{\@biblabel{#1}}% + \leftmargin\labelwidth + \advance\leftmargin\labelsep + \if at openbib + \advance\leftmargin\bibindent + \itemindent -\bibindent + \listparindent \itemindent + \parsep \z@ + \fi + \usecounter{enumiv}% + \let\p at enumiv\@empty + \renewcommand\theenumiv{\@arabic\c at enumiv}}% + \if at openbib + \renewcommand\newblock{\par}% + \else + \renewcommand\newblock{\hskip .11em \@plus.33em \@minus.07em}% + \fi + \sloppy\clubpenalty4000\widowpenalty4000% + \sfcode`\.=\@m} + {\def\@noitemerr + {\@latex at warning{Empty `thebibliography' environment}}% + \endlist} +\def\@lbibitem[#1]#2{\item[{[#1]}\hfill]\if at filesw + {\let\protect\noexpand\immediate + \write\@auxout{\string\bibcite{#2}{#1}}}\fi\ignorespaces} +\newcount\@tempcntc +\def\@citex[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi + \@tempcnta\z@\@tempcntb\m at ne\def\@citea{}\@cite{\@for\@citeb:=#2\do + {\@ifundefined + {b@\@citeb}{\@citeo\@tempcntb\m at ne\@citea\def\@citea{,}{\bfseries + ?}\@warning + {Citation `\@citeb' on page \thepage \space undefined}}% + {\setbox\z@\hbox{\global\@tempcntc0\csname b@\@citeb\endcsname\relax}% + \ifnum\@tempcntc=\z@ \@citeo\@tempcntb\m at ne + \@citea\def\@citea{,}\hbox{\csname b@\@citeb\endcsname}% + \else + \advance\@tempcntb\@ne + \ifnum\@tempcntb=\@tempcntc + \else\advance\@tempcntb\m at ne\@citeo + \@tempcnta\@tempcntc\@tempcntb\@tempcntc\fi\fi}}\@citeo}{#1}} +\def\@citeo{\ifnum\@tempcnta>\@tempcntb\else + \@citea\def\@citea{,\,\hskip\z at skip}% + \ifnum\@tempcnta=\@tempcntb\the\@tempcnta\else + {\advance\@tempcnta\@ne\ifnum\@tempcnta=\@tempcntb \else + \def\@citea{--}\fi + \advance\@tempcnta\m at ne\the\@tempcnta\@citea\the\@tempcntb}\fi\fi} +\else +\renewenvironment{thebibliography}[1] + {\section*{\refname} + \small + \list{}% + {\settowidth\labelwidth{}% + \leftmargin\parindent + \itemindent=-\parindent + \labelsep=\z@ + \if at openbib + \advance\leftmargin\bibindent + \itemindent -\bibindent + \listparindent \itemindent + \parsep \z@ + \fi + \usecounter{enumiv}% + \let\p at enumiv\@empty + \renewcommand\theenumiv{}}% + \if at openbib + \renewcommand\newblock{\par}% + \else + \renewcommand\newblock{\hskip .11em \@plus.33em \@minus.07em}% + \fi + \sloppy\clubpenalty4000\widowpenalty4000% + \sfcode`\.=\@m} + {\def\@noitemerr + {\@latex at warning{Empty `thebibliography' environment}}% + \endlist} + \def\@cite#1{#1}% + \def\@lbibitem[#1]#2{\item[]\if at filesw + {\def\protect##1{\string ##1\space}\immediate + \write\@auxout{\string\bibcite{#2}{#1}}}\fi\ignorespaces} + \fi +\else +\@cons\@openbib at code{\noexpand\small} +\fi + +\def\idxquad{\hskip 10\p@}% space that divides entry from number + +\def\@idxitem{\par\hangindent 10\p@} + +\def\subitem{\par\setbox0=\hbox{--\enspace}% second order + \noindent\hangindent\wd0\box0}% index entry + +\def\subsubitem{\par\setbox0=\hbox{--\,--\enspace}% third + \noindent\hangindent\wd0\box0}% order index entry + +\def\indexspace{\par \vskip 10\p@ plus5\p@ minus3\p@\relax} + +\renewenvironment{theindex} + {\@mkboth{\indexname}{\indexname}% + \thispagestyle{empty}\parindent\z@ + \parskip\z@ \@plus .3\p@\relax + \let\item\par + \def\,{\relax\ifmmode\mskip\thinmuskip + \else\hskip0.2em\ignorespaces\fi}% + \normalfont\small + \begin{multicols}{2}[\@makeschapterhead{\indexname}]% + } + {\end{multicols}} + +\renewcommand\footnoterule{% + \kern-3\p@ + \hrule\@width 2truecm + \kern2.6\p@} + \newdimen\fnindent + \fnindent1em +\long\def\@makefntext#1{% + \parindent \fnindent% + \leftskip \fnindent% + \noindent + \llap{\hb at xt@1em{\hss\@makefnmark\ }}\ignorespaces#1} + +\long\def\@makecaption#1#2{% + \vskip\abovecaptionskip + \sbox\@tempboxa{{\bfseries #1.} #2}% + \ifdim \wd\@tempboxa >\hsize + {\bfseries #1.} #2\par + \else + \global \@minipagefalse + \hb at xt@\hsize{\hfil\box\@tempboxa\hfil}% + \fi + \vskip\belowcaptionskip} + +\def\fps at figure{htbp} +\def\fnum at figure{\figurename\thinspace\thefigure} +\def \@floatboxreset {% + \reset at font + \small + \@setnobreak + \@setminipage +} +\def\fps at table{htbp} +\def\fnum at table{\tablename~\thetable} +\renewenvironment{table} + {\setlength\abovecaptionskip{0\p@}% + \setlength\belowcaptionskip{10\p@}% + \@float{table}} + {\end at float} +\renewenvironment{table*} + {\setlength\abovecaptionskip{0\p@}% + \setlength\belowcaptionskip{10\p@}% + \@dblfloat{table}} + {\end at dblfloat} + +\long\def\@caption#1[#2]#3{\par\addcontentsline{\csname + ext@#1\endcsname}{#1}{\protect\numberline{\csname + the#1\endcsname}{\ignorespaces #2}}\begingroup + \@parboxrestore + \@makecaption{\csname fnum@#1\endcsname}{\ignorespaces #3}\par + \endgroup} + +% LaTeX does not provide a command to enter the authors institute +% addresses. The \institute command is defined here. + +\newcounter{@inst} +\newcounter{@auth} +\newcounter{auco} +\newdimen\instindent +\newbox\authrun +\newtoks\authorrunning +\newtoks\tocauthor +\newbox\titrun +\newtoks\titlerunning +\newtoks\toctitle + +\def\clearheadinfo{\gdef\@author{No Author Given}% + \gdef\@title{No Title Given}% + \gdef\@subtitle{}% + \gdef\@institute{No Institute Given}% + \gdef\@thanks{}% + \global\titlerunning={}\global\authorrunning={}% + \global\toctitle={}\global\tocauthor={}} + +\def\institute#1{\gdef\@institute{#1}} + +\def\institutename{\par + \begingroup + \parskip=\z@ + \parindent=\z@ + \setcounter{@inst}{1}% + \def\and{\par\stepcounter{@inst}% + \noindent$^{\the at inst}$\enspace\ignorespaces}% + \setbox0=\vbox{\def\thanks##1{}\@institute}% + \ifnum\c@@inst=1\relax + \gdef\fnnstart{0}% + \else + \xdef\fnnstart{\c@@inst}% + \setcounter{@inst}{1}% + \noindent$^{\the at inst}$\enspace + \fi + \ignorespaces + \@institute\par + \endgroup} + +\def\@fnsymbol#1{\ensuremath{\ifcase#1\or\star\or{\star\star}\or + {\star\star\star}\or \dagger\or \ddagger\or + \mathchar "278\or \mathchar "27B\or \|\or **\or \dagger\dagger + \or \ddagger\ddagger \else\@ctrerr\fi}} + +\def\inst#1{\unskip$^{#1}$} +\def\fnmsep{\unskip$^,$} +\def\email#1{{\tt#1}} +\AtBeginDocument{\@ifundefined{url}{\def\url#1{#1}}{}% +\@ifpackageloaded{babel}{% +\@ifundefined{extrasenglish}{}{\addto\extrasenglish{\switcht at albion}}% +\@ifundefined{extrasfrenchb}{}{\addto\extrasfrenchb{\switcht at francais}}% +\@ifundefined{extrasgerman}{}{\addto\extrasgerman{\switcht at deutsch}}% +}{\switcht@@therlang}% +} +\def\homedir{\~{ }} + +\def\subtitle#1{\gdef\@subtitle{#1}} +\clearheadinfo +% +\renewcommand\maketitle{\newpage + \refstepcounter{chapter}% + \stepcounter{section}% + \setcounter{section}{0}% + \setcounter{subsection}{0}% + \setcounter{figure}{0} + \setcounter{table}{0} + \setcounter{equation}{0} + \setcounter{footnote}{0}% + \begingroup + \parindent=\z@ + \renewcommand\thefootnote{\@fnsymbol\c at footnote}% + \if at twocolumn + \ifnum \col at number=\@ne + \@maketitle + \else + \twocolumn[\@maketitle]% + \fi + \else + \newpage + \global\@topnum\z@ % Prevents figures from going at top of page. + \@maketitle + \fi + \thispagestyle{empty}\@thanks +% + \def\\{\unskip\ \ignorespaces}\def\inst##1{\unskip{}}% + \def\thanks##1{\unskip{}}\def\fnmsep{\unskip}% + \instindent=\hsize + \advance\instindent by-\headlineindent + \if!\the\toctitle!\addcontentsline{toc}{title}{\@title}\else + \addcontentsline{toc}{title}{\the\toctitle}\fi + \if at runhead + \if!\the\titlerunning!\else + \edef\@title{\the\titlerunning}% + \fi + \global\setbox\titrun=\hbox{\small\rm\unboldmath\ignorespaces\@title}% + \ifdim\wd\titrun>\instindent + \typeout{Title too long for running head. Please supply}% + \typeout{a shorter form with \string\titlerunning\space prior to + \string\maketitle}% + \global\setbox\titrun=\hbox{\small\rm + Title Suppressed Due to Excessive Length}% + \fi + \xdef\@title{\copy\titrun}% + \fi +% + \if!\the\tocauthor!\relax + {\def\and{\noexpand\protect\noexpand\and}% + \protected at xdef\toc at uthor{\@author}}% + \else + \def\\{\noexpand\protect\noexpand\newline}% + \protected at xdef\scratch{\the\tocauthor}% + \protected at xdef\toc at uthor{\scratch}% + \fi + \addtocontents{toc}{\noexpand\protect\noexpand\authcount{\the\c at auco}}% + \addcontentsline{toc}{author}{\toc at uthor}% + \if at runhead + \if!\the\authorrunning! + \value{@inst}=\value{@auth}% + \setcounter{@auth}{1}% + \else + \edef\@author{\the\authorrunning}% + \fi + \global\setbox\authrun=\hbox{\small\unboldmath\@author\unskip}% + \ifdim\wd\authrun>\instindent + \typeout{Names of authors too long for running head. Please supply}% + \typeout{a shorter form with \string\authorrunning\space prior to + \string\maketitle}% + \global\setbox\authrun=\hbox{\small\rm + Authors Suppressed Due to Excessive Length}% + \fi + \xdef\@author{\copy\authrun}% + \markboth{\@author}{\@title}% + \fi + \endgroup + \setcounter{footnote}{\fnnstart}% + \clearheadinfo} +% +\def\@maketitle{\newpage + \markboth{}{}% + \def\lastand{\ifnum\value{@inst}=2\relax + \unskip{} \andname\ + \else + \unskip \lastandname\ + \fi}% + \def\and{\stepcounter{@auth}\relax + \ifnum\value{@auth}=\value{@inst}% + \lastand + \else + \unskip, + \fi}% + \begin{center}% + \let\newline\\ + {\Large \bfseries\boldmath + \pretolerance=10000 + \@title \par}\vskip .8cm +\if!\@subtitle!\else {\large \bfseries\boldmath + \vskip -.65cm + \pretolerance=10000 + \@subtitle \par}\vskip .8cm\fi + \setbox0=\vbox{\setcounter{@auth}{1}\def\and{\stepcounter{@auth}}% + \def\thanks##1{}\@author}% + \global\value{@inst}=\value{@auth}% + \global\value{auco}=\value{@auth}% + \setcounter{@auth}{1}% +{\lineskip .5em +\noindent\ignorespaces +\@author\vskip.35cm} + {\small\institutename} + \end{center}% + } + +% definition of the "\spnewtheorem" command. +% +% Usage: +% +% \spnewtheorem{env_nam}{caption}[within]{cap_font}{body_font} +% or \spnewtheorem{env_nam}[numbered_like]{caption}{cap_font}{body_font} +% or \spnewtheorem*{env_nam}{caption}{cap_font}{body_font} +% +% New is "cap_font" and "body_font". It stands for +% fontdefinition of the caption and the text itself. +% +% "\spnewtheorem*" gives a theorem without number. +% +% A defined spnewthoerem environment is used as described +% by Lamport. +% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +\def\@thmcountersep{} +\def\@thmcounterend{.} + +\def\spnewtheorem{\@ifstar{\@sthm}{\@Sthm}} + +% definition of \spnewtheorem with number + +\def\@spnthm#1#2{% + \@ifnextchar[{\@spxnthm{#1}{#2}}{\@spynthm{#1}{#2}}} +\def\@Sthm#1{\@ifnextchar[{\@spothm{#1}}{\@spnthm{#1}}} + +\def\@spxnthm#1#2[#3]#4#5{\expandafter\@ifdefinable\csname #1\endcsname + {\@definecounter{#1}\@addtoreset{#1}{#3}% + \expandafter\xdef\csname the#1\endcsname{\expandafter\noexpand + \csname the#3\endcsname \noexpand\@thmcountersep \@thmcounter{#1}}% + \expandafter\xdef\csname #1name\endcsname{#2}% + \global\@namedef{#1}{\@spthm{#1}{\csname #1name\endcsname}{#4}{#5}}% + \global\@namedef{end#1}{\@endtheorem}}} + +\def\@spynthm#1#2#3#4{\expandafter\@ifdefinable\csname #1\endcsname + {\@definecounter{#1}% + \expandafter\xdef\csname the#1\endcsname{\@thmcounter{#1}}% + \expandafter\xdef\csname #1name\endcsname{#2}% + \global\@namedef{#1}{\@spthm{#1}{\csname #1name\endcsname}{#3}{#4}}% + \global\@namedef{end#1}{\@endtheorem}}} + +\def\@spothm#1[#2]#3#4#5{% + \@ifundefined{c@#2}{\@latexerr{No theorem environment `#2' defined}\@eha}% + {\expandafter\@ifdefinable\csname #1\endcsname + {\global\@namedef{the#1}{\@nameuse{the#2}}% + \expandafter\xdef\csname #1name\endcsname{#3}% + \global\@namedef{#1}{\@spthm{#2}{\csname #1name\endcsname}{#4}{#5}}% + \global\@namedef{end#1}{\@endtheorem}}}} + +\def\@spthm#1#2#3#4{\topsep 7\p@ \@plus2\p@ \@minus4\p@ +\refstepcounter{#1}% +\@ifnextchar[{\@spythm{#1}{#2}{#3}{#4}}{\@spxthm{#1}{#2}{#3}{#4}}} + +\def\@spxthm#1#2#3#4{\@spbegintheorem{#2}{\csname the#1\endcsname}{#3}{#4}% + \ignorespaces} + +\def\@spythm#1#2#3#4[#5]{\@spopargbegintheorem{#2}{\csname + the#1\endcsname}{#5}{#3}{#4}\ignorespaces} + +\def\@spbegintheorem#1#2#3#4{\trivlist + \item[\hskip\labelsep{#3#1\ #2\@thmcounterend}]#4} + +\def\@spopargbegintheorem#1#2#3#4#5{\trivlist + \item[\hskip\labelsep{#4#1\ #2}]{#4(#3)\@thmcounterend\ }#5} + +% definition of \spnewtheorem* without number + +\def\@sthm#1#2{\@Ynthm{#1}{#2}} + +\def\@Ynthm#1#2#3#4{\expandafter\@ifdefinable\csname #1\endcsname + {\global\@namedef{#1}{\@Thm{\csname #1name\endcsname}{#3}{#4}}% + \expandafter\xdef\csname #1name\endcsname{#2}% + \global\@namedef{end#1}{\@endtheorem}}} + +\def\@Thm#1#2#3{\topsep 7\p@ \@plus2\p@ \@minus4\p@ +\@ifnextchar[{\@Ythm{#1}{#2}{#3}}{\@Xthm{#1}{#2}{#3}}} + +\def\@Xthm#1#2#3{\@Begintheorem{#1}{#2}{#3}\ignorespaces} + +\def\@Ythm#1#2#3[#4]{\@Opargbegintheorem{#1} + {#4}{#2}{#3}\ignorespaces} + +\def\@Begintheorem#1#2#3{#3\trivlist + \item[\hskip\labelsep{#2#1\@thmcounterend}]} + +\def\@Opargbegintheorem#1#2#3#4{#4\trivlist + \item[\hskip\labelsep{#3#1}]{#3(#2)\@thmcounterend\ }} + +\if at envcntsect + \def\@thmcountersep{.} + \spnewtheorem{theorem}{Theorem}[section]{\bfseries}{\itshape} +\else + \spnewtheorem{theorem}{Theorem}{\bfseries}{\itshape} + \if at envcntreset + \@addtoreset{theorem}{section} + \else + \@addtoreset{theorem}{chapter} + \fi +\fi + +%definition of divers theorem environments +\spnewtheorem*{claim}{Claim}{\itshape}{\rmfamily} +\spnewtheorem*{proof}{Proof}{\itshape}{\rmfamily} +\if at envcntsame % alle Umgebungen wie Theorem. + \def\spn at wtheorem#1#2#3#4{\@spothm{#1}[theorem]{#2}{#3}{#4}} +\else % alle Umgebungen mit eigenem Zaehler + \if at envcntsect % mit section numeriert + \def\spn at wtheorem#1#2#3#4{\@spxnthm{#1}{#2}[section]{#3}{#4}} + \else % nicht mit section numeriert + \if at envcntreset + \def\spn at wtheorem#1#2#3#4{\@spynthm{#1}{#2}{#3}{#4} + \@addtoreset{#1}{section}} + \else + \def\spn at wtheorem#1#2#3#4{\@spynthm{#1}{#2}{#3}{#4} + \@addtoreset{#1}{chapter}}% + \fi + \fi +\fi +\spn at wtheorem{case}{Case}{\itshape}{\rmfamily} +\spn at wtheorem{conjecture}{Conjecture}{\itshape}{\rmfamily} +\spn at wtheorem{corollary}{Corollary}{\bfseries}{\itshape} +\spn at wtheorem{definition}{Definition}{\bfseries}{\itshape} +\spn at wtheorem{example}{Example}{\itshape}{\rmfamily} +\spn at wtheorem{exercise}{Exercise}{\itshape}{\rmfamily} +\spn at wtheorem{lemma}{Lemma}{\bfseries}{\itshape} +\spn at wtheorem{note}{Note}{\itshape}{\rmfamily} +\spn at wtheorem{problem}{Problem}{\itshape}{\rmfamily} +\spn at wtheorem{property}{Property}{\itshape}{\rmfamily} +\spn at wtheorem{proposition}{Proposition}{\bfseries}{\itshape} +\spn at wtheorem{question}{Question}{\itshape}{\rmfamily} +\spn at wtheorem{solution}{Solution}{\itshape}{\rmfamily} +\spn at wtheorem{remark}{Remark}{\itshape}{\rmfamily} + +\def\@takefromreset#1#2{% + \def\@tempa{#1}% + \let\@tempd\@elt + \def\@elt##1{% + \def\@tempb{##1}% + \ifx\@tempa\@tempb\else + \@addtoreset{##1}{#2}% + \fi}% + \expandafter\expandafter\let\expandafter\@tempc\csname cl@#2\endcsname + \expandafter\def\csname cl@#2\endcsname{}% + \@tempc + \let\@elt\@tempd} + +\def\theopargself{\def\@spopargbegintheorem##1##2##3##4##5{\trivlist + \item[\hskip\labelsep{##4##1\ ##2}]{##4##3\@thmcounterend\ }##5} + \def\@Opargbegintheorem##1##2##3##4{##4\trivlist + \item[\hskip\labelsep{##3##1}]{##3##2\@thmcounterend\ }} + } + +\renewenvironment{abstract}{% + \list{}{\advance\topsep by0.35cm\relax\small + \leftmargin=1cm + \labelwidth=\z@ + \listparindent=\z@ + \itemindent\listparindent + \rightmargin\leftmargin}\item[\hskip\labelsep + \bfseries\abstractname]} + {\endlist} + +\newdimen\headlineindent % dimension for space between +\headlineindent=1.166cm % number and text of headings. + +\def\ps at headings{\let\@mkboth\@gobbletwo + \let\@oddfoot\@empty\let\@evenfoot\@empty + \def\@evenhead{\normalfont\small\rlap{\thepage}\hspace{\headlineindent}% + \leftmark\hfil} + \def\@oddhead{\normalfont\small\hfil\rightmark\hspace{\headlineindent}% + \llap{\thepage}} + \def\chaptermark##1{}% + \def\sectionmark##1{}% + \def\subsectionmark##1{}} + +\def\ps at titlepage{\let\@mkboth\@gobbletwo + \let\@oddfoot\@empty\let\@evenfoot\@empty + \def\@evenhead{\normalfont\small\rlap{\thepage}\hspace{\headlineindent}% + \hfil} + \def\@oddhead{\normalfont\small\hfil\hspace{\headlineindent}% + \llap{\thepage}} + \def\chaptermark##1{}% + \def\sectionmark##1{}% + \def\subsectionmark##1{}} + +\if at runhead\ps at headings\else +\ps at empty\fi + +\setlength\arraycolsep{1.4\p@} +\setlength\tabcolsep{1.4\p@} + +\endinput +%end of file llncs.cls Added: pypy/extradoc/talk/ecoop2009/main.bbl ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/main.bbl Tue Dec 2 18:32:40 2008 @@ -0,0 +1,29 @@ +\begin{thebibliography}{1} + +\bibitem{AACM-DLS07} +D.~Ancona, M.~Ancona, A~Cuni, and N.~Matsakis. +\newblock R{P}ython: a {S}tep {T}owards {R}econciling {D}ynamically and + {S}tatically {T}yped {OO} {L}anguages. +\newblock In {\em O{OPSLA} 2007 {P}roceedings and {C}ompanion, {DLS}'07: + {P}roceedings of the 2007 {S}ymposium on {D}ynamic {L}anguages}, pages + 53--64. ACM, 2007. + +\bibitem{BolzEtAl08} +C.~F. Bolz, A.~Kuhn, A.~Lienhard, N.~D. Matsakis, O.~Nierstrasz, L.~Renggli, + A.~Rigo, and T.~Verwaest. +\newblock Back to the future in one week - implementing a smalltalk vm in pypy. +\newblock In {\em Self-Sustaining Systems, First Workshop, S3 2008, Potsdam, + Revised Selected Papers}, volume 5146 of {\em Lecture Notes in Computer + Science}, pages 123--139, 2008. + +\bibitem{Futamura99} +Yoshihiko Futamura. +\newblock Partial evaluation of computation process, revisited. +\newblock {\em Higher Order Symbol. Comput.}, 12(4):377--380, 1999. + +\bibitem{RigoPedroni06} +A.~Rigo and S.~Pedroni. +\newblock Py{P}y's approach to virtual machine construction. +\newblock In {\em OOPSLA Companion}, pages 944--953, 2006. + +\end{thebibliography} Added: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/main.tex Tue Dec 2 18:32:40 2008 @@ -0,0 +1,45 @@ +\documentclass{llncs} + +\usepackage{amssymb} +\usepackage{amsmath} +\usepackage[sans]{dsfont} +\usepackage{color} +\usepackage{xspace} +\usepackage{listings} +\usepackage[pdftex]{graphicx} + +%\input{macros} + +\pagestyle{plain} + +\lstset{mathescape=true,language=Java,basicstyle=\tt,keywordstyle=\bf} + +%\renewcommand{\baselinestretch}{.98} + +\begin{document} +\title{Automatic generation of JIT compilers for dynamic languages + in .NET\thanks{This work has been partially +supported by MIUR EOS DUE - Extensible Object Systems for Dynamic and +Unpredictable Environments.}} + + +\author{Davide Ancona\inst{1} \and Carl Friedrich Bolz\inst{2} \and Antonio Cuni\inst{1} \and Armin Rigo} + +\institute{DISI, University of Genova, Italy +\and +Softwaretechnik und Programmiersprachen + Heinrich-Heine-Universit\"at D\"usseldorf} + +\maketitle + +\input{abstract} +\input{intro} +\input{jitgen} +\input{rainbow} +\input{clibackend} +%\input{conclusion} + +\bibliographystyle{plain} +\bibliography{ALLMY} + +\end{document} Added: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Tue Dec 2 18:32:40 2008 @@ -0,0 +1,96 @@ +\section{The Rainbow Interpreter} + +The JIT compiler is implemented by running the "Rainbow interpreter", whose +output is a compiled function. + +Variables and opcodes are colored: + + - green values are known at compile time, and green operations are executed + when interpreting the bytecode; + + - red values are not known until runtime; the result of a red operation is a + piece of machine code that will compute the result at runtime. + +The Rainbow bytecode is produced at translation time, when the JIT compiler is +generated. + +Here are summarized the various phases of the JIT: + +Translation time: + + * Low-level flowgraphs are produced + + * The *hint-annotator* colors the variables + + * The *rainbow codewriter* translates flowgraphs into rainbow bytecode + + +Compile-time: + + * The rainbow interpreter executes the bytecode + + * As a result, it produces executable code + +Runtime: + + * The produced code is executed + + +\subsection{Example of Rainbow bytecode and execution} + +TODO + +\subsection{Promotion} + +There are values that, if known at compile time, allow the JIT compiler to +produce very efficient code. Unfortunately, these values are tipically red, +e.g. the exact type of a variable. + +"Promotion" is a particular operation that convert a red value into a green +value; i.e., after the promotion of a variable, the JIT compiler knows its +value at compile time. Since the value of a red variable is not known until +runtime, we need to postpone the compilation phase after the runtime phase. + +This is done by continuously intermixing compile time and runtime; a promotion +is implemented in this way: + + * (compile time): the rainbow interpreter produces machine code until it + hits a promotion point; e.g.:: + + \begin{lstlisting}[language=C] + y = hint(x, promote=True) + return y+10 + \end{lstlisting} + + * (compile time): at this point, it generates special machine code that when + reached calls the JIT compiler again; the JIT compilation stops:: + + \begin{lstlisting}[language=C] + switch(y) { + default: compile_more(y); + } + \end{lstlisting} + + * (runtime): the machine code is executed; when it reaches a promotion + point, it executes the special machine code we described in the previous + point; the JIT compiler is invoked again; + + * (compile time): now we finally know the exact value of our red variable, + and we can promote it to green; suppose that the value of 'y' is 32:: + + \begin{lstlisting}[language=C] + switch(y) { + 32: return 42; + default: compile_more(y); + } + \end{lstlisting} + + Note that the operation "y+10" has been constant-folded into "42", as it + was a green operation. + + * (runtime) the execution restart from the point it stopped, until a new + unhandled promotion point is reached. + +\subsection{Virtuals and virtualizables} + +TODO From antocuni at codespeak.net Tue Dec 2 18:42:20 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 2 Dec 2008 18:42:20 +0100 (CET) Subject: [pypy-svn] r60283 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081202174220.09DFA1683D8@codespeak.net> Author: antocuni Date: Tue Dec 2 18:42:18 2008 New Revision: 60283 Modified: pypy/extradoc/talk/ecoop2009/main.tex Log: tentative structure of the paper, as discussed today Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Tue Dec 2 18:42:18 2008 @@ -32,6 +32,30 @@ \maketitle +\section{Tentative structure} +\begin{itemize} + \item Introduction \& background; main contributions: + \begin{itemize} + \item Promotion + \item (unboxing) + \item JIT layering + \end{itemize} + \item How the generated JITs work + \begin{itemize} + \item Promotion: allows intermixing compile-time and runtime; how to + compare with polymorphic inline caches and partial evaluation + \item unboxing: how they compare with tracing JITs + \item (don't talk about merging) + \end{itemize} + \item .NET backend + \item How the JIT generator works + \item Benchmarks (TLC) + \item Future works + \item Conclusions +\end{itemize} + + + \input{abstract} \input{intro} \input{jitgen} From cfbolz at codespeak.net Wed Dec 3 17:12:52 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 3 Dec 2008 17:12:52 +0100 (CET) Subject: [pypy-svn] r60297 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20081203161252.E43DA168425@codespeak.net> Author: cfbolz Date: Wed Dec 3 17:12:52 2008 New Revision: 60297 Modified: pypy/trunk/pypy/rpython/lltypesystem/rlist.py Log: comment I had while reading resize Modified: pypy/trunk/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rlist.py Wed Dec 3 17:12:52 2008 @@ -201,6 +201,7 @@ if newsize <= 0: ll_assert(newsize == 0, "negative list length") new_allocated = 0 + # XXX why not use _ll_prebuilt_empty_array here? else: if newsize < 9: some = 3 From arigo at codespeak.net Wed Dec 3 17:17:22 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 3 Dec 2008 17:17:22 +0100 (CET) Subject: [pypy-svn] r60298 - in pypy/branch/oo-jit/pypy/rpython: lltypesystem ootypesystem test Message-ID: <20081203161722.109D816842D@codespeak.net> Author: arigo Date: Wed Dec 3 17:17:21 2008 New Revision: 60298 Modified: pypy/branch/oo-jit/pypy/rpython/lltypesystem/rstr.py pypy/branch/oo-jit/pypy/rpython/ootypesystem/rstr.py pypy/branch/oo-jit/pypy/rpython/test/test_rstr.py Log: A bug in rstr, for comparing with <=, <, >=, > with one or both of the elements being a None. Modified: pypy/branch/oo-jit/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/oo-jit/pypy/rpython/lltypesystem/rstr.py Wed Dec 3 17:17:21 2008 @@ -372,10 +372,13 @@ return result def ll_strcmp(s1, s2): - if not s1 and not s2: - return True - if not s1 or not s2: - return False + if not s1: + if not s2: + return 0 + else: + return -1 + elif not s2: + return 1 chars1 = s1.chars chars2 = s2.chars len1 = len(chars1) Modified: pypy/branch/oo-jit/pypy/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/ootypesystem/rstr.py (original) +++ pypy/branch/oo-jit/pypy/rpython/ootypesystem/rstr.py Wed Dec 3 17:17:21 2008 @@ -138,10 +138,13 @@ return s1.ll_streq(s2) def ll_strcmp(s1, s2): - if not s1 and not s2: - return True - if not s1 or not s2: - return False + if not s1: + if not s2: + return 0 + else: + return -1 + elif not s2: + return 1 return s1.ll_strcmp(s2) def ll_join(s, length_dummy, lst): Modified: pypy/branch/oo-jit/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/test/test_rstr.py (original) +++ pypy/branch/oo-jit/pypy/rpython/test/test_rstr.py Wed Dec 3 17:17:21 2008 @@ -151,58 +151,49 @@ def test_str_compare(self): const = self.const + s1 = [const('one'), const('two'), None] + s2 = [const('one'), const('two'), const('o'), + const('on'), const('twos'), const('foobar'), None] + def fn(i, j): - s1 = [const('one'), const('two'), None] - s2 = [const('one'), const('two'), const('o'), - const('on'), const('twos'), const('foobar'), None] return s1[i] == s2[j] - for i in range(3): - for j in range(7): + for i in range(len(s1)): + for j in range(len(s2)): res = self.interpret(fn, [i,j]) assert res is fn(i, j) def fn(i, j): - s1 = [const('one'), const('two')] - s2 = [const('one'), const('two'), const('o'), const('on'), const('twos'), const('foobar')] return s1[i] != s2[j] - for i in range(2): - for j in range(6): + for i in range(len(s1)): + for j in range(len(s2)): res = self.interpret(fn, [i, j]) assert res is fn(i, j) def fn(i, j): - s1 = [const('one'), const('two')] - s2 = [const('one'), const('two'), const('o'), const('on'), const('twos'), const('foobar')] return s1[i] < s2[j] - for i in range(2): - for j in range(6): + for i in range(len(s1)): + for j in range(len(s2)): res = self.interpret(fn, [i,j]) assert res is fn(i, j) def fn(i, j): - s1 = [const('one'), const('two')] - s2 = [const('one'), const('two'), const('o'), const('on'), const('twos'), const('foobar')] return s1[i] <= s2[j] - for i in range(2): - for j in range(6): + for i in range(len(s1)): + for j in range(len(s2)): res = self.interpret(fn, [i,j]) assert res is fn(i, j) def fn(i, j): - s1 = [const('one'), const('two')] - s2 = [const('one'), const('two'), const('o'), const('on'), const('twos'), const('foobar')] return s1[i] >= s2[j] - for i in range(2): - for j in range(6): + for i in range(len(s1)): + for j in range(len(s2)): res = self.interpret(fn, [i,j]) assert res is fn(i, j) def fn(i, j): - s1 = [const('one'), const('two')] - s2 = [const('one'), const('two'), const('o'), const('on'), const('twos'), const('foobar')] return s1[i] > s2[j] - for i in range(2): - for j in range(6): + for i in range(len(s1)): + for j in range(len(s2)): res = self.interpret(fn, [i,j]) assert res is fn(i, j) From antocuni at codespeak.net Thu Dec 4 00:30:40 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 4 Dec 2008 00:30:40 +0100 (CET) Subject: [pypy-svn] r60306 - in pypy/branch/oo-jit/pypy/jit/tl: . test Message-ID: <20081203233040.B0B59168422@codespeak.net> Author: antocuni Date: Thu Dec 4 00:30:40 2008 New Revision: 60306 Modified: pypy/branch/oo-jit/pypy/jit/tl/factorial.tlc pypy/branch/oo-jit/pypy/jit/tl/targettlc.py pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py Log: make tlc's object oriented features working also with targettlc, by storing the constant pool in the compiled file Modified: pypy/branch/oo-jit/pypy/jit/tl/factorial.tlc ============================================================================== Binary files. No diff available. Modified: pypy/branch/oo-jit/pypy/jit/tl/targettlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/targettlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/targettlc.py Thu Dec 4 00:30:40 2008 @@ -1,7 +1,7 @@ import time import py py.magic.autopath() -from pypy.jit.tl.tlc import interp, interp_eval, interp_nonjit +from pypy.jit.tl.tlc import interp, interp_eval, interp_nonjit, ConstantPool from pypy.jit.codegen.hlinfo import highleveljitinfo @@ -25,32 +25,40 @@ filename = args[0] x = int(args[1]) - bytecode = load_bytecode(filename) + bytecode, pool = load_bytecode(filename) if not onlyjit: start = time.clock() - res = interp_nonjit(bytecode, inputarg=x) + res = interp_nonjit(bytecode, inputarg=x, pool=pool) stop = time.clock() print 'Non jitted: %d (%f seconds)' % (res, stop-start) start = time.clock() - res = interp(bytecode, inputarg=x) + res = interp(bytecode, inputarg=x, pool=pool) stop = time.clock() print 'Warmup jitted: %d (%f seconds)' % (res, stop-start) start = time.clock() - res = interp(bytecode, inputarg=x) + res = interp(bytecode, inputarg=x, pool=pool) stop = time.clock() print 'Warmed jitted: %d (%f seconds)' % (res, stop-start) return 0 +def decode_poolcode(s): + pool = ConstantPool() + lists = s.split('|') + pool.strlists = [lst.split(',') for lst in lists] + return pool + def load_bytecode(filename): from pypy.rlib.streamio import open_file_as_stream f = open_file_as_stream(filename) - bytecode = f.readall() + poolcode = f.readline()[:-1] + bytecode = f.readall()[:-1] f.close() - return bytecode + pool = decode_poolcode(poolcode) + return bytecode, pool def target(driver, args): return entry_point, None Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py Thu Dec 4 00:30:40 2008 @@ -13,9 +13,11 @@ assert pool.strlists == [['foo', 'bar']] class TestTLC(test_tl.TestTL): - from pypy.jit.tl.tlc import interp - interp = staticmethod(interp) - + @staticmethod + def interp(code='', pc=0, inputarg=0): + from pypy.jit.tl.tlc import interp + return interp(code, pc, inputarg) + def test_basic_cons_cell(self): bytecode = compile(""" NIL Modified: pypy/branch/oo-jit/pypy/jit/tl/tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlc.py Thu Dec 4 00:30:40 2008 @@ -174,17 +174,21 @@ @specialize.argtype(0) def hint(x, global_merge_point=False, promote_class=False, + promote=False, + deepfreeze=False, forget=False, concrete=False): return x - def interp(code='', pc=0, inputarg=0): + def interp(code='', pc=0, inputarg=0, pool=None): if not isinstance(code,str): raise TypeError("code '%s' should be a string" % str(code)) - return interp_eval(code, pc, IntObj(inputarg)).int_o() + if pool is None: + pool = ConstantPool() + return interp_eval(code, pc, IntObj(inputarg), pool).int_o() - def interp_eval(code, pc, inputarg, pool2=ConstantPool()): + def interp_eval(code, pc, inputarg, pool2): code_len = len(code) stack = [] pool = hint(hint(pool2, concrete=True), deepfreeze=True) @@ -321,7 +325,7 @@ elif supports_call and opcode == CALL: offset = char2int(code[pc]) pc += 1 - res = interp_eval(code, pc + offset, zero) + res = interp_eval(code, pc + offset, zero, pool2) stack.append( res ) elif opcode == RETURN: @@ -371,8 +375,15 @@ if __name__ == '__main__': import sys from pypy.jit.tl.test.test_tl import FACTORIAL_SOURCE - bytecode = compile(FACTORIAL_SOURCE) - if len(sys.argv) >= 2 and sys.argv[1] == 'assemble': - print bytecode + if len(sys.argv) == 1: + src = FACTORIAL_SOURCE + elif len(sys.argv) == 2: + src = file(sys.argv[1]).read() else: - print ','.join([str(ord(n)) for n in bytecode]) + print >> sys.stderr, 'Usage: python tlc.py [sourcefile]' + sys.exit(2) + + pool = ConstantPool() + bytecode = compile(src, pool) + print serialize_pool(pool) + print bytecode Modified: pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py Thu Dec 4 00:30:40 2008 @@ -88,3 +88,7 @@ for label, pc in label_usage: bytecode[pc] = labels[label] - pc - 1 return ''.join([chr(i & 0xff) for i in bytecode]) + +def serialize_pool(pool): + lists = [','.join(lst) for lst in pool.strlists] + return '|'.join(lists) From mwh at codespeak.net Thu Dec 4 03:30:48 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Thu, 4 Dec 2008 03:30:48 +0100 (CET) Subject: [pypy-svn] r60314 - pypy/extradoc/talk/osdc2008 Message-ID: <20081204023048.017501683E0@codespeak.net> Author: mwh Date: Thu Dec 4 03:30:47 2008 New Revision: 60314 Modified: pypy/extradoc/talk/osdc2008/osdc08.pdf Log: incorpate fijal's comments, fix XXXs Modified: pypy/extradoc/talk/osdc2008/osdc08.pdf ============================================================================== Binary files. No diff available. From antocuni at codespeak.net Thu Dec 4 09:31:03 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 4 Dec 2008 09:31:03 +0100 (CET) Subject: [pypy-svn] r60315 - in pypy/branch/oo-jit/pypy/jit/tl: . test Message-ID: <20081204083103.B1C38168067@codespeak.net> Author: antocuni Date: Thu Dec 4 09:31:01 2008 New Revision: 60315 Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlc.py Log: implement truth value and equality for objects Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py Thu Dec 4 09:31:01 2008 @@ -157,3 +157,31 @@ """, pool) res = interp_eval(bytecode, 0, nil, pool) assert res.int_o() == 42 + + def test_obj_truth(self): + from pypy.jit.tl.tlc import interp_eval, nil + pool = ConstantPool() + bytecode = compile(""" + NEW foo,bar + BR_COND true + PUSH 12 + PUSH 1 + BR_COND exit + true: + PUSH 42 + exit: + RETURN + """, pool) + res = interp_eval(bytecode, 0, nil, pool) + assert res.int_o() == 42 + + def test_obj_equality(self): + from pypy.jit.tl.tlc import interp_eval, nil + pool = ConstantPool() + bytecode = compile(""" + NEW foo,bar + NEW foo,bar + EQ + """, pool) + res = interp_eval(bytecode, 0, nil, pool) + assert res.int_o() == 0 Modified: pypy/branch/oo-jit/pypy/jit/tl/tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlc.py Thu Dec 4 09:31:01 2008 @@ -73,6 +73,12 @@ cls = hint(self.cls, promote=True) return hint(cls, deepfreeze=True) + def t(self): + return True + + def eq(self, other): + return self is other + def getattr(self, name): i = self.getclass().attributes[name] return self.values[i] From xoraxax at codespeak.net Mon Dec 8 12:57:50 2008 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Mon, 8 Dec 2008 12:57:50 +0100 (CET) Subject: [pypy-svn] r60358 - in pypy/trunk/pypy/rlib: . test Message-ID: <20081208115750.0AD26168427@codespeak.net> Author: xoraxax Date: Mon Dec 8 12:57:50 2008 New Revision: 60358 Modified: pypy/trunk/pypy/rlib/rpoll.py pypy/trunk/pypy/rlib/test/test_rpoll.py Log: Fix rpolls select to adhere to the correct timeout. Modified: pypy/trunk/pypy/rlib/rpoll.py ============================================================================== --- pypy/trunk/pypy/rlib/rpoll.py (original) +++ pypy/trunk/pypy/rlib/rpoll.py Mon Dec 8 12:57:50 2008 @@ -105,7 +105,7 @@ ll_timeval = lltype.malloc(_c.timeval, flavor='raw') frac = math.fmod(timeout, 1.0) ll_timeval.c_tv_sec = int(timeout) - ll_timeval.c_tv_usec = int(timeout*1000000.0) + ll_timeval.c_tv_usec = int((timeout - int(timeout)) * 1000000.0) else: ll_timeval = lltype.nullptr(_c.timeval) try: Modified: pypy/trunk/pypy/rlib/test/test_rpoll.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_rpoll.py (original) +++ pypy/trunk/pypy/rlib/test/test_rpoll.py Mon Dec 8 12:57:50 2008 @@ -50,6 +50,7 @@ serv.close() def test_select(): + from time import time def f(): readend, writeend = os.pipe() try: @@ -59,10 +60,17 @@ iwtd, owtd, ewtd = select([readend], [], []) assert iwtd == [readend] assert owtd == ewtd == [] + finally: os.close(readend) os.close(writeend) + # once there was a bug where the sleeping time was doubled + a = time() + iwtd, owtd, ewtd = select([], [], [], 1.0) + diff = time() - a + assert 0.9 < diff < 1.1 + f() interpret(f, []) From davide at codespeak.net Thu Dec 11 15:59:00 2008 From: davide at codespeak.net (davide at codespeak.net) Date: Thu, 11 Dec 2008 15:59:00 +0100 (CET) Subject: [pypy-svn] r60425 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081211145900.8FE1B168452@codespeak.net> Author: davide Date: Thu Dec 11 15:58:58 2008 New Revision: 60425 Modified: pypy/extradoc/talk/ecoop2009/main.tex Log: test for password Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Thu Dec 11 15:58:58 2008 @@ -67,3 +67,4 @@ \bibliography{ALLMY} \end{document} + From davide at codespeak.net Thu Dec 11 16:10:15 2008 From: davide at codespeak.net (davide at codespeak.net) Date: Thu, 11 Dec 2008 16:10:15 +0100 (CET) Subject: [pypy-svn] r60426 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081211151015.EE31D168434@codespeak.net> Author: davide Date: Thu Dec 11 16:10:14 2008 New Revision: 60426 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex pypy/extradoc/talk/ecoop2009/main.tex Log: started integrating Antonio's post in the paper Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Thu Dec 11 16:10:14 2008 @@ -1,8 +1,218 @@ \section{CLI backend for Rainbow interpreter} -\subsection{Promotion on CLI} +%\subsection{Promotion on CLI} +% +%Implementing promotion on top of CLI is not straightforward, as it needs to +%patch and modify the already generated code, and this is not possible on .NET. +% +%To solve, we do xxx and yyy etc. etc. +% -Implementing promotion on top of CLI is not straightforward, as it needs to -patch and modify the already generated code, and this is not possible on .NET. +\subsection{Flexswitches} -To solve, we do xxx and yyy etc. etc. +As already explained, \dacom{I guess flexswitches will be introduced + in the previous sect.} \emph{flexswitch} is one of the key +concepts allowing the JIT compiler generator to produce code which can +be incrementally specialized and compiled at run time. + +A flexswitch is a special kind of switch which can be dynamically +extended with new cases; intuitively, its behavior can be described +well in terms of flow graphs. Indeed, a flexswitch can be considered +as a special flow graph block where links to newly created blocks are +dynamically added whenever new cases are needed. \dacom{in theory links could also be removed, but I + do not know whether this possibility has been considered in Rainbow} + +\begin{figure}[h] +\begin{center} +\includegraphics[height=5cm]{flexswitch1} +\includegraphics[height=5cm]{flexswitch2} +\caption{An example of a flexswitch evolution: in the picture on the + right a new block has been dynamically added.}\label{flexswitch-fig} +\end{center} +\end{figure} + +In the pictures of Figure~\ref{flexswitch-fig}, the cyan block +corresponds to a flexswitch; initially (picture on the left) +only the block containing the code to restart the JIT compilation +is connected to the flexswitch; the picture on the right +shows the graph after the first case has been dynamically added to the flexswitch, +by linking the cyan block with a freshly created new block. + + +\subsection{Implementing flexswitches in CLI} + +Implementing flexswitches for backends generating assembly code is +quite straightforward: basically, a new jump has to be inserted in the +existing code to point to the newly generated code fragment. + +Unfortunately, the CLI VM does not allow modification of code which +has been already loaded and linked, therefore the simplest approach +taken for low level architectures does not work for higher level +virtual machines as those for .NET and Java. + +Since in .NET methods are the basic units of compilation, a possible +solution consists in creating a new method +any time a new case has to be added to a flexswitch. +\dacom{comment for Antonio: I am not sure this is the best solution. This cannot work for Java where classes are the basic + units} + +\dacom{I still have to polish what comes next} + +\newcommand{\commentout}[1]{} +\commentout{ +Because of all these constraints we cannot simply map each graph to its own method, since we saw that our graphs can grow after they have already been executed few times. + +Hence, we need to distinguish between the two concepts: + + * a graph is the logical unit of code as seen by the JIT compiler: concretely, the CLI JIT backend renders it as one or more methods; + * a method is a collection of basic blocks; each method has the so called parent graph, i.e. the graph its blocks logically belongs to. + +The first method of a graph is called main method (which has nothing to do with the Main static methods found in .exe files); other methods are called children methods. + +When we want to add a new case to the flexswitch, we create a method containing all the new code; then we wrap the method inside a delegate (the .NET equivalent of a function pointer) and pass it to the flexswitch, so that it can later invoke it. +The hard bit: non-local links + +Using this approach, after a while the blocks of our original graph are scattered over a lot of different methods; however, there are no constraints about how these blocks can be linked together, so it happens to have links between blocks which are not in the same method. In the following, we will refer to them as non-local links. + +If the non-local block we want to jump to happens to be at the beginning of its containing method, it is enough to invoke the method; but, what if we want to jump somewhere in the middle? What we really want is to produce a method which has multiple entry-points; again, doing it in assembly would be trivial, but the virtual machine does not provide any support for it, so we need a work around. + +Each method in a graph is assigned an unique 16 bit method id; each block in a method is assigned a progressive 16 bit block number. From this two numbers, we can compute the block id as an unsigned integer, by storing the method id in the first 16 bits and the block number in the second 16 bits. By construction, the block id is guaranteed to be unique in the graph. + +The following picture shows a graph composed of three methods; the id of each method is shown in red, while the block ids are shown in red (for the method id part) and black (for the block number part). The graph contains three non-local links; in particular, note the link between blocks 0x00020001 and 0x00010001 which connects two block that resides in different methods. + +Every method contains a special dispatch block, (not shown in the picture above) whose goal is to jump to the specified block number inside the method itself. The first argument of a child method is always a block id; when the method starts, it immediately jumps to the dispatch block, and thus to the desired block. + +For example, suppose to have a method which contains 3 blocks numbered 0, 1, 2; here is how its dispatch blocks looks like; for simplicity it is shown as C# code, but it is actually generated as IL bytecode: + +// dispatch block +int methodid = (blockid & 0xFFFF0000) >> 16); // take the first 16 bits +int blocknum = blockid && 0x0000FFFF; // take the second 16 bits + +if (methodid != MY_METHOD_ID) { +// jump_to_unknown block +... +} + +switch(blocknum) { +case 0: +goto block0; +case 1: +goto block1; +case 2: +goto block2; +default: +throw new Exception("Invalid block id"); +} + +Whenever we want to jump to a non-local block, it is enough to store the block id in the appropriate variable and jump to the dispatch block. If the block resides in a different method, the jump_to_unknown block is entered; this special block is implemented differently by the main method and the child methods, as we will see soon. + +Each time a new method is added to the graph, we build a delegate for it, and store it in a special array called method_map; since we assign the method id sequentially starting from 0, we are sure that to fetch the method whose id is n we can simply load the n-th element of the array. + +The jump_to_unknown block of the main method uses this array to select the right method, and calls it (FlexSwitchCase is the type of delegates for all children methods): + +// jump_to_unknown block of the main method +FlexSwitchCase meth = method_map[methodid]; +blockid = meth(blockid, ...); // execute the method +goto dispatch_block; + +Each child method returns a block id specifying the next block to jump to; after its execution, we assign the return value to the blockid variable, and jump again to the dispatch block, which will jump again to the appropriate block. + +Keeping this in mind, it is straightforward to implement the jump_to_unknown block of children methods: it is enough to return the target block id to the caller, and let its dispatch loop do the right thing. If the caller is also a child method, it will return it again, until we reach the dispatch loop of the main method, which will finally do the jump. In theory, we could implement things differently and jumping directly from a child method to another one, but in that case the call stack could grows indefinitely in case of a tight loop between two blocks residing in different methods. + +To implement the dispatch block we can exploit the switch opcode of the CLI; if the .NET JIT is smart enough, it can render it using an indirect jump; overall, jumping to a non-local block consists of an indirect function call (by invoking the delegate) plus an indirect jump (by executing the switch opcode); even if this is more costly than a simple direct jump, we will see in the next section that this not the main source of overhead when following a non-local link. + +Obviously, the slow dispatching logic is needed only when we want to jump to a non-local block; if the target block happens to reside in the same method as the current one, we can directly jump to it, completely removing the overhead. + +Moreover, the dispatch blocks are emitted only if needed, i.e. if the parent graph contains at least one flexswitch; graphs without flexswitches are rendered in the obvious way, by making one method per graph. +The slow bit: passing arguments + +Jumping to the correct block is not enough to follow a link: as we said before, each link carries a set of arguments to be passed from the source to the target block. As usual, passing arguments across local links is easy, as we can just use local variables to hold their values; on the other hand, non-local links make things more complex. + +The only way to jump to a block is to invoke its containing method, so the first solution that comes to mind is to specify its input arguments as parameter of the method; however, each block has potentially a different number (and different types) of input arguments than every other block, so we need to think of something else. + +An alternative solution could be to compute the union of the sets of input arguments of all the blocks in the method, and use this set as a signature for the method; this way, there would be enough space to specify the input arguments for every block we might want to jump to, each block ignoring the exceeding unused parameters. + +Unfortunately, all the children methods must have the very same signature, as they are all called from the same calling site in the dispatch block of the main method. Since the union of the set of input arguments (and hence the computed signature) varies from method to method, this solution cannot work. + +We might think to determine the signature by computing the union of input arguments of all blocks in the graph; this way, all the children methods would have the same signature. But as we said above, the graph grows new blocks at runtime, so we cannot determine in advance which set of input arguments we will need. + +To solve the problem we need a way to pass a variable number of arguments without knowing in advance neither their number nor their types. Thus, we use an instance of this class: + +public class InputArgs { +public int[] ints; +public float[] floats; +public object[] objs; +... +} + +Since the fields are arrays, they can grow as needed to contain any number of arguments; arguments whose type is primitive are stored in the ints or floats array, depending on their type; arguments whose type is a reference type are stored in the objs array: it's up to each block to cast each argument back to the needed type. + +This solution impose a huge overhead on both writing and reading arguments: + + * when writing, we need to make sure that the arrays are big enough to contains all the arguments we need; if not, we need to allocate a bigger array. Moreover, for each argument we store into the array the virtual machine performs a bound-check, even if we know the index will never be out of bounds (because we checked the size of the array in advance); + * when reading, the same bound-check is performed for each argument read; moreover, for each value read from the objs array we need to insert a downcast. + +To mitigate the performance drop, we avoid to allocate a new InputArgs object each time we do a non-local jump; instead, we preallocate one at the beginning of the main method, and reuse it all the time. + +Our benchmarks show that passing arguments in arrays is about 10 times slower than passing them as real parameter of a method. Unfortunately, we couldn't come up with anything better. +Implement flexswitches + +Now, we can exploit all this machinery to implement flexswitches, as this is our ultimate goal. As described above, the point is to be able to add new cases at runtime, each case represented as a delegate. Here is an excerpt of the C# class that implements a flexswitch that switches over an integer value: + +public class IntLowLevelFlexSwitch: +{ +public uint default_blockid = 0xFFFFFFFF; +public int numcases = 0; +public int[] values = new int[4]; +public FlexSwitchCase[] cases = new FlexSwitchCase[4]; + +public void add_case(int value, FlexSwitchCase c) +{ +... +} + +public uint execute(int value, InputArgs args) +{ +for(int i=0; i Author: arigo Date: Thu Dec 11 19:15:25 2008 New Revision: 60432 Modified: pypy/branch/oo-jit/pypy/rpython/rpbc.py pypy/branch/oo-jit/pypy/rpython/test/test_remptydict.py Log: Corner case. Modified: pypy/branch/oo-jit/pypy/rpython/rpbc.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/rpbc.py (original) +++ pypy/branch/oo-jit/pypy/rpython/rpbc.py Thu Dec 11 19:15:25 2008 @@ -621,6 +621,13 @@ return pair(r_from.r_im_self, r_to.r_im_self).convert_from_to(v, llops) # __ None ____________________________________________________ +def null_hash(x): + assert x is None + return 0 +def null_eq(x, y): + assert x is y is None + return True + class NoneFrozenPBCRepr(Repr): lowleveltype = Void @@ -630,6 +637,12 @@ def none_call(self, hop): raise TyperError("attempt to call constant None") + def get_ll_hash_function(self): + return null_hash + + def get_ll_eq_function(self): + return null_eq + rtype_simple_call = none_call rtype_call_args = none_call Modified: pypy/branch/oo-jit/pypy/rpython/test/test_remptydict.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/test/test_remptydict.py (original) +++ pypy/branch/oo-jit/pypy/rpython/test/test_remptydict.py Thu Dec 11 19:15:25 2008 @@ -43,6 +43,15 @@ res = self.interpret(func, []) assert res is False + def test_none_key_dict(self): + def g(x): + return x + 17 + def func(x): + d = {None: g} + return d[None](x) + res = self.interpret(func, [25]) + assert res == 42 + class TestLLtype(BaseTestRemptydict, LLRtypeMixin): pass From pypy-svn at codespeak.net Fri Dec 12 10:19:23 2008 From: pypy-svn at codespeak.net (VIAGRA INC) Date: Fri, 12 Dec 2008 10:19:23 +0100 (CET) Subject: [pypy-svn] SALE 89% OFF Message-ID: <20081212111639.6697.qmail@host226-116-static.37-88-b.business.telecomitalia.it> An HTML attachment was scrubbed... URL: From davide at codespeak.net Fri Dec 12 11:36:04 2008 From: davide at codespeak.net (davide at codespeak.net) Date: Fri, 12 Dec 2008 11:36:04 +0100 (CET) Subject: [pypy-svn] r60446 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081212103604.3955F16841C@codespeak.net> Author: davide Date: Fri Dec 12 11:36:02 2008 New Revision: 60446 Added: pypy/extradoc/talk/ecoop2009/flexswitch1.png (contents, props changed) pypy/extradoc/talk/ecoop2009/flexswitch2.png (contents, props changed) Log: Pictures taken from Antonio's post Added: pypy/extradoc/talk/ecoop2009/flexswitch1.png ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/ecoop2009/flexswitch2.png ============================================================================== Binary file. No diff available. From pedronis at codespeak.net Fri Dec 12 13:16:52 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 12 Dec 2008 13:16:52 +0100 (CET) Subject: [pypy-svn] r60447 - in pypy/build/testrunner: . test Message-ID: <20081212121652.8F0B316846A@codespeak.net> Author: pedronis Date: Fri Dec 12 13:16:50 2008 New Revision: 60447 Modified: pypy/build/testrunner/runner.py pypy/build/testrunner/test/test_runner.py Log: - support a cleanup hook after each testdir - hook for choosing a test_driver per testdir Modified: pypy/build/testrunner/runner.py ============================================================================== --- pypy/build/testrunner/runner.py (original) +++ pypy/build/testrunner/runner.py Fri Dec 12 13:16:50 2008 @@ -99,10 +99,11 @@ def worker(num, n, run_param, testdirs, result_queue): sessdir = run_param.sessdir root = run_param.root - test_driver = run_param.test_driver + get_test_driver = run_param.get_test_driver interp = run_param.interp dry_run = run_param.dry_run timeout = run_param.timeout + cleanup = run_param.cleanup # xxx cfg thread start while 1: try: @@ -116,11 +117,12 @@ one_output = sessdir.join("%d-%s-output" % (num, basename)) num += n + test_driver = get_test_driver(test) exitcode = execute_test(root, test, one_output, logfname, interp, test_driver, do_dry_run=dry_run, timeout=timeout) - # xxx cfg cleanup after testdir + cleanup(test) output = one_output.read() if logfname.check(file=1): @@ -204,6 +206,9 @@ self.root = root self.self = self + def get_test_driver(self, testdir): + return self.test_driver + def is_test_py_file(self, p): name = p.basename return name.startswith('test_') and name.endswith('.py') @@ -235,6 +240,8 @@ if p1.check(dir=1, link=0): self.collect_testdirs(testdirs, p1) + def cleanup(self, testdir): + pass def main(args): Modified: pypy/build/testrunner/test/test_runner.py ============================================================================== --- pypy/build/testrunner/test/test_runner.py (original) +++ pypy/build/testrunner/test/test_runner.py Fri Dec 12 13:16:50 2008 @@ -163,7 +163,7 @@ run_param = runner.RunParam(self.one_test_dir) run_param.test_driver = test_driver - run_param.parallel_runs = 3 + run_param.parallel_runs = 3 res = runner.execute_tests(run_param, ['test_normal'], log, out) @@ -217,12 +217,18 @@ log = cStringIO.StringIO() out = cStringIO.StringIO() + cleanedup = [] + def cleanup(testdir): + cleanedup.append(testdir) + run_param = runner.RunParam(self.manydir) run_param.test_driver = test_driver run_param.parallel_runs = 3 + run_param.cleanup = cleanup testdirs = [] run_param.collect_testdirs(testdirs) + alltestdirs = testdirs[:] res = runner.execute_tests(run_param, testdirs, log, out) @@ -243,6 +249,8 @@ assert noutcomes == 3*107 assert nfailures == 3*6 + assert set(cleanedup) == set(alltestdirs) + def test_timeout(self): test_driver = [py.path.local(py.__file__).dirpath('bin', 'py.test')] From fijal at codespeak.net Fri Dec 12 14:37:51 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 12 Dec 2008 14:37:51 +0100 (CET) Subject: [pypy-svn] r60451 - pypy/branch/oo-jit/pypy/rpython Message-ID: <20081212133751.505EC1684E0@codespeak.net> Author: fijal Date: Fri Dec 12 14:37:50 2008 New Revision: 60451 Modified: pypy/branch/oo-jit/pypy/rpython/llinterp.py Log: catch KeyError here, don't know where it comes from though, revert please Modified: pypy/branch/oo-jit/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/llinterp.py (original) +++ pypy/branch/oo-jit/pypy/rpython/llinterp.py Fri Dec 12 14:37:50 2008 @@ -164,7 +164,10 @@ self.mallocs[ptr._obj] = llframe def remember_free(self, ptr): - del self.mallocs[ptr._obj] + try: + del self.mallocs[ptr._obj] + except KeyError: + pass def checkptr(ptr): assert isinstance(lltype.typeOf(ptr), lltype.Ptr) From fijal at codespeak.net Fri Dec 12 15:40:10 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 12 Dec 2008 15:40:10 +0100 (CET) Subject: [pypy-svn] r60454 - pypy/branch/oo-jit/pypy/rpython Message-ID: <20081212144010.88B66168491@codespeak.net> Author: fijal Date: Fri Dec 12 15:40:10 2008 New Revision: 60454 Modified: pypy/branch/oo-jit/pypy/rpython/llinterp.py Log: revert previous checkin Modified: pypy/branch/oo-jit/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/llinterp.py (original) +++ pypy/branch/oo-jit/pypy/rpython/llinterp.py Fri Dec 12 15:40:10 2008 @@ -164,10 +164,7 @@ self.mallocs[ptr._obj] = llframe def remember_free(self, ptr): - try: - del self.mallocs[ptr._obj] - except KeyError: - pass + del self.mallocs[ptr._obj] def checkptr(ptr): assert isinstance(lltype.typeOf(ptr), lltype.Ptr) From arigo at codespeak.net Fri Dec 12 15:46:23 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 12 Dec 2008 15:46:23 +0100 (CET) Subject: [pypy-svn] r60457 - pypy/branch/oo-jit/pypy/annotation Message-ID: <20081212144623.272DE168491@codespeak.net> Author: arigo Date: Fri Dec 12 15:46:22 2008 New Revision: 60457 Modified: pypy/branch/oo-jit/pypy/annotation/builtin.py Log: 'enumerate()' is not RPython so far. Modified: pypy/branch/oo-jit/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/oo-jit/pypy/annotation/builtin.py (original) +++ pypy/branch/oo-jit/pypy/annotation/builtin.py Fri Dec 12 15:46:22 2008 @@ -248,6 +248,9 @@ s = SomeInteger(nonneg=True, knowntype=s.knowntype) return s +def builtin_enumerate(*stuff): + raise Exception("'enumerate()' is not RPython so far") + def builtin_apply(*stuff): getbookkeeper().warning("ignoring apply%r" % (stuff,)) return SomeObject() From antocuni at codespeak.net Fri Dec 12 17:48:14 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 12 Dec 2008 17:48:14 +0100 (CET) Subject: [pypy-svn] r60461 - in pypy/branch/oo-jit/pypy/jit: rainbow/test tl tl/test Message-ID: <20081212164814.EDEB816847A@codespeak.net> Author: antocuni Date: Fri Dec 12 17:48:14 2008 New Revision: 60461 Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py Log: make ConstantPool able to contain more than just lists of string. E.g., it now can contain lists of ClassDescr, which represents info about classes, including methods (not implemented yet) Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py Fri Dec 12 17:48:14 2008 @@ -14,16 +14,55 @@ small = False def _get_interp(self): - def decode_strlist(s): - lists = s.split('|') - return [lst.split(',') for lst in lists] + def decode_descr(encdescr): + items = encdescr.split(',') + attributes = [] + methods = [] + for item in items: + if '=' in item: + methname, pc = item.split('=') + methods.append((methname, int(pc))) + else: + attributes.append(item) + return tlc.ClassDescr(attributes, methods) + + def encode_descr(descr): + parts = [] + parts += descr.attributes + parts += ['%s=%s' % item for item in descr.methods] + return ','.join(parts) - def interp(llbytecode, pc, inputarg, llstrlists): + def decode_pool(encpool): + """ + encpool is encoded in this way: + + attr1,attr2,foo=3|attr1,bar=5|... + attr1,attr2,foo,bar,hello,world,... + """ + if encpool == '': + return None + lines = encpool.split('\n') + assert len(lines) == 2 + encdescrs = lines[0].split('|') + classdescrs = [decode_descr(enc) for enc in encdescrs] + strings = lines[1].split(',') + pool = tlc.ConstantPool() + pool.classdescrs = classdescrs + pool.strings = strings + return pool + + def encode_pool(pool): + if pool is None: + return '' + encdescrs = '|'.join([encode_descr(descr) for descr in pool.classdescrs]) + encstrings = ','.join(pool.strings) + return '%s\n%s' % (encdescrs, encstrings) + + def interp(llbytecode, pc, inputarg, llencpool): from pypy.rpython.annlowlevel import hlstr bytecode = hlstr(llbytecode) - strlists = hlstr(llstrlists) - pool = tlc.ConstantPool() - pool.strlists = decode_strlist(strlists) + encpool = hlstr(llencpool) + pool = decode_pool(encpool) obj = tlc.interp_eval_without_call(bytecode, pc, tlc.IntObj(inputarg), @@ -34,14 +73,11 @@ def build_bytecode(s): result = ''.join([chr(int(t)) for t in s.split(',')]) return to_rstr(result) - def build_strlist(items): - lists = [','.join(lst) for lst in items] - return to_rstr('|'.join(lists)) - interp.convert_arguments = [build_bytecode, int, int, build_strlist] - + def build_pool(pool): + return to_rstr(encode_pool(pool)) + interp.convert_arguments = [build_bytecode, int, int, build_pool] return interp - def test_factorial(self): code = tlc.compile(FACTORIAL_SOURCE) bytecode = ','.join([str(ord(c)) for c in code]) @@ -52,7 +88,7 @@ interp = self._get_interp() res = self.timeshift_from_portal(interp, tlc.interp_eval_without_call, - [bytecode, 0, n, []], + [bytecode, 0, n, None], policy=P_OOPSPEC)#, backendoptimize=True) assert res == expected self.check_insns(malloc=1) @@ -75,7 +111,7 @@ interp = self._get_interp() res = self.timeshift_from_portal(interp, tlc.interp_eval_without_call, - [bytecode, 0, 1, []], + [bytecode, 0, 1, None], policy=P_OOPSPEC)#, backendoptimize=True) assert res == 20 @@ -93,13 +129,12 @@ interp = self._get_interp() res = self.timeshift_from_portal(interp, tlc.interp_eval_without_call, - [bytecode, 0, 0, pool.strlists], + [bytecode, 0, 0, pool], policy=P_OOPSPEC) assert res == 42 self.check_insns(malloc=1, direct_call=0) - class TestLLType(BaseTestTLC): type_system = "lltype" to_rstr = staticmethod(LLSupport.to_rstr) Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py Fri Dec 12 17:48:14 2008 @@ -1,16 +1,22 @@ import py -from pypy.jit.tl.tlopcode import compile, NEW +from pypy.jit.tl.tlopcode import compile, NEW, RETURN from pypy.jit.tl.test import test_tl from pypy.jit.tl.tlc import ConstantPool def test_constant_pool(): pool = ConstantPool() bytecode = compile(""" - NEW foo,bar + NEW foo,bar,meth=f + f: + RETURN """, pool) - expected = test_tl.list2bytecode([NEW, 0]) + expected = test_tl.list2bytecode([NEW, 0, RETURN]) assert expected == bytecode - assert pool.strlists == [['foo', 'bar']] + assert len(pool.classdescrs) == 1 + descr = pool.classdescrs[0] + assert descr.attributes == ['foo', 'bar'] + assert descr.methods == [('meth', 2)] + class TestTLC(test_tl.TestTL): @staticmethod @@ -139,7 +145,7 @@ NEW foo,bar PICK 0 PUSH 42 - SETATTR foo, + SETATTR foo """, pool) obj = interp_eval(bytecode, 0, None, pool) assert obj.values[0].int_o() == 42 @@ -152,8 +158,8 @@ NEW foo,bar PICK 0 PUSH 42 - SETATTR bar, - GETATTR bar, + SETATTR bar + GETATTR bar """, pool) res = interp_eval(bytecode, 0, nil, pool) assert res.int_o() == 42 Modified: pypy/branch/oo-jit/pypy/jit/tl/tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlc.py Fri Dec 12 17:48:14 2008 @@ -31,16 +31,32 @@ def setattr(self, name, value): raise TypeError +class ClassDescr(object): + + def __init__(self, attributes, methods): + self.attributes = attributes + self.methods = methods + class ConstantPool(object): def __init__(self): - self.strlists = [] + self.classdescrs = [] + self.strings = [] - def add_strlist(self, items): - idx = len(self.strlists) - self.strlists.append(items) + def add_classdescr(self, attributes, methods): + idx = len(self.classdescrs) + descr = ClassDescr(attributes, methods) + self.classdescrs.append(descr) return idx + def add_string(self, s): + try: + return self.strings.index(s) + except ValueError: + idx = len(self.strings) + self.strings.append(s) + return idx + class Class(object): classes = [] # [(attributes, cls), ...] @@ -343,15 +359,14 @@ elif opcode == NEW: idx = char2int(code[pc]) pc += 1 - attributes = pool.strlists[idx] - cls = Class.get(attributes) + descr = pool.classdescrs[idx] + cls = Class.get(descr.attributes) stack.append(InstanceObj(cls)) elif opcode == GETATTR: idx = char2int(code[pc]) pc += 1 - attributes = pool.strlists[idx] - name = attributes[0] + name = pool.strings[idx] a = stack.pop() hint(a, promote_class=True) stack.append(a.getattr(name)) @@ -359,8 +374,7 @@ elif opcode == SETATTR: idx = char2int(code[pc]) pc += 1 - attributes = pool.strlists[idx] - name = attributes[0] + name = pool.strings[idx] a, b = stack.pop(), stack.pop() hint(a, promote_class=True) hint(b, promote_class=True) Modified: pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py Fri Dec 12 17:48:14 2008 @@ -54,8 +54,9 @@ def compile(code='', pool=None): bytecode = [] - labels = {} #[key] = pc - label_usage = [] #(name, pc) + labels = {} #[key] = pc + label_usage = [] #(name, pc) + method_usage = [] #[methods] for s in code.split('\n'): for comment in '; # //'.split(): s = s.split(comment, 1)[0] @@ -73,13 +74,26 @@ try: bytecode.append( int(arg) ) except ValueError: - if ',' in arg: - # it's a list of strings + if t[0] == 'NEW': + # it's a class descr items = arg.split(',') - items = map(str.strip, items) - items = [x for x in items if x] + items = [x.strip() for x in items if x] + attributes = [] + methods = [] + for item in items: + if '=' in item: + methname, label = item.split('=') + methods.append((methname, label)) + else: + attributes.append(item) assert pool is not None - idx = pool.add_strlist(items) + idx = pool.add_classdescr(attributes, methods) + method_usage.append(methods) + bytecode.append(idx) + elif t[0] in ('GETATTR', 'SETATTR'): + # it's a string + assert pool is not None + idx = pool.add_string(arg) bytecode.append(idx) else: # it's a label @@ -87,6 +101,10 @@ bytecode.append( 0 ) for label, pc in label_usage: bytecode[pc] = labels[label] - pc - 1 + for methods in method_usage: + for i, (methname, label) in enumerate(methods): + pc = labels[label] + methods[i] = (methname, pc) return ''.join([chr(i & 0xff) for i in bytecode]) def serialize_pool(pool): From antocuni at codespeak.net Fri Dec 12 18:13:14 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 12 Dec 2008 18:13:14 +0100 (CET) Subject: [pypy-svn] r60464 - in pypy/branch/oo-jit/pypy/jit/tl: . test Message-ID: <20081212171314.38EEC168455@codespeak.net> Author: antocuni Date: Fri Dec 12 18:13:13 2008 New Revision: 60464 Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py Log: add support for calling methods on objects (without arguments) Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py Fri Dec 12 18:13:13 2008 @@ -191,3 +191,21 @@ """, pool) res = interp_eval(bytecode, 0, nil, pool) assert res.int_o() == 0 + + def test_method(self): + from pypy.jit.tl.tlc import interp_eval, nil + pool = ConstantPool() + bytecode = compile(""" + NEW foo,meth=meth + PICK 0 + PUSH 42 + SETATTR foo + SEND meth + RETURN + meth: + PUSHARG + GETATTR foo + RETURN + """, pool) + res = interp_eval(bytecode, 0, nil, pool) + assert res.int_o() == 42 Modified: pypy/branch/oo-jit/pypy/jit/tl/tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlc.py Fri Dec 12 18:13:13 2008 @@ -29,6 +29,7 @@ # object oriented features def getattr(self, name): raise TypeError def setattr(self, name, value): raise TypeError + def send(self, name): raise TypeError # return the bytecode position where the method starts class ClassDescr(object): @@ -59,11 +60,12 @@ class Class(object): - classes = [] # [(attributes, cls), ...] + classes = [] # [(descr, cls), ...] def get(key): - for attributes, cls in Class.classes: - if attributes == key: + for descr, cls in Class.classes: + if key.attributes == descr.attributes and\ + key.methods == descr.methods: return cls result = Class(key) Class.classes.append((key, result)) @@ -71,11 +73,14 @@ get._pure_function_ = True get = staticmethod(get) - def __init__(self, attrlist): + def __init__(self, descr): attributes = {} # attrname -> index - for name in attrlist: + for name in descr.attributes: attributes[name] = len(attributes) self.attributes = attributes + self.methods = {} + for methname, pc in descr.methods: + self.methods[methname] = pc class InstanceObj(Obj): @@ -104,6 +109,10 @@ self.values[i] = value return value + def send(self, name): + return self.getclass().methods[name] + + class IntObj(Obj): def __init__(self, value): @@ -360,7 +369,7 @@ idx = char2int(code[pc]) pc += 1 descr = pool.classdescrs[idx] - cls = Class.get(descr.attributes) + cls = Class.get(descr) stack.append(InstanceObj(cls)) elif opcode == GETATTR: @@ -380,6 +389,16 @@ hint(b, promote_class=True) b.setattr(name, a) + elif supports_call and opcode == SEND: + idx = char2int(code[pc]) + pc += 1 + name = pool.strings[idx] + a = stack.pop() + hint(a, promote_class=True) + method_pc = a.send(name) + res = interp_eval(code, method_pc, a, pool2) + stack.append( res ) + else: raise RuntimeError("unknown opcode: " + str(opcode)) Modified: pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py Fri Dec 12 18:13:13 2008 @@ -48,6 +48,7 @@ opcode(28, "NEW") opcode(29, "GETATTR") opcode(30, "SETATTR") +opcode(31, "SEND") del opcode @@ -90,9 +91,8 @@ idx = pool.add_classdescr(attributes, methods) method_usage.append(methods) bytecode.append(idx) - elif t[0] in ('GETATTR', 'SETATTR'): + elif t[0] in ('GETATTR', 'SETATTR', 'SEND'): # it's a string - assert pool is not None idx = pool.add_string(arg) bytecode.append(idx) else: From hruske at codespeak.net Fri Dec 12 21:35:11 2008 From: hruske at codespeak.net (hruske at codespeak.net) Date: Fri, 12 Dec 2008 21:35:11 +0100 (CET) Subject: [pypy-svn] r60468 - pypy/trunk/pypy/lib Message-ID: <20081212203511.E0DE3168481@codespeak.net> Author: hruske Date: Fri Dec 12 21:35:10 2008 New Revision: 60468 Added: pypy/trunk/pypy/lib/_sha256.py Modified: pypy/trunk/pypy/lib/hashlib.py Log: Pure Python SHA256 implementation. Needs a second look in case I forgot any compatibility calls and hashlib tests. Added: pypy/trunk/pypy/lib/_sha256.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/lib/_sha256.py Fri Dec 12 21:35:10 2008 @@ -0,0 +1,236 @@ +import struct + +SHA_BLOCKSIZE = 64 +SHA_DIGESTSIZE = 32 + + +def new_shaobject(): + return { + 'digest': [0]*8, + 'count_lo': 0, + 'count_hi': 0, + 'data': [0]* SHA_BLOCKSIZE, + 'local': 0, + 'digestsize': 0 + } + +ROR = lambda x, y: (((x & 0xffffffff) >> (y & 31)) | (x << (32 - (y & 31)))) & 0xffffffff +Ch = lambda x, y, z: (z ^ (x & (y ^ z))) +Maj = lambda x, y, z: (((x | y) & z) | (x & y)) +S = lambda x, n: ROR(x, n) +R = lambda x, n: (x & 0xffffffff) >> n +Sigma0 = lambda x: (S(x, 2) ^ S(x, 13) ^ S(x, 22)) +Sigma1 = lambda x: (S(x, 6) ^ S(x, 11) ^ S(x, 25)) +Gamma0 = lambda x: (S(x, 7) ^ S(x, 18) ^ R(x, 3)) +Gamma1 = lambda x: (S(x, 17) ^ S(x, 19) ^ R(x, 10)) + +def sha_transform(sha_info): + W = [] + + d = sha_info['data'] + for i in xrange(0,16): + W.append( (d[4*i]<<24) + (d[4*i+1]<<16) + (d[4*i+2]<<8) + d[4*i+3]) + + for i in xrange(16,64): + W.append( (Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]) & 0xffffffff ) + + ss = sha_info['digest'][:] + + def RND(a,b,c,d,e,f,g,h,i,ki): + t0 = h + Sigma1(e) + Ch(e, f, g) + ki + W[i]; + t1 = Sigma0(a) + Maj(a, b, c); + d += t0; + h = t0 + t1; + return d & 0xffffffff, h & 0xffffffff + + ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],0,0x428a2f98); + ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],1,0x71374491); + ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],2,0xb5c0fbcf); + ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],3,0xe9b5dba5); + ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],4,0x3956c25b); + ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],5,0x59f111f1); + ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],6,0x923f82a4); + ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],7,0xab1c5ed5); + ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],8,0xd807aa98); + ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],9,0x12835b01); + ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],10,0x243185be); + ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],11,0x550c7dc3); + ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],12,0x72be5d74); + ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],13,0x80deb1fe); + ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],14,0x9bdc06a7); + ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],15,0xc19bf174); + ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],16,0xe49b69c1); + ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],17,0xefbe4786); + ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],18,0x0fc19dc6); + ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],19,0x240ca1cc); + ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],20,0x2de92c6f); + ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],21,0x4a7484aa); + ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],22,0x5cb0a9dc); + ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],23,0x76f988da); + ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],24,0x983e5152); + ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],25,0xa831c66d); + ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],26,0xb00327c8); + ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],27,0xbf597fc7); + ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],28,0xc6e00bf3); + ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],29,0xd5a79147); + ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],30,0x06ca6351); + ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],31,0x14292967); + ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],32,0x27b70a85); + ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],33,0x2e1b2138); + ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],34,0x4d2c6dfc); + ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],35,0x53380d13); + ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],36,0x650a7354); + ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],37,0x766a0abb); + ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],38,0x81c2c92e); + ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],39,0x92722c85); + ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],40,0xa2bfe8a1); + ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],41,0xa81a664b); + ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],42,0xc24b8b70); + ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],43,0xc76c51a3); + ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],44,0xd192e819); + ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],45,0xd6990624); + ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],46,0xf40e3585); + ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],47,0x106aa070); + ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],48,0x19a4c116); + ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],49,0x1e376c08); + ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],50,0x2748774c); + ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],51,0x34b0bcb5); + ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],52,0x391c0cb3); + ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],53,0x4ed8aa4a); + ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],54,0x5b9cca4f); + ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],55,0x682e6ff3); + ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],56,0x748f82ee); + ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],57,0x78a5636f); + ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],58,0x84c87814); + ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],59,0x8cc70208); + ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],60,0x90befffa); + ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],61,0xa4506ceb); + ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],62,0xbef9a3f7); + ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],63,0xc67178f2); + + dig = [] + for i, x in enumerate(sha_info['digest']): + dig.append( (x + ss[i]) & 0xffffffff ) + sha_info['digest'] = dig + +def sha_init(): + sha_info = new_shaobject() + sha_info['digest'] = [0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19] + sha_info['count_lo'] = 0 + sha_info['count_hi'] = 0 + sha_info['local'] = 0 + sha_info['digestsize'] = 32 + return sha_info + +def sha224_init(sha_info): + sha_info['digest'] = [0xc1059ed8, 0x367cd507, 0x3070dd17, 0xf70e5939, 0xffc00b31, 0x68581511, 0x64f98fa7, 0xbefa4fa4] + sha_info['count_lo'] = 0 + sha_info['count_hi'] = 0 + sha_info['local'] = 0 + sha_info['digestsize'] = 28 + return sha_info + +def sha_update(sha_info, buffer): + count = len(buffer) + buffer_idx = 0 + clo = (sha_info['count_lo'] + (count << 3)) & 0xffffffff + if clo < sha_info['count_lo']: + sha_info['count_hi'] += 1 + sha_info['count_lo'] = clo + + sha_info['count_hi'] += (count >> 29) + + if sha_info['local']: + i = SHA_BLOCKSIZE - sha_info['local'] + if i > count: + i = count + + # copy buffer + for x in enumerate(buffer[buffer_idx:buffer_idx+i]): + sha_info['data'][sha_info['local']+x[0]] = struct.unpack('B', x[1])[0] + + count -= i + buffer_idx += i + + sha_info['local'] += i + if sha_info['local'] == SHA_BLOCKSIZE: + sha_transform(sha_info) + sha_info['local'] = 0 + else: + return + + while count >= SHA_BLOCKSIZE: + # copy buffer + sha_info['data'] = [struct.unpack('B',c)[0] for c in buffer[buffer_idx:buffer_idx + SHA_BLOCKSIZE]] + count -= SHA_BLOCKSIZE + buffer_idx += SHA_BLOCKSIZE + sha_transform(sha_info) + + + # copy buffer + pos = sha_info['local'] + sha_info['data'][pos:pos+count] = [struct.unpack('B',c)[0] for c in buffer[buffer_idx:buffer_idx + count]] + sha_info['local'] = count + +def sha_final(sha_info): + lo_bit_count = sha_info['count_lo'] + hi_bit_count = sha_info['count_hi'] + count = (lo_bit_count >> 3) & 0x3f + sha_info['data'][count] = 0x80; + count += 1 + if count > SHA_BLOCKSIZE - 8: + # zero the bytes in data after the count + sha_info['data'] = sha_info['data'][:count] + ([0] * (SHA_BLOCKSIZE - count)) + sha_transform(sha_info) + # zero bytes in data + sha_info['data'] = [0] * SHA_BLOCKSIZE + else: + sha_info['data'] = sha_info['data'][:count] + ([0] * (SHA_BLOCKSIZE - count)) + + sha_info['data'][56] = (hi_bit_count >> 24) & 0xff + sha_info['data'][57] = (hi_bit_count >> 16) & 0xff + sha_info['data'][58] = (hi_bit_count >> 8) & 0xff + sha_info['data'][59] = (hi_bit_count >> 0) & 0xff + sha_info['data'][60] = (lo_bit_count >> 24) & 0xff + sha_info['data'][61] = (lo_bit_count >> 16) & 0xff + sha_info['data'][62] = (lo_bit_count >> 8) & 0xff + sha_info['data'][63] = (lo_bit_count >> 0) & 0xff + + sha_transform(sha_info) + + dig = [] + for i in sha_info['digest']: + dig.extend([ ((i>>24) & 0xff), ((i>>16) & 0xff), ((i>>8) & 0xff), (i & 0xff) ]) + return ''.join([chr(i) for i in dig]) + +class sha256: + + def __init__(self, s=None): + self._sha = sha_init() + if s: + sha_update(self._sha, s) + + def update(self, s): + sha_update(self._sha, s) + + def digest(self): + return sha_final(self._sha.copy()) + + def hexdigest(self): + return ''.join(['%.2x' % ord(i) for i in self.digest()]) + +def test(): + a_str = "just a test string" + + assert 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' == sha256().hexdigest() + assert 'd7b553c6f09ac85d142415f857c5310f3bbbe7cdd787cce4b985acedd585266f' == sha256(a_str).hexdigest() + assert '8113ebf33c97daa9998762aacafe750c7cefc2b2f173c90c59663a57fe626f21' == sha256(a_str*7).hexdigest() + + s = sha256(a_str) + s.update(a_str) + assert '03d9963e05a094593190b6fc794cb1a3e1ac7d7883f0b5855268afeccc70d461' == s.hexdigest() + +if __name__ == "__main__": + test() + + Modified: pypy/trunk/pypy/lib/hashlib.py ============================================================================== --- pypy/trunk/pypy/lib/hashlib.py (original) +++ pypy/trunk/pypy/lib/hashlib.py Fri Dec 12 21:35:10 2008 @@ -60,6 +60,9 @@ elif name in ('MD5', 'md5'): import md5 return md5.new + elif name in ('SHA256', 'sha256'): + import _sha256 + return _sha256.sha256 raise ValueError, "unsupported hash type" def __hash_new(name, string=''): From antocuni at codespeak.net Sat Dec 13 00:03:30 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 13 Dec 2008 00:03:30 +0100 (CET) Subject: [pypy-svn] r60485 - in pypy/branch/oo-jit/pypy/jit/tl: . test Message-ID: <20081212230330.68D2D168481@codespeak.net> Author: antocuni Date: Sat Dec 13 00:03:29 2008 New Revision: 60485 Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py Log: add the possibility to pass arguments to methods Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py Sat Dec 13 00:03:29 2008 @@ -200,7 +200,7 @@ PICK 0 PUSH 42 SETATTR foo - SEND meth + SEND meth/0 RETURN meth: PUSHARG @@ -209,3 +209,24 @@ """, pool) res = interp_eval(bytecode, 0, nil, pool) assert res.int_o() == 42 + + def test_method_arg(self): + from pypy.jit.tl.tlc import interp_eval, nil + pool = ConstantPool() + bytecode = compile(""" + NEW foo,meth=meth + PICK 0 + PUSH 40 + SETATTR foo + PUSH 2 + SEND meth/1 + RETURN + meth: + PUSHARG + GETATTR foo + PUSHARGN 1 + ADD + RETURN + """, pool) + res = interp_eval(bytecode, 0, nil, pool) + assert res.int_o() == 42 Modified: pypy/branch/oo-jit/pypy/jit/tl/tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlc.py Sat Dec 13 00:03:29 2008 @@ -217,9 +217,10 @@ if pool is None: pool = ConstantPool() - return interp_eval(code, pc, IntObj(inputarg), pool).int_o() + args = [IntObj(inputarg)] + return interp_eval(code, pc, args, pool).int_o() - def interp_eval(code, pc, inputarg, pool2): + def interp_eval(code, pc, args, pool2): code_len = len(code) stack = [] pool = hint(hint(pool2, concrete=True), deepfreeze=True) @@ -356,14 +357,19 @@ elif supports_call and opcode == CALL: offset = char2int(code[pc]) pc += 1 - res = interp_eval(code, pc + offset, zero, pool2) + res = interp_eval(code, pc + offset, [zero], pool2) stack.append( res ) elif opcode == RETURN: break elif opcode == PUSHARG: - stack.append(inputarg) + stack.append(args[0]) + + elif opcode == PUSHARGN: + idx = char2int(code[pc]) + pc += 1 + stack.append(args[idx]) elif opcode == NEW: idx = char2int(code[pc]) @@ -392,11 +398,16 @@ elif supports_call and opcode == SEND: idx = char2int(code[pc]) pc += 1 + num_args = char2int(code[pc]) + pc += 1 + num_args += 1 # include self name = pool.strings[idx] - a = stack.pop() + meth_args = stack[-num_args:] + del stack[-num_args:] + a = meth_args[0] hint(a, promote_class=True) - method_pc = a.send(name) - res = interp_eval(code, method_pc, a, pool2) + meth_pc = a.send(name) + res = interp_eval(code, meth_pc, meth_args, pool2) stack.append( res ) else: Modified: pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py Sat Dec 13 00:03:29 2008 @@ -49,6 +49,7 @@ opcode(29, "GETATTR") opcode(30, "SETATTR") opcode(31, "SEND") +opcode(32, "PUSHARGN") del opcode @@ -91,10 +92,16 @@ idx = pool.add_classdescr(attributes, methods) method_usage.append(methods) bytecode.append(idx) - elif t[0] in ('GETATTR', 'SETATTR', 'SEND'): + elif t[0] in ('GETATTR', 'SETATTR'): # it's a string idx = pool.add_string(arg) bytecode.append(idx) + elif t[0] == 'SEND': + # 'methodname/num_args' + methname, num_args = arg.split('/') + idx = pool.add_string(methname) + bytecode.append(idx) + bytecode.append(int(num_args)) else: # it's a label label_usage.append( (arg, len(bytecode)) ) From antocuni at codespeak.net Sat Dec 13 10:16:11 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 13 Dec 2008 10:16:11 +0100 (CET) Subject: [pypy-svn] r60486 - in pypy/branch/oo-jit/pypy/jit: rainbow/test tl Message-ID: <20081213091611.5EB4B1684E2@codespeak.net> Author: antocuni Date: Sat Dec 13 10:16:09 2008 New Revision: 60486 Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlc.py Log: make test_0tlc using the 'with call' version of interp_eval, and some rpython fixes needed for it Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py Sat Dec 13 10:16:09 2008 @@ -63,10 +63,10 @@ bytecode = hlstr(llbytecode) encpool = hlstr(llencpool) pool = decode_pool(encpool) - obj = tlc.interp_eval_without_call(bytecode, - pc, - tlc.IntObj(inputarg), - pool) + obj = tlc.interp_eval(bytecode, + pc, + [tlc.IntObj(inputarg)], + pool) return obj.int_o() to_rstr = self.to_rstr @@ -78,18 +78,20 @@ interp.convert_arguments = [build_bytecode, int, int, build_pool] return interp - def test_factorial(self): - code = tlc.compile(FACTORIAL_SOURCE) + def exec_code(self, code, inputarg, pool=None): bytecode = ','.join([str(ord(c)) for c in code]) + interp = self._get_interp() + res = self.timeshift_from_portal(interp, + tlc.interp_eval, + [bytecode, 0, inputarg, pool], + policy=P_OOPSPEC) + return res + def test_factorial(self): + code = tlc.compile(FACTORIAL_SOURCE) n = 5 expected = 120 - - interp = self._get_interp() - res = self.timeshift_from_portal(interp, - tlc.interp_eval_without_call, - [bytecode, 0, n, None], - policy=P_OOPSPEC)#, backendoptimize=True) + res = self.exec_code(code, n) assert res == expected self.check_insns(malloc=1) @@ -106,18 +108,11 @@ PUSHARG DIV """) - bytecode = ','.join([str(ord(c)) for c in code]) - - interp = self._get_interp() - res = self.timeshift_from_portal(interp, - tlc.interp_eval_without_call, - [bytecode, 0, 1, None], - policy=P_OOPSPEC)#, backendoptimize=True) + res = self.exec_code(code, 1) assert res == 20 def test_getattr(self): - from pypy.jit.tl.tlc import interp_eval, nil, ConstantPool - pool = ConstantPool() + pool = tlc.ConstantPool() code = tlc.compile(""" NEW foo,bar PICK 0 @@ -125,12 +120,7 @@ SETATTR bar, GETATTR bar, """, pool) - bytecode = ','.join([str(ord(c)) for c in code]) - interp = self._get_interp() - res = self.timeshift_from_portal(interp, - tlc.interp_eval_without_call, - [bytecode, 0, 0, pool], - policy=P_OOPSPEC) + res = self.exec_code(code, 0, pool) assert res == 42 self.check_insns(malloc=1, direct_call=0) Modified: pypy/branch/oo-jit/pypy/jit/tl/tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlc.py Sat Dec 13 10:16:09 2008 @@ -220,6 +220,14 @@ args = [IntObj(inputarg)] return interp_eval(code, pc, args, pool).int_o() + def call_interp_eval(code2, pc2, args2, pool2): + # recursive calls to portal need to go through an helper + code = hint(code2, promote=True) + pc = hint(pc2, promote=True) + args = hint(args2, promote=True) + return interp_eval(code, pc, args, pool2) + + def interp_eval(code, pc, args, pool2): code_len = len(code) stack = [] @@ -357,7 +365,7 @@ elif supports_call and opcode == CALL: offset = char2int(code[pc]) pc += 1 - res = interp_eval(code, pc + offset, [zero], pool2) + res = call_interp_eval(code, pc + offset, [zero], pool2) stack.append( res ) elif opcode == RETURN: @@ -402,12 +410,17 @@ pc += 1 num_args += 1 # include self name = pool.strings[idx] - meth_args = stack[-num_args:] - del stack[-num_args:] + start = len(stack)-num_args + assert start >= 0 + meth_args = stack[start:] + # we can't use del because vlist.py doesn't support list_method_resize_le + #del stack[start:] + for i in range(num_args): + stack.pop() a = meth_args[0] hint(a, promote_class=True) meth_pc = a.send(name) - res = interp_eval(code, meth_pc, meth_args, pool2) + res = call_interp_eval(code, meth_pc, meth_args, pool2) stack.append( res ) else: From hruske at codespeak.net Sat Dec 13 11:07:21 2008 From: hruske at codespeak.net (hruske at codespeak.net) Date: Sat, 13 Dec 2008 11:07:21 +0100 (CET) Subject: [pypy-svn] r60487 - pypy/trunk/pypy/lib Message-ID: <20081213100721.5214A1684E4@codespeak.net> Author: hruske Date: Sat Dec 13 11:07:19 2008 New Revision: 60487 Modified: pypy/trunk/pypy/lib/_sha256.py pypy/trunk/pypy/lib/hashlib.py Log: Add sha224 support. Modified: pypy/trunk/pypy/lib/_sha256.py ============================================================================== --- pypy/trunk/pypy/lib/_sha256.py (original) +++ pypy/trunk/pypy/lib/_sha256.py Sat Dec 13 11:07:19 2008 @@ -122,7 +122,8 @@ sha_info['digestsize'] = 32 return sha_info -def sha224_init(sha_info): +def sha224_init(): + sha_info = new_shaobject() sha_info['digest'] = [0xc1059ed8, 0x367cd507, 0x3070dd17, 0xf70e5939, 0xffc00b31, 0x68581511, 0x64f98fa7, 0xbefa4fa4] sha_info['count_lo'] = 0 sha_info['count_hi'] = 0 @@ -204,7 +205,6 @@ return ''.join([chr(i) for i in dig]) class sha256: - def __init__(self, s=None): self._sha = sha_init() if s: @@ -214,11 +214,17 @@ sha_update(self._sha, s) def digest(self): - return sha_final(self._sha.copy()) + return sha_final(self._sha.copy())[:self._sha['digestsize']] def hexdigest(self): return ''.join(['%.2x' % ord(i) for i in self.digest()]) +class sha224(sha256): + def __init__(self, s=None): + self._sha = sha224_init() + if s: + sha_update(self._sha, s) + def test(): a_str = "just a test string" Modified: pypy/trunk/pypy/lib/hashlib.py ============================================================================== --- pypy/trunk/pypy/lib/hashlib.py (original) +++ pypy/trunk/pypy/lib/hashlib.py Sat Dec 13 11:07:19 2008 @@ -63,6 +63,9 @@ elif name in ('SHA256', 'sha256'): import _sha256 return _sha256.sha256 + elif name in ('SHA224', 'sha224'): + import _sha256 + return _sha256.sha224 raise ValueError, "unsupported hash type" def __hash_new(name, string=''): From antocuni at codespeak.net Sat Dec 13 14:05:02 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 13 Dec 2008 14:05:02 +0100 (CET) Subject: [pypy-svn] r60488 - in pypy/branch/oo-jit/pypy/jit: rainbow/test tl Message-ID: <20081213130502.E94FC1684E7@codespeak.net> Author: antocuni Date: Sat Dec 13 14:05:00 2008 New Revision: 60488 Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlc.py Log: don't make recursive calls to portal, but use our own stack of frames instead. This makes possible to remove the overhead of dynamic dispatch of methods Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py Sat Dec 13 14:05:00 2008 @@ -124,6 +124,25 @@ assert res == 42 self.check_insns(malloc=1, direct_call=0) + def test_method(self): + pool = tlc.ConstantPool() + code = tlc.compile(""" + NEW foo,meth=meth + PICK 0 + PUSH 40 + SETATTR foo + PUSH 2 + SEND meth/1 + RETURN + meth: + PUSHARG + GETATTR foo + PUSHARGN 1 + ADD + RETURN + """, pool) + res = self.exec_code(code, 0, pool) + assert res == 42 class TestLLType(BaseTestTLC): type_system = "lltype" Modified: pypy/branch/oo-jit/pypy/jit/tl/tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlc.py Sat Dec 13 14:05:00 2008 @@ -198,6 +198,27 @@ t = -(-ord(c) & 0xff) return t +class FrameStack: + def __init__(self): + self.pc_stack = [] + self.args_stack = [] + self.stack_stack = [] + + def push(self, pc, args, stack): + self.pc_stack.append(pc) + self.args_stack.append(args) + self.stack_stack.append(stack) + return self + + def pop(self): + pc = self.pc_stack.pop() + args = self.args_stack.pop() + stack = self.stack_stack.pop() + return pc, args, stack + + def isempty(self): + return len(self.pc_stack) == 0 + def make_interp(supports_call, jitted=True): if jitted: from pypy.rlib.jit import hint @@ -220,18 +241,11 @@ args = [IntObj(inputarg)] return interp_eval(code, pc, args, pool).int_o() - def call_interp_eval(code2, pc2, args2, pool2): - # recursive calls to portal need to go through an helper - code = hint(code2, promote=True) - pc = hint(pc2, promote=True) - args = hint(args2, promote=True) - return interp_eval(code, pc, args, pool2) - - def interp_eval(code, pc, args, pool2): code_len = len(code) - stack = [] pool = hint(hint(pool2, concrete=True), deepfreeze=True) + stack = [] + framestack = FrameStack() while pc < code_len: hint(None, global_merge_point=True) @@ -365,11 +379,18 @@ elif supports_call and opcode == CALL: offset = char2int(code[pc]) pc += 1 - res = call_interp_eval(code, pc + offset, [zero], pool2) - stack.append( res ) + framestack.push(pc, args, stack) + pc = pc + offset + args = [zero] + stack = [] elif opcode == RETURN: - break + if framestack.isempty(): + break + res = stack.pop() + pc2, args, stack = framestack.pop() + pc = hint(pc2, promote=True) + stack.append(res) elif opcode == PUSHARG: stack.append(args[0]) @@ -410,18 +431,16 @@ pc += 1 num_args += 1 # include self name = pool.strings[idx] - start = len(stack)-num_args - assert start >= 0 - meth_args = stack[start:] - # we can't use del because vlist.py doesn't support list_method_resize_le - #del stack[start:] + meth_args = [None] * num_args for i in range(num_args): - stack.pop() + meth_args[-i-1] = stack.pop() a = meth_args[0] hint(a, promote_class=True) - meth_pc = a.send(name) - res = call_interp_eval(code, meth_pc, meth_args, pool2) - stack.append( res ) + meth_pc = hint(a.send(name), promote=True) + framestack.push(pc, args, stack) + pc = meth_pc + args = meth_args + stack = [] else: raise RuntimeError("unknown opcode: " + str(opcode)) From antocuni at codespeak.net Sat Dec 13 17:30:51 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 13 Dec 2008 17:30:51 +0100 (CET) Subject: [pypy-svn] r60489 - in pypy/branch/oo-jit/pypy/jit: rainbow/test tl tl/test Message-ID: <20081213163051.AED241684AA@codespeak.net> Author: antocuni Date: Sat Dec 13 17:30:49 2008 New Revision: 60489 Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py pypy/branch/oo-jit/pypy/jit/tl/factorial.tlc pypy/branch/oo-jit/pypy/jit/tl/targettlc.py pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py Log: use the same (de)serialization code for both targettlc and the tests Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py Sat Dec 13 17:30:49 2008 @@ -7,6 +7,7 @@ from pypy.jit.conftest import Benchmark from pypy.jit.tl import tlc +from pypy.jit.tl.tlopcode import serialize_program, decode_program from pypy.jit.tl.test.test_tl import FACTORIAL_SOURCE @@ -14,90 +15,36 @@ small = False def _get_interp(self): - def decode_descr(encdescr): - items = encdescr.split(',') - attributes = [] - methods = [] - for item in items: - if '=' in item: - methname, pc = item.split('=') - methods.append((methname, int(pc))) - else: - attributes.append(item) - return tlc.ClassDescr(attributes, methods) - - def encode_descr(descr): - parts = [] - parts += descr.attributes - parts += ['%s=%s' % item for item in descr.methods] - return ','.join(parts) - - def decode_pool(encpool): - """ - encpool is encoded in this way: - - attr1,attr2,foo=3|attr1,bar=5|... - attr1,attr2,foo,bar,hello,world,... - """ - if encpool == '': - return None - lines = encpool.split('\n') - assert len(lines) == 2 - encdescrs = lines[0].split('|') - classdescrs = [decode_descr(enc) for enc in encdescrs] - strings = lines[1].split(',') - pool = tlc.ConstantPool() - pool.classdescrs = classdescrs - pool.strings = strings - return pool - - def encode_pool(pool): - if pool is None: - return '' - encdescrs = '|'.join([encode_descr(descr) for descr in pool.classdescrs]) - encstrings = ','.join(pool.strings) - return '%s\n%s' % (encdescrs, encstrings) - def interp(llbytecode, pc, inputarg, llencpool): + def interp(llprogram, inputarg): from pypy.rpython.annlowlevel import hlstr - bytecode = hlstr(llbytecode) - encpool = hlstr(llencpool) - pool = decode_pool(encpool) - obj = tlc.interp_eval(bytecode, - pc, - [tlc.IntObj(inputarg)], - pool) + program = hlstr(llprogram) + bytecode, pool = decode_program(program) + args = [tlc.IntObj(inputarg)] + obj = tlc.interp_eval(bytecode, 0, args, pool) return obj.int_o() - - to_rstr = self.to_rstr - def build_bytecode(s): - result = ''.join([chr(int(t)) for t in s.split(',')]) - return to_rstr(result) - def build_pool(pool): - return to_rstr(encode_pool(pool)) - interp.convert_arguments = [build_bytecode, int, int, build_pool] return interp - def exec_code(self, code, inputarg, pool=None): - bytecode = ','.join([str(ord(c)) for c in code]) + def exec_code(self, src, inputarg): #, pool=None): + pool = tlc.ConstantPool() + bytecode = tlc.compile(src, pool) + program = serialize_program(bytecode, pool) + llprogram = self.to_rstr(program) interp = self._get_interp() res = self.timeshift_from_portal(interp, tlc.interp_eval, - [bytecode, 0, inputarg, pool], + [llprogram, inputarg], policy=P_OOPSPEC) return res def test_factorial(self): - code = tlc.compile(FACTORIAL_SOURCE) - n = 5 - expected = 120 - res = self.exec_code(code, n) - assert res == expected + res = self.exec_code(FACTORIAL_SOURCE, 5) + assert res == 120 self.check_insns(malloc=1) def test_nth_item(self): # get the nth item of a chained list - code = tlc.compile(""" + code = """ NIL PUSH 40 CONS @@ -107,26 +54,24 @@ CONS PUSHARG DIV - """) + """ res = self.exec_code(code, 1) assert res == 20 def test_getattr(self): - pool = tlc.ConstantPool() - code = tlc.compile(""" + code = """ NEW foo,bar PICK 0 PUSH 42 SETATTR bar, GETATTR bar, - """, pool) - res = self.exec_code(code, 0, pool) + """ + res = self.exec_code(code, 0) assert res == 42 self.check_insns(malloc=1, direct_call=0) def test_method(self): - pool = tlc.ConstantPool() - code = tlc.compile(""" + code = """ NEW foo,meth=meth PICK 0 PUSH 40 @@ -140,8 +85,8 @@ PUSHARGN 1 ADD RETURN - """, pool) - res = self.exec_code(code, 0, pool) + """ + res = self.exec_code(code, 0) assert res == 42 class TestLLType(BaseTestTLC): Modified: pypy/branch/oo-jit/pypy/jit/tl/factorial.tlc ============================================================================== Binary files. No diff available. Modified: pypy/branch/oo-jit/pypy/jit/tl/targettlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/targettlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/targettlc.py Sat Dec 13 17:30:49 2008 @@ -45,20 +45,12 @@ return 0 -def decode_poolcode(s): - pool = ConstantPool() - lists = s.split('|') - pool.strlists = [lst.split(',') for lst in lists] - return pool def load_bytecode(filename): from pypy.rlib.streamio import open_file_as_stream + from pypy.jit.tl.tlopcode import decode_program f = open_file_as_stream(filename) - poolcode = f.readline()[:-1] - bytecode = f.readall()[:-1] - f.close() - pool = decode_poolcode(poolcode) - return bytecode, pool + return decode_program(f.readall()) def target(driver, args): return entry_point, None Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py Sat Dec 13 17:30:49 2008 @@ -17,6 +17,19 @@ assert descr.attributes == ['foo', 'bar'] assert descr.methods == [('meth', 2)] +def test_serialization(): + from pypy.jit.tl.tlopcode import serialize_program, decode_program + pool = ConstantPool() + bytecode = compile(""" + NEW foo,bar,meth=f + SETATTR foobar + f: + RETURN + """, pool) + s = serialize_program(bytecode, pool) + bytecode2, pool2 = decode_program(s) + assert bytecode == bytecode2 + assert pool == pool2 class TestTLC(test_tl.TestTL): @staticmethod Modified: pypy/branch/oo-jit/pypy/jit/tl/tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlc.py Sat Dec 13 17:30:49 2008 @@ -38,6 +38,10 @@ self.attributes = attributes self.methods = methods + def __eq__(self, other): + "NOT_RPYTHON" + return self.__dict__ == other.__dict__ + class ConstantPool(object): def __init__(self): @@ -58,6 +62,10 @@ self.strings.append(s) return idx + def __eq__(self, other): + "NOT_RPYTHON" + return self.__dict__ == other.__dict__ + class Class(object): classes = [] # [(descr, cls), ...] @@ -467,5 +475,4 @@ pool = ConstantPool() bytecode = compile(src, pool) - print serialize_pool(pool) - print bytecode + sys.stdout.write(serialize_program(bytecode, pool)) Modified: pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py Sat Dec 13 17:30:49 2008 @@ -114,6 +114,63 @@ methods[i] = (methname, pc) return ''.join([chr(i & 0xff) for i in bytecode]) + +def decode_descr(encdescr): + from pypy.jit.tl.tlc import ClassDescr + items = encdescr.split(',') + attributes = [] + methods = [] + for item in items: + if '=' in item: + methname, pc = item.split('=') + methods.append((methname, int(pc))) + else: + attributes.append(item) + return ClassDescr(attributes, methods) + +def decode_pool(encpool): + """ + encpool is encoded in this way: + + attr1,attr2,foo=3|attr1,bar=5|... + attr1,attr2,foo,bar,hello,world,... + """ + from pypy.jit.tl.tlc import ConstantPool + if encpool == '': + return None + lines = encpool.split('\n') + assert len(lines) == 2 + encdescrs = lines[0].split('|') + classdescrs = [decode_descr(enc) for enc in encdescrs] + strings = lines[1].split(',') + pool = ConstantPool() + pool.classdescrs = classdescrs + pool.strings = strings + return pool + +def serialize_descr(descr): + parts = [] + parts += descr.attributes + parts += ['%s=%s' % item for item in descr.methods] + return ','.join(parts) + def serialize_pool(pool): - lists = [','.join(lst) for lst in pool.strlists] - return '|'.join(lists) + if pool is None: + return '' + encdescrs = '|'.join([serialize_descr(descr) for descr in pool.classdescrs]) + encstrings = ','.join(pool.strings) + return '%s\n%s' % (encdescrs, encstrings) + +def serialize_program(bytecode, pool): + poolcode = serialize_pool(pool) + return '%s\n%s' % (poolcode, bytecode) + +def decode_program(s): + idx1 = s.find('\n') + assert idx1 >= 0 + idx2 = s.find('\n', idx1+1) + assert idx2 >= 0 + poolcode = s[:idx2] + bytecode = s[idx2+1:] # remove the 2nd newline + pool = decode_pool(poolcode) + return bytecode, pool From cfbolz at codespeak.net Sat Dec 13 19:15:26 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 13 Dec 2008 19:15:26 +0100 (CET) Subject: [pypy-svn] r60491 - pypy/branch/unicodedata-compression Message-ID: <20081213181526.406DF16849D@codespeak.net> Author: cfbolz Date: Sat Dec 13 19:15:24 2008 New Revision: 60491 Added: pypy/branch/unicodedata-compression/ - copied from r60490, pypy/trunk/ Log: branch to check in our attempts for compressing the unicode-database From fijal at codespeak.net Sun Dec 14 10:42:22 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 14 Dec 2008 10:42:22 +0100 (CET) Subject: [pypy-svn] r60495 - pypy/trunk/pypy/lib Message-ID: <20081214094222.E20C41684E3@codespeak.net> Author: fijal Date: Sun Dec 14 10:42:21 2008 New Revision: 60495 Modified: pypy/trunk/pypy/lib/_sha256.py Log: detabify Modified: pypy/trunk/pypy/lib/_sha256.py ============================================================================== --- pypy/trunk/pypy/lib/_sha256.py (original) +++ pypy/trunk/pypy/lib/_sha256.py Sun Dec 14 10:42:21 2008 @@ -5,14 +5,14 @@ def new_shaobject(): - return { - 'digest': [0]*8, - 'count_lo': 0, - 'count_hi': 0, - 'data': [0]* SHA_BLOCKSIZE, - 'local': 0, - 'digestsize': 0 - } + return { + 'digest': [0]*8, + 'count_lo': 0, + 'count_hi': 0, + 'data': [0]* SHA_BLOCKSIZE, + 'local': 0, + 'digestsize': 0 + } ROR = lambda x, y: (((x & 0xffffffff) >> (y & 31)) | (x << (32 - (y & 31)))) & 0xffffffff Ch = lambda x, y, z: (z ^ (x & (y ^ z))) @@ -25,218 +25,218 @@ Gamma1 = lambda x: (S(x, 17) ^ S(x, 19) ^ R(x, 10)) def sha_transform(sha_info): - W = [] - - d = sha_info['data'] - for i in xrange(0,16): - W.append( (d[4*i]<<24) + (d[4*i+1]<<16) + (d[4*i+2]<<8) + d[4*i+3]) - - for i in xrange(16,64): - W.append( (Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]) & 0xffffffff ) - - ss = sha_info['digest'][:] - - def RND(a,b,c,d,e,f,g,h,i,ki): - t0 = h + Sigma1(e) + Ch(e, f, g) + ki + W[i]; - t1 = Sigma0(a) + Maj(a, b, c); - d += t0; - h = t0 + t1; - return d & 0xffffffff, h & 0xffffffff - - ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],0,0x428a2f98); - ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],1,0x71374491); - ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],2,0xb5c0fbcf); - ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],3,0xe9b5dba5); - ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],4,0x3956c25b); - ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],5,0x59f111f1); - ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],6,0x923f82a4); - ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],7,0xab1c5ed5); - ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],8,0xd807aa98); - ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],9,0x12835b01); - ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],10,0x243185be); - ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],11,0x550c7dc3); - ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],12,0x72be5d74); - ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],13,0x80deb1fe); - ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],14,0x9bdc06a7); - ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],15,0xc19bf174); - ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],16,0xe49b69c1); - ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],17,0xefbe4786); - ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],18,0x0fc19dc6); - ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],19,0x240ca1cc); - ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],20,0x2de92c6f); - ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],21,0x4a7484aa); - ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],22,0x5cb0a9dc); - ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],23,0x76f988da); - ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],24,0x983e5152); - ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],25,0xa831c66d); - ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],26,0xb00327c8); - ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],27,0xbf597fc7); - ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],28,0xc6e00bf3); - ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],29,0xd5a79147); - ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],30,0x06ca6351); - ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],31,0x14292967); - ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],32,0x27b70a85); - ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],33,0x2e1b2138); - ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],34,0x4d2c6dfc); - ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],35,0x53380d13); - ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],36,0x650a7354); - ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],37,0x766a0abb); - ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],38,0x81c2c92e); - ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],39,0x92722c85); - ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],40,0xa2bfe8a1); - ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],41,0xa81a664b); - ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],42,0xc24b8b70); - ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],43,0xc76c51a3); - ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],44,0xd192e819); - ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],45,0xd6990624); - ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],46,0xf40e3585); - ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],47,0x106aa070); - ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],48,0x19a4c116); - ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],49,0x1e376c08); - ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],50,0x2748774c); - ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],51,0x34b0bcb5); - ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],52,0x391c0cb3); - ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],53,0x4ed8aa4a); - ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],54,0x5b9cca4f); - ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],55,0x682e6ff3); - ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],56,0x748f82ee); - ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],57,0x78a5636f); - ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],58,0x84c87814); - ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],59,0x8cc70208); - ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],60,0x90befffa); - ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],61,0xa4506ceb); - ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],62,0xbef9a3f7); - ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],63,0xc67178f2); - - dig = [] - for i, x in enumerate(sha_info['digest']): - dig.append( (x + ss[i]) & 0xffffffff ) - sha_info['digest'] = dig + W = [] + + d = sha_info['data'] + for i in xrange(0,16): + W.append( (d[4*i]<<24) + (d[4*i+1]<<16) + (d[4*i+2]<<8) + d[4*i+3]) + + for i in xrange(16,64): + W.append( (Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]) & 0xffffffff ) + + ss = sha_info['digest'][:] + + def RND(a,b,c,d,e,f,g,h,i,ki): + t0 = h + Sigma1(e) + Ch(e, f, g) + ki + W[i]; + t1 = Sigma0(a) + Maj(a, b, c); + d += t0; + h = t0 + t1; + return d & 0xffffffff, h & 0xffffffff + + ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],0,0x428a2f98); + ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],1,0x71374491); + ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],2,0xb5c0fbcf); + ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],3,0xe9b5dba5); + ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],4,0x3956c25b); + ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],5,0x59f111f1); + ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],6,0x923f82a4); + ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],7,0xab1c5ed5); + ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],8,0xd807aa98); + ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],9,0x12835b01); + ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],10,0x243185be); + ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],11,0x550c7dc3); + ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],12,0x72be5d74); + ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],13,0x80deb1fe); + ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],14,0x9bdc06a7); + ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],15,0xc19bf174); + ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],16,0xe49b69c1); + ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],17,0xefbe4786); + ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],18,0x0fc19dc6); + ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],19,0x240ca1cc); + ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],20,0x2de92c6f); + ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],21,0x4a7484aa); + ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],22,0x5cb0a9dc); + ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],23,0x76f988da); + ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],24,0x983e5152); + ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],25,0xa831c66d); + ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],26,0xb00327c8); + ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],27,0xbf597fc7); + ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],28,0xc6e00bf3); + ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],29,0xd5a79147); + ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],30,0x06ca6351); + ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],31,0x14292967); + ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],32,0x27b70a85); + ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],33,0x2e1b2138); + ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],34,0x4d2c6dfc); + ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],35,0x53380d13); + ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],36,0x650a7354); + ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],37,0x766a0abb); + ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],38,0x81c2c92e); + ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],39,0x92722c85); + ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],40,0xa2bfe8a1); + ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],41,0xa81a664b); + ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],42,0xc24b8b70); + ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],43,0xc76c51a3); + ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],44,0xd192e819); + ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],45,0xd6990624); + ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],46,0xf40e3585); + ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],47,0x106aa070); + ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],48,0x19a4c116); + ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],49,0x1e376c08); + ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],50,0x2748774c); + ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],51,0x34b0bcb5); + ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],52,0x391c0cb3); + ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],53,0x4ed8aa4a); + ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],54,0x5b9cca4f); + ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],55,0x682e6ff3); + ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],56,0x748f82ee); + ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],57,0x78a5636f); + ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],58,0x84c87814); + ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],59,0x8cc70208); + ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],60,0x90befffa); + ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],61,0xa4506ceb); + ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],62,0xbef9a3f7); + ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],63,0xc67178f2); + + dig = [] + for i, x in enumerate(sha_info['digest']): + dig.append( (x + ss[i]) & 0xffffffff ) + sha_info['digest'] = dig def sha_init(): - sha_info = new_shaobject() - sha_info['digest'] = [0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19] - sha_info['count_lo'] = 0 - sha_info['count_hi'] = 0 - sha_info['local'] = 0 - sha_info['digestsize'] = 32 - return sha_info + sha_info = new_shaobject() + sha_info['digest'] = [0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19] + sha_info['count_lo'] = 0 + sha_info['count_hi'] = 0 + sha_info['local'] = 0 + sha_info['digestsize'] = 32 + return sha_info def sha224_init(): - sha_info = new_shaobject() - sha_info['digest'] = [0xc1059ed8, 0x367cd507, 0x3070dd17, 0xf70e5939, 0xffc00b31, 0x68581511, 0x64f98fa7, 0xbefa4fa4] - sha_info['count_lo'] = 0 - sha_info['count_hi'] = 0 - sha_info['local'] = 0 - sha_info['digestsize'] = 28 - return sha_info + sha_info = new_shaobject() + sha_info['digest'] = [0xc1059ed8, 0x367cd507, 0x3070dd17, 0xf70e5939, 0xffc00b31, 0x68581511, 0x64f98fa7, 0xbefa4fa4] + sha_info['count_lo'] = 0 + sha_info['count_hi'] = 0 + sha_info['local'] = 0 + sha_info['digestsize'] = 28 + return sha_info def sha_update(sha_info, buffer): - count = len(buffer) - buffer_idx = 0 - clo = (sha_info['count_lo'] + (count << 3)) & 0xffffffff - if clo < sha_info['count_lo']: - sha_info['count_hi'] += 1 - sha_info['count_lo'] = clo - - sha_info['count_hi'] += (count >> 29) - - if sha_info['local']: - i = SHA_BLOCKSIZE - sha_info['local'] - if i > count: - i = count - - # copy buffer - for x in enumerate(buffer[buffer_idx:buffer_idx+i]): - sha_info['data'][sha_info['local']+x[0]] = struct.unpack('B', x[1])[0] - - count -= i - buffer_idx += i - - sha_info['local'] += i - if sha_info['local'] == SHA_BLOCKSIZE: - sha_transform(sha_info) - sha_info['local'] = 0 - else: - return - - while count >= SHA_BLOCKSIZE: - # copy buffer - sha_info['data'] = [struct.unpack('B',c)[0] for c in buffer[buffer_idx:buffer_idx + SHA_BLOCKSIZE]] - count -= SHA_BLOCKSIZE - buffer_idx += SHA_BLOCKSIZE - sha_transform(sha_info) - - - # copy buffer - pos = sha_info['local'] - sha_info['data'][pos:pos+count] = [struct.unpack('B',c)[0] for c in buffer[buffer_idx:buffer_idx + count]] - sha_info['local'] = count + count = len(buffer) + buffer_idx = 0 + clo = (sha_info['count_lo'] + (count << 3)) & 0xffffffff + if clo < sha_info['count_lo']: + sha_info['count_hi'] += 1 + sha_info['count_lo'] = clo + + sha_info['count_hi'] += (count >> 29) + + if sha_info['local']: + i = SHA_BLOCKSIZE - sha_info['local'] + if i > count: + i = count + + # copy buffer + for x in enumerate(buffer[buffer_idx:buffer_idx+i]): + sha_info['data'][sha_info['local']+x[0]] = struct.unpack('B', x[1])[0] + + count -= i + buffer_idx += i + + sha_info['local'] += i + if sha_info['local'] == SHA_BLOCKSIZE: + sha_transform(sha_info) + sha_info['local'] = 0 + else: + return + + while count >= SHA_BLOCKSIZE: + # copy buffer + sha_info['data'] = [struct.unpack('B',c)[0] for c in buffer[buffer_idx:buffer_idx + SHA_BLOCKSIZE]] + count -= SHA_BLOCKSIZE + buffer_idx += SHA_BLOCKSIZE + sha_transform(sha_info) + + + # copy buffer + pos = sha_info['local'] + sha_info['data'][pos:pos+count] = [struct.unpack('B',c)[0] for c in buffer[buffer_idx:buffer_idx + count]] + sha_info['local'] = count def sha_final(sha_info): - lo_bit_count = sha_info['count_lo'] - hi_bit_count = sha_info['count_hi'] - count = (lo_bit_count >> 3) & 0x3f - sha_info['data'][count] = 0x80; - count += 1 - if count > SHA_BLOCKSIZE - 8: - # zero the bytes in data after the count - sha_info['data'] = sha_info['data'][:count] + ([0] * (SHA_BLOCKSIZE - count)) - sha_transform(sha_info) - # zero bytes in data - sha_info['data'] = [0] * SHA_BLOCKSIZE - else: - sha_info['data'] = sha_info['data'][:count] + ([0] * (SHA_BLOCKSIZE - count)) - - sha_info['data'][56] = (hi_bit_count >> 24) & 0xff - sha_info['data'][57] = (hi_bit_count >> 16) & 0xff - sha_info['data'][58] = (hi_bit_count >> 8) & 0xff - sha_info['data'][59] = (hi_bit_count >> 0) & 0xff - sha_info['data'][60] = (lo_bit_count >> 24) & 0xff - sha_info['data'][61] = (lo_bit_count >> 16) & 0xff - sha_info['data'][62] = (lo_bit_count >> 8) & 0xff - sha_info['data'][63] = (lo_bit_count >> 0) & 0xff - - sha_transform(sha_info) - - dig = [] - for i in sha_info['digest']: - dig.extend([ ((i>>24) & 0xff), ((i>>16) & 0xff), ((i>>8) & 0xff), (i & 0xff) ]) - return ''.join([chr(i) for i in dig]) + lo_bit_count = sha_info['count_lo'] + hi_bit_count = sha_info['count_hi'] + count = (lo_bit_count >> 3) & 0x3f + sha_info['data'][count] = 0x80; + count += 1 + if count > SHA_BLOCKSIZE - 8: + # zero the bytes in data after the count + sha_info['data'] = sha_info['data'][:count] + ([0] * (SHA_BLOCKSIZE - count)) + sha_transform(sha_info) + # zero bytes in data + sha_info['data'] = [0] * SHA_BLOCKSIZE + else: + sha_info['data'] = sha_info['data'][:count] + ([0] * (SHA_BLOCKSIZE - count)) + + sha_info['data'][56] = (hi_bit_count >> 24) & 0xff + sha_info['data'][57] = (hi_bit_count >> 16) & 0xff + sha_info['data'][58] = (hi_bit_count >> 8) & 0xff + sha_info['data'][59] = (hi_bit_count >> 0) & 0xff + sha_info['data'][60] = (lo_bit_count >> 24) & 0xff + sha_info['data'][61] = (lo_bit_count >> 16) & 0xff + sha_info['data'][62] = (lo_bit_count >> 8) & 0xff + sha_info['data'][63] = (lo_bit_count >> 0) & 0xff + + sha_transform(sha_info) + + dig = [] + for i in sha_info['digest']: + dig.extend([ ((i>>24) & 0xff), ((i>>16) & 0xff), ((i>>8) & 0xff), (i & 0xff) ]) + return ''.join([chr(i) for i in dig]) class sha256: - def __init__(self, s=None): - self._sha = sha_init() - if s: - sha_update(self._sha, s) - - def update(self, s): - sha_update(self._sha, s) - - def digest(self): - return sha_final(self._sha.copy())[:self._sha['digestsize']] - - def hexdigest(self): - return ''.join(['%.2x' % ord(i) for i in self.digest()]) + def __init__(self, s=None): + self._sha = sha_init() + if s: + sha_update(self._sha, s) + + def update(self, s): + sha_update(self._sha, s) + + def digest(self): + return sha_final(self._sha.copy())[:self._sha['digestsize']] + + def hexdigest(self): + return ''.join(['%.2x' % ord(i) for i in self.digest()]) class sha224(sha256): - def __init__(self, s=None): - self._sha = sha224_init() - if s: - sha_update(self._sha, s) + def __init__(self, s=None): + self._sha = sha224_init() + if s: + sha_update(self._sha, s) def test(): - a_str = "just a test string" - - assert 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' == sha256().hexdigest() - assert 'd7b553c6f09ac85d142415f857c5310f3bbbe7cdd787cce4b985acedd585266f' == sha256(a_str).hexdigest() - assert '8113ebf33c97daa9998762aacafe750c7cefc2b2f173c90c59663a57fe626f21' == sha256(a_str*7).hexdigest() - - s = sha256(a_str) - s.update(a_str) - assert '03d9963e05a094593190b6fc794cb1a3e1ac7d7883f0b5855268afeccc70d461' == s.hexdigest() + a_str = "just a test string" + + assert 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' == sha256().hexdigest() + assert 'd7b553c6f09ac85d142415f857c5310f3bbbe7cdd787cce4b985acedd585266f' == sha256(a_str).hexdigest() + assert '8113ebf33c97daa9998762aacafe750c7cefc2b2f173c90c59663a57fe626f21' == sha256(a_str*7).hexdigest() + + s = sha256(a_str) + s.update(a_str) + assert '03d9963e05a094593190b6fc794cb1a3e1ac7d7883f0b5855268afeccc70d461' == s.hexdigest() if __name__ == "__main__": - test() + test() From antocuni at codespeak.net Sun Dec 14 14:11:14 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 14 Dec 2008 14:11:14 +0100 (CET) Subject: [pypy-svn] r60496 - in pypy/branch/oo-jit/pypy/jit/tl: . test Message-ID: <20081214131114.90E071684F5@codespeak.net> Author: antocuni Date: Sun Dec 14 14:11:12 2008 New Revision: 60496 Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlc.py Log: don't crash if we don't return anything from a function/method Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py Sun Dec 14 14:11:12 2008 @@ -243,3 +243,16 @@ """, pool) res = interp_eval(bytecode, 0, nil, pool) assert res.int_o() == 42 + + def test_call_without_return_value(self): + from pypy.jit.tl.tlc import interp_eval, nil + pool = ConstantPool() + bytecode = compile(""" + CALL foo + PUSH 42 + RETURN + foo: + RETURN + """, pool) + res = interp_eval(bytecode, 0, nil, pool) + assert res.int_o() == 42 Modified: pypy/branch/oo-jit/pypy/jit/tl/tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlc.py Sun Dec 14 14:11:12 2008 @@ -395,10 +395,14 @@ elif opcode == RETURN: if framestack.isempty(): break - res = stack.pop() + if stack: + res = stack.pop() + else: + res = None pc2, args, stack = framestack.pop() pc = hint(pc2, promote=True) - stack.append(res) + if res: + stack.append(res) elif opcode == PUSHARG: stack.append(args[0]) From antocuni at codespeak.net Sun Dec 14 14:21:36 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 14 Dec 2008 14:21:36 +0100 (CET) Subject: [pypy-svn] r60497 - in pypy/branch/oo-jit/pypy/jit/tl: . test Message-ID: <20081214132136.47FC01684F4@codespeak.net> Author: antocuni Date: Sun Dec 14 14:21:34 2008 New Revision: 60497 Added: pypy/branch/oo-jit/pypy/jit/tl/binarytree.tlc.src Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py Log: implement a binary search tree in tlc, with tests. This could become a benchmark in the future Added: pypy/branch/oo-jit/pypy/jit/tl/binarytree.tlc.src ============================================================================== --- (empty file) +++ pypy/branch/oo-jit/pypy/jit/tl/binarytree.tlc.src Sun Dec 14 14:21:34 2008 @@ -0,0 +1,138 @@ +main: + CALL newnode + PICK 0 + PUSH 20 + SEND insert/1 + + PICK 0 + PUSH 10 + SEND insert/1 + + PICK 0 + PUSH 15 + SEND insert/1 + + PICK 0 + PUSH 30 + SEND insert/1 + + PICK 0 + PUSHARG + SEND search/1 + + RETURN + + +newnode: + NEW value,left,right,isempty=isempty,insert=insert,search=search + RETURN + +isempty: + PUSHARG + GETATTR value + BR_COND isempty_not + PUSH 1 + RETURN + isempty_not: + PUSH 0 + RETURN + +insert: # (n) + # if self.isempty goto insert_empty + PUSHARG + SEND isempty/0 + BR_COND insert_empty + + # if n == self.value goto insert_found + PUSHARGN 1 + PUSHARG + GETATTR value + EQ + BR_COND insert_found + + # if n < self.value goto insert_left + PUSHARGN 1 + PUSHARG + GETATTR value + LT + BR_COND insert_left + + insert_right: + # self.right.insert(n) + PUSHARG + GETATTR right + PUSHARGN 1 + SEND insert/1 + RETURN + + insert_left: + # self.left.insert(n) + PUSHARG + GETATTR left + PUSHARGN 1 + SEND insert/1 + RETURN + + insert_found: + RETURN + + insert_empty: + # self.value = n + PUSHARG + PUSHARGN 1 + SETATTR value + + # self.left = Node() + PUSHARG + CALL newnode + SETATTR left + + # self.right = Node() + PUSHARG + CALL newnode + SETATTR right + + RETURN + + +search: # (n) + # if self.isempty goto search_empty + PUSHARG + SEND isempty/0 + BR_COND search_empty + + # if n == self.value goto search_found + PUSHARGN 1 + PUSHARG + GETATTR value + EQ + BR_COND search_found + + # if n < self.value goto search_left + PUSHARGN 1 + PUSHARG + GETATTR value + LT + BR_COND search_left + + search_right: + PUSHARG + GETATTR right + PUSHARGN 1 + SEND search/1 + RETURN + + search_left: + PUSHARG + GETATTR left + PUSHARGN 1 + SEND search/1 + RETURN + + search_found: + PUSH 1 + RETURN + + search_empty: + PUSH 0 + RETURN Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py Sun Dec 14 14:21:34 2008 @@ -141,12 +141,12 @@ py.test.raises(TypeError, self.interp, bytecode, 0, 0) def test_new_obj(self): - from pypy.jit.tl.tlc import interp_eval, InstanceObj + from pypy.jit.tl.tlc import interp_eval, InstanceObj, nil pool = ConstantPool() bytecode = compile(""" NEW foo,bar """, pool) - obj = interp_eval(bytecode, 0, None, pool) + obj = interp_eval(bytecode, 0, [nil], pool) assert isinstance(obj, InstanceObj) assert len(obj.values) == 2 assert sorted(obj.cls.attributes.keys()) == ['bar', 'foo'] @@ -160,7 +160,7 @@ PUSH 42 SETATTR foo """, pool) - obj = interp_eval(bytecode, 0, None, pool) + obj = interp_eval(bytecode, 0, [nil], pool) assert obj.values[0].int_o() == 42 assert obj.values[1] is nil @@ -174,7 +174,7 @@ SETATTR bar GETATTR bar """, pool) - res = interp_eval(bytecode, 0, nil, pool) + res = interp_eval(bytecode, 0, [nil], pool) assert res.int_o() == 42 def test_obj_truth(self): @@ -191,7 +191,7 @@ exit: RETURN """, pool) - res = interp_eval(bytecode, 0, nil, pool) + res = interp_eval(bytecode, 0, [nil], pool) assert res.int_o() == 42 def test_obj_equality(self): @@ -202,7 +202,7 @@ NEW foo,bar EQ """, pool) - res = interp_eval(bytecode, 0, nil, pool) + res = interp_eval(bytecode, 0, [nil], pool) assert res.int_o() == 0 def test_method(self): @@ -220,7 +220,7 @@ GETATTR foo RETURN """, pool) - res = interp_eval(bytecode, 0, nil, pool) + res = interp_eval(bytecode, 0, [nil], pool) assert res.int_o() == 42 def test_method_arg(self): @@ -241,7 +241,7 @@ ADD RETURN """, pool) - res = interp_eval(bytecode, 0, nil, pool) + res = interp_eval(bytecode, 0, [nil], pool) assert res.int_o() == 42 def test_call_without_return_value(self): @@ -254,5 +254,24 @@ foo: RETURN """, pool) - res = interp_eval(bytecode, 0, nil, pool) + res = interp_eval(bytecode, 0, [nil], pool) assert res.int_o() == 42 + + def test_binarytree(self): + from pypy.jit.tl.tlc import interp_eval, IntObj + pool = ConstantPool() + path = py.path.local(__file__).join('../../binarytree.tlc.src') + src = path.read() + bytecode = compile(src, pool) + def search(n): + obj = IntObj(n) + res = interp_eval(bytecode, 0, [obj], pool) + return res.int_o() + assert search(20) + assert search(10) + assert search(15) + assert search(30) + assert not search(1) + assert not search(40) + assert not search(12) + assert not search(27) From pedronis at codespeak.net Mon Dec 15 08:48:32 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 15 Dec 2008 08:48:32 +0100 (CET) Subject: [pypy-svn] r60501 - pypy/build/bot2/pypybuildbot Message-ID: <20081215074832.9DEB516845C@codespeak.net> Author: pedronis Date: Mon Dec 15 08:48:30 2008 New Revision: 60501 Modified: pypy/build/bot2/pypybuildbot/summary.py Log: attach classes such that we can do css coloring of failures Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Mon Dec 15 08:48:30 2008 @@ -301,12 +301,22 @@ line = [] for rev, outcome_set in by_rev: letter = outcome_set.get_outcome(failure) + failed = letter not in ('s', '.') if outcome_set.get_longrepr(failure): longrepr_url = self.make_longrepr_url_for(outcome_set, failure) - line.append([" ",html.a(letter, href=longrepr_url)]) + extra = {} + if failed: + extra = {'class': "failSummary failed"} + line.append([" ",html.a(letter, href=longrepr_url, + **extra)]) else: - line.append(" %s" % letter) + if failed: + line.append([" ", + html.span(letter, + class_="failSummary failed")]) + else: + line.append(" %s" % letter) for width, key in zip(colwidths, failure): line.append(" %-*s" % (width, key)) lines.append(line) From pedronis at codespeak.net Mon Dec 15 08:49:47 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 15 Dec 2008 08:49:47 +0100 (CET) Subject: [pypy-svn] r60502 - pypy/build/bot2/codespeak-html Message-ID: <20081215074947.B438216845C@codespeak.net> Author: pedronis Date: Mon Dec 15 08:49:46 2008 New Revision: 60502 Modified: pypy/build/bot2/codespeak-html/buildbot.css Log: some css dazu Modified: pypy/build/bot2/codespeak-html/buildbot.css ============================================================================== --- pypy/build/bot2/codespeak-html/buildbot.css (original) +++ pypy/build/bot2/codespeak-html/buildbot.css Mon Dec 15 08:49:46 2008 @@ -76,3 +76,19 @@ div.footer { font-size: 80%; } + +/* failure summary */ +.failSummary.failed { + color: #E00000; + font-weight: bold; +} + +a.failSummary.failed { + color: #E00000; + font-weight: bold; +} + +a:visited.failSummary.failed { + color: #E0B000; + font-weight: bold; +} From antocuni at codespeak.net Tue Dec 16 10:58:31 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 16 Dec 2008 10:58:31 +0100 (CET) Subject: [pypy-svn] r60505 - in pypy/branch/oo-jit/pypy/jit/rainbow: . test Message-ID: <20081216095831.28C49168420@codespeak.net> Author: antocuni Date: Tue Dec 16 10:58:30 2008 New Revision: 60505 Modified: pypy/branch/oo-jit/pypy/jit/rainbow/codewriter.py pypy/branch/oo-jit/pypy/jit/rainbow/interpreter.py pypy/branch/oo-jit/pypy/jit/rainbow/test/test_promotion.py Log: make isinstance working for objects whose class has been promoted Modified: pypy/branch/oo-jit/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/oo-jit/pypy/jit/rainbow/codewriter.py Tue Dec 16 10:58:30 2008 @@ -1787,6 +1787,10 @@ # serialize_op, since it works just fine raise NotImplementedError assert color == 'red', 'unknown color %s' % color + INSTANCE = op.args[0].concretetype + TYPES = INSTANCE._all_subclasses() + for T in TYPES: + self.register_typedesc_for_type(T) v1, cType = op.args arg1 = self.serialize_oparg('red', v1) index = self.structtypedesc_position(cType.value) @@ -1946,14 +1950,17 @@ self.register_redvar(op.result) + def register_typedesc_for_type(self, T): + descindex = self.structtypedesc_position(T) + desc = self.structtypedescs[descindex] + ooclass = ootype.runtimeClass(T) + self.interpreter.class2typedesc[ooclass] = desc + return desc + def fill_methodcodes(self, INSTANCE, methname, graph2tsgraph): - class2typedesc = self.interpreter.class2typedesc TYPES = INSTANCE._all_subclasses() for T in TYPES: - descindex = self.structtypedesc_position(T) - desc = self.structtypedescs[descindex] - ooclass = ootype.runtimeClass(T) - class2typedesc[ooclass] = desc + desc = self.register_typedesc_for_type(T) if methname in desc.methodcodes: break # we already filled the codes for this type _, meth = T._lookup(methname) Modified: pypy/branch/oo-jit/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/rainbow/interpreter.py Tue Dec 16 10:58:30 2008 @@ -1113,8 +1113,11 @@ typedesc) # else it's a vstruct content = objbox.content - assert isinstance(content, rcontainer.VirtualStruct) - objtypedesc = content.typedesc + if isinstance(content, rcontainer.PartialDataStruct): + objtypedesc = self.typedesc_from_partial_struct(content) + else: + assert isinstance(content, rcontainer.VirtualStruct) + objtypedesc = content.typedesc result = objtypedesc.issubtype(typedesc) return rvalue.ll_fromvalue(self.jitstate, result) @@ -1133,20 +1136,24 @@ if known_class: self.frame.pc = target + def typedesc_from_partial_struct(self, vstruct): + from pypy.rpython.ootypesystem.rclass import CLASSTYPE + classbox = vstruct.op_getfield(self.jitstate, self.class_fielddesc) + assert classbox.is_constant() + gv_meta = classbox.getgenvar(self.jitstate) + meta = gv_meta.revealconst(CLASSTYPE) + cls = meta.class_ # to be removed after the merging of the less-meta-instances branch + typedesc = self.class2typedesc[cls] + return typedesc + @arguments("green_varargs", "red_varargs", "string") def opimpl_const_oosend(self, greenargs, redargs, methname): - from pypy.rpython.ootypesystem.rclass import CLASSTYPE selfbox = redargs[0] assert isinstance(selfbox, rvalue.AbstractPtrRedBox) vstruct = selfbox.content assert vstruct is not None if isinstance(vstruct, rcontainer.PartialDataStruct): - classbox = vstruct.op_getfield(self.jitstate, self.class_fielddesc) - assert classbox.is_constant() - gv_meta = classbox.getgenvar(self.jitstate) - meta = gv_meta.revealconst(CLASSTYPE) - cls = meta.class_ # to be removed after the merging of the less-meta-instances branch - typedesc = self.class2typedesc[cls] + typedesc = self.typedesc_from_partial_struct(vstruct) else: assert isinstance(vstruct, rcontainer.VirtualStruct) typedesc = vstruct.typedesc Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_promotion.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/rainbow/test/test_promotion.py (original) +++ pypy/branch/oo-jit/pypy/jit/rainbow/test/test_promotion.py Tue Dec 16 10:58:30 2008 @@ -531,6 +531,26 @@ res = self.interpret(ll_function, [True], [], policy=StopAtXPolicy(make_obj)) self.check_flexswitches(2) + def test_isinstance_after_promotion(self): + class A: + pass + class B(A): + pass + + def make_obj(flag): + return flag and A() or B() + + def ll_function(flag): + hint(None, global_merge_point=True) + obj = make_obj(flag) + promoted_obj = hint(obj, promote_class=True) + return isinstance(promoted_obj, B) + + res = self.interpret(ll_function, [False], [], policy=StopAtXPolicy(make_obj)) + assert res + self.check_flexswitches(2) + + class TestLLType(BaseTestPromotion): type_system = "lltype" to_rstr = staticmethod(LLSupport.to_rstr) From davide at codespeak.net Tue Dec 16 13:03:37 2008 From: davide at codespeak.net (davide at codespeak.net) Date: Tue, 16 Dec 2008 13:03:37 +0100 (CET) Subject: [pypy-svn] r60508 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081216120337.1945916841E@codespeak.net> Author: davide Date: Tue Dec 16 13:03:36 2008 New Revision: 60508 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex pypy/extradoc/talk/ecoop2009/main.tex Log: Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Tue Dec 16 13:03:36 2008 @@ -19,8 +19,7 @@ extended with new cases; intuitively, its behavior can be described well in terms of flow graphs. Indeed, a flexswitch can be considered as a special flow graph block where links to newly created blocks are -dynamically added whenever new cases are needed. \dacom{in theory links could also be removed, but I - do not know whether this possibility has been considered in Rainbow} +dynamically added whenever new cases are needed. \begin{figure}[h] \begin{center} @@ -41,8 +40,8 @@ \subsection{Implementing flexswitches in CLI} -Implementing flexswitches for backends generating assembly code is -quite straightforward: basically, a new jump has to be inserted in the +Implementing flexswitches for backends generating machine code is +not too complex: basically, a new jump has to be inserted in the existing code to point to the newly generated code fragment. Unfortunately, the CLI VM does not allow modification of code which @@ -54,22 +53,33 @@ solution consists in creating a new method any time a new case has to be added to a flexswitch. \dacom{comment for Antonio: I am not sure this is the best solution. This cannot work for Java where classes are the basic - units} + units. Closures will be available only with Java Dolphin and I do + not know how much efficient will be} +In this way, whereas flow graphs without flexswitches are translated +to a single method, the translation of flow graphs which can dynamically grow because of +flexswitches will be scattered over several methods. +Summarizing, the backend behaves in the following way: +\begin{itemize} +\item Each flow graph is translated in a collection of methods which + can grow dynamically. \dacom{I propose primary/secondary instead of + the overloaded terms main/child} Each collection must contain at least one + method, called \emph{primary}, which is the first to be created. + All other methods, called \emph{secondary}, are added dynamically + whenever a new case is added to a flexswitch. + +\item Each either primary or secondary method corresponds to the + translation of some of the blocks of a single flow graph. +\end{itemize} + +When a new case is added to a flexswitch, new blocks are generated +and translated by the backend in a new single method pointed +by a delegate which is stored in the code implementing the flexswitch, +so that the method can be invoked later. -\dacom{I still have to polish what comes next} +\subsubsection{Non local links} -\newcommand{\commentout}[1]{} \commentout{ -Because of all these constraints we cannot simply map each graph to its own method, since we saw that our graphs can grow after they have already been executed few times. -Hence, we need to distinguish between the two concepts: - - * a graph is the logical unit of code as seen by the JIT compiler: concretely, the CLI JIT backend renders it as one or more methods; - * a method is a collection of basic blocks; each method has the so called parent graph, i.e. the graph its blocks logically belongs to. - -The first method of a graph is called main method (which has nothing to do with the Main static methods found in .exe files); other methods are called children methods. - -When we want to add a new case to the flexswitch, we create a method containing all the new code; then we wrap the method inside a delegate (the .NET equivalent of a function pointer) and pass it to the flexswitch, so that it can later invoke it. The hard bit: non-local links Using this approach, after a while the blocks of our original graph are scattered over a lot of different methods; however, there are no constraints about how these blocks can be linked together, so it happens to have links between blocks which are not in the same method. In the following, we will refer to them as non-local links. @@ -124,6 +134,7 @@ Obviously, the slow dispatching logic is needed only when we want to jump to a non-local block; if the target block happens to reside in the same method as the current one, we can directly jump to it, completely removing the overhead. Moreover, the dispatch blocks are emitted only if needed, i.e. if the parent graph contains at least one flexswitch; graphs without flexswitches are rendered in the obvious way, by making one method per graph. + The slow bit: passing arguments Jumping to the correct block is not enough to follow a link: as we said before, each link carries a set of arguments to be passed from the source to the target block. As usual, passing arguments across local links is easy, as we can just use local variables to hold their values; on the other hand, non-local links make things more complex. @@ -216,3 +227,5 @@ At the moment, the CLI JIT backend is almost complete, and all the hardest problems seems to be solved; the next step is to fix all the remaining bugs and implement some minor feature that it's still missing, then try to apply it to the full Python language and see what is the outcome. } + +% LocalWords: flexswitches backend flexswitch Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Tue Dec 16 13:03:36 2008 @@ -16,6 +16,7 @@ %\renewcommand{\baselinestretch}{.98} \newcommand{\dacom}[1]{{\small [{\bf DA}: #1]}} +\newcommand{\commentout}[1]{} \begin{document} \title{Automatic generation of JIT compilers for dynamic languages From fijal at codespeak.net Tue Dec 16 14:29:28 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 16 Dec 2008 14:29:28 +0100 (CET) Subject: [pypy-svn] r60510 - pypy/trunk/pypy/translator/c/test Message-ID: <20081216132928.03BB7168423@codespeak.net> Author: fijal Date: Tue Dec 16 14:29:27 2008 New Revision: 60510 Modified: pypy/trunk/pypy/translator/c/test/test_lltyped.py Log: This test passes if it's written sanely enough Modified: pypy/trunk/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/trunk/pypy/translator/c/test/test_lltyped.py Tue Dec 16 14:29:27 2008 @@ -630,15 +630,15 @@ fn() def test_prebuilt_nolength_char_array(self): - py.test.skip("fails on the trunk too") for lastchar in ('\x00', 'X'): A = Array(Char, hints={'nolength': True}) - a = malloc(A, 5, immortal=True) + a = malloc(A, 6, immortal=True) a[0] = '8' a[1] = '5' a[2] = '?' a[3] = '!' a[4] = lastchar + a[5] = '\x00' def llf(): s = '' for i in range(5): @@ -657,3 +657,6 @@ y = rffi.cast(TYPE, 0) fn = self.getcompiled(llf) fn() + + + From fijal at codespeak.net Tue Dec 16 15:48:20 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 16 Dec 2008 15:48:20 +0100 (CET) Subject: [pypy-svn] r60511 - in pypy/trunk/pypy: rpython/lltypesystem translator/c translator/c/test Message-ID: <20081216144820.DDEC4168451@codespeak.net> Author: fijal Date: Tue Dec 16 15:48:19 2008 New Revision: 60511 Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py pypy/trunk/pypy/translator/c/node.py pypy/trunk/pypy/translator/c/test/test_lltyped.py Log: enough hacks to pass those two tests. Really obscure and unsupported :( Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Tue Dec 16 15:48:19 2008 @@ -386,6 +386,19 @@ def setitem(self, index, value): self._storage._setitem(index, value, boundscheck=False) + def getitems(self): + _items = [] + i = 0 + while 1: + nextitem = self.getitem(i) + if nextitem == '\x00': + _items.append('\x00') + return _items + _items.append(nextitem) + i += 1 + + items = property(getitems) + # ____________________________________________________________ def _find_parent(llobj): Modified: pypy/trunk/pypy/translator/c/node.py ============================================================================== --- pypy/trunk/pypy/translator/c/node.py (original) +++ pypy/trunk/pypy/translator/c/node.py Tue Dec 16 15:48:19 2008 @@ -595,7 +595,10 @@ yield '\t%s' % length.rstrip(', ') yield '}' elif self.T.OF == Char: - s = ''.join(self.obj.items) + if len(self.obj.items) and self.obj.items[0] is None: + s = ''.join([self.obj.getitem(i) for i in range(len(self.obj.items))]) + else: + s = ''.join(self.obj.items) yield '\t%s%s' % (length, c_char_array_constant(s)) yield '}' else: Modified: pypy/trunk/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/trunk/pypy/translator/c/test/test_lltyped.py Tue Dec 16 15:48:19 2008 @@ -650,6 +650,49 @@ fn = self.getcompiled(llf) fn() + def test_prebuilt_ll2ctypes_array(self): + from pypy.rpython.lltypesystem import rffi, ll2ctypes + A = rffi.CArray(Char) + a = malloc(A, 6, flavor='raw') + a[0] = 'a' + a[1] = 'b' + a[2] = 'c' + a[3] = 'd' + a[4] = '\x00' + a[5] = '\x00' + # side effects when converting to c structure + ll2ctypes.lltype2ctypes(a) + def llf(): + s = '' + for i in range(4): + s += a[i] + return 'abcd' == s + + fn = self.getcompiled(llf) + assert fn() + + def test_ll2ctypes_array_from_c(self): + from pypy.rpython.lltypesystem import rffi, ll2ctypes + A = rffi.CArray(Char) + a = malloc(A, 6, flavor='raw') + a[0] = 'a' + a[1] = 'b' + a[2] = 'c' + a[3] = 'd' + a[4] = '\x00' + a[5] = '\x00' + # side effects when converting to c structure + c = ll2ctypes.lltype2ctypes(a) + a = ll2ctypes.ctypes2lltype(Ptr(A), c) + def llf(): + s = '' + for i in range(4): + s += a[i] + print s + return s == 'abcd' + fn = self.getcompiled(llf) + assert fn() + def test_cast_to_void_array(self): from pypy.rpython.lltypesystem import rffi def llf(): From antocuni at codespeak.net Tue Dec 16 17:47:03 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 16 Dec 2008 17:47:03 +0100 (CET) Subject: [pypy-svn] r60515 - pypy/branch/oo-jit/pypy/jit/tl Message-ID: <20081216164703.446BB16845A@codespeak.net> Author: antocuni Date: Tue Dec 16 17:47:02 2008 New Revision: 60515 Modified: pypy/branch/oo-jit/pypy/jit/tl/tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py Log: two new opcodes, useful for debugging Modified: pypy/branch/oo-jit/pypy/jit/tl/tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlc.py Tue Dec 16 17:47:02 2008 @@ -2,7 +2,7 @@ import autopath import py -from pypy.rlib.objectmodel import specialize +from pypy.rlib.objectmodel import specialize, we_are_translated from pypy.jit.tl.tlopcode import * from pypy.jit.tl import tlopcode from pypy.rlib.jit import hint @@ -12,6 +12,7 @@ def t(self): raise TypeError def int_o(self): raise TypeError + def to_string(self): raise TypeError def add(self, other): raise TypeError def sub(self, other): raise TypeError @@ -102,6 +103,9 @@ cls = hint(self.cls, promote=True) return hint(cls, deepfreeze=True) + def to_string(self): + return '' + def t(self): return True @@ -132,6 +136,9 @@ def int_o(self): return self.value + def to_string(self): + return str(self.value) + def add(self, other): return IntObj(self.value + other.int_o()) def sub(self, other): return IntObj(self.value - other.int_o()) def mul(self, other): return IntObj(self.value * other.int_o()) @@ -159,6 +166,9 @@ class NilObj(LispObj): + def to_string(self): + return 'nil' + def t(self): return False @@ -178,6 +188,9 @@ self._car = car self._cdr = cdr + def to_string(self): + return '' + def t(self): return True @@ -454,6 +467,19 @@ args = meth_args stack = [] + elif opcode == PRINT: + if not we_are_translated(): + a = stack.pop() + hint(a, promote_class=True) + + elif opcode == DUMP: + if not we_are_translated(): + parts = [] + for obj in stack: + hint(obj, promote_class=True) + parts.append(obj.to_string()) + print '[%s]' % ', '.join(parts) + else: raise RuntimeError("unknown opcode: " + str(opcode)) Modified: pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py Tue Dec 16 17:47:02 2008 @@ -51,6 +51,9 @@ opcode(31, "SEND") opcode(32, "PUSHARGN") +opcode(33, "PRINT") +opcode(34, "DUMP") + del opcode From antocuni at codespeak.net Tue Dec 16 17:52:34 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 16 Dec 2008 17:52:34 +0100 (CET) Subject: [pypy-svn] r60517 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081216165234.C1216168426@codespeak.net> Author: antocuni Date: Tue Dec 16 17:52:32 2008 New Revision: 60517 Added: pypy/extradoc/talk/ecoop2009/benchmarks.tex Modified: pypy/extradoc/talk/ecoop2009/main.tex Log: start the benchmark section Added: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Tue Dec 16 17:52:32 2008 @@ -0,0 +1,76 @@ +\section{The TLC language} + +\anto{maybe we should move this section somewhere else, if we want to use TLC + as a running example in other sections} + +In this section, we will briefly describe \emph{TLC}, a simple dynamic +language that we developed to exercise our JIT compiler generator. As most of +dynamic languages around, \emph{TLC} is implemented through a virtual machine +that interprets a custom bytecode. Since our main interest is in the runtime +performances of the VM, we did not implement the parser nor the bytecode +compiler, but only the VM itself. + +TLC provides four different types: +\begin{enumerate} +\item Integers +\item \lstinline{nil}, whose only value is the null value +\item Objects +\item Lisp-like lists +\end{enumerate} + +Objects represent a collection of named attributes (much like Javascript or +SELF) and named methods. At creation time, it is necessary to specify the set +of attributes of the object, as well as its methods. Once the object has been +created, it is not possible to add/remove attributes and methods. + +The virtual machine is stack-based, and provides several operations: + +\begin{itemize} +\item \textbf{Stack manipulation}: standard operations to manipulate the + stack, such as \lstinline{PUSH}, \lstinline{POP}, \lstinline{SWAP}, etc. +\item \textbf{Flow control} to do conditional and unconditional jumps. +\item \textbf{Arithmetic}: numerical operations on integers, like + \lstinline{ADD}, \lstinline{SUB}, etc. +\item \textbf{Comparisons} like \lstinline{EQ}, \lstinline{LT}, + \lstinline{GT}, etc. +\item \textbf{Object-oriented}: operations on objects: \lstinline{NEW}, + \lstinline{GETATTR}, \lstinline{SETATTR}, \lstinline{SEND}. +\item \textbf{List operations}: \lstinline{CONS}, \lstinline{CAR}, + \lstinline{CDR}. +\end{itemize} + +Obviously, not all the operations are applicable to all objects. For example, +it is not possibile to \lstinline{ADD} an integer and an object, or reading an +attribute from an object which does not provide it. Being a dynamic language, +the VM needs to do all these checks at runtime; in case one of the check +fails, the execution is simply aborted. + +\anto{should we try to invent a syntax for TLC and provide some examples?} + +\section{Benchmarks} + +Despite being very simple and minimalistic, \lstinline{TLC} is a good +candidate as a language to run benchmarks, as it has some of the features that +makes most of current dynamic languages so slow: + +\begin{itemize} + +\item \textbf{Stack based VM}: this kind of VM requires all the operands to be + on top of the evaluation stack. As a consequence programs spend a lot of + time pushing and popping values to/from the stack, or doing other stack + related operations. However, thanks ot its semplicity this is still the + most common and preferred way to implement VMs. + +\item \textbf{Boxed integers}: integer objects are internally represented as + an instance of the \lstinline{IntObj} class, whose field \lstinline{value} + contains the real value. By having boxed integers, common ariithmetic + operations are made very slow, because each time we want to load/store their + value we need to go through an extra level of indirection. Moreover, in + case of a complex expression, it is necessary to create a lot of temporary + objects to hold intermediate results. + +\item \textbf{Dynamic lookup}: attributes and methods are looked up at + runtime, because there is no way to know in advance if and where an object + have that particular attribute or method. +\end{itemize} + Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Tue Dec 16 17:52:32 2008 @@ -16,6 +16,7 @@ %\renewcommand{\baselinestretch}{.98} \newcommand{\dacom}[1]{{\small [{\bf DA}: #1]}} +\newcommand{\anto}[1]{{\small [{\bf ANTO}: #1]}} \newcommand{\commentout}[1]{} \begin{document} @@ -63,6 +64,7 @@ \input{jitgen} \input{rainbow} \input{clibackend} +\input{benchmarks} %\input{conclusion} \bibliographystyle{plain} From cfbolz at codespeak.net Tue Dec 16 18:18:09 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 16 Dec 2008 18:18:09 +0100 (CET) Subject: [pypy-svn] r60518 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081216171809.4795616845A@codespeak.net> Author: cfbolz Date: Tue Dec 16 18:18:09 2008 New Revision: 60518 Modified: pypy/extradoc/talk/ecoop2009/main.tex pypy/extradoc/talk/ecoop2009/rainbow.tex Log: make comments stand out more, visually Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Tue Dec 16 18:18:09 2008 @@ -4,6 +4,7 @@ \usepackage{amsmath} \usepackage[sans]{dsfont} \usepackage{color} +\usepackage{ifthen} \usepackage{xspace} \usepackage{listings} \usepackage[pdftex]{graphicx} @@ -15,8 +16,24 @@ \lstset{mathescape=true,language=Java,basicstyle=\tt,keywordstyle=\bf} %\renewcommand{\baselinestretch}{.98} -\newcommand{\dacom}[1]{{\small [{\bf DA}: #1]}} -\newcommand{\anto}[1]{{\small [{\bf ANTO}: #1]}} +\newboolean{showcomments} +\setboolean{showcomments}{true} +\ifthenelse{\boolean{showcomments}} + {\newcommand{\nb}[2]{ + \fbox{\bfseries\sffamily\scriptsize#1} + {\sf\small$\blacktriangleright$\textit{#2}$\blacktriangleleft$} + } + \newcommand{\version}{\emph{\scriptsize$-$Id: main.tex 19055 2008-06-05 11:20:31Z cfbolz $-$}} + } + {\newcommand{\nb}[2]{} + \newcommand{\version}{} + } + +\newcommand\AK[1]{\nb{akuhn}{#1}} +\newcommand\on[1]{\nb{oscar}{#1}} +\newcommand\dacom[1]{\nb{DA}{#1}} +\newcommand\cfbolz[1]{\nb{CFB}{#1}} +\newcommand\anto[1]{\nb{ANTO}{#1}} \newcommand{\commentout}[1]{} \begin{document} Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Tue Dec 16 18:18:09 2008 @@ -93,4 +93,5 @@ \subsection{Virtuals and virtualizables} +\cfbolz{do we even want to talk about virtualizables?} TODO From antocuni at codespeak.net Tue Dec 16 19:12:06 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 16 Dec 2008 19:12:06 +0100 (CET) Subject: [pypy-svn] r60520 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081216181206.8EB1816849D@codespeak.net> Author: antocuni Date: Tue Dec 16 19:12:06 2008 New Revision: 60520 Modified: pypy/extradoc/talk/ecoop2009/intro.tex Log: add an XXX Modified: pypy/extradoc/talk/ecoop2009/intro.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/intro.tex (original) +++ pypy/extradoc/talk/ecoop2009/intro.tex Tue Dec 16 19:12:06 2008 @@ -23,6 +23,10 @@ \subsection{PyPy and RPython} +\anto{as Armin points out, the two CLI backends can be easily confused; what + about renaming the ``CLI Backend for flowgraphs'' into ``CLI bytecode + compiler''? Any better idea for the name?} + \begin{figure}[h] \begin{center} \includegraphics[width=.6\textwidth]{diagram0} From cfbolz at codespeak.net Tue Dec 16 19:15:42 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 16 Dec 2008 19:15:42 +0100 (CET) Subject: [pypy-svn] r60521 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081216181542.DDD5E16845A@codespeak.net> Author: cfbolz Date: Tue Dec 16 19:15:42 2008 New Revision: 60521 Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex Log: steal some sections from the EU report Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Tue Dec 16 19:15:42 2008 @@ -24,6 +24,10 @@ Assume the Python bytecode to be constant, and constant-propagate it into the Python interpreter. +\cfbolz{note to self: steal bits from the master thesis?} + +\cfbolz{I would propose to use either TLC as an example here, or something that +looks at least like an interpreter loop} Example:: \begin{lstlisting}[language=Python] @@ -50,17 +54,172 @@ compilation requires feedback of runtime information into compile-time; for a dynamic language, types are a primary example. +Partial evaluation (PE) comes in two flavors: -\subsection{Execution steps} +\begin{itemize} +\item \emph{On-line} partial evaluation: a compiler-like algorithm takes the +source code of the function \texttt{f(x, y)} (or its intermediate representation, +i.e. its control flow graph), and some partial +information, e.g. \texttt{x=5}. From this, it produces the residual function +\texttt{g(y)} directly, by following in which operations the knowledge \texttt{x=5} can +be used, which loops can be unrolled, etc. + +\item \emph{Off-line} partial evalution: in many cases, the goal of partial +evaluation is to improve performance in a specific application. Assume that we +have a single known function \texttt{f(x, y)} in which we think that the value +of \texttt{x} will change slowly during the execution of our program ? much +more slowly than the value of \texttt{y}. An obvious example is a loop that +calls \texttt{f(x, y)} many times with always the same value \texttt{x}. We +could then use an on-line partial evaluator to produce a \texttt{g(y)} for each +new value of \texttt{x}. In practice, the overhead of the partial evaluator +might be too large for it to be executed at run-time. However, if we know the +function \texttt{f} in advance, and if we know \emph{which} arguments are the +ones that we will want to partially evaluate \texttt{f} with, then we do not +need a full compiler-like analysis of \texttt{f} every time the value of +\texttt{x} changes. We can precompute once and for all a specialized function +\texttt{f1(x)}, which when called produces the residual function \texttt{g(y)} +corresponding to \texttt{x}. This is \emph{off-line partial evaluation}; the +specialized function \texttt{f1(x)} is called a \emph{generating extension}. +\end{itemize} + +Off-line partial evaluation is usually based on \emph{binding-time analysis}, which +is the process of determining among the variables used in a function (or +a set of functions) which ones are going to be known in advance and +which ones are not. In the example of \texttt{f(x, y)}, such an analysis +would be able to infer that the constantness of the argument \texttt{x} +implies the constantness of many intermediate values used in the +function. The \emph{binding time} of a variable determines how early the +value of the variable will be known. + +\cfbolz{XXX: unclear how the next paragraph will fit into the text in the end. +it's certainly wrong as it is} +Once binding times have been determined, one possible approach to +producing the generating extension itself is by self-applying on-line +partial evaluators. This is known as the second Futamura projection +\cite{Futamura99}. So far it is unclear if this approach can lead to optimal +results, or even if it scales well. In PyPy we selected a more direct +approach: the generating extension is produced by transformation of the +control flow graphs of the interpreter, guided by the binding times. We +call this process \emph{timeshifting}. -* Translation time - - pypy-c-jit is translated into an executable +\subsection{Execution steps} + - - the JIT compiler is automatically generated +PyPy contains a framework for generating just-in-time compilers using +off-line partial evaluation. As such, there are three distinct phases: -* Compile-time: the JIT compiler runs +\begin{itemize} +\item \emph{Translation time:} during the normal translation of an RPython +program, say the TLC interpreter, we perform binding-time analysis on the +interpreter. This produces a generating extension, which is linked with the +rest of the program. \cfbolz{XXX not quite right} + +\item \emph{Compile time:} during the execution of the program, when a new +bytecode is about to be interpreted, the generating extension is invoked +instead. As the generating extension is a compiler, all the computations it +performs are called compile-time computations. Its sole effect is to produce +residual code. + +\item \emph{Run time:} the normal execution of the program (which includes the +time spent running the residual code created by the generating extension). +\end{itemize} + +Translation time is a purely off-line phase; compile time and run time are +actually highly interleaved during the execution of the program. + +\subsection{Binding Time Analysis} + +At translation time, PyPy performs binding-time analysis of the source +RPython program after it has been turned to low-level graphs, i.e. at +the level at which operations manipulate pointer-and-structure-like +objects. + +The binding-time terminology that we are using in PyPy is based on the +colors that we use when displaying the control flow graphs: + +\begin{itemize} +\item \emph{Green} variables contain values that are known at compile-time; +\item \emph{Red} variables contain values that are not known until run-time. +\end{itemize} + +The binding-time analyzer of our translation tool-chain is based on the +same type inference engine that is used on the source RPython program, +the annotator. In this mode, it is called the \emph{hint-annotator}; it +operates over input graphs that are already low-level instead of +RPython-level, and propagates annotations that do not track types but +value dependencies and manually-provided binding time hints. + +The normal process of the hint-annotator is to propagate the binding +time (i.e. color) of the variables using the following kind of rules: + +\begin{itemize} +\item For a foldable operation (i.e. one without side effect and which depends +only on its argument values), if all arguments are green, then the result can +be green too. + +\item Non-foldable operations always produce a red result. + +\item At join points, where multiple possible values (depending on control +flow) are meeting into a fresh variable, if any incoming value comes from a red +variable, the result is red. Otherwise, the color of the result might be +green. We do not make it eagerly green, because of the control flow +dependency: the residual function is basically a constant-folded copy of the +source function, so it might retain some of the same control flow. The value +that needs to be stored in the fresh join variable thus depends on which +branches are taken in the residual graph. +\end{itemize} + +\subsubsection{Hints} + +Our goal in designing our approach to binding-time analysis was to +minimize the number of explicit hints that the user must provide in +the source of the RPython program. This minimalism was not pushed to +extremes, though, to keep the hint-annotator reasonably simple. + +The driving idea was that hints should be need-oriented. Indeed, in a +program like an interpreter, there are a small number of places where it +would be clearly beneficial for a given value to be known at +compile-time, i.e. green: this is where we require the hints to be +added. + +The hint-annotator assumes that all variables are red by default, and +then propagates annotations that record dependency information. +When encountering the user-provided hints, the dependency information +is used to make some variables green. All +hints are in the form of an operation \texttt{hint(v1, someflag=True)} +which semantically just returns its first argument unmodified. + +The crucial need-oriented hint is \texttt{v2 = hint(v1, concrete=True)} +which should be used in places where the programmer considers the +knowledge of the value to be essential. This hint is interpreted by +the hint-annotator as a request for both \texttt{v1} and \texttt{v2} to be green. It +has a \emph{global} effect on the binding times: it means that not only +\texttt{v1} but all the values that \texttt{v1} depends on ? recursively ? +are forced to be green. The hint-annotator complains if the +dependencies of \texttt{v1} include a value that cannot be green, like +a value read out of a field of a non-immutable structure. + +Such a need-oriented backward propagation has advantages over the +commonly used forward propagation, in which a variable is compile-time +if and only if all the variables it depends on are also compile-time. A +known issue with forward propagation is that it may mark as compile-time +either more variables than expected (which leads to over-specialization +of the residual code), or less variables than expected (preventing +specialization to occur where it would be the most useful). Our +need-oriented approach reduces the problem of over-specialization, and +it prevents under-specialization: an unsatisfiable \texttt{hint(v1, +concrete=True)} is reported as an error. + +In our context, though, such an error can be corrected. This is done by +promoting a well-chosen variable among the ones that \texttt{v1} depends on. + +Promotion is invoked with the use of a hint as well: +\texttt{v2 = hint(v1, promote=True)}. +This hint is a \emph{local} request for \texttt{v2} to be green, without +requiring \texttt{v1} to be green. Note that this amounts to copying +a red value into a green one, which is not possible in classical +approaches to partial evaluation. See the Promotion section XXX ref for a +complete discussion of promotion. -* Runtime: the JIT compiled code runs -* Compile-time and runtime are intermixed From cfbolz at codespeak.net Tue Dec 16 19:25:01 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 16 Dec 2008 19:25:01 +0100 (CET) Subject: [pypy-svn] r60522 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081216182501.C78A41684A9@codespeak.net> Author: cfbolz Date: Tue Dec 16 19:24:58 2008 New Revision: 60522 Added: pypy/extradoc/talk/ecoop2009/background.tex Modified: pypy/extradoc/talk/ecoop2009/main.tex Log: I think we need a background section describing PyPy Added: pypy/extradoc/talk/ecoop2009/background.tex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/background.tex Tue Dec 16 19:24:58 2008 @@ -0,0 +1,7 @@ + + +\section{Background} + +\subsection{The PyPy Project} + +\cfbolz{need to describe a bit how PyPy works} Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Tue Dec 16 19:24:58 2008 @@ -78,6 +78,7 @@ \input{abstract} \input{intro} +\input{background} \input{jitgen} \input{rainbow} \input{clibackend} From davide at codespeak.net Tue Dec 16 19:34:12 2008 From: davide at codespeak.net (davide at codespeak.net) Date: Tue, 16 Dec 2008 19:34:12 +0100 (CET) Subject: [pypy-svn] r60523 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081216183412.18C361684A9@codespeak.net> Author: davide Date: Tue Dec 16 19:34:11 2008 New Revision: 60523 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex Log: Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Tue Dec 16 19:34:11 2008 @@ -68,7 +68,10 @@ whenever a new case is added to a flexswitch. \item Each either primary or secondary method corresponds to the - translation of some of the blocks of a single flow graph. + translation of some of the blocks of a single flow graph. Each + method has an initial block whose input variables are the + parameters of the method; the input variables of all other blocks + are local variables of the method. \end{itemize} When a new case is added to a flexswitch, new blocks are generated @@ -76,21 +79,32 @@ by a delegate which is stored in the code implementing the flexswitch, so that the method can be invoked later. -\subsubsection{Non local links} +\subsubsection{Internal and external links} + +A link is called \emph{internal} if it connects two blocks implemented in the same method, + \emph{external} otherwise. + +Following an internal link is not difficult in CLI bytecode: a jump to +the corresponding code fragment in the same method is emitted +to execute the new block, whereas the appropriate local variables are +used for passing arguments. +Also following an external link whose target is the initial block of a +method is not difficult: the corresponding method has to be invoked +with the appropriate arguments. + \commentout{ -The hard bit: non-local links -Using this approach, after a while the blocks of our original graph are scattered over a lot of different methods; however, there are no constraints about how these blocks can be linked together, so it happens to have links between blocks which are not in the same method. In the following, we will refer to them as non-local links. +Using this approach, after a while the blocks of our original graph are scattered over a lot of different methods; however, there are no constraints about how these blocks can be linked together, so it happens to have links between blocks which are not in the same method. In the following, we will refer to them as external links. -If the non-local block we want to jump to happens to be at the beginning of its containing method, it is enough to invoke the method; but, what if we want to jump somewhere in the middle? What we really want is to produce a method which has multiple entry-points; again, doing it in assembly would be trivial, but the virtual machine does not provide any support for it, so we need a work around. +If the external block we want to jump to happens to be at the beginning of its containing method, it is enough to invoke the method; but, what if we want to jump somewhere in the middle? What we really want is to produce a method which has multiple entry-points; again, doing it in assembly would be trivial, but the virtual machine does not provide any support for it, so we need a work around. Each method in a graph is assigned an unique 16 bit method id; each block in a method is assigned a progressive 16 bit block number. From this two numbers, we can compute the block id as an unsigned integer, by storing the method id in the first 16 bits and the block number in the second 16 bits. By construction, the block id is guaranteed to be unique in the graph. -The following picture shows a graph composed of three methods; the id of each method is shown in red, while the block ids are shown in red (for the method id part) and black (for the block number part). The graph contains three non-local links; in particular, note the link between blocks 0x00020001 and 0x00010001 which connects two block that resides in different methods. +The following picture shows a graph composed of three methods; the id of each method is shown in red, while the block ids are shown in red (for the method id part) and black (for the block number part). The graph contains three external links; in particular, note the link between blocks 0x00020001 and 0x00010001 which connects two block that resides in different methods. -Every method contains a special dispatch block, (not shown in the picture above) whose goal is to jump to the specified block number inside the method itself. The first argument of a child method is always a block id; when the method starts, it immediately jumps to the dispatch block, and thus to the desired block. +Every method contains a special dispatch block, (not shown in the picture above) whose goal is to jump to the specified block number inside the method itself. The first argument of a secondary method is always a block id; when the method starts, it immediately jumps to the dispatch block, and thus to the desired block. For example, suppose to have a method which contains 3 blocks numbered 0, 1, 2; here is how its dispatch blocks looks like; for simplicity it is shown as C# code, but it is actually generated as IL bytecode: @@ -114,38 +128,38 @@ throw new Exception("Invalid block id"); } -Whenever we want to jump to a non-local block, it is enough to store the block id in the appropriate variable and jump to the dispatch block. If the block resides in a different method, the jump_to_unknown block is entered; this special block is implemented differently by the main method and the child methods, as we will see soon. +Whenever we want to jump to a external block, it is enough to store the block id in the appropriate variable and jump to the dispatch block. If the block resides in a different method, the jump_to_unknown block is entered; this special block is implemented differently by the main method and the secondary methods, as we will see soon. Each time a new method is added to the graph, we build a delegate for it, and store it in a special array called method_map; since we assign the method id sequentially starting from 0, we are sure that to fetch the method whose id is n we can simply load the n-th element of the array. -The jump_to_unknown block of the main method uses this array to select the right method, and calls it (FlexSwitchCase is the type of delegates for all children methods): +The jump_to_unknown block of the main method uses this array to select the right method, and calls it (FlexSwitchCase is the type of delegates for all secondary methods): // jump_to_unknown block of the main method FlexSwitchCase meth = method_map[methodid]; blockid = meth(blockid, ...); // execute the method goto dispatch_block; -Each child method returns a block id specifying the next block to jump to; after its execution, we assign the return value to the blockid variable, and jump again to the dispatch block, which will jump again to the appropriate block. +Each secondary method returns a block id specifying the next block to jump to; after its execution, we assign the return value to the blockid variable, and jump again to the dispatch block, which will jump again to the appropriate block. -Keeping this in mind, it is straightforward to implement the jump_to_unknown block of children methods: it is enough to return the target block id to the caller, and let its dispatch loop do the right thing. If the caller is also a child method, it will return it again, until we reach the dispatch loop of the main method, which will finally do the jump. In theory, we could implement things differently and jumping directly from a child method to another one, but in that case the call stack could grows indefinitely in case of a tight loop between two blocks residing in different methods. +Keeping this in mind, it is straightforward to implement the jump_to_unknown block of secondary methods: it is enough to return the target block id to the caller, and let its dispatch loop do the right thing. If the caller is also a secondary method, it will return it again, until we reach the dispatch loop of the main method, which will finally do the jump. In theory, we could implement things differently and jumping directly from a secondary method to another one, but in that case the call stack could grows indefinitely in case of a tight loop between two blocks residing in different methods. -To implement the dispatch block we can exploit the switch opcode of the CLI; if the .NET JIT is smart enough, it can render it using an indirect jump; overall, jumping to a non-local block consists of an indirect function call (by invoking the delegate) plus an indirect jump (by executing the switch opcode); even if this is more costly than a simple direct jump, we will see in the next section that this not the main source of overhead when following a non-local link. +To implement the dispatch block we can exploit the switch opcode of the CLI; if the .NET JIT is smart enough, it can render it using an indirect jump; overall, jumping to a external block consists of an indirect function call (by invoking the delegate) plus an indirect jump (by executing the switch opcode); even if this is more costly than a simple direct jump, we will see in the next section that this not the main source of overhead when following a external link. -Obviously, the slow dispatching logic is needed only when we want to jump to a non-local block; if the target block happens to reside in the same method as the current one, we can directly jump to it, completely removing the overhead. +Obviously, the slow dispatching logic is needed only when we want to jump to a external block; if the target block happens to reside in the same method as the current one, we can directly jump to it, completely removing the overhead. Moreover, the dispatch blocks are emitted only if needed, i.e. if the parent graph contains at least one flexswitch; graphs without flexswitches are rendered in the obvious way, by making one method per graph. The slow bit: passing arguments -Jumping to the correct block is not enough to follow a link: as we said before, each link carries a set of arguments to be passed from the source to the target block. As usual, passing arguments across local links is easy, as we can just use local variables to hold their values; on the other hand, non-local links make things more complex. +Jumping to the correct block is not enough to follow a link: as we said before, each link carries a set of arguments to be passed from the source to the target block. As usual, passing arguments across internal links is easy, as we can just use local variables to hold their values; on the other hand, external links make things more complex. The only way to jump to a block is to invoke its containing method, so the first solution that comes to mind is to specify its input arguments as parameter of the method; however, each block has potentially a different number (and different types) of input arguments than every other block, so we need to think of something else. An alternative solution could be to compute the union of the sets of input arguments of all the blocks in the method, and use this set as a signature for the method; this way, there would be enough space to specify the input arguments for every block we might want to jump to, each block ignoring the exceeding unused parameters. -Unfortunately, all the children methods must have the very same signature, as they are all called from the same calling site in the dispatch block of the main method. Since the union of the set of input arguments (and hence the computed signature) varies from method to method, this solution cannot work. +Unfortunately, all the secondary methods must have the very same signature, as they are all called from the same calling site in the dispatch block of the main method. Since the union of the set of input arguments (and hence the computed signature) varies from method to method, this solution cannot work. -We might think to determine the signature by computing the union of input arguments of all blocks in the graph; this way, all the children methods would have the same signature. But as we said above, the graph grows new blocks at runtime, so we cannot determine in advance which set of input arguments we will need. +We might think to determine the signature by computing the union of input arguments of all blocks in the graph; this way, all the secondary methods would have the same signature. But as we said above, the graph grows new blocks at runtime, so we cannot determine in advance which set of input arguments we will need. To solve the problem we need a way to pass a variable number of arguments without knowing in advance neither their number nor their types. Thus, we use an instance of this class: @@ -163,7 +177,7 @@ * when writing, we need to make sure that the arrays are big enough to contains all the arguments we need; if not, we need to allocate a bigger array. Moreover, for each argument we store into the array the virtual machine performs a bound-check, even if we know the index will never be out of bounds (because we checked the size of the array in advance); * when reading, the same bound-check is performed for each argument read; moreover, for each value read from the objs array we need to insert a downcast. -To mitigate the performance drop, we avoid to allocate a new InputArgs object each time we do a non-local jump; instead, we preallocate one at the beginning of the main method, and reuse it all the time. +To mitigate the performance drop, we avoid to allocate a new InputArgs object each time we do a external jump; instead, we preallocate one at the beginning of the main method, and reuse it all the time. Our benchmarks show that passing arguments in arrays is about 10 times slower than passing them as real parameter of a method. Unfortunately, we couldn't come up with anything better. Implement flexswitches @@ -199,20 +213,20 @@ The value returned by execute is the next block id to jump to; if the value is not found in the values array, we return the default_blockid, whose value has been set before by the JIT compiler; default_blockid usually points to a block containing code to restart the JIT compiler again; when the JIT compiler restarts, it emits more code for the missing case, then calls add_case on the flexswitch; from now on, the new blocks are wired into the existing graph, and we finally managed to implement growable graphs. Performances -As we saw, implementing growable graphs for CLI is a pain, as the virtual machine offers very little support, so we need an incredible amount of workarounds. Moreover, the code generated is much worse than what an assembly backend could produce, and the cost of following a non-local link is very high compared to local links. +As we saw, implementing growable graphs for CLI is a pain, as the virtual machine offers very little support, so we need an incredible amount of workarounds. Moreover, the code generated is much worse than what an assembly backend could produce, and the cost of following a external link is very high compared to internal links. However, our first blog post showed that we still get very good performances; how is it possible? -As usual in computer science, most of the time of a running program in spent in a tiny fraction of the code; our benchmark is no exception, and the vast majority of the time is spent in the inner loop that multiplies numbers; the graph is built in such a way that all the blocks that are part of the inner loop reside in the same method, so that all links inside are local (and fast). +As usual in computer science, most of the time of a running program in spent in a tiny fraction of the code; our benchmark is no exception, and the vast majority of the time is spent in the inner loop that multiplies numbers; the graph is built in such a way that all the blocks that are part of the inner loop reside in the same method, so that all links inside are internal (and fast). -Flexswitches and non-local links play a key role to select the right specialized implementation of the inner loop, but once it is selected they are not executed anymore until we have finished the computation. +Flexswitches and external links play a key role to select the right specialized implementation of the inner loop, but once it is selected they are not executed anymore until we have finished the computation. -It is still unclear how things will look like when we will compile the full Python language instead of a toy one; depending on the code, it could be possible to have non-local links inside the inner loop, thus making performance much worse. +It is still unclear how things will look like when we will compile the full Python language instead of a toy one; depending on the code, it could be possible to have external links inside the inner loop, thus making performance much worse. Alternative implementations Before implementing the solution described here, we carefully studied a lot of possible alternatives, but all of them either didn't work because of a limitation of the virtual machine or they could work but with terrible performances. -In particular, in theory it is possible to implement non-local links using tail calls, by putting each block in its own method and doing a tail call instead of a jump; this would also solve the problem of how to pass arguments, as each method could have its own signature matching the input args of the block. I would like to explain this solution in a more detailed way as I think it's really elegant and nice, but since this post is already too long, I'll stop here :-). +In particular, in theory it is possible to implement external links using tail calls, by putting each block in its own method and doing a tail call instead of a jump; this would also solve the problem of how to pass arguments, as each method could have its own signature matching the input args of the block. I would like to explain this solution in a more detailed way as I think it's really elegant and nice, but since this post is already too long, I'll stop here :-). In theory, if the .NET JIT were smart enough it could inline and optimize away the tail calls (or at least many of those) and give us very efficient code. However, one benchmark I wrote shows that tail calls are up to 10 times slower (!!!) than normal calls, thus making impractical to use them for our purposes. Conclusion From pedronis at codespeak.net Tue Dec 16 20:23:27 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 16 Dec 2008 20:23:27 +0100 (CET) Subject: [pypy-svn] r60524 - in pypy/build/testrunner: . test Message-ID: <20081216192327.AA26D16847B@codespeak.net> Author: pedronis Date: Tue Dec 16 20:23:24 2008 New Revision: 60524 Modified: pypy/build/testrunner/runner.py pypy/build/testrunner/test/test_runner.py Log: don't hang and produce some error output if we fail to invoke the intepreter Modified: pypy/build/testrunner/runner.py ============================================================================== --- pypy/build/testrunner/runner.py (original) +++ pypy/build/testrunner/runner.py Tue Dec 16 20:23:24 2008 @@ -24,12 +24,20 @@ except OSError: pass +RUNFAILED = -1000 TIMEDOUT = -999 def run(args, cwd, out, timeout=None): f = out.open('w') try: - p = subprocess.Popen(args, cwd=str(cwd), stdout=f, stderr=f) + try: + p = subprocess.Popen(args, cwd=str(cwd), stdout=f, stderr=f) + except Exception, e: + f.write("Failed to run %s with cwd='%s' timeout=%s:\n" + " %s\n" + % (args, cwd, timeout, e)) + return RUNFAILED + if timeout is None: return p.wait() else: @@ -89,6 +97,8 @@ msg = "Exit code %d." % exitcode elif exitcode == TIMEDOUT: msg = "TIMEOUT" + elif exitcode == RUNFAILED: + msg = "Failed to run interp" else: msg = "Killed by %s." % getsignalname(-exitcode) extralog = "! %s\n %s\n" % (test, msg) Modified: pypy/build/testrunner/test/test_runner.py ============================================================================== --- pypy/build/testrunner/test/test_runner.py (original) +++ pypy/build/testrunner/test/test_runner.py Tue Dec 16 20:23:24 2008 @@ -270,6 +270,22 @@ log_lines = log.getvalue().splitlines() assert log_lines[1] == ' TIMEOUT' + def test_run_wrong_interp(self): + log = cStringIO.StringIO() + out = cStringIO.StringIO() + + run_param = runner.RunParam(self.one_test_dir) + run_param.interp=['wrong-interp'] + run_param.parallel_runs = 3 + + testdirs = [] + run_param.collect_testdirs(testdirs) + res = runner.execute_tests(run_param, testdirs, log, out) + assert res + + log_lines = log.getvalue().splitlines() + assert log_lines[1] == ' Failed to run interp' + class TestRunnerNoThreads(RunnerTests): with_thread = False From antocuni at codespeak.net Tue Dec 16 21:06:43 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 16 Dec 2008 21:06:43 +0100 (CET) Subject: [pypy-svn] r60525 - in pypy/branch/oo-jit/pypy/jit/tl: . test Message-ID: <20081216200643.ED24F1684BA@codespeak.net> Author: antocuni Date: Tue Dec 16 21:06:40 2008 New Revision: 60525 Added: pypy/branch/oo-jit/pypy/jit/tl/fibo.tlc (contents, props changed) pypy/branch/oo-jit/pypy/jit/tl/fibo.tlc.src Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py Log: add a tlc program that compute the nth fibonacci number Added: pypy/branch/oo-jit/pypy/jit/tl/fibo.tlc ============================================================================== Binary file. No diff available. Added: pypy/branch/oo-jit/pypy/jit/tl/fibo.tlc.src ============================================================================== --- (empty file) +++ pypy/branch/oo-jit/pypy/jit/tl/fibo.tlc.src Tue Dec 16 21:06:40 2008 @@ -0,0 +1,26 @@ +main: + PUSH 0 + PUSH 1 + PUSHARG + +loop: # [a, b, n] + PUSH 1 + SUB # [a, b, n-1] + PICK 0 + BR_COND true + PUSH 1 + BR_COND exit + +true: # [a, b, n] + SWAP + ROLL 3 # [n, b, a] + PICK 1 # [n, b, a, b] + ADD # [n, b, a+b] + ROLL 3 # [b, a+b, n] + + PUSH 1 + BR_COND loop + +exit: # [a, b, 0] + POP + RETURN Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py Tue Dec 16 21:06:40 2008 @@ -257,16 +257,20 @@ res = interp_eval(bytecode, 0, [nil], pool) assert res.int_o() == 42 - def test_binarytree(self): + def compile(self, filename): from pypy.jit.tl.tlc import interp_eval, IntObj pool = ConstantPool() - path = py.path.local(__file__).join('../../binarytree.tlc.src') + path = py.path.local(__file__).join(filename) src = path.read() bytecode = compile(src, pool) - def search(n): + def fn(n): obj = IntObj(n) res = interp_eval(bytecode, 0, [obj], pool) return res.int_o() + return fn + + def test_binarytree(self): + search = self.compile('../../binarytree.tlc.src') assert search(20) assert search(10) assert search(15) @@ -275,3 +279,10 @@ assert not search(40) assert not search(12) assert not search(27) + + def test_fibo(self): + fibo = self.compile('../../fibo.tlc.src') + assert fibo(1) == 1 + assert fibo(2) == 1 + assert fibo(3) == 2 + assert fibo(7) == 13 From davide at codespeak.net Tue Dec 16 23:05:31 2008 From: davide at codespeak.net (davide at codespeak.net) Date: Tue, 16 Dec 2008 23:05:31 +0100 (CET) Subject: [pypy-svn] r60526 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081216220531.C5C9A16851A@codespeak.net> Author: davide Date: Tue Dec 16 23:05:29 2008 New Revision: 60526 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex Log: added picture with method and block ids Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Tue Dec 16 23:05:29 2008 @@ -92,14 +92,37 @@ method is not difficult: the corresponding method has to be invoked with the appropriate arguments. +What cannot be easily implemented in CLI is following an external link +whose target is not an initial block; consider, for instance, the +outgoing link of the block dynamically added in the right-hand side +picture of Figure~\ref{flexswitch-fig}. How it is possible to pass the +right arguments to the target block? + +To solve this problem a special block, called \emph{dispatcher}, is +added to every method; whenever a method is invoked, its dispatcher is +executed first\footnote{Recall that the dispatcher is a special block +and must not be confused with the initial block of a method.} to +determine which block has to be executed. +This is done by passing to the method a 32 bits number, called +\emph{block id}, which uniquely identifies the next block of the graph to be executed. +The high word of a block id is the id of the method to which the block +belongs, whereas the low word is a progressive number identifying +each block implemented by the method. + +The picture in Figure~\ref{block-id-fig} shows a graph composed of three methods (for +simplicity, dispatchers are not shown); method ids are in red, whereas +block numbers are in black. +The graph contains three external links; in particular, note the link +between blocks \texttt{0x00020001} and \texttt{0x00010001} which +connects two blocks implemented by different methods. +\begin{figure}[h] +\begin{center} +\includegraphics[height=6cm]{blockid} +\caption{Method and block ids.}\label{block-id-fig} +\end{center} +\end{figure} \commentout{ - - -Using this approach, after a while the blocks of our original graph are scattered over a lot of different methods; however, there are no constraints about how these blocks can be linked together, so it happens to have links between blocks which are not in the same method. In the following, we will refer to them as external links. - -If the external block we want to jump to happens to be at the beginning of its containing method, it is enough to invoke the method; but, what if we want to jump somewhere in the middle? What we really want is to produce a method which has multiple entry-points; again, doing it in assembly would be trivial, but the virtual machine does not provide any support for it, so we need a work around. - Each method in a graph is assigned an unique 16 bit method id; each block in a method is assigned a progressive 16 bit block number. From this two numbers, we can compute the block id as an unsigned integer, by storing the method id in the first 16 bits and the block number in the second 16 bits. By construction, the block id is guaranteed to be unique in the graph. The following picture shows a graph composed of three methods; the id of each method is shown in red, while the block ids are shown in red (for the method id part) and black (for the block number part). The graph contains three external links; in particular, note the link between blocks 0x00020001 and 0x00010001 which connects two block that resides in different methods. From davide at codespeak.net Tue Dec 16 23:06:24 2008 From: davide at codespeak.net (davide at codespeak.net) Date: Tue, 16 Dec 2008 23:06:24 +0100 (CET) Subject: [pypy-svn] r60527 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081216220624.AA68316851A@codespeak.net> Author: davide Date: Tue Dec 16 23:06:24 2008 New Revision: 60527 Added: pypy/extradoc/talk/ecoop2009/blockid.png (contents, props changed) Log: Antonio's picture with method and block ids Added: pypy/extradoc/talk/ecoop2009/blockid.png ============================================================================== Binary file. No diff available. From antocuni at codespeak.net Wed Dec 17 10:57:11 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 17 Dec 2008 10:57:11 +0100 (CET) Subject: [pypy-svn] r60529 - pypy/branch/oo-jit/pypy/jit/tl Message-ID: <20081217095711.253671684CA@codespeak.net> Author: antocuni Date: Wed Dec 17 10:57:08 2008 New Revision: 60529 Modified: pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py Log: sanity check Modified: pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py Wed Dec 17 10:57:08 2008 @@ -110,7 +110,9 @@ label_usage.append( (arg, len(bytecode)) ) bytecode.append( 0 ) for label, pc in label_usage: - bytecode[pc] = labels[label] - pc - 1 + offset = labels[label] - pc - 1 + assert -128 <= offset <= 127 + bytecode[pc] = offset for methods in method_usage: for i, (methname, label) in enumerate(methods): pc = labels[label] From antocuni at codespeak.net Wed Dec 17 10:58:58 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 17 Dec 2008 10:58:58 +0100 (CET) Subject: [pypy-svn] r60530 - pypy/branch/oo-jit/pypy/jit/rainbow/test Message-ID: <20081217095858.E529D1684CA@codespeak.net> Author: antocuni Date: Wed Dec 17 10:58:58 2008 New Revision: 60530 Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_vlist.py Log: a failing test Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_vlist.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/rainbow/test/test_vlist.py (original) +++ pypy/branch/oo-jit/pypy/jit/rainbow/test/test_vlist.py Wed Dec 17 10:58:58 2008 @@ -214,6 +214,20 @@ res = self.interpret(f, [2], [0], policy=P_OOPSPEC) assert res == -7 + def test_vlist_of_vlist(self): + py.test.skip('infinite loop') + def fn(n): + stack = [] + for i in range(n): + mylist = [] + for j in range(i, n): + mylist.append(j) + stack.append(mylist) + mylist = stack.pop() + return mylist[0] + res = self.interpret(fn, [10], [], policy=P_OOPSPEC) + assert res == 9 + self.check_insns({}) class TestOOType(OOTypeMixin, OORtypeMixin, VListTest): type_system = "ootype" From antocuni at codespeak.net Wed Dec 17 11:01:41 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 17 Dec 2008 11:01:41 +0100 (CET) Subject: [pypy-svn] r60531 - in pypy/branch/oo-jit/pypy/jit: rainbow/test tl tl/test Message-ID: <20081217100141.F3FAE1684CA@codespeak.net> Author: antocuni Date: Wed Dec 17 11:01:40 2008 New Revision: 60531 Added: pypy/branch/oo-jit/pypy/jit/tl/accumulator.tlc (contents, props changed) pypy/branch/oo-jit/pypy/jit/tl/accumulator.tlc.src Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlc.py Log: a new oo benchmark for tlc, and two failing tests Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/rainbow/test/test_0tlc.py Wed Dec 17 11:01:40 2008 @@ -74,7 +74,7 @@ code = """ NEW foo,meth=meth PICK 0 - PUSH 40 + PUSHARG SETATTR foo PUSH 2 SEND meth/1 @@ -86,8 +86,39 @@ ADD RETURN """ - res = self.exec_code(code, 0) + res = self.exec_code(code, 40) assert res == 42 + self.check_insns(malloc=1) + + def test_method_loop(self): + path = py.path.local(__file__).join('../../../tl/accumulator.tlc.src') + code = path.read() + res = self.exec_code(code, 10) + assert res == sum(range(10)) + self.check_insns(malloc=1) + + def test_binarytree(self): + py.test.skip('fix me') + path = py.path.local(__file__).join('../../../tl/binarytree.tlc.src') + code = path.read() + res = self.exec_code(code, 15) + assert res == 1 + #self.check_insns(...) + + def test_bug(self): + py.test.skip('fix me') + code = """ + NEW foo,meth=meth + PUSHARG # if we use PUSH it works + SEND meth/1 + RETURN + meth: + PUSH 1 + RETURN + """ + res = self.exec_code(code, -1) + assert res == 1 + self.check_insns(malloc=1) class TestLLType(BaseTestTLC): type_system = "lltype" Added: pypy/branch/oo-jit/pypy/jit/tl/accumulator.tlc ============================================================================== Binary file. No diff available. Added: pypy/branch/oo-jit/pypy/jit/tl/accumulator.tlc.src ============================================================================== --- (empty file) +++ pypy/branch/oo-jit/pypy/jit/tl/accumulator.tlc.src Wed Dec 17 11:01:40 2008 @@ -0,0 +1,33 @@ +main: + NEW value,add=add + PICK 0 + PUSH 0 + SETATTR value + PUSHARG # [obj, n] + +loop: # [obj, n] + PICK 0 # [obj, n, n] + BR_COND continue + +exit: # [obj, n] + POP + GETATTR value + RETURN + +continue: # [obj, n] + PUSH 1 + SUB # [obj, n-1] + PICK 1 # [obj, n-1, obj] + PICK 1 # [obj, n-1, obj, n-1] + SEND add/1 # [obj, n-1] + PUSH 1 + BR_COND loop + +add: # (x) + PUSHARG # [self] + PUSHARG # [self, self] + GETATTR value # [self, self.value] + PUSHARGN 1 # [self, self.value, x] + ADD # [self, self.value+x] + SETATTR value # [] + RETURN Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py Wed Dec 17 11:01:40 2008 @@ -286,3 +286,9 @@ assert fibo(2) == 1 assert fibo(3) == 2 assert fibo(7) == 13 + + def test_accumulator(self): + acc = self.compile('../../accumulator.tlc.src') + assert acc(1) == 0 + assert acc(10) == sum(range(10)) + assert acc(20) == sum(range(20)) Modified: pypy/branch/oo-jit/pypy/jit/tl/tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlc.py Wed Dec 17 11:01:40 2008 @@ -471,6 +471,7 @@ if not we_are_translated(): a = stack.pop() hint(a, promote_class=True) + print a.to_string() elif opcode == DUMP: if not we_are_translated(): From cfbolz at codespeak.net Wed Dec 17 11:46:20 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 17 Dec 2008 11:46:20 +0100 (CET) Subject: [pypy-svn] r60532 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081217104620.A7C0B168516@codespeak.net> Author: cfbolz Date: Wed Dec 17 11:46:18 2008 New Revision: 60532 Removed: pypy/extradoc/talk/ecoop2009/background.tex Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex pypy/extradoc/talk/ecoop2009/intro.tex pypy/extradoc/talk/ecoop2009/jitgen.tex pypy/extradoc/talk/ecoop2009/main.tex pypy/extradoc/talk/ecoop2009/rainbow.tex Log: steal more text Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Wed Dec 17 11:46:18 2008 @@ -2,12 +2,14 @@ \anto{maybe we should move this section somewhere else, if we want to use TLC as a running example in other sections} +\cfbolz{yes please! how about making it the next section after the +introduction?} In this section, we will briefly describe \emph{TLC}, a simple dynamic language that we developed to exercise our JIT compiler generator. As most of dynamic languages around, \emph{TLC} is implemented through a virtual machine that interprets a custom bytecode. Since our main interest is in the runtime -performances of the VM, we did not implement the parser nor the bytecode +performance of the VM, we did not implement the parser nor the bytecode compiler, but only the VM itself. TLC provides four different types: @@ -18,8 +20,8 @@ \item Lisp-like lists \end{enumerate} -Objects represent a collection of named attributes (much like Javascript or -SELF) and named methods. At creation time, it is necessary to specify the set +Objects represent a collection of named attributes (much like JavaScript or +Self) and named methods. At creation time, it is necessary to specify the set of attributes of the object, as well as its methods. Once the object has been created, it is not possible to add/remove attributes and methods. @@ -40,15 +42,19 @@ \end{itemize} Obviously, not all the operations are applicable to all objects. For example, -it is not possibile to \lstinline{ADD} an integer and an object, or reading an +it is not possible to \lstinline{ADD} an integer and an object, or reading an attribute from an object which does not provide it. Being a dynamic language, the VM needs to do all these checks at runtime; in case one of the check fails, the execution is simply aborted. \anto{should we try to invent a syntax for TLC and provide some examples?} +\cfbolz{we should provide an example with the assembler syntax} \section{Benchmarks} +\cfbolz{I think this should go to the beginning of the description of the TLC as +it explains why it is written as it is written} + Despite being very simple and minimalistic, \lstinline{TLC} is a good candidate as a language to run benchmarks, as it has some of the features that makes most of current dynamic languages so slow: @@ -58,15 +64,15 @@ \item \textbf{Stack based VM}: this kind of VM requires all the operands to be on top of the evaluation stack. As a consequence programs spend a lot of time pushing and popping values to/from the stack, or doing other stack - related operations. However, thanks ot its semplicity this is still the + related operations. However, thanks to its simplicity this is still the most common and preferred way to implement VMs. \item \textbf{Boxed integers}: integer objects are internally represented as an instance of the \lstinline{IntObj} class, whose field \lstinline{value} - contains the real value. By having boxed integers, common ariithmetic + contains the real value. By having boxed integers, common arithmetic operations are made very slow, because each time we want to load/store their value we need to go through an extra level of indirection. Moreover, in - case of a complex expression, it is necessary to create a lot of temporary + case of a complex expression, it is necessary to create many temporary objects to hold intermediate results. \item \textbf{Dynamic lookup}: attributes and methods are looked up at Modified: pypy/extradoc/talk/ecoop2009/intro.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/intro.tex (original) +++ pypy/extradoc/talk/ecoop2009/intro.tex Wed Dec 17 11:46:18 2008 @@ -10,16 +10,19 @@ it still require a lot of work, as IronPython, Jython, JRuby demonstrate. Moreover, writing a static compiler is often not enough to get high -performances; IronPython and JRuby are going in the direction of JIT compiling +performance; IronPython and JRuby are going in the direction of JIT compiling specialized versions of the code depending on the actual values/types seen at -runtime; this approach seems to work, but write it manually requires an +runtime; this approach seems to work, but writing it manually requires an enormous effort. -PyPy's idea is to automatize the generation of static/JIT compilers in order -to reduce at minimun the effort required to get a fast implementation of a +\cfbolz{we should cite the dyla paper somewhere here} + +PyPy's idea is to automatize the generation of JIT compilers in order +to reduce to a minimum the effort required to get a fast implementation of a dynamic language; all you have to do is to write a high level specification of -the language (by writing an interpreter), and put it through PyPy's -translation toolchain. +the language (by writing an interpreter), and putting it through PyPy's +translation toolchain. The automatic generation of JIT compilers is done with +the help of partial evaluation techniques. \subsection{PyPy and RPython} @@ -49,7 +52,7 @@ Compilation of the interpreter is implemented as a stepwise refinement by means of a translation toolchain which performs type analysis, code optimizations and several transformations aiming at -incrementally providing implementation details as memory management or the threading model. +incrementally providing implementation details such as memory management or the threading model. The different kinds of intermediate codes which are refined during the translation process are all represented by a collection of control flow graphs, at several levels of abstractions. @@ -60,15 +63,23 @@ Currently, three fully developed backends are available to produce executable C/POSIX code, Java and CLI/.NET bytecode. -Despite the PyPy infrastructure was specifically developed -for Python, in fact it can be used for implementing -other languages. Indeed, PyPy has been successfully experimented with -several languages as Smalltalk \cite{BolzEtAl08}, JavaScript, Scheme and Prolog. +Despite having been specifically developed for Python, the PyPy infrastructure +can in fact be used for implementing other languages. Indeed, there were +successful experiments of using PyPy to implement several other languages such +as Smalltalk \cite{BolzEtAl08}, JavaScript, Scheme and Prolog. As suggested by Figure~\ref{pypy-fig}, a portable interpreter for a generic language $L$ can be -easily developed once an abstract interpreter for $L$ is implemented in +easily developed once an interpreter for $L$ has been implemented in RPython. +\subsection{PyPy and JIT-Generation} + +This section will give a high-level overview of how the JIT-generation process +works. More details will be given in subsequent sections. + + Another interesting feature of PyPy is that just-in-time compilers can be semi-automatically generated from the interpreter source. + +XXX list contributions clearly Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Wed Dec 17 11:46:18 2008 @@ -16,15 +16,46 @@ uses the same techniques but it's manually written instead of being automatically generated. -The original idea is by Futamura \cite{Futamura99}. He proposed to generate compilers -from interpreters with automatic specialization, but his work has had -relatively little practical impact so far. - \subsection{Partial evaluation} -Assume the Python bytecode to be constant, and constant-propagate it into the -Python interpreter. -\cfbolz{note to self: steal bits from the master thesis?} +In 1971 Yoshihiko Futamura published a paper \cite{Futamura99} that proposed a +technique to automatically transform an interpreter of a programming language +into a compiler for the same language. This would solve the problem of having to +write a compiler instead of a much simpler interpreter. He proposed to use +partial evaluation to achieve this goal. He defined partial evaluation along the following lines: + +Given a program $P$ with $m + n$ input variables $s_1, ..., s_m$ and $d_1, ..., +d_n$, the partial evaluation of $P$ with respect to concrete values $s'_1, ..., +s'_m$ for the first $m$ variables is a program $P'$. The program $P'$ takes only +the input variables $d_1, ..., d_n$ but behaves exactly like $P$ with the +concrete values (but is hopefully more efficient). This transformation is done +by a program $S$, the partial evaluator, which takes $P$ and $s_1, ..., s_m$ as +input: + + $$S(P, (s'_1, ..., s'_m)) = P'$$ + +The variables $s_1, ..., s_m$ are called the \emph{static} variables, the +variables $d_1, ..., d_n$ are called the \emph{dynamic} variables; $P'$ is the +\emph{residual code}. Partial evaluation creates a version of $P$ that works +only for a fixed set of inputs for the first $m$ arguments. This effect is +called \emph{specialization}. + +When $P$ is an interpreter for a programming language, then the $s_1, ..., s_m$ +are chosen such that they represent the program that the interpreter is +interpreting and the $d_1, ..., d_n$ represent the input of this program. Then +$P'$ can be regarded as a compiled version of the program that the chosen $s'_1, +..., s'_m$ represent, since it is a version of the interpreter that can only +interpret this program. Now once the partial evaluator $S$ is implemented, it is +actually enough to implement an interpreter for a new language and use $S$ +together with this interpreter to compile programs in that new language. + +A valid implementation for $S$ would be to just put the concrete values into $P$ +to get $P'$, which would not actually produce any performance benefits compared with +directly using $P$. A good implementation for $S$ should instead make use of the +information it has and evaluate all the parts of the program that actually +depend only on the $s_1, ..., s_m$ and to remove parts of $P$ that cannot be +reached given the concrete values. + \cfbolz{I would propose to use either TLC as an example here, or something that looks at least like an interpreter loop} @@ -102,8 +133,9 @@ control flow graphs of the interpreter, guided by the binding times. We call this process \emph{timeshifting}. +XXX write something about the problems of classical PE? -\subsection{Execution steps} +\subsection{Partial Evaluation in PyPy} PyPy contains a framework for generating just-in-time compilers using @@ -139,16 +171,20 @@ colors that we use when displaying the control flow graphs: \begin{itemize} -\item \emph{Green} variables contain values that are known at compile-time; -\item \emph{Red} variables contain values that are not known until run-time. +\item \emph{Green} variables contain values that are known at compile-time. +They correspond to static arguments. +\item \emph{Red} variables contain values that are usually not known +compile-time. They correspond to dynamic arguments. \end{itemize} -The binding-time analyzer of our translation tool-chain is based on the +The binding-time analyzer of our translation tool-chain is using a simple +abstract-interpretation based analysis. It is based on the same type inference engine that is used on the source RPython program, the annotator. In this mode, it is called the \emph{hint-annotator}; it operates over input graphs that are already low-level instead of RPython-level, and propagates annotations that do not track types but value dependencies and manually-provided binding time hints. +XXX the above needs rewriting when the background section is there The normal process of the hint-annotator is to propagate the binding time (i.e. color) of the variables using the following kind of rules: @@ -196,9 +232,9 @@ the hint-annotator as a request for both \texttt{v1} and \texttt{v2} to be green. It has a \emph{global} effect on the binding times: it means that not only \texttt{v1} but all the values that \texttt{v1} depends on ? recursively ? -are forced to be green. The hint-annotator complains if the +are forced to be green. The hint-annotator gives an error if the dependencies of \texttt{v1} include a value that cannot be green, like -a value read out of a field of a non-immutable structure. +a value read out of a field of a non-immutable instance. Such a need-oriented backward propagation has advantages over the commonly used forward propagation, in which a variable is compile-time @@ -211,9 +247,6 @@ it prevents under-specialization: an unsatisfiable \texttt{hint(v1, concrete=True)} is reported as an error. -In our context, though, such an error can be corrected. This is done by -promoting a well-chosen variable among the ones that \texttt{v1} depends on. - Promotion is invoked with the use of a hint as well: \texttt{v2 = hint(v1, promote=True)}. This hint is a \emph{local} request for \texttt{v2} to be green, without @@ -222,4 +255,3 @@ approaches to partial evaluation. See the Promotion section XXX ref for a complete discussion of promotion. - Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Wed Dec 17 11:46:18 2008 @@ -36,11 +36,16 @@ \newcommand\anto[1]{\nb{ANTO}{#1}} \newcommand{\commentout}[1]{} +\let\oldcite=\cite + +\renewcommand\cite[1]{\ifthenelse{\equal{#1}{XXX}}{[citation~needed]}{\oldcite{#1}}} + + \begin{document} \title{Automatic generation of JIT compilers for dynamic languages in .NET\thanks{This work has been partially supported by MIUR EOS DUE - Extensible Object Systems for Dynamic and -Unpredictable Environments.}} +Unpredictable Environments.\cfbolz{should we put the PyPy EU project here as well?}} \author{Davide Ancona\inst{1} \and Carl Friedrich Bolz\inst{2} \and Antonio Cuni\inst{1} \and Armin Rigo} @@ -78,7 +83,6 @@ \input{abstract} \input{intro} -\input{background} \input{jitgen} \input{rainbow} \input{clibackend} Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Wed Dec 17 11:46:18 2008 @@ -14,33 +14,78 @@ The Rainbow bytecode is produced at translation time, when the JIT compiler is generated. -Here are summarized the various phases of the JIT: +\cfbolz{XXX I think we should be very careful with the rainbow interp. it is a +total implementation-detail and we should only describe it as little as +possible} -Translation time: - - * Low-level flowgraphs are produced - - * The *hint-annotator* colors the variables - - * The *rainbow codewriter* translates flowgraphs into rainbow bytecode - - -Compile-time: - - * The rainbow interpreter executes the bytecode - - * As a result, it produces executable code +\subsection{Example of Rainbow bytecode and execution} -Runtime: +TODO - * The produced code is executed +\section{Promotion} +In the sequel, we describe in more details one of the main new +techniques introduced in our approach, which we call \emph{promotion}. In +short, it allows an arbitrary run-time value to be turned into a +compile-time value at any point in time. Promotion is thus the central way by +which we make use of the fact that the JIT is running interleaved with actual +program execution. Each promotion point is explicitly defined with a hint that +must be put in the source code of the interpreter. + +From a partial evaluation point of view, promotion is the converse of +the operation generally known as "lift" \cite{XXX}. Lifting a value means +copying a variable whose binding time is compile-time into a variable +whose binding time is run-time ? it corresponds to the compiler +"forgetting" a particular value that it knew about. By contrast, +promotion is a way for the compiler to gain \emph{more} information about +the run-time execution of a program. Clearly, this requires +fine-grained feedback from run-time to compile-time, thus a +dynamic setting. + +Promotion requires interleaving compile-time and run-time phases, +otherwise the compiler can only use information that is known ahead of +time. It is impossible in the "classical" approaches to partial +evaluation, in which the compiler always runs fully ahead of execution +This is a problem in many large use cases. For example, in an +interpreter for a dynamic language, there is mostly no information +that can be clearly and statically used by the compiler before any +code has run. + +A very different point of view on promotion is as a generalization of +techniques that already exist in dynamic compilers as found in modern +object-oriented language virtual machines. In this context feedback +techniques are crucial for good results. The main goal is to +optimize and reduce the overhead of dynamic dispatching and indirect +invocation. This is achieved with variations on the technique of +polymorphic inline caches \cite{XXX}: the dynamic lookups are cached and +the corresponding generated machine code contains chains of +compare-and-jump instructions which are modified at run-time. These +techniques also allow the gathering of information to direct inlining for even +better optimization results. + +In the presence of promotion, dispatch optimization can usually be +reframed as a partial evaluation task. Indeed, if the type of the +object being dispatched to is known at compile-time, the lookup can be +folded, and only a (possibly inlined) direct call remains in the +generated code. In the case where the type of the object is not known +at compile-time, it can first be read at run-time out of the object and +promoted to compile-time. As we will see in the sequel, this produces +very similar machine code \footnote{This can also be seen as a generalization of +a partial evaluation transformation called "The Trick" (see e.g. \cite{XXX}), +which again produces similar code but which is only applicable for finite sets +of values.}. + +The essential advantage is that it is no longer tied to the details of +the dispatch semantics of the language being interpreted, but applies in +more general situations. Promotion is thus the central enabling +primitive to make partial evaluation a practical approach to language +independent dynamic compiler generation. -\subsection{Example of Rainbow bytecode and execution} +\subsection{Promotion as Applied to the TLC} -TODO +XXX -\subsection{Promotion} +\subsection{Promotion in Practise} There are values that, if known at compile time, allow the JIT compiler to produce very efficient code. Unfortunately, these values are tipically red, @@ -54,7 +99,8 @@ This is done by continuously intermixing compile time and runtime; a promotion is implemented in this way: - * (compile time): the rainbow interpreter produces machine code until it +\begin{itemize} + \item (compile time): the rainbow interpreter produces machine code until it hits a promotion point; e.g.:: \begin{lstlisting}[language=C] @@ -62,7 +108,7 @@ return y+10 \end{lstlisting} - * (compile time): at this point, it generates special machine code that when + \item (compile time): at this point, it generates special machine code that when reached calls the JIT compiler again; the JIT compilation stops:: \begin{lstlisting}[language=C] @@ -71,11 +117,11 @@ } \end{lstlisting} - * (runtime): the machine code is executed; when it reaches a promotion + \item (runtime): the machine code is executed; when it reaches a promotion point, it executes the special machine code we described in the previous point; the JIT compiler is invoked again; - * (compile time): now we finally know the exact value of our red variable, + \item (compile time): now we finally know the exact value of our red variable, and we can promote it to green; suppose that the value of 'y' is 32:: \begin{lstlisting}[language=C] @@ -88,10 +134,83 @@ Note that the operation "y+10" has been constant-folded into "42", as it was a green operation. - * (runtime) the execution restart from the point it stopped, until a new + \item (runtime) the execution restart from the point it stopped, until a new unhandled promotion point is reached. +\end{itemize} -\subsection{Virtuals and virtualizables} +\section{Automatic Unboxing of Intermediate Results} -\cfbolz{do we even want to talk about virtualizables?} -TODO +XXX the following section needs a rewriting to be much more high-level and to +compare more directly with classical escape analysis + +Interpreters for dynamic languages typically allocate a lot of small +objects, for example due to boxing. For this reason, we +implemented a way for the compiler to generate residual memory +allocations as lazily as possible. The idea is to try to keep new +run-time structures "exploded": instead of a single run-time pointer to +a heap-allocated data structure, the structure is "virtualized" as a set +of fresh variables, one per field. In the compiler, the variable that +would normally contain the pointer to the structure gets instead a +content that is neither a run-time value nor a compile-time constant, +but a special \emph{virtual structure} ? a compile-time data structure that +recursively contains new variables, each of which can again store a +run-time, a compile-time, or a virtual structure value. + +This approach is based on the fact that the "run-time values" carried +around by the compiler really represent run-time locations ? the name of +a CPU register or a position in the machine stack frame. This is the +case for both regular variables and the fields of virtual structures. +It means that the compilation of a \texttt{getfield} or \texttt{setfield} +operation performed on a virtual structure simply loads or stores such a +location reference into the virtual structure; the actual value is not +copied around at run-time. + +It is not always possible to keep structures virtual. The main +situation in which it needs to be "forced" (i.e. actually allocated at +run-time) is when the pointer escapes to some non-virtual location like +a field of a real heap structure. + +Virtual structures still avoid the run-time allocation of most +short-lived objects, even in non-trivial situations. The following +example shows a typical case. Consider the Python expression \texttt{a+b+c}. +Assume that \texttt{a} contains an integer. The PyPy Python interpreter +implements application-level integers as boxes ? instances of a +\texttt{W\_IntObject} class with a single \texttt{intval} field. Here is the +addition of two integers: + +XXX needs to use TLC examples +\begin{verbatim} + def add(w1, w2): # w1, w2 are W_IntObject instances + value1 = w1.intval + value2 = w2.intval + result = value1 + value2 + return W_IntObject(result) +\end{verbatim} + +When interpreting the bytecode for \texttt{a+b+c}, two calls to \texttt{add()} are +issued; the intermediate \texttt{W\_IntObject} instance is built by the first +call and thrown away after the second call. By contrast, when the +interpreter is turned into a compiler, the construction of the +\texttt{W\_IntObject} object leads to a virtual structure whose \texttt{intval} +field directly references the register in which the run-time addition +put its result. This location is read out of the virtual structure at +the beginning of the second \texttt{add()}, and the second run-time addition +directly operates on the same register. + +An interesting effect of virtual structures is that they play nicely with +promotion. Indeed, before the interpreter can call the proper \texttt{add()} +function for integers, it must first determine that the two arguments +are indeed integer objects. In the corresponding dispatch logic, we +have added two hints to promote the type of each of the two arguments. +This produces a compiler that has the following behavior: in the general +case, the expression \texttt{a+b} will generate two consecutive run-time +switches followed by the residual code of the proper version of +\texttt{add()}. However, in \texttt{a+b+c}, the virtual structure representing +the intermediate value will contain a compile-time constant as type. +Promoting a compile-time constant is trivial ? no run-time code is +generated. The whole expression \texttt{a+b+c} thus only requires three +switches instead of four. It is easy to see that even more switches can +be skipped in larger examples; typically, in a tight loop manipulating +only integers, all objects are virtual structures for the compiler and +the residual code is theoretically optimal ? all type propagation and +boxing/unboxing occurs at compile-time. From antocuni at codespeak.net Wed Dec 17 11:47:39 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 17 Dec 2008 11:47:39 +0100 (CET) Subject: [pypy-svn] r60533 - in pypy/branch/oo-jit/pypy/jit/tl: . test Message-ID: <20081217104739.BF0F4168516@codespeak.net> Author: antocuni Date: Wed Dec 17 11:47:38 2008 New Revision: 60533 Modified: pypy/branch/oo-jit/pypy/jit/tl/accumulator.tlc pypy/branch/oo-jit/pypy/jit/tl/accumulator.tlc.src pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py Log: add another "class" to the benchmark, thanks cfbolz for the suggestion Modified: pypy/branch/oo-jit/pypy/jit/tl/accumulator.tlc ============================================================================== Binary files. No diff available. Modified: pypy/branch/oo-jit/pypy/jit/tl/accumulator.tlc.src ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/accumulator.tlc.src (original) +++ pypy/branch/oo-jit/pypy/jit/tl/accumulator.tlc.src Wed Dec 17 11:47:38 2008 @@ -1,9 +1,27 @@ main: - NEW value,add=add + PUSHARG + PUSH 0 # [n, 0] + LT + BR_COND negative + +positive: + + NEW value,accumulate=add PICK 0 PUSH 0 SETATTR value PUSHARG # [obj, n] + PUSH 1 + BR_COND loop + +negative: + NEW value,accumulate=count + PICK 0 + PUSH 0 + SETATTR value # [obj] + PUSH 0 # [obj, 0] + PUSHARG # [obj, 0, n] + SUB # [obj, -n] loop: # [obj, n] PICK 0 # [obj, n, n] @@ -19,7 +37,7 @@ SUB # [obj, n-1] PICK 1 # [obj, n-1, obj] PICK 1 # [obj, n-1, obj, n-1] - SEND add/1 # [obj, n-1] + SEND accumulate/1 # [obj, n-1] PUSH 1 BR_COND loop @@ -31,3 +49,12 @@ ADD # [self, self.value+x] SETATTR value # [] RETURN + +count: # (x) + PUSHARG # [self] + PUSHARG # [self, self] + GETATTR value # [self, self.value] + PUSH 1 + ADD # [self, self.value+1] + SETATTR value # [] + RETURN Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py Wed Dec 17 11:47:38 2008 @@ -289,6 +289,10 @@ def test_accumulator(self): acc = self.compile('../../accumulator.tlc.src') + assert acc(0) == 0 assert acc(1) == 0 assert acc(10) == sum(range(10)) assert acc(20) == sum(range(20)) + assert acc(-1) == 1 + assert acc(-2) == 2 + assert acc(-10) == 10 From antocuni at codespeak.net Wed Dec 17 12:04:54 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 17 Dec 2008 12:04:54 +0100 (CET) Subject: [pypy-svn] r60534 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081217110454.10290168516@codespeak.net> Author: antocuni Date: Wed Dec 17 12:04:53 2008 New Revision: 60534 Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex Log: more sections about benchmarks Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Wed Dec 17 12:04:53 2008 @@ -80,3 +80,75 @@ have that particular attribute or method. \end{itemize} +In the following sections, we will show some benchmarks that show how our +generated JIT can handle all the features above very well. + +To measure the speedup we get with the JIT, we run each program three times: + +\begin{enumerate} +\item By plain interpretation, without any jitting. +\item With the jit enabled: this run includes the time spent by doing the + compilation itself, plus the time spent by running the produced code. +\item Again with the jit enabled, but this time the compilation has already + been done, so we are actually measuring how good is the code we produced. +\end{enumerate} + +The benchmarks have been run on machine XXX with hardware YYY etc. etc. + +\subsection{Arithmetic operations} + +To benchmark arithmetic operations between integers, we wrote a simple program +that computes the factorial of a given number. The algorithm is +straightforward, and the loop contains only three operations: one +multiplication, one subtraction, and one comparison to check if we have +finished the job. + +When doing plain interpretation, we need to create and destroy three temporary +objects at each iterations. By contrast, the code generated by the JIT does +much better. At the first iteration, the classes of the two operands of the +multiplication are promoted; then, the JIT compiler knows that both are +integers, so it can inline the code to compute the result. Moreover, it can +\emph{virtualize} all the temporary objects, because they never escape from +the inner loop. The same remarks apply to the other two operations inside +the loop. + +As a result, the code executed after the first iteration is close to optimal: +the intermediate values are stored as \lstinline{int} local variables, and the +multiplication, subtraction and \emph{less-than} comparison are mapped to a +single CLI opcode (\lstinline{mul}, \lstinline{sub} and \lstinline{clt}, +respectively). + +Moreover, we also wrote a program to calculate the $n_{th}$ Fibonacci number, +for which we can do the same reasoning as above. + +Table XXX and figure XXX show the time spent to calculate the factorial of +various numbers, with and without the JIT. Table XXX and figure XXX show the +same informations for the Fibonacci program. + +\anto{Should we say that we get wrong results due to overflow but we don't care?} + +As we can see, the code generated by the JIT is almost 500 times faster than +the non-jitted case, and it is only about 1.5 times slower than the same +algorithm written in C\#, which can be considered the optimal goal. + +\subsection{Object-oriented features} + +To measure how the JIT handles object-oriented features, we wrote a very +simple benchmark that involves attribute lookups and method calls. We have an +\emph{accumulator} object, which has a field \lstinline{value} and a method +\lstinline{add}. The method \lstinline{add} takes a parameter and adds it the +field \lstinline{value}. + +XXX: update to the new version + +Our benchmark accepts a paramter \lstinline{n}, create an \emph{accumulator}, +and repeatedly calls \lstinline{add} on it, passing numbers from +\lstinline{n-1} to \lstinline{0}. + +The JIT is able to completely remove the overhead of object creation, +attribute lookups and method calls, and the generated code results in a simple +loop doing additions in-place. + +Table XXX and figure XXX show the time spent to run the benchmark with various +input arguments. Again, we can see that the jitted code is up to 500 times +faster than the interpreted one. From antocuni at codespeak.net Wed Dec 17 12:12:26 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 17 Dec 2008 12:12:26 +0100 (CET) Subject: [pypy-svn] r60535 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081217111226.83BF61684CA@codespeak.net> Author: antocuni Date: Wed Dec 17 12:12:26 2008 New Revision: 60535 Added: pypy/extradoc/talk/ecoop2009/tlc.tex Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex pypy/extradoc/talk/ecoop2009/main.tex Log: move the section about tlc into its own file, and put it just after the introduction Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Wed Dec 17 12:12:26 2008 @@ -1,55 +1,3 @@ -\section{The TLC language} - -\anto{maybe we should move this section somewhere else, if we want to use TLC - as a running example in other sections} -\cfbolz{yes please! how about making it the next section after the -introduction?} - -In this section, we will briefly describe \emph{TLC}, a simple dynamic -language that we developed to exercise our JIT compiler generator. As most of -dynamic languages around, \emph{TLC} is implemented through a virtual machine -that interprets a custom bytecode. Since our main interest is in the runtime -performance of the VM, we did not implement the parser nor the bytecode -compiler, but only the VM itself. - -TLC provides four different types: -\begin{enumerate} -\item Integers -\item \lstinline{nil}, whose only value is the null value -\item Objects -\item Lisp-like lists -\end{enumerate} - -Objects represent a collection of named attributes (much like JavaScript or -Self) and named methods. At creation time, it is necessary to specify the set -of attributes of the object, as well as its methods. Once the object has been -created, it is not possible to add/remove attributes and methods. - -The virtual machine is stack-based, and provides several operations: - -\begin{itemize} -\item \textbf{Stack manipulation}: standard operations to manipulate the - stack, such as \lstinline{PUSH}, \lstinline{POP}, \lstinline{SWAP}, etc. -\item \textbf{Flow control} to do conditional and unconditional jumps. -\item \textbf{Arithmetic}: numerical operations on integers, like - \lstinline{ADD}, \lstinline{SUB}, etc. -\item \textbf{Comparisons} like \lstinline{EQ}, \lstinline{LT}, - \lstinline{GT}, etc. -\item \textbf{Object-oriented}: operations on objects: \lstinline{NEW}, - \lstinline{GETATTR}, \lstinline{SETATTR}, \lstinline{SEND}. -\item \textbf{List operations}: \lstinline{CONS}, \lstinline{CAR}, - \lstinline{CDR}. -\end{itemize} - -Obviously, not all the operations are applicable to all objects. For example, -it is not possible to \lstinline{ADD} an integer and an object, or reading an -attribute from an object which does not provide it. Being a dynamic language, -the VM needs to do all these checks at runtime; in case one of the check -fails, the execution is simply aborted. - -\anto{should we try to invent a syntax for TLC and provide some examples?} -\cfbolz{we should provide an example with the assembler syntax} - \section{Benchmarks} \cfbolz{I think this should go to the beginning of the description of the TLC as Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Wed Dec 17 12:12:26 2008 @@ -83,6 +83,7 @@ \input{abstract} \input{intro} +\input{tlc} \input{jitgen} \input{rainbow} \input{clibackend} Added: pypy/extradoc/talk/ecoop2009/tlc.tex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/tlc.tex Wed Dec 17 12:12:26 2008 @@ -0,0 +1,47 @@ +\section{The TLC language} + +In this section, we will briefly describe \emph{TLC}, a simple dynamic +language that we developed to exercise our JIT compiler generator. As most of +dynamic languages around, \emph{TLC} is implemented through a virtual machine +that interprets a custom bytecode. Since our main interest is in the runtime +performance of the VM, we did not implement the parser nor the bytecode +compiler, but only the VM itself. + +TLC provides four different types: +\begin{enumerate} +\item Integers +\item \lstinline{nil}, whose only value is the null value +\item Objects +\item Lisp-like lists +\end{enumerate} + +Objects represent a collection of named attributes (much like JavaScript or +Self) and named methods. At creation time, it is necessary to specify the set +of attributes of the object, as well as its methods. Once the object has been +created, it is not possible to add/remove attributes and methods. + +The virtual machine is stack-based, and provides several operations: + +\begin{itemize} +\item \textbf{Stack manipulation}: standard operations to manipulate the + stack, such as \lstinline{PUSH}, \lstinline{POP}, \lstinline{SWAP}, etc. +\item \textbf{Flow control} to do conditional and unconditional jumps. +\item \textbf{Arithmetic}: numerical operations on integers, like + \lstinline{ADD}, \lstinline{SUB}, etc. +\item \textbf{Comparisons} like \lstinline{EQ}, \lstinline{LT}, + \lstinline{GT}, etc. +\item \textbf{Object-oriented}: operations on objects: \lstinline{NEW}, + \lstinline{GETATTR}, \lstinline{SETATTR}, \lstinline{SEND}. +\item \textbf{List operations}: \lstinline{CONS}, \lstinline{CAR}, + \lstinline{CDR}. +\end{itemize} + +Obviously, not all the operations are applicable to all objects. For example, +it is not possible to \lstinline{ADD} an integer and an object, or reading an +attribute from an object which does not provide it. Being a dynamic language, +the VM needs to do all these checks at runtime; in case one of the check +fails, the execution is simply aborted. + +\anto{should we try to invent a syntax for TLC and provide some examples?} +\cfbolz{we should provide an example with the assembler syntax} + From antocuni at codespeak.net Wed Dec 17 12:14:06 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 17 Dec 2008 12:14:06 +0100 (CET) Subject: [pypy-svn] r60536 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081217111406.9DA331684CA@codespeak.net> Author: antocuni Date: Wed Dec 17 12:14:06 2008 New Revision: 60536 Modified: pypy/extradoc/talk/ecoop2009/main.tex Log: fix syntax Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Wed Dec 17 12:14:06 2008 @@ -45,7 +45,7 @@ \title{Automatic generation of JIT compilers for dynamic languages in .NET\thanks{This work has been partially supported by MIUR EOS DUE - Extensible Object Systems for Dynamic and -Unpredictable Environments.\cfbolz{should we put the PyPy EU project here as well?}} +Unpredictable Environments.\cfbolz{should we put the PyPy EU project here as well?}}} \author{Davide Ancona\inst{1} \and Carl Friedrich Bolz\inst{2} \and Antonio Cuni\inst{1} \and Armin Rigo} From antocuni at codespeak.net Wed Dec 17 12:26:14 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 17 Dec 2008 12:26:14 +0100 (CET) Subject: [pypy-svn] r60537 - in pypy/branch/oo-jit/pypy/jit/tl: . test Message-ID: <20081217112614.A9A2F168516@codespeak.net> Author: antocuni Date: Wed Dec 17 12:26:11 2008 New Revision: 60537 Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlc.py pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py Log: add unconditional branches, to make examples easier to understand Modified: pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/test/test_tlc.py Wed Dec 17 12:26:11 2008 @@ -36,7 +36,20 @@ def interp(code='', pc=0, inputarg=0): from pypy.jit.tl.tlc import interp return interp(code, pc, inputarg) - + + def test_unconditional_branch(self): + bytecode = compile(""" + main: + BR target + PUSH 123 + RETURN + target: + PUSH 42 + RETURN + """) + res = self.interp(bytecode, 0, 0) + assert res == 42 + def test_basic_cons_cell(self): bytecode = compile(""" NIL Modified: pypy/branch/oo-jit/pypy/jit/tl/tlc.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlc.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlc.py Wed Dec 17 12:26:11 2008 @@ -385,6 +385,10 @@ hint(b, promote_class=True) stack.append(IntObj(not b.lt(a))) + elif opcode == BR: + pc += char2int(code[pc]) + pc += 1 + elif opcode == BR_COND: cond = stack.pop() hint(cond, promote_class=True) Modified: pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py (original) +++ pypy/branch/oo-jit/pypy/jit/tl/tlopcode.py Wed Dec 17 12:26:11 2008 @@ -53,6 +53,7 @@ opcode(33, "PRINT") opcode(34, "DUMP") +opcode(35, "BR") del opcode From antocuni at codespeak.net Wed Dec 17 12:35:09 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 17 Dec 2008 12:35:09 +0100 (CET) Subject: [pypy-svn] r60538 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081217113509.80BDD16851E@codespeak.net> Author: antocuni Date: Wed Dec 17 12:35:09 2008 New Revision: 60538 Modified: pypy/extradoc/talk/ecoop2009/tlc.tex Log: show an example of tlc assembler, and propose a higher level syntax to show more complex examples Modified: pypy/extradoc/talk/ecoop2009/tlc.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/tlc.tex (original) +++ pypy/extradoc/talk/ecoop2009/tlc.tex Wed Dec 17 12:35:09 2008 @@ -42,6 +42,40 @@ the VM needs to do all these checks at runtime; in case one of the check fails, the execution is simply aborted. -\anto{should we try to invent a syntax for TLC and provide some examples?} -\cfbolz{we should provide an example with the assembler syntax} +\subsection{TLC examples} +As we said above, TLC exists only at bytecode level; to ease the development +of TLC programs, we wrote an assembler that generates TLC bytecode. The +following example shows a simple program that computes the absolute value of +the given integer: + +\begin{lstlisting} +main: # stack: [] + PUSHARG # [n] + PUSH 0 # [n, 0] + LT # [n<0] + BR_COND neg + +pos: # [] + PUSHARG # [n] + RETURN + +neg: + PUSH 0 # [0] + PUSHARG # [0,n] + SUB # [-n] + RETURN +\end{lstlisting} + +Since reading TLC programs at bytecode level is hard, in this paper we will +use an invented Python-like syntax to describe examples, even if we need to +remind that the actual programs are written in the assembler language showed +above. The following listing shows the same example as above written in the +Python-like syntax: + +\begin{lstlisting} +def main(n): + if n<0: + return -n + return n +\end{lstlisting} From antocuni at codespeak.net Wed Dec 17 14:32:47 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 17 Dec 2008 14:32:47 +0100 (CET) Subject: [pypy-svn] r60539 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081217133247.51F8A168439@codespeak.net> Author: antocuni Date: Wed Dec 17 14:32:45 2008 New Revision: 60539 Modified: pypy/extradoc/talk/ecoop2009/tlc.tex Log: minor language tweaks Modified: pypy/extradoc/talk/ecoop2009/tlc.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/tlc.tex (original) +++ pypy/extradoc/talk/ecoop2009/tlc.tex Wed Dec 17 14:32:45 2008 @@ -38,7 +38,7 @@ Obviously, not all the operations are applicable to all objects. For example, it is not possible to \lstinline{ADD} an integer and an object, or reading an -attribute from an object which does not provide it. Being a dynamic language, +attribute from an object which does not provide it. Being dynamically typed, the VM needs to do all these checks at runtime; in case one of the check fails, the execution is simply aborted. @@ -70,8 +70,7 @@ Since reading TLC programs at bytecode level is hard, in this paper we will use an invented Python-like syntax to describe examples, even if we need to remind that the actual programs are written in the assembler language showed -above. The following listing shows the same example as above written in the -Python-like syntax: +above. Thus, the example above can be written in the following way: \begin{lstlisting} def main(n): From antocuni at codespeak.net Wed Dec 17 14:53:38 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 17 Dec 2008 14:53:38 +0100 (CET) Subject: [pypy-svn] r60540 - in pypy/extradoc/talk/ecoop2009: . benchmarks Message-ID: <20081217135338.4B2E2168441@codespeak.net> Author: antocuni Date: Wed Dec 17 14:53:36 2008 New Revision: 60540 Added: pypy/extradoc/talk/ecoop2009/benchmarks/ pypy/extradoc/talk/ecoop2009/benchmarks/factorial.cs pypy/extradoc/talk/ecoop2009/benchmarks/factorial.tlc - copied unchanged from r60489, pypy/branch/oo-jit/pypy/jit/tl/factorial.tlc pypy/extradoc/talk/ecoop2009/benchmarks/fibo.cs pypy/extradoc/talk/ecoop2009/benchmarks/fibo.tlc - copied unchanged from r60525, pypy/branch/oo-jit/pypy/jit/tl/fibo.tlc Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex pypy/extradoc/talk/ecoop2009/tlc.tex Log: move the paragraph about tlc features to the TLC section, and add the benchmarks written in C# Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Wed Dec 17 14:53:36 2008 @@ -1,35 +1,12 @@ \section{Benchmarks} -\cfbolz{I think this should go to the beginning of the description of the TLC as -it explains why it is written as it is written} - -Despite being very simple and minimalistic, \lstinline{TLC} is a good -candidate as a language to run benchmarks, as it has some of the features that -makes most of current dynamic languages so slow: - -\begin{itemize} - -\item \textbf{Stack based VM}: this kind of VM requires all the operands to be - on top of the evaluation stack. As a consequence programs spend a lot of - time pushing and popping values to/from the stack, or doing other stack - related operations. However, thanks to its simplicity this is still the - most common and preferred way to implement VMs. - -\item \textbf{Boxed integers}: integer objects are internally represented as - an instance of the \lstinline{IntObj} class, whose field \lstinline{value} - contains the real value. By having boxed integers, common arithmetic - operations are made very slow, because each time we want to load/store their - value we need to go through an extra level of indirection. Moreover, in - case of a complex expression, it is necessary to create many temporary - objects to hold intermediate results. - -\item \textbf{Dynamic lookup}: attributes and methods are looked up at - runtime, because there is no way to know in advance if and where an object - have that particular attribute or method. -\end{itemize} +In section \ref{sec:tlc-features}, we saw that TLC provides most of the +features that usaully make dynamically typed language so slow, such as +\emph{stack-based VM}, \emph{boxed arithmetic} and \emph{dynamic lookup} of +methods and attributes. In the following sections, we will show some benchmarks that show how our -generated JIT can handle all the features above very well. +generated JIT can handle all these features very well. To measure the speedup we get with the JIT, we run each program three times: @@ -41,6 +18,12 @@ been done, so we are actually measuring how good is the code we produced. \end{enumerate} +Moreover, for each benchmark we also show the time taken by running the +equivalent program written in C\#. By comparing the results against C\#, we +can see how far we are from the supposed optimal performances. \anto{I + don't really like the last sentence, but right now I can't think of another + way to phrase it. Rewording welcome :-)} + The benchmarks have been run on machine XXX with hardware YYY etc. etc. \subsection{Arithmetic operations} @@ -77,7 +60,10 @@ As we can see, the code generated by the JIT is almost 500 times faster than the non-jitted case, and it is only about 1.5 times slower than the same -algorithm written in C\#, which can be considered the optimal goal. +algorithm written in C\#: the difference in speed it is probably due to both +the fact that the current CLI backend emits slightly non-optimal code and that +the underyling .NET JIT compiler is highly optimized to handle bytecode +generated by C\# compilers. \subsection{Object-oriented features} Added: pypy/extradoc/talk/ecoop2009/benchmarks/factorial.cs ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/benchmarks/factorial.cs Wed Dec 17 14:53:36 2008 @@ -0,0 +1,23 @@ +using System; + +class Factorial +{ + public static void Main(string[] args) + { + int n = Convert.ToInt32(args[0]); + DateTime start, stop; + start = DateTime.UtcNow; + int res = factorial(n); + stop = DateTime.UtcNow; + double secs = (stop-start).TotalSeconds; + Console.WriteLine("C#: {0} ({1} seconds)", res, secs); + } + + public static int factorial(int n) + { + int res=1; + for(int i=1; i<=n; i++) + res *= i; + return res; + } +} Added: pypy/extradoc/talk/ecoop2009/benchmarks/fibo.cs ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/benchmarks/fibo.cs Wed Dec 17 14:53:36 2008 @@ -0,0 +1,27 @@ +using System; + +class Fibo +{ + public static void Main(string[] args) + { + int n = Convert.ToInt32(args[0]); + DateTime start, stop; + start = DateTime.UtcNow; + int res = fibo(n); + stop = DateTime.UtcNow; + double secs = (stop-start).TotalSeconds; + Console.WriteLine("C#: {0} ({1} seconds)", res, secs); + } + + public static int fibo(int n) + { + int a = 0; + int b = 1; + while (--n > 0) { + int sum = a+b; + a = b; + b = sum; + } + return b; + } +} Modified: pypy/extradoc/talk/ecoop2009/tlc.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/tlc.tex (original) +++ pypy/extradoc/talk/ecoop2009/tlc.tex Wed Dec 17 14:53:36 2008 @@ -42,6 +42,35 @@ the VM needs to do all these checks at runtime; in case one of the check fails, the execution is simply aborted. +\subsection{TLC features} +\label{sec:tlc-features} + +Despite being very simple and minimalistic, \lstinline{TLC} is a good +candidate as a language to test our JIT generator, as it has some of the +features that makes most of current dynamic languages so slow: + +\begin{itemize} + +\item \textbf{Stack based VM}: this kind of VM requires all the operands to be + on top of the evaluation stack. As a consequence programs spend a lot of + time pushing and popping values to/from the stack, or doing other stack + related operations. However, thanks to its simplicity this is still the + most common and preferred way to implement VMs. + +\item \textbf{Boxed integers}: integer objects are internally represented as + an instance of the \lstinline{IntObj} class, whose field \lstinline{value} + contains the real value. By having boxed integers, common arithmetic + operations are made very slow, because each time we want to load/store their + value we need to go through an extra level of indirection. Moreover, in + case of a complex expression, it is necessary to create many temporary + objects to hold intermediate results. + +\item \textbf{Dynamic lookup}: attributes and methods are looked up at + runtime, because there is no way to know in advance if and where an object + have that particular attribute or method. +\end{itemize} + + \subsection{TLC examples} As we said above, TLC exists only at bytecode level; to ease the development From antocuni at codespeak.net Wed Dec 17 15:03:00 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 17 Dec 2008 15:03:00 +0100 (CET) Subject: [pypy-svn] r60541 - pypy/extradoc/talk/ecoop2009/benchmarks Message-ID: <20081217140300.0233F168446@codespeak.net> Author: antocuni Date: Wed Dec 17 15:02:59 2008 New Revision: 60541 Added: pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cs pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.tlc - copied unchanged from r60533, pypy/branch/oo-jit/pypy/jit/tl/accumulator.tlc Log: c# version of the accumulator benchmark Added: pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cs ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cs Wed Dec 17 15:02:59 2008 @@ -0,0 +1,64 @@ +using System; + +interface Accumulator +{ + void accumulate(int x); + int getvalue(); +} + +class Add: Accumulator +{ + public int value = 0; + public void accumulate(int x) + { + value += x; + } + public int getvalue() + { + return value; + } +} + +class Count: Accumulator +{ + public int value = 0; + public void accumulate(int x) + { + value++; + } + public int getvalue() + { + return value; + } +} + +class Test +{ + public static void Main(string[] args) + { + int n = Convert.ToInt32(args[0]); + DateTime start, stop; + start = DateTime.UtcNow; + int res = accumulator(n); + stop = DateTime.UtcNow; + double secs = (stop-start).TotalSeconds; + Console.WriteLine("C#: {0} ({1} seconds)", res, secs); + } + + public static int accumulator(int n) + { + Accumulator acc = null; + if (n < 0) { + n = -n; + acc = new Count(); + } + else { + acc = new Add(); + } + + while (n-- > 0) { + acc.accumulate(n); + } + return acc.getvalue(); + } +} From fijal at codespeak.net Wed Dec 17 15:39:34 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 17 Dec 2008 15:39:34 +0100 (CET) Subject: [pypy-svn] r60542 - pypy/extradoc/talk/ecoop2009/benchmarks Message-ID: <20081217143934.AEF99168442@codespeak.net> Author: fijal Date: Wed Dec 17 15:39:32 2008 New Revision: 60542 Added: pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cpp Log: incomplete c++ solution for accumulator benchmark Added: pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cpp ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cpp Wed Dec 17 15:39:32 2008 @@ -0,0 +1,73 @@ + +#include +#include + +class Accumulator +{ + public: + virtual void accumulate(int x); + virtual int getvalue(); +}; + +class Add : public Accumulator +{ + int value; +public: + Add() + { + value = 0; + } + + void accumulate(int x) + { + value += x; + } + int getvalue() + { + return value; + } +}; + +class Count : public Accumulator +{ + int value; +public: + Count() + { + value = 0; + } + void accumulate(int x) + { + value++; + } + int getvalue() + { + return value; + } +}; + +int accumulator(int n) +{ + Accumulator* acc = NULL; + int res; + + if (n < 0) { + n = -n; + acc = new Count(); + } else { + acc = new Add(); + } + while (n-- > 0) { + acc->accumulate(n); + } + res = acc->getvalue(); + delete acc; + return res; +} + + +int main(int argc, char **argv) +{ + int arg; + arg = atoi(argv[1]); +} From fijal at codespeak.net Wed Dec 17 15:46:50 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 17 Dec 2008 15:46:50 +0100 (CET) Subject: [pypy-svn] r60543 - pypy/extradoc/talk/ecoop2009/benchmarks Message-ID: <20081217144650.C8BCD168442@codespeak.net> Author: fijal Date: Wed Dec 17 15:46:50 2008 New Revision: 60543 Modified: pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cpp Log: now I remember... Modified: pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cpp ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cpp (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cpp Wed Dec 17 15:46:50 2008 @@ -5,8 +5,8 @@ class Accumulator { public: - virtual void accumulate(int x); - virtual int getvalue(); + virtual void accumulate(int x) = 0; + virtual int getvalue() = 0; }; class Add : public Accumulator @@ -18,11 +18,11 @@ value = 0; } - void accumulate(int x) + virtual void accumulate(int x) { value += x; } - int getvalue() + virtual int getvalue() { return value; } @@ -36,11 +36,11 @@ { value = 0; } - void accumulate(int x) + virtual void accumulate(int x) { value++; } - int getvalue() + virtual int getvalue() { return value; } From fijal at codespeak.net Wed Dec 17 16:02:24 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 17 Dec 2008 16:02:24 +0100 (CET) Subject: [pypy-svn] r60544 - pypy/extradoc/talk/ecoop2009/benchmarks Message-ID: <20081217150224.36AF0168446@codespeak.net> Author: fijal Date: Wed Dec 17 16:02:22 2008 New Revision: 60544 Modified: pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cpp Log: working version Modified: pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cpp ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cpp (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cpp Wed Dec 17 16:02:22 2008 @@ -1,6 +1,7 @@ #include #include +#include class Accumulator { @@ -68,6 +69,15 @@ int main(int argc, char **argv) { - int arg; + int arg, res; + struct timeval t0, t1; + double time; + arg = atoi(argv[1]); + gettimeofday(&t0, NULL); + res = accumulator(arg); + gettimeofday(&t1, NULL); + time = ((t1.tv_sec + ((float)t1.tv_usec / 1000000)) - + (t0.tv_sec + ((float)t0.tv_usec / 1000000))); + std::cout << "C++ " << arg << " " << time << "\n"; } From fijal at codespeak.net Wed Dec 17 16:04:43 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 17 Dec 2008 16:04:43 +0100 (CET) Subject: [pypy-svn] r60545 - pypy/extradoc/talk/ecoop2009/benchmarks Message-ID: <20081217150443.5E8F016842F@codespeak.net> Author: fijal Date: Wed Dec 17 16:04:42 2008 New Revision: 60545 Modified: pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cpp Log: oops, typo Modified: pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cpp ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cpp (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cpp Wed Dec 17 16:04:42 2008 @@ -79,5 +79,5 @@ gettimeofday(&t1, NULL); time = ((t1.tv_sec + ((float)t1.tv_usec / 1000000)) - (t0.tv_sec + ((float)t0.tv_usec / 1000000))); - std::cout << "C++ " << arg << " " << time << "\n"; + std::cout << "C++ " << res << " " << time << "\n"; } From pedronis at codespeak.net Wed Dec 17 17:35:36 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 17 Dec 2008 17:35:36 +0100 (CET) Subject: [pypy-svn] r60547 - in pypy/build/testrunner: . test Message-ID: <20081217163536.47242168467@codespeak.net> Author: pedronis Date: Wed Dec 17 17:35:34 2008 New Revision: 60547 Modified: pypy/build/testrunner/runner.py pypy/build/testrunner/test/test_runner.py Log: more error handling Modified: pypy/build/testrunner/runner.py ============================================================================== --- pypy/build/testrunner/runner.py (original) +++ pypy/build/testrunner/runner.py Wed Dec 17 17:35:34 2008 @@ -24,6 +24,7 @@ except OSError: pass +EXECUTEFAILED = -1001 RUNFAILED = -1000 TIMEDOUT = -999 @@ -99,6 +100,8 @@ msg = "TIMEOUT" elif exitcode == RUNFAILED: msg = "Failed to run interp" + elif exitcode == EXECUTEFAILED: + msg = "Failed with exception in execute-test" else: msg = "Killed by %s." % getsignalname(-exitcode) extralog = "! %s\n %s\n" % (test, msg) @@ -127,14 +130,23 @@ one_output = sessdir.join("%d-%s-output" % (num, basename)) num += n - test_driver = get_test_driver(test) - exitcode = execute_test(root, test, one_output, logfname, - interp, test_driver, do_dry_run=dry_run, - timeout=timeout) + try: + test_driver = get_test_driver(test) + exitcode = execute_test(root, test, one_output, logfname, + interp, test_driver, do_dry_run=dry_run, + timeout=timeout) + + cleanup(test) + except: + print "execute-test for %r failed with:" % test + import traceback + traceback.print_exc() + exitcode = EXECUTEFAILED - cleanup(test) - - output = one_output.read() + if one_output.check(file=1): + output = one_output.read() + else: + output = "" if logfname.check(file=1): logdata = logfname.read() else: Modified: pypy/build/testrunner/test/test_runner.py ============================================================================== --- pypy/build/testrunner/test/test_runner.py (original) +++ pypy/build/testrunner/test/test_runner.py Wed Dec 17 17:35:34 2008 @@ -275,7 +275,7 @@ out = cStringIO.StringIO() run_param = runner.RunParam(self.one_test_dir) - run_param.interp=['wrong-interp'] + run_param.interp = ['wrong-interp'] run_param.parallel_runs = 3 testdirs = [] @@ -286,6 +286,27 @@ log_lines = log.getvalue().splitlines() assert log_lines[1] == ' Failed to run interp' + def test_run_bad_get_test_driver(self): + test_driver = [py.path.local(py.__file__).dirpath('bin', 'py.test')] + + log = cStringIO.StringIO() + out = cStringIO.StringIO() + + run_param = runner.RunParam(self.one_test_dir) + run_param.parallel_runs = 3 + def boom(testdir): + raise RuntimeError("Boom") + run_param.get_test_driver = boom + + testdirs = [] + run_param.collect_testdirs(testdirs) + res = runner.execute_tests(run_param, testdirs, log, out) + assert res + + log_lines = log.getvalue().splitlines() + assert log_lines[1] == ' Failed with exception in execute-test' + + class TestRunnerNoThreads(RunnerTests): with_thread = False From antocuni at codespeak.net Wed Dec 17 18:05:09 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 17 Dec 2008 18:05:09 +0100 (CET) Subject: [pypy-svn] r60548 - pypy/branch/oo-jit/pypy/translator/cli/src Message-ID: <20081217170509.069D316843C@codespeak.net> Author: antocuni Date: Wed Dec 17 18:05:08 2008 New Revision: 60548 Modified: pypy/branch/oo-jit/pypy/translator/cli/src/pypylib.cs Log: try to make time.clock() more precise Modified: pypy/branch/oo-jit/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/branch/oo-jit/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/branch/oo-jit/pypy/translator/cli/src/pypylib.cs Wed Dec 17 18:05:08 2008 @@ -1005,10 +1005,9 @@ return t.TotalSeconds; } - static DateTime ClockStart = DateTime.UtcNow; public static double ll_time_clock() { - return (DateTime.UtcNow - ClockStart).TotalSeconds; + return DateTime.UtcNow.Ticks * 1e-7; } public static void ll_time_sleep(double seconds) From antocuni at codespeak.net Wed Dec 17 18:14:34 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 17 Dec 2008 18:14:34 +0100 (CET) Subject: [pypy-svn] r60549 - pypy/extradoc/talk/ecoop2009/benchmarks Message-ID: <20081217171434.7B7CE16846D@codespeak.net> Author: antocuni Date: Wed Dec 17 18:14:34 2008 New Revision: 60549 Modified: pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cs pypy/extradoc/talk/ecoop2009/benchmarks/factorial.cs pypy/extradoc/talk/ecoop2009/benchmarks/fibo.cs Log: use higher precision timer also here Modified: pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cs ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cs (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.cs Wed Dec 17 18:14:34 2008 @@ -37,11 +37,11 @@ public static void Main(string[] args) { int n = Convert.ToInt32(args[0]); - DateTime start, stop; - start = DateTime.UtcNow; + long start, stop; + start = DateTime.UtcNow.Ticks; int res = accumulator(n); - stop = DateTime.UtcNow; - double secs = (stop-start).TotalSeconds; + stop = DateTime.UtcNow.Ticks; + double secs = (stop-start) * 1e-7; Console.WriteLine("C#: {0} ({1} seconds)", res, secs); } Modified: pypy/extradoc/talk/ecoop2009/benchmarks/factorial.cs ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks/factorial.cs (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks/factorial.cs Wed Dec 17 18:14:34 2008 @@ -5,11 +5,12 @@ public static void Main(string[] args) { int n = Convert.ToInt32(args[0]); - DateTime start, stop; - start = DateTime.UtcNow; + + long start, stop; + start = DateTime.UtcNow.Ticks; int res = factorial(n); - stop = DateTime.UtcNow; - double secs = (stop-start).TotalSeconds; + stop = DateTime.UtcNow.Ticks; + double secs = (stop-start) * 1e-7; Console.WriteLine("C#: {0} ({1} seconds)", res, secs); } Modified: pypy/extradoc/talk/ecoop2009/benchmarks/fibo.cs ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks/fibo.cs (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks/fibo.cs Wed Dec 17 18:14:34 2008 @@ -5,11 +5,11 @@ public static void Main(string[] args) { int n = Convert.ToInt32(args[0]); - DateTime start, stop; - start = DateTime.UtcNow; + long start, stop; + start = DateTime.UtcNow.Ticks; int res = fibo(n); - stop = DateTime.UtcNow; - double secs = (stop-start).TotalSeconds; + stop = DateTime.UtcNow.Ticks; + double secs = (stop-start) * 1e-7; Console.WriteLine("C#: {0} ({1} seconds)", res, secs); } From antocuni at codespeak.net Wed Dec 17 18:46:27 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 17 Dec 2008 18:46:27 +0100 (CET) Subject: [pypy-svn] r60550 - pypy/extradoc/talk/ecoop2009/benchmarks Message-ID: <20081217174627.00CF61684B6@codespeak.net> Author: antocuni Date: Wed Dec 17 18:46:25 2008 New Revision: 60550 Added: pypy/extradoc/talk/ecoop2009/benchmarks/results.txt Log: raw benchmark results Added: pypy/extradoc/talk/ecoop2009/benchmarks/results.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/benchmarks/results.txt Wed Dec 17 18:46:25 2008 @@ -0,0 +1,21 @@ +Factorial +N Interp JIT1 JIT2 C# +10 0,03125 0,421875 0 0 +10000000 30,98438 0,453132 0,046875 0,03125 +100000000 N/A 0,859375 0,453125 0,35937 +1000000000 N/A 4,84375 4,640625 3,4375 + + +Fibonacci +N Interp JIT1 JIT2 C# +10 0,0312576 0,45313 0 0 +10000000 29,359367 0,46875 0,015617 0,015625 +100000000 N/A 0,6875 0,25001 0,234375 +1000000000 N/A 2,9531 2,5 2,453125 + +Accumulator +N Interp JIT1 JIT2 C# +10 0,03125 0,453132 0 0 +10000000 43,0625 0,515625 0,04687 0,0625 +100000000 N/A 0,87500 0,453125 0,5625 +1000000000 N/A 4,18750 3,67187 5,953125 From antocuni at codespeak.net Wed Dec 17 18:59:32 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 17 Dec 2008 18:59:32 +0100 (CET) Subject: [pypy-svn] r60552 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081217175932.2996A168464@codespeak.net> Author: antocuni Date: Wed Dec 17 18:59:30 2008 New Revision: 60552 Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex Log: add high level syntax of two benchmarks. Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Wed Dec 17 18:59:30 2008 @@ -24,7 +24,9 @@ don't really like the last sentence, but right now I can't think of another way to phrase it. Rewording welcome :-)} -The benchmarks have been run on machine XXX with hardware YYY etc. etc. +The benchmarks have been run on machine with Intel Pentium 4 CPU running at +3.20 GHz and 2 GB of RAM, running Microsoft Windows XP and Microsoft .NET +Framework 2.0. \subsection{Arithmetic operations} @@ -32,7 +34,16 @@ that computes the factorial of a given number. The algorithm is straightforward, and the loop contains only three operations: one multiplication, one subtraction, and one comparison to check if we have -finished the job. +finished the job: + +\begin{lstlisting} +def main(n): + result = 1 + while n > 1: + result = result * n + n = n - 1 + return n +\end{lstlisting} When doing plain interpretation, we need to create and destroy three temporary objects at each iterations. By contrast, the code generated by the JIT does @@ -49,14 +60,31 @@ single CLI opcode (\lstinline{mul}, \lstinline{sub} and \lstinline{clt}, respectively). -Moreover, we also wrote a program to calculate the $n_{th}$ Fibonacci number, -for which we can do the same reasoning as above. +Similarly, we wrote a program to calculate the $n_{th}$ Fibonacci number, for +which we can do the same reasoning as above: + +\begin{lstlisting} +def main(n): + a = 0 + b = 1 + while n > 1: + sum = a + b + a = b + b = sum + n = n - 1 + return b +\end{lstlisting} + + Table XXX and figure XXX show the time spent to calculate the factorial of various numbers, with and without the JIT. Table XXX and figure XXX show the same informations for the Fibonacci program. -\anto{Should we say that we get wrong results due to overflow but we don't care?} +Note that do get meaningful timings, we had to calculate the factorial and +Fibonacci of very high numbers. This means that the results are incorrect due +to overflow, but since all the runnings overflows in the very same way, the +timings are still comparable. \anto{I think we should rephrase this sentence}. As we can see, the code generated by the JIT is almost 500 times faster than the non-jitted case, and it is only about 1.5 times slower than the same From antocuni at codespeak.net Wed Dec 17 19:01:34 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 17 Dec 2008 19:01:34 +0100 (CET) Subject: [pypy-svn] r60553 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081217180134.D496C168454@codespeak.net> Author: antocuni Date: Wed Dec 17 19:01:33 2008 New Revision: 60553 Modified: pypy/extradoc/talk/ecoop2009/main.tex Log: answer to a comment Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Wed Dec 17 19:01:33 2008 @@ -45,7 +45,7 @@ \title{Automatic generation of JIT compilers for dynamic languages in .NET\thanks{This work has been partially supported by MIUR EOS DUE - Extensible Object Systems for Dynamic and -Unpredictable Environments.\cfbolz{should we put the PyPy EU project here as well?}}} +Unpredictable Environments.\cfbolz{should we put the PyPy EU project here as well?}\anto{I think so}}} \author{Davide Ancona\inst{1} \and Carl Friedrich Bolz\inst{2} \and Antonio Cuni\inst{1} \and Armin Rigo} From davide at codespeak.net Thu Dec 18 00:12:59 2008 From: davide at codespeak.net (davide at codespeak.net) Date: Thu, 18 Dec 2008 00:12:59 +0100 (CET) Subject: [pypy-svn] r60556 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081217231259.CD530168449@codespeak.net> Author: davide Date: Thu Dec 18 00:12:59 2008 New Revision: 60556 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex Log: management of external links almost finished Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Thu Dec 18 00:12:59 2008 @@ -62,14 +62,14 @@ \begin{itemize} \item Each flow graph is translated in a collection of methods which can grow dynamically. \dacom{I propose primary/secondary instead of - the overloaded terms main/child} Each collection must contain at least one + the overloaded terms main/child} Each collection contains at least one method, called \emph{primary}, which is the first to be created. All other methods, called \emph{secondary}, are added dynamically whenever a new case is added to a flexswitch. -\item Each either primary or secondary method corresponds to the - translation of some of the blocks of a single flow graph. Each - method has an initial block whose input variables are the +\item Each either primary or secondary method implements a certain + number of blocks, all belonging to the same flow graph. Among these blocks + there always exists an initial block whose input variables are parameters of the method; the input variables of all other blocks are local variables of the method. \end{itemize} @@ -81,10 +81,11 @@ \subsubsection{Internal and external links} -A link is called \emph{internal} if it connects two blocks implemented in the same method, +A link is called \emph{internal} if it connects two blocks implemented +by the same method, \emph{external} otherwise. -Following an internal link is not difficult in CLI bytecode: a jump to +Following an internal link is not difficult in IL bytecode: a jump to the corresponding code fragment in the same method is emitted to execute the new block, whereas the appropriate local variables are used for passing arguments. @@ -98,15 +99,15 @@ picture of Figure~\ref{flexswitch-fig}. How it is possible to pass the right arguments to the target block? -To solve this problem a special block, called \emph{dispatcher}, is -added to every method; whenever a method is invoked, its dispatcher is -executed first\footnote{Recall that the dispatcher is a special block -and must not be confused with the initial block of a method.} to +To solve this problem every method contains a special code, called +\emph{dispatcher}; whenever a method is invoked, its dispatcher is +executed first\footnote{The dispatcher should not be +confused with the initial block of a method.} to determine which block has to be executed. This is done by passing to the method a 32 bits number, called \emph{block id}, which uniquely identifies the next block of the graph to be executed. The high word of a block id is the id of the method to which the block -belongs, whereas the low word is a progressive number identifying +belongs, whereas the low word is a progressive number univocally identifying each block implemented by the method. The picture in Figure~\ref{block-id-fig} shows a graph composed of three methods (for @@ -122,50 +123,61 @@ \end{center} \end{figure} -\commentout{ -Each method in a graph is assigned an unique 16 bit method id; each block in a method is assigned a progressive 16 bit block number. From this two numbers, we can compute the block id as an unsigned integer, by storing the method id in the first 16 bits and the block number in the second 16 bits. By construction, the block id is guaranteed to be unique in the graph. - -The following picture shows a graph composed of three methods; the id of each method is shown in red, while the block ids are shown in red (for the method id part) and black (for the block number part). The graph contains three external links; in particular, note the link between blocks 0x00020001 and 0x00010001 which connects two block that resides in different methods. - -Every method contains a special dispatch block, (not shown in the picture above) whose goal is to jump to the specified block number inside the method itself. The first argument of a secondary method is always a block id; when the method starts, it immediately jumps to the dispatch block, and thus to the desired block. - -For example, suppose to have a method which contains 3 blocks numbered 0, 1, 2; here is how its dispatch blocks looks like; for simplicity it is shown as C# code, but it is actually generated as IL bytecode: - +For instance, the code\footnote{For simplicity we write C\# code instead of +the actual IL bytecode.} generated for the dispatcher of method \texttt{0x0002} +is similar to the following fragment: +\begin{small} +\begin{lstlisting}[language={[Sharp]C}] // dispatch block -int methodid = (blockid & 0xFFFF0000) >> 16); // take the first 16 bits -int blocknum = blockid && 0x0000FFFF; // take the second 16 bits - +int methodid = (blockid && 0xFFFF0000) >> 16; +int blocknum = blockid && 0x0000FFFF; if (methodid != MY_METHOD_ID) { -// jump_to_unknown block -... + // jump_to_ext + ... } - switch(blocknum) { -case 0: -goto block0; -case 1: -goto block1; -case 2: -goto block2; -default: -throw new Exception("Invalid block id"); -} - -Whenever we want to jump to a external block, it is enough to store the block id in the appropriate variable and jump to the dispatch block. If the block resides in a different method, the jump_to_unknown block is entered; this special block is implemented differently by the main method and the secondary methods, as we will see soon. - -Each time a new method is added to the graph, we build a delegate for it, and store it in a special array called method_map; since we assign the method id sequentially starting from 0, we are sure that to fetch the method whose id is n we can simply load the n-th element of the array. - -The jump_to_unknown block of the main method uses this array to select the right method, and calls it (FlexSwitchCase is the type of delegates for all secondary methods): - -// jump_to_unknown block of the main method + case 0: goto block0; + case 1: goto block1; + default: throw new Exception("Invalid block id"); +} +\end{lstlisting} +\end{small} +If the next block to be executed is implemented in the same method +({\small\lstinline{methodid == MY_METHOD_ID}}), then the appropriate +jump to the corresponding code is executed. Otherwise, the \lstinline{jump_to_ext} +part of the dispatcher has to be executed. +The code that actually jumps to an external block is contained in +the dispatcher of the primary method, whereas the +\lstinline{jump_to_ext} code of dispatchers of secondary methods +simply delegates the dispatcher of the primary method of the same +graph (see later). + +The primary method is responsible for the bookkeeping of the secondary +methods which are added to the same graph dynamically. This can be +simply implemented with an array mapping method id of secondary methods +to the corresponding delegate. Therefore, the primary methods contain +the following \lstinline{jump_to_ext} code (where +\lstinline{FlexSwitchCase} is the type of delegates for secondary methods): +\begin{small} +\begin{lstlisting}[language={[Sharp]C}] +// jump_to_ext FlexSwitchCase meth = method_map[methodid]; blockid = meth(blockid, ...); // execute the method goto dispatch_block; +\end{lstlisting} +\end{small} +Each secondary method returns the block id of the next block to be +executed; therefore, after the secondary method has returned, the +dispatcher of the primary method will be executed again to jump +to the correct next block. + +To avoid mutual recursion and an undesired growth of the stack, +the \lstinline{jump_to_ext} code in dispatchers of secondary methods +just returns the block id of the next block; since the primary method +is always the first method of the graph which is called, the correct +jump will be eventually executed by the dispatcher of the primary method. -Each secondary method returns a block id specifying the next block to jump to; after its execution, we assign the return value to the blockid variable, and jump again to the dispatch block, which will jump again to the appropriate block. - -Keeping this in mind, it is straightforward to implement the jump_to_unknown block of secondary methods: it is enough to return the target block id to the caller, and let its dispatch loop do the right thing. If the caller is also a secondary method, it will return it again, until we reach the dispatch loop of the main method, which will finally do the jump. In theory, we could implement things differently and jumping directly from a secondary method to another one, but in that case the call stack could grows indefinitely in case of a tight loop between two blocks residing in different methods. - +\commentout{ To implement the dispatch block we can exploit the switch opcode of the CLI; if the .NET JIT is smart enough, it can render it using an indirect jump; overall, jumping to a external block consists of an indirect function call (by invoking the delegate) plus an indirect jump (by executing the switch opcode); even if this is more costly than a simple direct jump, we will see in the next section that this not the main source of overhead when following a external link. Obviously, the slow dispatching logic is needed only when we want to jump to a external block; if the target block happens to reside in the same method as the current one, we can directly jump to it, completely removing the overhead. @@ -265,4 +277,5 @@ At the moment, the CLI JIT backend is almost complete, and all the hardest problems seems to be solved; the next step is to fix all the remaining bugs and implement some minor feature that it's still missing, then try to apply it to the full Python language and see what is the outcome. } -% LocalWords: flexswitches backend flexswitch +% LocalWords: flexswitches backend flexswitch methodid blockid xFFFF blocknum +% LocalWords: FFFF goto FlexSwitchCase meth From antocuni at codespeak.net Thu Dec 18 11:26:05 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 18 Dec 2008 11:26:05 +0100 (CET) Subject: [pypy-svn] r60560 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081218102605.AC61916842F@codespeak.net> Author: antocuni Date: Thu Dec 18 11:26:03 2008 New Revision: 60560 Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex pypy/extradoc/talk/ecoop2009/tlc.tex Log: finish benchmark section, only the tables are missing Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Thu Dec 18 11:26:03 2008 @@ -96,21 +96,71 @@ \subsection{Object-oriented features} To measure how the JIT handles object-oriented features, we wrote a very -simple benchmark that involves attribute lookups and method calls. We have an -\emph{accumulator} object, which has a field \lstinline{value} and a method -\lstinline{add}. The method \lstinline{add} takes a parameter and adds it the -field \lstinline{value}. - -XXX: update to the new version - -Our benchmark accepts a paramter \lstinline{n}, create an \emph{accumulator}, -and repeatedly calls \lstinline{add} on it, passing numbers from -\lstinline{n-1} to \lstinline{0}. - -The JIT is able to completely remove the overhead of object creation, -attribute lookups and method calls, and the generated code results in a simple -loop doing additions in-place. - -Table XXX and figure XXX show the time spent to run the benchmark with various -input arguments. Again, we can see that the jitted code is up to 500 times -faster than the interpreted one. +simple benchmark that involves attribute lookups and polymorphic method calls: + +\begin{lstlisting} +def main(n): + if n < 0: + n = -n + obj = new(value, accumulate=count) + else: + obj = new(value, accumulate=add) + obj.value = 0 + while n > 0: + n = n - 1 + obj.accumulate(n) + return obj.value + +def count(x): + this.value = this.value + 1 + +def add(x): + this.value = this.value + x +\end{lstlisting} + +The two \lstinline{new} operations create an object with exactly one field +\lstinline{value} and one method \lstinline{accumulate}, whose implementation +is found in the functions \lstinline{count} and \lstinline{add}, respectively. +When calling a method, the receiver is implicity passed and can be accessed +through the special name \lstinline{this}. + +The computation \emph{per se} is trivial, as it calculates either $-n$ or +$1+2...+n-1$, depending on the sign of $n$. The interesting part is the +polymorphic call to \lstinline{accumulate} inside the loop, because the VM has +no way to know in advance which method to call (unless it does flow analysis, +which could be feasible in this case but not in general). The equivalent C\# +code we wrote uses two classes and a \lstinline{virtual} method call to +implement this behaviour. + +However, our generated JIT does not compile the whole function at +once. Instead, it compiles and executes code chunk by chunk, waiting until it +knows enough informations to generate highly efficient code. In particualr, +at the time when it emits the code for the inner loop it exactly knows the +type of \lstinline{obj}, thus it can remove the overhead of dynamic dispatch +and inline the method call. Moreover, since \lstinline{obj} never escapes the +function, it is \emph{virtualized} and its field \lstinline{value} is stored +as a local variable. As a result, the generated code results in a simple loop +doing additions in-place. + +Table XXX show the time spent to run the benchmark with various input +arguments. Again, we can see that the jitted code is up to 500 times faster +than the interpreted one. Moreover, the code generated by the JIT is +\textbf{faster} than the equivalent C\# code. + +Probably, the C\# code is slower because: + +\begin{itemize} +\item The object is still allocated on the heap, and thus there is an extra + level of indirection to access the \lstinline{value} field. +\item The method call is optimized through a \emph{polymorphic inline cache} + (XXX: citation needed), that requires a guard check at each iteration. +\end{itemize} + +\anto{maybe we should move the following paragraph to + abstract/introduction/conclusion?} + +Despite being only a microbenchark, this result is very important as it proves +that our strategy of intermixing compile time and runtime can yield to better +performances than current techniques. The result is even more impressive if +we consider dynamically typed languages as TLC are usually considered much +slower than the statically typed ones. Modified: pypy/extradoc/talk/ecoop2009/tlc.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/tlc.tex (original) +++ pypy/extradoc/talk/ecoop2009/tlc.tex Thu Dec 18 11:26:03 2008 @@ -36,7 +36,7 @@ \lstinline{CDR}. \end{itemize} -Obviously, not all the operations are applicable to all objects. For example, +Obviously, not all the operations are applicable to all types. For example, it is not possible to \lstinline{ADD} an integer and an object, or reading an attribute from an object which does not provide it. Being dynamically typed, the VM needs to do all these checks at runtime; in case one of the check @@ -97,7 +97,7 @@ \end{lstlisting} Since reading TLC programs at bytecode level is hard, in this paper we will -use an invented Python-like syntax to describe examples, even if we need to +use an invented Python-like syntax to describe examples, although we need to remind that the actual programs are written in the assembler language showed above. Thus, the example above can be written in the following way: From davide at codespeak.net Thu Dec 18 12:40:10 2008 From: davide at codespeak.net (davide at codespeak.net) Date: Thu, 18 Dec 2008 12:40:10 +0100 (CET) Subject: [pypy-svn] r60561 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081218114010.8E0EF168424@codespeak.net> Author: davide Date: Thu Dec 18 12:40:08 2008 New Revision: 60561 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex Log: almost finished. Need to discuss if we want to talk about alternative implementations Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Thu Dec 18 12:40:08 2008 @@ -144,7 +144,9 @@ \end{small} If the next block to be executed is implemented in the same method ({\small\lstinline{methodid == MY_METHOD_ID}}), then the appropriate -jump to the corresponding code is executed. Otherwise, the \lstinline{jump_to_ext} +jump to the corresponding code is executed, hence internal links +can be managed efficiently. +Otherwise, the \lstinline{jump_to_ext} part of the dispatcher has to be executed. The code that actually jumps to an external block is contained in the dispatcher of the primary method, whereas the @@ -177,105 +179,120 @@ is always the first method of the graph which is called, the correct jump will be eventually executed by the dispatcher of the primary method. -\commentout{ -To implement the dispatch block we can exploit the switch opcode of the CLI; if the .NET JIT is smart enough, it can render it using an indirect jump; overall, jumping to a external block consists of an indirect function call (by invoking the delegate) plus an indirect jump (by executing the switch opcode); even if this is more costly than a simple direct jump, we will see in the next section that this not the main source of overhead when following a external link. - -Obviously, the slow dispatching logic is needed only when we want to jump to a external block; if the target block happens to reside in the same method as the current one, we can directly jump to it, completely removing the overhead. - -Moreover, the dispatch blocks are emitted only if needed, i.e. if the parent graph contains at least one flexswitch; graphs without flexswitches are rendered in the obvious way, by making one method per graph. - -The slow bit: passing arguments - -Jumping to the correct block is not enough to follow a link: as we said before, each link carries a set of arguments to be passed from the source to the target block. As usual, passing arguments across internal links is easy, as we can just use local variables to hold their values; on the other hand, external links make things more complex. - -The only way to jump to a block is to invoke its containing method, so the first solution that comes to mind is to specify its input arguments as parameter of the method; however, each block has potentially a different number (and different types) of input arguments than every other block, so we need to think of something else. - -An alternative solution could be to compute the union of the sets of input arguments of all the blocks in the method, and use this set as a signature for the method; this way, there would be enough space to specify the input arguments for every block we might want to jump to, each block ignoring the exceeding unused parameters. - -Unfortunately, all the secondary methods must have the very same signature, as they are all called from the same calling site in the dispatch block of the main method. Since the union of the set of input arguments (and hence the computed signature) varies from method to method, this solution cannot work. - -We might think to determine the signature by computing the union of input arguments of all blocks in the graph; this way, all the secondary methods would have the same signature. But as we said above, the graph grows new blocks at runtime, so we cannot determine in advance which set of input arguments we will need. - -To solve the problem we need a way to pass a variable number of arguments without knowing in advance neither their number nor their types. Thus, we use an instance of this class: - +Clearly this complex translation is performed only for flow graphs +having at least one flexswitch; flow graphs without flexswitches +are implemented in a more efficient and direct way by a unique method +with no dispatcher. + +\subsubsection{Passing arguments to external links} + +The main drawback of our solution is that passing arguments across +external links cannot be done efficiently by using the parameters of +methods for the following reasons: +\begin{itemize} +\item In general, the number and type of arguments is different for every block in a graph; + +\item The number of blocks of a graph can grow dynamically, therefore + it is not possible to compute in advance the union of the arguments + of all blocks in a graph; + +\item Since external jumps are implemented with a delegate, all the + secondary methods of a graph must have the same signature. +\end{itemize} + +Therefore, the only solution we came up with is defining a class +\lstinline{InputArgs} for passing sequences of arguments whose length +and type is variable. +\begin{small} +\begin{lstlisting}[language={[Sharp]C}] public class InputArgs { -public int[] ints; -public float[] floats; -public object[] objs; -... -} - -Since the fields are arrays, they can grow as needed to contain any number of arguments; arguments whose type is primitive are stored in the ints or floats array, depending on their type; arguments whose type is a reference type are stored in the objs array: it's up to each block to cast each argument back to the needed type. - -This solution impose a huge overhead on both writing and reading arguments: - - * when writing, we need to make sure that the arrays are big enough to contains all the arguments we need; if not, we need to allocate a bigger array. Moreover, for each argument we store into the array the virtual machine performs a bound-check, even if we know the index will never be out of bounds (because we checked the size of the array in advance); - * when reading, the same bound-check is performed for each argument read; moreover, for each value read from the objs array we need to insert a downcast. - -To mitigate the performance drop, we avoid to allocate a new InputArgs object each time we do a external jump; instead, we preallocate one at the beginning of the main method, and reuse it all the time. - -Our benchmarks show that passing arguments in arrays is about 10 times slower than passing them as real parameter of a method. Unfortunately, we couldn't come up with anything better. -Implement flexswitches - -Now, we can exploit all this machinery to implement flexswitches, as this is our ultimate goal. As described above, the point is to be able to add new cases at runtime, each case represented as a delegate. Here is an excerpt of the C# class that implements a flexswitch that switches over an integer value: - -public class IntLowLevelFlexSwitch: -{ -public uint default_blockid = 0xFFFFFFFF; -public int numcases = 0; -public int[] values = new int[4]; -public FlexSwitchCase[] cases = new FlexSwitchCase[4]; - -public void add_case(int value, FlexSwitchCase c) -{ -... -} - -public uint execute(int value, InputArgs args) -{ -for(int i=0; i Author: davide Date: Thu Dec 18 12:47:06 2008 New Revision: 60562 Modified: pypy/extradoc/talk/ecoop2009/main.tex Log: removed useless macros, hidden tentative structure, updated biblio Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Thu Dec 18 12:47:06 2008 @@ -29,8 +29,6 @@ \newcommand{\version}{} } -\newcommand\AK[1]{\nb{akuhn}{#1}} -\newcommand\on[1]{\nb{oscar}{#1}} \newcommand\dacom[1]{\nb{DA}{#1}} \newcommand\cfbolz[1]{\nb{CFB}{#1}} \newcommand\anto[1]{\nb{ANTO}{#1}} @@ -57,6 +55,7 @@ \maketitle +\commentout{ \section{Tentative structure} \begin{itemize} \item Introduction \& background; main contributions: @@ -78,8 +77,7 @@ \item Future works \item Conclusions \end{itemize} - - +} \input{abstract} \input{intro} @@ -91,6 +89,6 @@ %\input{conclusion} \bibliographystyle{plain} -\bibliography{ALLMY} +\bibliography{main} \end{document} From davide at codespeak.net Thu Dec 18 12:49:38 2008 From: davide at codespeak.net (davide at codespeak.net) Date: Thu, 18 Dec 2008 12:49:38 +0100 (CET) Subject: [pypy-svn] r60563 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081218114938.643CA168434@codespeak.net> Author: davide Date: Thu Dec 18 12:49:37 2008 New Revision: 60563 Added: pypy/extradoc/talk/ecoop2009/main.bib Log: bib file is now available Added: pypy/extradoc/talk/ecoop2009/main.bib ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/main.bib Thu Dec 18 12:49:37 2008 @@ -0,0 +1,51 @@ + at string{lncs="Lecture Notes in Computer Science"} + + at InProceedings{AACM-DLS07, + Author = {Ancona, D. and Ancona, M. and Cuni, A and Matsakis, N.}, + Title = {R{P}ython: a {S}tep {T}owards {R}econciling + {D}ynamically and {S}tatically {T}yped {OO} {L}anguages}, + BookTitle = {O{OPSLA} 2007 {P}roceedings and {C}ompanion, {DLS}'07: + {P}roceedings of the 2007 {S}ymposium on {D}ynamic + {L}anguages}, + Pages = {53--64}, + Publisher = {ACM}, + year = 2007 +} + + at InProceedings{BolzEtAl08, + author = {C. F. Bolz and + A. Kuhn and + A. Lienhard and + N. D. Matsakis and + O. Nierstrasz and + L. Renggli and + A. Rigo and + T. Verwaest}, + title = {Back to the Future in One Week - Implementing a Smalltalk + VM in PyPy}, + booktitle = {Self-Sustaining Systems, First Workshop, S3 2008, Potsdam, Revised Selected Papers}, + series = lncs, + volume = {5146}, + year = {2008}, + pages = {123--139}, +} + + at Article{Futamura99, + author = {Yoshihiko Futamura}, + title = {Partial Evaluation of Computation Process, Revisited}, + journal = {Higher Order Symbol. Comput.}, + volume = {12}, + number = {4}, + year = {1999}, + pages = {377--380}, + publisher = {Kluwer Academic Publishers}, + } + + at InProceedings{RigoPedroni06, +author = {A.~Rigo and + S.~Pedroni}, +title = {Py{P}y's approach to virtual machine construction}, +booktitle = {OOPSLA Companion}, +year = {2006}, +pages = {944-953}, +} From antocuni at codespeak.net Thu Dec 18 14:15:46 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 18 Dec 2008 14:15:46 +0100 (CET) Subject: [pypy-svn] r60567 - in pypy/extradoc/talk/ecoop2009: . benchmarks Message-ID: <20081218131546.9E8EF168449@codespeak.net> Author: antocuni Date: Thu Dec 18 14:15:45 2008 New Revision: 60567 Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex pypy/extradoc/talk/ecoop2009/benchmarks/results.txt Log: add tables showing the results. They need some restyling as they are very ugly. Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Thu Dec 18 14:15:45 2008 @@ -75,23 +75,79 @@ return b \end{lstlisting} +\anto{these tables are ugly} - -Table XXX and figure XXX show the time spent to calculate the factorial of -various numbers, with and without the JIT. Table XXX and figure XXX show the -same informations for the Fibonacci program. - -Note that do get meaningful timings, we had to calculate the factorial and -Fibonacci of very high numbers. This means that the results are incorrect due -to overflow, but since all the runnings overflows in the very same way, the -timings are still comparable. \anto{I think we should rephrase this sentence}. - -As we can see, the code generated by the JIT is almost 500 times faster than -the non-jitted case, and it is only about 1.5 times slower than the same -algorithm written in C\#: the difference in speed it is probably due to both -the fact that the current CLI backend emits slightly non-optimal code and that -the underyling .NET JIT compiler is highly optimized to handle bytecode -generated by C\# compilers. +\begin{table}[ht] + \begin{tabular}{|l|r|r|r|r||r|r|} + \hline + \textbf{n} & + \textbf{Interp} & + \textbf{JIT} & + \textbf{JIT 2} & + \textbf{C\#} & + \textbf{Interp/JIT 2} & + \textbf{JIT 2/C\#} \\ + \hline + + $10$ & 0.031 & 0.422 & 0.000 & 0.000 & N/A & N/A \\ + $10^7$ & 30.984 & 0.453 & 0.047 & 0.031 & 661.000 & 1.500 \\ + $10^8$ & N/A & 0.859 & 0.453 & 0.359 & N/A & 1.261 \\ + $10^9$ & N/A & 4.844 & 4.641 & 3.438 & N/A & 1.350 \\ + + \hline + + \end{tabular} + \caption{Factorial benchmark} + \label{tab:factorial} +\end{table} + + +\begin{table}[ht] + \begin{tabular}{|l|r|r|r|r||r|r|} + \hline + \textbf{n} & + \textbf{Interp} & + \textbf{JIT} & + \textbf{JIT 2} & + \textbf{C\#} & + \textbf{Interp/JIT 2} & + \textbf{JIT 2/C\#} \\ + \hline + + $10$ & 0.031 & 0.453 & 0.000 & 0.000 & N/A & N/A \\ + $10^7$ & 29.359 & 0.469 & 0.016 & 0.016 & 1879.962 & 0.999 \\ + $10^8$ & N/A & 0.688 & 0.250 & 0.234 & N/A & 1.067 \\ + $10^9$ & N/A & 2.953 & 2.500 & 2.453 & N/A & 1.019 \\ + + \hline + + \end{tabular} + \caption{Fibonacci benchmark} + \label{tab:fibo} +\end{table} + + +Tables \ref{tab:factorial} and \ref{tab:fibo} show the time spent to calculate +the factorial and Fibonacci for various $n$. As we can see, for small values +of $n$ the time spent running the JIT compiler is much higher than the time +spent to simply interpret the program. This is an expected result, as till +now we only focused on optimizing the compiled code, not the compilation +process itself. + +On the other hand, to get meaningful timings we had to use very high values of +$n$. This means that the results are incorrect due to overflow, but since all +the runnings overflow in the very same way, the timings are still +comparable. \anto{I think we should rephrase this sentence}. For $n$ greater +than $10^7$, we did not run the interpreted program as it would have took too +much time, without adding anything to the discussion. + +As we can see, the code generated by the JIT can be up to ~1800 times faster +than the non-jitted case. Moreover, it often runs at the same speed as the +equivalent program written in C\#, being only 1.5 slower in the worst case. + +The difference in speed it is probably due to both the fact that the current +CLI backend emits slightly non-optimal code and that the underyling .NET JIT +compiler is highly optimized to handle bytecode generated by C\# compilers. \subsection{Object-oriented features} @@ -142,10 +198,36 @@ as a local variable. As a result, the generated code results in a simple loop doing additions in-place. -Table XXX show the time spent to run the benchmark with various input -arguments. Again, we can see that the jitted code is up to 500 times faster -than the interpreted one. Moreover, the code generated by the JIT is -\textbf{faster} than the equivalent C\# code. +\begin{table}[ht] + \begin{tabular}{|l|r|r|r|r||r|r|} + \hline + \textbf{n} & + \textbf{Interp} & + \textbf{JIT} & + \textbf{JIT 2} & + \textbf{C\#} & + \textbf{Interp/JIT 2} & + \textbf{JIT 2/C\#} \\ + \hline + + $10$ & 0.031 & 0.453 & 0.000 & 0.000 & N/A & N/A \\ + $10^7$ & 43.063 & 0.516 & 0.047 & 0.063 & 918.765 & 0.750 \\ + $10^8$ & N/A & 0.875 & 0.453 & 0.563 & N/A & 0.806 \\ + $10^9$ & N/A & 4.188 & 3.672 & 5.953 & N/A & 0.617 \\ + + \hline + + \end{tabular} + \caption{Accumulator benchmark} + \label{tab:accumulator} +\end{table} + + +Table \ref{tab:accumulator} show the results for the benchmark. Again, we can +see that the speedup of the JIT over the interpreter is comparable to the +other two benchmarks. However, the really interesting part is the comparison +with the equivalent C\# code, as the code generated by the JIT is +\textbf{faster}. Probably, the C\# code is slower because: Modified: pypy/extradoc/talk/ecoop2009/benchmarks/results.txt ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks/results.txt (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks/results.txt Thu Dec 18 14:15:45 2008 @@ -1,21 +1,24 @@ Factorial N Interp JIT1 JIT2 C# -10 0,03125 0,421875 0 0 -10000000 30,98438 0,453132 0,046875 0,03125 -100000000 N/A 0,859375 0,453125 0,35937 -1000000000 N/A 4,84375 4,640625 3,4375 +10 0.03125 0.421875 0 0 +10000000 30.98438 0.453132 0.046875 0.03125 +100000000 N/A 0.859375 0.453125 0.35937 +1000000000 N/A 4.84375 4.640625 3.4375 Fibonacci N Interp JIT1 JIT2 C# -10 0,0312576 0,45313 0 0 -10000000 29,359367 0,46875 0,015617 0,015625 -100000000 N/A 0,6875 0,25001 0,234375 -1000000000 N/A 2,9531 2,5 2,453125 +10 0.0312576 0.45313 0 0 +10000000 29.359367 0.46875 0.015617 0.015625 +100000000 N/A 0.6875 0.25001 0.234375 +1000000000 N/A 2.9531 2.5 2.453125 Accumulator N Interp JIT1 JIT2 C# -10 0,03125 0,453132 0 0 -10000000 43,0625 0,515625 0,04687 0,0625 -100000000 N/A 0,87500 0,453125 0,5625 -1000000000 N/A 4,18750 3,67187 5,953125 +10 0.03125 0.453132 0 0 +10000000 43.0625 0.515625 0.04687 0.0625 +100000000 N/A 0.87500 0.453125 0.5625 +1000000000 N/A 4.18750 3.67187 5.953125 + + + From antocuni at codespeak.net Thu Dec 18 14:18:39 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 18 Dec 2008 14:18:39 +0100 (CET) Subject: [pypy-svn] r60568 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081218131839.C683116840F@codespeak.net> Author: antocuni Date: Thu Dec 18 14:18:39 2008 New Revision: 60568 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex Log: add a comment Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Thu Dec 18 14:18:39 2008 @@ -284,6 +284,10 @@ \subsection{Alternative implementations} \dacom{need to be discussed with Antonio} + +\anto{I propose to wait until the rest of paper is more or less in a final + state, and see how much space is left} + \commentout{ Before implementing the solution described here, we carefully studied a lot of possible alternatives, but all of them either didn't work because of a limitation of the virtual machine or they could work but with terrible performances. From hpk at codespeak.net Thu Dec 18 14:23:07 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 18 Dec 2008 14:23:07 +0100 (CET) Subject: [pypy-svn] r60569 - pypy/build/benchmem Message-ID: <20081218132307.F100A16841C@codespeak.net> Author: hpk Date: Thu Dec 18 14:23:07 2008 New Revision: 60569 Modified: pypy/build/benchmem/report.py Log: avoid sorting executables because logic is incomplete Modified: pypy/build/benchmem/report.py ============================================================================== --- pypy/build/benchmem/report.py (original) +++ pypy/build/benchmem/report.py Thu Dec 18 14:23:07 2008 @@ -354,7 +354,7 @@ def run_rest(self, filename="table-basetime.txt"): p = py.path.local(filename) - executables = self.getexecutables(short=True) + executables = self.getexecutables(pythonfirst=False, short=True) row0 = ["startup"] + executables rows = [row0] # result.mintimings -> [(name, timings_dict)] From hpk at codespeak.net Thu Dec 18 15:14:29 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 18 Dec 2008 15:14:29 +0100 (CET) Subject: [pypy-svn] r60571 - pypy/trunk/pypy/doc Message-ID: <20081218141429.49A9E168440@codespeak.net> Author: hpk Date: Thu Dec 18 15:14:27 2008 New Revision: 60571 Modified: pypy/trunk/pypy/doc/maemo.txt Log: specify svn revision, use --opt=3 because mem requires --no-thread for now Modified: pypy/trunk/pypy/doc/maemo.txt ============================================================================== --- pypy/trunk/pypy/doc/maemo.txt (original) +++ pypy/trunk/pypy/doc/maemo.txt Thu Dec 18 15:14:27 2008 @@ -137,13 +137,18 @@ svn co https://codespeak.net/svn/pypy/trunk pypy-trunk +Several svn revisions since the 60000's are known to work and +the last manually tested one is currently 65011. You may +also look at http://test.pypy.org where integration +of automatic maemo builds and tests should soon appear. + Change to the ``pypy-trunk/pypy/translator/goal`` directory and execute:: - python translate.py --platform=maemo --opt=mem + python translate.py --platform=maemo --opt=3 You need to run translate.py using Python 2.5. This will last some 30-60 minutes on most machines. For compiling the C source code PyPy's tool chain -will use our scratchbox/Maemo cross-compilation environment. +will use our scratchbox/Maemo cross-compilation environment. When this step succeeds, your ``goal`` directory will contain a binary called ``pypy-c`` which is executable on the Maemo device. To run this binary From cfbolz at codespeak.net Thu Dec 18 15:18:01 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 18 Dec 2008 15:18:01 +0100 (CET) Subject: [pypy-svn] r60572 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081218141801.488B6168440@codespeak.net> Author: cfbolz Date: Thu Dec 18 15:18:00 2008 New Revision: 60572 Modified: pypy/extradoc/talk/ecoop2009/intro.tex Log: some notes about what the high-level overview should contain Modified: pypy/extradoc/talk/ecoop2009/intro.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/intro.tex (original) +++ pypy/extradoc/talk/ecoop2009/intro.tex Thu Dec 18 15:18:00 2008 @@ -15,15 +15,20 @@ runtime; this approach seems to work, but writing it manually requires an enormous effort. -\cfbolz{we should cite the dyla paper somewhere here} +\cfbolz{we should cite the dyla paper somewhere here. also, we generally need +more references} -PyPy's idea is to automatize the generation of JIT compilers in order +PyPy's approach is to automatize the generation of JIT compilers in order to reduce to a minimum the effort required to get a fast implementation of a dynamic language; all you have to do is to write a high level specification of the language (by writing an interpreter), and putting it through PyPy's translation toolchain. The automatic generation of JIT compilers is done with the help of partial evaluation techniques. +The contributions of this paper are \emph{promotion}, a generalization of +polymorphic inline caches that make partial evaluation practical for dynamic +languages and trying out the idea of JIT-layering XXX. + \subsection{PyPy and RPython} \anto{as Armin points out, the two CLI backends can be easily confused; what @@ -38,7 +43,7 @@ \end{center} \end{figure} -The \emph{PyPy} project\footnote{\texttt{http://codespeak.net/pypy/dist/pypy/doc/home.html}} +The \emph{PyPy} project\footnote{\texttt{http://codespeak.net/pypy/}} \cite{RigoPedroni06} was initially conceived to develop an implementation of Python which could be easily portable and extensible without renouncing efficiency. To achieve these aims, the PyPy implementation is based on a highly @@ -77,9 +82,26 @@ This section will give a high-level overview of how the JIT-generation process works. More details will be given in subsequent sections. +- write an interpreter +- add some hints to the interpreter + +translation time: +- start the translation process, flow graphs +- run a binding-time analysis using the hints, to figure out ... +- turn the flow graphs into rainbow bytecode +- turn the flow graphs of the rainbow interpreter and the original flow graphs +into .NET code + +runtime: +when the interpreter is started and a function should be interpreted: + - the generated JIT starts compiling the function + - it produces .NET bytecode until it does not have enough information to + continue. then it stops the compilation and puts a callback into the compiler + into the generated bytecode. + - at this point the .NET bytecode that is produced so far is executed. + - when the callback into the compiler is hit, execution stops and compilation + starts again + - afterwards execution continues after the callback (running the code that was + newly produced) -Another interesting feature of PyPy -is that just-in-time compilers can be semi-automatically generated from the -interpreter source. -XXX list contributions clearly From cfbolz at codespeak.net Thu Dec 18 15:19:43 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 18 Dec 2008 15:19:43 +0100 (CET) Subject: [pypy-svn] r60573 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081218141943.5F659168440@codespeak.net> Author: cfbolz Date: Thu Dec 18 15:19:42 2008 New Revision: 60573 Modified: pypy/extradoc/talk/ecoop2009/tlc.tex Log: fix some things in the tlc section Modified: pypy/extradoc/talk/ecoop2009/tlc.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/tlc.tex (original) +++ pypy/extradoc/talk/ecoop2009/tlc.tex Thu Dec 18 15:19:42 2008 @@ -1,11 +1,14 @@ \section{The TLC language} In this section, we will briefly describe \emph{TLC}, a simple dynamic -language that we developed to exercise our JIT compiler generator. As most of -dynamic languages around, \emph{TLC} is implemented through a virtual machine -that interprets a custom bytecode. Since our main interest is in the runtime -performance of the VM, we did not implement the parser nor the bytecode -compiler, but only the VM itself. +language that we developed to exercise our JIT compiler generator and that will +be used as a running example in this paper. The design goal of the language is +to be very simple (the interpreter of the full language consists of about 600 +lines of RPython code) but to still have the typical properties of dynamic +languages that make them hard to compile. \emph{TLC} is implemented with a small +interpreter that interprets a custom bytecode instruction set. Since our main +interest is in the runtime performance of the interpreter, we did not implement +the parser nor the bytecode compiler, but only the interpreter itself. TLC provides four different types: \begin{enumerate} @@ -20,12 +23,14 @@ of attributes of the object, as well as its methods. Once the object has been created, it is not possible to add/remove attributes and methods. -The virtual machine is stack-based, and provides several operations: +The interpreter for the language is stack-based and uses bytecode to represent +the program. It provides the following bytecode instructions: \begin{itemize} \item \textbf{Stack manipulation}: standard operations to manipulate the - stack, such as \lstinline{PUSH}, \lstinline{POP}, \lstinline{SWAP}, etc. + stack, such as \lstinline{POP}, \lstinline{PUSHARG}, \lstinline{SWAP}, etc. \item \textbf{Flow control} to do conditional and unconditional jumps. +\cfbolz{what is the bytecode for conditional jumps?} \item \textbf{Arithmetic}: numerical operations on integers, like \lstinline{ADD}, \lstinline{SUB}, etc. \item \textbf{Comparisons} like \lstinline{EQ}, \lstinline{LT}, @@ -44,18 +49,20 @@ \subsection{TLC features} \label{sec:tlc-features} +\cfbolz{calling this sections "features" is a bit obscure, since it is more +properties of the implementation} Despite being very simple and minimalistic, \lstinline{TLC} is a good candidate as a language to test our JIT generator, as it has some of the -features that makes most of current dynamic languages so slow: +properties that makes most of current dynamic languages so slow: \begin{itemize} -\item \textbf{Stack based VM}: this kind of VM requires all the operands to be +\item \textbf{Stack based interpreter}: this kind of interpreter requires all the operands to be on top of the evaluation stack. As a consequence programs spend a lot of time pushing and popping values to/from the stack, or doing other stack related operations. However, thanks to its simplicity this is still the - most common and preferred way to implement VMs. + most common and preferred way to implement interpreters. \item \textbf{Boxed integers}: integer objects are internally represented as an instance of the \lstinline{IntObj} class, whose field \lstinline{value} @@ -82,7 +89,7 @@ main: # stack: [] PUSHARG # [n] PUSH 0 # [n, 0] - LT # [n<0] + LT # [0 or 1] BR_COND neg pos: # [] From cfbolz at codespeak.net Thu Dec 18 15:20:12 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 18 Dec 2008 15:20:12 +0100 (CET) Subject: [pypy-svn] r60574 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081218142012.7C8FA168440@codespeak.net> Author: cfbolz Date: Thu Dec 18 15:20:12 2008 New Revision: 60574 Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex Log: lots of fixes Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Thu Dec 18 15:20:12 2008 @@ -26,8 +26,8 @@ In the sequel, we describe in more details one of the main new techniques introduced in our approach, which we call \emph{promotion}. In -short, it allows an arbitrary run-time value to be turned into a -compile-time value at any point in time. Promotion is thus the central way by +short, it allows an arbitrary run-time (i.e. red) value to be turned into a +compile-time (i.e. green) value at any point in time. Promotion is thus the central way by which we make use of the fact that the JIT is running interleaved with actual program execution. Each promotion point is explicitly defined with a hint that must be put in the source code of the interpreter. @@ -46,14 +46,14 @@ otherwise the compiler can only use information that is known ahead of time. It is impossible in the "classical" approaches to partial evaluation, in which the compiler always runs fully ahead of execution -This is a problem in many large use cases. For example, in an +This is a problem in many realistic use cases. For example, in an interpreter for a dynamic language, there is mostly no information that can be clearly and statically used by the compiler before any code has run. -A very different point of view on promotion is as a generalization of -techniques that already exist in dynamic compilers as found in modern -object-oriented language virtual machines. In this context feedback +A very different point of view on promotion is as a generalization of techniques +that already exist in dynamic compilers as found in modern virtual machines for +object-oriented language. In this context feedback techniques are crucial for good results. The main goal is to optimize and reduce the overhead of dynamic dispatching and indirect invocation. This is achieved with variations on the technique of @@ -66,38 +66,37 @@ In the presence of promotion, dispatch optimization can usually be reframed as a partial evaluation task. Indeed, if the type of the object being dispatched to is known at compile-time, the lookup can be -folded, and only a (possibly inlined) direct call remains in the +folded, and only a (possibly even inlined) direct call remains in the generated code. In the case where the type of the object is not known at compile-time, it can first be read at run-time out of the object and promoted to compile-time. As we will see in the sequel, this produces -very similar machine code \footnote{This can also be seen as a generalization of +machine code very similar to that of polymorphic inline +caches\footnote{Promotion can also be seen as a generalization of a partial evaluation transformation called "The Trick" (see e.g. \cite{XXX}), which again produces similar code but which is only applicable for finite sets of values.}. -The essential advantage is that it is no longer tied to the details of +The essential advantage of promotion is that it is no longer tied to the details of the dispatch semantics of the language being interpreted, but applies in more general situations. Promotion is thus the central enabling primitive to make partial evaluation a practical approach to language independent dynamic compiler generation. -\subsection{Promotion as Applied to the TLC} - -XXX - -\subsection{Promotion in Practise} +\subsection{Implementing Promotion} -There are values that, if known at compile time, allow the JIT compiler to -produce very efficient code. Unfortunately, these values are tipically red, -e.g. the exact type of a variable. +The implementation of promotion requires a tight coupling between +compile-time and run-time: a \emph{callback}, put in the generated code, +which can invoke the compiler again. When the callback is actually +reached at run-time, and only then, the compiler resumes and uses the +knowledge of the actual run-time value to generate more code. + +The new generated code is potentially different for each run-time value +seen. This implies that the generated code needs to contain some sort +of updatable switch, which can pick the right code path based on the +run-time value. -"Promotion" is a particular operation that convert a red value into a green -value; i.e., after the promotion of a variable, the JIT compiler knows its -value at compile time. Since the value of a red variable is not known until -runtime, we need to postpone the compilation phase after the runtime phase. - -This is done by continuously intermixing compile time and runtime; a promotion -is implemented in this way: +\cfbolz{I think this example is confusing, it is unclear in which order things +happen how. I will try to come up with something different}. \begin{itemize} \item (compile time): the rainbow interpreter produces machine code until it @@ -138,32 +137,27 @@ unhandled promotion point is reached. \end{itemize} +\subsection{Promotion as Applied to the TLC} + +XXX maybe a tlc example can be used for the example above? + + \section{Automatic Unboxing of Intermediate Results} XXX the following section needs a rewriting to be much more high-level and to compare more directly with classical escape analysis -Interpreters for dynamic languages typically allocate a lot of small -objects, for example due to boxing. For this reason, we -implemented a way for the compiler to generate residual memory -allocations as lazily as possible. The idea is to try to keep new -run-time structures "exploded": instead of a single run-time pointer to -a heap-allocated data structure, the structure is "virtualized" as a set -of fresh variables, one per field. In the compiler, the variable that -would normally contain the pointer to the structure gets instead a -content that is neither a run-time value nor a compile-time constant, -but a special \emph{virtual structure} ? a compile-time data structure that -recursively contains new variables, each of which can again store a -run-time, a compile-time, or a virtual structure value. - -This approach is based on the fact that the "run-time values" carried -around by the compiler really represent run-time locations ? the name of -a CPU register or a position in the machine stack frame. This is the -case for both regular variables and the fields of virtual structures. -It means that the compilation of a \texttt{getfield} or \texttt{setfield} -operation performed on a virtual structure simply loads or stores such a -location reference into the virtual structure; the actual value is not -copied around at run-time. +Interpreters for dynamic languages typically continuously allocate a lot of small +objects, for example due to boxing. This makes arithmetic operations extremely +inefficient. For this reason, we +implemented a way for the compiler to try to avoid memory allocations in the +residual code as long as possible. The idea is to try to keep new +run-time structures "exploded": instead of a single run-time object allocated on +the heap, the object is "virtualized" as a set +of fresh variables, one per field. Only when the object can be accessed by from +somewhere else is it actually allocated on the heap. The effect of this is similar to that of +escape analysis \cite{XXX}, which also prevents allocations of objects that can +be proven to not escape a method or set of methods. It is not always possible to keep structures virtual. The main situation in which it needs to be "forced" (i.e. actually allocated at @@ -172,8 +166,10 @@ Virtual structures still avoid the run-time allocation of most short-lived objects, even in non-trivial situations. The following -example shows a typical case. Consider the Python expression \texttt{a+b+c}. -Assume that \texttt{a} contains an integer. The PyPy Python interpreter +example shows a typical case. XXX use TLC example + + +The PyPy Python interpreter implements application-level integers as boxes ? instances of a \texttt{W\_IntObject} class with a single \texttt{intval} field. Here is the addition of two integers: @@ -187,6 +183,7 @@ return W_IntObject(result) \end{verbatim} +XXX kill the rest?!? When interpreting the bytecode for \texttt{a+b+c}, two calls to \texttt{add()} are issued; the intermediate \texttt{W\_IntObject} instance is built by the first call and thrown away after the second call. By contrast, when the From cfbolz at codespeak.net Thu Dec 18 15:21:48 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 18 Dec 2008 15:21:48 +0100 (CET) Subject: [pypy-svn] r60575 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081218142148.961CD168440@codespeak.net> Author: cfbolz Date: Thu Dec 18 15:21:48 2008 New Revision: 60575 Modified: pypy/extradoc/talk/ecoop2009/main.tex Log: add EU project Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Thu Dec 18 15:21:48 2008 @@ -43,7 +43,8 @@ \title{Automatic generation of JIT compilers for dynamic languages in .NET\thanks{This work has been partially supported by MIUR EOS DUE - Extensible Object Systems for Dynamic and -Unpredictable Environments.\cfbolz{should we put the PyPy EU project here as well?}\anto{I think so}}} +Unpredictable Environments and by the EU-funded project: IST 004779 PyPy +(PyPy: Implementing Python in Python).}} \author{Davide Ancona\inst{1} \and Carl Friedrich Bolz\inst{2} \and Antonio Cuni\inst{1} \and Armin Rigo} From cfbolz at codespeak.net Thu Dec 18 15:22:29 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 18 Dec 2008 15:22:29 +0100 (CET) Subject: [pypy-svn] r60576 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081218142229.F130F168440@codespeak.net> Author: cfbolz Date: Thu Dec 18 15:22:29 2008 New Revision: 60576 Modified: pypy/extradoc/talk/ecoop2009/main.bib Log: fix casing of PyPy Modified: pypy/extradoc/talk/ecoop2009/main.bib ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.bib (original) +++ pypy/extradoc/talk/ecoop2009/main.bib Thu Dec 18 15:22:29 2008 @@ -22,7 +22,7 @@ A. Rigo and T. Verwaest}, title = {Back to the Future in One Week - Implementing a Smalltalk - VM in PyPy}, + VM in {P}y{P}y}, booktitle = {Self-Sustaining Systems, First Workshop, S3 2008, Potsdam, Revised Selected Papers}, series = lncs, volume = {5146}, From cfbolz at codespeak.net Thu Dec 18 15:50:06 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 18 Dec 2008 15:50:06 +0100 (CET) Subject: [pypy-svn] r60577 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081218145006.28EE9168412@codespeak.net> Author: cfbolz Date: Thu Dec 18 15:50:05 2008 New Revision: 60577 Added: pypy/extradoc/talk/ecoop2009/tlc-folded.py (contents, props changed) pypy/extradoc/talk/ecoop2009/tlc-simplified.py (contents, props changed) Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex Log: add a tlc-based example for partial evaluation. Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Thu Dec 18 15:50:05 2008 @@ -56,36 +56,50 @@ depend only on the $s_1, ..., s_m$ and to remove parts of $P$ that cannot be reached given the concrete values. +\begin{figure}[h] +\label{fig:tlc-main} +\begin{center} +\input{tlc-simplified.py} +\caption{The main loop of the TLC interpreter} +\end{center} +\end{figure} + +Let us look at an example. Figure \ref{fig:tlc-main} shows parts of the +main-loop of the TLC interpreter (in slightly simplified form). The static +arguments of this functions would typically be the \lstinline{code} argument +(containing the bytecode as a string), the \lstinline{pc} argument (containing +the initial program counter) and the \lstinline{pool} argument (containing the +constant pool). The \lstinline{args} argument is dynamic since it contains the +user-input of the interpreted function. Since the \lstinline{while} loop and the +contained conditions depend only on static arguments it will typically be +unrolled by a partial evaluator. + +When the function is partially evaluated with respect to the TLC example +function shown in Figure XXX (which computes the absolute value of a number), +the residual code would look like in Figure \ref{fig:tlc-folded}. This version +is already a great improvement over pure interpretation, all the bytecode +dispatch overhead has been removed. However, the function as shown is still +rather inefficient, due to lots of superfluous stack handling and also due to +some indirect calls for implementing comparison (\lstinline{lt}) and +subtraction (\lstinline{sub}). + +\begin{figure}[h] +\label{fig:tlc-folded} +\begin{center} +\input{tlc-folded.py} +\caption{The residual code of the TLC main loop with respect to the +\lstinline{abs} function } +\end{center} +\end{figure} + +This shows a common shortcoming of partial evaluation when applied to a dynamic +language: The partial evaluator (just like an ahead-of-time compiler) cannot +make assumptions about the types of objects, which leads to poor results. +Effective dynamic compilation requires feedback of runtime information into +compile-time. This is no different for partial evaluation. -\cfbolz{I would propose to use either TLC as an example here, or something that -looks at least like an interpreter loop} - -Example:: -\begin{lstlisting}[language=Python] - def f(x, y): - x2 = x * x - y2 = y * y - return x2 + y2 - -**case x=3** :: - - def f_3(y): - y2 = y * y - return 9 + y2 - -**case x=10** :: - - def f_10(y): - y2 = y * y - return 100 + y2 -\end{lstlisting} - -A shortcoming of PE is that in many cases not much can be really assumed -constant at compile-time, and this leads to poor results. Effective dynamic -compilation requires feedback of runtime information into compile-time; for a -dynamic language, types are a primary example. - -Partial evaluation (PE) comes in two flavors: +Partial evaluation (PE) comes in two flavors: \cfbolz{if we have space problems +we should kill the bits about online and offline PE} \begin{itemize} \item \emph{On-line} partial evaluation: a compiler-like algorithm takes the Added: pypy/extradoc/talk/ecoop2009/tlc-folded.py ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/tlc-folded.py Thu Dec 18 15:50:05 2008 @@ -0,0 +1,18 @@ +\begin{lstlisting}[language=Python] +def interp_eval_abs(args): + stack = [] + stack.append(args[0]) + stack.append(IntObj(0)) + a, b = stack.pop(), stack.pop() + stack.append(IntObj(b.lt(a))) + cond = stack.pop() + if cond.istrue(): + stack.append(args[0]) + return stack[-1] + else: + stack.append(IntObj(0)) + stack.append(args[0]) + a, b = stack.pop(), stack.pop() + stack.append(b.sub(a)) + return stack[-1] +\end{lstlisting} Added: pypy/extradoc/talk/ecoop2009/tlc-simplified.py ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/tlc-simplified.py Thu Dec 18 15:50:05 2008 @@ -0,0 +1,29 @@ +\begin{lstlisting}[language=Python] +def interp_eval(code, pc, args, pool): + code_len = len(code) + stack = [] + while pc < code_len: + opcode = ord(code[pc]) + pc += 1 + + if opcode == PUSH: + stack.append(IntObj(char2int(code[pc]))) + pc += 1 + elif opcode == PUSHARG: + stack.append(args[0]) + elif opcode == SUB: + a, b = stack.pop(), stack.pop() + stack.append(b.sub(a)) + elif opcode == LT: + a, b = stack.pop(), stack.pop() + stack.append(IntObj(b.lt(a))) + elif opcode == BR_COND: + cond = stack.pop() + if cond.istrue(): + pc += char2int(code[pc]) + pc += 1 + elif opcode == RETURN: + break + ... + return stack[-1] +\end{lstlisting} From cfbolz at codespeak.net Thu Dec 18 16:25:00 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 18 Dec 2008 16:25:00 +0100 (CET) Subject: [pypy-svn] r60578 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081218152500.78FD616841C@codespeak.net> Author: cfbolz Date: Thu Dec 18 16:24:59 2008 New Revision: 60578 Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex pypy/extradoc/talk/ecoop2009/jitgen.tex pypy/extradoc/talk/ecoop2009/main.bib pypy/extradoc/talk/ecoop2009/rainbow.tex Log: kill a footnote, add a reference Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Thu Dec 18 16:24:59 2008 @@ -235,13 +235,13 @@ \item The object is still allocated on the heap, and thus there is an extra level of indirection to access the \lstinline{value} field. \item The method call is optimized through a \emph{polymorphic inline cache} - (XXX: citation needed), that requires a guard check at each iteration. + \cite{hoelzle_optimizing_1991}, that requires a guard check at each iteration. \end{itemize} \anto{maybe we should move the following paragraph to abstract/introduction/conclusion?} -Despite being only a microbenchark, this result is very important as it proves +Despite being only a microbenchmarks, this result is very important as it proves that our strategy of intermixing compile time and runtime can yield to better performances than current techniques. The result is even more impressive if we consider dynamically typed languages as TLC are usually considered much Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Thu Dec 18 16:24:59 2008 @@ -220,7 +220,7 @@ branches are taken in the residual graph. \end{itemize} -\subsubsection{Hints} +\subsection{Hints} Our goal in designing our approach to binding-time analysis was to minimize the number of explicit hints that the user must provide in Modified: pypy/extradoc/talk/ecoop2009/main.bib ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.bib (original) +++ pypy/extradoc/talk/ecoop2009/main.bib Thu Dec 18 16:24:59 2008 @@ -49,3 +49,23 @@ year = {2006}, pages = {944-953}, } + + at inproceedings{hoelzle_optimizing_1991, + title = {Optimizing Dynamically-Typed Object-Oriented Languages With Polymorphic Inline Caches}, + isbn = {3-540-54262-0}, + url = {http://portal.acm.org/citation.cfm?id=679193\&dl=ACM\&coll=portal}, + booktitle = {Proceedings of the European Conference on Object-Oriented Programming}, + publisher = {Springer-Verlag}, + author = {Urs H{\"o}lzle and Craig Chambers and David Ungar}, + year = {1991}, + pages = {21-38} +} + + at book{Jones:peval, + author = {Jones, Neil D. and Gomard, Carsten K. and Sestoft, Peter}, + title = {Partial Evaluation and Automatic Program Generation}, + publisher = {Prentice Hall}, + year = 1993, +} + + Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Thu Dec 18 16:24:59 2008 @@ -57,7 +57,7 @@ techniques are crucial for good results. The main goal is to optimize and reduce the overhead of dynamic dispatching and indirect invocation. This is achieved with variations on the technique of -polymorphic inline caches \cite{XXX}: the dynamic lookups are cached and +polymorphic inline caches \cite{hoelzle_optimizing_1991}: the dynamic lookups are cached and the corresponding generated machine code contains chains of compare-and-jump instructions which are modified at run-time. These techniques also allow the gathering of information to direct inlining for even @@ -71,10 +71,7 @@ at compile-time, it can first be read at run-time out of the object and promoted to compile-time. As we will see in the sequel, this produces machine code very similar to that of polymorphic inline -caches\footnote{Promotion can also be seen as a generalization of -a partial evaluation transformation called "The Trick" (see e.g. \cite{XXX}), -which again produces similar code but which is only applicable for finite sets -of values.}. +caches. The essential advantage of promotion is that it is no longer tied to the details of the dispatch semantics of the language being interpreted, but applies in From antocuni at codespeak.net Thu Dec 18 18:02:37 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 18 Dec 2008 18:02:37 +0100 (CET) Subject: [pypy-svn] r60582 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081218170237.42B7A1683FF@codespeak.net> Author: antocuni Date: Thu Dec 18 18:02:34 2008 New Revision: 60582 Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex pypy/extradoc/talk/ecoop2009/jitgen.tex pypy/extradoc/talk/ecoop2009/tlc.tex Log: answer to a comment, insert another one, fix a typo Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Thu Dec 18 18:02:34 2008 @@ -241,7 +241,7 @@ \anto{maybe we should move the following paragraph to abstract/introduction/conclusion?} -Despite being only a microbenchmarks, this result is very important as it proves +Despite being only a microbenchmark, this result is very important as it proves that our strategy of intermixing compile time and runtime can yield to better performances than current techniques. The result is even more impressive if we consider dynamically typed languages as TLC are usually considered much Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Thu Dec 18 18:02:34 2008 @@ -67,7 +67,7 @@ Let us look at an example. Figure \ref{fig:tlc-main} shows parts of the main-loop of the TLC interpreter (in slightly simplified form). The static arguments of this functions would typically be the \lstinline{code} argument -(containing the bytecode as a string), the \lstinline{pc} argument (containing +(containing the bytecode as a string \anto{what about using array of bytes instead of string? For .NET/JVM people, there is a strong difference between the twos}), the \lstinline{pc} argument (containing the initial program counter) and the \lstinline{pool} argument (containing the constant pool). The \lstinline{args} argument is dynamic since it contains the user-input of the interpreted function. Since the \lstinline{while} loop and the Modified: pypy/extradoc/talk/ecoop2009/tlc.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/tlc.tex (original) +++ pypy/extradoc/talk/ecoop2009/tlc.tex Thu Dec 18 18:02:34 2008 @@ -29,8 +29,8 @@ \begin{itemize} \item \textbf{Stack manipulation}: standard operations to manipulate the stack, such as \lstinline{POP}, \lstinline{PUSHARG}, \lstinline{SWAP}, etc. -\item \textbf{Flow control} to do conditional and unconditional jumps. -\cfbolz{what is the bytecode for conditional jumps?} +\item \textbf{Flow control} to do conditional (\lstinline{BR_COND}) and + unconditional (\lstinline{BR}) jumps. \item \textbf{Arithmetic}: numerical operations on integers, like \lstinline{ADD}, \lstinline{SUB}, etc. \item \textbf{Comparisons} like \lstinline{EQ}, \lstinline{LT}, From antocuni at codespeak.net Fri Dec 19 11:39:18 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 19 Dec 2008 11:39:18 +0100 (CET) Subject: [pypy-svn] r60585 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219103918.8F3C3168473@codespeak.net> Author: antocuni Date: Fri Dec 19 11:39:16 2008 New Revision: 60585 Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex pypy/extradoc/talk/ecoop2009/clibackend.tex pypy/extradoc/talk/ecoop2009/jitgen.tex pypy/extradoc/talk/ecoop2009/rainbow.tex pypy/extradoc/talk/ecoop2009/tlc.tex Log: minor fixes, some comments Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Fri Dec 19 11:39:16 2008 @@ -1,8 +1,11 @@ \section{Benchmarks} -In section \ref{sec:tlc-features}, we saw that TLC provides most of the +\anto{We should say somewhere that flexswitches are slow but benchmarks are so + good because they are not involved in the inner loops} + +In section \ref{sec:tlc-properties}, we saw that TLC provides most of the features that usaully make dynamically typed language so slow, such as -\emph{stack-based VM}, \emph{boxed arithmetic} and \emph{dynamic lookup} of +\emph{stack-based interpreter}, \emph{boxed arithmetic} and \emph{dynamic lookup} of methods and attributes. In the following sections, we will show some benchmarks that show how our @@ -12,9 +15,9 @@ \begin{enumerate} \item By plain interpretation, without any jitting. -\item With the jit enabled: this run includes the time spent by doing the +\item With the JIT enabled: this run includes the time spent by doing the compilation itself, plus the time spent by running the produced code. -\item Again with the jit enabled, but this time the compilation has already +\item Again with the JIT enabled, but this time the compilation has already been done, so we are actually measuring how good is the code we produced. \end{enumerate} @@ -50,7 +53,7 @@ much better. At the first iteration, the classes of the two operands of the multiplication are promoted; then, the JIT compiler knows that both are integers, so it can inline the code to compute the result. Moreover, it can -\emph{virtualize} all the temporary objects, because they never escape from +\emph{virtualize} (see section \ref{sec:virtuals} all the temporary objects, because they never escape from the inner loop. The same remarks apply to the other two operations inside the loop. @@ -182,7 +185,7 @@ The computation \emph{per se} is trivial, as it calculates either $-n$ or $1+2...+n-1$, depending on the sign of $n$. The interesting part is the -polymorphic call to \lstinline{accumulate} inside the loop, because the VM has +polymorphic call to \lstinline{accumulate} inside the loop, because the interpreter has no way to know in advance which method to call (unless it does flow analysis, which could be feasible in this case but not in general). The equivalent C\# code we wrote uses two classes and a \lstinline{virtual} method call to @@ -191,7 +194,7 @@ However, our generated JIT does not compile the whole function at once. Instead, it compiles and executes code chunk by chunk, waiting until it knows enough informations to generate highly efficient code. In particualr, -at the time when it emits the code for the inner loop it exactly knows the +at the time it emits the code for the inner loop it exactly knows the type of \lstinline{obj}, thus it can remove the overhead of dynamic dispatch and inline the method call. Moreover, since \lstinline{obj} never escapes the function, it is \emph{virtualized} and its field \lstinline{value} is stored Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Fri Dec 19 11:39:16 2008 @@ -52,17 +52,13 @@ Since in .NET methods are the basic units of compilation, a possible solution consists in creating a new method any time a new case has to be added to a flexswitch. -\dacom{comment for Antonio: I am not sure this is the best solution. This cannot work for Java where classes are the basic - units. Closures will be available only with Java Dolphin and I do - not know how much efficient will be} In this way, whereas flow graphs without flexswitches are translated to a single method, the translation of flow graphs which can dynamically grow because of flexswitches will be scattered over several methods. Summarizing, the backend behaves in the following way: \begin{itemize} \item Each flow graph is translated in a collection of methods which - can grow dynamically. \dacom{I propose primary/secondary instead of - the overloaded terms main/child} Each collection contains at least one + can grow dynamically. Each collection contains at least one method, called \emph{primary}, which is the first to be created. All other methods, called \emph{secondary}, are added dynamically whenever a new case is added to a flexswitch. @@ -71,12 +67,13 @@ number of blocks, all belonging to the same flow graph. Among these blocks there always exists an initial block whose input variables are parameters of the method; the input variables of all other blocks - are local variables of the method. + are local variables of the method. \anto{This is wrong: the signature of the secondary methods is fixed, and input args are passed inside the InputArgs class, not as methodo parameters} \end{itemize} When a new case is added to a flexswitch, new blocks are generated and translated by the backend in a new single method pointed -by a delegate which is stored in the code implementing the flexswitch, +by a delegate \footnote{\emph{Delegates} are the .NET equivalent of function pointers} + of which is stored in the code implementing the flexswitch, so that the method can be invoked later. \subsubsection{Internal and external links} @@ -88,7 +85,7 @@ Following an internal link is not difficult in IL bytecode: a jump to the corresponding code fragment in the same method is emitted to execute the new block, whereas the appropriate local variables are -used for passing arguments. +used for passing arguments. \anto{this is wrong for the same reason as above} Also following an external link whose target is the initial block of a method is not difficult: the corresponding method has to be invoked with the appropriate arguments. @@ -106,7 +103,7 @@ determine which block has to be executed. This is done by passing to the method a 32 bits number, called \emph{block id}, which uniquely identifies the next block of the graph to be executed. -The high word of a block id is the id of the method to which the block +The high word \anto{a word is 32 bit, block num and method id are 16 bit each} of a block id is the id of the method to which the block belongs, whereas the low word is a progressive number univocally identifying each block implemented by the method. @@ -145,7 +142,7 @@ If the next block to be executed is implemented in the same method ({\small\lstinline{methodid == MY_METHOD_ID}}), then the appropriate jump to the corresponding code is executed, hence internal links -can be managed efficiently. +can be managed efficiently. \anto{wrong: internal links don't go through the dispatcher} Otherwise, the \lstinline{jump_to_ext} part of the dispatcher has to be executed. The code that actually jumps to an external block is contained in @@ -267,10 +264,10 @@ the link and jumps to the right block by performing a linear search in array \lstinline{values}. -Recall that the first argument of delegate \lstinline{FlexSwitchCase} -is the block id to jump to; since the target of an external jump is -always the initial block of the method, the first argument will be -always 0. +Recall that the first argument of delegate \lstinline{FlexSwitchCase} is the +block id to jump to. By construction, the target block of a flexswitch is +always the first in a secondary method, and we use the special value +\lstinline{0} to signal this. The value returned by method \lstinline{execute} is the next block id to be executed; Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Fri Dec 19 11:39:16 2008 @@ -60,7 +60,7 @@ \label{fig:tlc-main} \begin{center} \input{tlc-simplified.py} -\caption{The main loop of the TLC interpreter} +\caption{The main loop of the TLC interpreter, written in RPython} \end{center} \end{figure} @@ -194,7 +194,7 @@ The binding-time analyzer of our translation tool-chain is using a simple abstract-interpretation based analysis. It is based on the same type inference engine that is used on the source RPython program, -the annotator. In this mode, it is called the \emph{hint-annotator}; it +the annotator \anto{I'm not sure we should mention the annotator, as it is not referred anywhere else}. In this mode, it is called the \emph{hint-annotator}; it operates over input graphs that are already low-level instead of RPython-level, and propagates annotations that do not track types but value dependencies and manually-provided binding time hints. Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Fri Dec 19 11:39:16 2008 @@ -140,6 +140,7 @@ \section{Automatic Unboxing of Intermediate Results} +\label{sec:virtuals} XXX the following section needs a rewriting to be much more high-level and to compare more directly with classical escape analysis @@ -151,7 +152,7 @@ residual code as long as possible. The idea is to try to keep new run-time structures "exploded": instead of a single run-time object allocated on the heap, the object is "virtualized" as a set -of fresh variables, one per field. Only when the object can be accessed by from +of fresh local variables, one per field. Only when the object can be accessed by from somewhere else is it actually allocated on the heap. The effect of this is similar to that of escape analysis \cite{XXX}, which also prevents allocations of objects that can be proven to not escape a method or set of methods. Modified: pypy/extradoc/talk/ecoop2009/tlc.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/tlc.tex (original) +++ pypy/extradoc/talk/ecoop2009/tlc.tex Fri Dec 19 11:39:16 2008 @@ -21,7 +21,8 @@ Objects represent a collection of named attributes (much like JavaScript or Self) and named methods. At creation time, it is necessary to specify the set of attributes of the object, as well as its methods. Once the object has been -created, it is not possible to add/remove attributes and methods. +created, it is possible to call methods and read or write attributes, but not +to add or remove them. The interpreter for the language is stack-based and uses bytecode to represent the program. It provides the following bytecode instructions: @@ -47,10 +48,8 @@ the VM needs to do all these checks at runtime; in case one of the check fails, the execution is simply aborted. -\subsection{TLC features} -\label{sec:tlc-features} -\cfbolz{calling this sections "features" is a bit obscure, since it is more -properties of the implementation} +\subsection{TLC properties} +\label{sec:tlc-properties} Despite being very simple and minimalistic, \lstinline{TLC} is a good candidate as a language to test our JIT generator, as it has some of the From antocuni at codespeak.net Fri Dec 19 12:48:58 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 19 Dec 2008 12:48:58 +0100 (CET) Subject: [pypy-svn] r60587 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219114858.B7A2E16847D@codespeak.net> Author: antocuni Date: Fri Dec 19 12:48:57 2008 New Revision: 60587 Added: pypy/extradoc/talk/ecoop2009/jitstrategy.tex Modified: pypy/extradoc/talk/ecoop2009/main.tex Log: start of the "how the generated jit works" section Added: pypy/extradoc/talk/ecoop2009/jitstrategy.tex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/jitstrategy.tex Fri Dec 19 12:48:57 2008 @@ -0,0 +1,36 @@ +\section{Effective JIT compilation of dynamic languages} + +Writing efficient compilers for dynamic languages is hard. Since these +languages are dynamically typed, usually the compiler does not have enough +information to produce efficient code, but instead it has to insert a lot of +runtime checks to select the appropriate implementation for each operation. + +By generating code at runtime, JIT compilers can exploit some extra knowledge +compared to traditional static compilers. However, we need to take special +care to choose a strategy for JIT compilation that lets the compiler to take +the best of this advantage. + +Most JIT compilers for dynamic languages around (such as IronPython, Jython, +JRuby \anto{XXX insert some reference}) compile code at the method +granularity. If on one hand they can exploit some of the knowledge gathered +at runtime (e.g. the types of method parameters), on the other hand they can +do little to optimize most of the operations inside, because their behaviour +depends on informations that are not available at compile time, because +e.g. the global state of the program can change at runtime. \anto{e.g., we can + add/remove methods to classes, etc. Should we insert some example here?} + +JIT compilers generated by PyPy solve this problem by delaying the compilation +until they know all the informations needed to generate efficient code. If at +some point the JIT compiler does not know about something it needs, it +generates a callback into itself. + +Later, when the generated code is executed, the callback is hit and the JIT +compiler is restarted again. At this point, the JIT knows exactly the state +of the program and can exploit all this extra knowledge to generate highly +efficient code. Finally, the old code is patched and linked to the newly +generated code, so that the next time the JIT compiler will not be invoked +again. As a result, \textbf{runtime and JIT-compile time are continuously + intermixed}. \anto{maybe we should put the translation/compile/run time + distinction here} + +XXX: insert an example of flexswitch Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Fri Dec 19 12:48:57 2008 @@ -83,6 +83,7 @@ \input{abstract} \input{intro} \input{tlc} +\input{jitstrategy} \input{jitgen} \input{rainbow} \input{clibackend} From davide at codespeak.net Fri Dec 19 13:07:24 2008 From: davide at codespeak.net (davide at codespeak.net) Date: Fri, 19 Dec 2008 13:07:24 +0100 (CET) Subject: [pypy-svn] r60589 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219120724.5B59E168481@codespeak.net> Author: davide Date: Fri Dec 19 13:07:23 2008 New Revision: 60589 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex Log: fig.6 on method and block ids should be changed Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Fri Dec 19 13:07:23 2008 @@ -116,7 +116,8 @@ \begin{figure}[h] \begin{center} \includegraphics[height=6cm]{blockid} -\caption{Method and block ids.}\label{block-id-fig} +\caption{Method and block ids.\dacom{in the picture ``Main method'' must be + replaced with ``Primary method'' and in the primary method the block with number 0 should be added}}\label{block-id-fig} \end{center} \end{figure} From davide at codespeak.net Fri Dec 19 14:42:51 2008 From: davide at codespeak.net (davide at codespeak.net) Date: Fri, 19 Dec 2008 14:42:51 +0100 (CET) Subject: [pypy-svn] r60591 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219134251.3E666168480@codespeak.net> Author: davide Date: Fri Dec 19 14:42:49 2008 New Revision: 60591 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex Log: changes in reaction of Antonio's comments Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Fri Dec 19 14:42:49 2008 @@ -65,9 +65,11 @@ \item Each either primary or secondary method implements a certain number of blocks, all belonging to the same flow graph. Among these blocks - there always exists an initial block whose input variables are - parameters of the method; the input variables of all other blocks - are local variables of the method. \anto{This is wrong: the signature of the secondary methods is fixed, and input args are passed inside the InputArgs class, not as methodo parameters} + there always exists an initial block whose input arguments + might be passed as arguments of the method; however, for + implementation reasons (see the details below) the input variables + of all blocks (including the initial one) + are implemented as local variables of the method. \end{itemize} When a new case is added to a flexswitch, new blocks are generated @@ -82,13 +84,12 @@ by the same method, \emph{external} otherwise. -Following an internal link is not difficult in IL bytecode: a jump to -the corresponding code fragment in the same method is emitted -to execute the new block, whereas the appropriate local variables are -used for passing arguments. \anto{this is wrong for the same reason as above} -Also following an external link whose target is the initial block of a -method is not difficult: the corresponding method has to be invoked -with the appropriate arguments. +Following an internal link would not be difficult in IL bytecode: a jump to +the corresponding code fragment in the same method can be emitted +to execute the new block, whereas the appropriate local variables can be +used for passing arguments. +Also following an external link whose target is an initial block could +be easily implemented, by just invoking the corresponding method. What cannot be easily implemented in CLI is following an external link whose target is not an initial block; consider, for instance, the @@ -103,8 +104,8 @@ determine which block has to be executed. This is done by passing to the method a 32 bits number, called \emph{block id}, which uniquely identifies the next block of the graph to be executed. -The high word \anto{a word is 32 bit, block num and method id are 16 bit each} of a block id is the id of the method to which the block -belongs, whereas the low word is a progressive number univocally identifying +The high 2 bytes \dacom{word was meant as a fixed-sized group of bits} of a block id constitute the id of the method to which the block +belongs, whereas the low 2 bytes constitute a progressive number univocally identifying each block implemented by the method. The picture in Figure~\ref{block-id-fig} shows a graph composed of three methods (for @@ -142,8 +143,7 @@ \end{small} If the next block to be executed is implemented in the same method ({\small\lstinline{methodid == MY_METHOD_ID}}), then the appropriate -jump to the corresponding code is executed, hence internal links -can be managed efficiently. \anto{wrong: internal links don't go through the dispatcher} +jump to the corresponding code is executed. Otherwise, the \lstinline{jump_to_ext} part of the dispatcher has to be executed. The code that actually jumps to an external block is contained in From cfbolz at codespeak.net Fri Dec 19 15:13:35 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Dec 2008 15:13:35 +0100 (CET) Subject: [pypy-svn] r60592 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219141335.34159168458@codespeak.net> Author: cfbolz Date: Fri Dec 19 15:13:34 2008 New Revision: 60592 Added: pypy/extradoc/talk/ecoop2009/conclusion.tex Modified: pypy/extradoc/talk/ecoop2009/main.tex Log: add (empty) conclusion file Added: pypy/extradoc/talk/ecoop2009/conclusion.tex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/conclusion.tex Fri Dec 19 15:13:34 2008 @@ -0,0 +1,6 @@ +\section{Related Word} +XXX add me + +\section{Conclusion} + +XXX add me Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Fri Dec 19 15:13:34 2008 @@ -88,7 +88,7 @@ \input{rainbow} \input{clibackend} \input{benchmarks} -%\input{conclusion} +\input{conclusion} \bibliographystyle{plain} \bibliography{main} From antocuni at codespeak.net Fri Dec 19 15:24:25 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 19 Dec 2008 15:24:25 +0100 (CET) Subject: [pypy-svn] r60593 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219142425.144B3168462@codespeak.net> Author: antocuni Date: Fri Dec 19 15:24:24 2008 New Revision: 60593 Removed: pypy/extradoc/talk/ecoop2009/jitstrategy.tex Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex pypy/extradoc/talk/ecoop2009/intro.tex pypy/extradoc/talk/ecoop2009/jitgen.tex pypy/extradoc/talk/ecoop2009/main.tex Log: remove the jitstrategy section and integrate its contents into the introduction. Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Fri Dec 19 15:24:24 2008 @@ -1,4 +1,5 @@ \section{CLI backend for Rainbow interpreter} +\label{sec:clibackend} %\subsection{Promotion on CLI} % Modified: pypy/extradoc/talk/ecoop2009/intro.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/intro.tex (original) +++ pypy/extradoc/talk/ecoop2009/intro.tex Fri Dec 19 15:24:24 2008 @@ -79,29 +79,69 @@ \subsection{PyPy and JIT-Generation} -This section will give a high-level overview of how the JIT-generation process -works. More details will be given in subsequent sections. - -- write an interpreter -- add some hints to the interpreter - -translation time: -- start the translation process, flow graphs -- run a binding-time analysis using the hints, to figure out ... -- turn the flow graphs into rainbow bytecode -- turn the flow graphs of the rainbow interpreter and the original flow graphs -into .NET code - -runtime: -when the interpreter is started and a function should be interpreted: - - the generated JIT starts compiling the function - - it produces .NET bytecode until it does not have enough information to - continue. then it stops the compilation and puts a callback into the compiler - into the generated bytecode. - - at this point the .NET bytecode that is produced so far is executed. - - when the callback into the compiler is hit, execution stops and compilation - starts again - - afterwards execution continues after the callback (running the code that was - newly produced) - - +One of the most important aspects that PyPy's translation toolchain can weave +in is the \emph{automatic generation of a JIT compiler}. This section will +give a high-level overview of how the JIT-generation process works. More +details will be given in subsequent sections. + +The first step is to write an interpreter for the chosen language. Since it +must be fed to the translation toolchain, the interpreter has to be written in +RPython. Then, to get best performances, we need to add few manual +annotations to the interpreter, in order to teach the JIT generator which +informations are important to know at compile-time. Annotations are inserted +as \emph{hints}, as described in section \ref{sec:hints}. + +It is important to distinguish between three distinct phases of execution: + +\begin{enumerate} +\item \textbf{Translation-time}: when the translation toolchain runs and the + JIT compiler is automatically generated from the interpreter. The result is + an executable that can be used to run programs written in the chosen + language. +\item \textbf{Compile-time}: when the JIT compiler runs, generating executable + code on the fly. +\item \textbf{Runtime}: when the code generated at compile-time is executed. +\end{enumerate} + +Note that in this schema translation-time happens only once, on the +developer's machine. By contrast, compile-time and runtime happens every time +the user wants to run some program. + + +\subsection{Effective JIT compilation of dynamic languages} + +Generating efficient compilers for dynamic languages is hard. Since these +languages are dynamically typed, usually the compiler does not have enough +information to produce efficient code, but instead it has to insert a lot of +runtime checks to select the appropriate implementation for each operation. + +By emitting code at runtime, JIT compilers can exploit some extra knowledge +compared to traditional static compilers. However, we need to take special +care to choose a strategy for JIT compilation that lets the compiler to take +the best of this advantage. + +Most JIT compilers for dynamic languages around (such as IronPython, Jython, +JRuby \anto{XXX insert some reference}) compile code at the method +granularity. If on one hand they can exploit some of the knowledge gathered +at runtime (e.g. the types of method parameters), on the other hand they can +do little to optimize most of the operations inside, because their behaviour +depends on informations that are not available at compile-time, because +e.g. the global state of the program can change at runtime. \anto{e.g., we can + add/remove methods to classes, etc. Should we insert some example here?} + +JIT compilers generated by PyPy solve this problem by delaying the compilation +until they know all the informations needed to generate efficient code. If at +some point the JIT compiler does not know about something it needs, it +generates a callback into itself. + +Later, when the generated code is executed, the callback is hit and the JIT +compiler is restarted again. At this point, the JIT knows exactly the state +of the program and can exploit all this extra knowledge to generate highly +efficient code. Finally, the old code is patched and linked to the newly +generated code, so that the next time the JIT compiler will not be invoked +again. As a result, \textbf{runtime and compile-time are continuously + intermixed}. + +Modifying the old code to link the newly generated one is very challenging on +.NET, as the framework does not offer any primitive to do this. Section +\ref{sec:clibackend} explains how it is possible to simulate this behaviour. Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Fri Dec 19 15:24:24 2008 @@ -221,6 +221,7 @@ \end{itemize} \subsection{Hints} +\label{sec:hints} Our goal in designing our approach to binding-time analysis was to minimize the number of explicit hints that the user must provide in Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Fri Dec 19 15:24:24 2008 @@ -83,7 +83,6 @@ \input{abstract} \input{intro} \input{tlc} -\input{jitstrategy} \input{jitgen} \input{rainbow} \input{clibackend} From cfbolz at codespeak.net Fri Dec 19 15:59:27 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Dec 2008 15:59:27 +0100 (CET) Subject: [pypy-svn] r60595 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219145927.E0030168480@codespeak.net> Author: cfbolz Date: Fri Dec 19 15:59:26 2008 New Revision: 60595 Modified: pypy/extradoc/talk/ecoop2009/intro.tex pypy/extradoc/talk/ecoop2009/jitgen.tex Log: some fixes Modified: pypy/extradoc/talk/ecoop2009/intro.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/intro.tex (original) +++ pypy/extradoc/talk/ecoop2009/intro.tex Fri Dec 19 15:59:26 2008 @@ -86,7 +86,7 @@ The first step is to write an interpreter for the chosen language. Since it must be fed to the translation toolchain, the interpreter has to be written in -RPython. Then, to get best performances, we need to add few manual +RPython. Then, to guide the process, we need to add few manual annotations to the interpreter, in order to teach the JIT generator which informations are important to know at compile-time. Annotations are inserted as \emph{hints}, as described in section \ref{sec:hints}. @@ -107,9 +107,6 @@ developer's machine. By contrast, compile-time and runtime happens every time the user wants to run some program. - -\subsection{Effective JIT compilation of dynamic languages} - Generating efficient compilers for dynamic languages is hard. Since these languages are dynamically typed, usually the compiler does not have enough information to produce efficient code, but instead it has to insert a lot of @@ -122,26 +119,31 @@ Most JIT compilers for dynamic languages around (such as IronPython, Jython, JRuby \anto{XXX insert some reference}) compile code at the method -granularity. If on one hand they can exploit some of the knowledge gathered +granularity. If on the one hand they can exploit some of the knowledge gathered at runtime (e.g. the types of method parameters), on the other hand they can do little to optimize most of the operations inside, because their behaviour depends on informations that are not available at compile-time, because e.g. the global state of the program can change at runtime. \anto{e.g., we can add/remove methods to classes, etc. Should we insert some example here?} +\cfbolz{the last sentence is waaay too complicated} JIT compilers generated by PyPy solve this problem by delaying the compilation until they know all the informations needed to generate efficient code. If at some point the JIT compiler does not know about something it needs, it -generates a callback into itself. +generates a callback into itself and stops execution. -Later, when the generated code is executed, the callback is hit and the JIT +Later, when the generated code is executed, the callback might be hit and the JIT compiler is restarted again. At this point, the JIT knows exactly the state of the program and can exploit all this extra knowledge to generate highly efficient code. Finally, the old code is patched and linked to the newly generated code, so that the next time the JIT compiler will not be invoked again. As a result, \textbf{runtime and compile-time are continuously - intermixed}. +interleaved}. The primitive to do this sort of interleaving is called promotion, +it is described in Section \ref{sec:promotion}. + +One of the most important optimizations that the generated JIT does is removing +unnecessary allocations. This is described in Section \ref{sec:virtuals} -Modifying the old code to link the newly generated one is very challenging on +Modifying the old code to link to the newly generated one is very challenging on .NET, as the framework does not offer any primitive to do this. Section \ref{sec:clibackend} explains how it is possible to simulate this behaviour. Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Fri Dec 19 15:59:26 2008 @@ -267,6 +267,7 @@ This hint is a \emph{local} request for \texttt{v2} to be green, without requiring \texttt{v1} to be green. Note that this amounts to copying a red value into a green one, which is not possible in classical -approaches to partial evaluation. See the Promotion section XXX ref for a +approaches to partial evaluation. See Section \ref{sec:promotion} for a complete discussion of promotion. + From arigo at codespeak.net Fri Dec 19 15:59:46 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Dec 2008 15:59:46 +0100 (CET) Subject: [pypy-svn] r60596 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219145946.192B2168480@codespeak.net> Author: arigo Date: Fri Dec 19 15:59:44 2008 New Revision: 60596 Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex Log: Move the \labels to their proper location. Otherwise the references to them are bogus. Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Fri Dec 19 15:59:44 2008 @@ -57,10 +57,10 @@ reached given the concrete values. \begin{figure}[h] -\label{fig:tlc-main} \begin{center} \input{tlc-simplified.py} \caption{The main loop of the TLC interpreter, written in RPython} +\label{fig:tlc-main} \end{center} \end{figure} @@ -84,11 +84,11 @@ subtraction (\lstinline{sub}). \begin{figure}[h] -\label{fig:tlc-folded} \begin{center} \input{tlc-folded.py} \caption{The residual code of the TLC main loop with respect to the \lstinline{abs} function } +\label{fig:tlc-folded} \end{center} \end{figure} From cfbolz at codespeak.net Fri Dec 19 16:11:34 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Dec 2008 16:11:34 +0100 (CET) Subject: [pypy-svn] r60597 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219151134.2FB8116848F@codespeak.net> Author: cfbolz Date: Fri Dec 19 16:11:33 2008 New Revision: 60597 Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex pypy/extradoc/talk/ecoop2009/main.bbl pypy/extradoc/talk/ecoop2009/rainbow.tex Log: kill a now-unnecessary section Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Fri Dec 19 16:11:33 2008 @@ -149,33 +149,9 @@ XXX write something about the problems of classical PE? -\subsection{Partial Evaluation in PyPy} - - -PyPy contains a framework for generating just-in-time compilers using -off-line partial evaluation. As such, there are three distinct phases: - -\begin{itemize} -\item \emph{Translation time:} during the normal translation of an RPython -program, say the TLC interpreter, we perform binding-time analysis on the -interpreter. This produces a generating extension, which is linked with the -rest of the program. \cfbolz{XXX not quite right} - -\item \emph{Compile time:} during the execution of the program, when a new -bytecode is about to be interpreted, the generating extension is invoked -instead. As the generating extension is a compiler, all the computations it -performs are called compile-time computations. Its sole effect is to produce -residual code. - -\item \emph{Run time:} the normal execution of the program (which includes the -time spent running the residual code created by the generating extension). -\end{itemize} - -Translation time is a purely off-line phase; compile time and run time are -actually highly interleaved during the execution of the program. - -\subsection{Binding Time Analysis} +\subsection{Binding Time Analysis in PyPy} +XXX need better transition text from last section At translation time, PyPy performs binding-time analysis of the source RPython program after it has been turned to low-level graphs, i.e. at the level at which operations manipulate pointer-and-structure-like Modified: pypy/extradoc/talk/ecoop2009/main.bbl ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.bbl (original) +++ pypy/extradoc/talk/ecoop2009/main.bbl Fri Dec 19 16:11:33 2008 @@ -11,7 +11,8 @@ \bibitem{BolzEtAl08} C.~F. Bolz, A.~Kuhn, A.~Lienhard, N.~D. Matsakis, O.~Nierstrasz, L.~Renggli, A.~Rigo, and T.~Verwaest. -\newblock Back to the future in one week - implementing a smalltalk vm in pypy. +\newblock Back to the future in one week - implementing a smalltalk vm in + {P}y{P}y. \newblock In {\em Self-Sustaining Systems, First Workshop, S3 2008, Potsdam, Revised Selected Papers}, volume 5146 of {\em Lecture Notes in Computer Science}, pages 123--139, 2008. @@ -21,6 +22,13 @@ \newblock Partial evaluation of computation process, revisited. \newblock {\em Higher Order Symbol. Comput.}, 12(4):377--380, 1999. +\bibitem{hoelzle_optimizing_1991} +Urs H{\"o}lzle, Craig Chambers, and David Ungar. +\newblock Optimizing dynamically-typed object-oriented languages with + polymorphic inline caches. +\newblock In {\em Proceedings of the European Conference on Object-Oriented + Programming}, pages 21--38. Springer-Verlag, 1991. + \bibitem{RigoPedroni06} A.~Rigo and S.~Pedroni. \newblock Py{P}y's approach to virtual machine construction. Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Fri Dec 19 16:11:33 2008 @@ -23,6 +23,7 @@ TODO \section{Promotion} +\label{sec:promotion} In the sequel, we describe in more details one of the main new techniques introduced in our approach, which we call \emph{promotion}. In @@ -164,13 +165,9 @@ Virtual structures still avoid the run-time allocation of most short-lived objects, even in non-trivial situations. The following -example shows a typical case. XXX use TLC example - - -The PyPy Python interpreter -implements application-level integers as boxes ? instances of a -\texttt{W\_IntObject} class with a single \texttt{intval} field. Here is the -addition of two integers: +example shows a typical case. The TLC interpreter implements application-level +integers as boxes ? instances of an \texttt{IntObject} class with a single +field. XXX needs to use TLC examples \begin{verbatim} From cfbolz at codespeak.net Fri Dec 19 16:13:10 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Dec 2008 16:13:10 +0100 (CET) Subject: [pypy-svn] r60598 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219151310.D0B50168487@codespeak.net> Author: cfbolz Date: Fri Dec 19 16:13:10 2008 New Revision: 60598 Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex Log: add note to not forget Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Fri Dec 19 16:13:10 2008 @@ -238,6 +238,7 @@ it prevents under-specialization: an unsatisfiable \texttt{hint(v1, concrete=True)} is reported as an error. +XXX move this to the promotion section Promotion is invoked with the use of a hint as well: \texttt{v2 = hint(v1, promote=True)}. This hint is a \emph{local} request for \texttt{v2} to be green, without From antocuni at codespeak.net Fri Dec 19 16:16:25 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 19 Dec 2008 16:16:25 +0100 (CET) Subject: [pypy-svn] r60599 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219151625.34F03168487@codespeak.net> Author: antocuni Date: Fri Dec 19 16:16:24 2008 New Revision: 60599 Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex Log: swap promotion and virtuals sections Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Fri Dec 19 16:16:24 2008 @@ -1,26 +1,76 @@ -\section{The Rainbow Interpreter} +\section{Automatic Unboxing of Intermediate Results} +\label{sec:virtuals} + +XXX the following section needs a rewriting to be much more high-level and to +compare more directly with classical escape analysis + +Interpreters for dynamic languages typically continuously allocate a lot of small +objects, for example due to boxing. This makes arithmetic operations extremely +inefficient. For this reason, we +implemented a way for the compiler to try to avoid memory allocations in the +residual code as long as possible. The idea is to try to keep new +run-time structures "exploded": instead of a single run-time object allocated on +the heap, the object is "virtualized" as a set +of fresh local variables, one per field. Only when the object can be accessed by from +somewhere else is it actually allocated on the heap. The effect of this is similar to that of +escape analysis \cite{XXX}, which also prevents allocations of objects that can +be proven to not escape a method or set of methods. -The JIT compiler is implemented by running the "Rainbow interpreter", whose -output is a compiled function. +It is not always possible to keep structures virtual. The main +situation in which it needs to be "forced" (i.e. actually allocated at +run-time) is when the pointer escapes to some non-virtual location like +a field of a real heap structure. -Variables and opcodes are colored: +Virtual structures still avoid the run-time allocation of most +short-lived objects, even in non-trivial situations. The following +example shows a typical case. The TLC interpreter implements application-level +integers as boxes ? instances of an \texttt{IntObject} class with a single +field. - - green values are known at compile time, and green operations are executed - when interpreting the bytecode; - - red values are not known until runtime; the result of a red operation is a - piece of machine code that will compute the result at runtime. +The PyPy Python interpreter +implements application-level integers as boxes ? instances of a +\texttt{W\_IntObject} class with a single \texttt{intval} field. Here is the +addition of two integers: -The Rainbow bytecode is produced at translation time, when the JIT compiler is -generated. +XXX needs to use TLC examples +\begin{verbatim} + def add(w1, w2): # w1, w2 are W_IntObject instances + value1 = w1.intval + value2 = w2.intval + result = value1 + value2 + return W_IntObject(result) +\end{verbatim} -\cfbolz{XXX I think we should be very careful with the rainbow interp. it is a -total implementation-detail and we should only describe it as little as -possible} +XXX kill the rest?!? +When interpreting the bytecode for \texttt{a+b+c}, two calls to \texttt{add()} are +issued; the intermediate \texttt{W\_IntObject} instance is built by the first +call and thrown away after the second call. By contrast, when the +interpreter is turned into a compiler, the construction of the +\texttt{W\_IntObject} object leads to a virtual structure whose \texttt{intval} +field directly references the register in which the run-time addition +put its result. This location is read out of the virtual structure at +the beginning of the second \texttt{add()}, and the second run-time addition +directly operates on the same register. -\subsection{Example of Rainbow bytecode and execution} +An interesting effect of virtual structures is that they play nicely with +promotion. Indeed, before the interpreter can call the proper \texttt{add()} +function for integers, it must first determine that the two arguments +are indeed integer objects. In the corresponding dispatch logic, we +have added two hints to promote the type of each of the two arguments. +This produces a compiler that has the following behavior: in the general +case, the expression \texttt{a+b} will generate two consecutive run-time +switches followed by the residual code of the proper version of +\texttt{add()}. However, in \texttt{a+b+c}, the virtual structure representing +the intermediate value will contain a compile-time constant as type. +Promoting a compile-time constant is trivial ? no run-time code is +generated. The whole expression \texttt{a+b+c} thus only requires three +switches instead of four. It is easy to see that even more switches can +be skipped in larger examples; typically, in a tight loop manipulating +only integers, all objects are virtual structures for the compiler and +the residual code is theoretically optimal ? all type propagation and +boxing/unboxing occurs at compile-time. -TODO \section{Promotion} \label{sec:promotion} @@ -140,69 +190,3 @@ XXX maybe a tlc example can be used for the example above? -\section{Automatic Unboxing of Intermediate Results} -\label{sec:virtuals} - -XXX the following section needs a rewriting to be much more high-level and to -compare more directly with classical escape analysis - -Interpreters for dynamic languages typically continuously allocate a lot of small -objects, for example due to boxing. This makes arithmetic operations extremely -inefficient. For this reason, we -implemented a way for the compiler to try to avoid memory allocations in the -residual code as long as possible. The idea is to try to keep new -run-time structures "exploded": instead of a single run-time object allocated on -the heap, the object is "virtualized" as a set -of fresh local variables, one per field. Only when the object can be accessed by from -somewhere else is it actually allocated on the heap. The effect of this is similar to that of -escape analysis \cite{XXX}, which also prevents allocations of objects that can -be proven to not escape a method or set of methods. - -It is not always possible to keep structures virtual. The main -situation in which it needs to be "forced" (i.e. actually allocated at -run-time) is when the pointer escapes to some non-virtual location like -a field of a real heap structure. - -Virtual structures still avoid the run-time allocation of most -short-lived objects, even in non-trivial situations. The following -example shows a typical case. The TLC interpreter implements application-level -integers as boxes ? instances of an \texttt{IntObject} class with a single -field. - -XXX needs to use TLC examples -\begin{verbatim} - def add(w1, w2): # w1, w2 are W_IntObject instances - value1 = w1.intval - value2 = w2.intval - result = value1 + value2 - return W_IntObject(result) -\end{verbatim} - -XXX kill the rest?!? -When interpreting the bytecode for \texttt{a+b+c}, two calls to \texttt{add()} are -issued; the intermediate \texttt{W\_IntObject} instance is built by the first -call and thrown away after the second call. By contrast, when the -interpreter is turned into a compiler, the construction of the -\texttt{W\_IntObject} object leads to a virtual structure whose \texttt{intval} -field directly references the register in which the run-time addition -put its result. This location is read out of the virtual structure at -the beginning of the second \texttt{add()}, and the second run-time addition -directly operates on the same register. - -An interesting effect of virtual structures is that they play nicely with -promotion. Indeed, before the interpreter can call the proper \texttt{add()} -function for integers, it must first determine that the two arguments -are indeed integer objects. In the corresponding dispatch logic, we -have added two hints to promote the type of each of the two arguments. -This produces a compiler that has the following behavior: in the general -case, the expression \texttt{a+b} will generate two consecutive run-time -switches followed by the residual code of the proper version of -\texttt{add()}. However, in \texttt{a+b+c}, the virtual structure representing -the intermediate value will contain a compile-time constant as type. -Promoting a compile-time constant is trivial ? no run-time code is -generated. The whole expression \texttt{a+b+c} thus only requires three -switches instead of four. It is easy to see that even more switches can -be skipped in larger examples; typically, in a tight loop manipulating -only integers, all objects are virtual structures for the compiler and -the residual code is theoretically optimal ? all type propagation and -boxing/unboxing occurs at compile-time. From arigo at codespeak.net Fri Dec 19 16:17:46 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Dec 2008 16:17:46 +0100 (CET) Subject: [pypy-svn] r60600 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219151746.E18AF168487@codespeak.net> Author: arigo Date: Fri Dec 19 16:17:45 2008 New Revision: 60600 Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex Log: A missing dot. Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Fri Dec 19 16:17:45 2008 @@ -96,7 +96,7 @@ Promotion requires interleaving compile-time and run-time phases, otherwise the compiler can only use information that is known ahead of time. It is impossible in the "classical" approaches to partial -evaluation, in which the compiler always runs fully ahead of execution +evaluation, in which the compiler always runs fully ahead of execution. This is a problem in many realistic use cases. For example, in an interpreter for a dynamic language, there is mostly no information that can be clearly and statically used by the compiler before any From cfbolz at codespeak.net Fri Dec 19 16:20:28 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Dec 2008 16:20:28 +0100 (CET) Subject: [pypy-svn] r60601 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219152028.E3733168487@codespeak.net> Author: cfbolz Date: Fri Dec 19 16:20:28 2008 New Revision: 60601 Modified: pypy/extradoc/talk/ecoop2009/diagram1.odg pypy/extradoc/talk/ecoop2009/diagram1.pdf Log: Remove word "rainbow interpreter" from diagram. Also, change ?CLI Backend for flowgraphs? into ?CLI bytecode compiler?. Modified: pypy/extradoc/talk/ecoop2009/diagram1.odg ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/ecoop2009/diagram1.pdf ============================================================================== Binary files. No diff available. From cfbolz at codespeak.net Fri Dec 19 16:26:31 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Dec 2008 16:26:31 +0100 (CET) Subject: [pypy-svn] r60602 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219152631.E627F168402@codespeak.net> Author: cfbolz Date: Fri Dec 19 16:26:29 2008 New Revision: 60602 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex pypy/extradoc/talk/ecoop2009/rainbow.tex Log: eh. not that many references were left to the rainbow interp anyway. die die Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Fri Dec 19 16:26:29 2008 @@ -1,4 +1,4 @@ -\section{CLI backend for Rainbow interpreter} +\section{The CLI backend for the Generated JIT} \label{sec:clibackend} %\subsection{Promotion on CLI} Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Fri Dec 19 16:26:29 2008 @@ -147,6 +147,7 @@ happen how. I will try to come up with something different}. \begin{itemize} + XXX remove mention of rainbow interp. but this needs to be rewritten anyway \item (compile time): the rainbow interpreter produces machine code until it hits a promotion point; e.g.:: From antocuni at codespeak.net Fri Dec 19 16:50:56 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 19 Dec 2008 16:50:56 +0100 (CET) Subject: [pypy-svn] r60603 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219155056.10AC3168405@codespeak.net> Author: antocuni Date: Fri Dec 19 16:50:56 2008 New Revision: 60603 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex Log: (arigato around) add an XXX Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Fri Dec 19 16:50:56 2008 @@ -9,6 +9,9 @@ %To solve, we do xxx and yyy etc. etc. % +\anto{XXX: add an overview section of the backend, else we only talk about + flexswitches} + \subsection{Flexswitches} As already explained, \dacom{I guess flexswitches will be introduced From cfbolz at codespeak.net Fri Dec 19 16:57:10 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Dec 2008 16:57:10 +0100 (CET) Subject: [pypy-svn] r60604 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219155710.5D60E168405@codespeak.net> Author: cfbolz Date: Fri Dec 19 16:57:09 2008 New Revision: 60604 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex Log: expand on the XXX Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Fri Dec 19 16:57:09 2008 @@ -11,6 +11,8 @@ \anto{XXX: add an overview section of the backend, else we only talk about flexswitches} +\cfbolz{it should maybe also have a nice introductory paragraph how the +interface between the JIT works?} \subsection{Flexswitches} From cfbolz at codespeak.net Fri Dec 19 16:57:59 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Dec 2008 16:57:59 +0100 (CET) Subject: [pypy-svn] r60605 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219155759.7EDC4168403@codespeak.net> Author: cfbolz Date: Fri Dec 19 16:57:58 2008 New Revision: 60605 Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex Log: remove note that we agreed on IRC to ignore Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Fri Dec 19 16:57:58 2008 @@ -67,7 +67,7 @@ Let us look at an example. Figure \ref{fig:tlc-main} shows parts of the main-loop of the TLC interpreter (in slightly simplified form). The static arguments of this functions would typically be the \lstinline{code} argument -(containing the bytecode as a string \anto{what about using array of bytes instead of string? For .NET/JVM people, there is a strong difference between the twos}), the \lstinline{pc} argument (containing +(containing the bytecode as a string), the \lstinline{pc} argument (containing the initial program counter) and the \lstinline{pool} argument (containing the constant pool). The \lstinline{args} argument is dynamic since it contains the user-input of the interpreted function. Since the \lstinline{while} loop and the From cfbolz at codespeak.net Fri Dec 19 17:05:51 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Dec 2008 17:05:51 +0100 (CET) Subject: [pypy-svn] r60606 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219160551.72884168413@codespeak.net> Author: cfbolz Date: Fri Dec 19 17:05:49 2008 New Revision: 60606 Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex pypy/extradoc/talk/ecoop2009/rainbow.tex pypy/extradoc/talk/ecoop2009/tlc.tex Log: make a figure Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Fri Dec 19 17:05:49 2008 @@ -75,7 +75,7 @@ unrolled by a partial evaluator. When the function is partially evaluated with respect to the TLC example -function shown in Figure XXX (which computes the absolute value of a number), +function shown in Figure \ref{fig:tlc-abs} (which computes the absolute value of a number), the residual code would look like in Figure \ref{fig:tlc-folded}. This version is already a great improvement over pure interpretation, all the bytecode dispatch overhead has been removed. However, the function as shown is still Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Fri Dec 19 17:05:49 2008 @@ -147,7 +147,7 @@ happen how. I will try to come up with something different}. \begin{itemize} - XXX remove mention of rainbow interp. but this needs to be rewritten anyway + %XXX remove mention of rainbow interp. but this needs to be rewritten anyway \item (compile time): the rainbow interpreter produces machine code until it hits a promotion point; e.g.:: Modified: pypy/extradoc/talk/ecoop2009/tlc.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/tlc.tex (original) +++ pypy/extradoc/talk/ecoop2009/tlc.tex Fri Dec 19 17:05:49 2008 @@ -80,10 +80,12 @@ \subsection{TLC examples} As we said above, TLC exists only at bytecode level; to ease the development -of TLC programs, we wrote an assembler that generates TLC bytecode. The -following example shows a simple program that computes the absolute value of -the given integer: +of TLC programs, we wrote an assembler that generates TLC bytecode. Figure \ref{fig:tlc-abs} +shows a simple program that computes the absolute value of +the given integer. +\begin{figure}[h] +\begin{center} \begin{lstlisting} main: # stack: [] PUSHARG # [n] @@ -101,6 +103,11 @@ SUB # [-n] RETURN \end{lstlisting} +\caption{The TLC bytecode for computing the absolute value of a function} +\label{fig:tlc-abs} +\end{center} +\end{figure} + Since reading TLC programs at bytecode level is hard, in this paper we will use an invented Python-like syntax to describe examples, although we need to From antocuni at codespeak.net Fri Dec 19 17:10:34 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 19 Dec 2008 17:10:34 +0100 (CET) Subject: [pypy-svn] r60607 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219161034.D28F0168471@codespeak.net> Author: antocuni Date: Fri Dec 19 17:10:34 2008 New Revision: 60607 Added: pypy/extradoc/talk/ecoop2009/tlc-folded-virtualized.py (contents, props changed) Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex Log: add tlc example about virtuals Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Fri Dec 19 17:10:34 2008 @@ -19,40 +19,34 @@ It is not always possible to keep structures virtual. The main situation in which it needs to be "forced" (i.e. actually allocated at run-time) is when the pointer escapes to some non-virtual location like -a field of a real heap structure. +a field of a real heap structure. Virtual structures still avoid the run-time + allocation of most short-lived objects, even in non-trivial situations. -Virtual structures still avoid the run-time allocation of most -short-lived objects, even in non-trivial situations. The following -example shows a typical case. The TLC interpreter implements application-level -integers as boxes ? instances of an \texttt{IntObject} class with a single -field. - - -The PyPy Python interpreter -implements application-level integers as boxes ? instances of a -\texttt{W\_IntObject} class with a single \texttt{intval} field. Here is the -addition of two integers: - -XXX needs to use TLC examples -\begin{verbatim} - def add(w1, w2): # w1, w2 are W_IntObject instances - value1 = w1.intval - value2 = w2.intval - result = value1 + value2 - return W_IntObject(result) -\end{verbatim} +In addition to virtual structures, the compiler can also handle virtual +containers, namely lists and dictionaries \footnote{(R)Python's dictionaries + are equivalent to .NET \lstinline{Hashtable}s}. If the indexing operations +can be evaluated at compile-time (i.e., if the variables holding the indexes +are green), the compiler internally keeps track of the state of the container +and store the items as local variables. + +Look again at figure \ref{fig:tlc-folded}: the list in the \lstinline{stack} +variable never escapes from the function. Moreover, all the indexing +operations (either done explicitly or implicitly by \lstinline{append} and +\lstinline{pop}) are evaluable at compile-time. Thus, the list is kept +\emph{virtual} and its elements are stored in variables $v_n$, where $n$ +represents the index in the list. Figure \ref{fig:tlc-folded-virtualized} +show how the resulting code looks like; to ease the reading, the state of the +\lstinline{stack} as kept by the compiler is shown in the comments. + +\begin{figure}[h] +\begin{center} +\input{tlc-folded-virtualized.py} +\caption{The result of virtualizing the \lstinline{stack} list} +\label{fig:tlc-main} +\end{center} +\end{figure} XXX kill the rest?!? -When interpreting the bytecode for \texttt{a+b+c}, two calls to \texttt{add()} are -issued; the intermediate \texttt{W\_IntObject} instance is built by the first -call and thrown away after the second call. By contrast, when the -interpreter is turned into a compiler, the construction of the -\texttt{W\_IntObject} object leads to a virtual structure whose \texttt{intval} -field directly references the register in which the run-time addition -put its result. This location is read out of the virtual structure at -the beginning of the second \texttt{add()}, and the second run-time addition -directly operates on the same register. - An interesting effect of virtual structures is that they play nicely with promotion. Indeed, before the interpreter can call the proper \texttt{add()} function for integers, it must first determine that the two arguments Added: pypy/extradoc/talk/ecoop2009/tlc-folded-virtualized.py ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/tlc-folded-virtualized.py Fri Dec 19 17:10:34 2008 @@ -0,0 +1,17 @@ +\begin{lstlisting}[language=Python] +def interp_eval_abs(args): + v0 = args[0] # stack = [v0] + v1 = IntObj(0) # [v0, v1] + a, b = v0, v1 # [] + v0 = IntObj(b.lt(a))) # [v0] + cond = v0 # [] + if cond.istrue(): + v0 = args[0] # [v0] + return v0 + else: + v0 = IntObj(0) # [v0] + v1 = args[0] # [v0, v1] + a, b = v0, v1 # [] + v0 = b.sub(a) # [v0] + return v +\end{lstlisting} From antocuni at codespeak.net Fri Dec 19 17:42:16 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 19 Dec 2008 17:42:16 +0100 (CET) Subject: [pypy-svn] r60608 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219164216.909A91683EA@codespeak.net> Author: antocuni Date: Fri Dec 19 17:42:14 2008 New Revision: 60608 Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex pypy/extradoc/talk/ecoop2009/rainbow.tex Log: add one more para about virtuals, and start the example about promotion Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Fri Dec 19 17:42:14 2008 @@ -247,4 +247,4 @@ approaches to partial evaluation. See Section \ref{sec:promotion} for a complete discussion of promotion. - +\anto{We should at least mention the promote_class hint} Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Fri Dec 19 17:42:14 2008 @@ -4,6 +4,9 @@ XXX the following section needs a rewriting to be much more high-level and to compare more directly with classical escape analysis +\anto{Maybe we should talk about ``virtual instances'' and not structures, + considering the context} + Interpreters for dynamic languages typically continuously allocate a lot of small objects, for example due to boxing. This makes arithmetic operations extremely inefficient. For this reason, we @@ -46,6 +49,14 @@ \end{center} \end{figure} +Even if not shown in the example, \lstinline{stack} is not the only +virtualized object. In particular the two objects created by +\lstinline{IntObj(0)} are also virtualized, and their fields are stored as +local variables as well. Virtualizion of instances is important not only +because it avoids the allocation of unneeded temporary objects, but also +because it makes possible to optimize method calls on them, as the JIT +compiler knows their exact type in advance. + XXX kill the rest?!? An interesting effect of virtual structures is that they play nicely with promotion. Indeed, before the interpreter can call the proper \texttt{add()} @@ -106,7 +117,10 @@ the corresponding generated machine code contains chains of compare-and-jump instructions which are modified at run-time. These techniques also allow the gathering of information to direct inlining for even -better optimization results. +better optimization results. +\anto{What about this: While traditional PICs are only applied to indirect + calls, promotion is a more general operation that can be applied to any kind + of value, including instances of user-defined classes or integer numbers} In the presence of promotion, dispatch optimization can usually be reframed as a partial evaluation task. Indeed, if the type of the @@ -134,54 +148,27 @@ The new generated code is potentially different for each run-time value seen. This implies that the generated code needs to contain some sort -of updatable switch, which can pick the right code path based on the +of updatable switch, or \emph{flexswitch}, which can pick the right code path based on the run-time value. -\cfbolz{I think this example is confusing, it is unclear in which order things -happen how. I will try to come up with something different}. - -\begin{itemize} - %XXX remove mention of rainbow interp. but this needs to be rewritten anyway - \item (compile time): the rainbow interpreter produces machine code until it - hits a promotion point; e.g.:: - - \begin{lstlisting}[language=C] - y = hint(x, promote=True) - return y+10 - \end{lstlisting} - - \item (compile time): at this point, it generates special machine code that when - reached calls the JIT compiler again; the JIT compilation stops:: - - \begin{lstlisting}[language=C] - switch(y) { - default: compile_more(y); - } - \end{lstlisting} - - \item (runtime): the machine code is executed; when it reaches a promotion - point, it executes the special machine code we described in the previous - point; the JIT compiler is invoked again; - - \item (compile time): now we finally know the exact value of our red variable, - and we can promote it to green; suppose that the value of 'y' is 32:: - - \begin{lstlisting}[language=C] - switch(y) { - 32: return 42; - default: compile_more(y); - } - \end{lstlisting} - - Note that the operation "y+10" has been constant-folded into "42", as it - was a green operation. - - \item (runtime) the execution restart from the point it stopped, until a new - unhandled promotion point is reached. -\end{itemize} - -\subsection{Promotion as Applied to the TLC} - -XXX maybe a tlc example can be used for the example above? +Let us look again at the TLC example. To ease the reading, figure +\ref{fig:tlc-main} showed a simplified version of TLC's main loop, which did +not include the hints. The real implementation of the \lstinline{LT} opcode +is shown in figure \ref{fig:tlc-main-hints}. +\begin{figure}[h] +\begin{center} +\begin{lstlisting}[language=Python] + elif opcode == LT: + a, b = stack.pop(), stack.pop() + hint(a, promote_class=True) + hint(b, promote_class=True) + stack.append(IntObj(b.lt(a))) +\end{lstlisting} +\caption{Usage of hints in TLC's main loop} +\label{fig:tlc-main-hints} +\end{center} +\end{figure} +By promoting the class of \lstlisting{a} and \lstlisting{b}, we tell the JIT +compiler not to generate code until it knows the exact RPython class of both. From afa at codespeak.net Fri Dec 19 18:31:34 2008 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 19 Dec 2008 18:31:34 +0100 (CET) Subject: [pypy-svn] r60609 - pypy/trunk/pypy/translator/platform Message-ID: <20081219173134.E814416846D@codespeak.net> Author: afa Date: Fri Dec 19 18:31:32 2008 New Revision: 60609 Modified: pypy/trunk/pypy/translator/platform/windows.py Log: On Windows, build pypy-c with /O2 compiler optimization; or the stack overflows during "import site"... Next to come: increase the process stack size for debug builds. Modified: pypy/trunk/pypy/translator/platform/windows.py ============================================================================== --- pypy/trunk/pypy/translator/platform/windows.py (original) +++ pypy/trunk/pypy/translator/platform/windows.py Fri Dec 19 18:31:32 2008 @@ -52,7 +52,7 @@ cc = 'cl.exe' link = 'link.exe' - cflags = ['/MD'] + cflags = ['/MD', '/O2'] link_flags = [] standalone_only = [] shared_only = [] @@ -62,7 +62,7 @@ # Install debug options only when interpreter is in debug mode if sys.executable.lower().endswith('_d.exe'): - self.cflags = ['/MDd', '/Z7'] + self.cflags = ['/MDd', '/Z7', '/Od'] self.link_flags = ['/debug'] self.add_cpython_dirs = True From antocuni at codespeak.net Fri Dec 19 19:16:24 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 19 Dec 2008 19:16:24 +0100 (CET) Subject: [pypy-svn] r60611 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219181624.DAB48168471@codespeak.net> Author: antocuni Date: Fri Dec 19 19:16:24 2008 New Revision: 60611 Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex Log: work in progress, but they are waiting for me for dinner :-) Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Fri Dec 19 19:16:24 2008 @@ -170,5 +170,87 @@ \end{center} \end{figure} +\begin{figure}[h] +\begin{center} +\begin{lstlisting}[language=Python] +class IntObj(Obj): + ... + def lt(self, other): + return self.value < other.int_o() +\end{lstlisting} +\caption{Excerpt of the \lstlisting{IntObj} class} +\label{fig:tlc-intobj} +\end{center} +\end{figure} + By promoting the class of \lstlisting{a} and \lstlisting{b}, we tell the JIT compiler not to generate code until it knows the exact RPython class of both. +Figure \ref{fig:tlc-abs-promotion-1} shows the +code \footnote{\lstinline{switch} is not a legal (R)Python statement, it is + used here only as a pseudocode example} generated while compiling the usual +\lstlisting{abs} function: note that, compared to figure +\ref{fig:tlc-folded-virtualized}, the code stops just before the calls +\lstlisting{b.lt(a)}. + +\begin{figure}[h] +\begin{center} +\begin{lstlisting}[language=Python] +def interp_eval_abs(args): + v0 = args[0] + v1 = IntObj(0) + a, b = v0, v1 + hint(a, promote_class=True) # no-op + cls_a = a.__class__ + switch cls_a: + default: + continue_compilation(jitstate, cls_a) +\end{lstlisting} +\caption{Promotion example 1} +\label{fig:tlc-abs-promotion-1} +\end{center} +\end{figure} + +\begin{figure}[h] +\begin{center} +\begin{lstlisting}[language=Python] +def interp_eval_abs(args): + v0 = args[0] + v1 = IntObj(0) + a, b = v0, v1 + hint(a, promote_class=True) # no-op + cls_a = a.__class__ + switch cls_a: + IntObj: + hint(b, promote_class=True) + v0 = IntObj(a.value < b.int_o()) + ... + default: + continue_compilation(jitstate, cls_a) +\end{lstlisting} +\caption{Promotion example 2} +\label{fig:tlc-abs-promotion-2} +\end{center} +\end{figure} + +The first time the flexswitch is executed, the \lstlisting{default} branch is +taken, and the special function \lstlisting{continue_compilation} restarts the +JIT compiler, passing it the just-seen value of \lstlisting{cls_a}. The JIT +compiler generates new specialized code, and \emph{patches} the flexswitch to +add the new case, which is then executed. + +If later an instance of \lstlisting{IntObj} hits the flexswitch again, the +code is executed without needing of more calls to the JIT compiler. On the +other hand, if the flexswitch is hit by an instance of some other class, the +\lstlisting{default} branch will be selected again and the whole process will +restart. + +Now, let us examine the content of the \lstlisting{IntObj} case: first, there +is a hint to promote the class of \lstlisting{b}. Although in general +promotion is implemented through a flexswitch, in this case it is not needed +as \lstlisting{b} holds a \emph{virtual instance}, whose class is already +known (as described in previous section). + + +%since at this +%point we know the exact class of \lstinline{a} as we can see, the body of the +%\lstlisting{IntObj.lt} method has been inlined, From cfbolz at codespeak.net Fri Dec 19 19:29:25 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Dec 2008 19:29:25 +0100 (CET) Subject: [pypy-svn] r60613 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219182925.F36AA16847E@codespeak.net> Author: cfbolz Date: Fri Dec 19 19:29:23 2008 New Revision: 60613 Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex pypy/extradoc/talk/ecoop2009/rainbow.tex Log: fix an XXX Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Fri Dec 19 19:29:23 2008 @@ -136,17 +136,6 @@ function. The \emph{binding time} of a variable determines how early the value of the variable will be known. -\cfbolz{XXX: unclear how the next paragraph will fit into the text in the end. -it's certainly wrong as it is} -Once binding times have been determined, one possible approach to -producing the generating extension itself is by self-applying on-line -partial evaluators. This is known as the second Futamura projection -\cite{Futamura99}. So far it is unclear if this approach can lead to optimal -results, or even if it scales well. In PyPy we selected a more direct -approach: the generating extension is produced by transformation of the -control flow graphs of the interpreter, guided by the binding times. We -call this process \emph{timeshifting}. - XXX write something about the problems of classical PE? \subsection{Binding Time Analysis in PyPy} @@ -154,8 +143,8 @@ XXX need better transition text from last section At translation time, PyPy performs binding-time analysis of the source RPython program after it has been turned to low-level graphs, i.e. at -the level at which operations manipulate pointer-and-structure-like -objects. +the level at which operations manipulate objects that have a fixed type +attached. The binding-time terminology that we are using in PyPy is based on the colors that we use when displaying the control flow graphs: @@ -169,12 +158,11 @@ The binding-time analyzer of our translation tool-chain is using a simple abstract-interpretation based analysis. It is based on the -same type inference engine that is used on the source RPython program, -the annotator \anto{I'm not sure we should mention the annotator, as it is not referred anywhere else}. In this mode, it is called the \emph{hint-annotator}; it +same type inference engine that is used on the source RPython program. +This is called the \emph{hint-annotator}; it operates over input graphs that are already low-level instead of RPython-level, and propagates annotations that do not track types but value dependencies and manually-provided binding time hints. -XXX the above needs rewriting when the background section is there The normal process of the hint-annotator is to propagate the binding time (i.e. color) of the variables using the following kind of rules: @@ -238,13 +226,3 @@ it prevents under-specialization: an unsatisfiable \texttt{hint(v1, concrete=True)} is reported as an error. -XXX move this to the promotion section -Promotion is invoked with the use of a hint as well: -\texttt{v2 = hint(v1, promote=True)}. -This hint is a \emph{local} request for \texttt{v2} to be green, without -requiring \texttt{v1} to be green. Note that this amounts to copying -a red value into a green one, which is not possible in classical -approaches to partial evaluation. See Section \ref{sec:promotion} for a -complete discussion of promotion. - -\anto{We should at least mention the promote_class hint} Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Fri Dec 19 19:29:23 2008 @@ -138,6 +138,17 @@ primitive to make partial evaluation a practical approach to language independent dynamic compiler generation. +Promotion is invoked with the use of a hint as well: +\texttt{v2 = hint(v1, promote=True)}. +This hint is a \emph{local} request for \texttt{v2} to be green, without +requiring \texttt{v1} to be green. Note that this amounts to copying +a red value into a green one, which is not possible in classical +approaches to partial evaluation. A slightly different hint can be used to +promote the \emph{class} of an instance. This is done with \lstinline{hint(v1, +promote_class=True)}. It does not have an effect on the bindings of any +variable. + + \subsection{Implementing Promotion} The implementation of promotion requires a tight coupling between From antocuni at codespeak.net Fri Dec 19 19:42:38 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 19 Dec 2008 19:42:38 +0100 (CET) Subject: [pypy-svn] r60614 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219184238.736FC168490@codespeak.net> Author: antocuni Date: Fri Dec 19 19:42:38 2008 New Revision: 60614 Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex Log: finish the promotion example Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Fri Dec 19 19:42:38 2008 @@ -188,6 +188,8 @@ ... def lt(self, other): return self.value < other.int_o() + def int_o(self): + return self.value \end{lstlisting} \caption{Excerpt of the \lstlisting{IntObj} class} \label{fig:tlc-intobj} @@ -233,7 +235,7 @@ switch cls_a: IntObj: hint(b, promote_class=True) - v0 = IntObj(a.value < b.int_o()) + v0 = IntObj(b.value < a.value) ... default: continue_compilation(jitstate, cls_a) @@ -243,25 +245,29 @@ \end{center} \end{figure} -The first time the flexswitch is executed, the \lstlisting{default} branch is -taken, and the special function \lstlisting{continue_compilation} restarts the -JIT compiler, passing it the just-seen value of \lstlisting{cls_a}. The JIT +The first time the flexswitch is executed, the \lstinline{default} branch is +taken, and the special function \lstinline{continue_compilation} restarts the +JIT compiler, passing it the just-seen value of \lstinline{cls_a}. The JIT compiler generates new specialized code, and \emph{patches} the flexswitch to add the new case, which is then executed. -If later an instance of \lstlisting{IntObj} hits the flexswitch again, the +If later an instance of \lstinline{IntObj} hits the flexswitch again, the code is executed without needing of more calls to the JIT compiler. On the other hand, if the flexswitch is hit by an instance of some other class, the -\lstlisting{default} branch will be selected again and the whole process will +\lstinline{default} branch will be selected again and the whole process will restart. -Now, let us examine the content of the \lstlisting{IntObj} case: first, there -is a hint to promote the class of \lstlisting{b}. Although in general +Now, let us examine the content of the \lstinline{IntObj} case: first, there +is a hint to promote the class of \lstinline{b}. Although in general promotion is implemented through a flexswitch, in this case it is not needed -as \lstlisting{b} holds a \emph{virtual instance}, whose class is already +as \lstinline{b} holds a \emph{virtual instance}, whose class is already known (as described in previous section). - -%since at this -%point we know the exact class of \lstinline{a} as we can see, the body of the -%\lstlisting{IntObj.lt} method has been inlined, +Then, the compiler knows the exact class of \lstinline{b}, thus it can inline +the calls to \lstinline{lt}. Moreover, inside \lstinline{lt} there is a +call to \lstinline{a.int_o()}, which is inlined as well for the very same +reason. + +Moreover, as we saw in section \ref{sec:virtuals}, the \lstinline{IntObj} +instance can be virtualized, so that the subsequent \lstinline{BR_COND} opcode +can be compiled efficiently without needing any more flexswitch. From arigo at codespeak.net Fri Dec 19 19:51:03 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Dec 2008 19:51:03 +0100 (CET) Subject: [pypy-svn] r60616 - in pypy: avm branch/avm Message-ID: <20081219185103.1DB0316841D@codespeak.net> Author: arigo Date: Fri Dec 19 19:51:02 2008 New Revision: 60616 Added: pypy/branch/avm/ - copied from r60615, pypy/avm/ Removed: pypy/avm/ Log: Move this branch to 'branch'. From arigo at codespeak.net Fri Dec 19 19:52:54 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 19 Dec 2008 19:52:54 +0100 (CET) Subject: [pypy-svn] r60617 - pypy/branch/avm Message-ID: <20081219185254.1354116841D@codespeak.net> Author: arigo Date: Fri Dec 19 19:52:52 2008 New Revision: 60617 Removed: pypy/branch/avm/ Log: Remove this again. From magcius at codespeak.net Fri Dec 19 19:53:38 2008 From: magcius at codespeak.net (magcius at codespeak.net) Date: Fri, 19 Dec 2008 19:53:38 +0100 (CET) Subject: [pypy-svn] r60618 - pypy/branch/avm Message-ID: <20081219185338.13288168458@codespeak.net> Author: magcius Date: Fri Dec 19 19:53:37 2008 New Revision: 60618 Added: pypy/branch/avm/ - copied from r60617, pypy/trunk/ Log: Initial branch of PyPy with the experimental AVM backend. From magcius at codespeak.net Fri Dec 19 20:01:17 2008 From: magcius at codespeak.net (magcius at codespeak.net) Date: Fri, 19 Dec 2008 20:01:17 +0100 (CET) Subject: [pypy-svn] r60619 - in pypy/branch/avm/pypy/translator/avm: . test Message-ID: <20081219190117.A7A991683FD@codespeak.net> Author: magcius Date: Fri Dec 19 20:01:17 2008 New Revision: 60619 Added: pypy/branch/avm/pypy/translator/avm/ pypy/branch/avm/pypy/translator/avm/__init__.py pypy/branch/avm/pypy/translator/avm/avm.py pypy/branch/avm/pypy/translator/avm/avm1.py pypy/branch/avm/pypy/translator/avm/avm1gen.py pypy/branch/avm/pypy/translator/avm/avm1types.py pypy/branch/avm/pypy/translator/avm/bootstrap.py pypy/branch/avm/pypy/translator/avm/database.py pypy/branch/avm/pypy/translator/avm/function.py pypy/branch/avm/pypy/translator/avm/log.py pypy/branch/avm/pypy/translator/avm/metavm.py pypy/branch/avm/pypy/translator/avm/records.py pypy/branch/avm/pypy/translator/avm/support.py pypy/branch/avm/pypy/translator/avm/swf.py pypy/branch/avm/pypy/translator/avm/tags.py pypy/branch/avm/pypy/translator/avm/test/ pypy/branch/avm/pypy/translator/avm/test/browsertest.py pypy/branch/avm/pypy/translator/avm/test/runtest.py pypy/branch/avm/pypy/translator/avm/util.py Log: Aaaand the experimental non-functioning (yet) AVM1 backend. Added: pypy/branch/avm/pypy/translator/avm/__init__.py ============================================================================== Added: pypy/branch/avm/pypy/translator/avm/avm.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm/avm.py Fri Dec 19 20:01:17 2008 @@ -0,0 +1,66 @@ + +import py, os + +from pypy.rpython.rmodel import inputconst +from pypy.rpython.typesystem import getfunctionptr +from pypy.rpython.lltypesystem import lltype +from pypy.rpython.ootypesystem import ootype +from pypy.tool.udir import udir +from pypy.translator.avm.log import log +from pypy.translator.avm.asmgen import AsmGen +from pypy.translator.avm.avm1types import AVM1TypeSystem +#from pypy.translator.js.opcodes import opcodes +from pypy.translator.avm.function import Function +from pypy.translator.avm.database import LowLevelDatabase +from pypy.translator.avm import bootstrap + +from pypy.translator.oosupport.genoo import GenOO +from heapq import heappush, heappop + + +class AVM1(GenOO): + TypeSystem = AVMTypeSystem + Function = Function + Database = LowLevelDatabase + + def __init__(self, translator, function=[], stackless=False, compress=False, \ + logging=False, use_debug=False): + if not isinstance(function, list): + functions = [functions] + GenOO.__init__(self, udir, translator, None) + + pending_graphs = [ translator.annotator.bookeeeper.getdesc(f).cachedgraph(None) for f in functions ] + for graph in pending_graphs: + self.db.pending_function(graph) + + self.db.translator = translator + self.use_debug = use_debug + self.assembly_name = self.translator.graphs[0].name + self.tmpfile = udir.join(self.assembly_name + '.swf') + + def stack_optimization(self): + pass + + def gen_pendings(self): + while self.db._pending_nodes: + node = self.db.pending_nodes.pop() + to_render = [] + nparent = node + while nparent-order != 0: + nparent = nparent.parent + to_render.append(nparent) + to_render.reverse() + for i in to_render: + i.render(self.ilasm) + + def create_assembler(self): + return bootstrap.create_assembler(self.assembly_name) + + def generate_swf(self): + self.ilasm = self.create_assembler() + self.fix_names() + self.gen_entrypoint() + while self.db._pending_nodes: + self.gen_pendings() + self.db.gen_constants(self.ilasm, self.db._pending_nodes) + # self.ilasm.close() Added: pypy/branch/avm/pypy/translator/avm/avm1.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm/avm1.py Fri Dec 19 20:01:17 2008 @@ -0,0 +1,625 @@ + +# AVM1 = ActionScript Virtual Machine 1 +# Used for ActionScript 1 and 2 + +from pypy.translator.avm.util import BitStream +import struct + +class DataTypes(object): + + def __init__(self, id, name, size): + self.id = id + self.name = name + self.size = size + + def __str__(self): + return self.name + + def __call__(self, *a, **b): + pass + + __unicode__ = __str__ + +DataTypes.STRING = DataTypes(0, "string", "Z") +DataTypes.FLOAT = DataTypes(1, "float", "f") +DataTypes.NULL = DataTypes(2, "null", "!") +DataTypes.UNDEFINED = DataTypes(3, "undefined", "!") +DataTypes.REGISTER = DataTypes(4, "register", "B") +DataTypes.BOOLEAN = DataTypes(5, "boolean", "B") +DataTypes.DOUBLE = DataTypes(6, "double", "d") +DataTypes.INTEGER = DataTypes(7, "integer", "l") +DataTypes.CONSTANT8 = DataTypes(8, "constant 8", "B") +DataTypes.CONSTANT16 = DataTypes(9, "constant 16", "H") + +class Index(object): + def __init__(self, index): + self.index = index + +class Value(object): + def __init__(self, value): + self.value = value + +Null = object() +Null.type = DataTypes.Null + +Undefined = object() +Undefined.type = DataTypes.UNDEFINED + +class Constant(Index): + def __getattr__(self, name): + if name == "type": + if self.index < 256: + return DataTypes.CONSTANT8 + return DataTypes.CONSTANT16 + return Index.__getattr__(self, name) + +class RegisterByIndex(Index): + type = DataTypes.REGISTER + +class RegisterByValue(Value): + type = DataTypes.REGISTER + +class Action(object): + + ACTION_NAME = "NotImplemented" + ACTION_ID = 0x00 + + def __init__(self): + self.offset = 0 + self.label_name = "" + + def serialize(self): + inner_data = self.gen_data() + outer_data = self.gen_outer_data() + header = struct.pack("BH", self.ACTION_ID, len(inner_data)) + return bytes + inner_data + outer_data + + def __len__(self): + return 6 + len(self.gen_data()) + len(self.gen_outer_data()) + + def gen_data(self): + raise NotImplementedError, "Action::gen_data is not implemented in the abstract class" + + def gen_outer_data(self): + raise NotImplementedError, "Action::gen_outer_data is not implemented in the abstract class" + + def get_block_props_early(self, block): + raise NotImplementedError, "Action::get_block_props_early is not implemented in the abstract class" + + def get_block_props_late(self, block): + raise NotImplementedError, "Action::get_block_props_late is not implemented in the abstract class" + +class RegisterError(IndexError): + pass + +global_registers = [] + +class Block(object): + + AUTO_LABEL_TEMPLATE = "label%d" + MAX_REGISTERS = 4 + FUNCTION_TYPE = 0 + + def __init__(self, insert_end=False): + self.code = "" + self.__sealed = False + self.insert_end = insert_end + + self.labels = {} + self.branch_blocks = {} + self.constants = ActionConstantPool() + self.actions = [self.constants] + + self.current_offset = 0 + self.label_count = 0 + self.registers = global_registers + + def get_free_register(self): + if None in self.registers: + return self.registers.index(None) + elif len(self.registers) < self.MAX_REGISTERS: + return len(self.registers) + else: + raise RegisterError, "maximum number of registers in use" + + def store_register(self, value, index=-1): + if value in self.registers: + return self.registers.index(value) + if index < 1: + index = self.get_free_register() + self.registers[index] = value + return index + + def find_register(self, value): + if value in self.registers: + return self.registers.index(value) + return -1 + + def free_register(self, index): + self.registers[index] = None + + def __len__(self): + return self.current_offset + (2 if self.insert_end else 0) + + def seal(self): + self.__sealed = True + return len(self) + + def is_sealed(self): + return self.__sealed + + def add_action(self, action): + if self.__sealed: + raise Exception, "Block is sealed. Cannot add new actions" + self.code = "" # Dirty the code. + action.offset = self.current_offset + action.get_block_props_early(self) + + # Do some early optimizations. Combine two pushes into one. + if isinstance(action, ActionPush) and isinstance(self.actions[-1], ActionPush): + old_action = self.actions[-1] + old_len = len(old_action) + self.actions[-1].values.extend(action.values) + self.current_offset += len(old_action) - old_len + return None + + # Two nots negate. Take them out. + if isinstance(action, ActionNot) and isinstance(self.actions[-1], ActionNot): + self.actions.pop() + self.current_offset -= 1 # len(ShortAction) is 1 + return None + + if not isinstance(action, Block): # Don't add block length until we've finalized. + self.current_offset += len(action) + + self.actions.append(action) + return action + + def serialize(self): + if len(self.code) > 0: + return self.code + bytes = [] + for action in self.actions: + action.get_block_props_late(self) + bytes += action.serialize() + if self.insert_end: + bytes += "\0\0" + self.code = "".join(bytes) + return self.code + + def new_label(self): + self.label_count += 1 + name = Block.AUTO_LABEL_TEMPLATE % self.label_count + self.labels[name] = -1 + return name + + def set_label_here(self, name): + self.labels[name] = self.current_offset + + def new_label_here(self): + name = self.new_label() + self.labels[name] = self.current_offset + return name + +class ActionCall(Action): + ACTION_NAME = "ActionCall" + ACTION_ID = 0x9e + +class ActionConstantPool(Action): + ACTION_NAME = "ActionConstantPool" + ACTION_ID = 0x88 + + def __init__(self, *args): + self.pool = [] + for string in args: + if not string in self.pool: + self.pool.append(args) + + def add_constant(self, string): + if not string in self.pool: + self.pool.append(string) + return len(self.pool)-1 + return self.pool.index(string) + + def gen_data(self): + return struct.pack("H", len(self.pool)) + "".join([item + "\0" for item in self.pool]) + +class ActionDefineFunction(Block, Action): + ACTION_NAME = "ActionDefineFunction" + ACTION_ID = 0x9b + FUNCTION_TYPE = 1 + + def __init__(self, name, parameters): + Block.__init__(self, False) + self.function_name = name + self.params = params + + def gen_data(self): + self.block_data = Block.serialize(self) + bytes = [self.function_name, "\0", struct.pack("H", len(self.params))] + bytes += [p + "\0" for p in self.params] + bytes += struct.pack("H", len(self.block_data)) + return "".join(bytes) + + def gen_outer_data(self): + return self.block_data + +class ActionDefineFunction2(Block, Action): + ACTION_NAME = "ActionDefineFunction2" + ACTION_ID = 0x8e + MAX_REGISTERS = 256 + FUNCTION_TYPE = 2 + + def __init__(self, name, parameters, flags=0): + Block.__init__(self, False) + self.function_name = name + self.params = params + self.flags = flags & 0xFF01 + self.eval_flags() + + def find_register(self): + if value in self.registers: + return self.registers.index(value)+1 + return -1 + + def eval_flags(self): # WARNING! eval_flags will clear registers! + bits = BitStream() + bits.write_bit_value(self.flags, 16) + bits.rewind() + preload_parent = bits.read_bit() + preload_root = bits.read_bit() + suppress_super = bits.read_bit() + preload_super = bits.read_bit() + suppress_args = bits.read_bit() + preload_args = bits.read_bit() + suppress_this = bits.read_bit() + preload_this = bits.read_bit() + bits.cursor += 7 # skip over 7 Reserved bits + preload_global = bits.read_bit() + + # According to the docs, this is the order of register allocation. + if preload_this: self.registers.append("this") + if preload_args: self.registers.append("arguments") + if preload_super: self.registers.append("super") + if preload_root: self.registers.append("_root") + if preload_parent: self.registers.append("_parent") + if preload_global: self.registers.append("_global") + + for name in self.params: + self.registers.append(name) + + def gen_data(self): + self.block_data = Block.serialize(self) + params = [p for p in self.params if isinstance(p, RegisterParam)] + bytes = [self.function_name, "\0", struct.pack("HBH", len(params), len(self.registers)-1, self.flags)] + for name in self.params: + bytes += [chr(self.registers.index(name)+1), name, "\0"] + + bytes += [struct.pack("H", len(self.block_data))] + return "".join(bytes) + + def gen_outer_data(self): + return self.block_data + +class ActionGetURL(Action): + ACTION_NAME = "ActionGetURL" + ACTION_ID = 0x83 + + def __init__(self, url, target=""): + self.url = url + self.target = target + + def gen_data(self): + return "%s\0%s\0" % (self.url, self.target) + +class ActionGetURL2(Action): + ACTION_NAME = "ActionGetURL2" + ACTION_ID = 0x9a + + METHOD_NONE = 0 + METHOD_GET = 1 + METHOD_POST = 2 + + def __init__(self, method, load_target=False, load_variables=False): + self.method = max(method, 2) + self.load_target = load_target + self.load_variables = load_varibles + + def gen_data(self): + bits = BitStream() + bits.write_bit_value(self.method, 2) + bits.zero_fill(4) + bits.write_boolean(load_target) + bits.write_boolean(load_variables) + return bits.serialize() + +class ActionGotoFrame(Action): + ACTION_NAME = "ActionGotoFrame" + ACTION_ID = 0x81 + + def __init__(self, index): + self.index = index + + def gen_data(self): + return struct.pack("H", self.index) + +class ActionGotoFrame2(Action): + ACTION_NAME = "ActionGotoFrame2" + ACTION_ID = 0x9f + + def __init__(self, play=False, scene_bias=0): + self.play = play + self.scene_bias = scene_bias + + def gen_data(self): + bits = BitStream() + bits.zero_fill(6) + bits.write_boolean(self.scene_bias > 0) + bits.write_boolean(self.play) + + if self.scene_bias > 0: + return bits.serialize() + struct.pack("H", sceneBias) + + return bits.serialize() + +class ActionGotoLabel(Action): + ACTION_NAME = "ActionGotoLabel" + ACTION_ID = 0x81 + + def __init__(self, label_name): + self.label_name = label_name + + def serialize(self): + return self.label_name + "\0" + +class BranchingActionBase(Action): + + def __init__(self, branch): + if isinstance(branch, str): + self.branch_label = branch + self.branch_offset = 0 + elif isinstance(branch, int): + self.branch_label = None + self.branch_offset = branch + + def get_block_props_late(self, block): + if len(self.branch_label) > 0: + self.branch_offset = block.labels[self.branch_label] - self.offset + + def gen_data(self): + return struct.pack("H", self.branch_offset) + +class ActionJump(BranchingActionBase): + ACTION_NAME = "ActionJump" + ACTION_ID = 0x99 + +class ActionIf(BranchingActionBase): + ACTION_NAME = "ActionIf" + ACTION_ID = 0x9d + +class ActionPush(Action): + ACTION_NAME = "ActionPush" + ACTION_ID = 0x96 + + def __init__(self, *args): + self.values = [] + self.add_element(args) + + def add_element(self, element): + if isinstance(element, (list, tuple)): + for el in element: + self.add_element(el) + elif hasattr(element, "type"): + self.values.append((0, type)) + elif isinstance(element, str): + self.values.append((element, DataTypes.STRING)) + elif isinstance(element, bool): + self.values.append((element, DataTypes.BOOLEAN)) + elif isinstance(element, int): + self.values.append((element, DataTypes.INTEGER)) + elif isinstance(element, float): + if element > 0xFFFFFFFF: + self.values.append((element, DataTypes.DOUBLE)) + else: + self.values.append((element, DataTypes.FLOAT)) + elif isinstance(element, Index): + self.values.append((element.index, element.type)) + elif isinstance(element, RegisterByValue): + self.values.append((element.value, RegisterByValue)) + + def get_block_props_early(self, block): + for index, (value, type) in enumerate(self.values): + if type == DataTypes.STRING: + constant_index = block.constants.add_constant(value) + self.values[index] = (constant_index, DataTypes.CONSTANT8 if constant_index < 256 else DataTypes.CONSTANT16) + elif type == RegisterByValue: + register_index = block.store_register(value) + self.values[index] = (register_index, DataTypes.REGISTER) + + def gen_data(self): + bytes = [] + for value, type in self.values: + bytes += chr(type.id) + if type.size == "Z": + bytes += [value, "\0"] + elif type.size != "!": + bytes += struct.pack(type.size, value)[0] + return "".join(bytes) + +class ActionSetTarget(object): + ACTION_NAME = "ActionSetTarget" + ACTION_ID = 0x8b + + def __init__(self, target): + self.target = target + + def gen_data(self): + return self.target + "\0" + +class ActionStoreRegister(object): + ACTION_NAME = "ActionStoreRegister" + ACTION_ID = 0x87 + + def __init__(self, index): + self.index = index + + def gen_data(self): + return chr(index) + +class ActionTry(object): + ACTION_NAME = "ActionTry" + ACTION_ID = 0x8f + + def __init__(self, catch_object, try_block=None, catch_block=None, finally_block=None): + + self.catch_object = catch_object + + self.try_block = try_block or Block() + self.catch_block = catch_block or Block() + self.finally_block = finally_block or Block() + + def gen_data(self): + has_catch_block = len(self.catch_block.actions) > 0 + bits = BitStream() + bits.zero_fill(5) + bits.write_boolean(isinstance(self.catch_object, Register)) + bits.write_boolean(len(self.finally_block.actions) > 0) + bits.write_boolean(has_catch_block) + bytes = [bits.serialize()] + bytes += [struct.pack("HHH", len(self.try_block) + 5 if has_catch_block else 0, len(self.catch_block), len(finallyBlock))] + bytes += self.catch_object.index if isinstance(self.catch_object, Register) else (self.catch_object + "\0") + return "".join(bytes) + + def gen_outer_data(self): + bytes = [self.try_block.serialize()] + if len(self.catch_block.actions) > 0: + bytes += ActionJump(len(self.catch_block)).serialize() + bytes += catchBlock.serialize() + bytes += self.finally_block.serialize() + +class ActionWaitForFrame(Action): + ACTION_NAME = "ActionWaitForFrame" + ACTION_ID = 0x8a + + def __init__(self, index, skip_count=0): + self.index = index + self.skip_count = skip_count + + def gen_data(self): + return struct.pack("HB", self.index, self.skip_count) + +class ActionWaitForFrame2(Action): + ACTION_NAME = "ActionWaitForFrame2" + ACTION_ID = 0x8d + + def __init__(self, skip_count=0): + self.skip_count = skip_count + + def gen_data(self): + return chr(self.skip_count) + +class ActionWith(Action): + ACTION_NAME = "ActionWith" + ACTION_ID = 0x94 + + def __init__(self, with_block): + self.block = with_block or Block() + + def gen_data(self): + return struct.pack("H", len(block)) + block.serialize() + +class ShortAction(Action): + + def __call__(self, *args, **kwargs): + return self + + def __init__(self, id, name): + self.ACTION_ID = id + self.ACTION_NAME = name + Action.__init__(self) + + def __len__(self): + return 1 # 1 (Action ID) + + def serialize(self) + return chr(self.ACTION_ID) + +ActionNextFrame = ShortAction(0x04, "ActionNextFrame") +ActionPreviousFrame = ShortAction(0x05, "ActionPreviousFrame") +ActionPlay = ShortAction(0x06, "ActionPlay") +ActionStop = ShortAction(0x07, "ActionStop") +ActionToggleQuality = ShortAction(0x08, "ActionToggleQuality") +ActionStopSounds = ShortAction(0x09, "ActionStopSounds") +ActionAdd = ShortAction(0x0a, "ActionAdd") +ActionSubtract = ShortAction(0x0b, "ActionSubtract") +ActionMultiply = ShortAction(0x0c, "ActionMultiply") +ActionDivide = ShortAction(0x0d, "ActionDivide") +ActionEquals = ShortAction(0x0e, "ActionEquals") +ActionLess = ShortAction(0x0f, "ActionLess") +ActionAnd = ShortAction(0x10, "ActionAnd") +ActionOr = ShortAction(0x11, "ActionOr") +ActionNot = ShortAction(0x12, "ActionNot") +ActionStringEquals = ShortAction(0x13, "ActionStringEquals") +ActionStringLength = ShortAction(0x14, "ActionStringLength") +ActionStringExtract = ShortAction(0x15, "ActionStringExtract") +ActionPop = ShortAction(0x17, "ActionPop") +ActionToInteger = ShortAction(0x18, "ActionToInteger") +ActionGetVariable = ShortAction(0x1c, "ActionGetVariable") +ActionSetVariable = ShortAction(0x1d, "ActionSetVariable") +ActionSetTarget2 = ShortAction(0x20, "ActionSetTarget2") +ActionStringAdd = ShortAction(0x21, "ActionStringAdd") +ActionGetProperty = ShortAction(0x22, "ActionGetProperty") +ActionSetProperty = ShortAction(0x23, "ActionSetProperty") +ActionCloneSprite = ShortAction(0x24, "ActionCloneSprite") +ActionRemoveSprite = ShortAction(0x25, "ActionRemoveSprite") +ActionTrace = ShortAction(0x26, "ActionTrace") +ActionStartDrag = ShortAction(0x27, "ActionStartDrag") +ActionEndDrag = ShortAction(0x28, "ActionEndDrag") +ActionStringLess = ShortAction(0x29, "ActionStringLess") +ActionThrow = ShortAction(0x2a, "ActionThrow") +ActionCastOp = ShortAction(0x2b, "ActionCastOp") +ActionImplementsOp = ShortAction(0x2c, "ActionImplementsOp") +ActionRandomNumber = ShortAction(0x30, "ActionRandomNumber") +ActionMBStringLength = ShortAction(0x31, "ActionMBStringLength") +ActionCharToAscii = ShortAction(0x32, "ActionCharToAscii") +ActionAsciiToChar = ShortAction(0x33, "ActionAsciiToChar") +ActionGetTime = ShortAction(0x34, "ActionGetTime") +ActionMBStringExtract = ShortAction(0x35, "ActionMBStringExtract") +ActionMBCharToAscii = ShortAction(0x36, "ActionMBCharToAscii") +ActionMBAsciiToChar = ShortAction(0x37, "ActionMBAsciiToChar") +ActionDelVar = ShortAction(0x3a, "ActionDelVar") +ActionDelThreadVars = ShortAction(0x3b, "ActionDelThreadVars") +ActionDefineLocalVal = ShortAction(0x3c, "ActionDefineLocalVal") +ActionCallFunction = ShortAction(0x3d, "ActionCallFunction") +ActionReturn = ShortAction(0x3e, "ActionReturn") +ActionModulo = ShortAction(0x3f, "ActionModulo") +ActionNewObject = ShortAction(0x40, "ActionNewObject") +ActionDefineLocal = ShortAction(0x41, "ActionDefineLocal") +ActionInitArray = ShortAction(0x42, "ActionInitArray") +ActionInitObject = ShortAction(0x43, "ActionInitObject") +ActionTypeof = ShortAction(0x44, "ActionTypeof") +ActionGetTargetPath = ShortAction(0x45, "ActionGetTargetPath") +ActionEnumerate = ShortAction(0x46, "ActionEnumerate") +ActionTypedAdd = ShortAction(0x47, "ActionTypedAdd") +ActionTypedLessThan = ShortAction(0x48, "ActionTypedLessThan") +ActionTypedEquals = ShortAction(0x49, "ActionTypedEquals") +ActionConvertToNumber = ShortAction(0x4a, "ActionConvertToNumber") +ActionConvertToString = ShortAction(0x4b, "ActionConvertToString") +ActionDuplicate = ShortAction(0x4c, "ActionDuplicate") +ActionSwap = ShortAction(0x4d, "ActionSwap") +ActionGetMember = ShortAction(0x4e, "ActionGetMember") +ActionSetMember = ShortAction(0x4f, "ActionSetMember") +ActionIncrement = ShortAction(0x50, "ActionIncrement") +ActionDecrement = ShortAction(0x51, "ActionDecrement") +ActionCallMethod = ShortAction(0x52, "ActionCallMethod") +ActionCallNewMethod = ShortAction(0x53, "ActionCallNewMethod") +ActionBitAnd = ShortAction(0x60, "ActionBitAnd") +ActionBitOr = ShortAction(0x61, "ActionBitOr") +ActionBitXor = ShortAction(0x62, "ActionBitXor") +ActionShiftLeft = ShortAction(0x63, "ActionShiftLeft") +ActionShiftRight = ShortAction(0x64, "ActionShiftRight") +ActionShiftUnsigned = ShortAction(0x65, "ActionShiftUnsigned") +ActionStrictEquals = ShortAction(0x66, "ActionStrictEquals") +ActionGreater = ShortAction(0x67, "ActionGreater") +ActionStringGreater = ShortAction(0x68, "ActionStringGreater") +ActionExtends = ShortAction(0x69, "ActionExtends") Added: pypy/branch/avm/pypy/translator/avm/avm1gen.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm/avm1gen.py Fri Dec 19 20:01:17 2008 @@ -0,0 +1,216 @@ + +""" backend generator routines +""" + +from pypy.translator.avm import avm1 + + +class AsmGen(object): + """ AVM1 'assembler' generator routines """ + def __init__(self, name, block=None): + self.name = name + self.block = block or Block(True) + self.scope = (self.block, None, None) # (Code object, Parent scope, Exit callback) + + def new_label(self): + return self.scope[0].new_label() + + def store_register(self, *args, **kwargs): + return self.scope[0].store_register(*args, **kwargs) + + def find_register(self, value): + return self.scope[0].find_register(value) + + def action(self, action): + return self.scope[0].add_action(action) + + def enter_scope(self, new_code_obj, exit_callback): + self.scope = (new_code_obj, self.scope, exit_callback) + + def in_function(self): + return self.block.FUNCTION_TYPE + + def exit_scope(self): + block_len = self.finalize_block(self, self.scope[0]) + exit_callback = self.scope[2] + + # Go up to the parent scope. + self.scope = self.scope[1] + + self.scope[0].current_offset += len(block_len) + if exit_callback is not None: + exit_callback(self) + + def finalize_block(self, block): + for label, branch_block in block.branch_blocks.iteritems(): + if not branch_block.is_sealed(): + raise Exception, "Branch block hasn't been finalized" + # Set the label. + block.labels[label] = block.current_offset + # Add the actions, which updates current_offset + for act in branch_block.actions: + block.add_action(act) + return block.seal() + + + def begin_function(self, name, arglist): + self.enter_scope(self.action(avm1.ActionDefineFunction2(name, arglist))) + + def begin_prototype_method(self, function_name, _class, arglist): + self.action(avm1.ActionPush(function_name, "prototype", _class)) + self.action(avm1.ActionGetVariable()) + self.action(avm1.ActionGetMember()) + self.enter_scope(self.action(avm1.ActionDefineFunction2("", arglist, 0)), lambda s: self.action(avm1.ActionSetMember())) + + def set_variable(self): + self.action(avm1.ActionSetVariable()) + + def set_member(self): + self.action(avm1.ActionSetMember()) + + def get_member(self): + self.action(avm1.ActionGetMember()) + + def push_arg(self, v): + if self.in_function() == 0: + raise Exception, "avm1::push_arg called while not in function scope." + elif self.in_function() == 1: + self.push_local(v.name) + else: + self.action(avm1.ActionPush(RegisterByValue(v.name))) + +# def push_value(self, v): +# self.action(avm1.ActionPush(v.name)) + + def push_this(self, v): + k = self.find_register("this") + if k > 0: + self.action(avm1.ActionPush(RegisterByIndex(k))) + else: + self.action(avm1.ActionPush("this")) + self.action(avm1.ActionGetVariable()) + + def push_local(self, v): + k = self.find_register(v.name) + if k > 0: + self.action(avm1.ActionPush(RegisterByIndex(k))) + else: + self.action(avm1.ActionPush(v.name)) + self.action(avm1.ActionGetVariable()) + + def push_const(self, v): + self.action(avm1.ActionPush(v)) + + def push_undefined(self): + self.action(avm1.ActionPush(avm1.Undefined)) + + def push_null(self): + self.action(avm1.ActionPush(avm1.Null)) + + def return_stmt(self): + self.action(avm1.ActionReturn()) + + def is_equal(self, value=None): + if value is not None: + self.action(avm1.ActionPush(value)) + self.action(avm1.ActionEquals()) + + def is_not_equal(self, value=None): + self.is_equal(value) + self.action(avm1.ActionNot()) + + def init_object(self, dict=None): + if dict is not None: + self.action(avm1.ActionPush([(b,a) for (a,b) in dict.iteritems()], len(dict))) + self.action(avm1.ActionInitObject()) + + def init_array(self, list=None): + if list is not None: + self.action(avm1.ActionPush(list, len(list))) + self.action(avm1.ActionInitArray()) + + def call_function(self, func_name): # Assumes the args and number of args are on the stack. + self.action(avm1.ActionPush(func_name)) + self.action(avm1.ActionCallFunction()) + + def call_function_constargs(self, func_name, *args): + self.action(avm1.ActionPush(args, len(args), func_name)) + self.action(avm1.ActionCallFunction()) + + def call_method(self, func_name): # Assumes the args and number of args and ScriptObject are on the stack, in that order. + self.action(avm1.ActionPush(func_name)) + self.action(avm1.ActionCallMethod()) + + def call_method_constvar(self, _class, func_name): # Assumes vars and number of args are on the stack. + self.action(avm1.ActionPush(_class)) + self.action(avm1.ActionGetVariable()) + self.action(avm1.ActionPush(func_name)) + self.action(avm1.ActionCallMethod()) + + def branch_if_true(self, label=""): # Boolean value should be on stack when this is called + if len(label) < 0: + label = self.new_label() + self.scope[0].branch_blocks[label] = Block(False) + self.action(avm1.ActionIf(label)) + return (label, self.scope[0].branch_blocks[label]) + + def set_proto_field(self, _class, member_name): # Assumes the value is on the stack. + self.action(avm1.ActionPush(member_name, "prototype", _class)) + self.action(avm1.ActionGetVariable()) + self.action(avm1.ActionGetMember()) + self.action(avm1.ActionSwap()) + self.action(avm1.ActionSetMember()) + + def set_static_field(self, _class, member_name): # Assumes the value is on the stack. + self.action(avm1.ActionPush(member_name, _class)) + self.action(avm1.ActionGetVariable()) + self.action(avm1.ActionSwap()) + self.action(avm1.ActionSetMember()) + + def newobject_constthis(self, obj, *args): # If no args are passed then it is assumed that the args and number of args are on the stack. + if len(args) > 0: + self.action(avm1.ActionPush(args, len(args), obj)) + else: + self.action(avm1.ActionPush(obj)) + self.action(avm1.ActionNewObject()) + + def newobject(self): + self.action(avm1.ActionNewObject()) + +# def store_void(self): +# if not len(self.right_hand): +# return +# v = self.right_hand.pop() +# if v is not None and v.find('('): +# self.codegenerator.writeline(v+";") + + # FIXME: will refactor later + #load_str = load_const + + def begin_switch_varname(self, varname): + self.action(avm1.ActionPush(varname)) + self.action(avm1.ActionGetVariable()) + self.switch_register = self.store_register(varname) + self.action(avm1.ActionStoreRegister(self.switch_register)) + + def write_case(self, testee): + self.action(avm1.ActionPush(avm1.RegisterByIndex(self.switch_register), testee)) + self.action(avm1.ActionStrictEquals()) + if len(self.case_label) < 1: + self.case_label, self.case_block = self.branch_if_true() + else: + self.branch_if_true(self.case_label) + + def write_break(self): + self.exit_scope() + self.case_flag = False + self.case_block = None + + def enter_case_branch(self): + self.enter_scope(self.case_block) + +# def inherits(self, subclass_name, parent_name): +# self.codegenerator.writeline("inherits(%s,%s);"%(subclass_name, parent_name)) + + def throw(self): # Assumes the value to be thrown is on the stack. + self.action(avm1.ActionThrow()) Added: pypy/branch/avm/pypy/translator/avm/avm1types.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm/avm1types.py Fri Dec 19 20:01:17 2008 @@ -0,0 +1,151 @@ +""" JavaScript type system +""" + +from pypy.rpython.ootypesystem import ootype +from pypy.rpython.lltypesystem import lltype +from pypy.translator.cli import oopspec + +from pypy.rpython.lltypesystem.lltype import Signed, Unsigned, Void, Bool, Float +from pypy.rpython.lltypesystem.lltype import SignedLongLong, UnsignedLongLong, Primitive +from pypy.rpython.lltypesystem.lltype import Char, UniChar +from pypy.rpython.ootypesystem.ootype import String, _string, List, StaticMethod +from pypy.rlib.objectmodel import Symbolic + +from pypy.translator.avm.log import log + +from types import FunctionType +from pypy.rpython.extfunc import is_external + +try: + set +except NameError: + from sets import Set as set + +class AVM1TypeSystem(object): + """ Class implementing JavaScript type system + calls with mapping similiar to cts + """ + def __init__(self, db): + self.db = db + + #def __class(self, name): + # return name.replace(".", "_") + + def escape_name(self, name): + return name.replace('.', '_') + + def llvar_to_cts(self, var): + return 'var ', var.name + + def lltype_to_cts(self, t): + if isinstance(t, ootype.Instance): + self.db.pending_class(t) + return self.escape_name(t._name) + elif isinstance(t, ootype.List): + return 'Array' + elif isinstance(t, ootype.Array): + return 'Array' + #elif isinstance(t, lltype.Primitive): + # return 'var' + elif isinstance(t, ootype.Record): + return 'Array' + elif isinstance(t, ootype.String.__class__): + return 'String' + elif isinstance(t, ootype.Dict): + return 'Object' + elif isinstance(t, ootype.DictItemsIterator): + return 'Object' + elif t is ootype.StringBuilder: + return 'StringBuilder' + #return "var" + raise NotImplementedError("Type %r" % (t,)) + + def graph_to_signature(self, graph, is_method = False, func_name = None): + func_name = func_name or self.db.get_uniquename(graph,graph.name) + + args = [arg for arg in graph.getargs() if + arg.concretetype is not ootype.Void] + if is_method: + args = args[1:] + + return func_name,args + + def method_signature(self, obj, name): + # TODO: use callvirt only when strictly necessary + if isinstance(obj, ootype.Instance): + owner, meth = obj._lookup(name) + METH = meth._TYPE + return obj._name, METH.ARGS + elif isinstance(obj, ootype.BuiltinType): + meth = oopspec.get_method(obj, name) + class_name = self.lltype_to_cts(obj) + return class_name,meth.ARGS + else: + assert False + + def obj_name(self, obj): + return self.lltype_to_cts(obj) + + def primitive_repr(self, _type, v): + if _type is Bool: + if v == False: + val = 'false' + else: + val = 'true' + elif _type is Void: + val = 'undefined' + elif isinstance(_type,String.__class__): + val = repr(v._str) + elif isinstance(_type,List): + # FIXME: It's not ok to use always empty list + val = repr(v._list) + elif isinstance(_type,StaticMethod): + if hasattr(v, 'graph') and not is_external(v): + self.db.pending_function(v.graph) + else: + self.db.pending_abstract_function(v) + val = v._name + val = val.replace('.', '_') + if val == '?': + val = 'undefined' + elif _type is UniChar or _type is Char: + #log("Constant %r"%v) + s = repr(v) + if s.startswith('u'): + s = s[1:] + if s != "'\''": + s.replace("'", '"') + val = s + elif isinstance(v, Symbolic): + val = v.expr + elif isinstance(_type, Primitive): + #log("Type: %r"%_type) + val = str(v) + else: + assert False, "Unknown constant %r"%_type + val = str(v) + return val + + #def lltype_to_cts(self, t, include_class=True): + # return 'var' +## if isinstance(t, ootype.Instance): +## self.db.pending_class(t) +## return self.__class(t._name, include_class) +## elif isinstance(t, ootype.Record): +## name = self.db.pending_record(t) +## return self.__class(name, include_class) +## elif isinstance(t, ootype.StaticMethod): +## return 'void' # TODO: is it correct to ignore StaticMethod? +## elif isinstance(t, ootype.List): +## item_type = self.lltype_to_cts(t.ITEM) +## return self.__class(PYPY_LIST % item_type, include_class) +## elif isinstance(t, ootype.Dict): +## key_type = self.lltype_to_cts(t._KEYTYPE) +## value_type = self.lltype_to_cts(t._VALUETYPE) +## return self.__class(PYPY_DICT % (key_type, value_type), include_class) +## elif isinstance(t, ootype.DictItemsIterator): +## key_type = self.lltype_to_cts(t._KEYTYPE) +## value_type = self.lltype_to_cts(t._VALUETYPE) +## return self.__class(PYPY_DICT_ITEMS_ITERATOR % (key_type, value_type), include_class) +## +## return _get_from_dict(_lltype_to_cts, t, 'Unknown type %s' % t) Added: pypy/branch/avm/pypy/translator/avm/bootstrap.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm/bootstrap.py Fri Dec 19 20:01:17 2008 @@ -0,0 +1,13 @@ + + +from pypy.translator.avm.swf import SwfData +from pypy.translator.avm.tags import DoAction, SetBackgroundColor + +def create_assembler(name): + return AsmGen(name, DoAction()) + +def bootstrap_avm1(asmgen): + data = SwfData() + data.add_tag(SetBackgroundColor(0x333333)) + data.add_tag(asmgen) + return data.serialize() Added: pypy/branch/avm/pypy/translator/avm/database.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm/database.py Fri Dec 19 20:01:17 2008 @@ -0,0 +1,472 @@ + +""" genjs constant database module +""" + +import py +from pypy.rpython.ootypesystem import ootype +from pypy.translator.avm import avm1 +#from pypy.translator.js.opcodes import opcodes +from pypy.translator.avm.function import Function +from pypy.translator.avm.log import log +#from pypy.translator.js._class import Class +from pypy.translator.avm.support import AVM1NameManager + +from pypy.rpython.lltypesystem.lltype import Signed, Unsigned, Void, Bool, Float +from pypy.rpython.lltypesystem.lltype import SignedLongLong, UnsignedLongLong, typeOf +from pypy.rpython.lltypesystem.lltype import Char, UniChar +from pypy.rpython.ootypesystem import ootype +from pypy.rpython.ootypesystem import bltregistry + +from pypy.objspace.flow.model import Variable, Constant + +try: + set +except NameError: + from sets import Set as set + +class LowLevelDatabase(object): + def __init__(self, genoo): + self._pending_nodes = set() + self.genoo = genoo + self._rendered_nodes = set() + self.classes = {} # classdef --> class_name + self.functions = {} # graph --> function_name + self.function_names = {} # graph --> real_name + self.methods = {} # graph --> method_name + self.consts = {} # value --> const_name + self.reverse_consts = {} + self.const_names = set() + self.rendered = set() + self.const_var = Variable("__consts") + self.name_manager = AVM1NameManager(self) + self.pending_consts = [] + self.cts = self.genoo.TypeSystem(self) +# self.proxies = [] + + def is_primitive(self, type_): + if type_ in (Void, Bool, Float, Signed, Unsigned, SignedLongLong, UnsignedLongLong, Char, UniChar, ootype.StringBuilder) or \ + isinstance(type_,ootype.StaticMethod): + return True + return False + + def pending_function(self, graph): + self.pending_node(self.genoo.Function(self, graph)) + + def pending_abstract_function(self, name): + pass + # XXX we want to implement it at some point (maybe...) + + def pending_class(self, classdef): + self.pending_node(classdef) + return c + + def pending_record(self, record): + r = Record(self, record) # I can't find Record anywhere, and it's not in the imports. + self.pending_node(r) + return r.get_name() + + def pending_node(self, node): + if node in self._pending_nodes or node in self._rendered_nodes: + return + self._pending_nodes.add(node) + + def record_function(self, graph, name): + self.functions[graph] = name + + def get_uniquename(self, graph, name): + try: + return self.function_names[graph] + except KeyError: + real_name = self.name_manager.uniquename(name, lenmax=1111111) + self.function_names[graph] = real_name + return real_name + + def record_class(self, classdef, name): + self.classes[classdef] = name + +# def register_comm_proxy(self, proxy_const, *args): +# """ Register external object which should be rendered as +# method call +# """ +# self.proxies.append(XmlHttp(proxy_const, *args)) + + def graph_name(self, graph): + return self.functions.get(graph, None) + + def class_name(self, classdef): + return self.classes.get(classdef, None) + + def record_const(self, value, type_ = None, retval='name'): + if type_ is None: + type_ = typeOf(value) + if self.is_primitive(type_): + return None + const = AbstractConst.make(self, value) + if not const: + return None + try: + if retval == 'name': + return self.consts[const] + else: + self.consts[const] + return self.reverse_consts[self.consts[const]] + except KeyError: + if self.genoo.config.translation.verbose: + log("New const:%r"%value) + if isinstance(value, ootype._string): + log(value._str) + else: + log.dot() + name = const.get_name() + if name in self.const_names: + name += '__%d' % len(self.consts) + self.consts[const] = name + self.reverse_consts[name] = const + self.const_names.add(name) + self.pending_consts.append((const,name)) + if retval == 'name': + return name + else: + return const + + def gen_constants(self, ilasm, pending): + try: + while True: + const,name = self.pending_consts.pop() + const.record_fields() + except IndexError: + pass + + if pending: + return + + #if not self.rendered: + # begin_consts(self.const_var.name) + + def generate_constants(consts): + all_c = [const for const,name in consts.iteritems()] + dep_ok = set() + while len(all_c) > 0: + const = all_c.pop() + if const not in self.rendered: + to_render = True + if hasattr(const, 'depends_on') and const.depends_on: + for i in const.depends_on: + if i not in self.rendered and i not in dep_ok: + assert i.depends is None or const in i.depends + to_render = False + continue + + if to_render and (not hasattr(const, 'depends')) or (not const.depends) or const in dep_ok: + yield const,consts[const] + self.rendered.add(const) + else: + all_c.append(const) + for i in const.depends: + all_c.append(i) + dep_ok.add(const) + + # We need to keep track of fields to make sure + # our items appear earlier than us + for const, name in generate_constants(self.consts): + if self.genoo.config.translation.verbose: + log("Recording %r %r"%(const,name)) + else: + log.dot() + + ilasm.push_const(name) + const.init(ilasm) + + def push_const(self, type_, value, ilasm): + if self.is_primitive(type_): + ilasm.push_const(self.cts.primitive_repr(type_, value)) + else: + try: + return self.consts[BuiltinConst(value)] + except KeyError: + name = self.record_const(value) + ilasm.push_local(self.const_var) + ilasm.get_field(name) + #assert False, 'Unknown constant %s' % const + + +class AbstractConst(object): + def __init__(self, db, const): + self.db = db + self.const = const + self.cts = db.genoo.TypeSystem(db) + self.depends = set() + self.depends_on = set() + + def __hash__(self): + return hash(self.get_key()) + + def __eq__(self, other): + return (other.__class__ is self.__class__ and + other.get_key() == self.get_key()) + + def __ne__(self, other): + return not (self == other) + + def make(db, const): + if isinstance(const, ootype._view): + static_type = const._TYPE + const = const._inst + else: + static_type = None + + if isinstance(const, ootype._instance): + return InstanceConst(db, const, static_type) + elif isinstance(const, ootype._list): + return ListConst(db, const) + elif isinstance(const, ootype._array): + return ListConst(db, const) + elif isinstance(const, ootype._record): + return RecordConst(db, const) + elif isinstance(const, ootype._string): + return StringConst(db, const) + elif isinstance(const, ootype._dict): + return DictConst(db, const) + elif isinstance(const, bltregistry._external_inst): + return ExtObject(db, const) + elif isinstance(const, ootype._class): + if const._INSTANCE: + return ClassConst(db, const) + else: + return None + else: + assert False, 'Unknown constant: %s %r' % (const, typeOf(const)) + make = staticmethod(make) + + def get_name(self): + pass + + def get_type(self): + pass + + def init(self, ilasm, name): + pass + + def record_fields(self): + pass + +class InstanceConst(AbstractConst): + def __init__(self, db, obj, static_type): + self.depends = set() + self.depends_on = set() + self.db = db + self.cts = db.genoo.TypeSystem(db) + self.obj = obj + if static_type is None: + self.static_type = obj._TYPE + else: + self.static_type = static_type + self.cts.lltype_to_cts(obj._TYPE) # force scheduling of obj's class + + def get_key(self): + return self.obj + + def get_name(self): + return self.obj._TYPE._name.replace('.', '_') + + def get_type(self): + return self.cts.lltype_to_cts(self.static_type) + + def init(self, ilasm): + if not self.obj: + ilasm.push_null() + return + + classdef = self.obj._TYPE + try: + classdef._hints['_suggested_external'] + #ilasm.new(classdef._name.split(".")) + pass + except KeyError: + #ilasm.new(classdef._name.replace(".", "_")) + pass + + def record_fields(self): + if not self.obj: + return + INSTANCE = self.obj._TYPE + #while INSTANCE: + for i, (_type, val) in INSTANCE._allfields().items(): + if _type is not ootype.Void: + name = self.db.record_const(getattr(self.obj, i), _type, 'const') + if name is not None: + self.depends.add(name) + name.depends_on.add(self) + +# def init_fields(self, ilasm, name): +# if not self.obj: +# return + +# INSTANCE = self.obj._TYPE +# #while INSTANCE: +# for i, (_type, el) in INSTANCE._allfields().items(): +# if _type is not ootype.Void: +# ilasm.push_local(const_var) +# self.db.push_const(_type, getattr(self.obj, i), ilasm) +# ilasm.set_field(None, "%s.%s"%(name, i)) +# ilasm.store_void() + +class RecordConst(AbstractConst): + def get_name(self): + return "const_tuple" + + def init(self, ilasm): + if not self.const: + ilasm.push_undefined() + + for i in reversed(self.const._items): + el = self.const._items[i] + self.db.push_const(typeOf(el), el, ilasm) + ilasm.init_array() + + def record_fields(self): + if not self.const: + return + + for i in self.const._items: + name = self.db.record_const(self.const._items[i], None, 'const') + if name is not None: + self.depends.add(name) + name.depends_on.add(self) + + def get_key(self): + return self.const + +class ListConst(AbstractConst): + + def _get_list(self): + if isinstance(self.const, ootype._list): + return self.const._list + else: + return self.const._array + + + def get_name(self): + return "const_list" + + def record_fields(self): + if not self.const: + return + + for i in self._get_list(): + name = self.db.record_const(i, None, 'const') + if name is not None: + self.depends.add(name) + name.depends_on.add(self) + + def get_key(self): + return self.const + + def init(self, ilasm): + if not self.const: + ilasm.push_undefined() + + lst = self._get_list() + for el in lst: + self.db.push_const(typeOf(el), el, ilasm) + ilasm.push_const(len(lst)) + ilasm.init_array() + +class StringConst(AbstractConst): + def get_name(self): + return "const_str" + + def get_key(self): + return self.const._str + + def init(self, ilasm): + if self.const: + ilasm.push_const(self.const._str) + else: + ilasm.push_undefined() + +class ClassConst(AbstractConst): + def __init__(self, db, const): + super(ClassConst, self).__init__(db, const) + self.cts.lltype_to_cts(const._INSTANCE) # force scheduling of class + + def get_key(self): + return self.get_name() + + def get_name(self): + return self.const._INSTANCE._name.replace(".", "_") + + def init(self, ilasm): + #ilasm.push_const(self.get_name()) + + #def init_fields(self, ilasm, const_var, name): + # pass + +class BuiltinConst(AbstractConst): + def __init__(self, name): + self.name = name + + def get_key(self): + return self.name + + def get_name(self): + return self.name + + def init(self, ilasm): + ilasm.push_const(self.name) + +class DictConst(RecordConst): + def record_const(self, co): + name = self.db.record_const(co, None, 'const') + if name is not None: + self.depends.add(name) + name.depends_on.add(self) + + def record_fields(self): + if not self.const: + return + + for i in self.const._dict: + self.record_const(i) + self.record_const(self.const._dict[i]) + + def init(self, ilasm): + if not self.const: + return + + for i in self.const._dict: + el = self.const._dict[i] + self.db.push_const(typeOf(el), el, ilasm) + self.db.push_const(typeOf(i), i, ilasm) + ilasm.push_const(len(_dict)) + ilasm.init_object() + +# class ExtObject(AbstractConst): +# def __init__(self, db, const): +# self.db = db +# self.const = const +# self.name = self.get_name() +# self.depends = set() +# self.depends_on = set() + +# def get_key(self): +# return self.name + +# def get_name(self): +# return self.const._TYPE._name + +# def init(self, ilasm): +# _class = self.const._TYPE._class_ +# if getattr(_class, '_render_xmlhttp', False): +# use_xml = getattr(_class, '_use_xml', False) +# base_url = getattr(_class, '_base_url', "") # XXX: should be +# method = getattr(_class, '_use_method', 'GET') +# # on per-method basis +# self.db.register_comm_proxy(self.const, self.name, use_xml, base_url, method) +# ilasm.new(self.get_name()) +# else: +# # Otherwise they just exist, or it's not implemented +# if not hasattr(self.const.value, '_render_name'): +# raise ValueError("Prebuilt constant %s has no attribute _render_name," +# "don't know how to render" % self.const.value) +# ilasm.push_str(self.const.value._render_name) Added: pypy/branch/avm/pypy/translator/avm/function.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm/function.py Fri Dec 19 20:01:17 2008 @@ -0,0 +1,236 @@ + +from pypy.objspace.flow import model as flowmodel +from pypy.rpython.lltypesystem.lltype import Signed, Unsigned, Void, Bool, Float +from pypy.rpython.lltypesystem.lltype import SignedLongLong, UnsignedLongLong +from pypy.rpython.ootypesystem import ootype1 +from pypy.translator.oosupport.metavm import Generator,InstructionList +from pypy.translator.oosupport import function + +from pypy.translator.avm.log import log +from types import FunctionType + +import re + +class BaseGenerator(object): + def push(self, v): + if isinstance(v, flowmodel.Variable): + if v.name in self.argset: + selftype, selfname = self.args[0] + if self.is_method and v.name == selfname: + self.ilasm.push_this() + else: + self.ilasm.push_arg(v) + else: + self.ilasm.push_local(v) + elif isinstance(v, flowmodel.Constant): + self.db.push_const(v.concretetype, v.value, self.ilasm) + elif isinstance(v, str): + self.ilasm.push_const(v) + else: + assert False + + def store(self, v): + assert isinstance(v, flowmodel.Variable) + if v.concretetype is not Void: + self.ilasm.(v) + else: + self.ilasm.push_null() + +# def change_name(self, name, to_name): +# self.ilasm.change_name(name, to_name) + +# def add_comment(self, text): +# pass + + def function_signature(self, graph): + return self.cts.graph_to_signature(graph, False) + + def class_name(self, ooinstance): + return ooinstance._name + + def emit(self, instr, *args): + self.ilasm.emit(instr, *args) + + def call_graph(self, graph): + self.db.pending_function(graph) + func_sig = self.function_signature(graph) + self.ilasm.call(func_sig) + + def call_external(self, name, args): + self.ilasm.call((name, args)) + + #def call_signature(self, signature): + # self.ilasm.call(signature) + + def cast_to(self, lltype): + cts_type = self.cts.lltype_to_cts(lltype, False) + self.ilasm.castclass(cts_type) + + def new(self, obj): + self.ilasm.new(self.cts.obj_name(obj)) + + def oonewarray(self, obj, length): + self.ilasm.oonewarray(self.cts.obj_name(obj), length) + + def set_field(self, obj, name): + self.ilasm.set_member() + #self.ilasm.set_field(self.field_name(obj,name)) + + def get_field(self, useless_stuff, name): + self.ilasm.get_field(name) + + def call_method(self, obj, name): + func_name, signature = self.cts.method_signature(obj, name) + self.ilasm.call_method(obj, name, signature) + + def call_external_method(self, name, arg_len): + self.ilasm.call_method(None, name, [0]*arg_len) + + def instantiate(self): + self.ilasm.runtimenew() + + def downcast(self, TYPE): + pass + + def load_special(self, v): + # special case for loading value + # when setting builtin field we need to load function instead of None + # FIXME: we cheat here + if isinstance(v, flowmodel.Constant) and v.concretetype is ootype.Void and isinstance(v.value, FunctionType): + graph = self.db.translator.annotator.bookkeeper.getdesc(v.value).cachedgraph(None) + self.db.pending_function(graph) + name = graph.name + self.ilasm.load_str(name) + else: + self.load(v) + + def cast_function(self, name, num): + self.ilasm.cast_function(name, num) + + def prefix_op(self, st): + self.ilasm.prefix_op(st) + + def push_const(self, s): + self.ilasm.push_const(s) + + def push_undefined(self): + self.ilasm.push_undefined() + + def push_primitive_constant(self, TYPE, value): + self.db.load_const(TYPE, value, self.ilasm) + + def branch_unconditionally(self, target_label): + self.ilasm.jump_block(self.block_map[target_label]) + + def branch_conditionally(self, exitcase, target_label): + self.ilasm.branch_if(exitcase) + self.ilasm.jump_block(self.block_map[target_label]) + self.ilasm.close_branch() + +class Function(function.Function, BaseGenerator): + def __init__(self, db, graph, name=None, is_method=False, + is_entrypoint=False, _class=None): + self._class = _class + super(Function, self).__init__(db, graph, name, is_method, is_entrypoint) + self._set_args() + self._set_locals() + self.order = 0 + self.name = name or self.db.get_uniquename(self.graph, self.graph.name) + + def _setup_link(self, link, is_exc_link = False): + target = link.target + for to_load, to_store in zip(link.args, target.inputargs): + if to_load.concretetype is not Void: + if is_exc_link and isinstance(to_load, flowmodel.Variable) and re.match("last_exc_value", to_load.name): + self.ilasm.load_str("exc") + else: + self.load(to_load) + self.store(to_store) + + def _create_generator(self, ilasm): + return self + + def begin_render(self): + block_map = {} + for blocknum, block in enumerate(self.graph.iterblocks()): + block_map[self._get_block_name(block)] = blocknum + self.block_map = block_map + + if self.is_method: + args = self.args[1:] # self is implicit + else: + args = self.args + if self.is_method: + self.ilasm.begin_method(self.name, self._class, [i[1] for i in args]) + else: + self.ilasm.begin_function(self.name, args) + self.ilasm.set_locals(",".join([i[1] for i in self.locals])) + self.ilasm.begin_for() + + def render_return_block(self, block): + return_var = block.inputargs[0] + if return_var.concretetype is not Void: + self.load(return_var) + self.ilasm.ret() + else: + self.ilasm.load_void() + self.ilasm.ret() + + def end_render(self): + self.ilasm.end_for() + self.ilasm.end_function() + + def render_raise_block(self, block): + self.ilasm.throw(block.inputargs[1]) + + def end_try(self, target_label, cond): + self.ilasm.jump_block(self.block_map[target_label]) + if cond: + self.ilasm.catch() + #self.ilasm.close_branch() + + # XXX: soon or later we should use the smarter version in oosupport + def render_bool_switch(self, block): + for link in block.exits: + self._setup_link(link) + target_label = self._get_block_name(link.target) + if link is block.exits[-1]: + self.generator.branch_unconditionally(target_label) + else: + assert type(link.exitcase) is bool + assert block.exitswitch is not None + self.generator.load(block.exitswitch) + self.generator.branch_conditionally(link.exitcase, target_label) + + def record_ll_meta_exc(self, ll_meta_exc): + pass + + def begin_catch(self, llexitcase): + real_name = self.cts.lltype_to_cts(llexitcase._INSTANCE) + s = "isinstanceof(exc, %s)"%real_name + self.ilasm.branch_if_string(s) + + def end_catch(self, target_label): + """ Ends the catch block, and branchs to the given target_label as the + last item in the catch block """ + self.ilasm.close_branch() + + def store_exception_and_link(self, link): + self._setup_link(link, True) + self.ilasm.jump_block(self.block_map[self._get_block_name(link.target)]) + + def after_except_block(self): + #self.ilasm.close_branch() + self.ilasm.throw_real("exc") + self.ilasm.close_branch() + + def set_label(self, label): + self.ilasm.write_case(self.block_map[label]) + #self.ilasm.label(label) + + def begin_try(self, cond): + if cond: + self.ilasm.begin_try() + + def clean_stack(self): + self.ilasm.clean_stack() Added: pypy/branch/avm/pypy/translator/avm/log.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm/log.py Fri Dec 19 20:01:17 2008 @@ -0,0 +1,4 @@ +import py +from pypy.tool.ansi_print import ansi_log +log = py.log.Producer("avm") +py.log.setconsumer("avm", ansi_log) Added: pypy/branch/avm/pypy/translator/avm/metavm.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm/metavm.py Fri Dec 19 20:01:17 2008 @@ -0,0 +1,301 @@ +""" Opcode meaning objects, descendants of MicroInstruction +""" + +#from pypy.translator.js.jsbuiltin import Builtins +from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, StoreResult,\ + InstructionList, New, GetField, MicroInstruction + +from pypy.translator.js.log import log +from pypy.rpython.ootypesystem import ootype +from types import FunctionType +from pypy.objspace.flow.model import Constant + +class NewBuiltin(MicroInstruction): + def __init__(self, arg): + self.arg = arg + + def render(self, generator, op): + generator.ilasm.new(self.arg) + +class _ListSetitem(MicroInstruction): + def render(self, generator, op): + # op.args = [function_name, list, value, index] + generator.push(op.args[1]) + generator.push(op.args[3]) + generator.push(op.args[2]) + generator.ilasm.set_member() +ListSetitem = _ListSetitem() + +class _ListGetitem(MicroInstruction): + def render(self, generator, op): + # op.args = [function_name, list, index] + generator.push(op.args[1]) + generator.push(op.args[2]) + generator.ilasm.get_member() +ListGetitem = _ListGetitem() + +class _ListContains(MicroInstruction): + def render(self, generator, op): + # op.args = [function_name, list, to_find] + generator.push(op.args[1]) + generator.push(op.args[2]) + generator.ilasm.is_not_equal_const(avm1.Null()) +ListContains = _ListContains() + +class _Call(MicroInstruction): + def render(self, generator, op): + graph = op.args[0].value.graph + self._render_function(generator, graph, op.args) + + def _render_builtin(self, generator, builtin, args): + for func_arg in args[1:]: # push parameters + generator.load(func_arg) + generator.call_external(builtin, args[1:]) + + def _render_builtin_prepared_args(self, generator, builtin, args): + for func_arg in args: + generator.load_str(func_arg) + generator.call_external(builtin, args) + + def _render_builtin_method(self, generator, builtin, args): + for func_arg in args: + generator.load_special(func_arg) + generator.call_external_method(builtin, len(args)-1) + + def _render_function(self, generator, graph, args): + for func_arg in args[1:]: # push parameters + if func_arg.concretetype is not ootype.Void: + generator.load(func_arg) + generator.call_graph(graph) + + def _render_method(self, generator, method_name, args): + this = args[0] + for arg in args: # push parametes + generator.load(arg) + generator.call_method(this.concretetype, method_name) + +Call = _Call() + +class CallBuiltin(_Call): + def __init__(self, builtin): + self.builtin = builtin + + def render(self, generator, op): + self._render_builtin(generator, self.builtin, op.args) + +class CallBuiltinMethod(_Call): + def __init__(self, builtin, slice=None, additional_args=[]): + self.builtin = builtin + self.slice = slice + self.additional_args = additional_args + + def render(self, generator, op): + if self.slice is not None: + args = op.args[self.slice] + else: + args = op.args + args += self.additional_args + self._render_builtin_method(generator, self.builtin, args) + +class _SameAs(MicroInstruction): + def render(self, generator, op): + generator.change_name(op.result, op.args[0]) + +class _CastFun(MicroInstruction): + def __init__(self, name, num): + self.name = name + self.num = num + + def render(self, generator, op): + log("Args: %r"%op.args) + generator.cast_function(self.name, self.num) + +class _Prefix(MicroInstruction): + def __init__(self, st): + self.st = st + + def render(self, generator, op): + generator.prefix_op(self.st) + +class _NotImplemented(MicroInstruction): + def __init__(self, reason): + self.reason = reason + + def render(self, generator, op): + raise NotImplementedError(self.reason) + +class _CastMethod(MicroInstruction): + def __init__(self, method_name, num=0): + self.method_name = method_name + self.num = num + + def render(self, generator, op): + generator.call_external_method(self.method_name, self.num) + +class _LoadConst(MicroInstruction): + def __init__(self, value): + self.value = value + + def render(self, generator, op): + generator.load(Constant(self.value, ootype.typeOf(self.value))) + +class _GetBuiltinField(MicroInstruction): + def render(self, generator, op): + this = op.args[0] + field = op.args[1].value[1:] + generator.load(this) + generator.get_field(None, field) + +class _GetPredefinedField(MicroInstruction): + def __init__(self, field, num=1): + self.field = field + self.num = num + + def render(self, generator, op): + if op.result.concretetype is ootype.Void: + return + this = op.args[self.num] + generator.load(this) + generator.get_field(None, self.field) + +GetBuiltinField = _GetBuiltinField() + +class _SetBuiltinField(MicroInstruction): + def render(self, generator, op): + this = op.args[0] + field = op.args[1].value + if not field.startswith('o'): + generator.load_void() + else: + value = op.args[2] + field_name = field[1:] + self.run_it(generator, this, field_name, value) + + def run_it(self, generator, this, field_name, value): + generator.load(this) + generator.load_special(value) + generator.set_field(None, field_name) + +class _SetPredefinedField(_SetBuiltinField): + def __init__(self, field): + self.field = field + + def render(self, generator, op): + value = op.args[2] + this = op.args[1] + self.run_it(generator, this, self.field, value) + +class _SetExternalField(_SetBuiltinField): + def render(self, generator, op): + self.run_it(generator, op.args[0], op.args[1].value, op.args[2]) + +SetBuiltinField = _SetBuiltinField() +SetExternalField = _SetExternalField() + +class _CallMethod(_Call): + def render(self, generator, op): + method = op.args[0] + self._render_method(generator, method.value, op.args[1:]) + +class _CallBuiltinObject(_Call): + def render(self, generator, op): + this = op.args[1].concretetype + method = op.args[0] + method_name = this._methods[method.value]._name[1:] + generator.load(op.args[1]) + self._render_builtin_method(generator, method_name, op.args[1:]) + +class _CallExternalObject(_Call): + def render(self, generator, op): + this = op.args[1].concretetype + method = op.args[0] + method_name = method.value + #generator.load(op.args[1]) + self._render_builtin_method(generator, method_name, op.args[1:]) + +CallBuiltinObject = _CallBuiltinObject() +CallExternalObject = _CallExternalObject() + +class _IsInstance(MicroInstruction): + def render(self, generator, op): + # FIXME: just temporary hack + generator.load(op.args[0]) + generator.ilasm.load_const(op.args[1].value._name.replace('.', '_'))#[-1]) + generator.cast_function("isinstanceof", 2) + +class _IndirectCall(MicroInstruction): + def render(self, generator, op): + for func_arg in op.args[1:]: # push parameters + generator.load(func_arg) + generator.call_external(op.args[0].name, op.args[1:]) + +class _SetTimeout(MicroInstruction): + # FIXME: Dirty hack for javascript callback stuff + def render(self, generator, op): + val = op.args[1].value + assert(isinstance(val, ootype._static_meth)) + #if isinstance(val, ootype.StaticMethod): + real_name = val._name + generator.db.pending_function(val.graph) + #generator.db.pending_function(val.graph) + #else: + # concrete = val.concretize() + # real_name = concrete.value._name + # generator.db.pending_function(concrete.value.graph) + generator.load_str("'%s()'" % real_name) + generator.load(op.args[2]) + generator.call_external('setTimeout',[0]*2) + +class _DiscardStack(MicroInstruction): + def render(self, generator, op): + generator.clean_stack() + +class SetOnEvent(MicroInstruction): + def __init__(self, field): + self.field = field + + # FIXME: Dirty hack for javascript callback stuff + def render(self, generator, op): + val = op.args[1].value + val = val.concretize().value + assert(isinstance(val, ootype._static_meth)) + real_name = val._name + generator.db.pending_function(val.graph) + generator.load_str("document") + generator.load_str(real_name) + generator.set_field(None, self.field) + +class _CheckLength(MicroInstruction): + def render(self, generator, op): + assert not generator.ilasm.right_hand + +class _ListRemove(MicroInstruction): + def render(self, generator, op): + generator.list_getitem(op.args[1], op.args[2]) + generator.call_external('delete', [0]) + +ListRemove = _ListRemove() +CheckLength = _CheckLength() +SetTimeout = _SetTimeout() +IndirectCall = _IndirectCall() +IsInstance = _IsInstance() +CallMethod = _CallMethod() +CopyName = [PushAllArgs, _SameAs ()] +CastString = _CastFun("convertToString", 1) +SameAs = CopyName +DiscardStack = _DiscardStack() + +def fix_opcodes(opcodes): + for key, value in opcodes.iteritems(): + if type(value) is str: + value = InstructionList([PushAllArgs, value, StoreResult, CheckLength]) + elif value == []: + value = InstructionList([CheckLength]) + elif value is not None: + if StoreResult not in value: + value.append(StoreResult) + if CheckLength not in value: + value.append(CheckLength) + value = InstructionList(value) + + opcodes[key] = value Added: pypy/branch/avm/pypy/translator/avm/records.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm/records.py Fri Dec 19 20:01:17 2008 @@ -0,0 +1,238 @@ + +from pypy.translator.avm.util import BitStream + +class RecordHeader(object): + + def __init__(self, type, length): + self.type = type + self.length = length + + def serialize(self): + bits = BitStream() + bits.write_bit_value(type, 6) + if length < 0x3F: + bits.write_bit_value(self.length, 10) + else: + bits.write_bit_value(0x3F, 10) + bits.write_bit_value(self.length, 32) + return bits + + def parse(self, bitstream): + self.type = bitstream.read_bit_value(6) + self.length = bitstream.read_bit_value(10) + if self.length >= 0x3F: + self.length = bits.read_bit_value(32) + +class Rect(object): + + def __init__(self, XMin=0, XMax=0, YMin=0, YMax=0): + self.XMin = XMin + self.XMax = XMax + self.YMin = YMin + self.YMax = YMax + + def union(self, rect): + return Rect(min(self.XMin, rect.XMin), + max(self.XMax, rect.XMax), + min(self.YMin, rect.YMin), + max(self.YMax, rect.YMax)) + + def serialize(self): + if XMin > XMax or YMin > Max: + raise ValueError, "Maximum values in a RECT must be larger than the minimum values." + + # Find our values in twips. + twpXMin = self.XMin * 20 + twpXMax = self.XMax * 20 + twpYMin = self.YMin * 20 + twpYMax = self.YMax * 20 + + # Find the number of bits required to store the longest + # value, then add one to account for the sign bit. + longest = max(abs(twpXMin), abs(twpXMax), abs(twpYMin), abs(twpYMax)) + import math + NBits = int(math.ceil(math.log(longest, 2))) + 1 + + if NBits > 31: + raise ValueError, "Number of bits per value field cannot exceede 31." + + # And write out our bits. + bits = BitStream() + bits.write_bit_value(NBits, 5) + bits.write_bit_value(twpXMin, NBits) + bits.write_bit_value(twpXMax, NBits) + bits.write_bit_value(twpYMin, NBits) + bits.write_bit_value(twpYMax, NBits) + + return bits + + def parse(self, bitstream): + + NBits = bits.read_bit_value(5) + self.XMin = bits.read_bit_value(NBits) + self.XMax = bits.read_bit_value(NBits) + self.YMin = bits.read_bit_value(NBits) + self.YMax = bits.read_bit_value(NBits) + +class XY(object): + + def __init__(self, X=0, Y=0): + self.X = 0 + self.Y = 0 + + def serialize(self): + # Convert to twips plz. + twpX = self.X * 20 + twpY = self.Y * 20 + + # Find the number of bits required to store the longest + # value, then add one to account for the sign bit. + longest = max(abs(twpX), abas(twpY)) + import math + NBits = int(math.ceil(math.log(longest, 2)))+1 + + bits = BitStream() + bits.write_bit_value(NBits, 5) + bits.write_bit_value(twpX, NBits) + bits.write_bit_value(twpY, NBits) + + return bits + + def parse(self, bitstream): + + NBits = bits.read_bit_value(5) + self.X = bits.read_bit_value(NBits) + self.Y = bits.read_bit_value(NBits) + +class RGB(object): + + def __init__(self, color): + self.color = color & 0xFFFFFF + + def serialize(self): + bits = BitStream() + bits.write_bit_value(self.color, 24) + return bits + + def parse(self, bitstream): + self.color = bitstream.read_bit_value(24) + +class RGBA(RGB): + + def __init__(self, color, alpha=1.0): + RGB.__init__(self, color) + self.alpha = alpha + + def serialize(self): + bits = RGB.serialize(self) + bits.write_bit_value(int(self.alpha * 0xFF), 8) + return bits + + def parse(self, bitstream): + RGB.parse(self, bitstream) + self.alpha = bitstream.read_bit_value(8) / 0xFF + +class Shape(object): + + def __init__(self): + self.shapes = [] + + self.edge_bounds = Rect() + self.shape_bounds = Rect() + + self.has_scaling = False + self.has_non_scaling = False + + self.bounds_calculated = False + + def add_shape_record(self, shape): + self.shapes.append(shape) + self.bounds_calculated = False + + def add_shape(self, shape): + self.shapes.expand(shape.shapes) + self.bounds_calculated = False + + def serialize(self): + if EndShapeRecord not in self.shapes: + shapes.append(EndShapeRecord()) + + bits = BitArray() + + bits.write_bit_value(0, 8) # NumFillBits and NumLineBits + for records in self.shapes: + bits += record.serialize() + + return bits + + def calculate_bounds(self): + + if self.bounds_calculated: + return + + last_x, last_y = 0, 0 + for record in shapes: + last_x, last_y, has_scale, has_non_scale = record.calculate_bounds(last, self.shape_bounds, self.edge_bounds) + if has_scale: + self.has_scaling = True + if has_non_scale: + self.has_non_scaling = True + + self.bounds_calculated = True + +def ShapeWithStyle(Shape): + + def __init__(self, fills=[], strokes=[]): + Shape.__init__(self) + self.fills = fills + self.strokes = strokes + + def add_fill_style(self, style): + self.fills.append(style) + + def add_line_style(self, style): + self.strokes.append(style) + + def add_shape(self, shape): + Shape.add_shape(self, shape) + try: + self.fills += shape.fills + self.strokes += shape.strokes + except AttributeError: + pass + + @static_method + def __serialize_style_list(list): + bits = BitStream() + + if len(list) <= 0xFF: + bits.write_bit_value(len(list), 8) + else: + bits.write_bit_value(0xFF, 8) + bits.write_bit_value(len(list), 16) + + for style in list: + bits += style.serialize() + + return bits + + def serialize(self): + bits = BitStream() + bits += __serialize_style_list(self.fills) + bits += __serialize_style_list(self.strokes) + import math + bits.write_bit_value(math.ceil(math.log(len(self.fills), 2)), 4) + bits.write_bit_value(math.ceil(math.log(len(self.strokes), 2)), 4) + return bits + +class LineStyle(object): + + def __init__(self, width=1, color=0, alpha=1.0): + self.width = width + self.color = RGBA(color, alpha) + + def serialize(self): + bits = BitStream() + bits.write_bit_value(self.width * 20, 16) + bits += color.serialize() + return bits Added: pypy/branch/avm/pypy/translator/avm/support.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm/support.py Fri Dec 19 20:01:17 2008 @@ -0,0 +1,50 @@ +from pypy.translator.gensupp import NameManager +#from pypy.translator.js.optimize import is_optimized_function + +class AVM1NameManager(NameManager): + def __init__(self, db): + NameManager.__init__(self) + self.db = db + self.reserved = {} + + + # Source: + # http://livedocs.adobe.com/flash/8/main/wwhelp/wwhimpl/common/html/wwhelp.htm?context=LiveDocs_Parts&file=00001236.html + reserved_words = '''add and break case catch class continue default + delete do dynamic else eq extends finally for function ge get gt if + ifFrameLoaded implements import in instanceof interface intrinsic + le lt ne new not on onClipEvent or private public return setw + static switch tellTarget this throw try typeof var void while with''' + + for name in reserved_words.split(): + self.reserved[name] = True + + predefined_classes_and_objects = ''' + Accessibility Accordion Alert Array Binding Boolean Button Camera + CellRenderer CheckBox Collection Color ComboBox ComponentMixins ContextMenu + ContextMenuItem CustomActions CustomFormatterCustomValidator DataGrid + DataHolder DataProvider DataSet DataType Date DateChooser DateField Delta + DeltaItem DeltaPacket DepthManager EndPoint Error FocusManager Form Function + Iterator Key Label List Loader LoadVars LocalConnection Log Math Media Menu + MenuBar Microphone Mouse MovieClip MovieClipLoader NetConnection NetStream + Number NumericStepper Object PendingCall PopUpManager PrintJob ProgressBar + RadioButton RDBMSResolver Screen ScrollPane Selection SharedObject Slide SOAPCall + Sound Stage String StyleManager System TextArea TextField TextFormat TextInput + TextSnapshot TransferObject Tree TreeDataProvider TypedValue UIComponent + UIEventDispatcher UIObject Video WebService WebServiceConnector Window XML + XMLConnector XUpdateResolver''' + + for name in predefined_classes_and_objects.split(): + self.reserved[name] = True + + self.make_reserved_names(' '.join(self.reserved)) + + self.predefined = set(predefined_classes_and_objects) + + #def uniquename(self, name, lenmax=0): + # return NameManager.uniquename(self, , lenmax) + + def ensure_non_reserved(self, name): + while name in self.reserved: + name += '_' + return name Added: pypy/branch/avm/pypy/translator/avm/swf.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm/swf.py Fri Dec 19 20:01:17 2008 @@ -0,0 +1,59 @@ + + +class SwfData(object): + + def __init__(self, width=600, height=400, fps=24, compress=False, version=10): + self.width = width + self.height = height + self.fps = fps + self.compress = compress + self.version = version + self.frame_count = 1 + + self.tags = [] + + def __getitem__(self, i): + return self.tags.__getitem__(i) + + def __iadd__(self, other): + if hasattr(other, "TAG_TYPE"): + self.add_tag(other) + else: + self.add_tags(other) + + def add_tag(self, tag): + if self.version > tag.min_version: + self.tags.append(tag) + + def add_tags(self, tag_container): + if hasattr(tag_container, "tags"): + self.tags += tag_container.tags + else: + self.tags += tag_container + + def serialize(self): + final_bytes = [] + + header = __gen_header() + data = __gen_data_stub() + data += [tag.serialize() for tag in self.tags] + + header[3] = len(header) + len("".join(data)) # FileSize + if self.compress: + import zlib + data = zlib.compress(data) + + return "".join(header + data) + + def __gen_header(self): + import struct + return ("CWS" if self.compress else "FWS") + struct.pack("BL", self.version, 0) + + def __gen_data_stub(self): + from util import BitStream + from records import Rect + data = BitStream() + data += Rect(XMax=width, YMax=height).serialize() + data.write_bit_value(fps, 16) + data.write_bit_value(frame_count, 16) + return data.serialize() Added: pypy/branch/avm/pypy/translator/avm/tags.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm/tags.py Fri Dec 19 20:01:17 2008 @@ -0,0 +1,46 @@ + +from pypy.translator.avm.util import BitStream +from pypy.translator.avm.avm1 import Block + +class SwfTag(object): + + TAG_TYPE = -1 + TAG_MIN_VERSION = -1 + + def serialize_data(self): + return "" + + def serialize(self): + data = self.serialize_data(self) + return RecordHeader(self.TAG_TYPE, len(data)).serialize().serialize() + data + +class SetBackgroundColor(SwfTag): + + TAG_TYPE = 9 + TAG_MIN_VERSION = 1 + + def __init__(self, color): + self.color = color + + def serialize_data(self): + import struct + return struct.pack("LB", color >> 8 & 0xFFFF, color & 0xFF) + +class DoAction(SwfTag, Block): + + TAG_TYPE = 12 + TAG_MIN_VERSION = 3 + + def __init__(self): + Block.__init__(self, True) + + def serialize_data(self): + return Block.serialize(self) + +class End(SwfTag): + + TAG_TYPE = 0 + TAG_MIN_VERSION = 0 + + def serialize(self): + return "\0\0" Added: pypy/branch/avm/pypy/translator/avm/test/browsertest.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm/test/browsertest.py Fri Dec 19 20:01:17 2008 @@ -0,0 +1,97 @@ +from BaseHTTPServer import HTTPServer as BaseHTTPServer, BaseHTTPRequestHandler +import py +from os import system +from cgi import parse_qs +from sys import platform +from time import sleep +import webbrowser +from pypy.translator.avm.log import log +log = log.browsertest + +class HTTPServer(BaseHTTPServer): + allow_reuse_address = True + +class config: + http_port = 10001 + + html_page = """ + +PyPy AVM1 Test Case: %s + + + + + + +""" + + crossdomain_xml=""" + +""" + +class TestCase(object): + def __init__(self, name, swfdata): + self.testcasename = name + self.swfdata = swfdata + self.result = None + +class TestHandler(BaseHTTPRequestHandler): + """The HTTP handler class that provides the tests and handles results""" + + def do_GET(self): + global do_status + if self.path == "/test.html": + data = config.html_page % testcase.xtestcasename + mime = 'text/html' + elif self.path == "/test.swf": + data = testcase.swfdata + mime = 'application/x-shockwave-flash' + elif self.path == "/crossdomain.xml": + data = config.crossdomain_xml + mime = 'text/xml' + self.serve_data(mime, data) + do_status = 'do_GET' + + def do_POST(self): + global do_status + if self.path == "/test.result": + form = parse_qs(self.rfile.read(int(self.headers['content-length']))) + testcase.result = form['result'][0] + do_status = 'do_POST' + + def serve_data(self, content_type, data): + self.send_response(200) + self.send_header("Content-type", content_type) + self.send_header("Content-length", len(data)) + self.end_headers() + self.wfile.write(data) + + +class BrowserTest(object): + """The browser driver""" + + def start_server(self, port, html_page, is_interactive): + server_address = ('', port) + self.httpd = HTTPServer(server_address, TestHandler) + self.httpd.is_interactive = is_interactive + self.httpd.html_page = html_page + + def get_result(self): + global do_status + do_status = None + while do_status != 'do_GET': + self.httpd.handle_request() + while do_status != 'do_POST': + self.httpd.handle_request() + return jstest.result + + +def browsertest(testcase, swfdata): + global driver, testcase + testcase = TestCase(str(testcase), str(swfdata)) + driver = BrowserTest() + driver.start_server(config.http_port, html_page, is_interactive) + webbrowser.open('http://localhost:%d/test.html' % config.http_port) + + result = driver.get_result() + return result Added: pypy/branch/avm/pypy/translator/avm/test/runtest.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm/test/runtest.py Fri Dec 19 20:01:17 2008 @@ -0,0 +1,140 @@ + +import py, os, re, subprocess +from pypy.translator.translator import TranslationContext +from pypy.translator.backendopt.all import backend_optimizations +from pypy.translator.avm.avm import AVM1 +from pypy.translator.avm.test.browsertest import browsertest, TestCase +# from pypy.translator.avm import conftest +from pypy.translator.avm.log import log +from pypy.conftest import option +from pypy.rpython.test.tool import BaseRtypingTest, OORtypeMixin +from pypy.rlib.nonconst import NonConstant +from pypy.rpython.ootypesystem import ootype + +from pypy.rpython.llinterp import LLException + +log = log.runtest + +class AVM1Exception(object): + +class compile_function(object): + def __init__(self, function, annotations, stackless=False, view=False, root=None, policy=None): + + t = TranslationContext() + + if policy is None: + from pypy.annotation.policy import AnnotatorPolicy + policy = AnnotatorPolicy() + policy.allow_someobjects = False + + self.root = root + + ann = t.buildannotator(policy=policy) + ann.build_types(function, annotations) + if view or option.view: + t.view() + + t.buildrtyper(type_system="ootype").specialize() + + if view or option.view: + t.view() + + self.avm = AVM1(t, function, stackless) + + def _conv(self, v): + if isinstance(v, str): + return repr(v) + return str(v).lower() + + def call(self, entry_function): + + if entry_function is None: + entry_function = self.avm.translator.graphs[0].name + else: + entry_function = self.avm.translator.annotator.bookeeper.getdesc(entry_function).cached_graph(None) + + output = browsertest("Test Name", self.avm.serialize()) + return self.reinterpret(output) + + @classmethod + def reinterpret(cls, s): + if s == 'false': + return False + elif s == 'true': + return True + elif s == 'undefined' or s == 'null': + return None + elif s == 'inf': + return 1e400 + elif s == 'NaN': + return 1e400 / 1e400 + elif s.startswith('[') or s.startswith('('): + contents = s[1:-1].split(',') + return [self.reintepret(i) for i in contents] + else: + try: + res = float(s) + if float(int(res)) == float(res): + return int(res) + return res + except ValueError: + return str(s) + +class AVM1Test(BaseRtypingTest, OORtypeMixin): + def _compile(self, _fn, args, policy=None): + argnames = _fn.func_code.co_varnames[:_fn.func_code.co_argcount] + func_name = _fn.func_name + if func_name == '': + func_name = 'func' + source = py.code.Source(""" + def %s(): + from pypy.rlib.nonconst import NonConstant + res = _fn(%s) + if isinstance(res, type(None)): + return None + else: + return str(res)""" + % (func_name, ",".join(["%s=NonConstant(%r)" % (name, i) for + name, i in zip(argnames, args)]))) + exec source.compile() in locals() + return compile_function(locals()[func_name], [], policy=policy) + + def interpret(self, fn, args, policy=None): + f = self.compile(fn, args, policy) + res = f(*args) + return res + + def interpret_raises(self, exception, fn, args): + try: + res = self.interpret(fn, args) + except AVM1Exception, e: + s = e.args[0] + assert s.startswith('uncaught exception') + assert re.search(exception.__name__, s) + else: + raise AssertionError("Did not raise, returned %s" % res) + + def string_to_ll(self, s): + return s + + def ll_to_string(self, s): + return str(s) + + def ll_to_list(self, l): + return l + + def ll_unpack_tuple(self, t, length): + assert len(t) == length + return tuple(t) + + def class_name(self, value): + return value[:-8].split('.')[-1] + + def is_of_instance_type(self, val): + m = re.match("^<.* object>$", val) + return bool(m) + + def read_attr(self, obj, name): + py.test.skip('read_attr not supported on genavm tests') + +def check_source_contains(compiled_function, pattern): Added: pypy/branch/avm/pypy/translator/avm/util.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm/util.py Fri Dec 19 20:01:17 2008 @@ -0,0 +1,196 @@ + +import struct, os + +ALIGN_LEFT = "left" +ALIGN_RIGHT = "right" + +class BitStream(object): + + """ BitStream is a class for taking care of data structures that are bit-packed, like SWF.""" + + def __init__(self, bits=[]): + """ + Constructor. + """ + self.bits = bits + self.cursor = 0 + + def read_bit(self): + """Reads a bit from the bit stream and returns it as either True or False. IndexError is thrown if reading past the end of the stream.""" + self.cursor += 1 + return self.bits[self.cursor-1] + + def read_bits(self, length): + """Reads length bits and return them in their own bit stream.""" + self.cursor += length + return BitStream(self.bits[self.cursor-length:self.cursor]) + + def write_bit(self, value): + """Writes the boolean value to the bit stream.""" + if self.cursor < len(self.bits): + self.bits[self.cursor] = bool(value) + else: + self.bits.append(bool(value)) + self.cursor += 1 + + def write_bits(self, bits, offset=0, length=0): + """Writes length bits from bits to this bit stream, starting reading at offset. If length + is 0, the entire stream is used.""" + if length < 1: length = len(bits) + self.bits[self.cursor:self.cursor+length] = bits[offset:offset+length] + self.cursor += length + + def read_bit_value(self, length): + """Read length bits and return a number containing those bits with the last bit read being + the least significant bit.""" + n = 0 + for i in xrange(length-1, -1, -1): + n |= self.read_bit() << i + return n + + def write_bit_value(self, value, length=-1): + """Writes an int to the specified number of bits in the stream, the most significant bit + first. If length is not specified or negative, the log base 2 of value is taken.""" + if int(value) != value: + self.write_fixed_value(value, length) + return + + if length < 0: + import math + try: + length = int(math.ceil(math.log(value, 2))) # Get the log base 2, or number of bits value will fit in. + except ValueError: + length = 1 + + for i in xrange(length-1, -1, -1): + self.write_bit(value & (1 << i)) + + def read_fixed_value(self, length, eight_bit_mantissa=False): + """Reads a fixed point number of length. If eight_bit_mantissa is True, an + 8.8 format is used instead of a 16.16 format.""" + min_length = 8 if eight_bit_mantissa else 16 + if length < min_length: + raise ValueError, "Length must be greater than or equal to %(m)s, as %(m)s.%(m)s FB requires at \ + least %(m)s bits to store." % {"m": min_length} + + return self.read_bit_value(length) / 0x100 if eight_bit_mantissa else 0x10000 + + def write_fixed_value(self, value, length=-1, eight_bit_mantissa=False): + """Writes a fixed point number of length, whole part first. If eight_bit_mantissa is True, + an 8.8 format is used instead of a 16.16 format. If length is negative, it will be calculated for.""" + self.writeBitValue( value * ( 0x100 if eight_bit_mantissa else 0x10000 ), length ) + + _EXPN_BIAS = {16: 16, 32: 127, 64: 1023} + _N_EXPN_BITS = {16: 5, 32: 8, 64: 52} + _N_FRAC_BITS = {16: 10, 32: 23, 64: 52} + _FLOAT_NAME = {16: "float16", 32: "float", 64: "double"} + + def read_float_value(self, length=16): + """Reads a floating point number of length, which must be 16 (float16), 32 (float), + or 64 (double). See: http://en.wikipedia.org/wiki/IEEE_floating-point_standard""" + if length not in _FLOAT_NAME: + raise ValueError, "Length in read_float_value is not 16, 32 or 64." + + sign = self.read_bit() + expn = self.read_bit_value(_N_EXPN_BITS[length]) + frac = self.read_bit_value(_N_FRAC_BITS[length]) + + frac_total = 1 << _N_FRAN_BITS[length] + + if expn == 0: + if frac == 0: + return 0 + else: + return ~frac + 1 if sign else frac + elif expn == frac_total - 1: + if frac == 0: + return float("-inf") if sign else float("inf") + else: + return float("nan") + + return (-1 if sign else 1) * ( 2**(expn-_EXPN_BIAS[length]) ) * ( 1 + frac / frac_total ) + + def write_float_value(self, value, length=16): + """Writes a floating point number of length, which must be 16 (float16), + 32 (float), or 64 (double). See: http://en.wikipedia.org/wiki/IEEE_floating-point_standard""" + if n == 0: # n is zero, so we don't care about length + self.write_bit_value(0, length) + + import math + if math.isnan(value): + self.one_fill(length) + return + elif value == float("-inf"): # negative infinity + self.one_fill(_N_EXPN_BITS[length] + 1) # sign merged + self.zero_fill(_N_FRAC_BITS[length]) + return + elif value == float("inf"): # positive infinity + self.write_bit(False) + self.one_fill(_N_EXPN_BITS[length]) + self.zero_fill(_N_FRAC_BITS[length]) + return + + if n < 0: + self.write_bit(True) + n = ~n + 1 + else: + self.write_bit(False) + + exp = _EXPN_BIAS[length] + if value < 1: + while int(value) != 1: + value *= 2 + exp -= 1 + else: + while int(value) != 1: + value /= 2 + exp += 1 + + if exp < 0 or exp > ( 1 << _N_EXPN_BITS[length] ): + raise ValueError, "Exponent out of range in %s [%d]." % (_FLOAT_NAME[length], length) + + frac_total = 1 << _N_FRAC_BITS + self.write_bit_value(exp, _N_EXPN_BITS[length]) + self.write_bit_value(int((value-1)*frac_total) & (frac_total - 1), _N_FRAC_BITS[length]) + + + def one_fill(self, amount): + """Fills amount bits with one. The equivalent of calling + self.write_boolean(True) amount times, but more efficient.""" + self.bits[self.cursor:self.cursor+amount] = [True] * amount + + def zero_fill(self, amount): + """Fills amount bits with zero. The equivalent of calling + self.write_boolean(False) amount times, but more efficient.""" + self.bits[self.cursor:self.cursor+amount] = [False] * amount + + def seek(self, offset, whence=os.SEEK_SET): + if whence == os.SEEK_SET: + self.cursor = offset + elif whence == os.SEEK_CUR: + self.cursor += offset + elif whence == os.SEEK_END: + self.cursor = len(self.bits) - abs(offset) + + def rewind(self): + self.seek(0, os.SEEK_SET) + + def skip_to_end(self): + self.seek(0, os.SEEK_END) + + def serialize(self, align=ALIGN_RIGHT, endianness="<"): + """Serialize bit array into a byte string, aligning either on the right + (ALIGN_RIGHT) or left (ALIGN_LEFT)""" + list = self[:] + leftover = len(list) % 8 + if leftover > 0 and align == BitStream.ALIGN_RIGHT: + list[:0] = [False] * (8-leftover) # Insert some False values to pad the list so it is aligned to the right. + list = BitStream(list) + bytes = [list.read_bit_value(8) for i in range(math.ceil(bits/8.0))] + return struct.pack("%s%dB" % (endianness, len(bytes)), *bytes) + + def parse(self, string, endianness="<"): + """Parse a bit array from a byte string into this BitStream.""" + bytes = list(struct.unpack("%s%dB" % (endianness, len(string)))) + for byte in bytes: + self.write_bit_value(byte, 8) From cfbolz at codespeak.net Fri Dec 19 20:32:08 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Dec 2008 20:32:08 +0100 (CET) Subject: [pypy-svn] r60620 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219193208.F0895168496@codespeak.net> Author: cfbolz Date: Fri Dec 19 20:32:06 2008 New Revision: 60620 Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex pypy/extradoc/talk/ecoop2009/rainbow.tex pypy/extradoc/talk/ecoop2009/tlc-folded-virtualized.py Log: some improvements, notes Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Fri Dec 19 20:32:06 2008 @@ -39,6 +39,7 @@ multiplication, one subtraction, and one comparison to check if we have finished the job: +\cfbolz{I think we can kill this for space reasons} \begin{lstlisting} def main(n): result = 1 @@ -66,6 +67,7 @@ Similarly, we wrote a program to calculate the $n_{th}$ Fibonacci number, for which we can do the same reasoning as above: +\cfbolz{I think we can kill this for space reasons} \begin{lstlisting} def main(n): a = 0 @@ -80,6 +82,7 @@ \anto{these tables are ugly} +\cfbolz{can we make one figure of all the tables?} \begin{table}[ht] \begin{tabular}{|l|r|r|r|r||r|r|} \hline Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Fri Dec 19 20:32:06 2008 @@ -57,25 +57,6 @@ because it makes possible to optimize method calls on them, as the JIT compiler knows their exact type in advance. -XXX kill the rest?!? -An interesting effect of virtual structures is that they play nicely with -promotion. Indeed, before the interpreter can call the proper \texttt{add()} -function for integers, it must first determine that the two arguments -are indeed integer objects. In the corresponding dispatch logic, we -have added two hints to promote the type of each of the two arguments. -This produces a compiler that has the following behavior: in the general -case, the expression \texttt{a+b} will generate two consecutive run-time -switches followed by the residual code of the proper version of -\texttt{add()}. However, in \texttt{a+b+c}, the virtual structure representing -the intermediate value will contain a compile-time constant as type. -Promoting a compile-time constant is trivial ? no run-time code is -generated. The whole expression \texttt{a+b+c} thus only requires three -switches instead of four. It is easy to see that even more switches can -be skipped in larger examples; typically, in a tight loop manipulating -only integers, all objects are virtual structures for the compiler and -the residual code is theoretically optimal ? all type propagation and -boxing/unboxing occurs at compile-time. - \section{Promotion} \label{sec:promotion} @@ -144,9 +125,9 @@ requiring \texttt{v1} to be green. Note that this amounts to copying a red value into a green one, which is not possible in classical approaches to partial evaluation. A slightly different hint can be used to -promote the \emph{class} of an instance. This is done with \lstinline{hint(v1, -promote_class=True)}. It does not have an effect on the bindings of any -variable. +promote the \emph{class} of an instance. This is done with +\lstinline{hint(v1, promote_class=True)}. It does not have an effect on the +bindings of any variable. \subsection{Implementing Promotion} @@ -164,12 +145,22 @@ Let us look again at the TLC example. To ease the reading, figure \ref{fig:tlc-main} showed a simplified version of TLC's main loop, which did -not include the hints. The real implementation of the \lstinline{LT} opcode -is shown in figure \ref{fig:tlc-main-hints}. +not include the hints. The implementation of the \lstinline{LT} opcode with +hints added is shown in figure \ref{fig:tlc-main-hints}. \begin{figure}[h] \begin{center} \begin{lstlisting}[language=Python] +def interp_eval(code, pc, args, pool): + code_len = len(code) + stack = [] + while pc < code_len: + opcode = ord(code[pc]) + opcode = hint(opcode, promote_class=True) + pc += 1 + + if opcode == PUSH: + ... elif opcode == LT: a, b = stack.pop(), stack.pop() hint(a, promote_class=True) @@ -188,22 +179,24 @@ ... def lt(self, other): return self.value < other.int_o() + def sub(self, other): + return IntObj(self.value - other.int_o()) def int_o(self): return self.value \end{lstlisting} -\caption{Excerpt of the \lstlisting{IntObj} class} +\caption{Excerpt of the \lstinline{IntObj} class} \label{fig:tlc-intobj} \end{center} \end{figure} -By promoting the class of \lstlisting{a} and \lstlisting{b}, we tell the JIT +By promoting the class of \lstinline{a} and \lstinline{b}, we tell the JIT compiler not to generate code until it knows the exact RPython class of both. Figure \ref{fig:tlc-abs-promotion-1} shows the code \footnote{\lstinline{switch} is not a legal (R)Python statement, it is used here only as a pseudocode example} generated while compiling the usual -\lstlisting{abs} function: note that, compared to figure +\lstinline{abs} function: note that, compared to figure \ref{fig:tlc-folded-virtualized}, the code stops just before the calls -\lstlisting{b.lt(a)}. +\lstinline{b.lt(a)}. \begin{figure}[h] \begin{center} @@ -212,7 +205,7 @@ v0 = args[0] v1 = IntObj(0) a, b = v0, v1 - hint(a, promote_class=True) # no-op + # hint(a, promote_class=True) implemented as follows: cls_a = a.__class__ switch cls_a: default: @@ -230,13 +223,20 @@ v0 = args[0] v1 = IntObj(0) a, b = v0, v1 - hint(a, promote_class=True) # no-op + # hint(a, promote_class=True) implemented as follows: cls_a = a.__class__ switch cls_a: IntObj: - hint(b, promote_class=True) + # hint(b, promote_class=True) needs no code v0 = IntObj(b.value < a.value) - ... + if v0.value: + return a + else: + v0 = IntObj(0) + v1 = args[0] + a, b = v0, v1 + v0 = IntObj(b.value - a.value) + return v0 default: continue_compilation(jitstate, cls_a) \end{lstlisting} Modified: pypy/extradoc/talk/ecoop2009/tlc-folded-virtualized.py ============================================================================== --- pypy/extradoc/talk/ecoop2009/tlc-folded-virtualized.py (original) +++ pypy/extradoc/talk/ecoop2009/tlc-folded-virtualized.py Fri Dec 19 20:32:06 2008 @@ -13,5 +13,5 @@ v1 = args[0] # [v0, v1] a, b = v0, v1 # [] v0 = b.sub(a) # [v0] - return v + return v0 \end{lstlisting} From cfbolz at codespeak.net Fri Dec 19 20:58:08 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Dec 2008 20:58:08 +0100 (CET) Subject: [pypy-svn] r60624 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219195808.F0FE3168481@codespeak.net> Author: cfbolz Date: Fri Dec 19 20:58:07 2008 New Revision: 60624 Modified: pypy/extradoc/talk/ecoop2009/main.tex Log: if it should become necessary: light compression to win some pages (commented out) Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Fri Dec 19 20:58:07 2008 @@ -1,5 +1,12 @@ \documentclass{llncs} + + %%%% Compression Light+: LNCS margin reduced by +/-7mm along all edges (RG). +%\textwidth=130mm % LNCS: 122mm +%\textheight=203mm % LNCS: 193mm + +%\renewcommand{\baselinestretch}{0.95} + \usepackage{amssymb} \usepackage{amsmath} \usepackage[sans]{dsfont} From cfbolz at codespeak.net Fri Dec 19 21:38:23 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Dec 2008 21:38:23 +0100 (CET) Subject: [pypy-svn] r60625 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219203823.5EB79168489@codespeak.net> Author: cfbolz Date: Fri Dec 19 21:38:21 2008 New Revision: 60625 Modified: pypy/extradoc/talk/ecoop2009/conclusion.tex pypy/extradoc/talk/ecoop2009/main.bbl pypy/extradoc/talk/ecoop2009/main.bib Log: add material to the related work section, needs an overhaul Modified: pypy/extradoc/talk/ecoop2009/conclusion.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/conclusion.tex (original) +++ pypy/extradoc/talk/ecoop2009/conclusion.tex Fri Dec 19 21:38:21 2008 @@ -1,6 +1,59 @@ \section{Related Word} -XXX add me +Promotion is a concept that we have already explored in other contexts. Psyco is +a run-time specialiser for Python that uses promotion (called ``unlift'' in +\cite{DBLP:conf/pepm/Rigo04}). However, Psyco is a manually written JIT, is +not applicable to other languages and cannot be retargetted. + + + - reference standard jit stuff, PICs \cite{hoelzle_optimizing_1991}, this one: http://www.cs.ucsb.edu/~urs/oocsb/self/papers/type-feedback.html + + - compare against tracing JITs \cite{gal_hotpathvm_2006}. tracing jits make + implementing jits much easier, maybe less need for JIT generation. also, they + concentrate on loops, which makes them produce a lot less code (which will need + to be addressed in our work as well). + + so far tracing JITs have less general allocation removal techniques, which + makes them get less speedup in a dynamic language with boxing + +in my master I had this snippet: + +""" +The generated code typically contains guards; in recent research +\cite{gal_incremental_2006} on Java, these guards' behaviour is extended to be +similar to our promotion. This has been used twice to implement a dynamic +language (JavaScript), by Tamarin\footnote{{\tt +http://www.mozilla.org/projects/tamarin/}} and in \cite{chang_efficient_2007}. +""" + +There has been an enormous amount of work on partial evaluation for compiler +generation. A good introduction is given in \cite{Jones:peval}. However, most of +it is for generating ahead-of-time compilers, which cannot produce very good +performance results for dynamic languages. + +However, there is also some research on runtime partial evaluation. One of the +earliest examples is Tempo for C +\cite{DBLP:conf/popl/ConselN96,DBLP:conf/dagstuhl/ConselHNNV96}. However, it is +essentially an offline specializer ``packaged as a library''; decisions about +what can be specialized and how are pre-determined. + +Another work in this direction is DyC \cite{grant_dyc_2000}, another runtime +specializer for C. Specialization decisions are also pre-determined, but +``polyvariant program-point specialization'' gives a coarse-grained equivalent +of our promotion. Targeting the C language makes higher-level specialization +difficult, though (e.g.\ \texttt{mallocs} are not removed). + +Greg Sullivan introduced "Dynamic Partial Evaluation", which is a special +form of partial evaluation at runtime \cite{sullivan_dynamic_2001} and describes +an implementation for a small dynamic language based on lambda calculus. This +work is conceptually very close to our own. +% XXX there are no performance figures, we have no clue how much of this is +% implemented. not sure how to write this + + - apart from that we maybe should talk about escape analysis + \cite{Blanchet99escapeanalysis}, \cite{Choi99escapeanalysis} + our algorithm is totally simple-minded in comparison, but is useful in practise \section{Conclusion} XXX add me + Modified: pypy/extradoc/talk/ecoop2009/main.bbl ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.bbl (original) +++ pypy/extradoc/talk/ecoop2009/main.bbl Fri Dec 19 21:38:21 2008 @@ -1,4 +1,4 @@ -\begin{thebibliography}{1} +\begin{thebibliography}{10} \bibitem{AACM-DLS07} D.~Ancona, M.~Ancona, A~Cuni, and N.~Matsakis. @@ -17,11 +17,50 @@ Revised Selected Papers}, volume 5146 of {\em Lecture Notes in Computer Science}, pages 123--139, 2008. +\bibitem{chang_efficient_2007} +Mason Chang, Michael Bebenita, Alexander Yermolovich, Andreas Gal, and Michael + Franz. +\newblock Efficient just-in-time execution of dynamically typed languages via + code specialization using precise runtime type inference. +\newblock Technical report, Donald Bren School of Information and Computer + Science, University of California, Irvine, 2007. + +\bibitem{DBLP:conf/dagstuhl/ConselHNNV96} +Charles Consel, Luke Hornof, Fran\c{c}ois No{\"e}l, Jacques Noy{\'e}, and + Nicolae Volansche. +\newblock A uniform approach for compile-time and run-time specialization. +\newblock In {\em Dagstuhl Seminar on Partial Evaluation}, pages 54--72, 1996. + +\bibitem{DBLP:conf/popl/ConselN96} +Charles Consel and Fran\c{c}ois No{\"e}l. +\newblock A general approach for run-time specialization and its application to + {C}. +\newblock In {\em POPL}, pages 145--156, 1996. + \bibitem{Futamura99} Yoshihiko Futamura. \newblock Partial evaluation of computation process, revisited. \newblock {\em Higher Order Symbol. Comput.}, 12(4):377--380, 1999. +\bibitem{gal_incremental_2006} +Andreas Gal and Michael Franz. +\newblock Incremental dynamic code generation with trace trees. +\newblock Technical report, Donald Bren School of Information and Computer + Science, University of California, Irvine, November 2006. + +\bibitem{gal_hotpathvm_2006} +Andreas Gal, Christian~W. Probst, and Michael Franz. +\newblock {HotpathVM}: an effective {JIT} compiler for resource-constrained + devices. +\newblock In {\em Proceedings of the 2nd international conference on Virtual + execution environments}, pages 144--153, Ottawa, Ontario, Canada, 2006. ACM. + +\bibitem{grant_dyc_2000} +Brian Grant, Markus Mock, Matthai Philipose, Craig Chambers, and Susan~J. + Eggers. +\newblock {DyC}: an expressive annotation-directed dynamic compiler for {C}. +\newblock {\em Theoretical Computer Science}, 248:147--199, 2000. + \bibitem{hoelzle_optimizing_1991} Urs H{\"o}lzle, Craig Chambers, and David Ungar. \newblock Optimizing dynamically-typed object-oriented languages with @@ -29,9 +68,26 @@ \newblock In {\em Proceedings of the European Conference on Object-Oriented Programming}, pages 21--38. Springer-Verlag, 1991. +\bibitem{Jones:peval} +Neil~D. Jones, Carsten~K. Gomard, and Peter Sestoft. +\newblock {\em Partial Evaluation and Automatic Program Generation}. +\newblock Prentice Hall, 1993. + \bibitem{RigoPedroni06} A.~Rigo and S.~Pedroni. \newblock Py{P}y's approach to virtual machine construction. \newblock In {\em OOPSLA Companion}, pages 944--953, 2006. +\bibitem{DBLP:conf/pepm/Rigo04} +Armin Rigo. +\newblock Representation-based just-in-time specialization and the psyco + prototype for python. +\newblock In {\em PEPM}, pages 15--26, 2004. + +\bibitem{sullivan_dynamic_2001} +Gregory~T. Sullivan. +\newblock Dynamic partial evaluation. +\newblock In {\em Proceedings of the Second Symposium on Programs as Data + Objects}, pages 238--256. Springer-Verlag, 2001. + \end{thebibliography} Modified: pypy/extradoc/talk/ecoop2009/main.bib ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.bib (original) +++ pypy/extradoc/talk/ecoop2009/main.bib Fri Dec 19 21:38:21 2008 @@ -68,4 +68,99 @@ year = 1993, } + at inproceedings{DBLP:conf/pepm/Rigo04, + author = {Armin Rigo}, + title = {Representation-based just-in-time specialization and the + psyco prototype for python}, + booktitle = {PEPM}, + year = {2004}, + pages = {15-26}, + ee = {http://doi.acm.org/10.1145/1014007.1014010}, + bibsource = {DBLP, http://dblp.uni-trier.de} +} + + + at inproceedings{gal_hotpathvm_2006, + address = {Ottawa, Ontario, Canada}, + title = {{HotpathVM}: an effective {JIT} compiler for resource-constrained devices}, + isbn = {1-59593-332-6}, + url = {http://portal.acm.org/citation.cfm?doid=1134760.1134780}, + doi = {10.1145/1134760.1134780}, + abstract = {We present a just-in-time compiler for a Java VM that is small enough to fit on resource-constrained devices, yet is surprisingly effective. Our system dynamically identifies traces of frequently executed bytecode instructions (which may span several basic blocks across several methods) and compiles them via Static Single Assignment (SSA) construction. Our novel use of SSA form in this context allows to hoist instructions across trace side-exits without necessitating expensive compensation code in off-trace paths. The overall memory consumption (code and data) of our system is only 150 kBytes, yet benchmarks show a speedup that in some cases rivals heavy-weight just-in-time compilers.}, + booktitle = {Proceedings of the 2nd international conference on Virtual execution environments}, + publisher = {ACM}, + author = {Andreas Gal and Christian W. Probst and Michael Franz}, + year = {2006}, + keywords = {dynamic compilation,embedded and resource-constrained systems,mixed-mode interpretive compiled systems,software trace scheduling,static single assignment form,virtual machines}, + pages = {144-153} +} + + at techreport{gal_incremental_2006, + title = {Incremental Dynamic Code Generation with Trace Trees}, + abstract = {The unit of compilation for traditional just-in-time compilers is the method. We have explored trace-based compilation, in which the unit of compilation is a loop, potentially spanning multiple methods and even library code. Using a new intermediate representation that is discovered and updated lazily on-demand while the program is being executed, our compiler generates code that is competitive with traditional dynamic compilers, but that uses only a fraction of the compile time and memory footprint.}, + institution = {Donald Bren School of Information and Computer Science, University of California, Irvine}, + author = {Andreas Gal and Michael Franz}, + month = nov, + year = {2006}, + pages = {11} +} + + + at techreport{chang_efficient_2007, + title = {Efficient Just-In-Time Execution of Dynamically Typed Languages +Via Code Specialization Using Precise Runtime Type Inference}, + abstract = {Dynamically typed languages such as JavaScript present a challenge to just-in-time compilers. In contrast to statically typed languages such as JVML, in which there are specific opcodes for common operations on primitive types (such as iadd for integer addition), all operations in dynamically typed language such as JavaScript are late-bound. Often enough, types cannot be inferred with certainty ahead of execution. As a result, just-in-time compilers for dynamically typed languages have tended to perform worse than their statically-typed counterparts. We present a new approach to compiling dynamically typed languages in which code traces observed during execution are dynamically specialized for each actually observed run-time type. For most benchmark programs, our prototype JavaScript virtual machine outperforms every other JavaScript platform known to us.}, + institution = {Donald Bren School of Information and Computer Science, University of California, Irvine}, + author = {Mason Chang and Michael Bebenita and Alexander Yermolovich and Andreas Gal and Michael Franz}, + year = {2007}, + keywords = {dynamic,javascript,JIT,traces} +} + + + + at inproceedings{DBLP:conf/popl/ConselN96, + author = {Charles Consel and + Fran\c{c}ois No{\"e}l}, + title = {A General Approach for Run-Time Specialization and its Application + to {C}}, + booktitle = {POPL}, + year = {1996}, + pages = {145-156}, + ee = {http://doi.acm.org/10.1145/237721.237767}, + bibsource = {DBLP, http://dblp.uni-trier.de} +} + at inproceedings{DBLP:conf/dagstuhl/ConselHNNV96, + author = {Charles Consel and + Luke Hornof and + Fran\c{c}ois No{\"e}l and + Jacques Noy{\'e} and + Nicolae Volansche}, + title = {A Uniform Approach for Compile-Time and Run-Time Specialization}, + booktitle = {Dagstuhl Seminar on Partial Evaluation}, + year = {1996}, + pages = {54-72}, + bibsource = {DBLP, http://dblp.uni-trier.de} +} + + at article{grant_dyc_2000, + title = {{DyC}: an expressive annotation-directed dynamic compiler for {C}}, + volume = {248}, + url = {http://citeseer.ist.psu.edu/grant97dyc.html}, + journal = {Theoretical Computer Science}, + author = {Brian Grant and Markus Mock and Matthai Philipose and Craig Chambers and Susan J. Eggers}, + year = {2000}, + keywords = {unread}, + pages = {147-199} +}, + + at inproceedings{sullivan_dynamic_2001, + title = {Dynamic Partial Evaluation}, + isbn = {3-540-42068-1}, + url = {http://portal.acm.org/citation.cfm?id=668117}, + booktitle = {Proceedings of the Second Symposium on Programs as Data Objects}, + publisher = {Springer-Verlag}, + author = {Gregory T. Sullivan}, + year = {2001}, + pages = {238-256} +} From cfbolz at codespeak.net Fri Dec 19 21:39:39 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Dec 2008 21:39:39 +0100 (CET) Subject: [pypy-svn] r60626 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219203939.B8C51168487@codespeak.net> Author: cfbolz Date: Fri Dec 19 21:39:38 2008 New Revision: 60626 Modified: pypy/extradoc/talk/ecoop2009/main.bbl pypy/extradoc/talk/ecoop2009/main.bib Log: oops, forgot to save Modified: pypy/extradoc/talk/ecoop2009/main.bbl ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.bbl (original) +++ pypy/extradoc/talk/ecoop2009/main.bbl Fri Dec 19 21:39:38 2008 @@ -8,6 +8,13 @@ {P}roceedings of the 2007 {S}ymposium on {D}ynamic {L}anguages}, pages 53--64. ACM, 2007. +\bibitem{Blanchet99escapeanalysis} +Bruno Blanchet. +\newblock Escape analysis for object oriented languages. application to java. +\newblock In {\em In Proceedings of the 14th Annual Conference on + Object-Oriented Programming Systems, Languages and Applications}, pages + 20--34, 1999. + \bibitem{BolzEtAl08} C.~F. Bolz, A.~Kuhn, A.~Lienhard, N.~D. Matsakis, O.~Nierstrasz, L.~Renggli, A.~Rigo, and T.~Verwaest. @@ -37,6 +44,14 @@ {C}. \newblock In {\em POPL}, pages 145--156, 1996. +\bibitem{Choi99escapeanalysis} +Jong deok Choi, Manish Gupta, Mauricio Serrano, Vugranam~C. Sreedhar, and Sam + Midkiff. +\newblock Escape analysis for java. +\newblock In {\em In Proceedings of the 14th Annual Conference on + Object-Oriented Programming Systems, Languages and Applications}, pages + 1--19. ACM Press, 1999. + \bibitem{Futamura99} Yoshihiko Futamura. \newblock Partial evaluation of computation process, revisited. Modified: pypy/extradoc/talk/ecoop2009/main.bib ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.bib (original) +++ pypy/extradoc/talk/ecoop2009/main.bib Fri Dec 19 21:39:38 2008 @@ -164,3 +164,20 @@ year = {2001}, pages = {238-256} } + + at INPROCEEDINGS{Blanchet99escapeanalysis, + author = {Bruno Blanchet}, + title = {Escape Analysis for Object Oriented Languages. Application to Java}, + booktitle = {In Proceedings of the 14th Annual Conference on Object-Oriented Programming Systems, Languages and Applications}, + year = {1999}, + pages = {20--34} +} + + at INPROCEEDINGS{Choi99escapeanalysis, + author = {Jong-deok Choi and Manish Gupta and Mauricio Serrano and Vugranam C. Sreedhar and Sam Midkiff}, + title = {Escape Analysis for Java}, + booktitle = {In Proceedings of the 14th Annual Conference on Object-Oriented Programming Systems, Languages and Applications}, + year = {1999}, + pages = {1--19}, + publisher = {ACM Press} +} From cfbolz at codespeak.net Fri Dec 19 21:42:06 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 19 Dec 2008 21:42:06 +0100 (CET) Subject: [pypy-svn] r60627 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081219204206.07B7B168488@codespeak.net> Author: cfbolz Date: Fri Dec 19 21:42:05 2008 New Revision: 60627 Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex Log: small fixes Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Fri Dec 19 21:42:05 2008 @@ -1,31 +1,27 @@ \section{Automatic Unboxing of Intermediate Results} \label{sec:virtuals} -XXX the following section needs a rewriting to be much more high-level and to -compare more directly with classical escape analysis - -\anto{Maybe we should talk about ``virtual instances'' and not structures, - considering the context} - Interpreters for dynamic languages typically continuously allocate a lot of small objects, for example due to boxing. This makes arithmetic operations extremely inefficient. For this reason, we implemented a way for the compiler to try to avoid memory allocations in the residual code as long as possible. The idea is to try to keep new -run-time structures "exploded": instead of a single run-time object allocated on +run-time instances "exploded": instead of a single run-time object allocated on the heap, the object is "virtualized" as a set of fresh local variables, one per field. Only when the object can be accessed by from somewhere else is it actually allocated on the heap. The effect of this is similar to that of -escape analysis \cite{XXX}, which also prevents allocations of objects that can -be proven to not escape a method or set of methods. +escape analysis \cite{Blanchet99escapeanalysis}, \cite{Choi99escapeanalysis}, +which also prevents allocations of objects that can be proven to not escape a +method or set of methods (the algorithms however are a lot more advanced than +our very simple analysis). -It is not always possible to keep structures virtual. The main +It is not always possible to keep instances virtual. The main situation in which it needs to be "forced" (i.e. actually allocated at run-time) is when the pointer escapes to some non-virtual location like -a field of a real heap structure. Virtual structures still avoid the run-time +a field of a real heap structure. Virtual instances still avoid the run-time allocation of most short-lived objects, even in non-trivial situations. -In addition to virtual structures, the compiler can also handle virtual +In addition to virtual instances, the compiler can also handle virtual containers, namely lists and dictionaries \footnote{(R)Python's dictionaries are equivalent to .NET \lstinline{Hashtable}s}. If the indexing operations can be evaluated at compile-time (i.e., if the variables holding the indexes @@ -211,7 +207,7 @@ default: continue_compilation(jitstate, cls_a) \end{lstlisting} -\caption{Promotion example 1} +\caption{Promotion step 1} \label{fig:tlc-abs-promotion-1} \end{center} \end{figure} @@ -240,7 +236,7 @@ default: continue_compilation(jitstate, cls_a) \end{lstlisting} -\caption{Promotion example 2} +\caption{Promotion step 2} \label{fig:tlc-abs-promotion-2} \end{center} \end{figure} From afa at codespeak.net Sat Dec 20 01:51:28 2008 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 20 Dec 2008 01:51:28 +0100 (CET) Subject: [pypy-svn] r60630 - pypy/trunk/lib-python/modified-2.5.2/ctypes Message-ID: <20081220005128.4184F168407@codespeak.net> Author: afa Date: Sat Dec 20 01:51:26 2008 New Revision: 60630 Added: pypy/trunk/lib-python/modified-2.5.2/ctypes/ - copied from r60629, pypy/trunk/lib-python/2.5.2/ctypes/ Modified: pypy/trunk/lib-python/modified-2.5.2/ctypes/__init__.py Log: Remove pythonapi from our ctypes module: there is no pypyAPI for the moment anyway. ctypes should now import on Windows Modified: pypy/trunk/lib-python/modified-2.5.2/ctypes/__init__.py ============================================================================== --- pypy/trunk/lib-python/2.5.2/ctypes/__init__.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/ctypes/__init__.py Sat Dec 20 01:51:26 2008 @@ -106,7 +106,7 @@ return CFunctionType if _os.name in ("nt", "ce"): - from _ctypes import LoadLibrary as _dlopen + from _ctypes import dlopen as _dlopen from _ctypes import FUNCFLAG_STDCALL as _FUNCFLAG_STDCALL if _os.name == "ce": # 'ce' doesn't have the stdcall calling convention @@ -433,12 +433,7 @@ cdll = LibraryLoader(CDLL) pydll = LibraryLoader(PyDLL) -if _os.name in ("nt", "ce"): - pythonapi = PyDLL("python dll", None, _sys.dllhandle) -elif _sys.platform == "cygwin": - pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2]) -else: - pythonapi = PyDLL(None) +pythonapi = None if _os.name in ("nt", "ce"): From antocuni at codespeak.net Sat Dec 20 10:19:03 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 10:19:03 +0100 (CET) Subject: [pypy-svn] r60631 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220091903.AD684168491@codespeak.net> Author: antocuni Date: Sat Dec 20 10:19:02 2008 New Revision: 60631 Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex Log: I guess this was a typo Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Sat Dec 20 10:19:02 2008 @@ -152,7 +152,7 @@ stack = [] while pc < code_len: opcode = ord(code[pc]) - opcode = hint(opcode, promote_class=True) + opcode = hint(opcode, concrete=True) pc += 1 if opcode == PUSH: From antocuni at codespeak.net Sat Dec 20 11:45:14 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 11:45:14 +0100 (CET) Subject: [pypy-svn] r60632 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220104514.C011316846D@codespeak.net> Author: antocuni Date: Sat Dec 20 11:45:12 2008 New Revision: 60632 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex pypy/extradoc/talk/ecoop2009/intro.tex pypy/extradoc/talk/ecoop2009/jitgen.tex pypy/extradoc/talk/ecoop2009/main.tex pypy/extradoc/talk/ecoop2009/rainbow.tex pypy/extradoc/talk/ecoop2009/tlc.tex Log: several small fixes. Some rewording in the cli backend section Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Sat Dec 20 11:45:12 2008 @@ -16,14 +16,13 @@ \subsection{Flexswitches} -As already explained, \dacom{I guess flexswitches will be introduced - in the previous sect.} \emph{flexswitch} is one of the key +As already explained, \emph{flexswitch} is one of the key concepts allowing the JIT compiler generator to produce code which can be incrementally specialized and compiled at run time. A flexswitch is a special kind of switch which can be dynamically -extended with new cases; intuitively, its behavior can be described -well in terms of flow graphs. Indeed, a flexswitch can be considered +extended with new cases. Intuitively, its behavior can be described +well in terms of flow graphs: a flexswitch can be considered as a special flow graph block where links to newly created blocks are dynamically added whenever new cases are needed. @@ -58,10 +57,15 @@ Since in .NET methods are the basic units of compilation, a possible solution consists in creating a new method any time a new case has to be added to a flexswitch. -In this way, whereas flow graphs without flexswitches are translated -to a single method, the translation of flow graphs which can dynamically grow because of -flexswitches will be scattered over several methods. -Summarizing, the backend behaves in the following way: + +It is important to underline the difference between flow graphs and a methods: +the first are the logical unit of code as seen by the JIT compiler, each of +them being concretely implemented by \emph{one or more} methods. + +In this way, whereas flow graphs without flexswitches are translated to a +single method, the translation of \emph{growable} flow graphs will be +scattered over several methods. Summarizing, the backend behaves in the +following way: \begin{itemize} \item Each flow graph is translated in a collection of methods which can grow dynamically. Each collection contains at least one @@ -70,49 +74,44 @@ whenever a new case is added to a flexswitch. \item Each either primary or secondary method implements a certain - number of blocks, all belonging to the same flow graph. Among these blocks - there always exists an initial block whose input arguments - might be passed as arguments of the method; however, for - implementation reasons (see the details below) the input variables - of all blocks (including the initial one) - are implemented as local variables of the method. + number of blocks, all belonging to the same flow graph. \end{itemize} -When a new case is added to a flexswitch, new blocks are generated -and translated by the backend in a new single method pointed -by a delegate \footnote{\emph{Delegates} are the .NET equivalent of function pointers} - of which is stored in the code implementing the flexswitch, -so that the method can be invoked later. +When a new case is added to a flexswitch, the backend generates the new blocks +into a new single method. The newly created method is pointed by a +delegate \footnote{\emph{Delegates} are the .NET equivalent of function + pointers} stored in the flexswitch, so that it can be invoked later when +needed. \subsubsection{Internal and external links} -A link is called \emph{internal} if it connects two blocks implemented -by the same method, +A link is called \emph{internal} if it connects two blocks contained +in the same method, \emph{external} otherwise. -Following an internal link would not be difficult in IL bytecode: a jump to +Following an internal link is easy in IL bytecode: a jump to the corresponding code fragment in the same method can be emitted to execute the new block, whereas the appropriate local variables can be used for passing arguments. -Also following an external link whose target is an initial block could -be easily implemented, by just invoking the corresponding method. +Following an external link whose target is an initial block could also +be easily implemented, by just invoking the corresponding method. What cannot be easily implemented in CLI is following an external link whose target is not an initial block; consider, for instance, the outgoing link of the block dynamically added in the right-hand side -picture of Figure~\ref{flexswitch-fig}. How it is possible to pass the -right arguments to the target block? +picture of Figure~\ref{flexswitch-fig}. How it is possible to jump in +the middle of a method? To solve this problem every method contains a special code, called -\emph{dispatcher}; whenever a method is invoked, its dispatcher is +\emph{dispatcher}: whenever a method is invoked, its dispatcher is executed first\footnote{The dispatcher should not be confused with the initial block of a method.} to determine which block has to be executed. This is done by passing to the method a 32 bits number, called \emph{block id}, which uniquely identifies the next block of the graph to be executed. -The high 2 bytes \dacom{word was meant as a fixed-sized group of bits} of a block id constitute the id of the method to which the block -belongs, whereas the low 2 bytes constitute a progressive number univocally identifying -each block implemented by the method. +The high 2 bytes of a block id constitute the \emph{method id}, which +univocally identifies a method in a graph, whereas the low 2 bytes constitute +a progressive number univocally identifying a block inside each method. The picture in Figure~\ref{block-id-fig} shows a graph composed of three methods (for simplicity, dispatchers are not shown); method ids are in red, whereas @@ -204,7 +203,7 @@ secondary methods of a graph must have the same signature. \end{itemize} -Therefore, the only solution we came up with is defining a class +Therefore, the solution we came up with is defining a class \lstinline{InputArgs} for passing sequences of arguments whose length and type is variable. \begin{small} Modified: pypy/extradoc/talk/ecoop2009/intro.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/intro.tex (original) +++ pypy/extradoc/talk/ecoop2009/intro.tex Sat Dec 20 11:45:12 2008 @@ -31,10 +31,6 @@ \subsection{PyPy and RPython} -\anto{as Armin points out, the two CLI backends can be easily confused; what - about renaming the ``CLI Backend for flowgraphs'' into ``CLI bytecode - compiler''? Any better idea for the name?} - \begin{figure}[h] \begin{center} \includegraphics[width=.6\textwidth]{diagram0} Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Sat Dec 20 11:45:12 2008 @@ -12,7 +12,7 @@ \end{figure} The JIT generation framework uses partial evaluation techniques to generate a -dynamic compiler from an interpreter; the idea is inspired by Psyco, which +dynamic compiler from an interpreter; the idea is inspired by Psyco \cite{DBLP:conf/pepm/Rigo04}, which uses the same techniques but it's manually written instead of being automatically generated. @@ -161,7 +161,9 @@ same type inference engine that is used on the source RPython program. This is called the \emph{hint-annotator}; it operates over input graphs that are already low-level instead of -RPython-level, and propagates annotations that do not track types but +RPython-level, \anto{we never make distinction between low-level and rpython-level +flowgraphs, do we? I propose to talk about ``intermediate flowgraphs''} +and propagates annotations that do not track types but value dependencies and manually-provided binding time hints. The normal process of the hint-annotator is to propagate the binding @@ -226,3 +228,5 @@ it prevents under-specialization: an unsatisfiable \texttt{hint(v1, concrete=True)} is reported as an error. +\anto{maybe we should at least mention promotion, and refer to the proper + section for details?} Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Sat Dec 20 11:45:12 2008 @@ -14,13 +14,23 @@ \usepackage{ifthen} \usepackage{xspace} \usepackage{listings} +\usepackage{fancyvrb} \usepackage[pdftex]{graphicx} %\input{macros} \pagestyle{plain} -\lstset{mathescape=true,language=Java,basicstyle=\tt,keywordstyle=\bf} +%\lstset{mathescape=true,language=Java,basicstyle=\tt,keywordstyle=\bf} +\lstset{language=Python, + basicstyle=\footnotesize\ttfamily, + keywordstyle=\color{blue}, % I couldn't find a way to make chars both bold and tt + frame=lines, + stringstyle=\color{blue}, + fancyvrb=true, + xleftmargin=20pt,xrightmargin=20pt, + showstringspaces=false} + %\renewcommand{\baselinestretch}{.98} \newboolean{showcomments} Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Sat Dec 20 11:45:12 2008 @@ -6,8 +6,8 @@ inefficient. For this reason, we implemented a way for the compiler to try to avoid memory allocations in the residual code as long as possible. The idea is to try to keep new -run-time instances "exploded": instead of a single run-time object allocated on -the heap, the object is "virtualized" as a set +run-time instances \emph{exploded}: instead of a single run-time object allocated on +the heap, the object is \emph{virtualized} as a set of fresh local variables, one per field. Only when the object can be accessed by from somewhere else is it actually allocated on the heap. The effect of this is similar to that of escape analysis \cite{Blanchet99escapeanalysis}, \cite{Choi99escapeanalysis}, @@ -16,7 +16,7 @@ our very simple analysis). It is not always possible to keep instances virtual. The main -situation in which it needs to be "forced" (i.e. actually allocated at +situation in which it needs to be \emph{forced} (i.e. actually allocated at run-time) is when the pointer escapes to some non-virtual location like a field of a real heap structure. Virtual instances still avoid the run-time allocation of most short-lived objects, even in non-trivial situations. @@ -66,10 +66,10 @@ must be put in the source code of the interpreter. From a partial evaluation point of view, promotion is the converse of -the operation generally known as "lift" \cite{XXX}. Lifting a value means +the operation generally known as \emph{lift} \cite{XXX}. Lifting a value means copying a variable whose binding time is compile-time into a variable whose binding time is run-time ? it corresponds to the compiler -"forgetting" a particular value that it knew about. By contrast, +``forgetting'' a particular value that it knew about. By contrast, promotion is a way for the compiler to gain \emph{more} information about the run-time execution of a program. Clearly, this requires fine-grained feedback from run-time to compile-time, thus a @@ -77,27 +77,25 @@ Promotion requires interleaving compile-time and run-time phases, otherwise the compiler can only use information that is known ahead of -time. It is impossible in the "classical" approaches to partial +time. It is impossible in the ``classical'' approaches to partial evaluation, in which the compiler always runs fully ahead of execution. This is a problem in many realistic use cases. For example, in an interpreter for a dynamic language, there is mostly no information that can be clearly and statically used by the compiler before any code has run. -A very different point of view on promotion is as a generalization of techniques -that already exist in dynamic compilers as found in modern virtual machines for -object-oriented language. In this context feedback -techniques are crucial for good results. The main goal is to -optimize and reduce the overhead of dynamic dispatching and indirect -invocation. This is achieved with variations on the technique of -polymorphic inline caches \cite{hoelzle_optimizing_1991}: the dynamic lookups are cached and -the corresponding generated machine code contains chains of -compare-and-jump instructions which are modified at run-time. These -techniques also allow the gathering of information to direct inlining for even -better optimization results. -\anto{What about this: While traditional PICs are only applied to indirect - calls, promotion is a more general operation that can be applied to any kind - of value, including instances of user-defined classes or integer numbers} +A very different point of view on promotion is as a generalization of +techniques that already exist in dynamic compilers as found in modern virtual +machines for object-oriented language, like \emph{Polymorphic Inline Cache} +(PIC, \cite{hoelzle_optimizing_1991}) and its variations, whose main goal is +to optimize and reduce the overhead of dynamic dispatching and indirect +invocation: the dynamic lookups are cached and the corresponding generated +machine code contains chains of compare-and-jump instructions which are +modified at run-time. These techniques also allow the gathering of +information to direct inlining for even better optimization results. Compared +to PICs, promotion is more general because it can be applied not only to +indirect calls but to any kind of value, including instances of user-defined +classes or integer numbers. In the presence of promotion, dispatch optimization can usually be reframed as a partial evaluation task. Indeed, if the type of the @@ -257,13 +255,13 @@ is a hint to promote the class of \lstinline{b}. Although in general promotion is implemented through a flexswitch, in this case it is not needed as \lstinline{b} holds a \emph{virtual instance}, whose class is already -known (as described in previous section). +known (as described in the previous section). Then, the compiler knows the exact class of \lstinline{b}, thus it can inline the calls to \lstinline{lt}. Moreover, inside \lstinline{lt} there is a call to \lstinline{a.int_o()}, which is inlined as well for the very same -reason. - -Moreover, as we saw in section \ref{sec:virtuals}, the \lstinline{IntObj} +reason. Moreover, as we saw in section \ref{sec:virtuals}, the \lstinline{IntObj} instance can be virtualized, so that the subsequent \lstinline{BR_COND} opcode can be compiled efficiently without needing any more flexswitch. + +\anto{We should show the very final version of the code, with all instances virtualized} Modified: pypy/extradoc/talk/ecoop2009/tlc.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/tlc.tex (original) +++ pypy/extradoc/talk/ecoop2009/tlc.tex Sat Dec 20 11:45:12 2008 @@ -36,7 +36,7 @@ \lstinline{ADD}, \lstinline{SUB}, etc. \item \textbf{Comparisons} like \lstinline{EQ}, \lstinline{LT}, \lstinline{GT}, etc. -\item \textbf{Object-oriented}: operations on objects: \lstinline{NEW}, +\item \textbf{Object-oriented} operations: \lstinline{NEW}, \lstinline{GETATTR}, \lstinline{SETATTR}, \lstinline{SEND}. \item \textbf{List operations}: \lstinline{CONS}, \lstinline{CAR}, \lstinline{CDR}. From arigo at codespeak.net Sat Dec 20 12:59:47 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Dec 2008 12:59:47 +0100 (CET) Subject: [pypy-svn] r60634 - pypy/branch/oo-jit/pypy/rpython/lltypesystem/test Message-ID: <20081220115947.C0CAA1683F8@codespeak.net> Author: arigo Date: Sat Dec 20 12:59:45 2008 New Revision: 60634 Modified: pypy/branch/oo-jit/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: More tests in test_ll2ctypes... passing. Modified: pypy/branch/oo-jit/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/branch/oo-jit/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sat Dec 20 12:59:45 2008 @@ -4,7 +4,7 @@ from pypy.rpython.lltypesystem import lltype, rffi, llmemory from pypy.rpython.tool import rffi_platform from pypy.rpython.lltypesystem.ll2ctypes import lltype2ctypes, ctypes2lltype -from pypy.rpython.lltypesystem.ll2ctypes import standard_c_lib +from pypy.rpython.lltypesystem.ll2ctypes import standard_c_lib, get_ctypes_type from pypy.rpython.lltypesystem.ll2ctypes import uninitialized2ctypes from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED from pypy.rpython.annlowlevel import llhelper @@ -899,3 +899,18 @@ res = eating_callback(X(), callback) assert res == 10 + + def test_recursive_struct_more(self): + NODE = lltype.ForwardReference() + NODE.become(lltype.Struct('NODE', ('value', lltype.Signed), + ('next', lltype.Ptr(NODE)))) + CNODEPTR = get_ctypes_type(NODE) + pc = CNODEPTR() + pc.value = 42 + pc.next = ctypes.pointer(pc) + p = ctypes2lltype(lltype.Ptr(NODE), ctypes.pointer(pc)) + assert p.value == 42 + assert p.next == p + pc2 = lltype2ctypes(p) + assert pc2.contents.value == 42 + assert pc2.contents.next.contents.value == 42 From antocuni at codespeak.net Sat Dec 20 13:01:10 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 13:01:10 +0100 (CET) Subject: [pypy-svn] r60636 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220120110.166D41683F8@codespeak.net> Author: antocuni Date: Sat Dec 20 13:01:09 2008 New Revision: 60636 Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex pypy/extradoc/talk/ecoop2009/main.tex Log: make sources a bit smaller, and comment out a big image which is not referenced by the text Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Sat Dec 20 13:01:09 2008 @@ -3,6 +3,7 @@ Traditional JIT compilers are hard to write, time consuming, hard to evolve, etc. etc. +\commentout{ \begin{figure}[h] \begin{center} \includegraphics[width=.6\textwidth]{diagram1} @@ -10,6 +11,7 @@ language $L$ with JIT compilation for the .NET platform} \end{center} \end{figure} +} The JIT generation framework uses partial evaluation techniques to generate a dynamic compiler from an interpreter; the idea is inspired by Psyco \cite{DBLP:conf/pepm/Rigo04}, which Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Sat Dec 20 13:01:09 2008 @@ -23,9 +23,9 @@ %\lstset{mathescape=true,language=Java,basicstyle=\tt,keywordstyle=\bf} \lstset{language=Python, - basicstyle=\footnotesize\ttfamily, + basicstyle=\scriptsize\ttfamily, keywordstyle=\color{blue}, % I couldn't find a way to make chars both bold and tt - frame=lines, + frame=none, stringstyle=\color{blue}, fancyvrb=true, xleftmargin=20pt,xrightmargin=20pt, From antocuni at codespeak.net Sat Dec 20 14:27:19 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 14:27:19 +0100 (CET) Subject: [pypy-svn] r60638 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220132719.064211683F8@codespeak.net> Author: antocuni Date: Sat Dec 20 14:27:16 2008 New Revision: 60638 Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex pypy/extradoc/talk/ecoop2009/tlc.tex Log: comment out a possible confusing example, and add an XXX Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Sat Dec 20 14:27:16 2008 @@ -40,6 +40,9 @@ finished the job: \cfbolz{I think we can kill this for space reasons} +\anto{If we decide to keep them, we must remember to explain the python-like + syntax, as it is no longer in tlc.tex} + \begin{lstlisting} def main(n): result = 1 Modified: pypy/extradoc/talk/ecoop2009/tlc.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/tlc.tex (original) +++ pypy/extradoc/talk/ecoop2009/tlc.tex Sat Dec 20 14:27:16 2008 @@ -77,7 +77,7 @@ \end{itemize} -\subsection{TLC examples} +\subsection{TLC example} As we said above, TLC exists only at bytecode level; to ease the development of TLC programs, we wrote an assembler that generates TLC bytecode. Figure \ref{fig:tlc-abs} @@ -108,6 +108,11 @@ \end{center} \end{figure} +\commentout{ + +\anto{I think this example could cause confusion, because the synatx is TOO + similar to (R)Python, hence in the next section it could be unclear which is + the leves of the examples} Since reading TLC programs at bytecode level is hard, in this paper we will use an invented Python-like syntax to describe examples, although we need to @@ -120,3 +125,4 @@ return -n return n \end{lstlisting} +} From antocuni at codespeak.net Sat Dec 20 14:57:21 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 14:57:21 +0100 (CET) Subject: [pypy-svn] r60639 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220135721.85F51168471@codespeak.net> Author: antocuni Date: Sat Dec 20 14:57:18 2008 New Revision: 60639 Modified: pypy/extradoc/talk/ecoop2009/conclusion.tex pypy/extradoc/talk/ecoop2009/rainbow.tex Log: put two code snippets side by side, and fix a typo Modified: pypy/extradoc/talk/ecoop2009/conclusion.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/conclusion.tex (original) +++ pypy/extradoc/talk/ecoop2009/conclusion.tex Sat Dec 20 14:57:18 2008 @@ -1,4 +1,4 @@ -\section{Related Word} +\section{Related Works} Promotion is a concept that we have already explored in other contexts. Psyco is a run-time specialiser for Python that uses promotion (called ``unlift'' in \cite{DBLP:conf/pepm/Rigo04}). However, Psyco is a manually written JIT, is Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Sat Dec 20 14:57:18 2008 @@ -41,7 +41,7 @@ \begin{center} \input{tlc-folded-virtualized.py} \caption{The result of virtualizing the \lstinline{stack} list} -\label{fig:tlc-main} +\label{fig:tlc-folded-virtualized} \end{center} \end{figure} @@ -114,7 +114,7 @@ independent dynamic compiler generation. Promotion is invoked with the use of a hint as well: -\texttt{v2 = hint(v1, promote=True)}. +\lstinline{v2 = hint(v1, promote=True)}. This hint is a \emph{local} request for \texttt{v2} to be green, without requiring \texttt{v1} to be green. Note that this amounts to copying a red value into a green one, which is not possible in classical @@ -144,43 +144,46 @@ \begin{figure}[h] \begin{center} -\begin{lstlisting}[language=Python] +\begin{tabular}{l|l} +\begin{lstlisting} def interp_eval(code, pc, args, pool): - code_len = len(code) - stack = [] - while pc < code_len: - opcode = ord(code[pc]) - opcode = hint(opcode, concrete=True) - pc += 1 - - if opcode == PUSH: - ... - elif opcode == LT: - a, b = stack.pop(), stack.pop() - hint(a, promote_class=True) - hint(b, promote_class=True) - stack.append(IntObj(b.lt(a))) + code_len = len(code) + stack = [] + while pc < code_len: + opcode = ord(code[pc]) + opcode = hint(opcode, concrete=True) + pc += 1 + + if opcode == PUSH: + ... + elif opcode == LT: + a, b = stack.pop(), stack.pop() + hint(a, promote_class=True) + hint(b, promote_class=True) + stack.append(IntObj(b.lt(a))) \end{lstlisting} -\caption{Usage of hints in TLC's main loop} -\label{fig:tlc-main-hints} -\end{center} -\end{figure} - -\begin{figure}[h] -\begin{center} -\begin{lstlisting}[language=Python] +& +\hspace{2pt} +\begin{lstlisting} class IntObj(Obj): - ... - def lt(self, other): - return self.value < other.int_o() - def sub(self, other): - return IntObj(self.value - other.int_o()) - def int_o(self): - return self.value + + def lt(self, other): + return (self.value < + other.int_o()) + + def sub(self, other): + return IntObj(self.value - + other.int_o()) + + def int_o(self): + return self.value + + ... \end{lstlisting} -\caption{Excerpt of the \lstinline{IntObj} class} -\label{fig:tlc-intobj} +\end{tabular} \end{center} +\caption{Usage of hints in TLC's main loop and excerpt of the \lstinline{IntObj} class} +\label{fig:tlc-main-hints} \end{figure} By promoting the class of \lstinline{a} and \lstinline{b}, we tell the JIT From antocuni at codespeak.net Sat Dec 20 16:40:10 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 16:40:10 +0100 (CET) Subject: [pypy-svn] r60640 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220154010.78D501684AB@codespeak.net> Author: antocuni Date: Sat Dec 20 16:40:08 2008 New Revision: 60640 Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex Log: add the final version of the code Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Sat Dec 20 16:40:08 2008 @@ -125,6 +125,7 @@ \subsection{Implementing Promotion} +\label{sec:implementing-promotion} The implementation of promotion requires a tight coupling between compile-time and run-time: a \emph{callback}, put in the generated code, @@ -226,7 +227,8 @@ IntObj: # hint(b, promote_class=True) needs no code v0 = IntObj(b.value < a.value) - if v0.value: + cond = v0 + if cond.value: return a else: v0 = IntObj(0) @@ -267,4 +269,27 @@ instance can be virtualized, so that the subsequent \lstinline{BR_COND} opcode can be compiled efficiently without needing any more flexswitch. -\anto{We should show the very final version of the code, with all instances virtualized} +Figure~\ref{fig:tlc-abs-final} shows the final, fully optimized version of the +code, with all the instances virtualized and the unneeded temporary variables +removed. + +\begin{figure}[h] +\begin{center} +\begin{lstlisting}[language=Python] +def interp_eval_abs(args): + a = args[0] + cls_a = a.__class__ + switch cls_a: + IntObj: + # 0 is the constant "value" field of the virtualized IntObj(0) + if 0 < a.value: + return a + else: + return IntObj(0 - a.value) + default: + continue_compilation(jitstate, cls_a) +\end{lstlisting} +\caption{Final optimized version of the \lstinline{abs} example} +\label{fig:tlc-abs-final} +\end{center} +\end{figure} From arigo at codespeak.net Sat Dec 20 16:43:23 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 20 Dec 2008 16:43:23 +0100 (CET) Subject: [pypy-svn] r60641 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220154323.DD7891684A8@codespeak.net> Author: arigo Date: Sat Dec 20 16:43:23 2008 New Revision: 60641 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex pypy/extradoc/talk/ecoop2009/main.tex pypy/extradoc/talk/ecoop2009/rainbow.tex Log: Standardize the footnote format. Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Sat Dec 20 16:43:23 2008 @@ -79,8 +79,8 @@ When a new case is added to a flexswitch, the backend generates the new blocks into a new single method. The newly created method is pointed by a -delegate \footnote{\emph{Delegates} are the .NET equivalent of function - pointers} stored in the flexswitch, so that it can be invoked later when +delegate\footnote{\emph{Delegates} are the .NET equivalent of function + pointers.} stored in the flexswitch, so that it can be invoked later when needed. \subsubsection{Internal and external links} Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Sat Dec 20 16:43:23 2008 @@ -49,6 +49,7 @@ \newcommand\dacom[1]{\nb{DA}{#1}} \newcommand\cfbolz[1]{\nb{CFB}{#1}} \newcommand\anto[1]{\nb{ANTO}{#1}} +\newcommand\arigo[1]{\nb{AR}{#1}} \newcommand{\commentout}[1]{} \let\oldcite=\cite Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Sat Dec 20 16:43:23 2008 @@ -22,7 +22,7 @@ allocation of most short-lived objects, even in non-trivial situations. In addition to virtual instances, the compiler can also handle virtual -containers, namely lists and dictionaries \footnote{(R)Python's dictionaries +containers, namely lists and dictionaries\footnote{(R)Python's dictionaries are equivalent to .NET \lstinline{Hashtable}s}. If the indexing operations can be evaluated at compile-time (i.e., if the variables holding the indexes are green), the compiler internally keeps track of the state of the container @@ -190,10 +190,10 @@ By promoting the class of \lstinline{a} and \lstinline{b}, we tell the JIT compiler not to generate code until it knows the exact RPython class of both. Figure \ref{fig:tlc-abs-promotion-1} shows the -code \footnote{\lstinline{switch} is not a legal (R)Python statement, it is - used here only as a pseudocode example} generated while compiling the usual +code\footnote{\lstinline{switch} is not a legal (R)Python statement, it is + used here only as a pseudocode example.} generated while compiling the usual \lstinline{abs} function: note that, compared to figure -\ref{fig:tlc-folded-virtualized}, the code stops just before the calls +\ref{fig:tlc-folded-virtualized}, the code stops just before the call \lstinline{b.lt(a)}. \begin{figure}[h] From antocuni at codespeak.net Sat Dec 20 16:48:04 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 16:48:04 +0100 (CET) Subject: [pypy-svn] r60642 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220154804.AADE31684AD@codespeak.net> Author: antocuni Date: Sat Dec 20 16:48:03 2008 New Revision: 60642 Added: pypy/extradoc/talk/ecoop2009/blockid.odg - copied, changed from r60336, user/antocuni/blog/flexswitch-cli.odg pypy/extradoc/talk/ecoop2009/blockid.pdf (contents, props changed) Removed: pypy/extradoc/talk/ecoop2009/blockid.png Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex pypy/extradoc/talk/ecoop2009/clibackend.tex Log: a introduction about jit backends and jit layering, fix an XXX Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Sat Dec 20 16:48:03 2008 @@ -1,4 +1,5 @@ \section{Benchmarks} +\label{sec:benchmarks} \anto{We should say somewhere that flexswitches are slow but benchmarks are so good because they are not involved in the inner loops} Copied: pypy/extradoc/talk/ecoop2009/blockid.odg (from r60336, user/antocuni/blog/flexswitch-cli.odg) ============================================================================== Binary files. No diff available. Added: pypy/extradoc/talk/ecoop2009/blockid.pdf ============================================================================== Binary file. No diff available. Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Sat Dec 20 16:48:03 2008 @@ -1,24 +1,46 @@ \section{The CLI backend for the Generated JIT} \label{sec:clibackend} -%\subsection{Promotion on CLI} -% -%Implementing promotion on top of CLI is not straightforward, as it needs to -%patch and modify the already generated code, and this is not possible on .NET. -% -%To solve, we do xxx and yyy etc. etc. -% - -\anto{XXX: add an overview section of the backend, else we only talk about - flexswitches} -\cfbolz{it should maybe also have a nice introductory paragraph how the -interface between the JIT works?} +\subsection{JIT layering} + +From the implementation point of view, the JIT generator is divided into a +frontend and several backends. The goal of the frontend is to generate a JIT +compiler which works as described in the previous sections. Internally, the +frontend represents the compiled code as \emph{flow graphs}, and the role of +the backends is to translate flowgraphs into machine code. + +At the moment of writing, three backends have been implemented: one for Intel +\emph{x86} processors, one for \emph{PowerPC} processors, and one for the +\emph{CLI Virtual Machine}. The latter is special because instead of emitting +code for a real CPU it emits code for a virtual machine: before being +executed, the generated code will be compiled again by .NET's own JIT +compiler. + +Thus, when using the CLI backend, we actually have two layers of +JIT-compilation, each layer removing away different kinds of overhead. By +operating at a higher level, our JIT can potentially do a better job than the +.NET one in some contexts, as our benchmarks demonstrate (see +section~\ref{sec:benchmarks}). On the other hand, the lower-level .NET JIT is +very good at producing machine code, much more than PyPy's own \emph{x86} +backend for example. By combining the strengths of both we can get highly +efficient machine code. + +As usual, the drawback is that programs that runs for a very short period of +time could run slower with JIT than without, due to the time spent doing the +initial (double) compilation. It is important to underline that so far we +mostly focused making the JIT able to produce very fast code, without trying +to optimzed the compilation phase itself. \subsection{Flexswitches} -As already explained, \emph{flexswitch} is one of the key -concepts allowing the JIT compiler generator to produce code which can -be incrementally specialized and compiled at run time. +For a large part, implementing the CLI backend is easy and straightforward, as +there is a close corrispondence between most of the operations used by +frontend's flowgraphs and the CLI instructions. Thus, we will not dig into +details for this part. + +However the concept of \emph{flexswitch}, as described in +Section~\ref{sec:implementing-promotion}, does not have any direct equivalent +in the CLI model, and it is hard to implement efficiently. A flexswitch is a special kind of switch which can be dynamically extended with new cases. Intuitively, its behavior can be described @@ -87,7 +109,7 @@ A link is called \emph{internal} if it connects two blocks contained in the same method, - \emph{external} otherwise. +\emph{external} otherwise. Following an internal link is easy in IL bytecode: a jump to the corresponding code fragment in the same method can be emitted @@ -122,8 +144,7 @@ \begin{figure}[h] \begin{center} \includegraphics[height=6cm]{blockid} -\caption{Method and block ids.\dacom{in the picture ``Main method'' must be - replaced with ``Primary method'' and in the primary method the block with number 0 should be added}}\label{block-id-fig} +\caption{Method and block ids.}\label{block-id-fig} \end{center} \end{figure} From antocuni at codespeak.net Sat Dec 20 17:01:37 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 17:01:37 +0100 (CET) Subject: [pypy-svn] r60643 - pypy/extradoc/talk/ecoop2009/benchmarks Message-ID: <20081220160137.EE69D1684A8@codespeak.net> Author: antocuni Date: Sat Dec 20 17:01:36 2008 New Revision: 60643 Added: pypy/extradoc/talk/ecoop2009/benchmarks/accumulator.tlc.src - copied unchanged from r60533, pypy/branch/oo-jit/pypy/jit/tl/accumulator.tlc.src pypy/extradoc/talk/ecoop2009/benchmarks/factorial.tlc.src pypy/extradoc/talk/ecoop2009/benchmarks/fibo.tlc.src - copied unchanged from r60525, pypy/branch/oo-jit/pypy/jit/tl/fibo.tlc.src Log: add tlc sources for the benchmarks Added: pypy/extradoc/talk/ecoop2009/benchmarks/factorial.tlc.src ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ecoop2009/benchmarks/factorial.tlc.src Sat Dec 20 17:01:36 2008 @@ -0,0 +1,21 @@ + PUSH 1 # accumulator + PUSHARG + +start: + PICK 0 + PUSH 1 + LE + BR_COND exit + + SWAP + PICK 1 + MUL + SWAP + PUSH 1 + SUB + PUSH 1 + BR_COND start + +exit: + POP + RETURN From antocuni at codespeak.net Sat Dec 20 17:10:24 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 17:10:24 +0100 (CET) Subject: [pypy-svn] r60644 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220161024.C04E91684A3@codespeak.net> Author: antocuni Date: Sat Dec 20 17:10:24 2008 New Revision: 60644 Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex Log: hide the python-like syntax for factorial and fibonacci, add a link to the sources Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Sat Dec 20 17:10:24 2008 @@ -23,10 +23,9 @@ \end{enumerate} Moreover, for each benchmark we also show the time taken by running the -equivalent program written in C\#. By comparing the results against C\#, we -can see how far we are from the supposed optimal performances. \anto{I - don't really like the last sentence, but right now I can't think of another - way to phrase it. Rewording welcome :-)} +equivalent program written in C\# \footnote{The sources for both TLC and C\# + programs are available at: + http://codespeak.net/svn/pypy/extradoc/talk/ecoop2009/benchmarks/} The benchmarks have been run on machine with Intel Pentium 4 CPU running at 3.20 GHz and 2 GB of RAM, running Microsoft Windows XP and Microsoft .NET @@ -36,10 +35,11 @@ To benchmark arithmetic operations between integers, we wrote a simple program that computes the factorial of a given number. The algorithm is -straightforward, and the loop contains only three operations: one +straightforward, thus we are not showing the source code. The loop contains only three operations: one multiplication, one subtraction, and one comparison to check if we have -finished the job: +finished the job. +\commentout{ \cfbolz{I think we can kill this for space reasons} \anto{If we decide to keep them, we must remember to explain the python-like syntax, as it is no longer in tlc.tex} @@ -52,9 +52,10 @@ n = n - 1 return n \end{lstlisting} +} When doing plain interpretation, we need to create and destroy three temporary -objects at each iterations. By contrast, the code generated by the JIT does +objects at each iteration. By contrast, the code generated by the JIT does much better. At the first iteration, the classes of the two operands of the multiplication are promoted; then, the JIT compiler knows that both are integers, so it can inline the code to compute the result. Moreover, it can @@ -69,8 +70,9 @@ respectively). Similarly, we wrote a program to calculate the $n_{th}$ Fibonacci number, for -which we can do the same reasoning as above: +which we can do the same reasoning as above. +\commentout{ \cfbolz{I think we can kill this for space reasons} \begin{lstlisting} def main(n): @@ -83,6 +85,7 @@ n = n - 1 return b \end{lstlisting} +} \anto{these tables are ugly} @@ -137,7 +140,7 @@ \end{table} -Tables \ref{tab:factorial} and \ref{tab:fibo} show the time spent to calculate +Tables \ref{tab:factorial} and \ref{tab:fibo} show the seconds spent to calculate the factorial and Fibonacci for various $n$. As we can see, for small values of $n$ the time spent running the JIT compiler is much higher than the time spent to simply interpret the program. This is an expected result, as till From antocuni at codespeak.net Sat Dec 20 17:15:20 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 17:15:20 +0100 (CET) Subject: [pypy-svn] r60645 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220161520.E4DDE1684A7@codespeak.net> Author: antocuni Date: Sat Dec 20 17:15:20 2008 New Revision: 60645 Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex Log: explain that the syntax is invented Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Sat Dec 20 17:15:20 2008 @@ -27,7 +27,7 @@ programs are available at: http://codespeak.net/svn/pypy/extradoc/talk/ecoop2009/benchmarks/} -The benchmarks have been run on machine with Intel Pentium 4 CPU running at +The benchmarks have been run on machine with an Intel Pentium 4 CPU running at 3.20 GHz and 2 GB of RAM, running Microsoft Windows XP and Microsoft .NET Framework 2.0. @@ -165,8 +165,13 @@ \subsection{Object-oriented features} To measure how the JIT handles object-oriented features, we wrote a very -simple benchmark that involves attribute lookups and polymorphic method calls: +simple benchmark that involves attribute lookups and polymorphic method calls. +Since the TLC assembler source is long and hard to read, +Figure~\ref{fig:accumulator} shows the equivalent program written in an +invented Python-like syntax. +\begin{figure}[h] +\begin{center} \begin{lstlisting} def main(n): if n < 0: @@ -186,6 +191,10 @@ def add(x): this.value = this.value + x \end{lstlisting} +\caption{The \emph{accumulator} example, written in a invented Python-like syntax} +\label{fig:accumulator} +\end{center} +\end{figure} The two \lstinline{new} operations create an object with exactly one field \lstinline{value} and one method \lstinline{accumulate}, whose implementation From antocuni at codespeak.net Sat Dec 20 17:50:49 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 17:50:49 +0100 (CET) Subject: [pypy-svn] r60646 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220165049.44B5B1683F7@codespeak.net> Author: antocuni Date: Sat Dec 20 17:50:47 2008 New Revision: 60646 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex Log: small tweaks in the flexswitch section, kill a section that we decided not to write Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Sat Dec 20 17:50:47 2008 @@ -148,8 +148,8 @@ \end{center} \end{figure} -For instance, the code\footnote{For simplicity we write C\# code instead of -the actual IL bytecode.} generated for the dispatcher of method \texttt{0x0002} +The code\footnote{For simplicity we write C\# code instead of +the actual IL bytecode.} generated for the dispatcher of methods is similar to the following fragment: \begin{small} \begin{lstlisting}[language={[Sharp]C}] @@ -171,12 +171,8 @@ ({\small\lstinline{methodid == MY_METHOD_ID}}), then the appropriate jump to the corresponding code is executed. Otherwise, the \lstinline{jump_to_ext} -part of the dispatcher has to be executed. -The code that actually jumps to an external block is contained in -the dispatcher of the primary method, whereas the -\lstinline{jump_to_ext} code of dispatchers of secondary methods -simply delegates the dispatcher of the primary method of the same -graph (see later). +part of the dispatcher has to be executed, which is implemented differently +by primary and secondary methods. The primary method is responsible for the bookkeeping of the secondary methods which are added to the same graph dynamically. This can be @@ -265,13 +261,11 @@ public int[] values = new int[4]; public FlexSwitchCase[] cases = new FlexSwitchCase[4]; - public void add_case(int value, FlexSwitchCase c) - { + public void add_case(int value, FlexSwitchCase c) { ... } - public uint execute(int value, InputArgs args) - { + public uint execute(int value, InputArgs args) { for(int i=0; i Author: antocuni Date: Sat Dec 20 18:33:21 2008 New Revision: 60650 Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex pypy/extradoc/talk/ecoop2009/main.tex Log: refactor tables Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Sat Dec 20 18:33:21 2008 @@ -87,60 +87,41 @@ \end{lstlisting} } -\anto{these tables are ugly} - -\cfbolz{can we make one figure of all the tables?} \begin{table}[ht] - \begin{tabular}{|l|r|r|r|r||r|r|} - \hline - \textbf{n} & - \textbf{Interp} & - \textbf{JIT} & - \textbf{JIT 2} & - \textbf{C\#} & - \textbf{Interp/JIT 2} & - \textbf{JIT 2/C\#} \\ - \hline + \begin{center} - $10$ & 0.031 & 0.422 & 0.000 & 0.000 & N/A & N/A \\ - $10^7$ & 30.984 & 0.453 & 0.047 & 0.031 & 661.000 & 1.500 \\ - $10^8$ & N/A & 0.859 & 0.453 & 0.359 & N/A & 1.261 \\ - $10^9$ & N/A & 4.844 & 4.641 & 3.438 & N/A & 1.350 \\ + \begin{tabular}{l|rrrrrr} + \multicolumn{5}{c}{\textbf{Factorial}} \\ [0.5ex] + \textbf{$n$} & $10$ & $10^7$ & $10^8$ & $10^9$ \\ \hline + \textbf{Interp} & 0.031 & 30.984 & N/A & N/A \\ + \textbf{JIT} & 0.422 & 0.453 & 0.859 & 4.844 \\ + \textbf{JIT 2} & 0.000 & 0.047 & 0.453 & 4.641 \\ + \textbf{C\#} & 0.000 & 0.031 & 0.359 & 3.438 \\ + \textbf{Interp/JIT 2} & N/A & \textbf{661.000} & N/A & N/A \\ + \textbf{JIT 2/C\#} & N/A & \textbf{1.500} & \textbf{1.261} & \textbf{1.350} \\ [3ex] - \end{tabular} - \caption{Factorial benchmark} - \label{tab:factorial} -\end{table} + \multicolumn{5}{c}{\textbf{Fibonacci}} \\ [0.5ex] -\begin{table}[ht] - \begin{tabular}{|l|r|r|r|r||r|r|} - \hline - \textbf{n} & - \textbf{Interp} & - \textbf{JIT} & - \textbf{JIT 2} & - \textbf{C\#} & - \textbf{Interp/JIT 2} & - \textbf{JIT 2/C\#} \\ + \textbf{$n$} & $10$ & $10^7$ & $10^8$ & $10^9$ \\ \hline - - $10$ & 0.031 & 0.453 & 0.000 & 0.000 & N/A & N/A \\ - $10^7$ & 29.359 & 0.469 & 0.016 & 0.016 & 1879.962 & 0.999 \\ - $10^8$ & N/A & 0.688 & 0.250 & 0.234 & N/A & 1.067 \\ - $10^9$ & N/A & 2.953 & 2.500 & 2.453 & N/A & 1.019 \\ - - \hline - + \textbf{Interp} & 0.031 & 29.359 & 0.000 & 0.000 \\ + \textbf{JIT} & 0.453 & 0.469 & 0.688 & 2.953 \\ + \textbf{JIT 2} & 0.000 & 0.016 & 0.250 & 2.500 \\ + \textbf{C\#} & 0.000 & 0.016 & 0.234 & 2.453 \\ + \textbf{Interp/JIT 2} & N/A & \textbf{1879.962}& N/A & N/A \\ + \textbf{JIT 2/C\#} & N/A & \textbf{0.999} & \textbf{1.067} & \textbf{1.019} \\ \end{tabular} - \caption{Fibonacci benchmark} - \label{tab:fibo} + + \end{center} + \caption{Factorial and Fibonacci benchmarks} + \label{tab:factorial-fibo} \end{table} -Tables \ref{tab:factorial} and \ref{tab:fibo} show the seconds spent to calculate +Table \ref{tab:factorial-fibo} shows the seconds spent to calculate the factorial and Fibonacci for various $n$. As we can see, for small values of $n$ the time spent running the JIT compiler is much higher than the time spent to simply interpret the program. This is an expected result, as till @@ -221,34 +202,34 @@ doing additions in-place. \begin{table}[ht] - \begin{tabular}{|l|r|r|r|r||r|r|} - \hline - \textbf{n} & - \textbf{Interp} & - \textbf{JIT} & - \textbf{JIT 2} & - \textbf{C\#} & - \textbf{Interp/JIT 2} & - \textbf{JIT 2/C\#} \\ - \hline + \begin{center} - $10$ & 0.031 & 0.453 & 0.000 & 0.000 & N/A & N/A \\ - $10^7$ & 43.063 & 0.516 & 0.047 & 0.063 & 918.765 & 0.750 \\ - $10^8$ & N/A & 0.875 & 0.453 & 0.563 & N/A & 0.806 \\ - $10^9$ & N/A & 4.188 & 3.672 & 5.953 & N/A & 0.617 \\ + \begin{tabular}{l|rrrrrr} + \multicolumn{5}{c}{\textbf{Accumulator}} \\ [0.5ex] + \textbf{$n$} & $10$ & $10^7$ & $10^8$ & $10^9$ \\ \hline + \textbf{Interp} & 0.031 & 43.063 & N/A & N/A \\ + \textbf{JIT} & 0.453 & 0.516 & 0.875 & 4.188 \\ + \textbf{JIT 2} & 0.000 & 0.047 & 0.453 & 3.672 \\ + \textbf{C\#} & 0.000 & 0.063 & 0.563 & 5.953 \\ + \textbf{Interp/JIT 2} & N/A & \textbf{918.765} & N/A & N/A \\ + \textbf{JIT 2/C\#} & N/A & \textbf{0.750} & \textbf{0.806} & \textbf{0.617} \\ \end{tabular} + \end{center} \caption{Accumulator benchmark} \label{tab:accumulator} \end{table} + + + Table \ref{tab:accumulator} show the results for the benchmark. Again, we can see that the speedup of the JIT over the interpreter is comparable to the other two benchmarks. However, the really interesting part is the comparison -with the equivalent C\# code, as the code generated by the JIT is +with the equivalent C\# code, as the code generated by the JIT is up to 1.62 times \textbf{faster}. Probably, the C\# code is slower because: Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Sat Dec 20 18:33:21 2008 @@ -15,6 +15,7 @@ \usepackage{xspace} \usepackage{listings} \usepackage{fancyvrb} +\usepackage{multirow} \usepackage[pdftex]{graphicx} %\input{macros} @@ -31,6 +32,8 @@ xleftmargin=20pt,xrightmargin=20pt, showstringspaces=false} +\setlength{\tabcolsep}{1ex} + %\renewcommand{\baselinestretch}{.98} \newboolean{showcomments} From antocuni at codespeak.net Sat Dec 20 18:44:17 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 18:44:17 +0100 (CET) Subject: [pypy-svn] r60651 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220174417.BC875168450@codespeak.net> Author: antocuni Date: Sat Dec 20 18:44:17 2008 New Revision: 60651 Modified: pypy/extradoc/talk/ecoop2009/intro.tex pypy/extradoc/talk/ecoop2009/main.bbl pypy/extradoc/talk/ecoop2009/main.bib pypy/extradoc/talk/ecoop2009/main.tex Log: add reference to dyla paper, add institute for armin Modified: pypy/extradoc/talk/ecoop2009/intro.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/intro.tex (original) +++ pypy/extradoc/talk/ecoop2009/intro.tex Sat Dec 20 18:44:17 2008 @@ -15,10 +15,7 @@ runtime; this approach seems to work, but writing it manually requires an enormous effort. -\cfbolz{we should cite the dyla paper somewhere here. also, we generally need -more references} - -PyPy's approach is to automatize the generation of JIT compilers in order +PyPy's approach \cite{RiBo07_223} is to automatize the generation of JIT compilers in order to reduce to a minimum the effort required to get a fast implementation of a dynamic language; all you have to do is to write a high level specification of the language (by writing an interpreter), and putting it through PyPy's Modified: pypy/extradoc/talk/ecoop2009/main.bbl ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.bbl (original) +++ pypy/extradoc/talk/ecoop2009/main.bbl Sat Dec 20 18:44:17 2008 @@ -99,6 +99,11 @@ prototype for python. \newblock In {\em PEPM}, pages 15--26, 2004. +\bibitem{RiBo07_223} +Armin Rigo and Carl~Friedrich Bolz. +\newblock {How to not write Virtual Machines for Dynamic Languages }. +\newblock In {\em Proceeding of Dyla 2007 (to appear)}, pages~--, 2007. + \bibitem{sullivan_dynamic_2001} Gregory~T. Sullivan. \newblock Dynamic partial evaluation. Modified: pypy/extradoc/talk/ecoop2009/main.bib ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.bib (original) +++ pypy/extradoc/talk/ecoop2009/main.bib Sat Dec 20 18:44:17 2008 @@ -181,3 +181,29 @@ pages = {1--19}, publisher = {ACM Press} } + + at inproceedings{RiBo07_223, + author = {Armin Rigo and Carl Friedrich Bolz}, + title = {{How to not write Virtual Machines for Dynamic Languages +}}, + booktitle = {Proceeding of Dyla 2007 (to appear)}, + abstract = {Typical modern dynamic languages have a growing number of + implementations. We explore the reasons for this situation, + and the limitations it imposes on open source or academic + communities that lack the resources to fine-tune and + maintain them all. It is sometimes proposed that + implementing dynamic languages on top of a standardized + general-purpose ob ject-oriented virtual machine (like Java + or .NET) would help reduce this burden. We propose a + complementary alternative to writing custom virtual machine + (VMs) by hand, validated by the PyPy pro ject: flexibly + generating VMs from a high-level specification, inserting + features and low-level details automatically ? including + good just-in-time compilers tuned to the dynamic language at + hand. We believe this to be ultimately a better investment + of efforts than the development of more and more advanced + general-purpose object oriented VMs. In this paper we + compare these two approaches in detail.}, +pages = {--}, + year = {2007}, +} \ No newline at end of file Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Sat Dec 20 18:44:17 2008 @@ -68,7 +68,7 @@ (PyPy: Implementing Python in Python).}} -\author{Davide Ancona\inst{1} \and Carl Friedrich Bolz\inst{2} \and Antonio Cuni\inst{1} \and Armin Rigo} +\author{Davide Ancona\inst{1} \and Carl Friedrich Bolz\inst{2} \and Antonio Cuni\inst{1} \and Armin Rigo\inst{2}} \institute{DISI, University of Genova, Italy \and From antocuni at codespeak.net Sat Dec 20 18:46:25 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 18:46:25 +0100 (CET) Subject: [pypy-svn] r60652 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220174625.3B709168450@codespeak.net> Author: antocuni Date: Sat Dec 20 18:46:24 2008 New Revision: 60652 Modified: pypy/extradoc/talk/ecoop2009/intro.tex Log: kill a picture, as it was only loosely referenced by the text Modified: pypy/extradoc/talk/ecoop2009/intro.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/intro.tex (original) +++ pypy/extradoc/talk/ecoop2009/intro.tex Sat Dec 20 18:46:24 2008 @@ -28,6 +28,7 @@ \subsection{PyPy and RPython} +\commentout{ \begin{figure}[h] \begin{center} \includegraphics[width=.6\textwidth]{diagram0} @@ -35,6 +36,7 @@ language $L$ for several platforms}\label{pypy-fig} \end{center} \end{figure} +} The \emph{PyPy} project\footnote{\texttt{http://codespeak.net/pypy/}} \cite{RigoPedroni06} was initially conceived to develop an implementation of Python which @@ -65,10 +67,13 @@ can in fact be used for implementing other languages. Indeed, there were successful experiments of using PyPy to implement several other languages such as Smalltalk \cite{BolzEtAl08}, JavaScript, Scheme and Prolog. + +\commentout{ As suggested by Figure~\ref{pypy-fig}, a portable interpreter for a generic language $L$ can be easily developed once an interpreter for $L$ has been implemented in RPython. +} \subsection{PyPy and JIT-Generation} From antocuni at codespeak.net Sat Dec 20 19:08:31 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 19:08:31 +0100 (CET) Subject: [pypy-svn] r60653 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220180831.BED781683E2@codespeak.net> Author: antocuni Date: Sat Dec 20 19:08:30 2008 New Revision: 60653 Modified: pypy/extradoc/talk/ecoop2009/conclusion.tex pypy/extradoc/talk/ecoop2009/main.bbl pypy/extradoc/talk/ecoop2009/main.bib Log: try to write the related work section in proper english Modified: pypy/extradoc/talk/ecoop2009/conclusion.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/conclusion.tex (original) +++ pypy/extradoc/talk/ecoop2009/conclusion.tex Sat Dec 20 19:08:30 2008 @@ -1,29 +1,29 @@ -\section{Related Works} +\section{Related Work} + Promotion is a concept that we have already explored in other contexts. Psyco is a run-time specialiser for Python that uses promotion (called ``unlift'' in \cite{DBLP:conf/pepm/Rigo04}). However, Psyco is a manually written JIT, is not applicable to other languages and cannot be retargetted. +Moreover, the idea of promotion of is a generalization of \emph{Polymorphic + Inline Caches} \cite{hoelzle_optimizing_1991}, as well as the idea of using +runtime feedback to produce more efficient code +\cite{hoelzle_type_feedback_1994}. + +PyPy-style JIT compilers are hard to write manually, thus we chose to write a +JIT generator. Tracing JIT compilers \cite{gal_hotpathvm_2006} also gives +good results but are much easier to write, making the need for an automatic +generator less urgent. However so far tracing JITs have less general +allocation removal techniques, which makes them get less speedup in a dynamic +language with boxing. Another difference is that tracing JITs concentrate on +loops, which makes them produce a lot less code. This issue will be addressed +by future research in PyPy. - - reference standard jit stuff, PICs \cite{hoelzle_optimizing_1991}, this one: http://www.cs.ucsb.edu/~urs/oocsb/self/papers/type-feedback.html - - - compare against tracing JITs \cite{gal_hotpathvm_2006}. tracing jits make - implementing jits much easier, maybe less need for JIT generation. also, they - concentrate on loops, which makes them produce a lot less code (which will need - to be addressed in our work as well). - - so far tracing JITs have less general allocation removal techniques, which - makes them get less speedup in a dynamic language with boxing - -in my master I had this snippet: - -""" -The generated code typically contains guards; in recent research +The code generated by tracing JITs code typically contains guards; in recent research \cite{gal_incremental_2006} on Java, these guards' behaviour is extended to be similar to our promotion. This has been used twice to implement a dynamic language (JavaScript), by Tamarin\footnote{{\tt http://www.mozilla.org/projects/tamarin/}} and in \cite{chang_efficient_2007}. -""" There has been an enormous amount of work on partial evaluation for compiler generation. A good introduction is given in \cite{Jones:peval}. However, most of @@ -49,9 +49,10 @@ % XXX there are no performance figures, we have no clue how much of this is % implemented. not sure how to write this - - apart from that we maybe should talk about escape analysis - \cite{Blanchet99escapeanalysis}, \cite{Choi99escapeanalysis} - our algorithm is totally simple-minded in comparison, but is useful in practise +Our algorithm to avoid allocation of unneeded intermediate objects fits into +the research area of escape analysis: in comparison to advanced techniques +\cite{Blanchet99escapeanalysis}, \cite{Choi99escapeanalysis} our algorithm is +totally simple-minded, but it is still useful in practise. \section{Conclusion} Modified: pypy/extradoc/talk/ecoop2009/main.bbl ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.bbl (original) +++ pypy/extradoc/talk/ecoop2009/main.bbl Sat Dec 20 19:08:30 2008 @@ -83,6 +83,13 @@ \newblock In {\em Proceedings of the European Conference on Object-Oriented Programming}, pages 21--38. Springer-Verlag, 1991. +\bibitem{hoelzle_type_feedback_1994} +Urs H\"{o}lzle and David Ungar. +\newblock Optimizing dynamically-dispatched calls with run-time type feedback. +\newblock In {\em PLDI '94: Proceedings of the ACM SIGPLAN 1994 conference on + Programming language design and implementation}, pages 326--336, New York, + NY, USA, 1994. ACM. + \bibitem{Jones:peval} Neil~D. Jones, Carsten~K. Gomard, and Peter Sestoft. \newblock {\em Partial Evaluation and Automatic Program Generation}. Modified: pypy/extradoc/talk/ecoop2009/main.bib ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.bib (original) +++ pypy/extradoc/talk/ecoop2009/main.bib Sat Dec 20 19:08:30 2008 @@ -206,4 +206,17 @@ compare these two approaches in detail.}, pages = {--}, year = {2007}, -} \ No newline at end of file +} + + at inproceedings{hoelzle_type_feedback_1994, + author = {Urs H\"{o}lzle and David Ungar}, + title = {Optimizing dynamically-dispatched calls with run-time type feedback}, + booktitle = {PLDI '94: Proceedings of the ACM SIGPLAN 1994 conference on Programming language design and implementation}, + year = {1994}, + isbn = {0-89791-662-X}, + pages = {326--336}, + location = {Orlando, Florida, United States}, + doi = {http://doi.acm.org/10.1145/178243.178478}, + publisher = {ACM}, + address = {New York, NY, USA}, + } From antocuni at codespeak.net Sat Dec 20 19:15:02 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 19:15:02 +0100 (CET) Subject: [pypy-svn] r60654 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220181502.49ABC168422@codespeak.net> Author: antocuni Date: Sat Dec 20 19:15:01 2008 New Revision: 60654 Modified: pypy/extradoc/talk/ecoop2009/conclusion.tex Log: add high-level structure of conclusions Modified: pypy/extradoc/talk/ecoop2009/conclusion.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/conclusion.tex (original) +++ pypy/extradoc/talk/ecoop2009/conclusion.tex Sat Dec 20 19:15:01 2008 @@ -56,5 +56,17 @@ \section{Conclusion} -XXX add me +high level structure: +1. We presented PyPy JIT generator + +2. interpreters can exploit the JIT generator by placing few hints here and +there to guide the generation of code + +3. generated JITs produce fast code thanks to \textbf{promotion} (our first +contribution) and automatic unboxing + +4. we presented a way to implement promotion/flexswitches on top of .NET + +5. our second contribution is to show that the idea of JIT layering can work +well, as for some cases we can get performances even better than C\# From antocuni at codespeak.net Sat Dec 20 19:25:32 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 19:25:32 +0100 (CET) Subject: [pypy-svn] r60655 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220182532.F4153168437@codespeak.net> Author: antocuni Date: Sat Dec 20 19:25:32 2008 New Revision: 60655 Modified: pypy/extradoc/talk/ecoop2009/intro.tex Log: expand a bit the idea of jit layering in the introduction Modified: pypy/extradoc/talk/ecoop2009/intro.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/intro.tex (original) +++ pypy/extradoc/talk/ecoop2009/intro.tex Sat Dec 20 19:25:32 2008 @@ -22,9 +22,16 @@ translation toolchain. The automatic generation of JIT compilers is done with the help of partial evaluation techniques. -The contributions of this paper are \emph{promotion}, a generalization of -polymorphic inline caches that make partial evaluation practical for dynamic -languages and trying out the idea of JIT-layering XXX. +Instead of emitting code for a real CPU, the generated JIT compilers described +in this paper emit code for a virtual machine: before being executed, the +emitted code will be compiled again by .NET's own JIT compiler, thus having +two \emph{layers} of JIT compilation. + +The contributions of this paper are twofold: (1) \emph{promotion} is a +generalization of polymorphic inline caches that make partial evaluation +practical for dynamic languages; (2) we demonstrated that the idea of +\emph{JIT layering} can give good results, as dynamic languages can be even +faster than their static counterparts. \subsection{PyPy and RPython} From antocuni at codespeak.net Sat Dec 20 19:31:23 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 19:31:23 +0100 (CET) Subject: [pypy-svn] r60656 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220183123.BDD8D168437@codespeak.net> Author: antocuni Date: Sat Dec 20 19:31:22 2008 New Revision: 60656 Modified: pypy/extradoc/talk/ecoop2009/intro.tex pypy/extradoc/talk/ecoop2009/jitgen.tex pypy/extradoc/talk/ecoop2009/rainbow.tex Log: fix few XXXs Modified: pypy/extradoc/talk/ecoop2009/intro.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/intro.tex (original) +++ pypy/extradoc/talk/ecoop2009/intro.tex Sat Dec 20 19:31:22 2008 @@ -122,15 +122,14 @@ care to choose a strategy for JIT compilation that lets the compiler to take the best of this advantage. -Most JIT compilers for dynamic languages around (such as IronPython, Jython, -JRuby \anto{XXX insert some reference}) compile code at the method -granularity. If on the one hand they can exploit some of the knowledge gathered -at runtime (e.g. the types of method parameters), on the other hand they can -do little to optimize most of the operations inside, because their behaviour -depends on informations that are not available at compile-time, because -e.g. the global state of the program can change at runtime. \anto{e.g., we can - add/remove methods to classes, etc. Should we insert some example here?} -\cfbolz{the last sentence is waaay too complicated} +Most JIT compilers for dynamic languages around (such as +IronPython \footnote{http://www.codeplex.com/IronPython}, +Jython \footnote{http://www.jython.org/} or +JRuby \footnote{http://jruby.codehaus.org/}) compile code at the method +granularity. If on the one hand they can exploit some of the knowledge +gathered at runtime (e.g. the types of method parameters), on the other hand +they can do little to optimize most of the operations inside, because few +assumptions can be made about the global state of the program. JIT compilers generated by PyPy solve this problem by delaying the compilation until they know all the informations needed to generate efficient code. If at Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Sat Dec 20 19:31:22 2008 @@ -138,17 +138,10 @@ function. The \emph{binding time} of a variable determines how early the value of the variable will be known. -XXX write something about the problems of classical PE? - \subsection{Binding Time Analysis in PyPy} -XXX need better transition text from last section At translation time, PyPy performs binding-time analysis of the source -RPython program after it has been turned to low-level graphs, i.e. at -the level at which operations manipulate objects that have a fixed type -attached. - -The binding-time terminology that we are using in PyPy is based on the +RPython program. The binding-time terminology that we are using in PyPy is based on the colors that we use when displaying the control flow graphs: \begin{itemize} @@ -162,9 +155,7 @@ abstract-interpretation based analysis. It is based on the same type inference engine that is used on the source RPython program. This is called the \emph{hint-annotator}; it -operates over input graphs that are already low-level instead of -RPython-level, \anto{we never make distinction between low-level and rpython-level -flowgraphs, do we? I propose to talk about ``intermediate flowgraphs''} +operates over input flowgraphs and propagates annotations that do not track types but value dependencies and manually-provided binding time hints. Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Sat Dec 20 19:31:22 2008 @@ -66,7 +66,7 @@ must be put in the source code of the interpreter. From a partial evaluation point of view, promotion is the converse of -the operation generally known as \emph{lift} \cite{XXX}. Lifting a value means +the operation generally known as \emph{lift}. Lifting a value means copying a variable whose binding time is compile-time into a variable whose binding time is run-time ? it corresponds to the compiler ``forgetting'' a particular value that it knew about. By contrast, From antocuni at codespeak.net Sat Dec 20 19:37:47 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 19:37:47 +0100 (CET) Subject: [pypy-svn] r60657 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220183747.0EBFB168403@codespeak.net> Author: antocuni Date: Sat Dec 20 19:37:46 2008 New Revision: 60657 Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex pypy/extradoc/talk/ecoop2009/clibackend.tex Log: fix another XXX Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Sat Dec 20 19:37:46 2008 @@ -1,9 +1,6 @@ \section{Benchmarks} \label{sec:benchmarks} -\anto{We should say somewhere that flexswitches are slow but benchmarks are so - good because they are not involved in the inner loops} - In section \ref{sec:tlc-properties}, we saw that TLC provides most of the features that usaully make dynamically typed language so slow, such as \emph{stack-based interpreter}, \emph{boxed arithmetic} and \emph{dynamic lookup} of @@ -39,21 +36,6 @@ multiplication, one subtraction, and one comparison to check if we have finished the job. -\commentout{ -\cfbolz{I think we can kill this for space reasons} -\anto{If we decide to keep them, we must remember to explain the python-like - syntax, as it is no longer in tlc.tex} - -\begin{lstlisting} -def main(n): - result = 1 - while n > 1: - result = result * n - n = n - 1 - return n -\end{lstlisting} -} - When doing plain interpretation, we need to create and destroy three temporary objects at each iteration. By contrast, the code generated by the JIT does much better. At the first iteration, the classes of the two operands of the @@ -72,21 +54,6 @@ Similarly, we wrote a program to calculate the $n_{th}$ Fibonacci number, for which we can do the same reasoning as above. -\commentout{ -\cfbolz{I think we can kill this for space reasons} -\begin{lstlisting} -def main(n): - a = 0 - b = 1 - while n > 1: - sum = a + b - a = b - b = sum - n = n - 1 - return b -\end{lstlisting} -} - \begin{table}[ht] \begin{center} @@ -143,6 +110,15 @@ CLI backend emits slightly non-optimal code and that the underyling .NET JIT compiler is highly optimized to handle bytecode generated by C\# compilers. +As we saw in Section~\ref{sec:flexswitches-cli}, the implementation of +flexswitches on top of CLI is hard and inefficient. However, our benchmarks +show that this inefficiency does not affect the overall performances of the +generated code. This is because in most programs the vast majority of the +time is spent in the inner loop: the graphs are built in such a way that all +the blocks that are part of the inner loop reside in the same method, so that +all links inside are internal (and fast). + + \subsection{Object-oriented features} To measure how the JIT handles object-oriented features, we wrote a very Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Sat Dec 20 19:37:46 2008 @@ -66,6 +66,7 @@ \subsection{Implementing flexswitches in CLI} +\label{sec:flexswitches-cli} Implementing flexswitches for backends generating machine code is not too complex: basically, a new jump has to be inserted in the From antocuni at codespeak.net Sat Dec 20 19:41:21 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 19:41:21 +0100 (CET) Subject: [pypy-svn] r60658 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220184121.E27B0168408@codespeak.net> Author: antocuni Date: Sat Dec 20 19:41:21 2008 New Revision: 60658 Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex Log: fix one of the last XXXs Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Sat Dec 20 19:41:21 2008 @@ -221,5 +221,8 @@ it prevents under-specialization: an unsatisfiable \texttt{hint(v1, concrete=True)} is reported as an error. -\anto{maybe we should at least mention promotion, and refer to the proper - section for details?} +There are cases in which having a green variable is essential for generating +good code, but it is not possible to use the \texttt{concrete} due to an +unsatisfiable dependency: Section~\ref{sec:promotion} introduces +\emph{promotion}, the novel technique that makes possible to solve this +problem. From antocuni at codespeak.net Sat Dec 20 20:58:57 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 20:58:57 +0100 (CET) Subject: [pypy-svn] r60659 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220195857.2CA5D168413@codespeak.net> Author: antocuni Date: Sat Dec 20 20:58:54 2008 New Revision: 60659 Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex pypy/extradoc/talk/ecoop2009/clibackend.tex pypy/extradoc/talk/ecoop2009/conclusion.tex pypy/extradoc/talk/ecoop2009/jitgen.tex pypy/extradoc/talk/ecoop2009/tlc.tex Log: some minore changes Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Sat Dec 20 20:58:54 2008 @@ -41,7 +41,7 @@ much better. At the first iteration, the classes of the two operands of the multiplication are promoted; then, the JIT compiler knows that both are integers, so it can inline the code to compute the result. Moreover, it can -\emph{virtualize} (see section \ref{sec:virtuals} all the temporary objects, because they never escape from +\emph{virtualize} (see Section \ref{sec:virtuals}) all the temporary objects, because they never escape from the inner loop. The same remarks apply to the other two operations inside the loop. @@ -167,7 +167,7 @@ code we wrote uses two classes and a \lstinline{virtual} method call to implement this behaviour. -However, our generated JIT does not compile the whole function at +As already discussed, our generated JIT does not compile the whole function at once. Instead, it compiles and executes code chunk by chunk, waiting until it knows enough informations to generate highly efficient code. In particualr, at the time it emits the code for the inner loop it exactly knows the Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Sat Dec 20 20:58:54 2008 @@ -6,7 +6,7 @@ From the implementation point of view, the JIT generator is divided into a frontend and several backends. The goal of the frontend is to generate a JIT compiler which works as described in the previous sections. Internally, the -frontend represents the compiled code as \emph{flow graphs}, and the role of +JIT represents the compiled code as \emph{flow graphs}, and the role of the backends is to translate flowgraphs into machine code. At the moment of writing, three backends have been implemented: one for Intel @@ -20,7 +20,7 @@ JIT-compilation, each layer removing away different kinds of overhead. By operating at a higher level, our JIT can potentially do a better job than the .NET one in some contexts, as our benchmarks demonstrate (see -section~\ref{sec:benchmarks}). On the other hand, the lower-level .NET JIT is +Section~\ref{sec:benchmarks}). On the other hand, the lower-level .NET JIT is very good at producing machine code, much more than PyPy's own \emph{x86} backend for example. By combining the strengths of both we can get highly efficient machine code. Modified: pypy/extradoc/talk/ecoop2009/conclusion.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/conclusion.tex (original) +++ pypy/extradoc/talk/ecoop2009/conclusion.tex Sat Dec 20 20:58:54 2008 @@ -54,6 +54,12 @@ \cite{Blanchet99escapeanalysis}, \cite{Choi99escapeanalysis} our algorithm is totally simple-minded, but it is still useful in practise. +\commentout{ +\section{Future work} + +XXX to be written +} + \section{Conclusion} high level structure: Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Sat Dec 20 20:58:54 2008 @@ -1,7 +1,7 @@ \section{Automatic generation of JIT compilers} Traditional JIT compilers are hard to write, time consuming, hard to evolve, -etc. etc. +etc. etc. \anto{we need a better introductive sentence} \commentout{ \begin{figure}[h] Modified: pypy/extradoc/talk/ecoop2009/tlc.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/tlc.tex (original) +++ pypy/extradoc/talk/ecoop2009/tlc.tex Sat Dec 20 20:58:54 2008 @@ -45,7 +45,7 @@ Obviously, not all the operations are applicable to all types. For example, it is not possible to \lstinline{ADD} an integer and an object, or reading an attribute from an object which does not provide it. Being dynamically typed, -the VM needs to do all these checks at runtime; in case one of the check +the interpreter needs to do all these checks at runtime; in case one of the check fails, the execution is simply aborted. \subsection{TLC properties} @@ -82,7 +82,8 @@ As we said above, TLC exists only at bytecode level; to ease the development of TLC programs, we wrote an assembler that generates TLC bytecode. Figure \ref{fig:tlc-abs} shows a simple program that computes the absolute value of -the given integer. +the given integer. In the subsequent sections, we will examine step-by-step +how the generated JIT compiler manages to produce a fully optimized version of it. \begin{figure}[h] \begin{center} From antocuni at codespeak.net Sat Dec 20 21:03:28 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 20 Dec 2008 21:03:28 +0100 (CET) Subject: [pypy-svn] r60660 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081220200328.B42D6168410@codespeak.net> Author: antocuni Date: Sat Dec 20 21:03:27 2008 New Revision: 60660 Modified: pypy/extradoc/talk/ecoop2009/tlc.tex Log: remove commented out code Modified: pypy/extradoc/talk/ecoop2009/tlc.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/tlc.tex (original) +++ pypy/extradoc/talk/ecoop2009/tlc.tex Sat Dec 20 21:03:27 2008 @@ -108,22 +108,3 @@ \label{fig:tlc-abs} \end{center} \end{figure} - -\commentout{ - -\anto{I think this example could cause confusion, because the synatx is TOO - similar to (R)Python, hence in the next section it could be unclear which is - the leves of the examples} - -Since reading TLC programs at bytecode level is hard, in this paper we will -use an invented Python-like syntax to describe examples, although we need to -remind that the actual programs are written in the assembler language showed -above. Thus, the example above can be written in the following way: - -\begin{lstlisting} -def main(n): - if n<0: - return -n - return n -\end{lstlisting} -} From antocuni at codespeak.net Sun Dec 21 10:27:31 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 21 Dec 2008 10:27:31 +0100 (CET) Subject: [pypy-svn] r60665 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081221092731.3F26916844F@codespeak.net> Author: antocuni Date: Sun Dec 21 10:27:28 2008 New Revision: 60665 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex pypy/extradoc/talk/ecoop2009/conclusion.tex pypy/extradoc/talk/ecoop2009/main.bbl pypy/extradoc/talk/ecoop2009/main.bib Log: add conclusion and future work Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Sun Dec 21 10:27:28 2008 @@ -3,6 +3,8 @@ \subsection{JIT layering} +%\anto{Mention Reflection.Emit} + From the implementation point of view, the JIT generator is divided into a frontend and several backends. The goal of the frontend is to generate a JIT compiler which works as described in the previous sections. Internally, the Modified: pypy/extradoc/talk/ecoop2009/conclusion.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/conclusion.tex (original) +++ pypy/extradoc/talk/ecoop2009/conclusion.tex Sun Dec 21 10:27:28 2008 @@ -54,25 +54,41 @@ \cite{Blanchet99escapeanalysis}, \cite{Choi99escapeanalysis} our algorithm is totally simple-minded, but it is still useful in practise. -\commentout{ -\section{Future work} +\section{Conclusion and Future Work} -XXX to be written -} +In this paper we presented PyPy's JIT compiler generator, based on partial +evaluation techniques, which can automatically turn an interpreter into a JIT +compiler, requiring the language developers to only add few \texttt{hint}s to +guide the generation process. + +We showed that classical partial evaluation cannot remove all the overhead +proper of dynamically typed languages, and how the new operation called +\emph{promotion} solves the problem, by delaying compile-time until the JIT +knows enough to produce efficient code, and by continuously intermixing +compile-time and runtime. Moreover, we showed that our simple but still +practically useful technique to avoid allocation of intermediate unnecessary +objects plays well with promotion and helps to produce even better code. + +Finally, we presented the CLI backend for PyPy's JIT compiler generator, whose +goal is to produce .NET bytecode at runtime. We showed how it is possible to +circumvent intrinsic limitations of the virtual machine to implement +promotion. As a result, we proved that the idea of \emph{JIT layering} is +worth of further exploration, as it makes possible for dynamically typed +languages to be even faster than their statically typed counterpart in some +circumstances. + +As a future work, we want to explore different strategies to make the frontend +producing less code, maintaining comparable or better performances. In +particular, we are working on a way to automatically detect loops in the user +code, as tracing JITs do \cite{gal_hotpathvm_2006}. By compilining whole +loops at once, the backends should be able to produce better code than today. + +At the moment, some bugs and minor missing features prevent the CLI JIT +backend to handle more complex languages such as Python and Smalltalk. We are +confident that once these problems will be fixed, we will get performance +results comparable to TLC, as the other backends already demonstrate +\cite{PyPyJIT}. Moreover, if the current implementation of flexswitches will +prove to be too slow for some purposes, we want to explore alternative +implementation strategies, also considering the new features that might be +integrated into virtual machines. -\section{Conclusion} - -high level structure: - -1. We presented PyPy JIT generator - -2. interpreters can exploit the JIT generator by placing few hints here and -there to guide the generation of code - -3. generated JITs produce fast code thanks to \textbf{promotion} (our first -contribution) and automatic unboxing - -4. we presented a way to implement promotion/flexswitches on top of .NET - -5. our second contribution is to show that the idea of JIT layering can work -well, as for some cases we can get performances even better than C\# Modified: pypy/extradoc/talk/ecoop2009/main.bbl ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.bbl (original) +++ pypy/extradoc/talk/ecoop2009/main.bbl Sun Dec 21 10:27:28 2008 @@ -111,6 +111,12 @@ \newblock {How to not write Virtual Machines for Dynamic Languages }. \newblock In {\em Proceeding of Dyla 2007 (to appear)}, pages~--, 2007. +\bibitem{PyPyJIT} +Armin Rigo and Samuele Pedroni. +\newblock {JIT} compiler architecture. +\newblock Technical Report D08.2, PyPy Consortium, 2007. +\newblock http://codespeak.net/pypy/dist/pypy/doc/index-report.html. + \bibitem{sullivan_dynamic_2001} Gregory~T. Sullivan. \newblock Dynamic partial evaluation. Modified: pypy/extradoc/talk/ecoop2009/main.bib ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.bib (original) +++ pypy/extradoc/talk/ecoop2009/main.bib Sun Dec 21 10:27:28 2008 @@ -220,3 +220,12 @@ publisher = {ACM}, address = {New York, NY, USA}, } + + at techreport{PyPyJIT, + title = "{JIT} Compiler Architecture", + author = {Armin Rigo and Samuele Pedroni}, + year = "2007", + institution = "PyPy Consortium", + number = "D08.2", + note = "http://codespeak.net/pypy/dist/pypy/doc/index-report.html" +} From antocuni at codespeak.net Sun Dec 21 10:33:17 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 21 Dec 2008 10:33:17 +0100 (CET) Subject: [pypy-svn] r60666 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081221093317.99D39168450@codespeak.net> Author: antocuni Date: Sun Dec 21 10:33:17 2008 New Revision: 60666 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex Log: add a footnote Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Sun Dec 21 10:33:17 2008 @@ -3,8 +3,6 @@ \subsection{JIT layering} -%\anto{Mention Reflection.Emit} - From the implementation point of view, the JIT generator is divided into a frontend and several backends. The goal of the frontend is to generate a JIT compiler which works as described in the previous sections. Internally, the @@ -14,7 +12,9 @@ At the moment of writing, three backends have been implemented: one for Intel \emph{x86} processors, one for \emph{PowerPC} processors, and one for the \emph{CLI Virtual Machine}. The latter is special because instead of emitting -code for a real CPU it emits code for a virtual machine: before being +code for a real CPU it emits code for a virtual machine \footnote{By using the +\lstinline{Reflection.Emit} namespace and creating \lstinline{DynamicMethod}s.}: +before being executed, the generated code will be compiled again by .NET's own JIT compiler. From antocuni at codespeak.net Sun Dec 21 10:50:21 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 21 Dec 2008 10:50:21 +0100 (CET) Subject: [pypy-svn] r60667 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081221095021.E9D70168450@codespeak.net> Author: antocuni Date: Sun Dec 21 10:50:21 2008 New Revision: 60667 Modified: pypy/extradoc/talk/ecoop2009/intro.tex pypy/extradoc/talk/ecoop2009/jitgen.tex Log: fix xxx Modified: pypy/extradoc/talk/ecoop2009/intro.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/intro.tex (original) +++ pypy/extradoc/talk/ecoop2009/intro.tex Sun Dec 21 10:50:21 2008 @@ -123,9 +123,9 @@ the best of this advantage. Most JIT compilers for dynamic languages around (such as -IronPython \footnote{http://www.codeplex.com/IronPython}, -Jython \footnote{http://www.jython.org/} or -JRuby \footnote{http://jruby.codehaus.org/}) compile code at the method +IronPython\footnote{http://www.codeplex.com/IronPython}, +Jython\footnote{http://www.jython.org/} or +JRuby\footnote{http://jruby.codehaus.org/}) compile code at the method granularity. If on the one hand they can exploit some of the knowledge gathered at runtime (e.g. the types of method parameters), on the other hand they can do little to optimize most of the operations inside, because few Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Sun Dec 21 10:50:21 2008 @@ -1,7 +1,9 @@ \section{Automatic generation of JIT compilers} -Traditional JIT compilers are hard to write, time consuming, hard to evolve, -etc. etc. \anto{we need a better introductive sentence} +Traditional JIT compilers are hard to write, time consuming and hard to +evolve. On the other hand, interpreters are easy both to write and maintain, +but they are usually slow. By automatically generating a JIT compiler out of +an interpreter, we take the best of both worlds. \commentout{ \begin{figure}[h] From antocuni at codespeak.net Sun Dec 21 11:08:35 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 21 Dec 2008 11:08:35 +0100 (CET) Subject: [pypy-svn] r60668 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081221100835.4FE08168460@codespeak.net> Author: antocuni Date: Sun Dec 21 11:08:34 2008 New Revision: 60668 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex Log: remove space before of footnote Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Sun Dec 21 11:08:34 2008 @@ -12,7 +12,7 @@ At the moment of writing, three backends have been implemented: one for Intel \emph{x86} processors, one for \emph{PowerPC} processors, and one for the \emph{CLI Virtual Machine}. The latter is special because instead of emitting -code for a real CPU it emits code for a virtual machine \footnote{By using the +code for a real CPU it emits code for a virtual machine\footnote{By using the \lstinline{Reflection.Emit} namespace and creating \lstinline{DynamicMethod}s.}: before being executed, the generated code will be compiled again by .NET's own JIT From antocuni at codespeak.net Sun Dec 21 11:09:30 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 21 Dec 2008 11:09:30 +0100 (CET) Subject: [pypy-svn] r60669 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081221100930.45625168460@codespeak.net> Author: antocuni Date: Sun Dec 21 11:09:29 2008 New Revision: 60669 Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex Log: typo Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Sun Dec 21 11:09:29 2008 @@ -31,7 +31,7 @@ time could run slower with JIT than without, due to the time spent doing the initial (double) compilation. It is important to underline that so far we mostly focused making the JIT able to produce very fast code, without trying -to optimzed the compilation phase itself. +to optimze the compilation phase itself. \subsection{Flexswitches} From antocuni at codespeak.net Sun Dec 21 11:15:21 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 21 Dec 2008 11:15:21 +0100 (CET) Subject: [pypy-svn] r60670 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081221101521.E347D168467@codespeak.net> Author: antocuni Date: Sun Dec 21 11:15:21 2008 New Revision: 60670 Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex Log: typo Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Sun Dec 21 11:15:21 2008 @@ -2,7 +2,7 @@ \label{sec:benchmarks} In section \ref{sec:tlc-properties}, we saw that TLC provides most of the -features that usaully make dynamically typed language so slow, such as +features that usually make dynamically typed language so slow, such as \emph{stack-based interpreter}, \emph{boxed arithmetic} and \emph{dynamic lookup} of methods and attributes. From davide at codespeak.net Sun Dec 21 11:23:20 2008 From: davide at codespeak.net (davide at codespeak.net) Date: Sun, 21 Dec 2008 11:23:20 +0100 (CET) Subject: [pypy-svn] r60671 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081221102320.BAEB5168476@codespeak.net> Author: davide Date: Sun Dec 21 11:23:20 2008 New Revision: 60671 Modified: pypy/extradoc/talk/ecoop2009/abstract.tex Log: we have a first draft of abstract Modified: pypy/extradoc/talk/ecoop2009/abstract.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/abstract.tex (original) +++ pypy/extradoc/talk/ecoop2009/abstract.tex Sun Dec 21 11:23:20 2008 @@ -1,3 +1,20 @@ \begin{abstract} -Dynamic languages are increasingly popular etc. etc. +Writing an optimizing static compiler for dynamic languages is not an +easy task, since quite complex static analysis is required. +On the other hand, recent developments show that JIT compilers +can exploit runtime type information to generate quite efficient code. +Unfortunately, writing a JIT compiler is far from being simple. + +In this paper we report our positive experience with automatic generation +of JIT compilers as supported by the PyPy infrastructure, by +focusing on JIT compilation for .NET. +The paper presents two main and novel contributions: we show that +partial evaluation can be used in practice for generating a JIT compiler, +and we experiment with the potentiality offered by the ability to +add a further level of JIT compilation on top of .NET. + +The practicality of the approach is demonstrated by showing some +promising experiments done with benchmarks written in a simple dynamic language. \end{abstract} + +% LocalWords: JIT PyPy From antocuni at codespeak.net Sun Dec 21 11:25:03 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 21 Dec 2008 11:25:03 +0100 (CET) Subject: [pypy-svn] r60672 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081221102503.5400C168476@codespeak.net> Author: antocuni Date: Sun Dec 21 11:25:02 2008 New Revision: 60672 Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex Log: minor modifications Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Sun Dec 21 11:25:02 2008 @@ -6,7 +6,7 @@ \emph{stack-based interpreter}, \emph{boxed arithmetic} and \emph{dynamic lookup} of methods and attributes. -In the following sections, we will show some benchmarks that show how our +In the following sections, we present some benchmarks that show how our generated JIT can handle all these features very well. To measure the speedup we get with the JIT, we run each program three times: @@ -20,7 +20,7 @@ \end{enumerate} Moreover, for each benchmark we also show the time taken by running the -equivalent program written in C\# \footnote{The sources for both TLC and C\# +equivalent program written in C\#\footnote{The sources for both TLC and C\# programs are available at: http://codespeak.net/svn/pypy/extradoc/talk/ecoop2009/benchmarks/} @@ -102,7 +102,7 @@ than $10^7$, we did not run the interpreted program as it would have took too much time, without adding anything to the discussion. -As we can see, the code generated by the JIT can be up to ~1800 times faster +As we can see, the code generated by the JIT can be up to about 1800 times faster than the non-jitted case. Moreover, it often runs at the same speed as the equivalent program written in C\#, being only 1.5 slower in the worst case. @@ -174,7 +174,7 @@ type of \lstinline{obj}, thus it can remove the overhead of dynamic dispatch and inline the method call. Moreover, since \lstinline{obj} never escapes the function, it is \emph{virtualized} and its field \lstinline{value} is stored -as a local variable. As a result, the generated code results in a simple loop +as a local variable. As a result, the generated code turns out to be a simple loop doing additions in-place. \begin{table}[ht] @@ -217,11 +217,8 @@ \cite{hoelzle_optimizing_1991}, that requires a guard check at each iteration. \end{itemize} -\anto{maybe we should move the following paragraph to - abstract/introduction/conclusion?} - Despite being only a microbenchmark, this result is very important as it proves that our strategy of intermixing compile time and runtime can yield to better performances than current techniques. The result is even more impressive if -we consider dynamically typed languages as TLC are usually considered much +we take in account that dynamically typed languages as TLC are usually considered much slower than the statically typed ones. From antocuni at codespeak.net Sun Dec 21 11:29:43 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 21 Dec 2008 11:29:43 +0100 (CET) Subject: [pypy-svn] r60673 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081221102943.38729168476@codespeak.net> Author: antocuni Date: Sun Dec 21 11:29:43 2008 New Revision: 60673 Modified: pypy/extradoc/talk/ecoop2009/abstract.tex Log: change a bit the abstract Modified: pypy/extradoc/talk/ecoop2009/abstract.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/abstract.tex (original) +++ pypy/extradoc/talk/ecoop2009/abstract.tex Sun Dec 21 11:29:43 2008 @@ -1,7 +1,6 @@ \begin{abstract} -Writing an optimizing static compiler for dynamic languages is not an -easy task, since quite complex static analysis is required. -On the other hand, recent developments show that JIT compilers +Implementing dynamically typed languages in an efficient way is a hard task. +Recent developments show that JIT compilers can exploit runtime type information to generate quite efficient code. Unfortunately, writing a JIT compiler is far from being simple. From antocuni at codespeak.net Sun Dec 21 11:33:43 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 21 Dec 2008 11:33:43 +0100 (CET) Subject: [pypy-svn] r60674 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081221103343.018BA168476@codespeak.net> Author: antocuni Date: Sun Dec 21 11:33:43 2008 New Revision: 60674 Modified: pypy/extradoc/talk/ecoop2009/abstract.tex Log: revert previous revision Modified: pypy/extradoc/talk/ecoop2009/abstract.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/abstract.tex (original) +++ pypy/extradoc/talk/ecoop2009/abstract.tex Sun Dec 21 11:33:43 2008 @@ -1,6 +1,7 @@ \begin{abstract} -Implementing dynamically typed languages in an efficient way is a hard task. -Recent developments show that JIT compilers +Writing an optimizing static compiler for dynamic languages is not an +easy task, since quite complex static analysis is required. +On the other hand, recent developments show that JIT compilers can exploit runtime type information to generate quite efficient code. Unfortunately, writing a JIT compiler is far from being simple. From antocuni at codespeak.net Sun Dec 21 11:36:13 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 21 Dec 2008 11:36:13 +0100 (CET) Subject: [pypy-svn] r60675 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081221103613.3BC16168476@codespeak.net> Author: antocuni Date: Sun Dec 21 11:36:12 2008 New Revision: 60675 Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex Log: kill bits about online/offline PE Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Sun Dec 21 11:36:12 2008 @@ -100,51 +100,15 @@ language: The partial evaluator (just like an ahead-of-time compiler) cannot make assumptions about the types of objects, which leads to poor results. Effective dynamic compilation requires feedback of runtime information into -compile-time. This is no different for partial evaluation. +compile-time. This is no different for partial evaluation, and subsequent sections shows how we managed to solve this problem. -Partial evaluation (PE) comes in two flavors: \cfbolz{if we have space problems -we should kill the bits about online and offline PE} - -\begin{itemize} -\item \emph{On-line} partial evaluation: a compiler-like algorithm takes the -source code of the function \texttt{f(x, y)} (or its intermediate representation, -i.e. its control flow graph), and some partial -information, e.g. \texttt{x=5}. From this, it produces the residual function -\texttt{g(y)} directly, by following in which operations the knowledge \texttt{x=5} can -be used, which loops can be unrolled, etc. - -\item \emph{Off-line} partial evalution: in many cases, the goal of partial -evaluation is to improve performance in a specific application. Assume that we -have a single known function \texttt{f(x, y)} in which we think that the value -of \texttt{x} will change slowly during the execution of our program ? much -more slowly than the value of \texttt{y}. An obvious example is a loop that -calls \texttt{f(x, y)} many times with always the same value \texttt{x}. We -could then use an on-line partial evaluator to produce a \texttt{g(y)} for each -new value of \texttt{x}. In practice, the overhead of the partial evaluator -might be too large for it to be executed at run-time. However, if we know the -function \texttt{f} in advance, and if we know \emph{which} arguments are the -ones that we will want to partially evaluate \texttt{f} with, then we do not -need a full compiler-like analysis of \texttt{f} every time the value of -\texttt{x} changes. We can precompute once and for all a specialized function -\texttt{f1(x)}, which when called produces the residual function \texttt{g(y)} -corresponding to \texttt{x}. This is \emph{off-line partial evaluation}; the -specialized function \texttt{f1(x)} is called a \emph{generating extension}. -\end{itemize} - -Off-line partial evaluation is usually based on \emph{binding-time analysis}, which -is the process of determining among the variables used in a function (or -a set of functions) which ones are going to be known in advance and -which ones are not. In the example of \texttt{f(x, y)}, such an analysis -would be able to infer that the constantness of the argument \texttt{x} -implies the constantness of many intermediate values used in the -function. The \emph{binding time} of a variable determines how early the -value of the variable will be known. \subsection{Binding Time Analysis in PyPy} -At translation time, PyPy performs binding-time analysis of the source -RPython program. The binding-time terminology that we are using in PyPy is based on the -colors that we use when displaying the control flow graphs: +At translation time, PyPy performs binding-time analysis of the source RPython +program, to determine which variables are static and which dynamic. The +binding-time terminology that we are using in PyPy is based on the colors that +we use when displaying the control flow graphs: \begin{itemize} \item \emph{Green} variables contain values that are known at compile-time. From antocuni at codespeak.net Sun Dec 21 11:44:33 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 21 Dec 2008 11:44:33 +0100 (CET) Subject: [pypy-svn] r60676 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081221104433.5CB37168476@codespeak.net> Author: antocuni Date: Sun Dec 21 11:44:33 2008 New Revision: 60676 Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex Log: remove reference to year, as the cited paper is more recent Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Sun Dec 21 11:44:33 2008 @@ -22,7 +22,7 @@ \subsection{Partial evaluation} -In 1971 Yoshihiko Futamura published a paper \cite{Futamura99} that proposed a +Yoshihiko Futamura published a paper \cite{Futamura99} that proposed a technique to automatically transform an interpreter of a programming language into a compiler for the same language. This would solve the problem of having to write a compiler instead of a much simpler interpreter. He proposed to use From davide at codespeak.net Sun Dec 21 11:45:59 2008 From: davide at codespeak.net (davide at codespeak.net) Date: Sun, 21 Dec 2008 11:45:59 +0100 (CET) Subject: [pypy-svn] r60677 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081221104559.30B4516847D@codespeak.net> Author: davide Date: Sun Dec 21 11:45:58 2008 New Revision: 60677 Modified: pypy/extradoc/talk/ecoop2009/intro.tex pypy/extradoc/talk/ecoop2009/main.tex Log: first submission Modified: pypy/extradoc/talk/ecoop2009/intro.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/intro.tex (original) +++ pypy/extradoc/talk/ecoop2009/intro.tex Sun Dec 21 11:45:58 2008 @@ -1,7 +1,4 @@ \section{Introduction} - -Dynamic languages are increasingly popular etc. etc. - The easiest way to implement a dynamic language such as Python is to write an interpreter; however, interpreters are slow. @@ -16,16 +13,16 @@ enormous effort. PyPy's approach \cite{RiBo07_223} is to automatize the generation of JIT compilers in order -to reduce to a minimum the effort required to get a fast implementation of a -dynamic language; all you have to do is to write a high level specification of -the language (by writing an interpreter), and putting it through PyPy's -translation toolchain. The automatic generation of JIT compilers is done with -the help of partial evaluation techniques. - -Instead of emitting code for a real CPU, the generated JIT compilers described -in this paper emit code for a virtual machine: before being executed, the -emitted code will be compiled again by .NET's own JIT compiler, thus having -two \emph{layers} of JIT compilation. +to minimize the effort required to get a fast implementation of a +dynamic language; automatic generation of JIT compilers is done with +the help of partial evaluation techniques and requires the user only +to provide an interpreter with some manual annotations which hint +the generator how interpretation and JIT compilation has to be interleaved. + +More precisely, in this paper we focus on the ability of generating a JIT compiler able to emit code +for the .NET virtual machine. To our knowledge, this is the first experiment with an interpreter with +two \emph{layers} of JIT compilation, since, before being executed, the +emitted code is eventually compiled again by .NET's own JIT compiler. The contributions of this paper are twofold: (1) \emph{promotion} is a generalization of polymorphic inline caches that make partial evaluation Modified: pypy/extradoc/talk/ecoop2009/main.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/main.tex (original) +++ pypy/extradoc/talk/ecoop2009/main.tex Sun Dec 21 11:45:58 2008 @@ -5,7 +5,7 @@ %\textwidth=130mm % LNCS: 122mm %\textheight=203mm % LNCS: 193mm -%\renewcommand{\baselinestretch}{0.95} +\renewcommand{\baselinestretch}{0.97} \usepackage{amssymb} \usepackage{amsmath} @@ -37,7 +37,7 @@ %\renewcommand{\baselinestretch}{.98} \newboolean{showcomments} -\setboolean{showcomments}{true} +\setboolean{showcomments}{false} \ifthenelse{\boolean{showcomments}} {\newcommand{\nb}[2]{ \fbox{\bfseries\sffamily\scriptsize#1} From davide at codespeak.net Sun Dec 21 12:10:24 2008 From: davide at codespeak.net (davide at codespeak.net) Date: Sun, 21 Dec 2008 12:10:24 +0100 (CET) Subject: [pypy-svn] r60678 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081221111024.81D6016848F@codespeak.net> Author: davide Date: Sun Dec 21 12:10:22 2008 New Revision: 60678 Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex Log: changed sentence which Antonio does not like Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Sun Dec 21 12:10:22 2008 @@ -91,15 +91,14 @@ Table \ref{tab:factorial-fibo} shows the seconds spent to calculate the factorial and Fibonacci for various $n$. As we can see, for small values of $n$ the time spent running the JIT compiler is much higher than the time -spent to simply interpret the program. This is an expected result, as till -now we only focused on optimizing the compiled code, not the compilation -process itself. - -On the other hand, to get meaningful timings we had to use very high values of -$n$. This means that the results are incorrect due to overflow, but since all -the runnings overflow in the very same way, the timings are still -comparable. \anto{I think we should rephrase this sentence}. For $n$ greater -than $10^7$, we did not run the interpreted program as it would have took too +spent to simply interpret the program. This is an expected result +which, however, can be improved once we will have time +to optimize compilation and not only the generated code. + +On the other, for reasonably high values of $n$ we obtain very good +results, which are valid despite the obvious overflow, since the +same operations are performed for all experiments. +For $n$ greater than $10^7$, we did not run the interpreted program as it would have took too much time, without adding anything to the discussion. As we can see, the code generated by the JIT can be up to about 1800 times faster From magcius at codespeak.net Sun Dec 21 17:19:50 2008 From: magcius at codespeak.net (magcius at codespeak.net) Date: Sun, 21 Dec 2008 17:19:50 +0100 (CET) Subject: [pypy-svn] r60679 - in pypy/branch/avm/pypy/translator/avm: . test Message-ID: <20081221161950.75860168455@codespeak.net> Author: magcius Date: Sun Dec 21 17:19:48 2008 New Revision: 60679 Added: pypy/branch/avm/pypy/translator/avm/test/test_bitstream.py Modified: pypy/branch/avm/pypy/translator/avm/function.py pypy/branch/avm/pypy/translator/avm/test/runtest.py pypy/branch/avm/pypy/translator/avm/util.py Log: Aaaaand my first test for the BitStream operations. Modified: pypy/branch/avm/pypy/translator/avm/function.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm/function.py (original) +++ pypy/branch/avm/pypy/translator/avm/function.py Sun Dec 21 17:19:48 2008 @@ -77,7 +77,7 @@ #self.ilasm.set_field(self.field_name(obj,name)) def get_field(self, useless_stuff, name): - self.ilasm.get_field(name) + self.ilasm.get_member(name) def call_method(self, obj, name): func_name, signature = self.cts.method_signature(obj, name) @@ -107,8 +107,8 @@ def cast_function(self, name, num): self.ilasm.cast_function(name, num) - def prefix_op(self, st): - self.ilasm.prefix_op(st) +# def prefix_op(self, st): +# self.ilasm.prefix_op(st) def push_const(self, s): self.ilasm.push_const(s) Modified: pypy/branch/avm/pypy/translator/avm/test/runtest.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm/test/runtest.py (original) +++ pypy/branch/avm/pypy/translator/avm/test/runtest.py Sun Dec 21 17:19:48 2008 @@ -16,6 +16,7 @@ log = log.runtest class AVM1Exception(object): + pass class compile_function(object): def __init__(self, function, annotations, stackless=False, view=False, root=None, policy=None): Added: pypy/branch/avm/pypy/translator/avm/test/test_bitstream.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm/test/test_bitstream.py Sun Dec 21 17:19:48 2008 @@ -0,0 +1,67 @@ + + +from pypy.translator.avm.util import BitStream, ALIGN_RIGHT, ALIGN_LEFT + +def test_write_bit(): + bits = BitStream() + bits.write_bit(True) + bits.write_bit(0) + bits.write_bit(1) + assert str(bits) == "101" + +def test_read_bit(): + bits = BitStream("101") + assert bits.read_bit() == True + assert bits.read_bit() == False + assert bits.read_bit() == True + +def test_write_bits(): + bits = BitStream() + bits.write_bits([1,1,0,1], 0, 3) # Length of 3! Stop after 3 bits, 110 + bits.write_bits([1,1,0,1], 2, 2) # Write the last two bits, 01 + assert str(bits) == "11001" + +def test_read_bits(): + bits = BitStream("01101") + assert str(bits.read_bits(4)) + +def test_write_int_value(): + bits = BitStream() + bits.write_int_value(51, 6) + assert str(bits) == "110011" + +def test_read_int_value(): + bits = BitStream("101010") + assert bits.read_int_value(6) == 42 + +def test_write_fixed_value(): + bits = BitStream() + bits.write_fixed_value(1.5) + assert str(bits) == "11000000000000000" + bits.rewind() + bits.write_fixed_value(6.2, 20) + assert str(bits) == "01100011001100110011" + +def test_read_fixed_value(): + bits = BitStream("01100011001100110011") # 6.2 + val = bits.read_fixed_value(20) + assert abs(6.2 - val) < 0.1 + +def test_write_float_value(): + bits = BitStream() + bits.write_float_value(0.15625, 32) + assert str(bits) == "00111110001000000000000000000000" + # 00111110001000000000000000000000 = 0.15625 + +def test_read_float_value(): + bits = BitStream("00111110001000000000000000000000") + val = bits.read_float_value(32) + assert val == 0.15625 + +def test_serialize(): + bits = BitStream("101010") + assert bits.serialize(ALIGN_RIGHT) == "\x2A" # 00101010 = 42 = 0x2A + assert bits.serialize(ALIGN_LEFT) == "\xA8" # 10101000 = 168 = 0xA8 + bits = BitStream("1100110011") + assert bits.serialize(ALIGN_RIGHT) == "\x03\x33" # 1100110011 = 00000011 00110011 = 03 51 = 03 33 + assert bits.serialize(ALIGN_LEFT) == "\xCC\xC0" # 1100110011 = 11001100 11000000 = 204 192 = CC C0 Modified: pypy/branch/avm/pypy/translator/avm/util.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm/util.py (original) +++ pypy/branch/avm/pypy/translator/avm/util.py Sun Dec 21 17:19:48 2008 @@ -1,5 +1,5 @@ -import struct, os +import struct, os, math ALIGN_LEFT = "left" ALIGN_RIGHT = "right" @@ -12,7 +12,7 @@ """ Constructor. """ - self.bits = bits + self.bits = [bool(b) and b != "0" for b in bits] self.cursor = 0 def read_bit(self): @@ -36,24 +36,27 @@ def write_bits(self, bits, offset=0, length=0): """Writes length bits from bits to this bit stream, starting reading at offset. If length is 0, the entire stream is used.""" - if length < 1: length = len(bits) - self.bits[self.cursor:self.cursor+length] = bits[offset:offset+length] + if length < 1: + length = len(bits) + + if length > self.bits_available(): + for i in range(length - self.bits_available()): + self.bits.append(False) + + self.bits[self.cursor:self.cursor+length] = (bool(x) for x in bits[offset:offset+length]) self.cursor += length - def read_bit_value(self, length): + def read_int_value(self, length): """Read length bits and return a number containing those bits with the last bit read being the least significant bit.""" n = 0 - for i in xrange(length-1, -1, -1): + for i in reversed(xrange(length)): n |= self.read_bit() << i return n - def write_bit_value(self, value, length=-1): + def write_int_value(self, value, length=-1): """Writes an int to the specified number of bits in the stream, the most significant bit first. If length is not specified or negative, the log base 2 of value is taken.""" - if int(value) != value: - self.write_fixed_value(value, length) - return if length < 0: import math @@ -61,8 +64,8 @@ length = int(math.ceil(math.log(value, 2))) # Get the log base 2, or number of bits value will fit in. except ValueError: length = 1 - - for i in xrange(length-1, -1, -1): + + for i in reversed(xrange(length)): self.write_bit(value & (1 << i)) def read_fixed_value(self, length, eight_bit_mantissa=False): @@ -73,29 +76,31 @@ raise ValueError, "Length must be greater than or equal to %(m)s, as %(m)s.%(m)s FB requires at \ least %(m)s bits to store." % {"m": min_length} - return self.read_bit_value(length) / 0x100 if eight_bit_mantissa else 0x10000 + return self.read_int_value(length) / (float(0x100) if eight_bit_mantissa else float(0x10000)) def write_fixed_value(self, value, length=-1, eight_bit_mantissa=False): """Writes a fixed point number of length, whole part first. If eight_bit_mantissa is True, an 8.8 format is used instead of a 16.16 format. If length is negative, it will be calculated for.""" - self.writeBitValue( value * ( 0x100 if eight_bit_mantissa else 0x10000 ), length ) + self.write_int_value(int(value * (0x100 if eight_bit_mantissa else 0x10000)), length) + # Precalculated, see the Wikipedia links below. _EXPN_BIAS = {16: 16, 32: 127, 64: 1023} - _N_EXPN_BITS = {16: 5, 32: 8, 64: 52} + _N_EXPN_BITS = {16: 5, 32: 8, 64: 8} _N_FRAC_BITS = {16: 10, 32: 23, 64: 52} _FLOAT_NAME = {16: "float16", 32: "float", 64: "double"} - def read_float_value(self, length=16): + def read_float_value(self, length): """Reads a floating point number of length, which must be 16 (float16), 32 (float), or 64 (double). See: http://en.wikipedia.org/wiki/IEEE_floating-point_standard""" - if length not in _FLOAT_NAME: - raise ValueError, "Length in read_float_value is not 16, 32 or 64." + + if length not in BitStream._FLOAT_NAME: + raise ValueError, "length is not 16, 32 or 64." sign = self.read_bit() - expn = self.read_bit_value(_N_EXPN_BITS[length]) - frac = self.read_bit_value(_N_FRAC_BITS[length]) + expn = self.read_int_value(BitStream._N_EXPN_BITS[length]) + frac = self.read_int_value(BitStream._N_FRAC_BITS[length]) - frac_total = 1 << _N_FRAN_BITS[length] + frac_total = float(1 << BitStream._N_FRAC_BITS[length]) if expn == 0: if frac == 0: @@ -108,35 +113,38 @@ else: return float("nan") - return (-1 if sign else 1) * ( 2**(expn-_EXPN_BIAS[length]) ) * ( 1 + frac / frac_total ) + return (-1 if sign else 1) * 2**(expn - BitStream._EXPN_BIAS[length]) * (1 + frac / frac_total) - def write_float_value(self, value, length=16): + def write_float_value(self, value, length): """Writes a floating point number of length, which must be 16 (float16), 32 (float), or 64 (double). See: http://en.wikipedia.org/wiki/IEEE_floating-point_standard""" - if n == 0: # n is zero, so we don't care about length - self.write_bit_value(0, length) - - import math + + if length not in BitStream._FLOAT_NAME: + raise ValueError, "length is not 16, 32 or 64." + + if value == 0: # value is zero, so we don't care about length + self.write_int_value(0, length) + if math.isnan(value): self.one_fill(length) return elif value == float("-inf"): # negative infinity - self.one_fill(_N_EXPN_BITS[length] + 1) # sign merged - self.zero_fill(_N_FRAC_BITS[length]) + self.one_fill(BitStream._N_EXPN_BITS[length] + 1) # sign merged + self.zero_fill(BitStream._N_FRAC_BITS[length]) return elif value == float("inf"): # positive infinity self.write_bit(False) - self.one_fill(_N_EXPN_BITS[length]) - self.zero_fill(_N_FRAC_BITS[length]) + self.one_fill(BitStream._N_EXPN_BITS[length]) + self.zero_fill(BitStream._N_FRAC_BITS[length]) return - if n < 0: + if value < 0: self.write_bit(True) - n = ~n + 1 + value = ~value + 1 else: self.write_bit(False) - exp = _EXPN_BIAS[length] + exp = BitStream._EXPN_BIAS[length] if value < 1: while int(value) != 1: value *= 2 @@ -146,22 +154,32 @@ value /= 2 exp += 1 - if exp < 0 or exp > ( 1 << _N_EXPN_BITS[length] ): - raise ValueError, "Exponent out of range in %s [%d]." % (_FLOAT_NAME[length], length) + if exp < 0 or exp > ( 1 << BitStream._N_EXPN_BITS[length] ): + raise ValueError, "Exponent out of range in %s [%d]." % (BitStream._FLOAT_NAME[length], length) - frac_total = 1 << _N_FRAC_BITS - self.write_bit_value(exp, _N_EXPN_BITS[length]) - self.write_bit_value(int((value-1)*frac_total) & (frac_total - 1), _N_FRAC_BITS[length]) + frac_total = 1 << BitStream._N_FRAC_BITS[length] + self.write_int_value(exp, BitStream._N_EXPN_BITS[length]) + self.write_int_value(int((value-1)*frac_total) & (frac_total - 1), BitStream._N_FRAC_BITS[length]) def one_fill(self, amount): """Fills amount bits with one. The equivalent of calling self.write_boolean(True) amount times, but more efficient.""" + + if amount > self.bits_available(): + for i in range(amount - self.bits_available()): + self.bits.append(True) + self.bits[self.cursor:self.cursor+amount] = [True] * amount def zero_fill(self, amount): """Fills amount bits with zero. The equivalent of calling self.write_boolean(False) amount times, but more efficient.""" + + if amount > self.bits_available(): + for i in range(amount - self.bits_available()): + self.bits.append(False) + self.bits[self.cursor:self.cursor+amount] = [False] * amount def seek(self, offset, whence=os.SEEK_SET): @@ -177,20 +195,39 @@ def skip_to_end(self): self.seek(0, os.SEEK_END) + + def bits_available(self): + return len(self.bits) - self.cursor + + def __len__(self): + return len(self.bits) + + def __getitem__(self, i): + return self.bits.__getitem__(i) + + def __setitem__(self, i, v): + return self.bits.__setitem__(i, v) + + def __str__(self): + return "".join("1" if b else "0" for b in self.bits) def serialize(self, align=ALIGN_RIGHT, endianness="<"): """Serialize bit array into a byte string, aligning either on the right (ALIGN_RIGHT) or left (ALIGN_LEFT)""" - list = self[:] + list = self.bits[:] leftover = len(list) % 8 - if leftover > 0 and align == BitStream.ALIGN_RIGHT: - list[:0] = [False] * (8-leftover) # Insert some False values to pad the list so it is aligned to the right. + if leftover > 0: + if align == ALIGN_RIGHT: + list[:0] = [False] * (8-leftover) # Insert some False values to pad the list so it is aligned to the right. + else: + list += [False] * (8-leftover) list = BitStream(list) - bytes = [list.read_bit_value(8) for i in range(math.ceil(bits/8.0))] + bytes = [list.read_int_value(8) for i in range(int(math.ceil(len(list)/8.0)))] return struct.pack("%s%dB" % (endianness, len(bytes)), *bytes) def parse(self, string, endianness="<"): """Parse a bit array from a byte string into this BitStream.""" bytes = list(struct.unpack("%s%dB" % (endianness, len(string)))) for byte in bytes: - self.write_bit_value(byte, 8) + self.write_int_value(byte, 8) + From arigo at codespeak.net Mon Dec 22 13:48:13 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 22 Dec 2008 13:48:13 +0100 (CET) Subject: [pypy-svn] r60681 - pypy/branch/oo-jit/pypy/rpython/lltypesystem Message-ID: <20081222124813.656AE168444@codespeak.net> Author: arigo Date: Mon Dec 22 13:48:11 2008 New Revision: 60681 Modified: pypy/branch/oo-jit/pypy/rpython/lltypesystem/ll_str.py pypy/branch/oo-jit/pypy/rpython/lltypesystem/rstr.py Log: Add a bunch of _pure_function_ and _look_inside_me_ attributes to string helpers, to avoid that the JIT looks inside them. Modified: pypy/branch/oo-jit/pypy/rpython/lltypesystem/ll_str.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/lltypesystem/ll_str.py (original) +++ pypy/branch/oo-jit/pypy/rpython/lltypesystem/ll_str.py Mon Dec 22 13:48:11 2008 @@ -5,6 +5,7 @@ def ll_int_str(repr, i): return ll_int2dec(i) +ll_int_str._pure_function_ = True def ll_int2dec(i): from pypy.rpython.lltypesystem.rstr import mallocstr @@ -36,6 +37,7 @@ result.chars[j] = temp[len-j-1] j += 1 return result +ll_int2dec._pure_function_ = True hex_chars = malloc(Array(Char), 16, immortal=True) @@ -77,6 +79,7 @@ result.chars[j] = temp[len-j-1] j += 1 return result +ll_int2hex._pure_function_ = True def ll_int2oct(i, addPrefix): from pypy.rpython.lltypesystem.rstr import mallocstr @@ -113,7 +116,8 @@ result.chars[j] = temp[len-j-1] j += 1 return result +ll_int2oct._pure_function_ = True def ll_float_str(repr, f): return formatd("%f", f) - +ll_float_str._pure_function_ = True Modified: pypy/branch/oo-jit/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/oo-jit/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/oo-jit/pypy/rpython/lltypesystem/rstr.py Mon Dec 22 13:48:11 2008 @@ -221,6 +221,7 @@ newstr.chars[j] = ch j += 1 return newstr + ll_char_mul._pure_function_ = True def ll_strlen(s): return len(s.chars) @@ -240,6 +241,7 @@ s = malloc(1) s.chars[0] = ch return s + ll_chr2str._pure_function_ = True def ll_str2unicode(str): lgt = len(str.chars) @@ -249,6 +251,7 @@ raise UnicodeDecodeError s.chars[i] = cast_primitive(UniChar, str.chars[i]) return s + ll_str2unicode._pure_function_ = True def ll_strhash(s): # unlike CPython, there is no reason to avoid to return -1 @@ -278,6 +281,7 @@ i += 1 j += 1 return newstr + ll_strconcat._pure_function_ = True def ll_strip(s, ch, left, right): s_len = len(s.chars) @@ -300,6 +304,7 @@ i += 1 j += 1 return result + ll_strip._pure_function_ = True def ll_upper(s): s_chars = s.chars @@ -315,6 +320,7 @@ result.chars[i] = ch i += 1 return result + ll_upper._pure_function_ = True def ll_lower(s): s_chars = s.chars @@ -330,6 +336,7 @@ result.chars[i] = ch i += 1 return result + ll_lower._pure_function_ = True def ll_join(s, length, items): s_chars = s.chars @@ -370,6 +377,7 @@ res_index += 1 i += 1 return result + ll_join._look_inside_me_ = False def ll_strcmp(s1, s2): if not s1: @@ -395,6 +403,7 @@ return diff i += 1 return len1 - len2 + ll_strcmp._pure_function_ = True def ll_streq(s1, s2): if s1 == s2: # also if both are NULLs @@ -414,6 +423,7 @@ j += 1 return True + ll_streq._pure_function_ = True def ll_startswith(s1, s2): len1 = len(s1.chars) @@ -429,6 +439,7 @@ j += 1 return True + ll_startswith._pure_function_ = True def ll_endswith(s1, s2): len1 = len(s1.chars) @@ -445,6 +456,7 @@ j += 1 return True + ll_endswith._pure_function_ = True def ll_find_char(s, ch, start, end): i = start @@ -455,6 +467,7 @@ return i i += 1 return -1 + ll_find_char._pure_function_ = True def ll_rfind_char(s, ch, start, end): if end > len(s.chars): @@ -465,6 +478,7 @@ if s.chars[i] == ch: return i return -1 + ll_rfind_char._pure_function_ = True def ll_count_char(s, ch, start, end): count = 0 @@ -476,6 +490,7 @@ count += 1 i += 1 return count + ll_count_char._pure_function_ = True def ll_find(cls, s1, s2, start, end): """Knuth Morris Prath algorithm for substring match""" @@ -509,6 +524,7 @@ m = m + i - e i = e return -1 + ll_find._pure_function_ = True ll_find = classmethod(ll_find) def ll_rfind(cls, s1, s2, start, end): @@ -554,6 +570,7 @@ m = m - i + e i = e return -1 + ll_rfind._pure_function_ = True ll_rfind = classmethod(ll_rfind) def ll_count(cls, s1, s2, start, end): @@ -591,6 +608,7 @@ m = m + i - e i = e return result + ll_count._pure_function_ = True ll_count = classmethod(ll_count) def ll_join_strs(length, items): @@ -618,6 +636,7 @@ res_index += 1 i += 1 return result + ll_join_strs._look_inside_me_ = False def ll_join_chars(length, chars): num_chars = length @@ -632,6 +651,7 @@ res_chars[i] = chars[i] i += 1 return result + ll_join_chars._look_inside_me_ = False def ll_stringslice_startonly(s1, start): len1 = len(s1.chars) @@ -642,6 +662,7 @@ start += 1 j += 1 return newstr + ll_stringslice_startonly._pure_function_ = True def ll_stringslice(s1, slice): start = slice.start @@ -657,6 +678,7 @@ start += 1 j += 1 return newstr + ll_stringslice._pure_function_ = True def ll_stringslice_minusone(s1): newlen = len(s1.chars) - 1 @@ -666,6 +688,7 @@ newstr.chars[j] = s1.chars[j] j += 1 return newstr + ll_stringslice_minusone._pure_function_ = True def ll_split_chr(LIST, s, c): chars = s.chars @@ -701,6 +724,7 @@ resindex += 1 return res + ll_split_chr._pure_function_ = True def ll_replace_chr_chr(s, c1, c2): length = len(s.chars) @@ -715,6 +739,7 @@ dst[j] = c j += 1 return newstr + ll_replace_chr_chr._pure_function_ = True def ll_contains(s, c): chars = s.chars @@ -725,6 +750,7 @@ return True i += 1 return False + ll_contains._pure_function_ = True def ll_int(s, base): if not 2 <= base <= 36: @@ -772,6 +798,7 @@ if not i == strlen: raise ValueError return sign * val + ll_int._pure_function_ = True # interface to build strings: # x = ll_build_start(n) From arigo at codespeak.net Tue Dec 23 13:46:43 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 23 Dec 2008 13:46:43 +0100 (CET) Subject: [pypy-svn] r60692 - pypy/branch/oo-jit/pypy/jit/rainbow/test Message-ID: <20081223124643.7EA461684F4@codespeak.net> Author: arigo Date: Tue Dec 23 13:46:42 2008 New Revision: 60692 Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_interpreter.py Log: Add support for extended 'view' attribute from the tests. Modified: pypy/branch/oo-jit/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/oo-jit/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/oo-jit/pypy/jit/rainbow/test/test_interpreter.py Tue Dec 23 13:46:42 2008 @@ -48,7 +48,7 @@ def hannotate(func, values, policy=None, inline=None, backendoptimize=False, - portal=None, type_system="lltype"): + portal=None, type_system="lltype", view="auto"): # build the normal ll graphs for ll_function t = TranslationContext() annpolicy = AnnotatorPolicy() @@ -80,7 +80,7 @@ [SomeLLAbstractConstant(v.concretetype, {OriginFlags(): True}) for v in graph1.getargs()]) hannotator.simplify() - if conftest.option.view: + if (view == "auto" and conftest.option.view) or view == True: hannotator.translator.view() return hs, hannotator, rtyper From arigo at codespeak.net Wed Dec 24 13:53:21 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 24 Dec 2008 13:53:21 +0100 (CET) Subject: [pypy-svn] r60702 - pypy/extradoc/talk/ecoop2009 Message-ID: <20081224125321.177FE168453@codespeak.net> Author: arigo Date: Wed Dec 24 13:53:20 2008 New Revision: 60702 Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex pypy/extradoc/talk/ecoop2009/clibackend.tex pypy/extradoc/talk/ecoop2009/conclusion.tex pypy/extradoc/talk/ecoop2009/jitgen.tex pypy/extradoc/talk/ecoop2009/rainbow.tex Log: Typos. Modified: pypy/extradoc/talk/ecoop2009/benchmarks.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/benchmarks.tex (original) +++ pypy/extradoc/talk/ecoop2009/benchmarks.tex Wed Dec 24 13:53:20 2008 @@ -20,11 +20,12 @@ \end{enumerate} Moreover, for each benchmark we also show the time taken by running the -equivalent program written in C\#\footnote{The sources for both TLC and C\# +equivalent program written in C\#.\footnote{The sources for both TLC and C\# programs are available at: + http://codespeak.net/svn/pypy/extradoc/talk/ecoop2009/benchmarks/} -The benchmarks have been run on machine with an Intel Pentium 4 CPU running at +The benchmarks have been run on a machine with an Intel Pentium 4 CPU running at 3.20 GHz and 2 GB of RAM, running Microsoft Windows XP and Microsoft .NET Framework 2.0. @@ -123,7 +124,7 @@ To measure how the JIT handles object-oriented features, we wrote a very simple benchmark that involves attribute lookups and polymorphic method calls. Since the TLC assembler source is long and hard to read, -Figure~\ref{fig:accumulator} shows the equivalent program written in an +figure~\ref{fig:accumulator} shows the equivalent program written in an invented Python-like syntax. \begin{figure}[h] @@ -168,7 +169,7 @@ As already discussed, our generated JIT does not compile the whole function at once. Instead, it compiles and executes code chunk by chunk, waiting until it -knows enough informations to generate highly efficient code. In particualr, +knows enough informations to generate highly efficient code. In particular, at the time it emits the code for the inner loop it exactly knows the type of \lstinline{obj}, thus it can remove the overhead of dynamic dispatch and inline the method call. Moreover, since \lstinline{obj} never escapes the Modified: pypy/extradoc/talk/ecoop2009/clibackend.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/clibackend.tex (original) +++ pypy/extradoc/talk/ecoop2009/clibackend.tex Wed Dec 24 13:53:20 2008 @@ -36,7 +36,7 @@ \subsection{Flexswitches} For a large part, implementing the CLI backend is easy and straightforward, as -there is a close corrispondence between most of the operations used by +there is a close correspondence between most of the operations used by frontend's flowgraphs and the CLI instructions. Thus, we will not dig into details for this part. @@ -103,7 +103,7 @@ \end{itemize} When a new case is added to a flexswitch, the backend generates the new blocks -into a new single method. The newly created method is pointed by a +into a new single method. The newly created method is pointed to by a delegate\footnote{\emph{Delegates} are the .NET equivalent of function pointers.} stored in the flexswitch, so that it can be invoked later when needed. @@ -124,7 +124,7 @@ What cannot be easily implemented in CLI is following an external link whose target is not an initial block; consider, for instance, the outgoing link of the block dynamically added in the right-hand side -picture of Figure~\ref{flexswitch-fig}. How it is possible to jump in +picture of Figure~\ref{flexswitch-fig}. How is it possible to jump into the middle of a method? To solve this problem every method contains a special code, called Modified: pypy/extradoc/talk/ecoop2009/conclusion.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/conclusion.tex (original) +++ pypy/extradoc/talk/ecoop2009/conclusion.tex Wed Dec 24 13:53:20 2008 @@ -5,13 +5,13 @@ \cite{DBLP:conf/pepm/Rigo04}). However, Psyco is a manually written JIT, is not applicable to other languages and cannot be retargetted. -Moreover, the idea of promotion of is a generalization of \emph{Polymorphic +Moreover, the idea of promotion is a generalization of \emph{Polymorphic Inline Caches} \cite{hoelzle_optimizing_1991}, as well as the idea of using runtime feedback to produce more efficient code \cite{hoelzle_type_feedback_1994}. PyPy-style JIT compilers are hard to write manually, thus we chose to write a -JIT generator. Tracing JIT compilers \cite{gal_hotpathvm_2006} also gives +JIT generator. Tracing JIT compilers \cite{gal_hotpathvm_2006} also give good results but are much easier to write, making the need for an automatic generator less urgent. However so far tracing JITs have less general allocation removal techniques, which makes them get less speedup in a dynamic Modified: pypy/extradoc/talk/ecoop2009/jitgen.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/jitgen.tex (original) +++ pypy/extradoc/talk/ecoop2009/jitgen.tex Wed Dec 24 13:53:20 2008 @@ -188,7 +188,7 @@ concrete=True)} is reported as an error. There are cases in which having a green variable is essential for generating -good code, but it is not possible to use the \texttt{concrete} due to an +good code, but it is not possible to use the \texttt{concrete} hint due to an unsatisfiable dependency: Section~\ref{sec:promotion} introduces \emph{promotion}, the novel technique that makes possible to solve this problem. Modified: pypy/extradoc/talk/ecoop2009/rainbow.tex ============================================================================== --- pypy/extradoc/talk/ecoop2009/rainbow.tex (original) +++ pypy/extradoc/talk/ecoop2009/rainbow.tex Wed Dec 24 13:53:20 2008 @@ -251,7 +251,7 @@ add the new case, which is then executed. If later an instance of \lstinline{IntObj} hits the flexswitch again, the -code is executed without needing of more calls to the JIT compiler. On the +code is executed without needing more calls to the JIT compiler. On the other hand, if the flexswitch is hit by an instance of some other class, the \lstinline{default} branch will be selected again and the whole process will restart. From pypy-svn at codespeak.net Wed Dec 24 14:59:24 2008 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Wed, 24 Dec 2008 14:59:24 +0100 (CET) Subject: [pypy-svn] I wait for you for an hour! Message-ID: <20081224135924.75104168488@codespeak.net> An HTML attachment was scrubbed... URL: From pypy-svn at codespeak.net Wed Dec 24 23:57:40 2008 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Wed, 24 Dec 2008 23:57:40 +0100 (CET) Subject: [pypy-svn] Joyful Christmas!, from colleagues Message-ID: <20081224225740.70E24168480@codespeak.net> An HTML attachment was scrubbed... URL: From pypy-svn at codespeak.net Fri Dec 26 06:15:03 2008 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Fri, 26 Dec 2008 06:15:03 +0100 (CET) Subject: [pypy-svn] This Christmas, don't sleep alone Message-ID: <20081226051503.64C4F1680C1@codespeak.net> An HTML attachment was scrubbed... URL: From pypy-svn at codespeak.net Fri Dec 26 16:59:56 2008 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Fri, 26 Dec 2008 16:59:56 +0100 (CET) Subject: [pypy-svn] Cheapest way to gain size Message-ID: <20081226155956.8FC481684AA@codespeak.net> An HTML attachment was scrubbed... URL: From pypy-svn at codespeak.net Sat Dec 27 08:05:26 2008 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Sat, 27 Dec 2008 08:05:26 +0100 (CET) Subject: [pypy-svn] the proven secret for manhood Message-ID: <20081227070526.CB89A1683F3@codespeak.net> An HTML attachment was scrubbed... URL: From pypy-svn at codespeak.net Sat Dec 27 17:00:36 2008 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Sat, 27 Dec 2008 17:00:36 +0100 (CET) Subject: [pypy-svn] She deep throated me Message-ID: <20081227160036.379A6168496@codespeak.net> An HTML attachment was scrubbed... URL: From pypy-svn at codespeak.net Sun Dec 28 03:40:28 2008 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Sun, 28 Dec 2008 03:40:28 +0100 (CET) Subject: [pypy-svn] A thicker, longer, more muscular tool Message-ID: <20081228024028.2B2EA168472@codespeak.net> An HTML attachment was scrubbed... URL: From pypy-svn at codespeak.net Sun Dec 28 21:01:32 2008 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Sun, 28 Dec 2008 21:01:32 +0100 (CET) Subject: [pypy-svn] Unbridled pleasure is yours Message-ID: <20081228200132.B4495168428@codespeak.net> An HTML attachment was scrubbed... URL: From pypy-svn at codespeak.net Mon Dec 29 14:40:25 2008 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Mon, 29 Dec 2008 14:40:25 +0100 (CET) Subject: [pypy-svn] You wont' be lonely with this Message-ID: <20081229134025.AA13016841A@codespeak.net> An HTML attachment was scrubbed... URL: From pedronis at codespeak.net Mon Dec 29 18:43:36 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 29 Dec 2008 18:43:36 +0100 (CET) Subject: [pypy-svn] r60727 - in pypy/build/bot2/pypybuildbot: . test Message-ID: <20081229174336.371691683FD@codespeak.net> Author: pedronis Date: Mon Dec 29 18:43:34 2008 New Revision: 60727 Modified: pypy/build/bot2/pypybuildbot/summary.py pypy/build/bot2/pypybuildbot/test/test_summary.py Log: some testing of body, recentRevisions Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Mon Dec 29 18:43:34 2008 @@ -81,7 +81,11 @@ self._misses = 0 self.cachesize = cachesize - def reset(self): + def clear(self): # for testing + self._outcome_sets = {} + self._lru = [] + + def reset_counters(self): self._hits = 0 self._misses = 0 @@ -370,7 +374,7 @@ def body(self, request): t0 = time.time() - outcome_set_cache.reset() + outcome_set_cache.reset_counters() builder = request.args.get('builder', []) build = request.args.get('build', []) @@ -472,7 +476,7 @@ def body(self, request): t0 = time.time() - outcome_set_cache.reset() + outcome_set_cache.reset_counters() status = self.getStatus(request) Modified: pypy/build/bot2/pypybuildbot/test/test_summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/test/test_summary.py (original) +++ pypy/build/bot2/pypybuildbot/test/test_summary.py Mon Dec 29 18:43:34 2008 @@ -1,3 +1,6 @@ +from zope.interface import implements +from buildbot import interfaces as buildbot_intefaces +from buildbot.status import builder as status_builder from pypybuildbot import summary from StringIO import StringIO @@ -220,5 +223,160 @@ assert len(revs) == 4 assert revs == {99: 199, 98: 198, 97: 197, 96: 196} - +class _BuilderToStatus(object): + + def __init__(self, status): + self.builder_status = status + +class FakeRequest(object): + + def __init__(self, builders, args={}): + status = status_builder.Status(self, '/tmp') + status.basedir = None + self.status = status + self.args = args + + self.builderNames = [] + self.builders = {} + for builder in builders: + name = builder.getName() + self.builderNames.append(name) + self.builders[name] = _BuilderToStatus(builder) + + self.site = self + self.buildbot_service = self + self.parent = self + self.buildbotURL = "http://buildbot/" + + def getStatus(self): + return self.status + +def witness_branches(summary): + ref = [None] + recentRevisions = summary.recentRevisions + def witness(*args, **kwds): + branches = recentRevisions(*args, **kwds) + ref[0] = branches + return branches + summary.recentRevisions = witness + + return lambda: ref[0] + +class FakeLog(object): + implements(buildbot_intefaces.IStatusLog) + + def __init__(self, step, name, cont=""): + self.step = step + self.name = name + self.cont = cont + + def getStep(self): + return self.step + + def getName(self): + return self.name + + def hasContents(self): + return True + + def readlines(self): + return [l+'\n' for l in self.cont.splitlines()] + +def add_builds(builder, builds): + n = getattr(builder, 'nextBuildNumber', 0) + for rev, reslog in builds: + build = status_builder.BuildStatus(builder, n) + build.setProperty('got_revision', rev, None) + step = build.addStepWithName('pytest') + step.logs.extend([FakeLog(step, 'pytestLog', reslog), + FakeLog(step, 'stdio')]) + build.buildFinished() + builder.addBuildToCache(build) + n += 1 + builder.nextBuildNumber = n + + +class TestSummary(object): + + def setup_method(self, meth): + summary.outcome_set_cache.clear() + + def test_sanity(self): + s = summary.Summary() + res = witness_branches(s) + req = FakeRequest([]) + s.body(req) + branches = res() + + assert branches == {} + + def test_one_build_no_rev(self): + builder = status_builder.BuilderStatus('builder0') + build = status_builder.BuildStatus(builder, 0) + build.buildFinished() + builder.addBuildToCache(build) + builder.nextBuildNumber = len(builder.buildCache) + + s = summary.Summary() + res = witness_branches(s) + req = FakeRequest([builder]) + s.body(req) + branches = res() + + assert branches == {None: ({}, [build])} + + def test_one_build(self): + builder = status_builder.BuilderStatus('builder0') + add_builds(builder, [(60000, ". a")]) + + s = summary.Summary() + res = witness_branches(s) + req = FakeRequest([builder]) + s.body(req) + branches = res() + + revs = branches[None][0] + assert revs.keys() == [60000] + outcome = revs[60000]['builder0'] + assert outcome.revision == 60000 + assert outcome.key == ('builder0', 0) + + def test_two_builds(self): + builder = status_builder.BuilderStatus('builder0') + add_builds(builder, [(60000, ". a"), + (60001, ". a")]) + + s = summary.Summary() + res = witness_branches(s) + req = FakeRequest([builder]) + s.body(req) + branches = res() + + revs = branches[None][0] + assert sorted(revs.keys()) == [60000, 60001] + outcome = revs[60000]['builder0'] + assert outcome.revision == 60000 + assert outcome.key == ('builder0', 0) + outcome = revs[60001]['builder0'] + assert outcome.revision == 60001 + assert outcome.key == ('builder0', 1) + + def test_two_builds_samerev(self): + builder = status_builder.BuilderStatus('builder0') + add_builds(builder, [(60000, ". a"), + (60000, ". a")]) + + s = summary.Summary() + res = witness_branches(s) + req = FakeRequest([builder]) + out = s.body(req) + branches = res() + + revs = branches[None][0] + assert sorted(revs.keys()) == [60000] + outcome = revs[60000]['builder0'] + assert outcome.revision == 60000 + assert outcome.key == ('builder0', 1) + + From pedronis at codespeak.net Mon Dec 29 19:56:22 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 29 Dec 2008 19:56:22 +0100 (CET) Subject: [pypy-svn] r60729 - pypy/build/bot2/pypybuildbot/test Message-ID: <20081229185622.9275C16840D@codespeak.net> Author: pedronis Date: Mon Dec 29 19:56:20 2008 New Revision: 60729 Modified: pypy/build/bot2/pypybuildbot/test/test_summary.py Log: - minimal test for recentrev query after fixing add_builds behavior, got_revision is a string Modified: pypy/build/bot2/pypybuildbot/test/test_summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/test/test_summary.py (original) +++ pypy/build/bot2/pypybuildbot/test/test_summary.py Mon Dec 29 19:56:20 2008 @@ -287,7 +287,7 @@ n = getattr(builder, 'nextBuildNumber', 0) for rev, reslog in builds: build = status_builder.BuildStatus(builder, n) - build.setProperty('got_revision', rev, None) + build.setProperty('got_revision', str(rev), None) step = build.addStepWithName('pytest') step.logs.extend([FakeLog(step, 'pytestLog', reslog), FakeLog(step, 'stdio')]) @@ -379,4 +379,42 @@ assert outcome.revision == 60000 assert outcome.key == ('builder0', 1) + def test_two_builds(self): + builder = status_builder.BuilderStatus('builder0') + add_builds(builder, [(60000, ". a"), + (60001, ". a")]) + + s = summary.Summary() + res = witness_branches(s) + req = FakeRequest([builder]) + s.body(req) + branches = res() + + revs = branches[None][0] + assert sorted(revs.keys()) == [60000, 60001] + outcome = revs[60000]['builder0'] + assert outcome.revision == 60000 + assert outcome.key == ('builder0', 0) + outcome = revs[60001]['builder0'] + assert outcome.revision == 60001 + assert outcome.key == ('builder0', 1) + + def test_two_builds_recentrev(self): + builder = status_builder.BuilderStatus('builder0') + add_builds(builder, [(60000, ". a"), + (60001, ". a")]) + + s = summary.Summary() + res = witness_branches(s) + req = FakeRequest([builder]) + req.args = {'recentrev': ['60000']} + s.body(req) + branches = res() + + revs = branches[None][0] + assert sorted(revs.keys()) == [60000] + outcome = revs[60000]['builder0'] + assert outcome.revision == 60000 + assert outcome.key == ('builder0', 0) + From pedronis at codespeak.net Mon Dec 29 20:00:26 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 29 Dec 2008 20:00:26 +0100 (CET) Subject: [pypy-svn] r60730 - pypy/build/bot2/pypybuildbot/test Message-ID: <20081229190026.7BD7A16840F@codespeak.net> Author: pedronis Date: Mon Dec 29 20:00:24 2008 New Revision: 60730 Modified: pypy/build/bot2/pypybuildbot/test/test_summary.py Log: oops, remove exra copy of test Modified: pypy/build/bot2/pypybuildbot/test/test_summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/test/test_summary.py (original) +++ pypy/build/bot2/pypybuildbot/test/test_summary.py Mon Dec 29 20:00:24 2008 @@ -379,26 +379,6 @@ assert outcome.revision == 60000 assert outcome.key == ('builder0', 1) - def test_two_builds(self): - builder = status_builder.BuilderStatus('builder0') - add_builds(builder, [(60000, ". a"), - (60001, ". a")]) - - s = summary.Summary() - res = witness_branches(s) - req = FakeRequest([builder]) - s.body(req) - branches = res() - - revs = branches[None][0] - assert sorted(revs.keys()) == [60000, 60001] - outcome = revs[60000]['builder0'] - assert outcome.revision == 60000 - assert outcome.key == ('builder0', 0) - outcome = revs[60001]['builder0'] - assert outcome.revision == 60001 - assert outcome.key == ('builder0', 1) - def test_two_builds_recentrev(self): builder = status_builder.BuilderStatus('builder0') add_builds(builder, [(60000, ". a"), From pedronis at codespeak.net Mon Dec 29 20:15:56 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 29 Dec 2008 20:15:56 +0100 (CET) Subject: [pypy-svn] r60731 - pypy/build/bot2/pypybuildbot Message-ID: <20081229191556.A958A168415@codespeak.net> Author: pedronis Date: Mon Dec 29 20:15:56 2008 New Revision: 60731 Modified: pypy/build/bot2/pypybuildbot/summary.py Log: oops, remove exra copy of test Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Mon Dec 29 20:15:56 2008 @@ -261,7 +261,10 @@ branch_anchor = html.a(branch, href="/summary?branch=%s" % branch) self.sections.append(html.h2(branch_anchor)) - def _rev_anchor(self, rev): + def _label(self, outcome_set): + return outcome_set.revision + + def _label_anchor(self, rev): rev_anchor = html.a(str(rev), href="/summary?branch=%s&recentrev=%d" % (self.cur_branch, rev)) return rev_anchor @@ -269,18 +272,18 @@ def add_section(self, outcome_sets): if not outcome_sets: return - revs = sorted(outcome_set.revision for outcome_set in outcome_sets) - by_rev = sorted((outcome_set.revision, outcome_set) for outcome_set + labels = sorted(self._label(outcome_set) for outcome_set in outcome_sets) + by_label = sorted((self._label(outcome_set), outcome_set) for outcome_set in outcome_sets) lines = [] - align = 2*len(revs)-1+len(str(revs[-1])) + align = 2*len(labels)-1+len(str(labels[-1])) def bars(): return ' |'*len(lines) - for rev, outcome_set in by_rev: + for label, outcome_set in by_label: count_failures = len(outcome_set.failed) count_skipped = len(outcome_set.skipped) - line = [bars(), ' ', self._rev_anchor(rev)] + line = [bars(), ' ', self._label_anchor(label)] line.append((align-len(line[0]))*" ") line.append(self.make_stdio_anchors_for(outcome_set)) line.append('\n') @@ -289,7 +292,7 @@ failed = set() exploded = set() - for rev, outcome_set in by_rev: + for label, outcome_set in by_label: for failure in outcome_set.failed: letter = outcome_set.get_outcome(failure) if letter == '!': @@ -303,7 +306,7 @@ for failure in sorted(failed, key=sorting): line = [] - for rev, outcome_set in by_rev: + for label, outcome_set in by_label: letter = outcome_set.get_outcome(failure) failed = letter not in ('s', '.') if outcome_set.get_longrepr(failure): From pedronis at codespeak.net Mon Dec 29 20:17:45 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 29 Dec 2008 20:17:45 +0100 (CET) Subject: [pypy-svn] r60732 - in pypy/build/bot2/pypybuildbot: . test Message-ID: <20081229191745.484821683D7@codespeak.net> Author: pedronis Date: Mon Dec 29 20:17:44 2008 New Revision: 60732 Modified: pypy/build/bot2/pypybuildbot/summary.py pypy/build/bot2/pypybuildbot/test/test_summary.py Log: oops, premature commit Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Mon Dec 29 20:17:44 2008 @@ -261,10 +261,7 @@ branch_anchor = html.a(branch, href="/summary?branch=%s" % branch) self.sections.append(html.h2(branch_anchor)) - def _label(self, outcome_set): - return outcome_set.revision - - def _label_anchor(self, rev): + def _rev_anchor(self, rev): rev_anchor = html.a(str(rev), href="/summary?branch=%s&recentrev=%d" % (self.cur_branch, rev)) return rev_anchor @@ -272,18 +269,18 @@ def add_section(self, outcome_sets): if not outcome_sets: return - labels = sorted(self._label(outcome_set) for outcome_set in outcome_sets) - by_label = sorted((self._label(outcome_set), outcome_set) for outcome_set + revs = sorted(outcome_set.revision for outcome_set in outcome_sets) + by_rev = sorted((outcome_set.revision, outcome_set) for outcome_set in outcome_sets) lines = [] - align = 2*len(labels)-1+len(str(labels[-1])) + align = 2*len(revs)-1+len(str(revs[-1])) def bars(): return ' |'*len(lines) - for label, outcome_set in by_label: + for rev, outcome_set in by_rev: count_failures = len(outcome_set.failed) count_skipped = len(outcome_set.skipped) - line = [bars(), ' ', self._label_anchor(label)] + line = [bars(), ' ', self._rev_anchor(rev)] line.append((align-len(line[0]))*" ") line.append(self.make_stdio_anchors_for(outcome_set)) line.append('\n') @@ -292,7 +289,7 @@ failed = set() exploded = set() - for label, outcome_set in by_label: + for rev, outcome_set in by_rev: for failure in outcome_set.failed: letter = outcome_set.get_outcome(failure) if letter == '!': @@ -306,7 +303,7 @@ for failure in sorted(failed, key=sorting): line = [] - for label, outcome_set in by_label: + for rev, outcome_set in by_rev: letter = outcome_set.get_outcome(failure) failed = letter not in ('s', '.') if outcome_set.get_longrepr(failure): Modified: pypy/build/bot2/pypybuildbot/test/test_summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/test/test_summary.py (original) +++ pypy/build/bot2/pypybuildbot/test/test_summary.py Mon Dec 29 20:17:44 2008 @@ -379,6 +379,26 @@ assert outcome.revision == 60000 assert outcome.key == ('builder0', 1) + def test_two_builds(self): + builder = status_builder.BuilderStatus('builder0') + add_builds(builder, [(60000, ". a"), + (60001, ". a")]) + + s = summary.Summary() + res = witness_branches(s) + req = FakeRequest([builder]) + s.body(req) + branches = res() + + revs = branches[None][0] + assert sorted(revs.keys()) == [60000, 60001] + outcome = revs[60000]['builder0'] + assert outcome.revision == 60000 + assert outcome.key == ('builder0', 0) + outcome = revs[60001]['builder0'] + assert outcome.revision == 60001 + assert outcome.key == ('builder0', 1) + def test_two_builds_recentrev(self): builder = status_builder.BuilderStatus('builder0') add_builds(builder, [(60000, ". a"), From pedronis at codespeak.net Mon Dec 29 20:19:01 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 29 Dec 2008 20:19:01 +0100 (CET) Subject: [pypy-svn] r60733 - pypy/build/bot2/pypybuildbot/test Message-ID: <20081229191901.9EF07168412@codespeak.net> Author: pedronis Date: Mon Dec 29 20:19:01 2008 New Revision: 60733 Modified: pypy/build/bot2/pypybuildbot/test/test_summary.py Log: this was wanted though, oops^2 Modified: pypy/build/bot2/pypybuildbot/test/test_summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/test/test_summary.py (original) +++ pypy/build/bot2/pypybuildbot/test/test_summary.py Mon Dec 29 20:19:01 2008 @@ -379,26 +379,6 @@ assert outcome.revision == 60000 assert outcome.key == ('builder0', 1) - def test_two_builds(self): - builder = status_builder.BuilderStatus('builder0') - add_builds(builder, [(60000, ". a"), - (60001, ". a")]) - - s = summary.Summary() - res = witness_branches(s) - req = FakeRequest([builder]) - s.body(req) - branches = res() - - revs = branches[None][0] - assert sorted(revs.keys()) == [60000, 60001] - outcome = revs[60000]['builder0'] - assert outcome.revision == 60000 - assert outcome.key == ('builder0', 0) - outcome = revs[60001]['builder0'] - assert outcome.revision == 60001 - assert outcome.key == ('builder0', 1) - def test_two_builds_recentrev(self): builder = status_builder.BuilderStatus('builder0') add_builds(builder, [(60000, ". a"), From pedronis at codespeak.net Mon Dec 29 21:52:07 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 29 Dec 2008 21:52:07 +0100 (CET) Subject: [pypy-svn] r60734 - in pypy/build/bot2/pypybuildbot: . test Message-ID: <20081229205207.71F67168452@codespeak.net> Author: pedronis Date: Mon Dec 29 21:52:04 2008 New Revision: 60734 Modified: pypy/build/bot2/pypybuildbot/summary.py pypy/build/bot2/pypybuildbot/test/test_summary.py Log: drill down by builder and builds, in these cases all builds not only the last one are shown for a given revision Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Mon Dec 29 21:52:04 2008 @@ -160,7 +160,7 @@ self._skipped = None self._numpassed = None self.revision = map.values()[0].revision - + @property def failed(self): if self._failed is None: @@ -231,6 +231,7 @@ def __init__(self): self.sections = [] self.cur_branch=None + self.fixed_builder = False def make_longrepr_url_for(self, outcome_set, namekey): cachekey, namekey = outcome_set.get_key_namekey(namekey) @@ -261,26 +262,43 @@ branch_anchor = html.a(branch, href="/summary?branch=%s" % branch) self.sections.append(html.h2(branch_anchor)) - def _rev_anchor(self, rev): - rev_anchor = html.a(str(rev), href="/summary?branch=%s&recentrev=%d" % - (self.cur_branch, rev)) + def _builder_num(self, outcome_set): + return outcome_set.map.values()[0].key + + def _label(self, outcome_set): + if self.fixed_builder: + # (rev, buildNumber) + buildNumber = self._builder_num(outcome_set)[1] + return (outcome_set.revision, buildNumber) + else: + # rev + return outcome_set.revision + + def _label_anchor(self, outcome_set): + rev = outcome_set.revision + if self.fixed_builder: + pick = "builder=%s&builds=%d" % self._builder_num(outcome_set) + else: + pick = "recentrev=%d" % rev + rev_anchor = html.a(str(rev), href="/summary?branch=%s&%s" % + (self.cur_branch, pick)) return rev_anchor def add_section(self, outcome_sets): if not outcome_sets: return - revs = sorted(outcome_set.revision for outcome_set in outcome_sets) - by_rev = sorted((outcome_set.revision, outcome_set) for outcome_set + labels = sorted(self._label(outcome_set) for outcome_set in outcome_sets) + by_label = sorted((self._label(outcome_set), outcome_set) for outcome_set in outcome_sets) lines = [] - align = 2*len(revs)-1+len(str(revs[-1])) + align = 2*len(labels)-1+len(str(labels[-1])) def bars(): return ' |'*len(lines) - for rev, outcome_set in by_rev: + for label, outcome_set in by_label: count_failures = len(outcome_set.failed) count_skipped = len(outcome_set.skipped) - line = [bars(), ' ', self._rev_anchor(rev)] + line = [bars(), ' ', self._label_anchor(outcome_set)] line.append((align-len(line[0]))*" ") line.append(self.make_stdio_anchors_for(outcome_set)) line.append('\n') @@ -289,7 +307,7 @@ failed = set() exploded = set() - for rev, outcome_set in by_rev: + for label, outcome_set in by_label: for failure in outcome_set.failed: letter = outcome_set.get_outcome(failure) if letter == '!': @@ -303,7 +321,7 @@ for failure in sorted(failed, key=sorting): line = [] - for rev, outcome_set in by_rev: + for label, outcome_set in by_label: letter = outcome_set.get_outcome(failure) failed = letter not in ('s', '.') if outcome_set.get_longrepr(failure): @@ -418,7 +436,12 @@ trunk_name = make_subst(None, "") trunk_value = make_subst("", None) - +def safe_int(v): + try: + return int(v) + except ValueError: + return None + class Summary(HtmlResource): def __init__(self): @@ -431,20 +454,35 @@ N) @staticmethod - def _prune_revs(revs, cutnum): - if len(revs) > cutnum: - for rev in sorted(revs.keys())[:-cutnum]: - del revs[rev] + def _prune_runs(runs, cutnum): + if len(runs) > cutnum: + for rev in sorted(runs.keys())[:-cutnum]: + del runs[rev] - def recentRevisions(self, status, only_recentrevs=None, only_branches=None): + def recentRuns(self, status, only_recentrevs=None, only_branches=None, + only_builder=None, only_builds=None): test_rev = make_test(only_recentrevs) test_branch = make_test(only_branches) + test_builder = make_test(only_builder) + fixed_builder = bool(only_builder) branches = {} for builderName in status.getBuilderNames(): + if not test_builder(builderName): + continue builderStatus = status.getBuilder(builderName) - for build in builderStatus.generateFinishedBuilds(num_builds=5*N): + if only_builds: + def builditer(): + for num in only_builds: + b = builderStatus.getBuild(num) + if b is not None: + yield b + builditer = builditer() + else: + builditer = builderStatus.generateFinishedBuilds(num_builds=5*N) + + for build in builditer: branch = getProp(build, 'branch') if not test_branch(branch): continue @@ -452,27 +490,49 @@ if not test_rev(got_rev): continue - revs, no_revision_builds = branches.setdefault(branch, + runs, no_revision_builds = branches.setdefault(branch, ({}, [])) if got_rev is None: no_revision_builds.append(build) else: rev = int(got_rev) - revBuilds = revs.setdefault(rev, {}) - # pick the most recent or ? - if builderName not in revBuilds: - revBuilds[builderName] = build.getNumber() - - for branch, (revs, no_revision_builds) in branches.items(): - self._prune_revs(revs, N) - for rev, revBuilds in revs.iteritems(): - for builderName, buildNumber in revBuilds.items(): + buildNumber = build.getNumber() + if fixed_builder: + builds = runs.setdefault((rev, buildNumber), {}) + else: + builds = runs.setdefault(rev, {}) + # pick the most recent or ? + + if builderName not in builds: + builds[builderName] = build.getNumber() + + for branch, (runs, no_revision_builds) in branches.items(): + self._prune_runs(runs, N) + for label, runBuilds in runs.iteritems(): + for builderName, buildNumber in runBuilds.items(): key = (builderName, buildNumber) outcome_set = outcome_set_cache.get(status, key) - revBuilds[builderName] = outcome_set + runBuilds[builderName] = outcome_set return branches + + @staticmethod + def _parse_builds(build_select): + builds = set() + for sel in build_select: + for onesel in sel.split(','): + build = safe_int(onesel) + if build is not None: + builds.add(build) + continue + build_start_end = onesel.split('-') + if len(build_start_end) == 2: + build_start = safe_int(build_start_end[0]) + build_end = safe_int(build_start_end[1]) + if (build_start is not None and build_end is not None): + builds.update(range(build_start, build_end+1)) + return builds def body(self, request): t0 = time.time() @@ -487,14 +547,24 @@ only_recentrevs = request.args.get('recentrev', None) if only_branches is not None: only_branches = map(trunk_value, only_branches) - - branches = self.recentRevisions(status, - only_recentrevs=only_recentrevs, - only_branches=only_branches) + only_builder = request.args.get('builder', None) + only_builds = None + if only_builder is not None: + only_builder = only_builder[-1:] # pick exactly one + page.fixed_builder = True + build_select = request.args.get('builds', None) + if build_select is not None: + only_builds = self._parse_builds(build_select) + + branches = self.recentRuns(status, + only_recentrevs=only_recentrevs, + only_branches=only_branches, + only_builder=only_builder, + only_builds=only_builds) - for branch, (revs, no_revision_builds) in sorted(branches.iteritems()): + for branch, (runs, no_revision_builds) in sorted(branches.iteritems()): outcome_sets = [] - for rev, by_build in revs.items(): + for label, by_build in runs.items(): outcome_sets.append(GatherOutcomeSet(by_build)) branch = trunk_name(branch) page.start_branch(branch) Modified: pypy/build/bot2/pypybuildbot/test/test_summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/test/test_summary.py (original) +++ pypy/build/bot2/pypybuildbot/test/test_summary.py Mon Dec 29 21:52:04 2008 @@ -3,6 +3,7 @@ from buildbot.status import builder as status_builder from pypybuildbot import summary from StringIO import StringIO +import re class TestOutcomes(object): @@ -215,14 +216,14 @@ assert res == [2,3,2] -def test__prune_revs(): - revs = dict(zip(range(100), range(100, 200))) +def test__prune_runs(): + runs = dict(zip(range(100), range(100, 200))) - summary.Summary._prune_revs(revs, 4) + summary.Summary._prune_runs(runs, 4) - assert len(revs) == 4 + assert len(runs) == 4 - assert revs == {99: 199, 98: 198, 97: 197, 96: 196} + assert runs == {99: 199, 98: 198, 97: 197, 96: 196} class _BuilderToStatus(object): @@ -254,12 +255,12 @@ def witness_branches(summary): ref = [None] - recentRevisions = summary.recentRevisions + recentRuns = summary.recentRuns def witness(*args, **kwds): - branches = recentRevisions(*args, **kwds) + branches = recentRuns(*args, **kwds) ref[0] = branches return branches - summary.recentRevisions = witness + summary.recentRuns = witness return lambda: ref[0] @@ -328,7 +329,7 @@ def test_one_build(self): builder = status_builder.BuilderStatus('builder0') - add_builds(builder, [(60000, ". a")]) + add_builds(builder, [(60000, "F a\n. b")]) s = summary.Summary() res = witness_branches(s) @@ -344,13 +345,13 @@ def test_two_builds(self): builder = status_builder.BuilderStatus('builder0') - add_builds(builder, [(60000, ". a"), - (60001, ". a")]) + add_builds(builder, [(60000, "F a\n. b"), + (60001, "F a\n. b")]) s = summary.Summary() res = witness_branches(s) req = FakeRequest([builder]) - s.body(req) + out = s.body(req) branches = res() revs = branches[None][0] @@ -360,12 +361,18 @@ assert outcome.key == ('builder0', 0) outcome = revs[60001]['builder0'] assert outcome.revision == 60001 - assert outcome.key == ('builder0', 1) + assert outcome.key == ('builder0', 1) + + revs = [] + for m in re.finditer(r'recentrev=(\d+)', out): + revs.append(int(m.group(1))) + + assert revs == [60000, 60001] def test_two_builds_samerev(self): builder = status_builder.BuilderStatus('builder0') - add_builds(builder, [(60000, ". a"), - (60000, ". a")]) + add_builds(builder, [(60000, "F a\n. b"), + (60000, "F a\n. b")]) s = summary.Summary() res = witness_branches(s) @@ -381,8 +388,8 @@ def test_two_builds_recentrev(self): builder = status_builder.BuilderStatus('builder0') - add_builds(builder, [(60000, ". a"), - (60001, ". a")]) + add_builds(builder, [(60000, "F a\n. b"), + (60001, "F a\n. b")]) s = summary.Summary() res = witness_branches(s) @@ -397,4 +404,66 @@ assert outcome.revision == 60000 assert outcome.key == ('builder0', 0) - + def test_many_builds_query_builder(self): + builder = status_builder.BuilderStatus('builder0') + add_builds(builder, [(60000, "F a\n. b"), + (60000, ". a\n. b"), + (60001, "F a\n. b")]) + + s = summary.Summary() + res = witness_branches(s) + req = FakeRequest([builder]) + req.args={'builder': ['builder0']} + out = s.body(req) + branches = res() + + runs = branches[None][0] + assert sorted(runs.keys()) == [(60000,0), (60000,1), (60001,2)] + outcome = runs[(60000,0)]['builder0'] + assert outcome.revision == 60000 + assert outcome.key == ('builder0', 0) + outcome = runs[(60000,1)]['builder0'] + assert outcome.revision == 60000 + assert outcome.key == ('builder0', 1) + outcome = runs[(60001,2)]['builder0'] + assert outcome.revision == 60001 + assert outcome.key == ('builder0', 2) + + runs = [] + for m in re.finditer(r'builder=(\w+)&builds=(\d+)', out): + runs.append((m.group(1), int(m.group(2)))) + + assert runs == [('builder0', 0), + ('builder0', 1), + ('builder0', 2)] + + + def test_many_builds_query_builder_builds(self): + builder = status_builder.BuilderStatus('builder0') + add_builds(builder, [(60000, "F a\n. b"), + (60000, ". a\n. b"), + (60001, "F a\n. b")]) + + s = summary.Summary() + res = witness_branches(s) + req = FakeRequest([builder]) + req.args={'builder': ['builder0'], + 'builds': ['0','2-2', '7']} + out = s.body(req) + branches = res() + + runs = branches[None][0] + assert sorted(runs.keys()) == [(60000,0), (60001,2)] + outcome = runs[(60000,0)]['builder0'] + assert outcome.revision == 60000 + assert outcome.key == ('builder0', 0) + outcome = runs[(60001,2)]['builder0'] + assert outcome.revision == 60001 + assert outcome.key == ('builder0', 2) + + runs = [] + for m in re.finditer(r'builder=(\w+)&builds=(\d+)', out): + runs.append((m.group(1), int(m.group(2)))) + + assert runs == [('builder0', 0), + ('builder0', 2)] From pedronis at codespeak.net Tue Dec 30 18:13:28 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 30 Dec 2008 18:13:28 +0100 (CET) Subject: [pypy-svn] r60739 - in pypy/build/bot2/pypybuildbot: . test Message-ID: <20081230171328.CE8D61684DD@codespeak.net> Author: pedronis Date: Tue Dec 30 18:13:26 2008 New Revision: 60739 Modified: pypy/build/bot2/pypybuildbot/summary.py pypy/build/bot2/pypybuildbot/test/test_summary.py Log: - support multiple pytest log step for a builder - have the run links go to the build pages as some people requested, from there is possible to go to the various logs - links to builder drill down to drill page - on builder drill-down pages show also the elapsed time of pytest runs Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Tue Dec 30 18:13:26 2008 @@ -1,4 +1,4 @@ -import urllib, time +import time, urlparse, urllib import py html = py.xml.html @@ -6,16 +6,33 @@ from buildbot.status.web.base import HtmlResource from buildbot.status.builder import FAILURE, EXCEPTION +def host_agnostic(url): + parts = urlparse.urlsplit(url) + return urlparse.urlunsplit(('','')+parts[2:]) + +def show_elapsed(secs): + if secs < 5: + return "%.02fs" % secs + secs = int(round(secs)) + if secs < 60: + return "%ds" % secs + if secs < 5*60: + return "%dm%d" % (secs/60, secs%60) + mins = int(round(secs/60.)) + if mins < 60: + return "%dm" % mins + return "%dh%d" % (mins/60, mins%60) + class RevisionOutcomeSet(object): - def __init__(self, rev, key=None, run_stdio=None): + def __init__(self, rev, key=None, run_info=None): self.revision = rev self.key = key self._outcomes = {} self.failed = set() self.skipped = set() self.longreprs = {} - self._run_stdio = run_stdio + self._run_info = run_info def populate_one(self, name, shortrepr, longrepr=None): if shortrepr == '!': @@ -68,8 +85,8 @@ def get_key_namekey(self, namekey): return (self.key, namekey) - def get_run_stdios(self): - return {self.key: (self, self._run_stdio)} + def get_run_infos(self): + return {self.key: (self, self._run_info)} class RevisionOutcomeSetCache(object): @@ -96,40 +113,42 @@ builderName, buildNumber = key builderStatus = status.getBuilder(builderName) build = builderStatus.getBuild(buildNumber) + run_url = status.getURLForThing(build) rev = int(build.getProperty("got_revision")) - pytest_log = None - stdio_log = None + pytest_logs = [] + pytest_elapsed = 0 failure = None - aborted = False for step in build.getSteps(): logs = dict((log.getName(), log) for log in step.getLogs()) if 'pytestLog' in logs: - stdio_log = logs['stdio'] - if 'aborted' in step.getText(): - aborted = True - pytest_log = logs['pytestLog'] - break - elif (stdio_log is None and + aborted = 'aborted' in step.getText() + pytest_logs.append((step.getName(), logs['pytestLog'], aborted)) + ts = step.getTimes() + if ts[0] is not None and ts[1] is not None: + pytest_elapsed += ts[1]-ts[0] + elif (failure is None and step.getResults()[0] in (FAILURE, EXCEPTION)): failure = ' '.join(step.getText()) - stdio_log = logs.get('stdio') - if stdio_log is None: - stdio_url = "no_log" - else: - stdio_url = status.getURLForThing(stdio_log) - # builbot is broken in this :( - stdio_url = stdio_url[:-1]+"stdio" - - outcome_set = RevisionOutcomeSet(rev, key, stdio_url) - if pytest_log is None or not pytest_log.hasContents(): - name = failure or '' - outcome_set.populate_one(name, '!', "no log from the test run") - else: - if aborted: - outcome_set.populate_one(' aborted', '!', "") - outcome_set.populate(pytest_log) + run_info = {'URL': run_url, 'elapsed': pytest_elapsed or None} + outcome_set = RevisionOutcomeSet(rev, key, run_info) + someresult = False + if pytest_logs: + for stepName, resultLog, aborted in pytest_logs: + if resultLog.hasContents(): + someresult = True + if aborted: + outcome_set.populate_one(stepName+' aborted', '!', "") + outcome_set.populate(resultLog) + + if not someresult: + if failure: + name = '"%s"' % failure # quote + else: + name = '' + outcome_set.populate_one(name, '!', "no logs from the test run") + return outcome_set def get(self, status, key): @@ -203,10 +222,10 @@ def get_key_namekey(self, namekey): return self.map[namekey[0]].get_key_namekey(namekey[1:]) - def get_run_stdios(self): + def get_run_infos(self): all = {} for outcome_set in self.map.itervalues(): - all.update(outcome_set.get_run_stdios()) + all.update(outcome_set.get_run_infos()) return all # ________________________________________________________________ @@ -228,10 +247,11 @@ class SummaryPage(object): - def __init__(self): + def __init__(self, status): self.sections = [] self.cur_branch=None self.fixed_builder = False + self.status = status def make_longrepr_url_for(self, outcome_set, namekey): cachekey, namekey = outcome_set.get_key_namekey(namekey) @@ -244,24 +264,39 @@ qs = urllib.urlencode(parms) return "/summary/longrepr?" + qs - def make_stdio_anchors_for(self, outcome_set): + def make_run_anchors_for(self, outcome_set): anchors = [] - stdios = sorted(outcome_set.get_run_stdios().items()) - for cachekey, (run, url) in stdios: + infos = sorted(outcome_set.get_run_infos().items()) + for cachekey, (run, info) in infos: builder = cachekey[0] anchors.append(' ') - text = "%s [%d, %d F, %d s]" % (builder, + timing = "" + if self.fixed_builder and info['elapsed'] is not None: + timing = " in %s" % show_elapsed(info['elapsed']) + text = "%s [%d, %d F, %d s%s]" % (builder, run.numpassed, len(run.failed), - len(run.skipped)) - anchors.append(html.a(text, href=url)) + len(run.skipped), + timing) + anchors.append(html.a(text, href=host_agnostic(info['URL']))) return anchors def start_branch(self, branch): self.cur_branch = branch - branch_anchor = html.a(branch, href="/summary?branch=%s" % branch) + branch_anchor = html.a(branch, href="/summary?branch=%s" % branch, + class_="failSummary branch") self.sections.append(html.h2(branch_anchor)) + def _builder_anchor(self, builder): + if self.fixed_builder: + url = self.status.getURLForThing(self.status.getBuilder(builder)) + cls = "builder" + else: + url = "/summary?builder=%s" % builder + cls = "builderquery" + return html.a(builder, href=host_agnostic(url), + class_=' '.join(["failSummary", cls])) + def _builder_num(self, outcome_set): return outcome_set.map.values()[0].key @@ -287,9 +322,10 @@ def add_section(self, outcome_sets): if not outcome_sets: return - labels = sorted(self._label(outcome_set) for outcome_set in outcome_sets) - by_label = sorted((self._label(outcome_set), outcome_set) for outcome_set - in outcome_sets) + labels = sorted(self._label(outcome_set) + for outcome_set in outcome_sets) + by_label = sorted((self._label(outcome_set), outcome_set) + for outcome_set in outcome_sets) lines = [] align = 2*len(labels)-1+len(str(labels[-1])) @@ -300,7 +336,7 @@ count_skipped = len(outcome_set.skipped) line = [bars(), ' ', self._label_anchor(outcome_set)] line.append((align-len(line[0]))*" ") - line.append(self.make_stdio_anchors_for(outcome_set)) + line.append(self.make_run_anchors_for(outcome_set)) line.append('\n') lines.append(line) lines.append([bars(), "\n"]) @@ -339,7 +375,14 @@ class_="failSummary failed")]) else: line.append(" %s" % letter) - for width, key in zip(colwidths, failure): + # builder + builder_width = colwidths[0] + builder = failure[0] + spacing = (" %-*s" % (builder_width, 'x'*len(builder))).rstrip('x') + builder_anchor = self._builder_anchor(builder) + line.append([spacing, builder_anchor]) + + for width, key in zip(colwidths[1:], failure[1:]): line.append(" %-*s" % (width, key)) lines.append(line) lines.append("\n") @@ -356,7 +399,7 @@ num = build.getNumber() descr = "%s #%d" % (builderName, num) url = status.getURLForThing(build) - section.append(html.a(descr, href=url)) + section.append(html.a(descr, href=host_agnostic(url))) section.append(html.br()) self.sections.append(section) @@ -540,7 +583,7 @@ status = self.getStatus(request) - page = SummaryPage() + page = SummaryPage(status) #page.sections.append(repr(request.args)) only_branches = request.args.get('branch', None) Modified: pypy/build/bot2/pypybuildbot/test/test_summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/test/test_summary.py (original) +++ pypy/build/bot2/pypybuildbot/test/test_summary.py Tue Dec 30 18:13:26 2008 @@ -225,6 +225,30 @@ assert runs == {99: 199, 98: 198, 97: 197, 96: 196} +def test_show_elapsed(): + res = summary.show_elapsed(0.25) + assert res == "0.25s" + res = summary.show_elapsed(1.0) + assert res == "1.00s" + res = summary.show_elapsed(1.25) + assert res == "1.25s" + res = summary.show_elapsed(4.5) + assert res == "4.50s" + res = summary.show_elapsed(5.25) + assert res == "5s" + res = summary.show_elapsed(5.5) + assert res == "6s" + res = summary.show_elapsed(2*60+30) + assert res == "2m30" + res = summary.show_elapsed(4*60+30) + assert res == "4m30" + res = summary.show_elapsed(5*60+30) + assert res == "6m" + res = summary.show_elapsed(61*60) + assert res == "1h1" + res = summary.show_elapsed(90*60) + assert res == "1h30" + class _BuilderToStatus(object): def __init__(self, status): @@ -286,12 +310,16 @@ def add_builds(builder, builds): n = getattr(builder, 'nextBuildNumber', 0) + t = 1000 for rev, reslog in builds: build = status_builder.BuildStatus(builder, n) build.setProperty('got_revision', str(rev), None) step = build.addStepWithName('pytest') step.logs.extend([FakeLog(step, 'pytestLog', reslog), FakeLog(step, 'stdio')]) + step.started = t + step.finished = t + (n+1)*60 + t = step.finished + 30 build.buildFinished() builder.addBuildToCache(build) n += 1 @@ -322,19 +350,64 @@ s = summary.Summary() res = witness_branches(s) req = FakeRequest([builder]) - s.body(req) + out = s.body(req) branches = res() assert branches == {None: ({}, [build])} + def test_one_build_no_logs(self): + builder = status_builder.BuilderStatus('builder0') + build = status_builder.BuildStatus(builder, 0) + build.setProperty('got_revision', '50000', None) + build.buildFinished() + builder.addBuildToCache(build) + builder.nextBuildNumber = len(builder.buildCache) + + s = summary.Summary() + res = witness_branches(s) + req = FakeRequest([builder]) + out = s.body(req) + branches = res() + + revs = branches[None][0] + assert revs.keys() == [50000] + + assert '<run>' in out + + def test_one_build_no_logs_failure(self): + builder = status_builder.BuilderStatus('builder0') + build = status_builder.BuildStatus(builder, 0) + build.setProperty('got_revision', '50000', None) + step = build.addStepWithName('step') + step.setText(['step', 'borken']) + step.stepFinished(summary.FAILURE) + step1 = build.addStepWithName('other') + step1.setText(['other', 'borken']) + step1.stepFinished(summary.FAILURE) + build.buildFinished() + builder.addBuildToCache(build) + builder.nextBuildNumber = len(builder.buildCache) + + s = summary.Summary() + res = witness_branches(s) + req = FakeRequest([builder]) + out = s.body(req) + branches = res() + + revs = branches[None][0] + assert revs.keys() == [50000] + + assert 'step borken' in out + assert 'other borken' not in out + def test_one_build(self): builder = status_builder.BuilderStatus('builder0') - add_builds(builder, [(60000, "F a\n. b")]) + add_builds(builder, [(60000, "F TEST1\n. b")]) s = summary.Summary() res = witness_branches(s) req = FakeRequest([builder]) - s.body(req) + out = s.body(req) branches = res() revs = branches[None][0] @@ -343,10 +416,12 @@ assert outcome.revision == 60000 assert outcome.key == ('builder0', 0) + assert 'TEST1' in out + def test_two_builds(self): builder = status_builder.BuilderStatus('builder0') - add_builds(builder, [(60000, "F a\n. b"), - (60001, "F a\n. b")]) + add_builds(builder, [(60000, "F TEST1\n. b"), + (60001, "F TEST1\n. b")]) s = summary.Summary() res = witness_branches(s) @@ -369,10 +444,12 @@ assert revs == [60000, 60001] + assert 'TEST1' in out + def test_two_builds_samerev(self): builder = status_builder.BuilderStatus('builder0') - add_builds(builder, [(60000, "F a\n. b"), - (60000, "F a\n. b")]) + add_builds(builder, [(60000, "F TEST1\n. b"), + (60000, "F TEST1\n. b")]) s = summary.Summary() res = witness_branches(s) @@ -386,16 +463,18 @@ assert outcome.revision == 60000 assert outcome.key == ('builder0', 1) + assert 'TEST1' in out + def test_two_builds_recentrev(self): builder = status_builder.BuilderStatus('builder0') - add_builds(builder, [(60000, "F a\n. b"), - (60001, "F a\n. b")]) + add_builds(builder, [(60000, "F TEST1\n. b"), + (60001, "F TEST1\n. b")]) s = summary.Summary() res = witness_branches(s) req = FakeRequest([builder]) req.args = {'recentrev': ['60000']} - s.body(req) + out = s.body(req) branches = res() revs = branches[None][0] @@ -404,11 +483,13 @@ assert outcome.revision == 60000 assert outcome.key == ('builder0', 0) + assert 'TEST1' in out + def test_many_builds_query_builder(self): builder = status_builder.BuilderStatus('builder0') - add_builds(builder, [(60000, "F a\n. b"), + add_builds(builder, [(60000, "F TEST1\n. b"), (60000, ". a\n. b"), - (60001, "F a\n. b")]) + (60001, "F TEST1\n. b")]) s = summary.Summary() res = witness_branches(s) @@ -437,12 +518,14 @@ ('builder0', 1), ('builder0', 2)] + assert 'TEST1' in out + def test_many_builds_query_builder_builds(self): builder = status_builder.BuilderStatus('builder0') - add_builds(builder, [(60000, "F a\n. b"), + add_builds(builder, [(60000, "F TEST1\n. b"), (60000, ". a\n. b"), - (60001, "F a\n. b")]) + (60001, "F TEST1\n. b")]) s = summary.Summary() res = witness_branches(s) @@ -467,3 +550,27 @@ assert runs == [('builder0', 0), ('builder0', 2)] + + assert 'TEST1' in out + + def test_many_pytestLogs(self): + builder = status_builder.BuilderStatus('builder1') + build = status_builder.BuildStatus(builder, 0) + build.setProperty('got_revision', '70000', None) + step = build.addStepWithName('pytest') + step.logs.extend([FakeLog(step, 'pytestLog', "F TEST1")]) + step2 = build.addStepWithName('pytest2') + step2.logs.extend([FakeLog(step, 'pytestLog', ". x\nF TEST2")]) + step2.setText(["pytest2", "aborted"]) + build.buildFinished() + builder.addBuildToCache(build) + builder.nextBuildNumber = 1 + + s = summary.Summary() + req = FakeRequest([builder]) + out = s.body(req) + + assert 'TEST1' in out + assert 'TEST2' in out + assert 'pytest aborted' not in out + assert 'pytest2 aborted' in out From pedronis at codespeak.net Tue Dec 30 18:22:49 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 30 Dec 2008 18:22:49 +0100 (CET) Subject: [pypy-svn] r60740 - pypy/build/bot2/pypybuildbot Message-ID: <20081230172249.82E81168480@codespeak.net> Author: pedronis Date: Tue Dec 30 18:22:49 2008 New Revision: 60740 Modified: pypy/build/bot2/pypybuildbot/summary.py Log: fix the hack Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Tue Dec 30 18:22:49 2008 @@ -378,9 +378,11 @@ # builder builder_width = colwidths[0] builder = failure[0] - spacing = (" %-*s" % (builder_width, 'x'*len(builder))).rstrip('x') + spacing = (" %-*s" % (builder_width, 'x'*len(builder))).split('x') + spaceleft = spacing[0] + spaceright = spacing[1] builder_anchor = self._builder_anchor(builder) - line.append([spacing, builder_anchor]) + line.append([spaceleft, builder_anchor, spaceright]) for width, key in zip(colwidths[1:], failure[1:]): line.append(" %-*s" % (width, key)) From pedronis at codespeak.net Tue Dec 30 18:25:59 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 30 Dec 2008 18:25:59 +0100 (CET) Subject: [pypy-svn] r60741 - pypy/build/bot2/pypybuildbot Message-ID: <20081230172559.3DC2F168480@codespeak.net> Author: pedronis Date: Tue Dec 30 18:25:58 2008 New Revision: 60741 Modified: pypy/build/bot2/pypybuildbot/summary.py Log: fix the hack, part 2 Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Tue Dec 30 18:25:58 2008 @@ -380,7 +380,7 @@ builder = failure[0] spacing = (" %-*s" % (builder_width, 'x'*len(builder))).split('x') spaceleft = spacing[0] - spaceright = spacing[1] + spaceright = spacing[-1] builder_anchor = self._builder_anchor(builder) line.append([spaceleft, builder_anchor, spaceright]) From pedronis at codespeak.net Tue Dec 30 18:46:39 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 30 Dec 2008 18:46:39 +0100 (CET) Subject: [pypy-svn] r60742 - pypy/build/bot2/codespeak-html Message-ID: <20081230174639.896451684DF@codespeak.net> Author: pedronis Date: Tue Dec 30 18:46:35 2008 New Revision: 60742 Modified: pypy/build/bot2/codespeak-html/buildbot.css Log: some css to go with the last tweaks Modified: pypy/build/bot2/codespeak-html/buildbot.css ============================================================================== --- pypy/build/bot2/codespeak-html/buildbot.css (original) +++ pypy/build/bot2/codespeak-html/buildbot.css Tue Dec 30 18:46:35 2008 @@ -92,3 +92,15 @@ color: #E0B000; font-weight: bold; } + +a.failSummary.branch, a:visited.failSummary.branch { + color: black; +} + +a.failSummary.builderquery, a:visited.failSummary.builderquery { + color: black; +} + +a.failSummary.builder, a:visited.failSummary.builder { + color: grey; +}