[pypy-svn] r32004 - in pypy/dist/pypy: annotation objspace/std rpython rpython/lltypesystem rpython/ootypesystem translator translator/test
arigo at codespeak.net
arigo at codespeak.net
Tue Sep 5 13:25:02 CEST 2006
Author: arigo
Date: Tue Sep 5 13:24:57 2006
New Revision: 32004
Modified:
pypy/dist/pypy/annotation/listdef.py
pypy/dist/pypy/annotation/unaryop.py
pypy/dist/pypy/objspace/std/setobject.py
pypy/dist/pypy/rpython/lltypesystem/rlist.py
pypy/dist/pypy/rpython/ootypesystem/rlist.py
pypy/dist/pypy/rpython/rmodel.py
pypy/dist/pypy/rpython/rtyper.py
pypy/dist/pypy/translator/geninterplevel.py
pypy/dist/pypy/translator/simplify.py
pypy/dist/pypy/translator/test/test_simplify.py
pypy/dist/pypy/translator/translator.py
Log:
Finished the list comprehension optimization. Disabled by default
for now, until we know a bit more if it really gives any benefit
and/or what kind of subtle bugs it contains.
Modified: pypy/dist/pypy/annotation/listdef.py
==============================================================================
--- pypy/dist/pypy/annotation/listdef.py (original)
+++ pypy/dist/pypy/annotation/listdef.py Tue Sep 5 13:24:57 2006
@@ -170,7 +170,9 @@
self.listitem.generalize(s_value)
def __repr__(self):
- return '<[%r]>' % (self.listitem.s_value,)
+ return '<[%r]%s%s>' % (self.listitem.s_value,
+ self.listitem.mutated and 'm' or '',
+ self.listitem.resized and 'r' or '')
def mutate(self):
self.listitem.mutate()
Modified: pypy/dist/pypy/annotation/unaryop.py
==============================================================================
--- pypy/dist/pypy/annotation/unaryop.py (original)
+++ pypy/dist/pypy/annotation/unaryop.py Tue Sep 5 13:24:57 2006
@@ -24,7 +24,7 @@
'iter', 'next', 'invert', 'type', 'issubtype',
'pos', 'neg', 'nonzero', 'abs', 'hex', 'oct',
'ord', 'int', 'float', 'long', 'id',
- 'neg_ovf', 'abs_ovf'])
+ 'neg_ovf', 'abs_ovf', 'hint'])
for opname in UNARY_OPERATIONS:
missing_operation(SomeObject, opname)
@@ -177,6 +177,9 @@
def op_contains(obj, s_element):
return SomeBool()
+ def hint(self, *args_s):
+ return self
+
class __extend__(SomeFloat):
def pos(flt):
@@ -314,6 +317,20 @@
lst.listdef.generalize(s_element)
return SomeBool()
+ def hint(lst, *args_s):
+ hints = args_s[-1].const
+ if 'maxlength' in hints:
+ # only for iteration over lists or dicts at the moment,
+ # not over an iterator object (because it has no known length)
+ s_iterable = args_s[0]
+ if isinstance(s_iterable, (SomeList, SomeDict)):
+ lst.listdef.resize()
+ lst.listdef.listitem.hint_maxlength = True
+ elif 'fence' in hints:
+ lst = lst.listdef.offspring()
+ return lst
+
+
class __extend__(SomeDict):
def len(dct):
Modified: pypy/dist/pypy/objspace/std/setobject.py
==============================================================================
--- pypy/dist/pypy/objspace/std/setobject.py (original)
+++ pypy/dist/pypy/objspace/std/setobject.py Tue Sep 5 13:24:57 2006
@@ -116,7 +116,7 @@
def _is_eq(w_left, w_right):
if len(w_left.setdata) != len(w_right.setdata):
return False
- for w_key in w_left.setdata.iterkeys():
+ for w_key in w_left.setdata:
if w_key not in w_right.setdata:
return False
return True
@@ -135,7 +135,7 @@
else:
ld = ldict.copy()
del_list_w = []
- for w_key in ld.iterkeys():
+ for w_key in ld:
if w_key in rdict:
del_list_w.append(w_key)
for w_key in del_list_w:
@@ -149,7 +149,7 @@
else:
ld = ldict.copy()
del_list_w = []
- for w_key in ld.iterkeys():
+ for w_key in ld:
if w_key not in rdict:
del_list_w.append(w_key)
@@ -166,11 +166,11 @@
ld = ldict.copy()
del_list_w = []
add_list_w = []
- for w_key in ld.iterkeys():
+ for w_key in ld:
if w_key in rdict:
del_list_w.append(w_key)
- for w_key in rdict.iterkeys():
+ for w_key in rdict:
if w_key not in ld:
add_list_w.append(w_key)
@@ -387,7 +387,7 @@
return space.wrap(w_set.hash)
hash = 1927868237
hash *= (len(w_set.setdata) + 1)
- for w_item in w_set.setdata.iterkeys():
+ for w_item in w_set.setdata:
h = space.hash_w(w_item)
value = ((h ^ (h << 16) ^ 89869747) * multi)
hash = intmask(hash ^ value)
Modified: pypy/dist/pypy/rpython/lltypesystem/rlist.py
==============================================================================
--- pypy/dist/pypy/rpython/lltypesystem/rlist.py (original)
+++ pypy/dist/pypy/rpython/lltypesystem/rlist.py Tue Sep 5 13:24:57 2006
@@ -81,6 +81,20 @@
temp),
rstr.list_str_close_bracket))
+ def get_itemarray_lowleveltype(self):
+ ITEM = self.item_repr.lowleveltype
+ ITEMARRAY = GcArray(ITEM,
+ adtmeths = ADTIFixedList({
+ "ll_newlist": ll_fixed_newlist,
+ "ll_length": ll_fixed_length,
+ "ll_items": ll_fixed_items,
+ ##"list_builder": self.list_builder,
+ "ITEM": ITEM,
+ "ll_getitem_fast": ll_fixed_getitem_fast,
+ "ll_setitem_fast": ll_fixed_setitem_fast,
+ }))
+ return ITEMARRAY
+
##class ListBuilder(object):
## """Interface to allow lazy list building by the JIT."""
@@ -179,7 +193,7 @@
self.external_item_repr, self.item_repr = externalvsinternal(self.rtyper, self._item_repr_computer())
if isinstance(self.LIST, GcForwardReference):
ITEM = self.item_repr.lowleveltype
- ITEMARRAY = GcArray(ITEM)
+ ITEMARRAY = self.get_itemarray_lowleveltype()
# XXX we might think of turning length stuff into Unsigned
self.LIST.become(GcStruct("list", ("length", Signed),
("items", Ptr(ITEMARRAY)),
@@ -206,6 +220,41 @@
result.items = malloc(self.LIST.items.TO, n)
return result
+ def rtype_method_append(self, hop):
+ if getattr(self.listitem, 'hint_maxlength', False):
+ v_lst, v_value = hop.inputargs(self, self.item_repr)
+ hop.exception_cannot_occur()
+ hop.gendirectcall(ll_append_noresize, v_lst, v_value)
+ else:
+ AbstractListRepr.rtype_method_append(self, hop)
+
+ def rtype_hint(self, hop):
+ optimized = getattr(self.listitem, 'hint_maxlength', False)
+ hints = hop.args_s[-1].const
+ if 'maxlength' in hints:
+ if optimized:
+ s_iterable = hop.args_s[1]
+ r_iterable = hop.args_r[1]
+ v_list = hop.inputarg(self, arg=0)
+ v_iterable = hop.inputarg(r_iterable, arg=1)
+ hop2 = hop.copy()
+ while hop2.nb_args > 0:
+ hop2.r_s_popfirstarg()
+ hop2.v_s_insertfirstarg(v_iterable, s_iterable)
+ v_maxlength = r_iterable.rtype_len(hop2)
+ hop.llops.gendirectcall(ll_set_maxlength, v_list, v_maxlength)
+ return v_list
+ if 'fence' in hints:
+ v_list = hop.inputarg(self, arg=0)
+ if isinstance(hop.r_result, FixedSizeListRepr):
+ if optimized and 'exactlength' in hints:
+ llfn = ll_list2fixed_exact
+ else:
+ llfn = ll_list2fixed
+ v_list = hop.llops.gendirectcall(llfn, v_list)
+ return v_list
+ return AbstractListRepr.rtype_hint(self, hop)
+
class FixedSizeListRepr(AbstractFixedSizeListRepr, BaseListRepr):
@@ -214,17 +263,7 @@
self.external_item_repr, self.item_repr = externalvsinternal(self.rtyper, self._item_repr_computer())
if isinstance(self.LIST, GcForwardReference):
ITEM = self.item_repr.lowleveltype
- ITEMARRAY = GcArray(ITEM,
- adtmeths = ADTIFixedList({
- "ll_newlist": ll_fixed_newlist,
- "ll_length": ll_fixed_length,
- "ll_items": ll_fixed_items,
- ##"list_builder": self.list_builder,
- "ITEM": ITEM,
- "ll_getitem_fast": ll_fixed_getitem_fast,
- "ll_setitem_fast": ll_fixed_setitem_fast,
- }))
-
+ ITEMARRAY = self.get_itemarray_lowleveltype()
self.LIST.become(ITEMARRAY)
def compact_repr(self):
@@ -317,6 +356,12 @@
_ll_list_resize_really(l, newsize)
+def ll_append_noresize(l, newitem):
+ length = l.length
+ l.length = length + 1
+ l.ll_setitem_fast(length, newitem)
+ll_append_noresize.oopspec = 'list.append(l, newitem)'
+
TEMP = GcArray(Ptr(rstr.STR))
@@ -378,6 +423,25 @@
llops.gendirectcall(ll_setitem_nonneg, v_func, v_result, ci, v_item)
return v_result
+# special operations for list comprehension optimization
+def ll_set_maxlength(l, n):
+ LIST = typeOf(l).TO
+ l.items = malloc(LIST.items.TO, n)
+
+def ll_list2fixed(l):
+ n = l.length
+ olditems = l.items
+ if n == len(olditems):
+ return olditems
+ else:
+ LIST = typeOf(l).TO
+ newitems = malloc(LIST.items.TO, n)
+ for i in range(n):
+ newitems[i] = olditems[i]
+ return newitems
+
+def ll_list2fixed_exact(l):
+ return l.items
# ____________________________________________________________
#
Modified: pypy/dist/pypy/rpython/ootypesystem/rlist.py
==============================================================================
--- pypy/dist/pypy/rpython/ootypesystem/rlist.py (original)
+++ pypy/dist/pypy/rpython/ootypesystem/rlist.py Tue Sep 5 13:24:57 2006
@@ -76,6 +76,16 @@
buf.ll_append_char(']')
return buf.ll_build()
+ def rtype_hint(self, hop):
+ hints = hop.args_s[-1].const
+ if 'maxlength' in hints:
+ v_list = hop.inputarg(self, arg=0)
+ # XXX give a hint to pre-allocate the list (see lltypesystem/rlist)
+ return v_list
+ if 'fence' in hints:
+ return hop.inputarg(self, arg=0)
+ return AbstractBaseListRepr.rtype_hint(self, hop)
+
class ListRepr(AbstractListRepr, BaseListRepr):
Modified: pypy/dist/pypy/rpython/rmodel.py
==============================================================================
--- pypy/dist/pypy/rpython/rmodel.py (original)
+++ pypy/dist/pypy/rpython/rmodel.py Tue Sep 5 13:24:57 2006
@@ -232,6 +232,9 @@
def make_iterator_repr(self, *variant):
raise TyperError("%s is not iterable" % (self,))
+ def rtype_hint(self, hop):
+ return hop.inputarg(hop.r_result, arg=0)
+
# hlinvoke helpers
def get_r_implfunc(self):
Modified: pypy/dist/pypy/rpython/rtyper.py
==============================================================================
--- pypy/dist/pypy/rpython/rtyper.py (original)
+++ pypy/dist/pypy/rpython/rtyper.py Tue Sep 5 13:24:57 2006
@@ -551,13 +551,13 @@
# __________ irregular operations __________
def translate_op_newlist(self, hop):
- return self.type_system.rlist.rtype_newlist(hop)
+ return rlist.rtype_newlist(hop)
def translate_op_newdict(self, hop):
return self.type_system.rdict.rtype_newdict(hop)
def translate_op_alloc_and_set(self, hop):
- return self.type_system.rlist.rtype_alloc_and_set(hop)
+ return rlist.rtype_alloc_and_set(hop)
def translate_op_newtuple(self, hop):
return self.type_system.rtuple.rtype_newtuple(hop)
@@ -903,7 +903,7 @@
from pypy.rpython import robject
from pypy.rpython import rint, rbool, rfloat
from pypy.rpython import rslice, rrange
-from pypy.rpython import rstr, rdict
+from pypy.rpython import rstr, rdict, rlist
from pypy.rpython import rclass, rbuiltin, rpbc, rspecialcase
from pypy.rpython import rexternalobj
from pypy.rpython import rptr
Modified: pypy/dist/pypy/translator/geninterplevel.py
==============================================================================
--- pypy/dist/pypy/translator/geninterplevel.py (original)
+++ pypy/dist/pypy/translator/geninterplevel.py Tue Sep 5 13:24:57 2006
@@ -282,6 +282,9 @@
"shape": repr(shape),
"data_w": self.arglist(op.args[2:], localscope),
'Arg': self.nameof(Arguments) }
+ if op.opname == "hint":
+ return "%s = %s" % (self.expr(op.result, localscope),
+ self.expr(op.args[0], localscope))
if op.opname in self.has_listarg:
fmt = "%s = %s([%s])"
else:
@@ -1486,7 +1489,8 @@
entrypoint = dic
t = TranslationContext(verbose=False, simplifying=needed_passes,
do_imports_immediately=do_imports_immediately,
- builtins_can_raise_exceptions=True)
+ builtins_can_raise_exceptions=True,
+ list_comprehension_operations=False)
gen = GenRpy(t, entrypoint, modname, dic)
finally:
Modified: pypy/dist/pypy/translator/simplify.py
==============================================================================
--- pypy/dist/pypy/translator/simplify.py (original)
+++ pypy/dist/pypy/translator/simplify.py Tue Sep 5 13:24:57 2006
@@ -710,15 +710,16 @@
# ____________________________________________________________
def detect_list_comprehension(graph):
- """Look for the pattern: Replace it with marker operations:
+ """Look for the pattern: Replace it with marker operations:
- v1 = newlist() v0 = newlistbuilder(length)
- loop start loop start
- ... ...
- exactly one append per loop v0.append(..)
- and nothing else done with v1
- ... ...
- loop end v1 = listbuilder_done(v0)
+ v0 = newlist()
+ v2 = newlist() v1 = hint(v0, iterable, {'maxlength'})
+ loop start loop start
+ ... ...
+ exactly one append per loop v1.append(..)
+ and nothing else done with v2
+ ... ...
+ loop end v2 = hint(v1, {'fence'})
"""
# NB. this assumes RPythonicity: we can only iterate over something
# that has a len(), and this len() cannot change as long as we are
@@ -781,12 +782,17 @@
return
detector = ListComprehensionDetector(graph, loops, newlist_v,
variable_families)
+ graphmutated = False
for location in append_v:
+ if graphmutated:
+ # new variables introduced, must restart the whole process
+ return detect_list_comprehension(graph)
try:
detector.run(*location)
except DetectorFailed:
pass
-
+ else:
+ graphmutated = True
class DetectorFailed(Exception):
pass
@@ -798,6 +804,7 @@
self.loops = loops
self.newlist_v = newlist_v
self.variable_families = variable_families
+ self.reachable_cache = {}
def enum_blocks_from(self, fromblock, avoid):
found = {avoid: True}
@@ -811,7 +818,7 @@
for exit in block.exits:
pending.append(exit.target)
- def enum_reachable_blocks(self, fromblock, stop_at):
+ def enum_reachable_blocks(self, fromblock, stop_at, stay_within=None):
if fromblock is stop_at:
return
found = {stop_at: True}
@@ -822,15 +829,35 @@
continue
found[block] = True
for exit in block.exits:
- yield exit.target
- pending.append(exit.target)
+ if stay_within is None or exit.target in stay_within:
+ yield exit.target
+ pending.append(exit.target)
+
+ def reachable_within(self, fromblock, toblock, avoid, stay_within):
+ if toblock is avoid:
+ return False
+ for block in self.enum_reachable_blocks(fromblock, avoid, stay_within):
+ if block is toblock:
+ return True
+ return False
def reachable(self, fromblock, toblock, avoid):
if toblock is avoid:
return False
+ try:
+ return self.reachable_cache[fromblock, toblock, avoid]
+ except KeyError:
+ pass
+ future = [fromblock]
for block in self.enum_reachable_blocks(fromblock, avoid):
+ self.reachable_cache[fromblock, block, avoid] = True
if block is toblock:
return True
+ future.append(block)
+ # 'toblock' is unreachable from 'fromblock', so it is also
+ # unreachable from any of the 'future' blocks
+ for block in future:
+ self.reachable_cache[block, toblock, avoid] = False
return False
def vlist_alive(self, block):
@@ -965,29 +992,21 @@
raise DetectorFailed # no suitable loop
# Found a suitable loop, let's patch the graph:
-
- # - remove newlist()
- for op in newlistblock.operations:
- if op.opname == 'newlist':
- res = self.variable_families.find_rep(op.result)
- if res is self.vlistfamily:
- newlistblock.operations.remove(op)
- break
- else:
- raise AssertionError("bad newlistblock")
-
- # - remove the vlist variable from all blocks of the loop header,
- # up to the iterblock
- for block in loopheader:
- if block is not newlistblock:
- self.remove_vlist(block.inputargs)
- if block is not iterblock:
- for link in block.exits:
- self.remove_vlist(link.args)
-
- # - add a newlistbuilder() in the iterblock, where we can compute
- # the known maximum length
- vlist = self.contains_vlist(iterblock.exits[0].args)
+ assert iterblock not in loopbody
+ assert loopnextblock in loopbody
+ assert stopblock not in loopbody
+
+ # at StopIteration, the new list is exactly of the same length as
+ # the one we iterate over if it's not possible to skip the appendblock
+ # in the body:
+ exactlength = not self.reachable_within(loopnextblock, loopnextblock,
+ avoid = appendblock,
+ stay_within = loopbody)
+
+ # - add a hint(vlist, iterable, {'maxlength'}) in the iterblock,
+ # where we can compute the known maximum length
+ link = iterblock.exits[0]
+ vlist = self.contains_vlist(link.args)
assert vlist
for op in iterblock.operations:
res = self.variable_families.find_rep(op.result)
@@ -996,11 +1015,16 @@
else:
raise AssertionError("lost 'iter' operation")
vlength = Variable('maxlength')
+ vlist2 = Variable(vlist)
+ chint = Constant({'maxlength': True})
iterblock.operations += [
- SpaceOperation('len', [op.args[0]], vlength),
- SpaceOperation('newlistbuilder', [vlength], vlist)]
+ SpaceOperation('hint', [vlist, op.args[0], chint], vlist2)]
+ link.args = list(link.args)
+ for i in range(len(link.args)):
+ if link.args[i] is vlist:
+ link.args[i] = vlist2
- # - wherever the list exits the loop body, add a 'listbuilder_done'
+ # - wherever the list exits the loop body, add a 'hint({fence})'
from pypy.translator.unsimplify import insert_empty_block
for block in loopbody:
for link in block.exits:
@@ -1008,13 +1032,18 @@
vlist = self.contains_vlist(link.args)
if vlist is None:
continue # list not passed along this link anyway
+ hints = {'fence': True}
+ if (exactlength and block is loopnextblock and
+ link.target is stopblock):
+ hints['exactlength'] = True
+ chints = Constant(hints)
newblock = insert_empty_block(None, link)
index = link.args.index(vlist)
vlist2 = newblock.inputargs[index]
- vlist3 = Variable('listbuilder')
+ vlist3 = Variable(vlist2)
newblock.inputargs[index] = vlist3
newblock.operations.append(
- SpaceOperation('listbuilder_done', [vlist3], vlist2))
+ SpaceOperation('hint', [vlist3, chints], vlist2))
# done!
Modified: pypy/dist/pypy/translator/test/test_simplify.py
==============================================================================
--- pypy/dist/pypy/translator/test/test_simplify.py (original)
+++ pypy/dist/pypy/translator/test/test_simplify.py Tue Sep 5 13:24:57 2006
@@ -198,12 +198,91 @@
if conftest.option.view:
graph.show()
assert summary(graph) == {
+ 'newlist': 1,
'iter': 1,
- 'len': 1,
- 'newlistbuilder': 1,
'next': 1,
'mul': 1,
'getattr': 1,
'simple_call': 1,
- 'listbuilder_done': 1,
+ 'hint': 2,
}
+
+class TestLLSpecializeListComprehension:
+ typesystem = 'lltype'
+
+ def specialize(self, func, argtypes):
+ from pypy.rpython.llinterp import LLInterpreter
+ t = TranslationContext(list_comprehension_operations=True)
+ t.buildannotator().build_types(func, argtypes)
+ if conftest.option.view:
+ t.view()
+ t.buildrtyper(self.typesystem).specialize()
+ if self.typesystem == 'lltype':
+ backend_optimizations(t)
+ if conftest.option.view:
+ t.view()
+ graph = graphof(t, func)
+ interp = LLInterpreter(t.rtyper)
+ return interp, graph
+
+ def test_simple(self):
+ def main(n):
+ lst = [x*17 for x in range(n)]
+ return lst[5]
+ interp, graph = self.specialize(main, [int])
+ res = interp.eval_graph(graph, [10])
+ assert res == 5 * 17
+
+ def test_mutated_after_listcomp(self):
+ def main(n):
+ lst = [x*17 for x in range(n)]
+ lst.append(-42)
+ return lst[5]
+ interp, graph = self.specialize(main, [int])
+ res = interp.eval_graph(graph, [10])
+ assert res == 5 * 17
+ res = interp.eval_graph(graph, [5])
+ assert res == -42
+
+ def test_two_loops(self):
+ def main(n, m):
+ lst1 = []
+ lst2 = []
+ for i in range(n):
+ lst1.append(i)
+ for i in range(m):
+ lst2.append(i)
+ sum = 0
+ for i in lst1:
+ sum += i
+ for i in lst2:
+ sum -= i
+ return sum
+ interp, graph = self.specialize(main, [int, int])
+ res = interp.eval_graph(graph, [8, 3])
+ assert res == 28 - 3
+
+ def test_list_iterator(self):
+ # for now, this is not optimized as a list comp
+ def main(n):
+ r = range(n)
+ lst = [i*17 for i in iter(r)]
+ return lst[5]
+ interp, graph = self.specialize(main, [int])
+ res = interp.eval_graph(graph, [8])
+ assert res == 5 * 17
+
+ def test_dict_iterator(self):
+ # for now, this is not optimized as a list comp
+ def main(n, m):
+ d = {n: m, m: n}
+ lst = [i*17 for i in d.iterkeys()]
+ return len(lst) + lst[0] + lst[-1]
+ interp, graph = self.specialize(main, [int, int])
+ res = interp.eval_graph(graph, [8, 5])
+ assert res == 2 + 8 * 17 + 5 * 17
+ res = interp.eval_graph(graph, [4, 4])
+ assert res == 1 + 4 * 17 + 4 * 17
+
+class TestOOSpecializeListComprehension(TestLLSpecializeListComprehension):
+ typesystem = 'ootype'
Modified: pypy/dist/pypy/translator/translator.py
==============================================================================
--- pypy/dist/pypy/translator/translator.py (original)
+++ pypy/dist/pypy/translator/translator.py Tue Sep 5 13:24:57 2006
@@ -21,7 +21,7 @@
'simplifying': True,
'do_imports_immediately': True,
'builtins_can_raise_exceptions': False,
- 'list_comprehension_operations': False,
+ 'list_comprehension_operations': False, # True, - not super-tested
}
def __init__(self, **flowing_flags):
More information about the Pypy-commit
mailing list