[pypy-commit] pypy default: merge space-iterator-improvements. this branch preallocates a correctly size list for unpackiterable
fijal
noreply at buildbot.pypy.org
Mon Sep 12 16:10:54 CEST 2011
Author: Maciej Fijalkowski <fijall at gmail.com>
Branch:
Changeset: r47229:b4ee85d4978b
Date: 2011-09-12 16:10 +0200
http://bitbucket.org/pypy/pypy/changeset/b4ee85d4978b/
Log: merge space-iterator-improvements. this branch preallocates a
correctly size list for unpackiterable
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -8,13 +8,13 @@
from pypy.interpreter.miscutils import ThreadLocals
from pypy.tool.cache import Cache
from pypy.tool.uid import HUGEVAL_BYTES
-from pypy.rlib.objectmodel import we_are_translated
+from pypy.rlib.objectmodel import we_are_translated, newlist
from pypy.rlib.debug import make_sure_not_resized
from pypy.rlib.timer import DummyTimer, Timer
from pypy.rlib.rarithmetic import r_uint
from pypy.rlib import jit
from pypy.tool.sourcetools import func_with_new_name
-import os, sys, py
+import os, sys
__all__ = ['ObjSpace', 'OperationError', 'Wrappable', 'W_Root']
@@ -757,7 +757,18 @@
w_iterator = self.iter(w_iterable)
# If we know the expected length we can preallocate.
if expected_length == -1:
- items = []
+ try:
+ lgt_estimate = self.len_w(w_iterable)
+ except OperationError, o:
+ if (not o.match(self, self.w_AttributeError) and
+ not o.match(self, self.w_TypeError)):
+ raise
+ items = []
+ else:
+ try:
+ items = newlist(lgt_estimate)
+ except MemoryError:
+ items = [] # it might have lied
else:
items = [None] * expected_length
idx = 0
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -71,6 +71,23 @@
assert err.value.match(space, space.w_ValueError)
err = raises(OperationError, space.unpackiterable, w_l, 5)
assert err.value.match(space, space.w_ValueError)
+ w_a = space.appexec((), """():
+ class A(object):
+ def __iter__(self):
+ return self
+ def next(self):
+ raise StopIteration
+ def __len__(self):
+ 1/0
+ return A()
+ """)
+ try:
+ space.unpackiterable(w_a)
+ except OperationError, o:
+ if not o.match(space, space.w_ZeroDivisionError):
+ raise Exception("DID NOT RAISE")
+ else:
+ raise Exception("DID NOT RAISE")
def test_fixedview(self):
space = self.space
diff --git a/pypy/objspace/std/iterobject.py b/pypy/objspace/std/iterobject.py
--- a/pypy/objspace/std/iterobject.py
+++ b/pypy/objspace/std/iterobject.py
@@ -72,10 +72,6 @@
w_seqiter.index += 1
return w_item
-# XXX __length_hint__()
-##def len__SeqIter(space, w_seqiter):
-## return w_seqiter.getlength(space)
-
def iter__FastTupleIter(space, w_seqiter):
return w_seqiter
@@ -93,10 +89,6 @@
w_seqiter.index = index + 1
return w_item
-# XXX __length_hint__()
-##def len__FastTupleIter(space, w_seqiter):
-## return w_seqiter.getlength(space)
-
def iter__FastListIter(space, w_seqiter):
return w_seqiter
@@ -114,10 +106,6 @@
w_seqiter.index = index + 1
return w_item
-# XXX __length_hint__()
-##def len__FastListIter(space, w_seqiter):
-## return w_seqiter.getlength(space)
-
def iter__ReverseSeqIter(space, w_seqiter):
return w_seqiter
@@ -135,20 +123,5 @@
raise OperationError(space.w_StopIteration, space.w_None)
return w_item
-# XXX __length_hint__()
-##def len__ReverseSeqIter(space, w_seqiter):
-## if w_seqiter.w_seq is None:
-## return space.wrap(0)
-## index = w_seqiter.index+1
-## w_length = space.len(w_seqiter.w_seq)
-## # if length of sequence is less than index :exhaust iterator
-## if space.is_true(space.gt(space.wrap(w_seqiter.index), w_length)):
-## w_len = space.wrap(0)
-## w_seqiter.w_seq = None
-## else:
-## w_len =space.wrap(index)
-## if space.is_true(space.lt(w_len,space.wrap(0))):
-## w_len = space.wrap(0)
-## return w_len
register_all(vars())
diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -19,6 +19,8 @@
# def f(...
#
+from pypy.rpython.extregistry import ExtRegistryEntry
+
class _Specialize(object):
def memo(self):
""" Specialize functions based on argument values. All arguments has
@@ -177,6 +179,34 @@
obj.__class__ = FREED_OBJECT
# ____________________________________________________________
+
+def newlist(sizehint=0):
+ """ Create a new list, but pass a hint how big the size should be
+ preallocated
+ """
+ return []
+
+class Entry(ExtRegistryEntry):
+ _about_ = newlist
+
+ def compute_result_annotation(self, s_sizehint):
+ from pypy.annotation.model import SomeInteger
+
+ assert isinstance(s_sizehint, SomeInteger)
+ return self.bookkeeper.newlist()
+
+ def specialize_call(self, orig_hop, i_sizehint=None):
+ from pypy.rpython.rlist import rtype_newlist
+ # fish a bit hop
+ hop = orig_hop.copy()
+ v = hop.args_v[0]
+ r, s = hop.r_s_popfirstarg()
+ if s.is_constant():
+ v = hop.inputconst(r, s.const)
+ hop.exception_is_here()
+ return rtype_newlist(hop, v_sizehint=v)
+
+# ____________________________________________________________
#
# id-like functions. The idea is that calling hash() or id() is not
# allowed in RPython. You have to call one of the following more
@@ -301,8 +331,6 @@
# ----------
-from pypy.rpython.extregistry import ExtRegistryEntry
-
class Entry(ExtRegistryEntry):
_about_ = compute_hash
diff --git a/pypy/rlib/test/test_objectmodel.py b/pypy/rlib/test/test_objectmodel.py
--- a/pypy/rlib/test/test_objectmodel.py
+++ b/pypy/rlib/test/test_objectmodel.py
@@ -424,3 +424,32 @@
if option.view:
graph.show()
return graph
+
+
+def test_newlist():
+ from pypy.annotation.model import SomeInteger
+ def f(z):
+ x = newlist(sizehint=38)
+ if z < 0:
+ x.append(1)
+ return len(x)
+
+ graph = getgraph(f, [SomeInteger()])
+ for llop in graph.startblock.operations:
+ if llop.opname == 'malloc_varsize':
+ break
+ assert llop.args[2].value == 38
+
+def test_newlist_nonconst():
+ from pypy.annotation.model import SomeInteger
+ def f(z):
+ x = newlist(sizehint=z)
+ return len(x)
+
+ graph = getgraph(f, [SomeInteger()])
+ for llop in graph.startblock.operations:
+ if llop.opname == 'malloc_varsize':
+ break
+ assert llop.args[2] is graph.startblock.inputargs[0]
+
+
diff --git a/pypy/rpython/lltypesystem/rlist.py b/pypy/rpython/lltypesystem/rlist.py
--- a/pypy/rpython/lltypesystem/rlist.py
+++ b/pypy/rpython/lltypesystem/rlist.py
@@ -1,21 +1,14 @@
from pypy.tool.pairtype import pairtype, pair
-from pypy.annotation import model as annmodel
-from pypy.rpython.error import TyperError
-from pypy.rpython.rmodel import Repr, IntegerRepr, inputconst
+from pypy.rpython.rmodel import Repr, inputconst
from pypy.rpython.rmodel import externalvsinternal
from pypy.rpython.rlist import AbstractBaseListRepr, AbstractListRepr, \
- AbstractFixedSizeListRepr, AbstractListIteratorRepr, rtype_newlist, \
- rtype_alloc_and_set, ll_setitem_nonneg, ADTIList, ADTIFixedList
-from pypy.rpython.rlist import dum_nocheck, dum_checkidx
-from pypy.rpython.lltypesystem.lltype import \
- GcForwardReference, Ptr, GcArray, GcStruct, \
- Void, Signed, malloc, typeOf, Primitive, \
- Bool, nullptr, typeMethod
+ AbstractFixedSizeListRepr, AbstractListIteratorRepr, \
+ ll_setitem_nonneg, ADTIList, ADTIFixedList
+from pypy.rpython.rlist import dum_nocheck
+from pypy.rpython.lltypesystem.lltype import GcForwardReference, Ptr, GcArray,\
+ GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod
from pypy.rpython.lltypesystem import rstr
-from pypy.rpython import robject
from pypy.rlib.debug import ll_assert
-from pypy.rpython.lltypesystem import rffi
-from pypy.rpython.lltypesystem.lloperation import llop
from pypy.rlib import rgc
# ____________________________________________________________
@@ -67,6 +60,7 @@
ITEMARRAY = GcArray(ITEM,
adtmeths = ADTIFixedList({
"ll_newlist": ll_fixed_newlist,
+ "ll_newlist_hint": ll_fixed_newlist,
"ll_newemptylist": ll_fixed_newemptylist,
"ll_length": ll_fixed_length,
"ll_items": ll_fixed_items,
@@ -100,6 +94,7 @@
("items", Ptr(ITEMARRAY)),
adtmeths = ADTIList({
"ll_newlist": ll_newlist,
+ "ll_newlist_hint": ll_newlist_hint,
"ll_newemptylist": ll_newemptylist,
"ll_length": ll_length,
"ll_items": ll_items,
@@ -267,6 +262,15 @@
ll_newlist = typeMethod(ll_newlist)
ll_newlist.oopspec = 'newlist(length)'
+def ll_newlist_hint(LIST, lengthhint):
+ ll_assert(lengthhint >= 0, "negative list length")
+ l = malloc(LIST)
+ l.length = 0
+ l.items = malloc(LIST.items.TO, lengthhint)
+ return l
+ll_newlist_hint = typeMethod(ll_newlist_hint)
+ll_newlist_hint.oopspec = 'newlist(lengthhint)'
+
# should empty lists start with no allocated memory, or with a preallocated
# minimal number of entries? XXX compare memory usage versus speed, and
# check how many always-empty lists there are in a typical pypy-c run...
@@ -337,11 +341,15 @@
l[index] = item
ll_fixed_setitem_fast.oopspec = 'list.setitem(l, index, item)'
-def newlist(llops, r_list, items_v):
+def newlist(llops, r_list, items_v, v_sizehint=None):
LIST = r_list.LIST
if len(items_v) == 0:
- v_result = llops.gendirectcall(LIST.ll_newemptylist)
+ if v_sizehint is None:
+ v_result = llops.gendirectcall(LIST.ll_newemptylist)
+ else:
+ v_result = llops.gendirectcall(LIST.ll_newlist_hint, v_sizehint)
else:
+ assert v_sizehint is None
cno = inputconst(Signed, len(items_v))
v_result = llops.gendirectcall(LIST.ll_newlist, cno)
v_func = inputconst(Void, dum_nocheck)
diff --git a/pypy/rpython/ootypesystem/rlist.py b/pypy/rpython/ootypesystem/rlist.py
--- a/pypy/rpython/ootypesystem/rlist.py
+++ b/pypy/rpython/ootypesystem/rlist.py
@@ -124,7 +124,7 @@
else:
return ootype.List()
- def _generate_newlist(self, llops, items_v):
+ def _generate_newlist(self, llops, items_v, v_sizehint):
c_list = inputconst(ootype.Void, self.lowleveltype)
v_result = llops.genop("new", [c_list], resulttype=self.lowleveltype)
c_resize = inputconst(ootype.Void, "_ll_resize")
@@ -150,8 +150,8 @@
-def newlist(llops, r_list, items_v):
- v_result = r_list._generate_newlist(llops, items_v)
+def newlist(llops, r_list, items_v, v_sizehint=None):
+ v_result = r_list._generate_newlist(llops, items_v, v_sizehint)
c_setitem = inputconst(ootype.Void, "ll_setitem_fast")
for i, v_item in enumerate(items_v):
@@ -224,7 +224,7 @@
def make_iterator_repr(self):
return ListIteratorRepr(self)
- def _generate_newlist(self, llops, items_v):
+ def _generate_newlist(self, llops, items_v, v_sizehint):
c_array = inputconst(ootype.Void, self.lowleveltype)
c_length = inputconst(ootype.Signed, len(items_v))
v_result = llops.genop("oonewarray", [c_array, c_length], resulttype=self.lowleveltype)
diff --git a/pypy/rpython/rlist.py b/pypy/rpython/rlist.py
--- a/pypy/rpython/rlist.py
+++ b/pypy/rpython/rlist.py
@@ -2,7 +2,7 @@
from pypy.objspace.flow.model import Constant
from pypy.annotation import model as annmodel
from pypy.rpython.error import TyperError
-from pypy.rpython.rmodel import Repr, IteratorRepr, IntegerRepr, inputconst
+from pypy.rpython.rmodel import Repr, IteratorRepr, IntegerRepr
from pypy.rpython.rstr import AbstractStringRepr, AbstractCharRepr
from pypy.rpython.lltypesystem.lltype import typeOf, Ptr, Void, Signed, Bool
from pypy.rpython.lltypesystem.lltype import nullptr, Char, UniChar, Number
@@ -344,7 +344,7 @@
return hop.genop('bool_not', [flag], resulttype=Bool)
-def rtype_newlist(hop):
+def rtype_newlist(hop, v_sizehint=None):
nb_args = hop.nb_args
r_list = hop.r_result
if r_list == robject.pyobj_repr: # special case: SomeObject lists!
@@ -358,7 +358,8 @@
return v_result
r_listitem = r_list.item_repr
items_v = [hop.inputarg(r_listitem, arg=i) for i in range(nb_args)]
- return hop.rtyper.type_system.rlist.newlist(hop.llops, r_list, items_v)
+ return hop.rtyper.type_system.rlist.newlist(hop.llops, r_list, items_v,
+ v_sizehint=v_sizehint)
def rtype_alloc_and_set(hop):
r_list = hop.r_result
diff --git a/pypy/rpython/test/test_rlist.py b/pypy/rpython/test/test_rlist.py
--- a/pypy/rpython/test/test_rlist.py
+++ b/pypy/rpython/test/test_rlist.py
@@ -1360,6 +1360,19 @@
assert ('foldable' in func.func_name) == \
("y[*]" in immutable_fields)
+ def test_hints(self):
+ from pypy.rlib.objectmodel import newlist
+ from pypy.rpython.annlowlevel import hlstr
+
+ def f(z):
+ z = hlstr(z)
+ x = newlist(sizehint=13)
+ x += z
+ return ''.join(x)
+
+ res = self.interpret(f, [self.string_to_ll('abc')])
+ assert self.ll_to_string(res) == 'abc'
+
class TestLLtype(BaseTestRlist, LLRtypeMixin):
type_system = 'lltype'
rlist = ll_rlist
More information about the pypy-commit
mailing list