[pypy-svn] r77300 - in pypy/branch/jit-str/pypy/jit: backend/llgraph metainterp metainterp/optimizeopt metainterp/test
arigo at codespeak.net
arigo at codespeak.net
Thu Sep 23 15:54:28 CEST 2010
Author: arigo
Date: Thu Sep 23 15:54:26 2010
New Revision: 77300
Modified:
pypy/branch/jit-str/pypy/jit/backend/llgraph/llimpl.py
pypy/branch/jit-str/pypy/jit/metainterp/executor.py
pypy/branch/jit-str/pypy/jit/metainterp/optimizeopt/optimizer.py
pypy/branch/jit-str/pypy/jit/metainterp/optimizeopt/virtualize.py
pypy/branch/jit-str/pypy/jit/metainterp/resoperation.py
pypy/branch/jit-str/pypy/jit/metainterp/test/test_optimizefindnode.py
pypy/branch/jit-str/pypy/jit/metainterp/test/test_optimizeopt.py
pypy/branch/jit-str/pypy/jit/metainterp/test/test_string.py
Log:
Finish optimizing str_concats. It builds the final string at once
from parts that can be substrings or single characters. See the
tests in test_optimizeopt.
Modified: pypy/branch/jit-str/pypy/jit/backend/llgraph/llimpl.py
==============================================================================
--- pypy/branch/jit-str/pypy/jit/backend/llgraph/llimpl.py (original)
+++ pypy/branch/jit-str/pypy/jit/backend/llgraph/llimpl.py Thu Sep 23 15:54:26 2010
@@ -1382,6 +1382,16 @@
uni = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string)
uni.chars[index] = unichr(newvalue)
+def do_copystrcontent(src, dst, srcstart, dststart, length):
+ src = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), src)
+ dst = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), dst)
+ rstr.copy_string_contents(src, dst, srcstart, dststart, length)
+
+def do_copyunicodecontent(src, dst, srcstart, dststart, length):
+ src = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), src)
+ dst = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), dst)
+ rstr.copy_unicode_contents(src, dst, srcstart, dststart, length)
+
# ---------- call ----------
_call_args_i = []
Modified: pypy/branch/jit-str/pypy/jit/metainterp/executor.py
==============================================================================
--- pypy/branch/jit-str/pypy/jit/metainterp/executor.py (original)
+++ pypy/branch/jit-str/pypy/jit/metainterp/executor.py Thu Sep 23 15:54:26 2010
@@ -2,7 +2,7 @@
"""
import py
-from pypy.rpython.lltypesystem import lltype, llmemory
+from pypy.rpython.lltypesystem import lltype, llmemory, rstr
from pypy.rpython.ootypesystem import ootype
from pypy.rpython.lltypesystem.lloperation import llop
from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask
@@ -203,6 +203,24 @@
def do_same_as(cpu, _, box):
return box.clonebox()
+def do_copystrcontent(cpu, _, srcbox, dstbox,
+ srcstartbox, dststartbox, lengthbox):
+ src = srcbox.getptr(lltype.Ptr(rstr.STR))
+ dst = dstbox.getptr(lltype.Ptr(rstr.STR))
+ srcstart = srcstartbox.getint()
+ dststart = dststartbox.getint()
+ length = lengthbox.getint()
+ rstr.copy_string_contents(src, dst, srcstart, dststart, length)
+
+def do_copyunicodecontent(cpu, _, srcbox, dstbox,
+ srcstartbox, dststartbox, lengthbox):
+ src = srcbox.getptr(lltype.Ptr(rstr.UNICODE))
+ dst = dstbox.getptr(lltype.Ptr(rstr.UNICODE))
+ srcstart = srcstartbox.getint()
+ dststart = dststartbox.getint()
+ length = lengthbox.getint()
+ rstr.copy_unicode_contents(src, dst, srcstart, dststart, length)
+
# ____________________________________________________________
##def do_force_token(cpu):
Modified: pypy/branch/jit-str/pypy/jit/metainterp/optimizeopt/optimizer.py
==============================================================================
--- pypy/branch/jit-str/pypy/jit/metainterp/optimizeopt/optimizer.py (original)
+++ pypy/branch/jit-str/pypy/jit/metainterp/optimizeopt/optimizer.py Thu Sep 23 15:54:26 2010
@@ -126,6 +126,16 @@
def setitem(self, index, value):
raise NotImplementedError
+ def getstrlen(self, newoperations):
+ box = self.force_box()
+ lengthbox = BoxInt()
+ newoperations.append(ResOperation(rop.STRLEN, [box], lengthbox))
+ return lengthbox
+
+ def string_copy_parts(self, *args):
+ from pypy.jit.metainterp.optimizeopt import virtualize
+ return virtualize.default_string_copy_parts(self, *args)
+
class ConstantValue(OptValue):
def __init__(self, box):
self.make_constant(box)
Modified: pypy/branch/jit-str/pypy/jit/metainterp/optimizeopt/virtualize.py
==============================================================================
--- pypy/branch/jit-str/pypy/jit/metainterp/optimizeopt/virtualize.py (original)
+++ pypy/branch/jit-str/pypy/jit/metainterp/optimizeopt/virtualize.py Thu Sep 23 15:54:26 2010
@@ -7,7 +7,7 @@
from pypy.jit.metainterp.optimizeutil import _findall
from pypy.rlib.objectmodel import we_are_translated
from pypy.jit.metainterp.optimizeopt.optimizer import *
-from pypy.jit.codewriter.effectinfo import EffectInfo, callinfo_for_oopspec
+from pypy.jit.codewriter.effectinfo import EffectInfo
from pypy.rlib.unroll import unrolling_iterable
@@ -198,8 +198,13 @@
class VAbstractStringValue(AbstractVirtualValue):
- def getlengthvalue(self):
- raise NotImplementedError
+ def _really_force(self):
+ assert self.source_op is not None
+ self.box = box = self.source_op.result
+ newoperations = self.optimizer.newoperations
+ lengthbox = self.getstrlen(newoperations)
+ newoperations.append(ResOperation(rop.NEWSTR, [lengthbox], box))
+ self.string_copy_parts(newoperations, box, CONST_0)
class VStringPlainValue(VAbstractStringValue):
@@ -207,12 +212,12 @@
def setup(self, size):
self._chars = [CVAL_ZERO] * size
- self._lengthvalue = None # cache only
+ self._lengthbox = None # cache only
- def getlengthvalue(self):
- if self._lengthvalue is None:
- self._lengthvalue = ConstantValue(ConstInt(len(self._chars)))
- return self._lengthvalue
+ def getstrlen(self, _):
+ if self._lengthbox is None:
+ self._lengthbox = ConstInt(len(self._chars))
+ return self._lengthbox
def getitem(self, index):
return self._chars[index]
@@ -221,16 +226,14 @@
assert isinstance(charvalue, OptValue)
self._chars[index] = charvalue
- def _really_force(self):
- assert self.source_op is not None
- newoperations = self.optimizer.newoperations
- newoperations.append(self.source_op)
- self.box = box = self.source_op.result
+ def string_copy_parts(self, newoperations, targetbox, offsetbox):
for i in range(len(self._chars)):
charbox = self._chars[i].force_box()
- op = ResOperation(rop.STRSETITEM,
- [box, ConstInt(i), charbox], None)
- newoperations.append(op)
+ newoperations.append(ResOperation(rop.STRSETITEM, [targetbox,
+ offsetbox,
+ charbox], None))
+ offsetbox = _int_add(newoperations, offsetbox, CONST_1)
+ return offsetbox
def get_args_for_fail(self, modifier):
if self.box is None and not modifier.already_seen_virtual(self.keybox):
@@ -246,24 +249,20 @@
class VStringConcatValue(VAbstractStringValue):
"""The concatenation of two other strings."""
- def setup(self, left, right, length):
+ def setup(self, left, right, lengthbox):
self.left = left
self.right = right
- self.lengthvalue = length
+ self.lengthbox = lengthbox
- def getlengthvalue(self):
- return self.lengthvalue
+ def getstrlen(self, _):
+ return self.lengthbox
- def _really_force(self):
- assert self.source_op is not None
- calldescr, func = callinfo_for_oopspec(EffectInfo.OS_STR_CONCAT)
- leftbox = self.left.force_box()
- rightbox = self.right.force_box()
- self.box = box = self.source_op.result
- newoperations = self.optimizer.newoperations
- newoperations.append(ResOperation(rop.CALL,
- [ConstInt(func), leftbox, rightbox],
- box, calldescr))
+ def string_copy_parts(self, newoperations, targetbox, offsetbox):
+ offsetbox = self.left.string_copy_parts(newoperations, targetbox,
+ offsetbox)
+ offsetbox = self.right.string_copy_parts(newoperations, targetbox,
+ offsetbox)
+ return offsetbox
def get_args_for_fail(self, modifier):
if self.box is None and not modifier.already_seen_virtual(self.keybox):
@@ -279,6 +278,32 @@
return modifier.make_vstrconcat()
+def default_string_copy_parts(srcvalue, newoperations, targetbox, offsetbox):
+ # Copies the pointer-to-string 'srcvalue' into the target string
+ # given by 'targetbox', at the specified offset. Returns the offset
+ # at the end of the copy.
+ srcbox = srcvalue.force_box()
+ lengthbox = BoxInt()
+ newoperations.append(ResOperation(rop.STRLEN, [srcbox], lengthbox))
+ nextoffsetbox = _int_add(newoperations, offsetbox, lengthbox)
+ newoperations.append(ResOperation(rop.COPYSTRCONTENT, [srcbox, targetbox,
+ CONST_0, offsetbox,
+ lengthbox], None))
+ return nextoffsetbox
+
+def _int_add(newoperations, box1, box2):
+ if isinstance(box1, ConstInt):
+ if box1.value == 0:
+ return box2
+ if isinstance(box2, ConstInt):
+ return ConstInt(box1.value + box2.value)
+ elif isinstance(box2, ConstInt) and box2.value == 0:
+ return box1
+ resbox = BoxInt()
+ newoperations.append(ResOperation(rop.INT_ADD, [box1, box2], resbox))
+ return resbox
+
+
class __extend__(SpecNode):
def setup_virtual_node(self, optimizer, box, newinputargs):
raise NotImplementedError
@@ -584,23 +609,23 @@
def optimize_STRLEN(self, op):
value = self.getvalue(op.args[0])
if isinstance(value, VStringPlainValue): # even if no longer virtual
- self.make_equal_to(op.result, value.getlengthvalue())
+ lengthbox = value.getstrlen(self.optimizer.newoperations)
+ self.make_equal_to(op.result, self.getvalue(lengthbox))
else:
value.ensure_nonnull()
self.emit_operation(op)
def opt_call_oopspec_STR_CONCAT(self, op):
- lengthbox = BoxInt()
- len1box = BoxInt()
- len2box = BoxInt()
- seo = self.optimizer.send_extra_operation
- seo(ResOperation(rop.STRLEN, [op.args[1]], len1box))
- seo(ResOperation(rop.STRLEN, [op.args[2]], len2box))
- seo(ResOperation(rop.INT_ADD, [len1box, len2box], lengthbox))
+ vleft = self.getvalue(op.args[1])
+ vright = self.getvalue(op.args[2])
+ newoperations = self.optimizer.newoperations
+ len1box = vleft.getstrlen(newoperations)
+ len2box = vright.getstrlen(newoperations)
+ lengthbox = _int_add(newoperations, len1box, len2box)
value = self.make_vstring_concat(op.result, op)
value.setup(left = self.getvalue(op.args[1]),
right = self.getvalue(op.args[2]),
- length = self.getvalue(lengthbox))
+ lengthbox = lengthbox)
return True
def propagate_forward(self, op):
Modified: pypy/branch/jit-str/pypy/jit/metainterp/resoperation.py
==============================================================================
--- pypy/branch/jit-str/pypy/jit/metainterp/resoperation.py (original)
+++ pypy/branch/jit-str/pypy/jit/metainterp/resoperation.py Thu Sep 23 15:54:26 2010
@@ -221,6 +221,8 @@
'COND_CALL_GC_WB', # [objptr, newvalue] (for the write barrier)
'DEBUG_MERGE_POINT/1', # debugging only
'VIRTUAL_REF_FINISH/2', # removed before it's passed to the backend
+ 'COPYSTRCONTENT/5', # src, dst, srcstart, dststart, length
+ 'COPYUNICODECONTENT/5',
'_CANRAISE_FIRST', # ----- start of can_raise operations -----
'CALL',
Modified: pypy/branch/jit-str/pypy/jit/metainterp/test/test_optimizefindnode.py
==============================================================================
--- pypy/branch/jit-str/pypy/jit/metainterp/test/test_optimizefindnode.py (original)
+++ pypy/branch/jit-str/pypy/jit/metainterp/test/test_optimizefindnode.py Thu Sep 23 15:54:26 2010
@@ -117,6 +117,8 @@
EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE))
arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
EffectInfo([], [], [], oopspecindex=EffectInfo.OS_ARRAYCOPY))
+ strconcatdescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
+ EffectInfo([], [], [], oopspecindex=EffectInfo.OS_STR_CONCAT))
class LoopToken(AbstractDescr):
pass
asmdescr = LoopToken() # it can be whatever, it's not a descr though
Modified: pypy/branch/jit-str/pypy/jit/metainterp/test/test_optimizeopt.py
==============================================================================
--- pypy/branch/jit-str/pypy/jit/metainterp/test/test_optimizeopt.py (original)
+++ pypy/branch/jit-str/pypy/jit/metainterp/test/test_optimizeopt.py Thu Sep 23 15:54:26 2010
@@ -3891,7 +3891,130 @@
"""
self.optimize_loop(ops, 'Not, Not', expected)
+ def test_newstr_1(self):
+ ops = """
+ [i0]
+ p1 = newstr(1)
+ strsetitem(p1, 0, i0)
+ i1 = strgetitem(p1, 0)
+ jump(i1)
+ """
+ expected = """
+ [i0]
+ jump(i0)
+ """
+ self.optimize_loop(ops, 'Not', expected)
+
+ def test_newstr_2(self):
+ ops = """
+ [i0, i1]
+ p1 = newstr(2)
+ strsetitem(p1, 0, i0)
+ strsetitem(p1, 1, i1)
+ i2 = strgetitem(p1, 1)
+ i3 = strgetitem(p1, 0)
+ jump(i2, i3)
+ """
+ expected = """
+ [i0, i1]
+ jump(i1, i0)
+ """
+ self.optimize_loop(ops, 'Not, Not', expected)
+
+ def test_concat_1(self):
+ ops = """
+ [p1, p2]
+ p3 = call(0, p1, p2, descr=strconcatdescr)
+ jump(p2, p3)
+ """
+ expected = """
+ [p1, p2]
+ i1 = strlen(p1)
+ i2 = strlen(p2)
+ i3 = int_add(i1, i2)
+ p3 = newstr(i3)
+ i4 = strlen(p1)
+ copystrcontent(p1, p3, 0, 0, i4)
+ i5 = strlen(p2)
+ i6 = int_add(i4, i5) # will be killed by the backend
+ copystrcontent(p2, p3, 0, i4, i5)
+ jump(p2, p3)
+ """
+ self.optimize_loop(ops, 'Not, Not', expected)
+ def test_concat_vstr2_str(self):
+ ops = """
+ [i0, i1, p2]
+ p1 = newstr(2)
+ strsetitem(p1, 0, i0)
+ strsetitem(p1, 1, i1)
+ p3 = call(0, p1, p2, descr=strconcatdescr)
+ jump(i1, i0, p3)
+ """
+ expected = """
+ [i0, i1, p2]
+ i2 = strlen(p2)
+ i3 = int_add(2, i2)
+ p3 = newstr(i3)
+ strsetitem(p3, 0, i0)
+ strsetitem(p3, 1, i1)
+ i4 = strlen(p2)
+ i5 = int_add(2, i4) # will be killed by the backend
+ copystrcontent(p2, p3, 0, 2, i4)
+ jump(i1, i0, p3)
+ """
+ self.optimize_loop(ops, 'Not, Not, Not', expected)
+
+ def test_concat_str_vstr2(self):
+ ops = """
+ [i0, i1, p2]
+ p1 = newstr(2)
+ strsetitem(p1, 0, i0)
+ strsetitem(p1, 1, i1)
+ p3 = call(0, p2, p1, descr=strconcatdescr)
+ jump(i1, i0, p3)
+ """
+ expected = """
+ [i0, i1, p2]
+ i2 = strlen(p2)
+ i3 = int_add(i2, 2)
+ p3 = newstr(i3)
+ i4 = strlen(p2)
+ copystrcontent(p2, p3, 0, 0, i4)
+ strsetitem(p3, i4, i0)
+ i5 = int_add(i4, 1)
+ strsetitem(p3, i5, i1)
+ i6 = int_add(i5, 1) # will be killed by the backend
+ jump(i1, i0, p3)
+ """
+ self.optimize_loop(ops, 'Not, Not, Not', expected)
+
+ def test_concat_str_str_str(self):
+ ops = """
+ [p1, p2, p3]
+ p4 = call(0, p1, p2, descr=strconcatdescr)
+ p5 = call(0, p4, p3, descr=strconcatdescr)
+ jump(p2, p3, p5)
+ """
+ expected = """
+ [p1, p2, p3]
+ i1 = strlen(p1)
+ i2 = strlen(p2)
+ i12 = int_add(i1, i2)
+ i3 = strlen(p3)
+ i123 = int_add(i12, i3)
+ p5 = newstr(i123)
+ i1b = strlen(p1)
+ copystrcontent(p1, p5, 0, 0, i1b)
+ i2b = strlen(p2)
+ i12b = int_add(i1b, i2b)
+ copystrcontent(p2, p5, 0, i1b, i2b)
+ i3b = strlen(p3)
+ i123b = int_add(i12b, i3b) # will be killed by the backend
+ copystrcontent(p3, p5, 0, i12b, i3b)
+ jump(p2, p3, p5)
+ """
+ self.optimize_loop(ops, 'Not, Not, Not', expected)
##class TestOOtype(BaseTestOptimizeOpt, OOtypeMixin):
Modified: pypy/branch/jit-str/pypy/jit/metainterp/test/test_string.py
==============================================================================
--- pypy/branch/jit-str/pypy/jit/metainterp/test/test_string.py (original)
+++ pypy/branch/jit-str/pypy/jit/metainterp/test/test_string.py Thu Sep 23 15:54:26 2010
@@ -175,25 +175,94 @@
newunicode=0, unicodesetitem=0,
call=0, call_pure=0)
- def test_strconcat_escape(self):
- for somestr in ["abc", ]: #u"def"]:
- jitdriver = JitDriver(greens = [], reds = ['m', 'n'])
- @dont_look_inside
- def escape(x):
- pass
- mylist = [somestr+str(i) for i in range(10)]
- def f(n, m):
- while m >= 0:
- jitdriver.can_enter_jit(m=m, n=n)
- jitdriver.jit_merge_point(m=m, n=n)
- s = mylist[n] + mylist[m]
- escape(s)
- m -= 1
- return 42
- self.meta_interp(f, [6, 7])
- self.check_loops(newstr=0, strsetitem=0,
- newunicode=0, unicodesetitem=0,
- call=2, call_pure=0) # ll_strconcat, escape
+ def test_strconcat_escape_str_str(self):
+ jitdriver = JitDriver(greens = [], reds = ['m', 'n'])
+ @dont_look_inside
+ def escape(x):
+ pass
+ mylist = ["somestr"+str(i) for i in range(10)]
+ def f(n, m):
+ while m >= 0:
+ jitdriver.can_enter_jit(m=m, n=n)
+ jitdriver.jit_merge_point(m=m, n=n)
+ s = mylist[n] + mylist[m]
+ escape(s)
+ m -= 1
+ return 42
+ self.meta_interp(f, [6, 7])
+ self.check_loops(newstr=1, strsetitem=0, copystrcontent=2,
+ call=1, call_pure=0) # escape
+
+ def test_strconcat_escape_str_char(self):
+ jitdriver = JitDriver(greens = [], reds = ['m', 'n'])
+ @dont_look_inside
+ def escape(x):
+ pass
+ mylist = ["somestr"+str(i) for i in range(10)]
+ def f(n, m):
+ while m >= 0:
+ jitdriver.can_enter_jit(m=m, n=n)
+ jitdriver.jit_merge_point(m=m, n=n)
+ s = mylist[n] + chr(m)
+ escape(s)
+ m -= 1
+ return 42
+ self.meta_interp(f, [6, 7])
+ self.check_loops(newstr=1, strsetitem=1, copystrcontent=1,
+ call=1, call_pure=0) # escape
+
+ def test_strconcat_escape_char_str(self):
+ jitdriver = JitDriver(greens = [], reds = ['m', 'n'])
+ @dont_look_inside
+ def escape(x):
+ pass
+ mylist = ["somestr"+str(i) for i in range(10)]
+ def f(n, m):
+ while m >= 0:
+ jitdriver.can_enter_jit(m=m, n=n)
+ jitdriver.jit_merge_point(m=m, n=n)
+ s = chr(n) + mylist[m]
+ escape(s)
+ m -= 1
+ return 42
+ self.meta_interp(f, [6, 7])
+ self.check_loops(newstr=1, strsetitem=1, copystrcontent=1,
+ call=1, call_pure=0) # escape
+
+ def test_strconcat_escape_char_char(self):
+ jitdriver = JitDriver(greens = [], reds = ['m', 'n'])
+ @dont_look_inside
+ def escape(x):
+ pass
+ def f(n, m):
+ while m >= 0:
+ jitdriver.can_enter_jit(m=m, n=n)
+ jitdriver.jit_merge_point(m=m, n=n)
+ s = chr(n) + chr(m)
+ escape(s)
+ m -= 1
+ return 42
+ self.meta_interp(f, [6, 7])
+ self.check_loops(newstr=1, strsetitem=2, copystrcontent=0,
+ call=1, call_pure=0) # escape
+
+ def test_strconcat_escape_str_char_str(self):
+ jitdriver = JitDriver(greens = [], reds = ['m', 'n'])
+ @dont_look_inside
+ def escape(x):
+ pass
+ mylist = ["somestr"+str(i) for i in range(10)]
+ def f(n, m):
+ while m >= 0:
+ jitdriver.can_enter_jit(m=m, n=n)
+ jitdriver.jit_merge_point(m=m, n=n)
+ s = mylist[n] + chr(n) + mylist[m]
+ escape(s)
+ m -= 1
+ return 42
+ self.meta_interp(f, [6, 7])
+ self.check_loops(newstr=1, strsetitem=1, copystrcontent=2,
+ call=1, call_pure=0) # escape
def test_strconcat_guard_fail(self):
for somestr in ["abc", ]: #u"def"]:
@@ -212,8 +281,6 @@
m -= 1
return 42
self.meta_interp(f, [6, 10])
- self.check_loops(newstr=0, strsetitem=0,
- newunicode=0, unicodesetitem=0)
class TestOOtype(StringTests, OOJitMixin):
More information about the Pypy-commit
mailing list