[pypy-svn] r69849 - in pypy/trunk/pypy/jit/metainterp: . test
cfbolz at codespeak.net
cfbolz at codespeak.net
Wed Dec 2 17:33:11 CET 2009
Author: cfbolz
Date: Wed Dec 2 17:33:10 2009
New Revision: 69849
Modified:
pypy/trunk/pypy/jit/metainterp/optimizeopt.py
pypy/trunk/pypy/jit/metainterp/resume.py
pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py
pypy/trunk/pypy/jit/metainterp/test/test_resume.py
Log:
Compress the numbering lists. This seems to give a saving of about 10% and makes
things slower. I will revert it afterwards, just want to have it checked in as
not to lose it.
Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original)
+++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Wed Dec 2 17:33:10 2009
@@ -160,13 +160,13 @@
self._really_force()
return self.box
- def make_virtual_info(self, modifier, fieldnums):
+ def make_virtual_info(self, modifier, fieldnums_compressed):
vinfo = self._cached_vinfo
- if vinfo is not None and resume.tagged_list_eq(
- vinfo.fieldnums, fieldnums):
+ if (vinfo is not None and
+ vinfo.fieldnums_compressed == fieldnums_compressed):
return vinfo
vinfo = self._make_virtual(modifier)
- vinfo.fieldnums = fieldnums
+ vinfo.fieldnums_compressed = fieldnums_compressed
self._cached_vinfo = vinfo
return vinfo
Modified: pypy/trunk/pypy/jit/metainterp/resume.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/resume.py (original)
+++ pypy/trunk/pypy/jit/metainterp/resume.py Wed Dec 2 17:33:10 2009
@@ -60,48 +60,97 @@
storage.rd_snapshot = snapshot
class Numbering(object):
- __slots__ = ('prev', 'nums')
+ __slots__ = ('prev', 'nums_compressed')
def __init__(self, prev, nums):
self.prev = prev
- self.nums = nums
+ self.nums_compressed = compress_tagged_list(nums)
+
+ def nums(self):
+ return uncompress_tagged_list(self.nums_compressed)
+
+# _________________________________________________________
+# tagging helpers
TAGMASK = 3
def tag(value, tagbits):
if tagbits >> 2:
raise ValueError
- sx = value >> 13
- if sx != 0 and sx != -1:
+ if rarithmetic.intmask((value << 2)) >> 2 != value:
raise ValueError
- return rffi.r_short(value<<2|tagbits)
+ return value<<2|tagbits
def untag(value):
- value = rarithmetic.widen(value)
- tagbits = value&TAGMASK
+ tagbits = value & TAGMASK
return value>>2, tagbits
-def tagged_eq(x, y):
- # please rpython :(
- return rarithmetic.widen(x) == rarithmetic.widen(y)
-
-def tagged_list_eq(tl1, tl2):
- if len(tl1) != len(tl2):
- return False
- for i in range(len(tl1)):
- if not tagged_eq(tl1[i], tl2[i]):
- return False
- return True
-
TAGCONST = 0
TAGINT = 1
TAGBOX = 2
TAGVIRTUAL = 3
-UNASSIGNED = tag(-2 ** 12 - 1, TAGBOX)
-UNASSIGNEDVIRTUAL = tag(-2 ** 12 - 1, TAGVIRTUAL)
+MINIMUM_VALUE = -2 ** 12
+UNASSIGNED = tag(MINIMUM_VALUE, TAGBOX)
+UNASSIGNEDVIRTUAL = tag(MINIMUM_VALUE, TAGVIRTUAL)
NULLREF = tag(-1, TAGCONST)
+def compress_tagged_list(l):
+ res = ['\x00'] * (len(l) * 4) # maximum size
+ resindex = 0
+ for tagged in l:
+ while True:
+ rest = tagged >> 7
+ # tagged fits into 7 bits, if the remaining int is all zeroes or
+ # all ones
+ fits = (rest == 0) | (rest == -1)
+ # if the highest bit of tagged (which will corresponds to the sign
+ # bit on uncompressing) does not actually correspond with the sign
+ # of tagged, we need to output another byte
+ fits = fits & (bool(tagged & 0x40) == (tagged < 0))
+ res[resindex] = chr((tagged & 0x7f) | 0x80 * (not fits))
+ resindex += 1
+ if fits:
+ break
+ tagged = rest
+ return "".join(res[:resindex])
+
+def _decompress_next(s, i):
+ res = 0
+ shift = 0
+ while True:
+ byte = ord(s[i])
+ i += 1
+ more = bool(byte & 0x80)
+ byte &= 0x7f
+ res += byte << shift
+ if not more:
+ # sign-extend
+ if byte & 0x40:
+ res |= -1 << (shift + 7)
+ break
+ shift += 7
+ return res, i
+
+def uncompress_tagged_list(s):
+ result = [-1] * len(s) # maximum size
+ i = 0
+ resindex = 0
+ while i < len(s):
+ res, i = _decompress_next(s, i)
+ result[resindex] = res
+ resindex += 1
+ return result[:resindex]
+
+def compressed_length(s):
+ res = 0
+ for char in s:
+ if ord(char) & 0x80 == 0:
+ res += 1
+ return res
+
+
+# ____________________________________________________________
class ResumeDataLoopMemo(object):
@@ -130,7 +179,7 @@
except ValueError:
pass
tagged = self.large_ints.get(val, UNASSIGNED)
- if not tagged_eq(tagged, UNASSIGNED):
+ if not tagged == UNASSIGNED:
return tagged
tagged = self._newconst(const)
self.large_ints[val] = tagged
@@ -140,7 +189,7 @@
if not val:
return NULLREF
tagged = self.refs.get(val, UNASSIGNED)
- if not tagged_eq(tagged, UNASSIGNED):
+ if not tagged == UNASSIGNED:
return tagged
tagged = self._newconst(const)
self.refs[val] = tagged
@@ -310,13 +359,13 @@
i, tagbits = untag(tagged)
if tagbits == TAGBOX:
assert box not in self.liveboxes_from_env
- assert tagged_eq(tagged, UNASSIGNED)
+ assert tagged == UNASSIGNED
index = memo.assign_number_to_box(box, new_liveboxes)
self.liveboxes[box] = tag(index, TAGBOX)
count += 1
else:
assert tagbits == TAGVIRTUAL
- if tagged_eq(tagged, UNASSIGNEDVIRTUAL):
+ if tagged == UNASSIGNEDVIRTUAL:
assert box not in self.liveboxes_from_env
index = memo.assign_number_to_virtual(box)
self.liveboxes[box] = tag(index, TAGVIRTUAL)
@@ -337,10 +386,11 @@
value = values[virtualbox]
fieldnums = [self._gettagged(box)
for box in fieldboxes]
- vinfo = value.make_virtual_info(self, fieldnums)
- # if a new vinfo instance is made, we get the fieldnums list we
+ fieldnums_compressed = compress_tagged_list(fieldnums)
+ vinfo = value.make_virtual_info(self, fieldnums_compressed)
+ # if a new vinfo instance is made, we get the string we
# pass in as an attribute. hackish.
- if vinfo.fieldnums is not fieldnums:
+ if vinfo.fieldnums_compressed is not fieldnums_compressed:
memo.nvreused += 1
virtuals[num] = vinfo
@@ -370,25 +420,34 @@
def setfields(self, metainterp, box, fn_decode_box):
raise NotImplementedError
+ def fieldnums(self):
+ return uncompress_tagged_list(self.fieldnums_compressed)
+
class AbstractVirtualStructInfo(AbstractVirtualInfo):
def __init__(self, fielddescrs):
self.fielddescrs = fielddescrs
- #self.fieldnums = ...
+ #self.fieldnums_compressed = ...
def setfields(self, metainterp, box, fn_decode_box):
- for i in range(len(self.fielddescrs)):
- fieldbox = fn_decode_box(self.fieldnums[i])
+ fieldnums_compressed = self.fieldnums_compressed
+ i = 0
+ j = 0
+ while i < len(fieldnums_compressed):
+ tagged, i = _decompress_next(fieldnums_compressed, i)
+ fieldbox = fn_decode_box(tagged)
metainterp.execute_and_record(rop.SETFIELD_GC,
- self.fielddescrs[i],
+ self.fielddescrs[j],
box, fieldbox)
+ j += 1
def debug_prints(self):
- assert len(self.fielddescrs) == len(self.fieldnums)
+ fieldnums = self.fieldnums()
+ assert len(self.fielddescrs) == len(fieldnums)
for i in range(len(self.fielddescrs)):
debug_print("\t\t",
str(self.fielddescrs[i]),
- str(untag(self.fieldnums[i])))
+ str(untag(fieldnums[i])))
class VirtualInfo(AbstractVirtualStructInfo):
def __init__(self, known_class, fielddescrs):
@@ -418,24 +477,30 @@
class VArrayInfo(AbstractVirtualInfo):
def __init__(self, arraydescr):
self.arraydescr = arraydescr
- #self.fieldnums = ...
+ #self.fieldnums_compressed = ...
def allocate(self, metainterp):
- length = len(self.fieldnums)
+ length = compressed_length(self.fieldnums_compressed)
return metainterp.execute_and_record(rop.NEW_ARRAY,
self.arraydescr,
ConstInt(length))
def setfields(self, metainterp, box, fn_decode_box):
- for i in range(len(self.fieldnums)):
- itembox = fn_decode_box(self.fieldnums[i])
+ fieldnums_compressed = self.fieldnums_compressed
+ i = 0
+ j = 0
+ while i < len(fieldnums_compressed):
+ tagged, i = _decompress_next(fieldnums_compressed, i)
+ itembox = fn_decode_box(tagged)
metainterp.execute_and_record(rop.SETARRAYITEM_GC,
self.arraydescr,
- box, ConstInt(i), itembox)
+ box, ConstInt(j), itembox)
+ j += 1
def debug_prints(self):
debug_print("\tvarrayinfo", self.arraydescr)
- for i in self.fieldnums:
+ fieldnums = self.fieldnums()
+ for i in fieldnums:
debug_print("\t\t", str(untag(i)))
@@ -490,18 +555,20 @@
def consume_boxes(self):
numb = self.cur_numb
assert numb is not None
- nums = numb.nums
- n = len(nums)
+ nums_compressed = numb.nums_compressed
+ n = compressed_length(nums_compressed)
boxes = [None] * n
+ j = 0
for i in range(n):
- boxes[i] = self._decode_box(nums[i])
+ tagged, j = _decompress_next(nums_compressed, j)
+ boxes[i] = self._decode_box(tagged)
self.cur_numb = numb.prev
return boxes
def _decode_box(self, tagged):
num, tag = untag(tagged)
if tag == TAGCONST:
- if tagged_eq(tagged, NULLREF):
+ if tagged == NULLREF:
return self.cpu.ts.CONST_NULL
return self.consts[num]
elif tag == TAGVIRTUAL:
@@ -534,7 +601,7 @@
frameinfo = frameinfo.prev
numb = storage.rd_numb
while numb is not None:
- debug_print('\tnumb', str([untag(i) for i in numb.nums]),
+ debug_print('\tnumb', str([untag(i) for i in numb.nums()]),
'at', compute_unique_id(numb))
numb = numb.prev
for const in storage.rd_consts:
Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original)
+++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Wed Dec 2 17:33:10 2009
@@ -53,12 +53,12 @@
#
opt.store_final_boxes_in_guard(op)
if op.fail_args == [b0, b1]:
- assert fdescr.rd_numb.nums == [tag(1, TAGBOX)]
- assert fdescr.rd_numb.prev.nums == [tag(0, TAGBOX)]
+ assert fdescr.rd_numb.nums() == [tag(1, TAGBOX)]
+ assert fdescr.rd_numb.prev.nums() == [tag(0, TAGBOX)]
else:
assert op.fail_args == [b1, b0]
- assert fdescr.rd_numb.nums == [tag(0, TAGBOX)]
- assert fdescr.rd_numb.prev.nums == [tag(1, TAGBOX)]
+ assert fdescr.rd_numb.nums() == [tag(0, TAGBOX)]
+ assert fdescr.rd_numb.prev.nums() == [tag(1, TAGBOX)]
assert fdescr.rd_virtuals is None
assert fdescr.rd_consts == []
Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original)
+++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Wed Dec 2 17:33:10 2009
@@ -1,4 +1,5 @@
import py
+import sys
from pypy.rpython.lltypesystem import lltype, llmemory, rffi
from pypy.jit.metainterp.optimizeopt import VirtualValue, OptValue, VArrayValue
from pypy.jit.metainterp.optimizeopt import VStructValue
@@ -19,10 +20,10 @@
assert tag((1<<13)-1, 3) == rffi.r_short(((1<<15)-1)|3)
assert tag(-1<<13, 3) == rffi.r_short((-1<<15)|3)
py.test.raises(ValueError, tag, 3, 5)
- py.test.raises(ValueError, tag, 1<<13, 0)
- py.test.raises(ValueError, tag, (1<<13)+1, 0)
- py.test.raises(ValueError, tag, (-1<<13)-1, 0)
- py.test.raises(ValueError, tag, (-1<<13)-5, 0)
+ py.test.raises(ValueError, tag, sys.maxint, 0)
+ py.test.raises(ValueError, tag, sys.maxint//2, 0)
+ py.test.raises(ValueError, tag, -sys.maxint-1, 0)
+ py.test.raises(ValueError, tag, -sys.maxint//2, 0)
def test_untag():
assert untag(tag(3, 1)) == (3, 1)
@@ -30,15 +31,20 @@
assert untag(tag((1<<13)-1, 3)) == ((1<<13)-1, 3)
assert untag(tag(-1<<13, 3)) == (-1<<13, 3)
-def test_tagged_eq():
- assert tagged_eq(UNASSIGNED, UNASSIGNED)
- assert not tagged_eq(tag(1, TAGBOX), UNASSIGNED)
-
-def test_tagged_list_eq():
- assert tagged_list_eq([UNASSIGNED, tag(1, TAGBOX), tag(-2, TAGVIRTUAL)],
- [UNASSIGNED, tag(1, TAGBOX), tag(-2, TAGVIRTUAL)])
- assert not tagged_list_eq([tag(1, TAGBOX)], [tag(-2, TAGBOX)])
- assert not tagged_list_eq([tag(1, TAGBOX), tag(-2, TAGBOX)], [tag(1, TAGBOX)])
+def test_compress_tagged_list():
+ assert compress_tagged_list([0, 1, 12, 1024]) == "\x00\x01\x0c\x80\x08"
+ assert compress_tagged_list([-1, 1, 12]) == "\x7f\x01\x0c"
+ for i in range(256):
+ assert uncompress_tagged_list(compress_tagged_list([i])) == [i]
+ l = range(-10000, 10000) + range(-100000000, 100000000, 10000)
+ compressed = compress_tagged_list(l)
+ assert uncompress_tagged_list(compressed) == l
+ assert compressed_length(compressed) == len(l)
+ for i in range(-64, 64):
+ assert len(compress_tagged_list([i])) == 1
+ for i in range(-8192, -64) + range(64, 8192):
+ assert len(compress_tagged_list([i])) == 2
+
class MyMetaInterp:
_already_allocated_resume_virtuals = None
@@ -190,12 +196,12 @@
l = [1, 2]
numb = Numbering(None, l)
assert numb.prev is None
- assert numb.nums is l
+ assert numb.nums_compressed == "\x01\x02"
- l1 = ['b3']
+ l1 = [3]
numb1 = Numbering(numb, l1)
assert numb1.prev is numb
- assert numb1.nums is l1
+ assert numb1.nums_compressed == "\x03"
def test_capture_resumedata():
b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()]
@@ -394,8 +400,8 @@
modifier = ResumeDataVirtualAdder(storage, memo)
liveboxes = modifier.finish(values)
assert len(storage.rd_virtuals) == 1
- assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX),
- tag(0, TAGCONST)]
+ assert storage.rd_virtuals[0].fieldnums() == [tag(-1, TAGBOX),
+ tag(0, TAGCONST)]
b6 = BoxPtr()
v6 = virtual_value(b6, c2, None)
@@ -405,10 +411,10 @@
modifier = ResumeDataVirtualAdder(storage2, memo)
liveboxes2 = modifier.finish(values)
assert len(storage2.rd_virtuals) == 2
- assert storage2.rd_virtuals[0].fieldnums == [tag(len(liveboxes2)-1, TAGBOX),
- tag(-1, TAGVIRTUAL)]
- assert storage2.rd_virtuals[1].fieldnums == [tag(2, TAGINT),
- tag(-1, TAGVIRTUAL)]
+ assert storage2.rd_virtuals[0].fieldnums() == [tag(len(liveboxes2)-1, TAGBOX),
+ tag(-1, TAGVIRTUAL)]
+ assert storage2.rd_virtuals[1].fieldnums() == [tag(2, TAGINT),
+ tag(-1, TAGVIRTUAL)]
# now on to resuming
metainterp = MyMetaInterp()
@@ -450,8 +456,8 @@
modifier = ResumeDataVirtualAdder(storage, memo)
liveboxes = modifier.finish(values)
assert len(storage.rd_virtuals) == 1
- assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX),
- tag(0, TAGCONST)]
+ assert storage.rd_virtuals[0].fieldnums() == [tag(-1, TAGBOX),
+ tag(0, TAGCONST)]
storage2 = Storage()
fs = [FakeFrame("code0", 0, -1, b1, b4, b2)]
@@ -460,7 +466,7 @@
modifier = ResumeDataVirtualAdder(storage2, memo)
liveboxes = modifier.finish(values)
assert len(storage2.rd_virtuals) == 2
- assert storage2.rd_virtuals[1].fieldnums == storage.rd_virtuals[0].fieldnums
+ assert storage2.rd_virtuals[1].fieldnums() == storage.rd_virtuals[0].fieldnums()
assert storage2.rd_virtuals[1] is storage.rd_virtuals[0]
@@ -479,10 +485,10 @@
liveboxes = modifier.finish(values)
assert liveboxes == [b3]
assert len(storage.rd_virtuals) == 2
- assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX),
- tag(1, TAGVIRTUAL)]
- assert storage.rd_virtuals[1].fieldnums == [tag(-1, TAGBOX),
- tag(0, TAGVIRTUAL)]
+ assert storage.rd_virtuals[0].fieldnums() == [tag(-1, TAGBOX),
+ tag(1, TAGVIRTUAL)]
+ assert storage.rd_virtuals[1].fieldnums() == [tag(-1, TAGBOX),
+ tag(0, TAGVIRTUAL)]
# ____________________________________________________________
@@ -494,12 +500,12 @@
assert untag(tagged) == (44, TAGINT)
tagged = memo.getconst(ConstInt(-3))
assert untag(tagged) == (-3, TAGINT)
- const = ConstInt(50000)
+ const = ConstInt(sys.maxint)
tagged = memo.getconst(const)
index, tagbits = untag(tagged)
assert tagbits == TAGCONST
assert memo.consts[index] is const
- tagged = memo.getconst(ConstInt(50000))
+ tagged = memo.getconst(ConstInt(sys.maxint))
index2, tagbits = untag(tagged)
assert tagbits == TAGCONST
assert index2 == index
@@ -554,10 +560,10 @@
assert liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
b3: tag(2, TAGBOX)}
- assert numb.nums == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX),
- tag(1, TAGINT)]
- assert numb.prev.nums == [tag(0, TAGBOX), tag(1, TAGINT), tag(1, TAGBOX),
- tag(0, TAGBOX), tag(2, TAGINT)]
+ assert numb.nums() == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX),
+ tag(1, TAGINT)]
+ assert numb.prev.nums() == [tag(0, TAGBOX), tag(1, TAGINT), tag(1, TAGBOX),
+ tag(0, TAGBOX), tag(2, TAGINT)]
assert numb.prev.prev is None
numb2, liveboxes2, v = memo.number({}, snap2)
@@ -566,8 +572,8 @@
assert liveboxes2 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
b3: tag(2, TAGBOX)}
assert liveboxes2 is not liveboxes
- assert numb2.nums == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX),
- tag(3, TAGINT)]
+ assert numb2.nums() == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX),
+ tag(3, TAGINT)]
assert numb2.prev is numb.prev
env3 = [c3, b3, b1, c3]
@@ -589,8 +595,8 @@
assert v == 0
assert liveboxes3 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX)}
- assert numb3.nums == [tag(3, TAGINT), tag(4, TAGINT), tag(0, TAGBOX),
- tag(3, TAGINT)]
+ assert numb3.nums() == [tag(3, TAGINT), tag(4, TAGINT), tag(0, TAGBOX),
+ tag(3, TAGINT)]
assert numb3.prev is numb.prev
# virtual
@@ -602,8 +608,8 @@
assert liveboxes4 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
b4: tag(0, TAGVIRTUAL)}
- assert numb4.nums == [tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX),
- tag(3, TAGINT)]
+ assert numb4.nums() == [tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX),
+ tag(3, TAGINT)]
assert numb4.prev is numb.prev
env5 = [b1, b4, b5]
@@ -615,8 +621,8 @@
assert liveboxes5 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
b4: tag(0, TAGVIRTUAL), b5: tag(1, TAGVIRTUAL)}
- assert numb5.nums == [tag(0, TAGBOX), tag(0, TAGVIRTUAL),
- tag(1, TAGVIRTUAL)]
+ assert numb5.nums() == [tag(0, TAGBOX), tag(0, TAGVIRTUAL),
+ tag(1, TAGVIRTUAL)]
assert numb5.prev is numb4
def test_ResumeDataLoopMemo_number_boxes():
@@ -732,14 +738,14 @@
memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
modifier = ResumeDataVirtualAdder(storage, memo)
modifier.finish({})
- assert len(memo.consts) == 2
+ assert len(memo.consts) == 1
assert storage.rd_consts is memo.consts
- b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**17), ConstInt(-65)]
+ b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(-sys.maxint), ConstInt(-65)]
storage2 = make_storage(b1s, b2s, b3s)
modifier2 = ResumeDataVirtualAdder(storage2, memo)
modifier2.finish({})
- assert len(memo.consts) == 3
+ assert len(memo.consts) == 2
assert storage2.rd_consts is memo.consts
More information about the Pypy-commit
mailing list