[pypy-svn] r65263 - in pypy/branch/pyjitpl5/pypy: jit/backend/x86 jit/backend/x86/test rpython/memory/gctransform
arigo at codespeak.net
arigo at codespeak.net
Thu May 14 15:38:21 CEST 2009
Author: arigo
Date: Thu May 14 15:38:19 2009
New Revision: 65263
Modified:
pypy/branch/pyjitpl5/pypy/jit/backend/x86/gc.py
pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_gc.py
pypy/branch/pyjitpl5/pypy/rpython/memory/gctransform/asmgcroot.py
pypy/branch/pyjitpl5/pypy/rpython/memory/gctransform/framework.py
Log:
In-progress: stack root enumeration.
Modified: pypy/branch/pyjitpl5/pypy/jit/backend/x86/gc.py
==============================================================================
--- pypy/branch/pyjitpl5/pypy/jit/backend/x86/gc.py (original)
+++ pypy/branch/pyjitpl5/pypy/jit/backend/x86/gc.py Thu May 14 15:38:19 2009
@@ -5,6 +5,7 @@
from pypy.translator.tool.cbuild import ExternalCompilationInfo
from pypy.jit.backend.x86 import symbolic
from pypy.jit.backend.x86.runner import ConstDescr3
+from pypy.jit.backend.x86.ri386 import MODRM
# ____________________________________________________________
@@ -41,9 +42,9 @@
def gc_malloc_array(self, arraydescr, num_elem):
assert isinstance(arraydescr, ConstDescr3)
- size_of_field = arraydescr.v0
- ofs = arraydescr.v1
- size = ofs + (1 << size_of_field) * num_elem
+ basesize = arraydescr.v0
+ itemsize = arraydescr.v1
+ size = basesize + itemsize * num_elem
return self.funcptr_for_new(size)
def args_for_new(self, descrsize):
@@ -121,6 +122,94 @@
return addr_ref
+class GcRootMap_asmgcc:
+ """Handles locating the stack roots in the assembler.
+ This is the class supporting --gcrootfinder=asmgcc.
+ """
+ LOC_NOWHERE = 0
+ LOC_REG = 1
+ LOC_EBP_BASED = 2
+ LOC_ESP_BASED = 3
+
+ GCMAP_ARRAY = rffi.CArray(llmemory.Address)
+ CALLSHAPE_ARRAY = rffi.CArray(rffi.UCHAR)
+
+ def __init__(self):
+ self._gcmap = lltype.malloc(self.GCMAP_ARRAY, 0, flavor='raw')
+ self._gcmap_curlength = 0
+ self._gcmap_maxlength = 0
+
+ def gcmapstart(self):
+ return llmemory.cast_ptr_to_adr(self._gcmap)
+
+ def gcmapend(self):
+ start = self.gcmapstart()
+ return start + llmemory.sizeof(lltype.Signed) * self._gcmap_curlength
+
+ def put(self, retaddr, callshapeaddr):
+ """'retaddr' is the address just after the CALL.
+ 'callshapeaddr' is the address returned by encode_callshape()."""
+ index = self._gcmap_curlength
+ if index + 2 > self._gcmap_maxlength:
+ self._enlarge_gcmap()
+ self._gcmap[index] = retaddr
+ self._gcmap[index+1] = callshapeaddr
+ self._gcmap_curlength = index + 2
+
+ def _enlarge_gcmap(self):
+ newlength = 128 + self._gcmap_maxlength // 4
+ newgcmap = lltype.malloc(self.GCMAP_ARRAY, newlength, flavor='raw')
+ oldgcmap = self._gcmap
+ for i in range(self._gcmap_curlength):
+ newgcmap[i] = oldgcmap[i]
+ self._gcmap = newgcmap
+ self._gcmap_maxlength = newlength
+ lltype.free(oldgcmap, flavor='raw')
+
+ def encode_callshape(self, gclocs, framesize):
+ """Encode a callshape from the list of locations containing GC
+ pointers and from the frame size of the current (caller) frame.
+ The framesize gives the offset from %esp to the return address
+ of the current frame."""
+ shape = self._get_callshape(gclocs, framesize)
+ return self._compress_callshape(shape)
+
+ def _get_callshape(self, gclocs, framesize):
+ # the four registers %ebx, %esi, %edi, %ebp are not used at all
+ # so far, so their value always comes from the caller.
+ shape = [self.LOC_ESP_BASED | framesize,
+ self.LOC_REG | 0,
+ self.LOC_REG | 4,
+ self.LOC_REG | 8,
+ self.LOC_REG | 12,
+ 0]
+ for loc in gclocs:
+ assert isinstance(loc, MODRM)
+ shape.append(self.LOC_ESP_BASED | (4 * loc.position))
+ return shape
+
+ def _compress_callshape(self, shape):
+ # Similar to compress_callshape() in trackgcroot.py. XXX a bit slowish
+ result = []
+ for loc in shape:
+ assert loc >= 0
+ loc = loc * 2
+ flag = 0
+ while loc >= 0x80:
+ result.append(int(loc & 0x7F) | flag)
+ flag = 0x80
+ loc >>= 7
+ result.append(int(loc) | flag)
+ # XXX so far, we always allocate a new small array (we could regroup
+ # them inside bigger arrays) and we never try to share them.
+ length = len(result)
+ compressed = lltype.malloc(self.CALLSHAPE_ARRAY, length,
+ flavor='raw')
+ for i in range(length):
+ compressed[length-1-i] = rffi.cast(rffi.UCHAR, result[i])
+ return llmemory.cast_ptr_to_adr(compressed)
+
+
class GcLLDescr_framework(GcLLDescription):
GcRefList = GcRefList
@@ -129,10 +218,23 @@
from pypy.rpython.memory.gctransform import framework
self.translator = mixlevelann.rtyper.annotator.translator
+ # to find roots in the assembler, make a GcRootMap
+ name = gcdescr.config.translation.gcrootfinder
+ try:
+ cls = globals()['GcRootMap_' + name]
+ except KeyError:
+ raise NotImplementedError("--gcrootfinder=%s not implemented"
+ " with the JIT" % (name,))
+ self.gcrootmap = cls()
+
# make a TransformerLayoutBuilder and save it on the translator
# where it can be fished and reused by the FrameworkGCTransformer
self.layoutbuilder = framework.TransformerLayoutBuilder()
- self.translator._transformerlayoutbuilder_from_jit = self.layoutbuilder
+ self.translator._jit2gc = {
+ 'layoutbuilder': self.layoutbuilder,
+ 'gcmapstart': self.gcrootmap.gcmapstart,
+ 'gcmapend': self.gcrootmap.gcmapend,
+ }
GCClass, _ = choose_gc_from_config(gcdescr.config)
self.moving_gc = GCClass.moving_gc
@@ -145,10 +247,6 @@
self.GC_MALLOC_BASIC = lltype.Ptr(lltype.FuncType(
[lltype.Signed, lltype.Signed, lltype.Bool], llmemory.GCREF))
- assert gcdescr.config.translation.gcrootfinder == "asmgcc", (
- "with the framework GCs, you must use"
- " --gcrootfinder=asmgcc for now")
-
def sizeof(self, S, translate_support_code):
from pypy.rpython.memory.gctypelayout import weakpointer_offset
assert translate_support_code, "required with the framework GC"
Modified: pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_gc.py
==============================================================================
--- pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_gc.py (original)
+++ pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_gc.py Thu May 14 15:38:19 2009
@@ -11,45 +11,39 @@
from pypy.rpython.lltypesystem.lloperation import llop
from pypy.rlib.jit import JitDriver
from pypy.jit.backend.x86.runner import CPU386
-from pypy.jit.backend.x86.gc import GcRefList
-
-
-myjitdriver = JitDriver(greens = [], reds = ['n', 'x'])
+from pypy.jit.backend.x86.gc import GcRefList, GcRootMap_asmgcc
+from pypy.jit.backend.x86.regalloc import stack_pos
class X(object):
pass
-def main(n, x):
- while n > 0:
- myjitdriver.can_enter_jit(n=n, x=x)
- myjitdriver.jit_merge_point(n=n, x=x)
- y = X()
- y.foo = x.foo
- n -= y.foo
-main._dont_inline_ = True
-
-def g(n):
- x = X()
- x.foo = 2
- main(n, x)
- x.foo = 5
- return weakref.ref(x)
-g._dont_inline_ = True
-
-def entrypoint(args):
- r_list = []
- for i in range(20):
- r = g(1000)
- r_list.append(r)
- rgc.collect()
- rgc.collect(); rgc.collect()
- freed = 0
- for r in r_list:
- if r() is None:
- freed += 1
- print freed
- return 0
+def get_test(main):
+ main._dont_inline_ = True
+
+ def g(n):
+ x = X()
+ x.foo = 2
+ main(n, x)
+ x.foo = 5
+ return weakref.ref(x)
+ g._dont_inline_ = True
+
+ def entrypoint(args):
+ r_list = []
+ for i in range(20):
+ r = g(1000)
+ r_list.append(r)
+ rgc.collect()
+ rgc.collect(); rgc.collect()
+ freed = 0
+ for r in r_list:
+ if r() is None:
+ freed += 1
+ print freed
+ return 0
+
+ return entrypoint
def compile_and_run(f, gc, **kwds):
@@ -74,7 +68,15 @@
def test_compile_boehm():
- res = compile_and_run(entrypoint, "boehm", jit=True)
+ myjitdriver = JitDriver(greens = [], reds = ['n', 'x'])
+ def main(n, x):
+ while n > 0:
+ myjitdriver.can_enter_jit(n=n, x=x)
+ myjitdriver.jit_merge_point(n=n, x=x)
+ y = X()
+ y.foo = x.foo
+ n -= y.foo
+ res = compile_and_run(get_test(main), "boehm", jit=True)
assert int(res) >= 16
def test_GcRefList():
@@ -93,8 +95,47 @@
return 0
compile_and_run(fn, "hybrid", gcrootfinder="asmgcc", jit=False)
-def test_compile_hybrid():
- # a moving GC, with a write barrier. Supports malloc_varsize_nonmovable.
- res = compile_and_run(entrypoint, "hybrid", gcrootfinder="asmgcc",
+def test_compile_hybrid_1():
+ # a moving GC. Supports malloc_varsize_nonmovable. Simple test, works
+ # without write_barriers and root stack enumeration.
+ myjitdriver = JitDriver(greens = [], reds = ['n', 'x'])
+ def main(n, x):
+ while n > 0:
+ myjitdriver.can_enter_jit(n=n, x=x)
+ myjitdriver.jit_merge_point(n=n, x=x)
+ y = X()
+ y.foo = x.foo
+ n -= y.foo
+ res = compile_and_run(get_test(main), "hybrid", gcrootfinder="asmgcc",
+ jit=True)
+ assert int(res) == 20
+
+def test_GcRootMap_asmgcc():
+ gcrootmap = GcRootMap_asmgcc()
+ shape = gcrootmap._get_callshape([stack_pos(1), stack_pos(55)], 236)
+ assert shape == [236|3, 1, 5, 9, 13, 0, 4|3, 220|3]
+ #
+ addr = gcrootmap.encode_callshape([stack_pos(1), stack_pos(55)], 236)
+ PCALLSHAPE = lltype.Ptr(GcRootMap_asmgcc.CALLSHAPE_ARRAY)
+ p = llmemory.cast_adr_to_ptr(addr, PCALLSHAPE)
+ for i, expected in enumerate([131, 62, 14, 0, 26, 18, 10, 2, 131, 94]):
+ assert p[i] == expected
+
+def test_compile_hybrid_2():
+ py.test.skip("in-progress")
+ # a moving GC. Supports malloc_varsize_nonmovable. More complex test,
+ # requires root stack enumeration but not write_barriers.
+ myjitdriver = JitDriver(greens = [], reds = ['n', 'x'])
+ def main(n, x):
+ while n > 0:
+ myjitdriver.can_enter_jit(n=n, x=x)
+ myjitdriver.jit_merge_point(n=n, x=x)
+ prev = x
+ for j in range(101): # main() runs 20'000 times, thus allocates
+ y = X() # a total of 2'020'000 objects
+ y.foo = prev.foo
+ prev = y
+ n -= prev.foo
+ res = compile_and_run(get_test(main), "hybrid", gcrootfinder="asmgcc",
jit=True)
assert int(res) == 20
Modified: pypy/branch/pyjitpl5/pypy/rpython/memory/gctransform/asmgcroot.py
==============================================================================
--- pypy/branch/pyjitpl5/pypy/rpython/memory/gctransform/asmgcroot.py (original)
+++ pypy/branch/pyjitpl5/pypy/rpython/memory/gctransform/asmgcroot.py Thu May 14 15:38:19 2009
@@ -46,6 +46,14 @@
self.walk_stack_from(initialframedata)
self._asm_callback = _asm_callback
self._shape_decompressor = ShapeDecompressor()
+ if hasattr(gctransformer.translator, '_jit2gc'):
+ jit2gc = gctransformer.translator._jit2gc
+ self._extra_gcmapstart = jit2gc['gcmapstart']
+ self._extra_gcmapend = jit2gc['gcmapend']
+ else:
+ returns_null = lambda: llmemory.NULL
+ self._extra_gcmapstart = returns_null
+ self._extra_gcmapend = returns_null
def walk_stack_roots(self, collect_stack_root):
gcdata = self.gcdata
@@ -128,25 +136,12 @@
retaddr = callee.frame_address.address[0]
#
# try to locate the caller function based on retaddr.
+ # set up self._shape_decompressor.
#
- gcmapstart = llop.llvm_gcmapstart(llmemory.Address)
- gcmapend = llop.llvm_gcmapend(llmemory.Address)
- item = search_in_gcmap(gcmapstart, gcmapend, retaddr)
- if not item:
- # the item may have been not found because the array was
- # not sorted. Sort it and try again.
- sort_gcmap(gcmapstart, gcmapend)
- item = search_in_gcmap(gcmapstart, gcmapend, retaddr)
- if not item:
- llop.debug_fatalerror(lltype.Void, "cannot find gc roots!")
- return False
+ self.locate_caller_based_on_retaddr(retaddr)
#
# found! Enumerate the GC roots in the caller frame
#
- shape = item.signed[1]
- if shape < 0:
- shape = ~ shape # can ignore this "range" marker here
- self._shape_decompressor.setpos(shape)
collect_stack_root = self.gcdata._gc_collect_stack_root
gc = self.gc
while True:
@@ -173,6 +168,36 @@
# of the entry point, stop walking"
return caller.frame_address != llmemory.NULL
+ def locate_caller_based_on_retaddr(self, retaddr):
+ gcmapstart = llop.llvm_gcmapstart(llmemory.Address)
+ gcmapend = llop.llvm_gcmapend(llmemory.Address)
+ item = search_in_gcmap(gcmapstart, gcmapend, retaddr)
+ if item:
+ self._shape_decompressor.setpos(item.signed[1])
+ return
+ gcmapstart2 = self._extra_gcmapstart()
+ gcmapend2 = self._extra_gcmapend()
+ if gcmapstart2 != gcmapend2:
+ # we have a non-empty JIT-produced table to look in
+ item = search_in_gcmap(gcmapstart2, gcmapend2, retaddr)
+ if item:
+ self._shape_decompressor.setaddr(item.address[1])
+ return
+ # maybe the JIT-produced table is not sorted?
+ sort_gcmap(gcmapstart2, gcmapend2)
+ item = search_in_gcmap(gcmapstart2, gcmapend2, retaddr)
+ if item:
+ self._shape_decompressor.setaddr(item.address[1])
+ return
+ # the item may have been not found because the main array was
+ # not sorted. Sort it and try again.
+ sort_gcmap(gcmapstart, gcmapend)
+ item = search_in_gcmap(gcmapstart, gcmapend, retaddr)
+ if item:
+ self._shape_decompressor.setpos(item.signed[1])
+ return
+ llop.debug_fatalerror(lltype.Void, "cannot find gc roots!")
+
def getlocation(self, callee, location):
"""Get the location in the 'caller' frame of a variable, based
on the integer 'location' that describes it. All locations are
@@ -261,9 +286,14 @@
_alloc_flavor_ = "raw"
def setpos(self, pos):
+ if pos < 0:
+ pos = ~ pos # can ignore this "range" marker here
gccallshapes = llop.llvm_gccallshapes(llmemory.Address)
self.addr = gccallshapes + pos
+ def setaddr(self, addr):
+ self.addr = addr
+
def next(self):
value = 0
addr = self.addr
Modified: pypy/branch/pyjitpl5/pypy/rpython/memory/gctransform/framework.py
==============================================================================
--- pypy/branch/pyjitpl5/pypy/rpython/memory/gctransform/framework.py (original)
+++ pypy/branch/pyjitpl5/pypy/rpython/memory/gctransform/framework.py Thu May 14 15:38:19 2009
@@ -124,8 +124,8 @@
# for regular translation: pick the GC from the config
GCClass, GC_PARAMS = choose_gc_from_config(translator.config)
- if hasattr(translator, '_transformerlayoutbuilder_from_jit'):
- self.layoutbuilder = translator._transformerlayoutbuilder_from_jit
+ if hasattr(translator, '_jit2gc'):
+ self.layoutbuilder = translator._jit2gc['layoutbuilder']
else:
self.layoutbuilder = TransformerLayoutBuilder()
self.layoutbuilder.transformer = self
More information about the Pypy-commit
mailing list