[pypy-svn] r32857 - in pypy/dist/pypy/rpython: . memory/gctransform2 memory/gctransform2/test
mwh at codespeak.net
mwh at codespeak.net
Tue Oct 3 19:46:45 CEST 2006
Author: mwh
Date: Tue Oct 3 19:46:43 2006
New Revision: 32857
Added:
pypy/dist/pypy/rpython/memory/gctransform2/
pypy/dist/pypy/rpython/memory/gctransform2/__init__.py (contents, props changed)
pypy/dist/pypy/rpython/memory/gctransform2/refcounting.py (contents, props changed)
pypy/dist/pypy/rpython/memory/gctransform2/support.py (contents, props changed)
pypy/dist/pypy/rpython/memory/gctransform2/test/
pypy/dist/pypy/rpython/memory/gctransform2/test/__init__.py (contents, props changed)
pypy/dist/pypy/rpython/memory/gctransform2/test/test_refcounting.py (contents, props changed)
pypy/dist/pypy/rpython/memory/gctransform2/test/test_transform.py (contents, props changed)
pypy/dist/pypy/rpython/memory/gctransform2/transform.py (contents, props changed)
Modified:
pypy/dist/pypy/rpython/llinterp.py
Log:
start a much more rtyper-ish rewrite of the gctransformer.
only the base class and the refcounting transformer done so far.
Modified: pypy/dist/pypy/rpython/llinterp.py
==============================================================================
--- pypy/dist/pypy/rpython/llinterp.py (original)
+++ pypy/dist/pypy/rpython/llinterp.py Tue Oct 3 19:46:43 2006
@@ -635,6 +635,7 @@
assert not isinstance(getattr(lltype.typeOf(obj).TO, field),
lltype.ContainerType)
return getattr(obj, field)
+ op_bare_getfield = op_getfield
def op_cast_int_to_ptr(self, RESTYPE, int1):
return lltype.cast_int_to_ptr(RESTYPE, int1)
Added: pypy/dist/pypy/rpython/memory/gctransform2/__init__.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/rpython/memory/gctransform2/__init__.py Tue Oct 3 19:46:43 2006
@@ -0,0 +1 @@
+#
Added: pypy/dist/pypy/rpython/memory/gctransform2/refcounting.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/rpython/memory/gctransform2/refcounting.py Tue Oct 3 19:46:43 2006
@@ -0,0 +1,367 @@
+import py
+from pypy.rpython.memory.gctransform2.transform import GCTransformer
+from pypy.rpython.memory.gctransform2.support import find_gc_ptrs_in_type
+from pypy.rpython.lltypesystem import lltype, llmemory
+from pypy.rpython.lltypesystem.lloperation import llop
+from pypy.translator.backendopt.support import var_needsgc
+from pypy.rpython import rmodel
+from pypy.rpython.memory import lladdress
+from pypy.rpython.memory.gcheader import GCHeaderBuilder
+#from pypy.rpython.extregistry import ExtRegistryEntry
+from pypy.rpython.rarithmetic import ovfcheck
+from pypy.rpython.extregistry import ExtRegistryEntry
+from pypy.annotation import model as annmodel
+from pypy.rpython.rbuiltin import gen_cast
+import os, sys
+
+
+class LLTransformerOp(object):
+ """Objects that can be called in ll functions.
+ Their calls are replaced by a simple operation of the GC transformer,
+ e.g. ll_pop_alive.
+ """
+ def __init__(self, transformer_method):
+ self.transformer_method = transformer_method
+
+class LLTransformerOpEntry(ExtRegistryEntry):
+ "Annotation and specialization of LLTransformerOp() instances."
+ _type_ = LLTransformerOp
+
+ def compute_result_annotation(self, s_arg):
+ return annmodel.s_None
+
+ def specialize_call(self, hop):
+ op = self.instance # the LLTransformerOp instance
+ op.transformer_method(hop.args_v[0], hop.llops)
+ hop.exception_cannot_occur()
+ return hop.inputconst(hop.r_result.lowleveltype, hop.s_result.const)
+
+def ll_call_destructor(destrptr, destr_v):
+ try:
+ destrptr(destr_v)
+ except:
+ try:
+ os.write(2, "a destructor raised an exception, ignoring it\n")
+ except:
+ pass
+
+def _static_deallocator_body_for_type(v, TYPE, depth=1):
+ if isinstance(TYPE, lltype.Array):
+ inner = list(_static_deallocator_body_for_type('v_%i'%depth, TYPE.OF, depth+1))
+ if inner:
+ yield ' '*depth + 'i_%d = 0'%(depth,)
+ yield ' '*depth + 'l_%d = len(%s)'%(depth, v)
+ yield ' '*depth + 'while i_%d < l_%d:'%(depth, depth)
+ yield ' '*depth + ' v_%d = %s[i_%d]'%(depth, v, depth)
+ for line in inner:
+ yield line
+ yield ' '*depth + ' i_%d += 1'%(depth,)
+ elif isinstance(TYPE, lltype.Struct):
+ for name in TYPE._names:
+ inner = list(_static_deallocator_body_for_type(
+ v + '_' + name, TYPE._flds[name], depth))
+ if inner:
+ yield ' '*depth + v + '_' + name + ' = ' + v + '.' + name
+ for line in inner:
+ yield line
+ elif isinstance(TYPE, lltype.Ptr) and TYPE._needsgc():
+ yield ' '*depth + 'pop_alive(%s)'%v
+
+counts = {}
+
+## def print_call_chain(ob):
+## import sys
+## f = sys._getframe(1)
+## stack = []
+## flag = False
+## while f:
+## if f.f_locals.get('self') is ob:
+## stack.append((f.f_code.co_name, f.f_locals.get('TYPE')))
+## if not flag:
+## counts[f.f_code.co_name] = counts.get(f.f_code.co_name, 0) + 1
+## print counts
+## flag = True
+## f = f.f_back
+## stack.reverse()
+## for i, (a, b) in enumerate(stack):
+## print ' '*i, a, repr(b)[:100-i-len(a)], id(b)
+
+ADDRESS_VOID_FUNC = lltype.FuncType([llmemory.Address], lltype.Void)
+
+def get_rtti(TYPE):
+ if isinstance(TYPE, lltype.RttiStruct):
+ try:
+ return lltype.getRuntimeTypeInfo(TYPE)
+ except ValueError:
+ pass
+ return None
+
+class RefcountingGCTransformer(GCTransformer):
+
+ HDR = lltype.Struct("header", ("refcount", lltype.Signed))
+
+ def __init__(self, translator):
+ super(RefcountingGCTransformer, self).__init__(translator, inline=True)
+ self.gcheaderbuilder = GCHeaderBuilder(self.HDR)
+ gc_header_offset = self.gcheaderbuilder.size_gc_header
+ self.deallocator_graphs_needing_transforming = []
+
+ # create incref, etc graph
+
+ HDRPTR = lltype.Ptr(self.HDR)
+ def ll_incref(adr):
+ if adr:
+ gcheader = llmemory.cast_adr_to_ptr(adr - gc_header_offset, HDRPTR)
+ gcheader.refcount = gcheader.refcount + 1
+ def ll_decref(adr, dealloc):
+ if adr:
+ gcheader = llmemory.cast_adr_to_ptr(adr - gc_header_offset, HDRPTR)
+ refcount = gcheader.refcount - 1
+ gcheader.refcount = refcount
+ if refcount == 0:
+ dealloc(adr)
+ def ll_decref_simple(adr):
+ if adr:
+ gcheader = llmemory.cast_adr_to_ptr(adr - gc_header_offset, HDRPTR)
+ refcount = gcheader.refcount - 1
+ if refcount == 0:
+ llop.gc_free(lltype.Void, adr)
+ else:
+ gcheader.refcount = refcount
+ def ll_no_pointer_dealloc(adr):
+ llop.gc_free(lltype.Void, adr)
+
+ def ll_malloc_fixedsize(size):
+ size = gc_header_offset + size
+ result = lladdress.raw_malloc(size)
+ lladdress.raw_memclear(result, size)
+ result += gc_header_offset
+ return result
+ def ll_malloc_varsize_no_length(length, size, itemsize):
+ try:
+ fixsize = gc_header_offset + size
+ varsize = ovfcheck(itemsize * length)
+ tot_size = ovfcheck(fixsize + varsize)
+ except OverflowError:
+ raise MemoryError
+ result = lladdress.raw_malloc(tot_size)
+ lladdress.raw_memclear(result, tot_size)
+ result += gc_header_offset
+ return result
+ def ll_malloc_varsize(length, size, itemsize, lengthoffset):
+ result = ll_malloc_varsize_no_length(length, size, itemsize)
+ (result + lengthoffset).signed[0] = length
+ return result
+
+ if self.translator:
+ self.increfptr = self.inittime_helper(
+ ll_incref, [llmemory.Address], lltype.Void)
+ self.decref_ptr = self.inittime_helper(
+ ll_decref, [llmemory.Address, lltype.Ptr(ADDRESS_VOID_FUNC)],
+ lltype.Void)
+ self.decref_simple_ptr = self.inittime_helper(
+ ll_decref_simple, [llmemory.Address], lltype.Void)
+ self.no_pointer_dealloc_ptr = self.inittime_helper(
+ ll_no_pointer_dealloc, [llmemory.Address], lltype.Void)
+ self.malloc_fixedsize_ptr = self.inittime_helper(
+ ll_malloc_fixedsize, [lltype.Signed], llmemory.Address)
+ self.malloc_varsize_no_length_ptr = self.inittime_helper(
+ ll_malloc_varsize_no_length, [lltype.Signed]*3, llmemory.Address)
+ self.malloc_varsize_ptr = self.inittime_helper(
+ ll_malloc_varsize, [lltype.Signed]*4, llmemory.Address)
+ self.mixlevelannotator.finish() # for now
+ # cache graphs:
+ self.decref_funcptrs = {}
+ self.static_deallocator_funcptrs = {}
+ self.dynamic_deallocator_funcptrs = {}
+ self.queryptr2dynamic_deallocator_funcptr = {}
+
+ def var_needs_set_transform(self, var):
+ return var_needsgc(var)
+
+ def push_alive_nopyobj(self, var, llops):
+ v_adr = gen_cast(llops, llmemory.Address, var)
+ llops.genop("direct_call", [self.increfptr, v_adr])
+
+ def pop_alive_nopyobj(self, var, llops):
+ PTRTYPE = var.concretetype
+ v_adr = gen_cast(llops, llmemory.Address, var)
+
+ dealloc_fptr = self.dynamic_deallocation_funcptr_for_type(PTRTYPE.TO)
+ if dealloc_fptr is self.no_pointer_dealloc_ptr.value:
+ # simple case
+ llops.genop("direct_call", [self.decref_simple_ptr, v_adr])
+ else:
+ cdealloc_fptr = rmodel.inputconst(
+ lltype.typeOf(dealloc_fptr), dealloc_fptr)
+ llops.genop("direct_call", [self.decref_ptr, v_adr, cdealloc_fptr])
+
+ def gct_gc_protect(self, hop):
+ """ protect this object from gc (make it immortal) """
+ self.push_alive(hop.spaceop.args[0])
+
+ def gct_gc_unprotect(self, hop):
+ """ get this object back into gc control """
+ self.pop_alive(hop.spaceop.args[0])
+
+ def gct_malloc(self, hop):
+ TYPE = hop.spaceop.result.concretetype.TO
+ assert not TYPE._is_varsize()
+ c_size = rmodel.inputconst(lltype.Signed, llmemory.sizeof(TYPE))
+ v_raw = hop.genop("direct_call", [self.malloc_fixedsize_ptr, c_size],
+ resulttype=llmemory.Address)
+ hop.cast_result(v_raw)
+
+ def gct_malloc_varsize(self, hop):
+ def intconst(c): return rmodel.inputconst(lltype.Signed, c)
+
+ op = hop.spaceop
+ TYPE = op.result.concretetype.TO
+ assert TYPE._is_varsize()
+
+ if isinstance(TYPE, lltype.Struct):
+ ARRAY = TYPE._flds[TYPE._arrayfld]
+ else:
+ ARRAY = TYPE
+ assert isinstance(ARRAY, lltype.Array)
+ if ARRAY._hints.get('isrpystring', False):
+ c_const_size = intconst(llmemory.sizeof(TYPE, 1))
+ else:
+ c_const_size = intconst(llmemory.sizeof(TYPE, 0))
+ c_item_size = intconst(llmemory.sizeof(ARRAY.OF))
+
+ if ARRAY._hints.get("nolength", False):
+ v_raw = hop.genop("direct_call",
+ [self.malloc_varsize_no_length_ptr, op.args[-1],
+ c_const_size, c_item_size],
+ resulttype=llmemory.Address)
+ else:
+ if isinstance(TYPE, lltype.Struct):
+ offset_to_length = llmemory.FieldOffset(TYPE, TYPE._arrayfld) + \
+ llmemory.ArrayLengthOffset(ARRAY)
+ else:
+ offset_to_length = llmemory.ArrayLengthOffset(ARRAY)
+ v_raw = hop.genop("direct_call",
+ [self.malloc_varsize_ptr, op.args[-1],
+ c_const_size, c_item_size, intconst(offset_to_length)],
+ resulttype=llmemory.Address)
+ hop.cast_result(v_raw)
+
+ def gct_gc_deallocate(self, hop):
+ TYPE = hop.spaceop.args[0].value
+ v_addr = hop.spaceop.args[1]
+ dealloc_fptr = self.dynamic_deallocation_funcptr_for_type(TYPE)
+ cdealloc_fptr = rmodel.inputconst(
+ lltype.typeOf(dealloc_fptr), dealloc_fptr)
+ hop.genop("direct_call", [cdealloc_fptr, v_addr])
+
+ def consider_constant(self, TYPE, value):
+ if value is not lltype.top_container(value):
+ return
+ if isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)):
+ p = value._as_ptr()
+ if not self.gcheaderbuilder.get_header(p):
+ hdr = self.gcheaderbuilder.new_header(p)
+ hdr.refcount = sys.maxint // 2
+
+ def static_deallocation_funcptr_for_type(self, TYPE):
+ if TYPE in self.static_deallocator_funcptrs:
+ return self.static_deallocator_funcptrs[TYPE]
+ #print_call_chain(self)
+
+ rtti = get_rtti(TYPE)
+ if rtti is not None and hasattr(rtti._obj, 'destructor_funcptr'):
+ destrptr = rtti._obj.destructor_funcptr
+ DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
+ else:
+ destrptr = None
+ DESTR_ARG = None
+
+ if destrptr is None and not find_gc_ptrs_in_type(TYPE):
+ #print repr(TYPE)[:80], 'is dealloc easy'
+ p = self.no_pointer_dealloc_ptr.value
+ self.static_deallocator_funcptrs[TYPE] = p
+ return p
+
+ if destrptr is not None:
+ body = '\n'.join(_static_deallocator_body_for_type('v', TYPE, 3))
+ src = """
+def ll_deallocator(addr):
+ exc_instance = llop.gc_fetch_exception(EXC_INSTANCE_TYPE)
+ try:
+ v = cast_adr_to_ptr(addr, PTR_TYPE)
+ gcheader = cast_adr_to_ptr(addr - gc_header_offset, HDRPTR)
+ # refcount is at zero, temporarily bump it to 1:
+ gcheader.refcount = 1
+ destr_v = cast_pointer(DESTR_ARG, v)
+ ll_call_destructor(destrptr, destr_v)
+ refcount = gcheader.refcount - 1
+ gcheader.refcount = refcount
+ if refcount == 0:
+%s
+ llop.gc_free(lltype.Void, addr)
+ except:
+ pass
+ llop.gc_restore_exception(lltype.Void, exc_instance)
+ pop_alive(exc_instance)
+ # XXX layering of exceptiontransform versus gcpolicy
+
+""" % (body, )
+ else:
+ call_del = None
+ body = '\n'.join(_static_deallocator_body_for_type('v', TYPE))
+ src = ('def ll_deallocator(addr):\n v = cast_adr_to_ptr(addr, PTR_TYPE)\n' +
+ body + '\n llop.gc_free(lltype.Void, addr)\n')
+ d = {'pop_alive': LLTransformerOp(self.pop_alive),
+ 'llop': llop,
+ 'lltype': lltype,
+ 'destrptr': destrptr,
+ 'gc_header_offset': self.gcheaderbuilder.size_gc_header,
+ 'cast_adr_to_ptr': llmemory.cast_adr_to_ptr,
+ 'cast_pointer': lltype.cast_pointer,
+ 'PTR_TYPE': lltype.Ptr(TYPE),
+ 'DESTR_ARG': DESTR_ARG,
+ 'EXC_INSTANCE_TYPE': self.translator.rtyper.exceptiondata.lltype_of_exception_value,
+ 'll_call_destructor': ll_call_destructor,
+ 'HDRPTR':lltype.Ptr(self.HDR)}
+ exec src in d
+ this = d['ll_deallocator']
+ fptr = self.annotate_helper(this, [llmemory.Address], lltype.Void)
+ self.static_deallocator_funcptrs[TYPE] = fptr
+ for p in find_gc_ptrs_in_type(TYPE):
+ self.static_deallocation_funcptr_for_type(p.TO)
+ return fptr
+
+ def dynamic_deallocation_funcptr_for_type(self, TYPE):
+ if TYPE in self.dynamic_deallocator_funcptrs:
+ return self.dynamic_deallocator_funcptrs[TYPE]
+ #print_call_chain(self)
+
+ rtti = get_rtti(TYPE)
+ if rtti is None:
+ p = self.static_deallocation_funcptr_for_type(TYPE)
+ self.dynamic_deallocator_funcptrs[TYPE] = p
+ return p
+
+ queryptr = rtti._obj.query_funcptr
+ if queryptr._obj in self.queryptr2dynamic_deallocator_funcptr:
+ return self.queryptr2dynamic_deallocator_funcptr[queryptr._obj]
+
+ RTTI_PTR = lltype.Ptr(lltype.RuntimeTypeInfo)
+ QUERY_ARG_TYPE = lltype.typeOf(queryptr).TO.ARGS[0]
+ gc_header_offset = self.gcheaderbuilder.size_gc_header
+ HDRPTR = lltype.Ptr(self.HDR)
+ def ll_dealloc(addr):
+ # bump refcount to 1
+ gcheader = llmemory.cast_adr_to_ptr(addr - gc_header_offset, HDRPTR)
+ gcheader.refcount = 1
+ v = llmemory.cast_adr_to_ptr(addr, QUERY_ARG_TYPE)
+ rtti = queryptr(v)
+ gcheader.refcount = 0
+ llop.gc_call_rtti_destructor(lltype.Void, rtti, addr)
+ fptr = self.annotate_helper(ll_dealloc, [llmemory.Address], lltype.Void)
+ self.dynamic_deallocator_funcptrs[TYPE] = fptr
+ self.queryptr2dynamic_deallocator_funcptr[queryptr._obj] = fptr
+ return fptr
+
+
Added: pypy/dist/pypy/rpython/memory/gctransform2/support.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/rpython/memory/gctransform2/support.py Tue Oct 3 19:46:43 2006
@@ -0,0 +1,31 @@
+from pypy.rpython.lltypesystem import lltype
+
+def var_ispyobj(var):
+ if hasattr(var, 'concretetype'):
+ if isinstance(var.concretetype, lltype.Ptr):
+ return var.concretetype.TO._gckind == 'cpy'
+ else:
+ return False
+ else:
+ # assume PyObjPtr
+ return True
+
+PyObjPtr = lltype.Ptr(lltype.PyObject)
+
+def find_gc_ptrs_in_type(TYPE):
+ if isinstance(TYPE, lltype.Array):
+ return find_gc_ptrs_in_type(TYPE.OF)
+ elif isinstance(TYPE, lltype.Struct):
+ result = []
+ for name in TYPE._names:
+ result.extend(find_gc_ptrs_in_type(TYPE._flds[name]))
+ return result
+ elif isinstance(TYPE, lltype.Ptr) and TYPE._needsgc():
+ return [TYPE]
+ elif isinstance(TYPE, lltype.GcOpaqueType):
+ # heuristic: in theory the same problem exists with OpaqueType, but
+ # we use OpaqueType for other things too that we know are safely
+ # empty of further gc pointers
+ raise Exception("don't know what is in %r" % (TYPE,))
+ else:
+ return []
Added: pypy/dist/pypy/rpython/memory/gctransform2/test/__init__.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/rpython/memory/gctransform2/test/__init__.py Tue Oct 3 19:46:43 2006
@@ -0,0 +1 @@
+#
Added: pypy/dist/pypy/rpython/memory/gctransform2/test/test_refcounting.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/rpython/memory/gctransform2/test/test_refcounting.py Tue Oct 3 19:46:43 2006
@@ -0,0 +1,121 @@
+from pypy.rpython.memory.gctransform2.test.test_transform import rtype
+from pypy.rpython.memory.gctransform2.refcounting import RefcountingGCTransformer
+from pypy.translator.c.database import LowLevelDatabase
+from pypy.translator.c.gc import RefcountingGcPolicy
+from pypy import conftest
+
+class RefcountingGcPolicy2(RefcountingGcPolicy):
+ transformerclass = RefcountingGCTransformer
+
+def llinterpreter_for_refcounted_graph(f, args_s):
+ from pypy.rpython.llinterp import LLInterpreter
+ from pypy.translator.c.genc import CStandaloneBuilder
+ from pypy.translator.c import gc
+
+ t = rtype(f, args_s)
+ cbuild = CStandaloneBuilder(t, f, RefcountingGcPolicy2)
+ db = cbuild.generate_graphs_for_llinterp()
+ graph = cbuild.getentrypointptr()._obj.graph
+ llinterp = LLInterpreter(t.rtyper)
+ if conftest.option.view:
+ t.view()
+ return llinterp, graph
+ res = llinterp.eval_graph(graph, [0])
+ assert res == f(0)
+ res = llinterp.eval_graph(graph, [1])
+ assert res == f(1)
+
+
+def test_llinterp_refcounted_graph():
+ from pypy.annotation.model import SomeInteger
+
+ class C:
+ pass
+ c = C()
+ c.x = 1
+ def g(x):
+ if x:
+ return c
+ else:
+ d = C()
+ d.x = 2
+ return d
+ def f(x):
+ return g(x).x
+
+ llinterp, graph = llinterpreter_for_refcounted_graph(f, [SomeInteger()])
+
+ res = llinterp.eval_graph(graph, [0])
+ assert res == f(0)
+ res = llinterp.eval_graph(graph, [1])
+ assert res == f(1)
+
+def test_llinterp_refcounted_graph_varsize():
+ from pypy.annotation.model import SomeInteger
+
+ def f(x):
+ r = []
+ for i in range(x):
+ if i % 2:
+ r.append(x)
+ return len(r)
+
+
+ llinterp, graph = llinterpreter_for_refcounted_graph(f, [SomeInteger()])
+
+ res = llinterp.eval_graph(graph, [0])
+ assert res == f(0)
+ res = llinterp.eval_graph(graph, [10])
+ assert res == f(10)
+
+def test_llinterp_refcounted_graph_str():
+ from pypy.annotation.model import SomeString
+ from pypy.rpython.lltypesystem.rstr import string_repr
+
+ def f(x):
+ return len(x + 'a')
+
+
+ llinterp, graph = llinterpreter_for_refcounted_graph(f, [SomeString()])
+
+ cc = string_repr.convert_const
+
+ res = llinterp.eval_graph(graph, [cc('a')])
+ assert res == f('a')
+ res = llinterp.eval_graph(graph, [cc('brrrrrr')])
+ assert res == f('brrrrrr')
+
+def test_llinterp_refcounted_graph_with_del():
+ from pypy.annotation.model import SomeInteger
+
+ class D:
+ pass
+
+ delcounter = D()
+ delcounter.dels = 0
+
+ class C:
+ def __del__(self):
+ delcounter.dels += 1
+ c = C()
+ c.x = 1
+ def h(x):
+ if x:
+ return c
+ else:
+ d = C()
+ d.x = 2
+ return d
+ def g(x):
+ return h(x).x
+ def f(x):
+ r = g(x)
+ return r + delcounter.dels
+
+ llinterp, graph = llinterpreter_for_refcounted_graph(f, [SomeInteger()])
+
+ res = llinterp.eval_graph(graph, [1])
+ assert res == 1
+ res = llinterp.eval_graph(graph, [0])
+ assert res == 3
+
Added: pypy/dist/pypy/rpython/memory/gctransform2/test/test_transform.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/rpython/memory/gctransform2/test/test_transform.py Tue Oct 3 19:46:43 2006
@@ -0,0 +1,209 @@
+from pypy.rpython.memory.gctransform2.transform import GCTransformer
+from pypy.objspace.flow.model import c_last_exception, Variable
+from pypy.rpython.memory.gctransform import var_needsgc, var_ispyobj
+from pypy.translator.translator import TranslationContext, graphof
+from pypy.translator.c.exceptiontransform import ExceptionTransformer
+from pypy.rpython.lltypesystem import lltype
+from pypy.objspace.flow.model import Variable
+from pypy.annotation import model as annmodel
+from pypy.rpython.extregistry import ExtRegistryEntry
+from pypy import conftest
+
+class _TestGCTransformer(GCTransformer):
+
+ def push_alive_nopyobj(self, var):
+ self.llops.genop("gc_push_alive", [var])
+
+ def pop_alive_nopyobj(self, var):
+ self.llops.genop("gc_pop_alive", [var])
+
+
+def checkblock(block, is_borrowed):
+ if block.operations == ():
+ # a return/exception block -- don't want to think about them
+ # (even though the test passes for somewhat accidental reasons)
+ return
+ if block.isstartblock:
+ refs_in = 0
+ else:
+ refs_in = len([v for v in block.inputargs if isinstance(v, Variable)
+ and var_needsgc(v)
+ and not is_borrowed(v)])
+ push_alives = len([op for op in block.operations
+ if op.opname == 'gc_push_alive'])
+ pyobj_push_alives = len([op for op in block.operations
+ if op.opname == 'gc_push_alive_pyobj'])
+
+ # implicit_pyobj_pushalives included calls to things that return pyobject*
+ implicit_pyobj_pushalives = len([op for op in block.operations
+ if var_ispyobj(op.result)
+ and op.opname not in ('bare_getfield', 'bare_getarrayitem', 'same_as')])
+ nonpyobj_gc_returning_calls = len([op for op in block.operations
+ if op.opname in ('direct_call', 'indirect_call')
+ and var_needsgc(op.result)
+ and not var_ispyobj(op.result)])
+
+ pop_alives = len([op for op in block.operations
+ if op.opname == 'gc_pop_alive'])
+ pyobj_pop_alives = len([op for op in block.operations
+ if op.opname == 'gc_pop_alive_pyobj'])
+ if pop_alives == len(block.operations):
+ # it's a block we inserted
+ return
+ for link in block.exits:
+ assert block.exitswitch is not c_last_exception
+ refs_out = 0
+ for v2 in link.target.inputargs:
+ if var_needsgc(v2) and not is_borrowed(v2):
+ refs_out += 1
+ pyobj_pushes = pyobj_push_alives + implicit_pyobj_pushalives
+ nonpyobj_pushes = push_alives + nonpyobj_gc_returning_calls
+ assert refs_in + pyobj_pushes + nonpyobj_pushes == pop_alives + pyobj_pop_alives + refs_out
+
+def rtype(func, inputtypes, specialize=True):
+ t = TranslationContext()
+ t.buildannotator().build_types(func, inputtypes)
+ if specialize:
+ t.buildrtyper().specialize()
+ return t
+
+def rtype_and_transform(func, inputtypes, transformcls, specialize=True, check=True):
+ t = rtype(func, inputtypes, specialize)
+ transformer = transformcls(t)
+ etrafo = ExceptionTransformer(t)
+ etrafo.transform_completely()
+ graphs_borrowed = {}
+ for graph in t.graphs:
+ graphs_borrowed[graph] = transformer.transform_graph(graph)
+ if conftest.option.view:
+ t.view()
+ t.checkgraphs()
+ if check:
+ for graph, is_borrowed in graphs_borrowed.iteritems():
+ for block in graph.iterblocks():
+ checkblock(block, is_borrowed)
+ return t, transformer
+
+def getops(graph):
+ ops = {}
+ for block in graph.iterblocks():
+ for op in block.operations:
+ ops.setdefault(op.opname, []).append(op)
+ return ops
+
+def test_simple():
+ def f():
+ return 1
+ rtype_and_transform(f, [], _TestGCTransformer)
+
+def test_fairly_simple():
+ class C:
+ pass
+ def f():
+ c = C()
+ c.x = 1
+ return c.x
+ t, transformer = rtype_and_transform(f, [], _TestGCTransformer)
+
+def test_return_gcpointer():
+ class C:
+ pass
+ def f():
+ c = C()
+ c.x = 1
+ return c
+ t, transformer = rtype_and_transform(f, [], _TestGCTransformer)
+
+def test_call_function():
+ class C:
+ pass
+ def f():
+ c = C()
+ c.x = 1
+ return c
+ def g():
+ return f().x
+ t, transformer = rtype_and_transform(g, [], _TestGCTransformer)
+ ggraph = graphof(t, g)
+ for i, op in enumerate(ggraph.startblock.operations):
+ if op.opname == "direct_call":
+ break
+ else:
+ assert False, "direct_call not found!"
+ assert ggraph.startblock.operations[i + 1].opname != 'gc_push_alive'
+
+
+def test_multiply_passed_var():
+ S = lltype.GcStruct("S", ('x', lltype.Signed))
+ def f(x):
+ if x:
+ a = lltype.malloc(S)
+ a.x = 1
+ b = a
+ else:
+ a = lltype.malloc(S)
+ a.x = 1
+ b = lltype.malloc(S)
+ b.x = 2
+ return a.x + b.x
+ t, transformer = rtype_and_transform(f, [int], _TestGCTransformer)
+
+def test_pyobj():
+ def f(x):
+ if x:
+ a = 1
+ else:
+ a = "1"
+ return int(a)
+ t, transformer = rtype_and_transform(f, [int], _TestGCTransformer)
+ fgraph = graphof(t, f)
+ gcops = [op for op in fgraph.startblock.exits[0].target.operations
+ if op.opname.startswith("gc_")]
+ for op in gcops:
+ assert op.opname.endswith("_pyobj")
+
+def test_call_return_pyobj():
+ def g(factory):
+ return factory()
+ def f(factory):
+ g(factory)
+ t, transformer = rtype_and_transform(f, [object], _TestGCTransformer)
+ fgraph = graphof(t, f)
+ ops = getops(fgraph)
+ calls = ops['direct_call']
+ for call in calls:
+ if call.result.concretetype is not lltype.Bool: #RPyExceptionOccurred()
+ assert var_ispyobj(call.result)
+
+def test_getfield_pyobj():
+ class S:
+ pass
+ def f(thing):
+ s = S()
+ s.x = thing
+ return s.x
+ t, transformer = rtype_and_transform(f, [object], _TestGCTransformer)
+ fgraph = graphof(t, f)
+ pyobj_getfields = 0
+ pyobj_setfields = 0
+ for b in fgraph.iterblocks():
+ for op in b.operations:
+ if op.opname == 'bare_getfield' and var_ispyobj(op.result):
+ pyobj_getfields += 1
+ elif op.opname == 'setfield' and var_ispyobj(op.args[2]):
+ pyobj_setfields += 1
+ # although there's only one explicit getfield in the code, a
+ # setfield on a pyobj must get the old value out and decref it
+ assert pyobj_getfields >= 2
+ assert pyobj_setfields >= 1
+
+def test_pass_gc_pointer():
+ S = lltype.GcStruct("S", ('x', lltype.Signed))
+ def f(s):
+ s.x = 1
+ def g():
+ s = lltype.malloc(S)
+ f(s)
+ return s.x
+ t, transformer = rtype_and_transform(g, [], _TestGCTransformer)
+
Added: pypy/dist/pypy/rpython/memory/gctransform2/transform.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/rpython/memory/gctransform2/transform.py Tue Oct 3 19:46:43 2006
@@ -0,0 +1,333 @@
+import py
+from pypy.rpython.lltypesystem import lltype, llmemory
+from pypy.objspace.flow.model import SpaceOperation, Variable, Constant, \
+ c_last_exception, checkgraph
+from pypy.translator.unsimplify import insert_empty_block
+from pypy.translator.unsimplify import insert_empty_startblock
+from pypy.translator.unsimplify import starts_with_empty_block
+from pypy.translator.unsimplify import remove_empty_startblock
+from pypy.translator.backendopt.support import var_needsgc
+from pypy.translator.backendopt import inline
+from pypy.translator.backendopt import graphanalyze
+from pypy.translator.backendopt.canraise import RaiseAnalyzer
+from pypy.translator.backendopt.ssa import DataFlowFamilyBuilder
+from pypy.annotation import model as annmodel
+from pypy.rpython import rmodel, annlowlevel
+from pypy.rpython.memory import gc, lladdress
+from pypy.rpython.annlowlevel import MixLevelHelperAnnotator
+from pypy.rpython.rtyper import LowLevelOpList
+from pypy.rpython.rbuiltin import gen_cast
+from pypy.rpython.rarithmetic import ovfcheck
+import sets, os, sys
+
+def var_ispyobj(var):
+ if hasattr(var, 'concretetype'):
+ if isinstance(var.concretetype, lltype.Ptr):
+ return var.concretetype.TO._gckind == 'cpy'
+ else:
+ return False
+ else:
+ # assume PyObjPtr
+ return True
+
+PyObjPtr = lltype.Ptr(lltype.PyObject)
+
+class GcHighLevelOp(object):
+ def __init__(self, gctransformer, op, llops):
+ self.gctransformer = gctransformer
+ self.spaceop = op
+ self.llops = llops
+
+ def dispatch(self):
+ gct = self.gctransformer
+ opname = self.spaceop.opname
+ v_result = self.spaceop.result
+
+ meth = getattr(gct, 'gct_' + opname, gct.default)
+ meth(self)
+
+ if var_needsgc(v_result):
+ gct.livevars.append(v_result)
+ if var_ispyobj(v_result):
+ if opname in ('getfield', 'getarrayitem', 'same_as',
+ 'cast_pointer', 'getsubstruct'):
+ # XXX more operations?
+ gct.push_alive(v_result)
+ elif opname not in ('direct_call', 'indirect_call'):
+ gct.push_alive(v_result)
+
+ def rename(self, newopname):
+ self.llops.append(
+ SpaceOperation(newopname, self.spaceop.args, self.spaceop.result))
+
+ def inputargs(self):
+ return self.spaceop.args
+
+ def genop(self, opname, args, resulttype=None, resultvar=None):
+ assert resulttype is None or resultvar is None
+ if resultvar is None:
+ return self.llops.genop(opname, args,
+ resulttype=resulttype)
+ else:
+ newop = SpaceOperation(opname, args, resultvar)
+ self.llops.append(newop)
+ return resultvar
+
+ def cast_result(self, var):
+ v_result = self.spaceop.result
+ resulttype = getattr(v_result, 'concretetype', PyObjPtr)
+ curtype = getattr(var, 'concretetype', PyObjPtr)
+ if curtype == resulttype:
+ self.genop('same_as', [var], resultvar=v_result)
+ else:
+ v_new = gen_cast(self.llops, resulttype, var)
+ assert v_new != var
+ self.llops[-1].result = v_result
+
+
+class GCTransformer(object):
+ finished_helpers = False
+
+ def __init__(self, translator, inline=False):
+ self.translator = translator
+ self.seen_graphs = {}
+ self.minimal_transform = {}
+ if translator:
+ self.mixlevelannotator = MixLevelHelperAnnotator(translator.rtyper)
+ else:
+ self.mixlevelannotator = None
+ self.inline = inline
+ if translator and inline:
+ self.lltype_to_classdef = translator.rtyper.lltype_to_classdef_mapping()
+ self.graphs_to_inline = {}
+ if self.MinimalGCTransformer:
+ self.minimalgctransformer = self.MinimalGCTransformer(self)
+ else:
+ self.minimalgctransformer = None
+
+ def get_lltype_of_exception_value(self):
+ if self.translator is not None:
+ exceptiondata = self.translator.rtyper.getexceptiondata()
+ return exceptiondata.lltype_of_exception_value
+ else:
+ return lltype.Ptr(lltype.PyObject)
+
+ def need_minimal_transform(self, graph):
+ self.seen_graphs[graph] = True
+ self.minimal_transform[graph] = True
+
+ def inline_helpers(self, graph):
+ if self.inline:
+ raise_analyzer = RaiseAnalyzer(self.translator)
+ for inline_graph in self.graphs_to_inline:
+ try:
+ # XXX quite inefficient: we go over the function lots of times
+ inline.inline_function(self.translator, inline_graph, graph,
+ self.lltype_to_classdef,
+ raise_analyzer)
+ except inline.CannotInline, e:
+ print 'CANNOT INLINE:', e
+ print '\t%s into %s' % (inline_graph, graph)
+ checkgraph(graph)
+
+ def compute_borrowed_vars(self, graph):
+ # the input args are borrowed, and stay borrowed for as long as they
+ # are not merged with other values.
+ var_families = DataFlowFamilyBuilder(graph).get_variable_families()
+ borrowed_reps = {}
+ for v in graph.getargs():
+ borrowed_reps[var_families.find_rep(v)] = True
+ # no support for returning borrowed values so far
+ retvar = graph.getreturnvar()
+
+ def is_borrowed(v1):
+ return (var_families.find_rep(v1) in borrowed_reps
+ and v1 is not retvar)
+ return is_borrowed
+
+ def transform_block(self, block, is_borrowed):
+ self.llops = LowLevelOpList()
+ #self.curr_block = block
+ self.livevars = [var for var in block.inputargs
+ if var_needsgc(var) and not is_borrowed(var)]
+ for op in block.operations:
+ hop = GcHighLevelOp(self, op, self.llops)
+ hop.dispatch()
+
+ if len(block.exits) != 0: # i.e not the return block
+ assert block.exitswitch is not c_last_exception
+
+ deadinallexits = sets.Set(self.livevars)
+ for link in block.exits:
+ deadinallexits.difference_update(sets.Set(link.args))
+
+ for var in deadinallexits:
+ self.pop_alive(var)
+
+ for link in block.exits:
+ livecounts = dict.fromkeys(sets.Set(self.livevars) - deadinallexits, 1)
+ for v, v2 in zip(link.args, link.target.inputargs):
+ if is_borrowed(v2):
+ continue
+ if v in livecounts:
+ livecounts[v] -= 1
+ elif var_needsgc(v):
+ # 'v' is typically a Constant here, but it can be
+ # a borrowed variable going into a non-borrowed one
+ livecounts[v] = -1
+ self.links_to_split[link] = livecounts
+
+ block.operations[:] = self.llops
+ self.llops = None
+ self.livevars = None
+
+ def transform_graph(self, graph):
+ if graph in self.minimal_transform:
+ if self.minimalgctransformer:
+ self.minimalgctransformer.transform_graph(graph)
+ del self.minimal_transform[graph]
+ return
+ if graph in self.seen_graphs:
+ return
+ self.seen_graphs[graph] = True
+ self.links_to_split = {} # link -> vars to pop_alive across the link
+
+ # for sanity, we need an empty block at the start of the graph
+ if not starts_with_empty_block(graph):
+ insert_empty_startblock(self.translator.annotator, graph)
+ is_borrowed = self.compute_borrowed_vars(graph)
+
+ for block in graph.iterblocks():
+ self.transform_block(block, is_borrowed)
+
+ for link, livecounts in self.links_to_split.iteritems():
+ llops = LowLevelOpList()
+ for var, livecount in livecounts.iteritems():
+ for i in range(livecount):
+ self.pop_alive(var, llops)
+ for i in range(-livecount):
+ self.push_alive(var, llops)
+ if llops:
+ if link.prevblock.exitswitch is None:
+ link.prevblock.operations.extend(self.llops)
+ else:
+ insert_empty_block(self.translator.annotator, link, llops)
+
+ # remove the empty block at the start of the graph, which should
+ # still be empty (but let's check)
+ if starts_with_empty_block(graph):
+ remove_empty_startblock(graph)
+
+ checkgraph(graph)
+
+ self.links_to_split = None
+ v = Variable('vanishing_exc_value')
+ v.concretetype = self.get_lltype_of_exception_value()
+ llops = LowLevelOpList()
+ self.pop_alive(v, llops)
+ graph.exc_cleanup = (v, list(llops))
+ return is_borrowed # xxx for tests only
+
+ def annotate_helper(self, ll_helper, ll_args, ll_result, inline=False):
+ assert not self.finished_helpers
+ args_s = map(annmodel.lltype_to_annotation, ll_args)
+ s_result = annmodel.lltype_to_annotation(ll_result)
+ graph = self.mixlevelannotator.getgraph(ll_helper, args_s, s_result)
+ # the produced graphs does not need to be fully transformed
+ self.need_minimal_transform(graph)
+ if inline:
+ self.graphs_to_inline[graph] = True
+ return self.mixlevelannotator.graph2delayed(graph)
+
+ def inittime_helper(self, ll_helper, ll_args, ll_result):
+ ptr = self.annotate_helper(ll_helper, ll_args, ll_result, inline=True)
+ return Constant(ptr, lltype.typeOf(ptr))
+
+ def finish_helpers(self):
+ if self.translator is not None:
+ self.mixlevelannotator.finish_annotate()
+ self.finished_helpers = True
+ if self.translator is not None:
+ self.mixlevelannotator.finish_rtype()
+
+ def finish_tables(self):
+ pass
+
+ def finish(self):
+ self.finish_helpers()
+ self.finish_tables()
+
+ def transform_generic_set(self, hop):
+ v_new = hop.spaceop.args[-1]
+ v_old = hop.genop('bare_g' + hop.spaceop.opname[1:],
+ hop.inputargs()[:-1],
+ resulttype=v_new.concretetype)
+ self.push_alive(v_new)
+ hop.llops.append(hop.spaceop)
+ self.pop_alive(v_old)
+
+
+ def push_alive(self, var, llops=None):
+ if llops is None:
+ llops = self.llops
+ if var_ispyobj(var):
+ self.push_alive_pyobj(var, llops)
+ else:
+ self.push_alive_nopyobj(var, llops)
+
+ def pop_alive(self, var, llops=None):
+ if llops is None:
+ llops = self.llops
+ if var_ispyobj(var):
+ self.pop_alive_pyobj(var, llops)
+ else:
+ self.pop_alive_nopyobj(var, llops)
+
+ def push_alive_pyobj(self, var, llops):
+ if hasattr(var, 'concretetype') and var.concretetype != PyObjPtr:
+ var = gen_cast(llops, PyObjPtr, var)
+ llops.genop("gc_push_alive_pyobj", [var])
+
+ def pop_alive_pyobj(self, var, llops):
+ if hasattr(var, 'concretetype') and var.concretetype != PyObjPtr:
+ var = gen_cast(llops, PyObjPtr, var)
+ llops.genop("gc_pop_alive_pyobj", [var])
+
+ def push_alive_nopyobj(self, var, llops):
+ pass
+
+ def pop_alive_nopyobj(self, var, llops):
+ pass
+
+ def var_needs_set_transform(self, var):
+ return var_ispyobj(var)
+
+ def default(self, hop):
+ hop.llops.append(hop.spaceop)
+
+ def gct_getfield(self, hop):
+ hop.rename('bare_' + hop.spaceop.opname)
+
+ def gct_setfield(self, hop):
+ if self.var_needs_set_transform(hop.spaceop.args[-1]):
+ self.transform_generic_set(hop)
+ else:
+ self.default(hop)
+ gct_setarrayitem = gct_setfield
+
+ def gct_safe_call(self, hop):
+ hop.rename("direct_call")
+
+class MinimalGCTransformer(GCTransformer):
+ def __init__(self, parenttransformer):
+ GCTransformer.__init__(self, parenttransformer.translator)
+ self.parenttransformer = parenttransformer
+
+ def push_alive(self, var, llops=None):
+ pass
+
+ def pop_alive(self, var, llops=None):
+ pass
+
+GCTransformer.MinimalGCTransformer = MinimalGCTransformer
+MinimalGCTransformer.MinimalGCTransformer = None
More information about the Pypy-commit
mailing list