[pypy-svn] r17110 - in pypy/dist/pypy/rpython/memory: . test
cfbolz at codespeak.net
cfbolz at codespeak.net
Wed Aug 31 01:21:27 CEST 2005
Author: cfbolz
Date: Wed Aug 31 01:21:18 2005
New Revision: 17110
Modified:
pypy/dist/pypy/rpython/memory/gc.py
pypy/dist/pypy/rpython/memory/gclltype.py
pypy/dist/pypy/rpython/memory/gcwrapper.py
pypy/dist/pypy/rpython/memory/test/test_gc.py
Log:
let the GCs run on the llinterpreter as well (this makes the tests really
slow, btw). Some trickery is required in the setup area, because a real GC
instance is needed during setup time for the conversion of the constants in
the graphs. This instance can not be reused on llinterp level, so we need to
allocate one there.
Modified: pypy/dist/pypy/rpython/memory/gc.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gc.py (original)
+++ pypy/dist/pypy/rpython/memory/gc.py Wed Aug 31 01:21:18 2005
@@ -10,12 +10,14 @@
class GCError(Exception):
pass
-def get_dummy_annotate(gc):
+def get_dummy_annotate(gc_class):
def dummy_annotate():
+ gc = gc_class()
gc.get_roots = dummy_get_roots1 #prevent the get_roots attribute to
gc.get_roots = dummy_get_roots2 #be constants
a = gc.malloc(1, 2)
b = gc.malloc(2, 3)
+ gc.write_barrier(raw_malloc(1), raw_malloc(2), raw_malloc(1))
gc.collect()
return a - b
return dummy_annotate
@@ -58,6 +60,11 @@
def write_barrier(self, addr, addr_to, addr_struct):
addr_to.address[0] = addr
+ def free_memory(self):
+ #this will never be called at runtime, just during setup
+ "NOT_RPYTHON"
+ pass
+
class MarkSweepGC(GCBase):
_alloc_flavor_ = "raw"
@@ -99,7 +106,8 @@
gc_info = curr.address[0] - self.size_gc_header()
# constants roots are not malloced and thus don't have their mark
# bit reset
- gc_info.signed[0] = 0
+ gc_info.signed[0] = 0
+ free_non_gc_object(roots)
while 1: #mark
curr = objects.pop()
## print "object: ", curr
@@ -131,6 +139,7 @@
j += 1
i += 1
gc_info.signed[0] = 1
+ free_non_gc_object(objects)
newmo = AddressLinkedList()
curr_heap_size = 0
freed_size = 0
@@ -177,6 +186,13 @@
self.set_query_functions(None, None, None, None, None, None, None)
self.get_roots = get_roots
+ def free_memory(self):
+ "NOT_RPYTHON"
+ raw_free(self.tospace)
+ self.tospace = NULL
+ raw_free(self.fromspace)
+ self.fromspace = NULL
+
def malloc(self, typeid, length=0):
size = self.fixed_size(typeid)
if self.is_varsize(typeid):
@@ -185,7 +201,7 @@
if self.free + totalsize > self.top_of_space:
self.collect()
#XXX need to increase the space size if the object is too big
- #for bonus points do big blocks differently
+ #for bonus points do big objects differently
return self.malloc(typeid, length)
result = self.free
self.init_gc_object(result, typeid)
@@ -193,7 +209,6 @@
self.free += totalsize
return result + self.size_gc_header()
-
def collect(self):
## print "collecting"
self.fromspace, self.tospace = self.tospace, self.fromspace
@@ -206,6 +221,7 @@
break
## print "root", root, root.address[0]
root.address[0] = self.copy(root.address[0])
+ free_non_gc_object(roots)
while scan < self.free:
curr = scan + self.size_gc_header()
self.trace_and_copy(curr)
@@ -321,14 +337,23 @@
if curr.address[0] == NULL:
break
curr = curr.address[0]
+ dealloc_list = AddressLinkedList()
while 1:
candidate = self.zero_ref_counts.pop()
self.length_zero_ref_counts -= 1
if candidate == NULL:
break
refcount = self.refcount(candidate)
- if refcount == 0:
- self.deallocate(candidate)
+ if (refcount == 0 and
+ (candidate - self.size_gc_header()).signed[1] != -1):
+ (candidate - self.size_gc_header()).signed[1] = -1
+ dealloc_list.append(candidate)
+ while 1:
+ deallocate = dealloc_list.pop()
+ if deallocate == NULL:
+ break
+ self.deallocate(deallocate)
+ free_non_gc_object(dealloc_list)
while 1:
root = roots.pop()
if root == NULL:
Modified: pypy/dist/pypy/rpython/memory/gclltype.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gclltype.py (original)
+++ pypy/dist/pypy/rpython/memory/gclltype.py Wed Aug 31 01:21:18 2005
@@ -36,9 +36,15 @@
from pypy.rpython.memory.gc import MarkSweepGC, SemiSpaceGC
use_gc = MarkSweepGC
-def create_mark_sweep_gc(llinterp, flowgraphs):
- from pypy.rpython.memory.gcwrapper import GcWrapper
+def create_gc(llinterp, flowgraphs):
+ from pypy.rpython.memory.gcwrapper import GcWrapper, AnnotatingGcWrapper
wrapper = GcWrapper(llinterp, flowgraphs, use_gc)
return wrapper
+
+def create_gc_run_on_llinterp(llinterp, flowgraphs):
+ from pypy.rpython.memory.gcwrapper import GcWrapper, AnnotatingGcWrapper
+ wrapper = AnnotatingGcWrapper(llinterp, flowgraphs, use_gc)
+ return wrapper
+
prepare_graphs_and_create_gc = create_no_gc
Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gcwrapper.py (original)
+++ pypy/dist/pypy/rpython/memory/gcwrapper.py Wed Aug 31 01:21:18 2005
@@ -124,6 +124,17 @@
self.varsize_offsets_to_gcpointers_in_var_part)
+def getfunctionptr(translator, graphfunc):
+ """Make a functionptr from the given Python function."""
+ graph = translator.getflowgraph(graphfunc)
+ llinputs = [v.concretetype for v in graph.getargs()]
+ lloutput = graph.getreturnvar().concretetype
+ FT = lltype.FuncType(llinputs, lloutput)
+ _callable = graphfunc
+ return lltypesimulation.functionptr(FT, graphfunc.func_name,
+ graph=graph, _callable=_callable)
+
+
class GcWrapper(object):
def __init__(self, llinterp, flowgraphs, gc_class):
self.query_types = QueryTypes(llinterp)
@@ -183,15 +194,15 @@
def update_changed_addresses(self):
for i, root in enumerate(self.roots):
- if root._address != self.pseudo_root_pointers.address[i]:
- print "address changed:", root._address, self.pseudo_root_pointers.address[i]
root.__dict__['_address'] = self.pseudo_root_pointers.address[i]
- def get_roots(self):
- print "getting roots"
+ def get_roots_from_llinterp(self):
if self.pseudo_root_pointers != NULL:
raw_free(self.pseudo_root_pointers)
- self.roots = self.llinterp.find_roots() + self.constantroots
+ roots = [r for r in self.llinterp.find_roots()
+ if isinstance(r._TYPE.TO,
+ (lltype.GcStruct, lltype.GcArray))]
+ self.roots = roots + self.constantroots
self.roots = [r for r in self.roots
if isinstance(r._TYPE.TO,
(lltype.Struct, lltype.Array))]
@@ -199,6 +210,10 @@
self.pseudo_root_pointers = NULL
else:
self.pseudo_root_pointers = raw_malloc(len(self.roots) * INT_SIZE)
+ return self.roots
+
+ def get_roots(self):
+ self.get_roots_from_llinterp()
ll = AddressLinkedList()
for i, root in enumerate(self.roots):
self.pseudo_root_pointers.address[i] = root._address
@@ -206,17 +221,79 @@
return ll
class AnnotatingGcWrapper(GcWrapper):
- def __init__(self, llinterp, gc, qt, constantroots):
- super(AnnotatingGcWrapper, self).__init__(llinterp, gc, qt,
- constantroots)
+ def __init__(self, llinterp, flowgraphs, gc_class):
+ super(AnnotatingGcWrapper, self).__init__(llinterp, flowgraphs, gc_class)
+ # tell the real-built gc to free its memory as it is only used for
+ # initialisation
+ self.gc.free_memory()
self.annotate_rtype_gc()
def annotate_rtype_gc(self):
- #XXXXX unfinished
- func = gc.get_dummy_annotate(self.gc)
- self.gc.get_roots = gc.dummy_get_roots
+ # annotate and specialize functions
+ gc_class = self.gc.__class__
+ def instantiate_linked_list():
+ return AddressLinkedList()
+ f1, f2, f3, f4, f5, f6, f7 = self.query_types.create_query_functions()
+ def instantiate_gc():
+ gc = gc_class()
+ gc.set_query_functions(f1, f2, f3, f4, f5, f6, f7)
+ return gc
+ func = gc.get_dummy_annotate(self.gc.__class__)
+ self.gc.get_roots = gc.dummy_get_roots1
a = RPythonAnnotator()
- res = a.build_types(func, [])
- a.translator.view()
+ a.build_types(instantiate_gc, [])
+ a.build_types(func, [])
+ a.build_types(instantiate_linked_list, [])
a.translator.specialize()
- self.gc.get_roots = self.get_roots
+ self.annotator = a
+
+ # convert constants
+ fgcc = FlowGraphConstantConverter(a.translator.flowgraphs)
+ fgcc.convert()
+ self.malloc_graph = a.translator.flowgraphs[self.gc.malloc.im_func]
+
+ # create a gc via invoking instantiate_gc
+ self.gcptr = self.llinterp.eval_function(
+ instantiate_gc, graph=a.translator.flowgraphs[instantiate_gc])
+ GETROOTS_FUNCTYPE = lltype.typeOf(
+ getfunctionptr(a.translator, gc.dummy_get_roots1)).TO
+ setattr(self.gcptr, "inst_get_roots",
+ lltypesimulation.functionptr(GETROOTS_FUNCTYPE, "get_roots",
+ _callable=self.get_roots))
+
+ #get funcptrs neccessary to build the result of get_roots
+ self.instantiate_linked_list = getfunctionptr(
+ a.translator, instantiate_linked_list)
+ self.append_linked_list = getfunctionptr(
+ a.translator, AddressLinkedList.append.im_func)
+ self.pop_linked_list = getfunctionptr(
+ a.translator, AddressLinkedList.pop.im_func)
+ self.gc.get_roots = None
+ self.translator = a.translator
+# a.translator.view()
+
+ def get_arg_malloc(self, TYPE, size=0):
+ typeid = self.query_types.get_typeid(TYPE)
+ return [self.gcptr, typeid, size]
+
+ def get_funcptr_malloc(self):
+ return self.llinterp.llt.functionptr(gc.gc_interface["malloc"], "malloc",
+ _callable=self.gc.malloc,
+ graph=self.malloc_graph)
+
+ def adjust_result_malloc(self, address, TYPE, size=0):
+ result = lltypesimulation.init_object_on_address(address, TYPE, size)
+ self.update_changed_addresses()
+ return result
+
+ def get_roots(self):
+ # call the llinterpreter to construct the result in a suitable way
+ self.get_roots_from_llinterp()
+ ll = self.llinterp.active_frame.op_direct_call(
+ self.instantiate_linked_list)
+ for i, root in enumerate(self.roots):
+ self.pseudo_root_pointers.address[i] = root._address
+ self.llinterp.active_frame.op_direct_call(
+ self.append_linked_list, ll,
+ self.pseudo_root_pointers + INT_SIZE * i)
+ return ll
Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/test/test_gc.py (original)
+++ pypy/dist/pypy/rpython/memory/test/test_gc.py Wed Aug 31 01:21:18 2005
@@ -1,4 +1,5 @@
import py
+import sys
from pypy.annotation import model as annmodel
from pypy.translator.annrpython import RPythonAnnotator
@@ -14,87 +15,21 @@
from pypy.rpython.objectmodel import free_non_gc_object
def setup_module(mod):
+ def stdout_ignore_ll_functions(msg):
+ strmsg = str(msg)
+ if "evaluating" in strmsg and "ll_" in strmsg:
+ return
+ print >>sys.stdout, strmsg
mod.logstate = py.log._getstate()
py.log.setconsumer("llinterp", py.log.STDOUT)
+ py.log.setconsumer("llinterp frame", stdout_ignore_ll_functions)
py.log.setconsumer("llinterp operation", None)
- gclltype.prepare_graphs_and_create_gc = gclltype.create_mark_sweep_gc
+ gclltype.prepare_graphs_and_create_gc = gclltype.create_gc
def teardown_module(mod):
gclltype.prepare_graphs_and_create_gc = gclltype.create_no_gc
-class PseudoObjectModel(object):
- """Object model for testing purposes: you can specify roots and a
- layout_mapping which is a dictionary of typeids to a list of offsets of
- pointers in an object"""
- def __init__(self, roots, layout_mapping, size_mapping):
- self.roots = roots
- self.layout_mapping = layout_mapping
- self.size_mapping = size_mapping
-
- def get_roots(self):
- self.roots
- ll = AddressLinkedList()
- for root in self.roots:
- ll.append(root)
- return ll
-
- def is_varsize(self, typeid):
- False
-
- def fixed_size(self, typeid):
- return self.size_mapping[typeid]
-
- def offsets_to_gc_pointers(self, typeid):
- return self.layout_mapping[typeid]
-
class TestMarkSweepGC(object):
- def DONOTtest_simple(self):
- variables = raw_malloc(4 * INT_SIZE)
- roots = [variables + i * INT_SIZE for i in range(4)]
- layout0 = [] #int
- layout1 = [0, INT_SIZE] #(ptr, ptr)
- om = PseudoObjectModel(roots, {0: layout0, 1: layout1}, {0: INT_SIZE, 1: 2 * INT_SIZE})
- gc = MarkSweepGC(om, 2 ** 16)
- variables.address[0] = gc.malloc(0)
- variables.address[1] = gc.malloc(0)
- variables.address[2] = gc.malloc(0)
- variables.address[3] = gc.malloc(0)
- variables.address[0].signed[0] = 0
- variables.address[1].signed[0] = 1
- variables.address[2].signed[0] = 2
- variables.address[3].signed[0] = 3
- print "roots", roots
- gc.collect() #does not crash
- assert variables.address[0].signed[0] == 0
- assert variables.address[1].signed[0] == 1
- assert variables.address[2].signed[0] == 2
- assert variables.address[3].signed[0] == 3
- addr = gc.malloc(0)
- addr.signed[0] = 1
- print "roots", roots
- gc.collect()
- py.test.raises(MemorySimulatorError, "addr.signed[0]")
- variables.address[0] = gc.malloc(1)
- variables.address[0].address[0] = variables.address[1]
- variables.address[0].address[1] = NULL
- print "roots", roots
- gc.collect() #does not crash
- assert variables.address[0].address[0] == variables.address[1]
- assert variables.address[0].address[1] == NULL
- addr0 = gc.malloc(1)
- addr0.address[1] = NULL
- addr1 = gc.malloc(1)
- addr1.address[0] = addr1.address[1] = NULL
- addr0.address[0] = addr1
- addr2 = variables.address[1]
- print "addr0, addr1, addr2 =", addr0, addr1, addr2
- variables.address[1] == NULL
- variables.address[0].address[0] = NULL
- print "roots", roots
- gc.collect()
- py.test.raises(MemorySimulatorError, "addr0.signed[0]")
- py.test.raises(MemorySimulatorError, "addr1.signed[0]")
-
def test_llinterp_lists(self):
curr = simulator.current_size
def malloc_a_lot():
@@ -129,7 +64,7 @@
def test_global_list(self):
lst = []
def append_to_list(i, j):
- lst.append([i] * 500)
+ lst.append([i] * 50)
return lst[j][0]
res = interpret(append_to_list, [0, 0])
assert res == 0
@@ -148,157 +83,36 @@
assert res == concat(100)
assert simulator.current_size - curr < 16000
+class TestMarkSweepGCRunningOnLLinterp(TestMarkSweepGC):
+ def setup_class(cls):
+ cls.prep_old = gclltype.prepare_graphs_and_create_gc
+ gclltype.prepare_graphs_and_create_gc = gclltype.create_gc_run_on_llinterp
+ def teardown_class(cls):
+ gclltype.prepare_graphs_and_create_gc = cls.prep_old.im_func
-class TestSemiSpaceGC(object):
+class TestSemiSpaceGC(TestMarkSweepGC):
def setup_class(cls):
gclltype.use_gc = SemiSpaceGC
cls.old = gclltype.use_gc
def teardown_class(cls):
gclltype.use_gc = cls.old
- def DONOTtest_simple(self):
- variables = raw_malloc(4 * INT_SIZE)
- roots = [variables + i * INT_SIZE for i in range(4)]
- layout0 = [] #int
- layout1 = [0, INT_SIZE] #(ptr, ptr)
- om = PseudoObjectModel(roots, {0: layout0, 1: layout1}, {0: INT_SIZE, 1: 2 * INT_SIZE})
- gc = SemiSpaceGC(om, 2 ** 16)
- variables.address[0] = gc.malloc(0)
- variables.address[1] = gc.malloc(0)
- variables.address[2] = gc.malloc(0)
- variables.address[3] = gc.malloc(0)
- variables.address[0].signed[0] = 0
- variables.address[1].signed[0] = 1
- variables.address[2].signed[0] = 2
- variables.address[3].signed[0] = 3
- print "roots", roots
- gc.collect() #does not crash
- assert variables.address[0].signed[0] == 0
- assert variables.address[1].signed[0] == 1
- assert variables.address[2].signed[0] == 2
- assert variables.address[3].signed[0] == 3
- addr = gc.malloc(0)
- addr.signed[0] = 1
- print "roots", roots
- gc.collect()
-## py.test.raises(MemorySimulatorError, "addr.signed[0]")
- variables.address[0] = gc.malloc(1)
- variables.address[0].address[0] = variables.address[1]
- variables.address[0].address[1] = NULL
- print "roots", roots
- gc.collect() #does not crash
- assert variables.address[0].address[0] == variables.address[1]
- assert variables.address[0].address[1] == NULL
- addr0 = gc.malloc(1)
- addr0.address[1] = NULL
- addr1 = gc.malloc(1)
- addr1.address[0] = addr1.address[1] = NULL
- addr0.address[0] = addr1
- addr2 = variables.address[1]
- print "addr0, addr1, addr2 =", addr0, addr1, addr2
- variables.address[1] == NULL
- variables.address[0].address[0] = NULL
- print "roots", roots
- gc.collect()
-
- def test_llinterp_lists(self):
- curr = simulator.current_size
- def malloc_a_lot():
- i = 0
- while i < 10:
- i += 1
- a = [1] * 10
- j = 0
- while j < 20:
- j += 1
- a.append(j)
- res = interpret(malloc_a_lot, [])
- assert simulator.current_size - curr < 16000
- print "size before: %s, size after %s" % (curr, simulator.current_size)
-
- def test_llinterp_tuples(self):
- curr = simulator.current_size
- def malloc_a_lot():
- i = 0
- while i < 10:
- i += 1
- a = (1, 2, i)
- b = [a] * 10
- j = 0
- while j < 20:
- j += 1
- b.append((1, j, i))
- res = interpret(malloc_a_lot, [])
- assert simulator.current_size - curr < 16000
- print "size before: %s, size after %s" % (curr, simulator.current_size)
+class TestSemiSpaceGCRunningOnLLinterp(TestMarkSweepGC):
+ def setup_class(cls):
+ cls.prep_old = gclltype.prepare_graphs_and_create_gc
+ gclltype.prepare_graphs_and_create_gc = gclltype.create_gc_run_on_llinterp
+ gclltype.use_gc = SemiSpaceGC
+ cls.old = gclltype.use_gc
- def test_global_list(self):
- lst = []
- def append_to_list(i, j):
- lst.append([i] * 50)
- return lst[j][0]
- res = interpret(append_to_list, [0, 0])
- assert res == 0
- for i in range(1, 15):
- res = interpret(append_to_list, [i, i - 1])
- assert res == i - 1 # crashes if constants are not considered roots
+ def teardown_class(cls):
+ gclltype.prepare_graphs_and_create_gc = cls.prep_old.im_func
+ gclltype.use_gc = cls.old
- def test_string_concatenation(self):
- curr = simulator.current_size
- def concat(j):
- lst = []
- for i in range(j):
- lst.append(str(i))
- return len("".join(lst))
- res = interpret(concat, [100])
- assert res == concat(100)
- assert simulator.current_size - curr < 16000
-class DONOTTestDeferredRefcountingGC(object):
+class DONOTTestDeferredRefcountingGC(TestMarkSweepGC):
def setup_class(cls):
gclltype.use_gc = DeferredRefcountingGC
cls.old = gclltype.use_gc
def teardown_class(cls):
gclltype.use_gc = cls.old
- def test_llinterp_lists(self):
- curr = simulator.current_size
- def malloc_a_lot():
- i = 0
- while i < 10:
- i += 1
- a = [1] * 10
- j = 0
- while j < 20:
- j += 1
- a.append(j)
- res = interpret(malloc_a_lot, [])
- assert simulator.current_size - curr < 16000
- print "size before: %s, size after %s" % (curr, simulator.current_size)
-
- def test_llinterp_tuples(self):
- curr = simulator.current_size
- def malloc_a_lot():
- i = 0
- while i < 10:
- i += 1
- a = (1, 2, i)
- b = [a] * 10
- j = 0
- while j < 20:
- j += 1
- b.append((1, j, i))
- res = interpret(malloc_a_lot, [])
- assert simulator.current_size - curr < 16000
- print "size before: %s, size after %s" % (curr, simulator.current_size)
-
- def test_global_list(self):
- lst = []
- def append_to_list(i, j):
- lst.append([i] * 500)
- return lst[j][0]
- res = interpret(append_to_list, [0, 0])
- assert res == 0
- for i in range(1, 15):
- res = interpret(append_to_list, [i, i - 1])
- assert res == i - 1 # crashes if constants are not considered roots
More information about the Pypy-commit
mailing list