[pypy-svn] r27506 - in pypy/dist/pypy/rpython/memory: . test
arigo at codespeak.net
arigo at codespeak.net
Sat May 20 15:26:21 CEST 2006
Author: arigo
Date: Sat May 20 15:26:20 2006
New Revision: 27506
Modified:
pypy/dist/pypy/rpython/memory/gc.py
pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py
Log:
Intermediate check-in to show the structure of the linked lists used by
x_clone(). The test fails in obscure heap-corruption-like ways, but I
hope it is "just" a matter of bug fixes in x_clone() or x_swap_list().
Modified: pypy/dist/pypy/rpython/memory/gc.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gc.py (original)
+++ pypy/dist/pypy/rpython/memory/gc.py Sat May 20 15:26:20 2006
@@ -121,6 +121,11 @@
POOL = lltype.GcStruct('gc_pool')
POOLPTR = lltype.Ptr(POOL)
+ POOLNODE = lltype.ForwardReference()
+ POOLNODEPTR = lltype.Ptr(POOLNODE)
+ POOLNODE.become(lltype.Struct('gc_pool_node', ('linkedlist', HDRPTR),
+ ('nextnode', POOLNODEPTR)))
+
def __init__(self, AddressLinkedList, start_heap_size=4096, get_roots=None):
self.heap_usage = 0 # at the end of the latest collection
self.bytes_malloced = 0 # since the latest collection
@@ -129,9 +134,22 @@
self.AddressLinkedList = AddressLinkedList
#self.set_query_functions(None, None, None, None, None, None, None)
self.malloced_objects = lltype.nullptr(self.HDR)
- self.curpool = lltype.nullptr(self.POOL)
self.get_roots = get_roots
self.gcheaderbuilder = GCHeaderBuilder(self.HDR)
+ # pools, for x_swap_pool():
+ # 'curpool' is the current pool, lazily allocated (i.e. NULL means
+ # the current POOL object is not yet malloc'ed). POOL objects are
+ # usually at the start of a linked list of objects, via the HDRs.
+ # The exception is 'curpool' whose linked list of objects is in
+ # 'self.malloced_objects' instead of in the header of 'curpool'.
+ # POOL objects are never in the middle of a linked list themselves.
+ self.curpool = lltype.nullptr(self.POOL)
+ # 'poolnodes' is a linked list of all such linked lists. Each
+ # linked list will usually start with a POOL object, but it can
+ # also contain only normal objects if the POOL object at the head
+ # was already freed. The objects in 'malloced_objects' are not
+ # found via 'poolnodes'.
+ self.poolnodes = lltype.nullptr(self.POOLNODE)
def malloc(self, typeid, length=0):
size = self.fixed_size(typeid)
@@ -235,30 +253,53 @@
i += 1
hdr.typeid = hdr.typeid | 1
objects.delete()
- newmo = self.AddressLinkedList()
+ # also mark self.curpool
+ if self.curpool:
+ gc_info = llmemory.cast_ptr_to_adr(self.curpool) - size_gc_header
+ hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR)
+ hdr.typeid = hdr.typeid | 1
+
curr_heap_size = 0
freed_size = 0
- hdr = self.malloced_objects
- newmo = lltype.nullptr(self.HDR)
- while hdr: #sweep
- typeid = hdr.typeid >> 1
- next = hdr.next
- addr = llmemory.cast_ptr_to_adr(hdr)
- size = self.fixed_size(typeid)
- if self.is_varsize(typeid):
- length = (addr + size_gc_header + self.varsize_offset_to_length(typeid)).signed[0]
- size += self.varsize_item_sizes(typeid) * length
- estimate = raw_malloc_usage(size_gc_header + size)
- if hdr.typeid & 1:
- hdr.typeid = hdr.typeid & (~1)
- hdr.next = newmo
- newmo = hdr
- curr_heap_size += estimate
+ firstpoolnode = lltype.malloc(self.POOLNODE, flavor='raw')
+ firstpoolnode.linkedlist = self.malloced_objects
+ firstpoolnode.nextnode = self.poolnodes
+ prevpoolnode = lltype.nullptr(self.POOLNODE)
+ poolnode = firstpoolnode
+ while poolnode: #sweep
+ ppnext = lltype.direct_fieldptr(poolnode, 'linkedlist')
+ hdr = poolnode.linkedlist
+ while hdr: #sweep
+ typeid = hdr.typeid >> 1
+ next = hdr.next
+ addr = llmemory.cast_ptr_to_adr(hdr)
+ size = self.fixed_size(typeid)
+ if self.is_varsize(typeid):
+ length = (addr + size_gc_header + self.varsize_offset_to_length(typeid)).signed[0]
+ size += self.varsize_item_sizes(typeid) * length
+ estimate = raw_malloc_usage(size_gc_header + size)
+ if hdr.typeid & 1:
+ hdr.typeid = hdr.typeid & (~1)
+ ppnext[0] = hdr
+ ppnext = lltype.direct_fieldptr(hdr, 'next')
+ curr_heap_size += estimate
+ else:
+ freed_size += estimate
+ raw_free(addr)
+ hdr = next
+ ppnext[0] = lltype.nullptr(self.HDR)
+ next = poolnode.nextnode
+ if not poolnode.linkedlist and prevpoolnode:
+ # completely empty node
+ prevpoolnode.nextnode = next
+ lltype.free(poolnode, flavor='raw')
else:
- freed_size += estimate
- raw_free(addr)
- hdr = next
- self.malloced_objects = newmo
+ prevpoolnode = poolnode
+ poolnode = next
+ self.malloced_objects = firstpoolnode.linkedlist
+ self.poolnodes = firstpoolnode.nextnode
+ lltype.free(firstpoolnode, flavor='raw')
+
if curr_heap_size > self.bytes_malloced_threshold:
self.bytes_malloced_threshold = curr_heap_size
end_time = time.time()
@@ -305,6 +346,14 @@
# make a fresh pool object, which is automatically inserted at the
# front of the current list
oldpool = lltype.malloc(self.POOL)
+ addr = llmemory.cast_ptr_to_adr(oldpool)
+ addr -= size_gc_header
+ hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR)
+ # put this new POOL object in the poolnodes list
+ node = lltype.malloc(self.POOLNODE, flavor="raw")
+ node.linkedlist = hdr
+ node.nextnode = self.poolnodes
+ self.poolnodes = node
else:
# manually insert oldpool at the front of the current list
addr = llmemory.cast_ptr_to_adr(oldpool)
@@ -319,12 +368,13 @@
addr -= size_gc_header
hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR)
self.malloced_objects = hdr.next
+ # invariant: now that objects in the hdr.next list are accessible
+ # through self.malloced_objects, make sure they are not accessible
+ # via poolnodes (which has a node pointing to newpool):
+ hdr.next = lltype.nullptr(self.HDR)
else:
# start a fresh new linked list
self.malloced_objects = lltype.nullptr(self.HDR)
- # note that newpool will not be freed by a collect as long as
- # it sits in self.malloced_objects_box, because it is not itself
- # part of any linked list
self.curpool = newpool
return lltype.cast_opaque_ptr(X_POOL_PTR, oldpool)
@@ -334,13 +384,19 @@
# in the specified pool. A new pool is built to contain the
# copies, and the 'gcobjectptr' and 'pool' fields of clonedata
# are adjusted to refer to the result.
+
+ # install a new pool into which all the mallocs go
+ curpool = self.x_swap_pool(lltype.nullptr(X_POOL))
+
size_gc_header = self.gcheaderbuilder.size_gc_header
oldobjects = self.AddressLinkedList()
- oldpool = lltype.cast_opaque_ptr(self.POOLPTR, clonedata.pool)
+ # if no pool specified, use the current pool as the 'source' pool
+ oldpool = clonedata.pool or curpool
+ oldpool = lltype.cast_opaque_ptr(self.POOLPTR, oldpool)
addr = llmemory.cast_ptr_to_adr(oldpool)
addr -= size_gc_header
hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR)
- hdr = hdr.next
+ hdr = hdr.next # skip the POOL object itself
while hdr:
next = hdr.next
hdr.typeid |= 1 # mark all objects from malloced_list
@@ -353,10 +409,11 @@
stack = self.AddressLinkedList()
stack.append(llmemory.cast_ptr_to_adr(clonedata)
+ llmemory.offsetof(X_CLONE, 'gcobjectptr'))
- newobjectlist = lltype.nullptr(self.HDR)
while stack.non_empty():
gcptr_addr = stack.pop()
oldobj_addr = gcptr_addr.address[0]
+ if not oldobj_addr:
+ continue # pointer is NULL
oldhdr = llmemory.cast_adr_to_ptr(oldobj_addr - size_gc_header,
self.HDRPTR)
typeid = oldhdr.typeid
@@ -369,12 +426,11 @@
raise NotImplementedError
else:
size = self.fixed_size(typeid)
- newmem = raw_malloc(size_gc_header + size)
- newhdr = llmemory.cast_adr_to_ptr(newmem, self.HDRPTR)
- newhdr.typeid = typeid << 1
- newhdr.next = newobjectlist
- newobjectlist = newhdr
- newobj_addr = newmem + size_gc_header
+ # XXX! collect() at the beginning if the free heap is low
+ newobj = self.malloc_fixedsize(typeid, size, False)
+ newobj_addr = llmemory.cast_ptr_to_adr(newobj)
+ newhdr_addr = newobj_addr - size_gc_header
+ newhdr = llmemory.cast_adr_to_ptr(newhdr_addr, self.HDRPTR)
raw_memcopy(oldobj_addr, newobj_addr, size)
offsets = self.offsets_to_gc_pointers(typeid)
i = 0
@@ -395,10 +451,8 @@
next = hdr
oldobjects.delete()
- # make the new pool object to collect newobjectlist
- curpool = self.x_swap_pool(lltype.nullptr(X_POOL))
- assert not self.malloced_objects
- self.malloced_objects = newobjectlist
+ # build the new pool object collecting the new objects, and
+ # reinstall the pool that was current at the beginning of x_clone()
clonedata.pool = self.x_swap_pool(curpool)
Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original)
+++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Sat May 20 15:26:20 2006
@@ -295,7 +295,8 @@
def test_cloning(self):
B = lltype.GcStruct('B', ('x', lltype.Signed))
- A = lltype.GcStruct('A', ('b', lltype.Ptr(B)))
+ A = lltype.GcStruct('A', ('b', lltype.Ptr(B)),
+ ('unused', lltype.Ptr(B)))
def make(n):
b = lltype.malloc(B)
b.x = n
@@ -333,6 +334,85 @@
class transformerclass(gctransform.StacklessFrameworkGCTransformer):
GC_PARAMS = {'start_heap_size': 4096 }
+ def test_tree_cloning(self):
+ py.test.skip("aaaaaaaaaaaaaaaaaaaaaaargh later")
+ import os
+ # this makes a tree of calls. Each leaf stores its path (a linked
+ # list) in 'result'. Paths are mutated in-place but the leaves don't
+ # see each other's mutations because of x_clone.
+ NODE = lltype.GcForwardReference()
+ NODE.become(lltype.GcStruct('node', ('index', lltype.Signed),
+ ('counter', lltype.Signed),
+ ('next', lltype.Ptr(NODE))))
+ PATHARRAY = lltype.GcArray(lltype.Ptr(NODE))
+ clonedata = lltype.malloc(X_CLONE)
+
+ def clone(node):
+ # that's for testing if the test is correct...
+ if not node:
+ return node
+ newnode = lltype.malloc(NODE)
+ newnode.index = node.index
+ newnode.counter = node.counter
+ newnode.next = clone(node.next)
+ return newnode
+
+ def do_call(result, path, index, remaining_depth):
+ # clone the while path
+ clonedata.gcobjectptr = lltype.cast_opaque_ptr(llmemory.GCREF,
+ path)
+ clonedata.pool = lltype.nullptr(X_POOL)
+ llop.gc_x_clone(lltype.Void, clonedata)
+ # install the new pool as the current one
+ parentpool = llop.gc_x_swap_pool(X_POOL_PTR, clonedata.pool)
+ path = lltype.cast_opaque_ptr(lltype.Ptr(NODE),
+ clonedata.gcobjectptr)
+
+ # The above should have the same effect as:
+ # path = clone(path)
+
+ # bump all the path's counters by one
+ p = path
+ while p:
+ p.counter += 1
+ p = p.next
+
+ if remaining_depth == 0:
+ result[index] = path # leaf
+ else:
+ node = lltype.malloc(NODE)
+ node.index = index * 2
+ node.counter = 0
+ node.next = path
+ do_call(result, node, node.index, remaining_depth - 1)
+ node.index += 1 # mutation!
+ do_call(result, node, node.index, remaining_depth - 1)
+
+ # restore the parent pool
+ llop.gc_x_swap_pool(X_POOL_PTR, parentpool)
+
+ def check(path, index, level, depth):
+ if level == depth:
+ assert index == 0
+ assert not path
+ else:
+ assert path.index == index
+ assert path.counter == level + 1
+ check(path.next, index >> 1, level + 1, depth)
+
+ def func(depth, dummy):
+ result = lltype.malloc(PATHARRAY, 1 << depth)
+ os.write(2, 'building tree... ')
+ do_call(result, lltype.nullptr(NODE), 0, depth)
+ os.write(2, 'checking tree... ')
+ for i in range(1 << depth):
+ check(result[i], i, 0, depth)
+ os.write(2, 'ok\n')
+ return 1
+ run = self.runner(func, nbargs=2)
+ res = run([5, 0])
+ assert res == 1
+
class TestSemiSpaceGC(TestMarkSweepGC):
def setup_class(cls):
More information about the Pypy-commit
mailing list