[pypy-svn] r51485 - in pypy/dist/pypy: rpython/memory/gc rpython/memory/test translator/c/test
arigo at codespeak.net
arigo at codespeak.net
Thu Feb 14 14:13:15 CET 2008
Author: arigo
Date: Thu Feb 14 14:13:14 2008
New Revision: 51485
Added:
pypy/dist/pypy/rpython/memory/test/snippet.py
- copied unchanged from r51484, pypy/branch/finalizer-order/pypy/rpython/memory/test/snippet.py
Modified:
pypy/dist/pypy/rpython/memory/gc/generation.py
pypy/dist/pypy/rpython/memory/gc/semispace.py
pypy/dist/pypy/rpython/memory/test/test_gc.py
pypy/dist/pypy/translator/c/test/test_newgc.py
Log:
(cfbolz, arigo) (Merge of branch/finalizer-order/pypy)
The straightforward implementation of pypy/doc/discussion/finalizer-order.txt.
Tests pass.
Modified: pypy/dist/pypy/rpython/memory/gc/generation.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gc/generation.py (original)
+++ pypy/dist/pypy/rpython/memory/gc/generation.py Thu Feb 14 14:13:14 2008
@@ -1,6 +1,5 @@
import sys
-from pypy.rpython.memory.gc.semispace import SemiSpaceGC, GCFLAGSHIFT, \
- GCFLAG_IMMORTAL
+from pypy.rpython.memory.gc.semispace import SemiSpaceGC, GCFLAG_IMMORTAL
from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage
from pypy.rpython.lltypesystem import lltype, llmemory, llarena
from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE
@@ -12,11 +11,11 @@
# in the nursery. It is initially set on all prebuilt and old objects,
# and gets cleared by the write_barrier() when we write in them a
# pointer to a young object.
-GCFLAG_NO_YOUNG_PTRS = 1 << (GCFLAGSHIFT+1)
+GCFLAG_NO_YOUNG_PTRS = SemiSpaceGC.first_unused_gcflag << 1
# The following flag is set for static roots which are not on the list
# of static roots yet, but will appear with write barrier
-GCFLAG_NO_HEAP_PTRS = 1 << (GCFLAGSHIFT+2)
+GCFLAG_NO_HEAP_PTRS = SemiSpaceGC.first_unused_gcflag << 2
DEBUG_PRINT = False
@@ -30,6 +29,7 @@
inline_simple_malloc_varsize = True
needs_write_barrier = True
prebuilt_gc_objects_are_static_roots = False
+ first_unused_gcflag = SemiSpaceGC.first_unused_gcflag << 3
def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE,
nursery_size=128,
Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gc/semispace.py (original)
+++ pypy/dist/pypy/rpython/memory/gc/semispace.py Thu Feb 14 14:13:14 2008
@@ -2,7 +2,7 @@
from pypy.rpython.lltypesystem.llmemory import raw_memcopy, raw_memclear
from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage
from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE
-from pypy.rpython.memory.support import get_address_stack
+from pypy.rpython.memory.support import get_address_stack, get_address_deque
from pypy.rpython.memory.gcheader import GCHeaderBuilder
from pypy.rpython.lltypesystem import lltype, llmemory, llarena
from pypy.rlib.objectmodel import free_non_gc_object
@@ -14,8 +14,9 @@
import sys, os
TYPEID_MASK = 0xffff
-GCFLAGSHIFT = 16
-GCFLAG_IMMORTAL = 1 << GCFLAGSHIFT
+first_gcflag = 1 << 16
+GCFLAG_IMMORTAL = first_gcflag
+GCFLAG_FINALIZATION_ORDERING = first_gcflag << 1
memoryError = MemoryError()
@@ -24,6 +25,7 @@
inline_simple_malloc = True
inline_simple_malloc_varsize = True
needs_zero_gc_pointers = False
+ first_unused_gcflag = first_gcflag << 2
HDR = lltype.Struct('header', ('forw', llmemory.Address),
('tid', lltype.Signed))
@@ -35,6 +37,7 @@
self.max_space_size = max_space_size
self.gcheaderbuilder = GCHeaderBuilder(self.HDR)
self.AddressStack = get_address_stack(chunk_size)
+ self.AddressDeque = get_address_deque(chunk_size)
def setup(self):
self.tospace = llarena.arena_malloc(self.space_size, True)
@@ -43,8 +46,8 @@
self.fromspace = llarena.arena_malloc(self.space_size, True)
ll_assert(bool(self.fromspace), "couldn't allocate fromspace")
self.free = self.tospace
- self.objects_with_finalizers = self.AddressStack()
- self.run_finalizers = self.AddressStack()
+ self.objects_with_finalizers = self.AddressDeque()
+ self.run_finalizers = self.AddressDeque()
self.objects_with_weakrefs = self.AddressStack()
self.finalizer_lock_count = 0
self.red_zone = 0
@@ -189,12 +192,11 @@
self.top_of_space = tospace + self.space_size
scan = self.free = tospace
self.collect_roots()
- scan = self.scan_copied(scan)
if self.run_finalizers.non_empty():
self.update_run_finalizers()
- if self.objects_with_finalizers.non_empty():
- self.deal_with_objects_with_finalizers()
scan = self.scan_copied(scan)
+ if self.objects_with_finalizers.non_empty():
+ scan = self.deal_with_objects_with_finalizers(scan)
if self.objects_with_weakrefs.non_empty():
self.invalidate_weakrefs()
self.notify_objects_just_moved()
@@ -308,20 +310,105 @@
else:
hdr.forw = NULL
- def deal_with_objects_with_finalizers(self):
+ def deal_with_objects_with_finalizers(self, scan):
# walk over list of objects with finalizers
# if it is not copied, add it to the list of to-be-called finalizers
# and copy it, to me make the finalizer runnable
- # NOTE: the caller is calling scan_copied, so no need to do it here
- new_with_finalizer = self.AddressStack()
+ # We try to run the finalizers in a "reasonable" order, like
+ # CPython does. The details of this algorithm are in
+ # pypy/doc/discussion/finalizer-order.txt.
+ new_with_finalizer = self.AddressDeque()
+ marked = self.AddressDeque()
+ pending = self.AddressStack()
while self.objects_with_finalizers.non_empty():
- obj = self.objects_with_finalizers.pop()
- if self.is_forwarded(obj):
- new_with_finalizer.append(self.get_forwarding_address(obj))
+ x = self.objects_with_finalizers.popleft()
+ ll_assert(self._finalization_state(x) != 1,
+ "bad finalization state 1")
+ if self.is_forwarded(x):
+ new_with_finalizer.append(self.get_forwarding_address(x))
+ continue
+ marked.append(x)
+ pending.append(x)
+ while pending.non_empty():
+ y = pending.pop()
+ state = self._finalization_state(y)
+ if state == 0:
+ self._bump_finalization_state_from_0_to_1(y)
+ elif state == 2:
+ self._bump_finalization_state_from_2_to_3(y)
+ else:
+ continue # don't need to recurse inside y
+ self.trace(y, self._append_if_nonnull, pending)
+ scan = self._recursively_bump_finalization_state_from_1_to_2(
+ x, scan)
+
+ while marked.non_empty():
+ x = marked.popleft()
+ state = self._finalization_state(x)
+ ll_assert(state >= 2, "unexpected finalization state < 2")
+ newx = self.get_forwarding_address(x)
+ if state == 2:
+ self.run_finalizers.append(newx)
+ # we must also fix the state from 2 to 3 here, otherwise
+ # we leave the GCFLAG_FINALIZATION_ORDERING bit behind
+ # which will confuse the next collection
+ pending.append(x)
+ while pending.non_empty():
+ y = pending.pop()
+ state = self._finalization_state(y)
+ if state == 2:
+ self._bump_finalization_state_from_2_to_3(y)
+ self.trace(y, self._append_if_nonnull, pending)
else:
- self.run_finalizers.append(self.copy(obj))
+ new_with_finalizer.append(newx)
+
+ pending.delete()
+ marked.delete()
self.objects_with_finalizers.delete()
self.objects_with_finalizers = new_with_finalizer
+ return scan
+
+ def _append_if_nonnull(pointer, stack):
+ if pointer.address[0] != NULL:
+ stack.append(pointer.address[0])
+ _append_if_nonnull = staticmethod(_append_if_nonnull)
+
+ def _finalization_state(self, obj):
+ if self.is_forwarded(obj):
+ newobj = self.get_forwarding_address(obj)
+ hdr = self.header(newobj)
+ if hdr.tid & GCFLAG_FINALIZATION_ORDERING:
+ return 2
+ else:
+ return 3
+ else:
+ hdr = self.header(obj)
+ if hdr.tid & GCFLAG_FINALIZATION_ORDERING:
+ return 1
+ else:
+ return 0
+
+ def _bump_finalization_state_from_0_to_1(self, obj):
+ ll_assert(self._finalization_state(obj) == 0,
+ "unexpected finalization state != 0")
+ hdr = self.header(obj)
+ hdr.tid |= GCFLAG_FINALIZATION_ORDERING
+
+ def _bump_finalization_state_from_2_to_3(self, obj):
+ ll_assert(self._finalization_state(obj) == 2,
+ "unexpected finalization state != 2")
+ newobj = self.get_forwarding_address(obj)
+ hdr = self.header(newobj)
+ hdr.tid &= ~GCFLAG_FINALIZATION_ORDERING
+
+ def _recursively_bump_finalization_state_from_1_to_2(self, obj, scan):
+ # recursively convert objects from state 1 to state 2.
+ # Note that copy() copies all bits, including the
+ # GCFLAG_FINALIZATION_ORDERING. The mapping between
+ # state numbers and the presence of this bit was designed
+ # for the following to work :-)
+ self.copy(obj)
+ return self.scan_copied(scan)
def invalidate_weakrefs(self):
# walk over list of objects that contain weakrefs
@@ -349,9 +436,9 @@
def update_run_finalizers(self):
# we are in an inner collection, caused by a finalizer
# the run_finalizers objects need to be copied
- new_run_finalizer = self.AddressStack()
+ new_run_finalizer = self.AddressDeque()
while self.run_finalizers.non_empty():
- obj = self.run_finalizers.pop()
+ obj = self.run_finalizers.popleft()
new_run_finalizer.append(self.copy(obj))
self.run_finalizers.delete()
self.run_finalizers = new_run_finalizer
@@ -363,7 +450,7 @@
try:
while self.run_finalizers.non_empty():
#print "finalizer"
- obj = self.run_finalizers.pop()
+ obj = self.run_finalizers.popleft()
finalizer = self.getfinalizer(self.get_type_id(obj))
finalizer(obj)
finally:
Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/test/test_gc.py (original)
+++ pypy/dist/pypy/rpython/memory/test/test_gc.py Thu Feb 14 14:13:14 2008
@@ -3,8 +3,10 @@
#from pypy.rpython.memory.support import INT_SIZE
from pypy.rpython.memory import gcwrapper
+from pypy.rpython.memory.test import snippet
from pypy.rpython.test.test_llinterp import get_interpreter
from pypy.rpython.lltypesystem import lltype
+from pypy.rpython.lltypesystem.rstr import STR
from pypy.rpython.lltypesystem.lloperation import llop
from pypy.rlib.objectmodel import we_are_translated
from pypy.rlib.objectmodel import compute_unique_id
@@ -35,6 +37,12 @@
self.GC_PARAMS)
return interp.eval_graph(graph, values)
+ def run(self, func): # for snippet.py
+ res = self.interpret(func, [])
+ if lltype.typeOf(res) == lltype.Ptr(STR):
+ res = ''.join(res.chars)
+ return res
+
def test_llinterp_lists(self):
#curr = simulator.current_size
def malloc_a_lot():
@@ -384,108 +392,9 @@
class TestMarkSweepGC(GCTest):
from pypy.rpython.memory.gc.marksweep import MarkSweepGC as GCClass
-class TestSemiSpaceGC(GCTest):
+class TestSemiSpaceGC(GCTest, snippet.SemiSpaceGCTests):
from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass
- def test_finalizer_order(self):
- py.test.skip("in-progress")
- import random
- from pypy.tool.algo import graphlib
-
- examples = []
- letters = 'abcdefghijklmnopqrstuvwxyz'
- for i in range(20):
- input = []
- edges = {}
- for c in letters:
- edges[c] = []
- # make up a random graph
- for c in letters:
- for j in range(random.randrange(0, 4)):
- d = random.choice(letters)
- edges[c].append(graphlib.Edge(c, d))
- input.append((c, d))
- # find the expected order in which destructors should be called
- components = list(graphlib.strong_components(edges, edges))
- head = {}
- for component in components:
- c = component.keys()[0]
- for d in component:
- assert d not in head
- head[d] = c
- assert len(head) == len(letters)
- strict = []
- for c, d in input:
- if head[c] != head[d]:
- strict.append((c, d))
- examples.append((input, components, strict))
-
- class State:
- pass
- state = State()
- class A:
- def __init__(self, key):
- self.key = key
- self.refs = []
- def __del__(self):
- assert state.age[self.key] == -1
- state.age[self.key] = state.time
- state.progress = True
-
- def build_example(input):
- state.time = 0
- state.age = {}
- vertices = {}
- for c in letters:
- vertices[c] = A(c)
- state.age[c] = -1
- for c, d in input:
- vertices[c].refs.append(d)
-
- def f():
- i = 0
- while i < len(examples):
- input, components, strict = examples[i]
- build_example(input)
- while state.time < len(letters):
- state.progress = False
- llop.gc__collect(lltype.Void)
- if not state.progress:
- break
- state.time += 1
- # check that all instances have been finalized
- if -1 in state.age.values():
- return i * 10 + 1
- # check that if a -> b and a and b are not in the same
- # strong component, then a is finalized strictly before b
- for c, d in strict:
- if state.age[c] >= state.age[d]:
- return i * 10 + 2
- # check that two instances in the same strong component
- # are never finalized during the same collection
- for component in components:
- seen = {}
- for c in component:
- age = state.age[c]
- if age in seen:
- return i * 10 + 3
- seen[age] = True
- i += 1
- return 0
-
- res = self.interpret(f, [])
- if res != 0:
- import pprint
- pprint.pprint(examples[res / 10])
- if res % 10 == 1:
- py.test.fail("some instances have not been finalized at all")
- if res % 10 == 2:
- py.test.fail("the strict order is not respected")
- if res % 10 == 3:
- py.test.fail("two instances from the same component "
- "have been finalized together")
- assert 0
-
class TestGrowingSemiSpaceGC(TestSemiSpaceGC):
GC_PARAMS = {'space_size': 64}
Modified: pypy/dist/pypy/translator/c/test/test_newgc.py
==============================================================================
--- pypy/dist/pypy/translator/c/test/test_newgc.py (original)
+++ pypy/dist/pypy/translator/c/test/test_newgc.py Thu Feb 14 14:13:14 2008
@@ -9,6 +9,7 @@
from pypy.translator.c import genc, gc
from pypy.rpython.lltypesystem import lltype, llmemory
from pypy.rpython.lltypesystem.lloperation import llop
+from pypy.rpython.memory.test import snippet
from pypy import conftest
def compile_func(fn, inputtypes, t=None, gcpolicy="ref"):
@@ -281,6 +282,12 @@
gcpolicy = "marksweep"
should_be_moving = False
+ # interface for snippet.py
+ large_tests_ok = True
+ def run(self, func):
+ fn = self.getcompiled(func)
+ return fn()
+
def test_empty_collect(self):
def f():
llop.gc__collect(lltype.Void)
@@ -836,7 +843,7 @@
def test_weakref(self):
py.test.skip("fails for some reason I couldn't figure out yet :-(")
-class TestSemiSpaceGC(TestUsingFramework):
+class TestSemiSpaceGC(TestUsingFramework, snippet.SemiSpaceGCTests):
gcpolicy = "semispace"
should_be_moving = True
More information about the Pypy-commit
mailing list