[pypy-commit] pypy incremental-gc: refactoring tests and initial write barrier code
andrewchambers
noreply at buildbot.pypy.org
Tue Aug 13 10:27:41 CEST 2013
Author: Andrew Chambers <andrewchamberss at gmail.com>
Branch: incremental-gc
Changeset: r66112:fdbbced1820f
Date: 2013-08-13 16:54 +1200
http://bitbucket.org/pypy/pypy/changeset/fdbbced1820f/
Log: refactoring tests and initial write barrier code
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -133,15 +133,19 @@
# The scanning phase, next step call will scan the current roots
# This state must complete in a single step
-STATE_SCANNING = 0
+STATE_SCANNING = 1 << 0
#XXX describe
# marking of objects can be done over multiple
-STATE_MARKING = 1
-STATE_SWEEPING_RAWMALLOC = 2
-STATE_SWEEPING_ARENA_1 = 3
-STATE_SWEEPING_ARENA_2 = 4
-STATE_FINALIZING = 5
+STATE_MARKING = 1 << 1
+STATE_SWEEPING_RAWMALLOC = 1 << 2
+STATE_SWEEPING_ARENA_1 = 1 << 3
+STATE_SWEEPING_ARENA_2 = 1 << 4
+STATE_FINALIZING = 1 << 5
+
+MASK_SWEEPING = (STATE_SWEEPING_RAWMALLOC |
+ STATE_SWEEPING_ARENA_1 |
+ STATE_SWEEPING_ARENA_2)
@@ -333,7 +337,7 @@
self.rawmalloced_total_size = r_uint(0)
self.gc_state = r_uint(0) #XXX Only really needs to be a byte
-
+ self.gc_state = STATE_SCANNING
#
# A list of all objects with finalizers (these are never young).
self.objects_with_finalizers = self.AddressDeque()
@@ -1176,13 +1180,24 @@
def write_barrier(self, newvalue, addr_struct):
if self.header(addr_struct).tid & GCFLAG_TRACK_YOUNG_PTRS:
self.remember_young_pointer(addr_struct, newvalue)
+
+ if self.gc_state == STATE_MARKING:
+ if self.header(addr_struct).tid & GCFLAG_VISITED:
+ self.write_to_visited_object_forward(addr_struct,new_value)
+
def write_barrier_from_array(self, newvalue, addr_array, index):
+
if self.header(addr_array).tid & GCFLAG_TRACK_YOUNG_PTRS:
if self.card_page_indices > 0: # <- constant-folded
self.remember_young_pointer_from_array2(addr_array, index)
else:
self.remember_young_pointer(addr_array, newvalue)
+
+ if self.gc_state == STATE_MARKING:
+ if self.header(addr_struct).tid & GCFLAG_VISITED:
+ self.write_to_visited_object_backward(addr_struct,new_value)
+
def _init_writebarrier_logic(self):
DEBUG = self.DEBUG
@@ -1190,6 +1205,33 @@
# instead of keeping it as a regular method is to
# make the code in write_barrier() marginally smaller
# (which is important because it is inlined *everywhere*).
+
+ # move marking process forward
+ def write_to_visited_object_forward(addr_struct, new_value):
+ ll_assert(self.gc_state == STATE_MARKING,"expected MARKING state")
+ if self.header(new_value).tid & (GCFLAG_GRAY | GCFLAG_VISITED) == 0:
+ # writing a white object into black, make new object gray and
+ # add to objects_to_trace
+ #
+ self.header(new_value).tid |= GCFLAG_GRAY
+ self.objects_to_trace.append(new_value)
+ write_to_visited_object_forward._dont_inline_ = True
+ self.write_to_visited_object_forward = write_to_visited_object_forward
+
+ # move marking process backward
+ def write_to_visited_object_backward(addr_struct, new_value):
+ ll_assert(self.gc_state == STATE_MARKING,"expected MARKING state")
+ if self.header(new_value).tid & (GCFLAG_GRAY | GCFLAG_VISITED) == 0:
+ # writing a white object into black, make black gray and
+ # readd to objects_to_trace
+ # this is useful for arrays because it stops the writebarrier
+ # from being re-triggered on successive writes
+ self.header(addr_struct).tid &= ~GCFLAG_VISITED
+ self.header(addr_struct).tid |= GCFLAG_GRAY
+ self.objects_to_trace.append(addr_struct)
+ write_to_visited_object_backward._dont_inline_ = True
+ self.write_to_visited_object_backward = write_to_visited_object_backward
+
def remember_young_pointer(addr_struct, newvalue):
# 'addr_struct' is the address of the object in which we write.
# 'newvalue' is the address that we are going to write in there.
@@ -1728,9 +1770,20 @@
old.append(new.pop())
new.delete()
+ def debug_gc_step_until(self,state):
+ while self.gc_state != state:
+ self.minor_collection()
+ self.major_collection_step()
+
+ def debug_gc_step_n(self,n):
+ while n > 0:
+ self.minor_collection()
+ self.major_collection_step()
+ n -= 1
+
# Note - minor collections seem fast enough so that one
# is done before every major collection step
- def major_collection_step(self,reserving_size):
+ def major_collection_step(self,reserving_size=0):
debug_start("gc-collect-step")
debug_print("stating gc state: ",self.gc_state)
# Debugging checks
diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -584,109 +584,14 @@
class TestMiniMarkGCFull(DirectGCTest):
from rpython.memory.gc.minimark import MiniMarkGC as GCClass
+class TestIncrementalMiniMarkGCSimple(TestMiniMarkGCSimple):
+ from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass
+
+
+
+ def test_write_barrier(self):
+ pass
+
-class TestIncrementalMiniMarkGCSimple(DirectGCTest):
+class TestIncrementalMiniMarkGCFull(TestMiniMarkGCFull):
from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass
- #simple arena doesnt change for incremental.
- from rpython.memory.gc.minimarktest import SimpleArenaCollection
- # test the GC itself, providing a simple class for ArenaCollection
- GC_PARAMS = {'ArenaCollectionClass': SimpleArenaCollection}
-
- def test_card_marker(self):
- for arraylength in (range(4, 17)
- + [69] # 3 bytes
- + [300]): # 10 bytes
- print 'array length:', arraylength
- nums = {}
- a = self.malloc(VAR, arraylength)
- self.stackroots.append(a)
- for i in range(50):
- p = self.malloc(S)
- p.x = -i
- a = self.stackroots[-1]
- index = (i*i) % arraylength
- self.writearray(a, index, p)
- nums[index] = p.x
- #
- for index, expected_x in nums.items():
- assert a[index].x == expected_x
- self.stackroots.pop()
- test_card_marker.GC_PARAMS = {"card_page_indices": 4}
-
- def test_writebarrier_before_copy(self):
- from rpython.memory.gc import incminimark
- largeobj_size = self.gc.nonlarge_max + 1
- self.gc.next_major_collection_threshold = 99999.0
- p_src = self.malloc(VAR, largeobj_size)
- p_dst = self.malloc(VAR, largeobj_size)
- # make them old
- self.stackroots.append(p_src)
- self.stackroots.append(p_dst)
- self.gc.collect()
- p_dst = self.stackroots.pop()
- p_src = self.stackroots.pop()
- #
- addr_src = llmemory.cast_ptr_to_adr(p_src)
- addr_dst = llmemory.cast_ptr_to_adr(p_dst)
- hdr_src = self.gc.header(addr_src)
- hdr_dst = self.gc.header(addr_dst)
- #
- assert hdr_src.tid & incminimark.GCFLAG_TRACK_YOUNG_PTRS
- assert hdr_dst.tid & incminimark.GCFLAG_TRACK_YOUNG_PTRS
- #
- res = self.gc.writebarrier_before_copy(addr_src, addr_dst, 0, 0, 10)
- assert res
- assert hdr_dst.tid & incminimark.GCFLAG_TRACK_YOUNG_PTRS
- #
- hdr_src.tid &= ~incminimark.GCFLAG_TRACK_YOUNG_PTRS # pretend we have young ptrs
- res = self.gc.writebarrier_before_copy(addr_src, addr_dst, 0, 0, 10)
- assert res # we optimized it
- assert hdr_dst.tid & incminimark.GCFLAG_TRACK_YOUNG_PTRS == 0 # and we copied the flag
- #
- hdr_src.tid |= incminimark.GCFLAG_TRACK_YOUNG_PTRS
- hdr_dst.tid |= incminimark.GCFLAG_TRACK_YOUNG_PTRS
- hdr_src.tid |= incminimark.GCFLAG_HAS_CARDS
- hdr_src.tid |= incminimark.GCFLAG_CARDS_SET
- # hdr_dst.tid does not have minimark.GCFLAG_HAS_CARDS
- res = self.gc.writebarrier_before_copy(addr_src, addr_dst, 0, 0, 10)
- assert not res # there might be young ptrs, let ll_arraycopy to find them
-
- def test_writebarrier_before_copy_preserving_cards(self):
- from rpython.rtyper.lltypesystem import llarena
- from rpython.memory.gc import incminimark
- tid = self.get_type_id(VAR)
- largeobj_size = self.gc.nonlarge_max + 1
- self.gc.next_major_collection_threshold = 99999.0
- addr_src = self.gc.external_malloc(tid, largeobj_size)
- addr_dst = self.gc.external_malloc(tid, largeobj_size)
- hdr_src = self.gc.header(addr_src)
- hdr_dst = self.gc.header(addr_dst)
- #
- assert hdr_src.tid & incminimark.GCFLAG_HAS_CARDS
- assert hdr_dst.tid & incminimark.GCFLAG_HAS_CARDS
- #
- young_p = self.malloc(S)
- self.gc.write_barrier_from_array(young_p, addr_src, 0)
- index_in_third_page = int(2.5 * self.gc.card_page_indices)
- assert index_in_third_page < largeobj_size
- self.gc.write_barrier_from_array(young_p, addr_src,
- index_in_third_page)
- #
- assert hdr_src.tid & incminimark.GCFLAG_CARDS_SET
- addr_byte = self.gc.get_card(addr_src, 0)
- assert ord(addr_byte.char[0]) == 0x01 | 0x04 # bits 0 and 2
- #
- res = self.gc.writebarrier_before_copy(addr_src, addr_dst,
- 0, 0, 2*self.gc.card_page_indices)
- assert res
- #
- assert hdr_dst.tid & incminimark.GCFLAG_CARDS_SET
- addr_byte = self.gc.get_card(addr_dst, 0)
- assert ord(addr_byte.char[0]) == 0x01 | 0x04 # bits 0 and 2
-
- test_writebarrier_before_copy_preserving_cards.GC_PARAMS = {
- "card_page_indices": 4}
-
-
-class TestIncrementalMiniMarkGCFull(DirectGCTest):
- from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass
diff --git a/rpython/memory/test/test_incminimark_gc.py b/rpython/memory/test/test_incminimark_gc.py
--- a/rpython/memory/test/test_incminimark_gc.py
+++ b/rpython/memory/test/test_incminimark_gc.py
@@ -1,11 +1,6 @@
from rpython.rlib.rarithmetic import LONG_BIT
-from rpython.memory.test import test_semispace_gc
+from rpython.memory.test import test_minimark_gc
-WORD = LONG_BIT // 8
-
-class TestIncrementalMiniMarkGC(test_semispace_gc.TestSemiSpaceGC):
+class TestIncrementalMiniMarkGC(test_minimark_gc.TestMiniMarkGC):
from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass
- GC_CAN_SHRINK_BIG_ARRAY = False
- GC_CAN_MALLOC_NONMOVABLE = True
- BUT_HOW_BIG_IS_A_BIG_STRING = 11*WORD
diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py
--- a/rpython/memory/test/test_transformed_gc.py
+++ b/rpython/memory/test/test_transformed_gc.py
@@ -1259,48 +1259,8 @@
res = run([])
assert res == 123
-class TestIncrementalMiniMarkGC(TestHybridGC):
+class TestIncrementalMiniMarkGC(TestMiniMarkGC):
gcname = "incminimark"
- GC_CAN_TEST_ID = True
-
- class gcpolicy(gc.BasicFrameworkGcPolicy):
- class transformerclass(shadowstack.ShadowStackFrameworkGCTransformer):
- from rpython.memory.gc.incminimark \
- import IncrementalMiniMarkGC as GCClass
- GC_PARAMS = {'nursery_size': 32*WORD,
- 'page_size': 16*WORD,
- 'arena_size': 64*WORD,
- 'small_request_threshold': 5*WORD,
- 'large_object': 8*WORD,
- 'card_page_indices': 4,
- 'translated_to_c': False,
- }
- root_stack_depth = 200
-
- def define_no_clean_setarrayitems(cls):
- # The optimization find_clean_setarrayitems() in
- # gctransformer/framework.py does not work with card marking.
- # Check that it is turned off.
- S = lltype.GcStruct('S', ('x', lltype.Signed))
- A = lltype.GcArray(lltype.Ptr(S))
- def sub(lst):
- lst[15] = lltype.malloc(S) # 'lst' is set the single mark "12-15"
- lst[15].x = 123
- lst[0] = lst[15] # that would be a "clean_setarrayitem"
- def f():
- lst = lltype.malloc(A, 16) # 16 > 10
- rgc.collect()
- sub(lst)
- null = lltype.nullptr(S)
- lst[15] = null # clear, so that A() is only visible via lst[0]
- rgc.collect() # -> crash
- return lst[0].x
- return f
-
- def test_no_clean_setarrayitems(self):
- run = self.runner("no_clean_setarrayitems")
- res = run([])
- assert res == 123
# ________________________________________________________________
More information about the pypy-commit
mailing list