[pypy-commit] pypy cpyext-gc-cycle: Fixed bugs in rrc incmark
stevie_92
pypy.commits at gmail.com
Sat Sep 21 10:43:29 EDT 2019
Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r97578:9543fe367251
Date: 2019-09-21 16:42 +0200
http://bitbucket.org/pypy/pypy/changeset/9543fe367251/
Log: Fixed bugs in rrc incmark Added incremental rrc tests and debug
output of snapshot
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -35,11 +35,13 @@
# Now take a snapshot
self._take_snapshot(self.pyobj_list)
+ self._debug_print_snap(print_label="after-snapshot")
# collect all rawrefcounted roots
self._collect_roots()
# TODO: execute incrementally (own phase, save index)
+ self._debug_print_snap(print_label="roots-marked")
self._debug_check_consistency(print_label="roots-marked")
self.state = self.STATE_MARKING
return False
@@ -52,6 +54,7 @@
if (all_rrc_marked and not self.gc.objects_to_trace.non_empty() and
not self.gc.more_objects_to_trace.non_empty()):
# all objects have been marked, dead objects will stay dead
+ self._debug_print_snap(print_label="before-fin")
self._debug_check_consistency(print_label="before-fin")
self.state = self.STATE_GARBAGE_MARKING
else:
@@ -112,7 +115,9 @@
self._gc_list_remove(pygchdr)
self._gc_list_add(self.pyobj_old_list, pygchdr)
else:
- pygchdr.c_gc_refs = 1 # new object, keep alive
+ # new object, keep alive
+ pyobj = self.gc_as_pyobj(pygchdr)
+ pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
# TODO: also keep reachable objects alive (in case rc proxy -> non-rc -> non-rc proxy -> rc obj!!!)
pygchdr = next_old
@@ -122,11 +127,11 @@
while free_p_list.non_empty():
self.p_list_old.append(free_p_list.pop())
while pygchdr <> self.pyobj_list: # continue previous loop
- pygchdr.c_gc_refs = 1
+ pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
pygchdr = pygchdr.c_gc_next
pygchdr = self.pyobj_old_list.c_gc_next
while pygchdr <> self.pyobj_old_list: # resurrect "dead" objects
- pygchdr.c_gc_refs = 1
+ pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
pygchdr = pygchdr.c_gc_next
if not self._gc_list_is_empty(self.pyobj_old_list):
self._gc_list_merge(self.pyobj_old_list, self.pyobj_list)
@@ -158,6 +163,15 @@
self._debug_check_consistency(print_label="end-mark")
return True
+ def _debug_print_snap(self, print_label=None):
+ debug_start("snap " + print_label)
+ for i in range(0, self.total_objs):
+ snapobj = self.snapshot_objs[i]
+ debug_print("item", snapobj.pyobj, ": snapobj", snapobj,
+ "refcnt", snapobj.refcnt,
+ "refcnt original", snapobj.refcnt_original,
+ "link", snapobj.pypy_link)
+
def _free_p_list(self, pyobject, foo):
from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
@@ -186,7 +200,7 @@
# refcount > 0
def _mark_rawrefcount(self):
- self._gc_list_init(self.pyobj_old_list)
+ self._gc_list_init(self.pyobj_old_list) # TODO: move???
# as long as new objects with cyclic a refcount > 0 or alive border
# objects are found, increment the refcount of all referenced objects
# of those newly found objects
diff --git a/rpython/memory/gc/rrc/mark.py b/rpython/memory/gc/rrc/mark.py
--- a/rpython/memory/gc/rrc/mark.py
+++ b/rpython/memory/gc/rrc/mark.py
@@ -198,8 +198,6 @@
pyobj = self.gc_as_pyobj(gchdr)
obj = llmemory.NULL
if pyobj.c_ob_pypy_link <> 0:
- #intobj = pyobj.c_ob_pypy_link
- #obj = llmemory.cast_int_to_adr(intobj)
pyobject = llmemory.cast_ptr_to_adr(pyobj)
obj = self.refcnt_dict.get(pyobject)
if not alive and self.gc.header(obj).tid & (
@@ -218,8 +216,6 @@
self._traverse(pyobj, 1)
# mark recursively, if it is a pypyobj
if pyobj.c_ob_pypy_link <> 0:
- #intobj = pyobj.c_ob_pypy_link
- #obj = llmemory.cast_int_to_adr(intobj)
self.gc.objects_to_trace.append(obj)
self.gc.visit_all_objects()
return alive
diff --git a/rpython/memory/gc/test/dot/keep_cpython_self.dot b/rpython/memory/gc/test/dot/keep_cpython_inc_1.dot
copy from rpython/memory/gc/test/dot/keep_cpython_self.dot
copy to rpython/memory/gc/test/dot/keep_cpython_inc_1.dot
--- a/rpython/memory/gc/test/dot/keep_cpython_self.dot
+++ b/rpython/memory/gc/test/dot/keep_cpython_inc_1.dot
@@ -1,4 +1,18 @@
digraph G {
- "a" [type=C, alive=y, ext_refcnt=1];
- "a" -> "a";
+ "a" [type=B, alive=y, ext_refcnt=1];
+ "b" [type=P, alive=n];
+ "c" [type=B, alive=y];
+ "d" [type=P, alive=y];
+ "e" [type=B, alive=y];
+ "f" [type=C, alive=y];
+ "g" [type=C, alive=y];
+ "h" [type=C, alive=y, ext_refcnt=1];
+ "a" -> "b" [removed=after_snap];
+ "b"-> "c";
+ "c" -> "d";
+ "c" -> "f";
+ "f" -> "c";
+ "d" -> "e";
+ "e" -> "g";
+ "h" -> "f" [added=after_snap];
}
diff --git a/rpython/memory/gc/test/dot/keep_cpython_self.dot b/rpython/memory/gc/test/dot/keep_cpython_inc_2.dot
copy from rpython/memory/gc/test/dot/keep_cpython_self.dot
copy to rpython/memory/gc/test/dot/keep_cpython_inc_2.dot
--- a/rpython/memory/gc/test/dot/keep_cpython_self.dot
+++ b/rpython/memory/gc/test/dot/keep_cpython_inc_2.dot
@@ -1,4 +1,15 @@
digraph G {
- "a" [type=C, alive=y, ext_refcnt=1];
- "a" -> "a";
+ "a" [type=C, alive=y, ext_refcnt=1, added=after_snap];
+ "b" [type=B, alive=y, added=after_snap];
+ "c" [type=P, alive=y, added=after_snap];
+ "d" [type=P, alive=y, added=after_snap];
+ "e" [type=B, alive=y, added=after_snap];
+ "f" [type=C, alive=y, added=after_snap];
+ "g" [type=C, alive=y, added=after_snap];
+ "a" -> "b" [added=after_snap];
+ "b" -> "c" [added=after_snap];
+ "c" -> "d" [added=after_snap];
+ "d" -> "e" [added=after_snap];
+ "e" -> "f" [added=after_snap];
+ "f" -> "g" [added=after_snap];
}
diff --git a/rpython/memory/gc/test/dot/keep_cpython_self.dot b/rpython/memory/gc/test/dot/keep_cpython_inc_3.dot
copy from rpython/memory/gc/test/dot/keep_cpython_self.dot
copy to rpython/memory/gc/test/dot/keep_cpython_inc_3.dot
--- a/rpython/memory/gc/test/dot/keep_cpython_self.dot
+++ b/rpython/memory/gc/test/dot/keep_cpython_inc_3.dot
@@ -1,4 +1,15 @@
digraph G {
- "a" [type=C, alive=y, ext_refcnt=1];
- "a" -> "a";
+ "a" [type=B, alive=y, ext_refcnt=1];
+ "b" [type=P, alive=n];
+ "c" [type=P, alive=y];
+ "d" [type=B, alive=y];
+ "e" [type=C, alive=y];
+ "f" [type=C, alive=y, ext_refcnt=1, added=after_snap];
+ "g" [type=B, alive=y, added=linked_after_snap];
+ "a" -> "b" [removed=after_snap];
+ "b" -> "c";
+ "c" -> "d";
+ "d" -> "e";
+ "f" -> "g" [added=after_snap];
+ "g" -> "c" [added=after_snap];
}
diff --git a/rpython/memory/gc/test/dot/keep_cross_simple.dot b/rpython/memory/gc/test/dot/keep_cross_simple_3a.dot
copy from rpython/memory/gc/test/dot/keep_cross_simple.dot
copy to rpython/memory/gc/test/dot/keep_cross_simple_3a.dot
--- a/rpython/memory/gc/test/dot/keep_cross_simple.dot
+++ b/rpython/memory/gc/test/dot/keep_cross_simple_3a.dot
@@ -1,5 +1,5 @@
digraph G {
"a" [type=B, alive=y, rooted=y];
- "b" [type=C, alive=y];
+ "b" [type=B, alive=y];
"a" -> "b";
}
diff --git a/rpython/memory/gc/test/dot/keep_cross_simple.dot b/rpython/memory/gc/test/dot/keep_cross_simple_3b.dot
copy from rpython/memory/gc/test/dot/keep_cross_simple.dot
copy to rpython/memory/gc/test/dot/keep_cross_simple_3b.dot
--- a/rpython/memory/gc/test/dot/keep_cross_simple.dot
+++ b/rpython/memory/gc/test/dot/keep_cross_simple_3b.dot
@@ -1,5 +1,5 @@
digraph G {
- "a" [type=B, alive=y, rooted=y];
- "b" [type=C, alive=y];
+ "a" [type=B, alive=y, ext_refcnt=1];
+ "b" [type=B, alive=y];
"a" -> "b";
}
diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -511,7 +511,7 @@
class NodeInfo:
def __init__(self, type, alive, ext_refcnt, finalizer, resurrect,
- delete, garbage, tuple, gc):
+ delete, garbage, tuple, gc, rooted, tracked):
self.type = type
self.alive = alive
self.ext_refcnt = ext_refcnt
@@ -521,6 +521,8 @@
self.garbage = garbage
self.tuple = tuple
self.gc = gc
+ self.rooted = rooted
+ self.tracked = tracked
class WeakrefNode(BorderNode):
def __init__(self, p, pref, r, raddr, check_alive, info, r_dest,
@@ -540,6 +542,13 @@
g = pydot.graph_from_dot_file(path)[0]
nodes = {}
+ add_pyobj_after_snap = []
+ add_pypy_after_snap = []
+ add_border_after_snap = []
+ add_linked_pyobj_after_snap = []
+ add_after_snap = []
+ remove_after_snap = []
+
# create objects from graph (always create old to prevent moving)
finalizers = False
i = 0
@@ -559,27 +568,51 @@
garbage = True if 'garbage' in attr else False
tuple = attr['tuple'] == "y" if 'tuple' in attr else False
gc = attr['gc'] == "y" if 'gc' in attr else True
+ added = attr['added'] if 'added' in attr else None
info = NodeInfo(type, alive, ext_refcnt, finalizer, resurrect,
- delete, garbage, tuple, gc)
+ delete, garbage, tuple, gc, rooted, tracked)
if type == "C":
- r, raddr, check_alive = self._rawrefcount_pyobj(
- tracked=tracked, tuple=tuple)
- r.c_ob_refcnt += ext_refcnt
- nodes[name] = CPythonNode(r, raddr, check_alive, info)
+ if added == "after_snap":
+ nodes[name] = CPythonNode(None, None, None, info)
+ add_pyobj_after_snap.append(nodes[name])
+ else:
+ r, raddr, check_alive = self._rawrefcount_pyobj(
+ tracked=tracked, tuple=tuple)
+ r.c_ob_refcnt += ext_refcnt
+ nodes[name] = CPythonNode(r, raddr, check_alive, info)
elif type == "P":
- p, pref, check_alive = \
- self._rawrefcount_pypyobj(42 + i, rooted=rooted,
- create_old=True)
- nodes[name] = PyPyNode(p, pref, check_alive, info)
- i += 1
+ if added == "after_snap":
+ nodes[name] = PyPyNode(None, None, None, info)
+ add_pypy_after_snap.append(nodes[name])
+ else:
+ p, pref, check_alive = \
+ self._rawrefcount_pypyobj(42 + i, rooted=rooted,
+ create_old=True)
+ nodes[name] = PyPyNode(p, pref, check_alive, info)
+ i += 1
elif type == "B": # TODO: add to correct list (now always p_list)
- p, pref, r, raddr, check_alive =\
- self._rawrefcount_pair(42 + i, rooted=rooted,
- create_old=True, tracked=tracked,
- tuple=tuple, is_gc=gc)
- r.c_ob_refcnt += ext_refcnt
- nodes[name] = BorderNode(p, pref, r, raddr, check_alive, info)
- i += 1
+ if added == "after_snap":
+ nodes[name] = BorderNode(None, None, None, None, None,
+ info)
+ add_border_after_snap.append(nodes[name])
+ elif added == "linked_after_snap":
+ p, pref, check_alive = \
+ self._rawrefcount_pypyobj(42 + i, rooted=rooted,
+ create_old=True)
+ nodes[name] = BorderNode(p, pref, None, None, check_alive,
+ info)
+ add_linked_pyobj_after_snap.append(nodes[name])
+ i += 1
+ else:
+ p, pref, r, raddr, check_alive =\
+ self._rawrefcount_pair(42 + i, rooted=rooted,
+ create_old=True,
+ tracked=tracked, tuple=tuple,
+ is_gc=gc)
+ r.c_ob_refcnt += ext_refcnt
+ nodes[name] = BorderNode(p, pref, r, raddr, check_alive,
+ info)
+ i += 1
# add references between objects from graph
for e in g.get_edges():
@@ -588,6 +621,8 @@
attr = e.obj_dict['attributes']
weakref = attr['weakref'] == "y" if 'weakref' in attr else False
callback = attr['callback'] == "y" if 'callback' in attr else False
+ added = attr['added'] if 'added' in attr else None
+ removed = attr['removed'] if 'removed' in attr else None
clear_callback = attr['clear_callback'] == "y" \
if 'clear_callback' in attr else False
if source.info.type == "C" or dest.info.type == "C":
@@ -602,21 +637,37 @@
self._rawrefcount_addweakref(source.r, weakref)
i += 1
else:
- self._rawrefcount_addref(source.r, dest.r)
- if source.info.alive:
- dest.info.ext_refcnt += 1
+ if added == "after_snap":
+ add_after_snap.append(('C', source, dest))
+ else:
+ self._rawrefcount_addref(source.r, dest.r)
+ if source.info.alive:
+ dest.info.ext_refcnt += 1
+ if removed == "after_snap":
+ remove_after_snap.append(('C', source, dest))
elif source.info.type == "P" or dest.info.type == "P":
- if llmemory.cast_ptr_to_adr(source.p.next) == llmemory.NULL:
- source.p.next = dest.p
+ if (source.p is None or llmemory.cast_ptr_to_adr(source.p.next)
+ == llmemory.NULL):
+ if added == "after_snap":
+ add_after_snap.append(('P', 'next', source, dest))
+ else:
+ source.p.next = dest.p
+ if removed == "after_snap":
+ remove_after_snap.append(('P', 'next', source))
elif llmemory.cast_ptr_to_adr(source.p.prev) == llmemory.NULL:
- source.p.prev = dest.p
+ if added == "after_snap":
+ add_after_snap.append(('P', 'prev', source, dest))
+ else:
+ source.p.prev = dest.p
+ if removed == "after_snap":
+ remove_after_snap.append(('P', 'prev', source))
else:
assert False # only 2 refs supported from pypy obj in tests
# add finalizers
for name in nodes:
n = nodes[name]
- if hasattr(n, "r"):
+ if hasattr(n, "r") and n.r is not None:
index = self.pyobjs.index(n.r)
resurrect = n.info.resurrect
delete = n.info.delete
@@ -651,12 +702,15 @@
for wr in wrs:
dests_by_source[source].append(wr.r)
else:
- dests_by_source[source].append(dest.r)
+ if attr['added'] != "after_snap" if "added" in attr else \
+ True:
+ dests_by_source[source].append(dest.r)
for source in dests_by_source:
dests_target = dests_by_source[source]
def append(pyobj, ignore):
dests_target.remove(pyobj)
- self.gc.rrc_gc.tp_traverse(source.r, append, None)
+ if source.r is not None:
+ self.gc.rrc_gc.tp_traverse(source.r, append, None)
assert len(dests_target) == 0
garbage_pypy = []
@@ -756,7 +810,86 @@
# do a collection to find cyclic isolates and clean them, if there are
# no finalizers
- self.gc.collect()
+ if True:
+ from rpython.rlib import rgc
+ state = -1
+ after_snap = False
+ while state <> 0:
+ states = self.gc.collect_step()
+ state = rgc.new_state(states)
+ if (self.gc.rrc_gc.state == RawRefCountBaseGC.STATE_MARKING and
+ not after_snap):
+ for obj in add_pyobj_after_snap:
+ r, raddr, check_alive = self._rawrefcount_pyobj(
+ tracked=obj.info.tracked, tuple=obj.info.tuple)
+ r.c_ob_refcnt += obj.info.ext_refcnt
+ obj.r = r
+ obj.raddr = raddr
+ obj.check_alive = check_alive
+ for obj in add_pypy_after_snap:
+ p, pref, check_alive = \
+ self._rawrefcount_pypyobj(42 + i, rooted=obj.info
+ .rooted, create_old=True)
+ obj.p = p
+ obj.pref = pref
+ obj.check_alive = check_alive
+ i += 1
+ for obj in add_border_after_snap:
+ p, pref, r, raddr, check_alive = \
+ self._rawrefcount_pair(42 + i, rooted=obj.info
+ .rooted, create_old=True,
+ tracked=obj.info.tracked,
+ tuple=obj.info.tuple,
+ is_gc=obj.info.gc)
+ r.c_ob_refcnt += obj.info.ext_refcnt
+ obj.r = r
+ obj.raddr = raddr
+ obj.p = p
+ obj.pref = pref
+ obj.check_alive = check_alive
+ i += 1
+ for obj in add_linked_pyobj_after_snap:
+ r, raddr, check_alive = self._rawrefcount_pyobj(
+ tracked=obj.info.tracked, tuple=obj.info.tuple)
+ r.c_ob_refcnt += obj.info.ext_refcnt
+ obj.r = r
+ obj.raddr = raddr
+ def double_check():
+ obj.check_alive()
+ check_alive()
+ obj.check_alive = double_check
+ self.gc.rawrefcount_create_link_pypy(obj.pref, raddr)
+
+ for add in add_after_snap:
+ if add[0] == "C":
+ (type, source, dest) = add
+ self._rawrefcount_addref(source.r, dest.r)
+ if source.info.alive:
+ dest.info.ext_refcnt += 1
+ elif add[0] == "P":
+ (type, prop, source, dest) = add
+ if prop == "next":
+ source.p.next = dest.p
+ elif prop == "prev":
+ source.p.prev = dest.p
+ else:
+ assert False, "not yet supported"
+ else:
+ assert False, "not yet supported"
+ for remove in remove_after_snap:
+ if remove[0] == "P":
+ if remove[1] == "next":
+ remove[2].p.next = remove[2].p
+ elif prop == "prev":
+ remove[2].p.prev = remove[2].p
+ else:
+ assert False, "not yet supported"
+ else:
+ assert False, "not yet supported"
+ after_snap = True
+ else:
+ self.gc.collect()
+
self.gc.rrc_gc.invoke_callback()
if self.trigger <> []:
cleanup()
More information about the pypy-commit
mailing list