[pypy-commit] pypy cpyext-gc-cycle: Added interface for rawrefcount finalizers to the gc
stevie_92
pypy.commits at gmail.com
Wed Feb 13 09:17:35 EST 2019
Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r95997:a316d3f47e9f
Date: 2019-02-13 15:16 +0100
http://bitbucket.org/pypy/pypy/changeset/a316d3f47e9f/
Log: Added interface for rawrefcount finalizers to the gc Added support
for rawrefcount finalizers to dot-tests Potentially dead cross-heap
cycles are kept alive until the following collection cycle (still
missing optimization for cycles without finalizers)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1187,6 +1187,9 @@
state.C._PyPy_pyobj_as_gc = rffi.llexternal(
'_PyPy_pyobj_as_gc', [GCHdr_PyObject], PyGC_HeadPtr,
compilation_info=eci, _nowrapper=True)
+ state.C._PyPy_finalizer_type = rffi.llexternal(
+ '_PyPy_finalizer_type', [PyGC_HeadPtr], lltype.Signed,
+ compilation_info=eci, _nowrapper=True)
def init_function(func):
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -438,6 +438,7 @@
PyAPI_FUNC(PyGC_Head *) _PyPy_init_pyobj_list();
PyAPI_FUNC(GCHdr_PyObject *) _PyPy_gc_as_pyobj(PyGC_Head *);
PyAPI_FUNC(PyGC_Head *) _PyPy_pyobj_as_gc(GCHdr_PyObject *);
+PyAPI_FUNC(Py_ssize_t) _PyPy_finalizer_type(PyGC_Head *);
#ifdef __cplusplus
}
diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c
--- a/pypy/module/cpyext/src/object.c
+++ b/pypy/module/cpyext/src/object.c
@@ -61,6 +61,12 @@
}
}
+Py_ssize_t
+_PyPy_finalizer_type(PyGC_Head *g)
+{
+ return 0;
+}
+
void
_Py_Dealloc(PyObject *obj)
{
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -174,7 +174,8 @@
llhelper(rawrefcount.RAWREFCOUNT_TRAVERSE,
self.tp_traverse),
pypyobj_list,
- self.C._PyPy_gc_as_pyobj, self.C._PyPy_pyobj_as_gc)
+ self.C._PyPy_gc_as_pyobj, self.C._PyPy_pyobj_as_gc,
+ self.C._PyPy_finalizer_type)
self.builder.attach_all(space)
setup_new_method_def(space)
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3011,6 +3011,8 @@
PYOBJ_HDR_PTR))
RAWREFCOUNT_PYOBJ_AS_GC = lltype.Ptr(lltype.FuncType([PYOBJ_HDR_PTR],
PYOBJ_GC_HDR_PTR))
+ RAWREFCOUNT_FINALIZER_TYPE = lltype.Ptr(lltype.FuncType([PYOBJ_GC_HDR_PTR],
+ lltype.Signed))
def _pyobj(self, pyobjaddr):
return llmemory.cast_adr_to_ptr(pyobjaddr, self.PYOBJ_HDR_PTR)
@@ -3018,7 +3020,7 @@
return llmemory.cast_adr_to_ptr(pygchdraddr, self.PYOBJ_GC_HDR_PTR)
def rawrefcount_init(self, dealloc_trigger_callback, tp_traverse,
- pyobj_list, gc_as_pyobj, pyobj_as_gc):
+ pyobj_list, gc_as_pyobj, pyobj_as_gc, finalizer_type):
# see pypy/doc/discussion/rawrefcount.rst
if not self.rrc_enabled:
self.rrc_p_list_young = self.AddressStack()
@@ -3041,6 +3043,7 @@
self.rrc_pyobj_garbage_list.c_gc_prev = self.rrc_pyobj_garbage_list
self.rrc_gc_as_pyobj = gc_as_pyobj
self.rrc_pyobj_as_gc = pyobj_as_gc
+ self.rrc_finalizer_type = finalizer_type
self.rrc_enabled = True
def check_no_more_rawrefcount_state(self):
@@ -3268,13 +3271,19 @@
# TODO: pypy objects
def _rrc_major_trace(self, pyobject, ignore):
- pygchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject))
- if pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR):
- rc = pygchdr.c_gc_refs
- else:
- rc = self._pyobj(pyobject).c_ob_refcnt
+ from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
+ from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
+ #
+ # TODO: optimization: if no finalizers are found the cyclic rc
+ # TODO: can be used instead of the real rc, because the objects
+ # TODO: cannot be resurrected anyway
+ # pygchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject))
+ # if pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR):
+ # rc = pygchdr.c_gc_refs
+ # else:
+ rc = self._pyobj(pyobject).c_ob_refcnt
- if rc == 0:
+ if rc == REFCNT_FROM_PYPY or rc == REFCNT_FROM_PYPY_LIGHT: # or rc == 0
pass # the corresponding object may die
else:
# force the corresponding object to be alive
diff --git a/rpython/memory/gc/test/dot/free_finalizer_nocycle.dot b/rpython/memory/gc/test/dot/free_finalizer_nocycle.dot
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/free_finalizer_nocycle.dot
@@ -0,0 +1,13 @@
+digraph G {
+ "a" [type=P, alive=n];
+ "b" [type=B, alive=n];
+ "c" [type=C, alive=n, finalizer=modern];
+ "d" [type=C, alive=n];
+ "e" [type=B, alive=n];
+ "f" [type=P, alive=n];
+ "a" -> "b";
+ "b" -> "c";
+ "c" -> "d";
+ "d" -> "e";
+ "e" -> "f";
+}
diff --git a/rpython/memory/gc/test/dot/free_finalizer_simple.dot b/rpython/memory/gc/test/dot/free_finalizer_simple.dot
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/free_finalizer_simple.dot
@@ -0,0 +1,14 @@
+digraph G {
+ "a" [type=P, alive=n];
+ "b" [type=B, alive=n];
+ "c" [type=C, alive=n, finalizer=modern];
+ "d" [type=C, alive=n];
+ "e" [type=B, alive=n];
+ "f" [type=P, alive=n];
+ "a" -> "b";
+ "b" -> "c";
+ "c" -> "d";
+ "d" -> "e";
+ "e" -> "f";
+ "f" -> "a";
+}
diff --git a/rpython/memory/gc/test/dot/keep_finalizer_simple.dot b/rpython/memory/gc/test/dot/keep_finalizer_simple.dot
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/keep_finalizer_simple.dot
@@ -0,0 +1,14 @@
+digraph G {
+ "a" [type=P, alive=y];
+ "b" [type=B, alive=y];
+ "c" [type=C, alive=y, finalizer=modern, resurrect=d];
+ "d" [type=C, alive=y];
+ "e" [type=B, alive=y];
+ "f" [type=P, alive=y];
+ "a" -> "b";
+ "b" -> "c";
+ "c" -> "d";
+ "d" -> "e";
+ "e" -> "f";
+ "f" -> "a";
+}
diff --git a/rpython/memory/gc/test/dot/partial_free_finalizer_nocycle.dot b/rpython/memory/gc/test/dot/partial_free_finalizer_nocycle.dot
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/partial_free_finalizer_nocycle.dot
@@ -0,0 +1,13 @@
+digraph G {
+ "a" [type=P, alive=n];
+ "b" [type=B, alive=n];
+ "c" [type=C, alive=n, finalizer=modern, resurrect=d];
+ "d" [type=C, alive=y];
+ "e" [type=B, alive=y];
+ "f" [type=P, alive=y];
+ "a" -> "b";
+ "b" -> "c";
+ "c" -> "d";
+ "d" -> "e";
+ "e" -> "f";
+}
diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -39,6 +39,9 @@
def rawrefcount_pyobj_as_gc(pyobj):
return self.gcobjs[self.pyobjs.index(pyobj)]
+ def rawrefcount_finalizer_type(gc):
+ return 0
+
self.pyobj_list = lltype.malloc(PYOBJ_GC_HDR_PTR.TO, flavor='raw',
immortal=True)
self.pyobj_list.c_gc_next = self.pyobj_list
@@ -47,7 +50,8 @@
rawrefcount_tp_traverse,
llmemory.cast_ptr_to_adr(self.pyobj_list),
rawrefcount_gc_as_pyobj,
- rawrefcount_pyobj_as_gc)
+ rawrefcount_pyobj_as_gc,
+ rawrefcount_finalizer_type)
def _collect(self, major, expected_trigger=0):
if major:
@@ -392,6 +396,8 @@
@py.test.mark.parametrize("file", dot_files)
def test_dots(self, file):
from rpython.memory.gc.test.dot import pydot
+ from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
+ from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
class Node:
def __init__(self, info):
@@ -421,10 +427,12 @@
self.info = info
class NodeInfo:
- def __init__(self, type, alive, ext_refcnt):
+ def __init__(self, type, alive, ext_refcnt, finalizer, resurrect):
self.type = type
self.alive = alive
self.ext_refcnt = ext_refcnt
+ self.finalizer = finalizer
+ self.resurrect = resurrect
path = os.path.join(self.dot_dir, file)
g = pydot.graph_from_dot_file(path)[0]
@@ -439,7 +447,9 @@
alive = attr['alive'] == "y"
rooted = attr['rooted'] == "y" if 'rooted' in attr else False
ext_refcnt = int(attr['ext_refcnt']) if 'ext_refcnt' in attr else 0
- info = NodeInfo(type, alive, ext_refcnt)
+ finalizer = attr['finalizer'] if 'finalizer' in attr else None
+ resurrect = attr['resurrect'] if 'resurrect' in attr else None
+ info = NodeInfo(type, alive, ext_refcnt, finalizer, resurrect)
if type == "C":
r, raddr, check_alive = self._rawrefcount_pyobj()
r.c_ob_refcnt += ext_refcnt
@@ -490,11 +500,7 @@
self.gc.rrc_tp_traverse(source.r, append, None)
assert len(dests_target) == 0
- # do collection
- self.gc.collect()
-
- self.gc.rrc_invoke_callback()
- if self.trigger <> []:
+ def cleanup():
# do cleanup after collection (clear all dead pyobjects)
def decref_children(pyobj):
self.gc.rrc_tp_traverse(pyobj, decref, None)
@@ -513,19 +519,26 @@
next_dead = self.gc.rawrefcount_next_dead()
while next_dead <> llmemory.NULL:
- pyobj = llmemory.cast_adr_to_ptr(next_dead, self.gc.PYOBJ_HDR_PTR)
+ pyobj = llmemory.cast_adr_to_ptr(next_dead,
+ self.gc.PYOBJ_HDR_PTR)
+ print "nextdead:", pyobj, "refcnt:", pyobj.c_ob_refcnt
decref(pyobj, None)
next_dead = self.gc.rawrefcount_next_dead()
+ # TODO: call finalizers here and during the next collection it
+ # will be checked if the CI is really trash
+
next_dead = self.gc.rawrefcount_cyclic_garbage_head()
while next_dead <> llmemory.NULL:
- pyobj = llmemory.cast_adr_to_ptr(next_dead, self.gc.PYOBJ_HDR_PTR)
+ pyobj = llmemory.cast_adr_to_ptr(next_dead,
+ self.gc.PYOBJ_HDR_PTR)
pyobj.c_ob_refcnt += 1
def clear(pyobj_to, pyobj_from):
refs = self.pyobj_refs[self.pyobjs.index(pyobj_from)]
refs.remove(pyobj_to)
decref(pyobj_to, None)
+
self.gc.rrc_tp_traverse(pyobj, clear, pyobj)
decref(pyobj, None)
@@ -537,6 +550,21 @@
self.gc.rawrefcount_cyclic_garbage_remove()
next_dead = self.gc.rawrefcount_cyclic_garbage_head()
+ # do a collection to find cyclic isolates
+ self.gc.collect()
+
+ self.gc.rrc_invoke_callback()
+ if self.trigger <> []:
+ cleanup()
+
+ # now do another collection, to clean up cyclic trash
+ # TODO: maybe optimize, so that we don't need another major collection
+ self.gc.collect()
+
+ self.gc.rrc_invoke_callback()
+ if self.trigger <> []:
+ cleanup()
+
# check livelihood of objects, according to graph
for name in nodes:
n = nodes[name]
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -478,7 +478,8 @@
[s_gc, SomePtr(GCClass.RAWREFCOUNT_DEALLOC_TRIGGER),
SomePtr(GCClass.RAWREFCOUNT_TRAVERSE), SomeAddress(),
SomePtr(GCClass.RAWREFCOUNT_GC_AS_PYOBJ),
- SomePtr(GCClass.RAWREFCOUNT_PYOBJ_AS_GC)],
+ SomePtr(GCClass.RAWREFCOUNT_PYOBJ_AS_GC),
+ SomePtr(GCClass.RAWREFCOUNT_FINALIZER_TYPE)],
annmodel.s_None)
self.rawrefcount_create_link_pypy_ptr = getfn(
GCClass.rawrefcount_create_link_pypy,
@@ -1319,7 +1320,8 @@
self.pop_roots(hop, livevars)
def gct_gc_rawrefcount_init(self, hop):
- [v_fnptr, v_fnptr2, v_pyobj_list, v_fnptr3, v_fnptr4] = hop.spaceop.args
+ [v_fnptr, v_fnptr2, v_pyobj_list, v_fnptr3, v_fnptr4,
+ v_fnptr5] = hop.spaceop.args
assert v_fnptr.concretetype == self.GCClass.RAWREFCOUNT_DEALLOC_TRIGGER
assert v_fnptr2.concretetype == self.GCClass.RAWREFCOUNT_TRAVERSE
# TODO add assert for v_pyobj_list, improve asserts (types not same but equal)
@@ -1327,7 +1329,7 @@
# assert v_fnptr4.concretetype == self.GCClass.RAWREFCOUNT_PYOBJ_AS_GC
hop.genop("direct_call",
[self.rawrefcount_init_ptr, self.c_const_gc, v_fnptr,
- v_fnptr2, v_pyobj_list, v_fnptr3, v_fnptr4])
+ v_fnptr2, v_pyobj_list, v_fnptr3, v_fnptr4, v_fnptr5])
def gct_gc_rawrefcount_create_link_pypy(self, hop):
[v_gcobj, v_pyobject] = hop.spaceop.args
diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py
--- a/rpython/rlib/rawrefcount.py
+++ b/rpython/rlib/rawrefcount.py
@@ -286,19 +286,22 @@
_about_ = init
def compute_result_annotation(self, s_dealloc_callback, s_tp_traverse,
- s_pyobj_list, s_as_gc, s_as_pyobj):
+ s_pyobj_list, s_as_gc, s_as_pyobj,
+ a_finalizer_type):
from rpython.rtyper.llannotation import SomePtr
assert isinstance(s_dealloc_callback, SomePtr) # ll-ptr-to-function
assert isinstance(s_tp_traverse, SomePtr)
assert isinstance(s_as_gc, SomePtr)
assert isinstance(s_as_pyobj, SomePtr)
+ assert isinstance(a_finalizer_type, SomePtr)
def specialize_call(self, hop):
hop.exception_cannot_occur()
v_dealloc_callback, v_tp_traverse, v_pyobj_list, v_as_gc, \
- v_as_pyobj = hop.inputargs(*hop.args_r)
+ v_as_pyobj, v_finalizer_type = hop.inputargs(*hop.args_r)
hop.genop('gc_rawrefcount_init', [v_dealloc_callback, v_tp_traverse,
- v_pyobj_list, v_as_gc, v_as_pyobj])
+ v_pyobj_list, v_as_gc, v_as_pyobj,
+ v_finalizer_type])
class Entry(ExtRegistryEntry):
More information about the pypy-commit
mailing list