[pypy-commit] pypy cpyext-gc-cycle: Added support for rawrefcount finalizers in incminimark
stevie_92
pypy.commits at gmail.com
Sat Feb 16 08:35:47 EST 2019
Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r96024:09b2440acb51
Date: 2019-02-16 14:35 +0100
http://bitbucket.org/pypy/pypy/changeset/09b2440acb51/
Log: Added support for rawrefcount finalizers in incminimark Added call
to tp_finalize in cpyext, if the gc found an unreachable object
(still needs some testing)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1190,7 +1190,10 @@
state.C._PyPy_finalizer_type = rffi.llexternal(
'_PyPy_finalizer_type', [PyGC_HeadPtr], lltype.Signed,
compilation_info=eci, _nowrapper=True)
-
+ state.C._Py_Finalize = rffi.llexternal('_Py_Finalize',
+ [PyObject], lltype.Void,
+ compilation_info=eci,
+ _nowrapper=True)
def init_function(func):
INIT_FUNCTIONS.append(func)
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -51,6 +51,7 @@
PyAPI_FUNC(void) Py_DecRef(PyObject *);
extern Py_ssize_t _pypy_rawrefcount_w_marker_deallocating;
PyAPI_FUNC(void) _Py_Dealloc(PyObject *);
+PyAPI_FUNC(void) _Py_Finalize(PyObject *);
#define Py_CLEAR(op) \
do { \
@@ -244,6 +245,8 @@
#define Py_TPFLAGS_DEFAULT Py_TPFLAGS_DEFAULT_EXTERNAL
+#define Py_TPFLAGS_HAVE_FINALIZE (1UL << 0)
+
#define PyType_HasFeature(t,f) (((t)->tp_flags & (f)) != 0)
#define PyType_FastSubclass(t,f) PyType_HasFeature(t,f)
@@ -317,7 +320,7 @@
#define _PyGCHead_FINALIZED(g) (((g)->gc_refs & _PyGC_REFS_MASK_FINALIZED) != 0)
#define _PyGCHead_SET_FINALIZED(g, v) do { \
- (g)->gc_refs = ((g)->gc_refs & ~_PyGC_REFS_MASK_FINALIZED) \
+ (g)->gc_refs = ((g)->gc_refs & ~_PyGC_REFS_MASK_FINALIZED) \
| (v != 0); \
} while (0)
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -311,6 +311,8 @@
/* Type attribute cache version tag. Added in version 2.6 */
unsigned int tp_version_tag;
+ destructor tp_finalize;
+
} PyTypeObject;
typedef struct _heaptypeobject {
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -406,6 +406,13 @@
# if w_obj is not None:
# assert pyobj.c_ob_refcnt >= rawrefcount.REFCNT_FROM_PYPY
+ at specialize.ll()
+def finalize(space, pyobj):
+ from pypy.module.cpyext.api import generic_cpy_call
+ assert is_pyobj(pyobj)
+ pyobj = rffi.cast(PyObject, pyobj)
+ state = space.fromcache(State)
+ generic_cpy_call(space, state.C._Py_Finalize, pyobj)
@init_function
def write_w_marker_deallocating(space):
diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c
--- a/pypy/module/cpyext/src/object.c
+++ b/pypy/module/cpyext/src/object.c
@@ -62,9 +62,17 @@
}
Py_ssize_t
-_PyPy_finalizer_type(PyGC_Head *g)
+_PyPy_finalizer_type(PyGC_Head *gc)
{
- return 0;
+ PyObject *op = FROM_GC(gc);
+ if (!_PyGCHead_FINALIZED(gc) &&
+ PyType_HasFeature(Py_TYPE(op), Py_TPFLAGS_HAVE_FINALIZE) &&
+ Py_TYPE(op)->tp_finalize != NULL) {
+ return 1;
+ } else {
+ return 0;
+ }
+ // TODO: legacy finalizer (tp_del)
}
void
@@ -77,6 +85,23 @@
}
void
+_Py_Finalize(PyObject *op)
+{
+ PyGC_Head *gc = _Py_AS_GC(op);
+ destructor finalize;
+
+ if (!_PyGCHead_FINALIZED(gc) &&
+ PyType_HasFeature(Py_TYPE(op), Py_TPFLAGS_HAVE_FINALIZE) &&
+ (finalize = Py_TYPE(op)->tp_finalize) != NULL) {
+ _PyGCHead_SET_FINALIZED(gc, 0);
+ Py_INCREF(op);
+ finalize(op);
+ assert(!PyErr_Occurred());
+ Py_DECREF(op);
+ }
+}
+
+void
_PyPy_object_dealloc(PyObject *obj)
{
PyTypeObject *pto;
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -61,7 +61,7 @@
if not self.space.config.translating:
def dealloc_trigger():
from pypy.module.cpyext.pyobject import PyObject, decref, \
- incref, cts
+ incref, cts, finalize
print 'dealloc_trigger...'
while True:
ob = rawrefcount.next_dead(PyObject)
@@ -72,6 +72,11 @@
print 'deallocating PyObject', ob, 'of type', name
decref(space, ob)
while True:
+ py_obj = rawrefcount.next_cyclic_isolate(PyObject)
+ if not py_obj:
+ break
+ finalize(space, py_obj)
+ while True:
ob = rawrefcount.cyclic_garbage_head(PyObject)
if not ob:
break
@@ -252,7 +257,7 @@
def _rawrefcount_perform(space):
- from pypy.module.cpyext.pyobject import PyObject, incref, decref
+ from pypy.module.cpyext.pyobject import PyObject, incref, decref, finalize
while True:
py_obj = rawrefcount.next_dead(PyObject)
if not py_obj:
@@ -260,6 +265,12 @@
decref(space, py_obj)
while True:
+ py_obj = rawrefcount.next_cyclic_isolate(PyObject)
+ if not py_obj:
+ break
+ finalize(space, py_obj)
+
+ while True:
py_obj = rawrefcount.cyclic_garbage_head(PyObject)
if not py_obj:
break
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -1046,6 +1046,8 @@
Test if a simple collect is working
TODO: make more precise
"""
+ skip('does not work right now, because of how the test is set up, '
+ 'see comment below')
if self.runappdirect:
skip('cannot import module with undefined functions')
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3013,6 +3013,11 @@
PYOBJ_GC_HDR_PTR))
RAWREFCOUNT_FINALIZER_TYPE = lltype.Ptr(lltype.FuncType([PYOBJ_GC_HDR_PTR],
lltype.Signed))
+ RAWREFCOUNT_FINALIZER_NONE = 0
+ RAWREFCOUNT_FINALIZER_MODERN = 1
+ RAWREFCOUNT_FINALIZER_LEGACY = 2
+ RAWREFCOUNT_REFS_SHIFT = 1
+ RAWREFCOUNT_REFS_MASK_FINALIZED = 1
def _pyobj(self, pyobjaddr):
return llmemory.cast_adr_to_ptr(pyobjaddr, self.PYOBJ_HDR_PTR)
@@ -3037,6 +3042,10 @@
lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw', immortal=True)
self.rrc_pyobj_old_list.c_gc_next = self.rrc_pyobj_old_list
self.rrc_pyobj_old_list.c_gc_prev = self.rrc_pyobj_old_list
+ self.rrc_pyobj_isolate_list = \
+ lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw', immortal=True)
+ self.rrc_pyobj_isolate_list.c_gc_next = self.rrc_pyobj_isolate_list
+ self.rrc_pyobj_isolate_list.c_gc_prev = self.rrc_pyobj_isolate_list
self.rrc_pyobj_garbage_list = \
lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw', immortal=True)
self.rrc_pyobj_garbage_list.c_gc_next = self.rrc_pyobj_garbage_list
@@ -3107,9 +3116,15 @@
return self.rrc_dealloc_pending.pop()
return llmemory.NULL
+ def rawrefcount_next_cyclic_isolate(self):
+ if not self._rrc_gc_list_is_empty(self.rrc_pyobj_isolate_list):
+ gchdr = self._rrc_gc_list_pop(self.rrc_pyobj_isolate_list)
+ self._rrc_gc_list_add(self.rrc_pyobj_old_list, gchdr)
+ return llmemory.cast_ptr_to_adr(self.rrc_gc_as_pyobj(gchdr))
+ return llmemory.NULL
+
def rawrefcount_cyclic_garbage_head(self):
- if self.rrc_pyobj_garbage_list.c_gc_next <> \
- self.rrc_pyobj_garbage_list:
+ if not self._rrc_gc_list_is_empty(self.rrc_pyobj_garbage_list):
return llmemory.cast_ptr_to_adr(
self.rrc_gc_as_pyobj(self.rrc_pyobj_garbage_list.c_gc_next))
else:
@@ -3130,6 +3145,8 @@
def rrc_invoke_callback(self):
if self.rrc_enabled and (self.rrc_dealloc_pending.non_empty() or
+ self.rrc_pyobj_isolate_list.c_gc_next <>
+ self.rrc_pyobj_isolate_list or
self.rrc_pyobj_garbage_list.c_gc_next <>
self.rrc_pyobj_garbage_list):
self.rrc_dealloc_trigger_callback()
@@ -3241,10 +3258,41 @@
_rrc_free._always_inline_ = True
def rrc_major_collection_trace(self):
- self._rrc_collect_rawrefcount_roots()
+ if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list):
+ # Check, if the cyclic isolate from the last collection cycle
+ # is reachable from outside, after the finalizers have been
+ # executed.
+ self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_old_list)
+ found_alive = False
+ gchdr = self.rrc_pyobj_old_list.c_gc_next
+ while gchdr <> self.rrc_pyobj_old_list:
+ if (gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT) > 0:
+ found_alive = True
+ break
+ gchdr = gchdr.c_gc_next
+ if found_alive:
+ self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
+ self.rrc_pyobj_list)
+ else:
+ self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
+ self.rrc_pyobj_garbage_list)
+
+ self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_list)
self._rrc_mark_rawrefcount()
- self.rrc_p_list_old.foreach(self._rrc_major_trace, None)
- self.rrc_o_list_old.foreach(self._rrc_major_trace, None)
+
+ found_finalizer = False
+ gchdr = self.rrc_pyobj_old_list.c_gc_next
+ while gchdr <> self.rrc_pyobj_old_list:
+ if self.rrc_finalizer_type(gchdr) == \
+ self.RAWREFCOUNT_FINALIZER_MODERN:
+ found_finalizer = True
+ gchdr = gchdr.c_gc_next
+ if found_finalizer:
+ self._rrc_gc_list_move(self.rrc_pyobj_old_list,
+ self.rrc_pyobj_isolate_list)
+
+ self.rrc_p_list_old.foreach(self._rrc_major_trace, found_finalizer)
+ self.rrc_o_list_old.foreach(self._rrc_major_trace, found_finalizer)
# TODO: for all unreachable objects, which are marked potentially
# TODO: uncollectable, move them to the set of uncollectable objs
@@ -3260,30 +3308,20 @@
# TODO: a list of callbacks, which has to be called after the
# TODO: the GC runs
- # TODO: call tp_finalize for unreachable objects
- # TODO: (could resurrect objects, so we have to do it now)
- # TODO: (set finalizer flag before calling and check if
- # TODO: finalizer was not called before)
-
- # TODO: for all objects in unreachable, check if they
- # TODO: are still unreachable. if not, abort and move all
- # TODO: unreachable back to pyobj_list and mark all reachable
- # TODO: pypy objects
-
- def _rrc_major_trace(self, pyobject, ignore):
+ def _rrc_major_trace(self, pyobject, found_finalizer):
from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
#
- # TODO: optimization: if no finalizers are found the cyclic rc
- # TODO: can be used instead of the real rc, because the objects
- # TODO: cannot be resurrected anyway
- # pygchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject))
- # if pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR):
- # rc = pygchdr.c_gc_refs
- # else:
- rc = self._pyobj(pyobject).c_ob_refcnt
-
- if rc == REFCNT_FROM_PYPY or rc == REFCNT_FROM_PYPY_LIGHT: # or rc == 0
+ if not found_finalizer:
+ pygchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject))
+ if pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR):
+ rc = pygchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT
+ else:
+ rc = self._pyobj(pyobject).c_ob_refcnt
+ else:
+ rc = self._pyobj(pyobject).c_ob_refcnt
+
+ if rc == REFCNT_FROM_PYPY or rc == REFCNT_FROM_PYPY_LIGHT or rc == 0:
pass # the corresponding object may die
else:
# force the corresponding object to be alive
@@ -3312,17 +3350,9 @@
self.rrc_o_list_old.delete()
self.rrc_o_list_old = new_o_list
- # merge old_list into garbage_list and clear old_list
- if self.rrc_pyobj_old_list.c_gc_next <> self.rrc_pyobj_old_list:
- next = self.rrc_pyobj_garbage_list.c_gc_next
- next_old = self.rrc_pyobj_old_list.c_gc_next
- prev_old = self.rrc_pyobj_old_list.c_gc_prev
- self.rrc_pyobj_garbage_list.c_gc_next = next_old
- next_old.c_gc_prev = self.rrc_pyobj_garbage_list
- prev_old.c_gc_next = next
- next.c_gc_prev = prev_old
- self.rrc_pyobj_old_list.c_gc_next = self.rrc_pyobj_old_list
- self.rrc_pyobj_old_list.c_gc_prev = self.rrc_pyobj_old_list
+ if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list):
+ self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
+ self.rrc_pyobj_garbage_list)
def _rrc_major_free(self, pyobject, surviving_list, surviving_dict):
# The pyobject survives if the corresponding obj survives.
@@ -3338,18 +3368,21 @@
else:
self._rrc_free(pyobject, True)
- def _rrc_collect_rawrefcount_roots(self):
+ def _rrc_collect_rawrefcount_roots(self, pygclist):
from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
#
# Initialize the cyclic refcount with the real refcount.
- pygchdr = self.rrc_pyobj_list.c_gc_next
- while pygchdr <> self.rrc_pyobj_list:
- pygchdr.c_gc_refs = self.rrc_gc_as_pyobj(pygchdr).c_ob_refcnt
- if pygchdr.c_gc_refs >= REFCNT_FROM_PYPY_LIGHT:
- pygchdr.c_gc_refs -= REFCNT_FROM_PYPY_LIGHT
- elif pygchdr.c_gc_refs >= REFCNT_FROM_PYPY:
- pygchdr.c_gc_refs -= REFCNT_FROM_PYPY
+ pygchdr = pygclist.c_gc_next
+ while pygchdr <> pygclist:
+ refcnt = self.rrc_gc_as_pyobj(pygchdr).c_ob_refcnt
+ if refcnt >= REFCNT_FROM_PYPY_LIGHT:
+ refcnt -= REFCNT_FROM_PYPY_LIGHT
+ elif refcnt >= REFCNT_FROM_PYPY:
+ refcnt -= REFCNT_FROM_PYPY
+ pygchdr.c_gc_refs = pygchdr.c_gc_refs & \
+ self.RAWREFCOUNT_REFS_MASK_FINALIZED
+ pygchdr.c_gc_refs = refcnt << self.RAWREFCOUNT_REFS_SHIFT
pygchdr = pygchdr.c_gc_next
# For every object in this set, if it is marked, add 1 as a real
@@ -3359,8 +3392,8 @@
# Subtract all internal refcounts from the cyclic refcount
# of rawrefcounted objects
- pygchdr = self.rrc_pyobj_list.c_gc_next
- while pygchdr <> self.rrc_pyobj_list:
+ pygchdr = pygclist.c_gc_next
+ while pygchdr <> pygclist:
pyobj = self.rrc_gc_as_pyobj(pygchdr)
self._rrc_traverse(pyobj, -1)
pygchdr = pygchdr.c_gc_next
@@ -3374,22 +3407,16 @@
gchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject))
if gchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
if self.header(obj).tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS):
- gchdr.c_gc_refs += 1
+ gchdr.c_gc_refs += 1 << self.RAWREFCOUNT_REFS_SHIFT
def _rrc_mark_rawrefcount(self):
- if self.rrc_pyobj_list.c_gc_next == self.rrc_pyobj_list:
- self.rrc_pyobj_old_list.c_gc_next = self.rrc_pyobj_old_list
- self.rrc_pyobj_old_list.c_gc_prev = self.rrc_pyobj_old_list
+ if self._rrc_gc_list_is_empty(self.rrc_pyobj_list):
+ self._rrc_gc_list_init(self.rrc_pyobj_old_list)
return
# as long as new objects with cyclic a refcount > 0 or alive border
# objects are found, increment the refcount of all referenced objects
# of those newly found objects
- self.rrc_pyobj_old_list.c_gc_next = self.rrc_pyobj_list.c_gc_next
- self.rrc_pyobj_old_list.c_gc_prev = self.rrc_pyobj_list.c_gc_prev
- self.rrc_pyobj_old_list.c_gc_next.c_gc_prev = self.rrc_pyobj_old_list
- self.rrc_pyobj_old_list.c_gc_prev.c_gc_next = self.rrc_pyobj_old_list
- self.rrc_pyobj_list.c_gc_next = self.rrc_pyobj_list
- self.rrc_pyobj_list.c_gc_prev = self.rrc_pyobj_list
+ self._rrc_gc_list_move(self.rrc_pyobj_list, self.rrc_pyobj_old_list)
found_alive = True
#
while found_alive:
@@ -3397,7 +3424,7 @@
gchdr = self.rrc_pyobj_old_list.c_gc_next
while gchdr <> self.rrc_pyobj_old_list:
next_old = gchdr.c_gc_next
- alive = gchdr.c_gc_refs > 0
+ alive = (gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT) > 0
pyobj = self.rrc_gc_as_pyobj(gchdr)
if pyobj.c_ob_pypy_link <> 0:
intobj = pyobj.c_ob_pypy_link
@@ -3405,7 +3432,7 @@
if not alive and self.header(obj).tid & (
GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS):
# add fake refcount, to mark it as live
- gchdr.c_gc_refs += 1
+ gchdr.c_gc_refs += 1 << self.RAWREFCOUNT_REFS_SHIFT
alive = True
if alive:
# remove from old list
@@ -3413,11 +3440,7 @@
next.c_gc_prev = gchdr.c_gc_prev
gchdr.c_gc_prev.c_gc_next = next
# add to new list
- next = self.rrc_pyobj_list.c_gc_next
- self.rrc_pyobj_list.c_gc_next = gchdr
- gchdr.c_gc_prev = self.rrc_pyobj_list
- gchdr.c_gc_next = next
- next.c_gc_prev = gchdr
+ self._rrc_gc_list_add(self.rrc_pyobj_list, gchdr)
# increment refcounts
self._rrc_traverse(pyobj, 1)
# mark recursively, if it is a pypyobj
@@ -3443,7 +3466,8 @@
def _rrc_visit_action(self, pyobj, ignore):
pygchdr = self.rrc_pyobj_as_gc(pyobj)
if pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
- pygchdr.c_gc_refs += self.rrc_refcnt_add
+ pygchdr.c_gc_refs += self.rrc_refcnt_add << \
+ self.RAWREFCOUNT_REFS_SHIFT
def _rrc_traverse(self, pyobj, refcnt_add):
from rpython.rlib.objectmodel import we_are_translated
@@ -3462,3 +3486,38 @@
def _rrc_gc_list_init(self, pygclist):
pygclist.c_gc_next = pygclist
pygclist.c_gc_prev = pygclist
+
+ def _rrc_gc_list_add(self, pygclist, gchdr):
+ next = pygclist.c_gc_next
+ pygclist.c_gc_next = gchdr
+ gchdr.c_gc_prev = pygclist
+ gchdr.c_gc_next = next
+ next.c_gc_prev = gchdr
+
+ def _rrc_gc_list_pop(self, pygclist):
+ ret = pygclist.c_gc_next
+ pygclist.c_gc_next = ret.c_gc_next
+ ret.c_gc_next.c_gc_prev = pygclist
+ return ret
+
+ def _rrc_gc_list_move(self, pygclist_source, pygclist_dest):
+ pygclist_dest.c_gc_next = pygclist_source.c_gc_next
+ pygclist_dest.c_gc_prev = pygclist_source.c_gc_prev
+ pygclist_dest.c_gc_next.c_gc_prev = pygclist_dest
+ pygclist_dest.c_gc_prev.c_gc_next = pygclist_dest
+ pygclist_source.c_gc_next = pygclist_source
+ pygclist_source.c_gc_prev = pygclist_source
+
+ def _rrc_gc_list_merge(self, pygclist_source, pygclist_dest):
+ next = pygclist_dest.c_gc_next
+ next_old = pygclist_source.c_gc_next
+ prev_old = pygclist_source.c_gc_prev
+ pygclist_dest.c_gc_next = next_old
+ next_old.c_gc_prev = pygclist_dest
+ prev_old.c_gc_next = next
+ next.c_gc_prev = prev_old
+ pygclist_source.c_gc_next = pygclist_source
+ pygclist_source.c_gc_prev = pygclist_source
+
+ def _rrc_gc_list_is_empty(self, pygclist):
+ return pygclist.c_gc_next == pygclist
diff --git a/rpython/memory/gc/test/dot/keep_finalizer_simple.dot b/rpython/memory/gc/test/dot/keep_finalizer_simple.dot
--- a/rpython/memory/gc/test/dot/keep_finalizer_simple.dot
+++ b/rpython/memory/gc/test/dot/keep_finalizer_simple.dot
@@ -1,7 +1,7 @@
digraph G {
"a" [type=P, alive=y];
"b" [type=B, alive=y];
- "c" [type=C, alive=y, finalizer=modern, resurrect=d];
+ "c" [type=C, alive=y, finalizer=modern, resurrect="d"];
"d" [type=C, alive=y];
"e" [type=B, alive=y];
"f" [type=P, alive=y];
diff --git a/rpython/memory/gc/test/dot/partial_free_finalizer_nocycle.dot b/rpython/memory/gc/test/dot/partial_free_finalizer_nocycle.dot
--- a/rpython/memory/gc/test/dot/partial_free_finalizer_nocycle.dot
+++ b/rpython/memory/gc/test/dot/partial_free_finalizer_nocycle.dot
@@ -1,7 +1,7 @@
digraph G {
"a" [type=P, alive=n];
"b" [type=B, alive=n];
- "c" [type=C, alive=n, finalizer=modern, resurrect=d];
+ "c" [type=C, alive=n, finalizer=modern, resurrect="d"];
"d" [type=C, alive=y];
"e" [type=B, alive=y];
"f" [type=P, alive=y];
diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -9,6 +9,9 @@
RAWREFCOUNT_VISIT = IncrementalMiniMarkGC.RAWREFCOUNT_VISIT
PYOBJ_GC_HDR = IncrementalMiniMarkGC.PYOBJ_GC_HDR
PYOBJ_GC_HDR_PTR = IncrementalMiniMarkGC.PYOBJ_GC_HDR_PTR
+RAWREFCOUNT_FINALIZER_MODERN = \
+ IncrementalMiniMarkGC.RAWREFCOUNT_FINALIZER_MODERN
+RAWREFCOUNT_FINALIZER_NONE = IncrementalMiniMarkGC.RAWREFCOUNT_FINALIZER_NONE
S = lltype.GcForwardReference()
S.become(lltype.GcStruct('S',
@@ -27,6 +30,8 @@
self.gcobjs = []
self.pyobjs = []
self.pyobj_refs = []
+ self.pyobj_finalizer = {}
+ self.pyobj_resurrect = {}
def rawrefcount_tp_traverse(obj, callback, args):
refs = self.pyobj_refs[self.pyobjs.index(obj)]
@@ -40,7 +45,14 @@
return self.gcobjs[self.pyobjs.index(pyobj)]
def rawrefcount_finalizer_type(gc):
- return 0
+ pyobj = self.pyobjs[self.gcobjs.index(gc)]
+ if pyobj in self.pyobjs and \
+ self.pyobj_finalizer.has_key(self.pyobjs.index(pyobj)):
+ # TODO: improve test, so that NONE is returned, if finalizer
+ # has already been called
+ return self.pyobj_finalizer[self.pyobjs.index(pyobj)]
+ else:
+ return RAWREFCOUNT_FINALIZER_NONE
self.pyobj_list = lltype.malloc(PYOBJ_GC_HDR_PTR.TO, flavor='raw',
immortal=True)
@@ -69,6 +81,10 @@
refs.append(pyobj_to)
pyobj_to.c_ob_refcnt += 1
+ def _rawrefcount_add_resurrect(self, pyobj_source, pyobj_target):
+ refs = self.pyobj_resurrect[self.pyobjs.index(pyobj_source)] = []
+ refs.append(pyobj_target)
+
def _rawrefcount_pypyobj(self, intval, rooted=False, create_old=True):
p1 = self.malloc(S)
p1.x = intval
@@ -439,6 +455,7 @@
nodes = {}
# create objects from graph (always create old to prevent moving)
+ finalizers = False
i = 0
for n in g.get_nodes():
name = n.get_name()
@@ -448,6 +465,8 @@
rooted = attr['rooted'] == "y" if 'rooted' in attr else False
ext_refcnt = int(attr['ext_refcnt']) if 'ext_refcnt' in attr else 0
finalizer = attr['finalizer'] if 'finalizer' in attr else None
+ if finalizer <> None:
+ finalizers = True
resurrect = attr['resurrect'] if 'resurrect' in attr else None
info = NodeInfo(type, alive, ext_refcnt, finalizer, resurrect)
if type == "C":
@@ -484,6 +503,19 @@
else:
assert False # only 2 refs supported from pypy obj
+ # add finalizers
+ for name in nodes:
+ n = nodes[name]
+ if hasattr(n, "r"):
+ index = self.pyobjs.index(n.r)
+ resurrect = n.info.resurrect
+ if n.info.finalizer == "modern" and resurrect is not None:
+ self.pyobj_finalizer[index] = RAWREFCOUNT_FINALIZER_MODERN
+ self._rawrefcount_add_resurrect(n.r, nodes[resurrect].r)
+ nodes[resurrect].info.ext_refcnt += 1
+ else:
+ self.pyobj_finalizer[index] = RAWREFCOUNT_FINALIZER_NONE
+
# quick self check, if traverse works properly
dests_by_source = {}
for e in g.get_edges():
@@ -508,6 +540,13 @@
def decref(pyobj, ignore):
pyobj.c_ob_refcnt -= 1
if pyobj.c_ob_refcnt == 0:
+ if self.pyobj_resurrect.has_key(self.pyobjs.index(pyobj)):
+ resurrect = self.pyobj_resurrect[
+ self.pyobjs.index(pyobj)]
+ for r in resurrect:
+ r.c_ob_refcnt += 1
+ resurrect.remove(r)
+ if pyobj.c_ob_refcnt == 0:
gchdr = self.gc.rrc_pyobj_as_gc(pyobj)
next = gchdr.c_gc_next
next.c_gc_prev = gchdr.c_gc_prev
@@ -521,12 +560,20 @@
while next_dead <> llmemory.NULL:
pyobj = llmemory.cast_adr_to_ptr(next_dead,
self.gc.PYOBJ_HDR_PTR)
- print "nextdead:", pyobj, "refcnt:", pyobj.c_ob_refcnt
decref(pyobj, None)
next_dead = self.gc.rawrefcount_next_dead()
- # TODO: call finalizers here and during the next collection it
- # will be checked if the CI is really trash
+ next = self.gc.rawrefcount_next_cyclic_isolate()
+ while next <> llmemory.NULL:
+ pyobj = llmemory.cast_adr_to_ptr(next,
+ self.gc.PYOBJ_HDR_PTR)
+ if self.pyobj_resurrect.has_key(self.pyobjs.index(pyobj)):
+ resurrect = self.pyobj_resurrect[self.pyobjs.index(pyobj)]
+ for r in resurrect:
+ r.c_ob_refcnt += 1
+ # TODO: improve test, use flag in gc_refs instead
+ resurrect.remove(r)
+ next = self.gc.rawrefcount_next_cyclic_isolate()
next_dead = self.gc.rawrefcount_cyclic_garbage_head()
while next_dead <> llmemory.NULL:
@@ -550,20 +597,20 @@
self.gc.rawrefcount_cyclic_garbage_remove()
next_dead = self.gc.rawrefcount_cyclic_garbage_head()
- # do a collection to find cyclic isolates
+ # do a collection to find cyclic isolates and clean them, if there are
+ # no finalizers
self.gc.collect()
-
self.gc.rrc_invoke_callback()
if self.trigger <> []:
cleanup()
- # now do another collection, to clean up cyclic trash
- # TODO: maybe optimize, so that we don't need another major collection
- self.gc.collect()
-
- self.gc.rrc_invoke_callback()
- if self.trigger <> []:
- cleanup()
+ if finalizers:
+ # now do another collection, to clean up cyclic trash, if there
+ # were finalizers involved
+ self.gc.collect()
+ self.gc.rrc_invoke_callback()
+ if self.trigger <> []:
+ cleanup()
# check livelihood of objects, according to graph
for name in nodes:
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -502,6 +502,9 @@
self.rawrefcount_next_dead_ptr = getfn(
GCClass.rawrefcount_next_dead, [s_gc], SomeAddress(),
inline = True)
+ self.rawrefcount_next_cyclic_isolate_ptr = getfn(
+ GCClass.rawrefcount_next_cyclic_isolate, [s_gc], SomeAddress(),
+ inline = True)
self.rawrefcount_cyclic_garbage_head_ptr = getfn(
GCClass.rawrefcount_cyclic_garbage_head, [s_gc], SomeAddress(),
inline = True)
@@ -1377,6 +1380,12 @@
[self.rawrefcount_next_dead_ptr, self.c_const_gc],
resultvar=hop.spaceop.result)
+ def gct_gc_rawrefcount_next_cyclic_isolate(self, hop):
+ assert hop.spaceop.result.concretetype == llmemory.Address
+ hop.genop("direct_call",
+ [self.rawrefcount_next_cyclic_isolate_ptr, self.c_const_gc],
+ resultvar=hop.spaceop.result)
+
def gct_gc_rawrefcount_cyclic_garbage_head(self, hop):
assert hop.spaceop.result.concretetype == llmemory.Address
hop.genop("direct_call",
@@ -1385,7 +1394,8 @@
def gct_gc_rawrefcount_cyclic_garbage_remove(self, hop):
hop.genop("direct_call",
- [self.rawrefcount_cyclic_garbage_remove_ptr, self.c_const_gc])
+ [self.rawrefcount_cyclic_garbage_remove_ptr,
+ self.c_const_gc])
def _set_into_gc_array_part(self, op):
if op.opname == 'setarrayitem':
diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py
--- a/rpython/rlib/rawrefcount.py
+++ b/rpython/rlib/rawrefcount.py
@@ -142,6 +142,10 @@
old_pyobj_list.remove(old_pyobj_list[0])
@not_rpython
+def next_cyclic_isolate(OB_PTR_TYPE):
+ return lltype.nullptr(OB_PTR_TYPE.TO)
+
+ at not_rpython
def _collect(track_allocation=True):
"""for tests only. Emulates a GC collection.
Will invoke dealloc_trigger_callback() once if there are objects
@@ -363,7 +367,7 @@
return _spec_p(hop, v_p)
class Entry(ExtRegistryEntry):
- _about_ = (next_dead, cyclic_garbage_head)
+ _about_ = (next_dead, cyclic_garbage_head, next_cyclic_isolate)
def compute_result_annotation(self, s_OB_PTR_TYPE):
from rpython.rtyper.llannotation import lltype_to_annotation
@@ -375,6 +379,8 @@
name = 'gc_rawrefcount_next_dead'
elif self.instance is cyclic_garbage_head:
name = 'gc_rawrefcount_cyclic_garbage_head'
+ elif self.instance is next_cyclic_isolate:
+ name = 'gc_rawrefcount_next_cyclic_isolate'
hop.exception_cannot_occur()
v_ob = hop.genop(name, [], resulttype = llmemory.Address)
return _spec_ob(hop, v_ob)
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -972,6 +972,9 @@
def op_gc_rawrefcount_next_dead(self, *args):
raise NotImplementedError("gc_rawrefcount_next_dead")
+ def op_gc_rawrefcount_next_cyclic_isolate(self, *args):
+ raise NotImplementedError("gc_rawrefcount_next_cyclic_isolate")
+
def op_gc_rawrefcount_cyclic_garbage_head(self, *args):
raise NotImplementedError("gc_rawrefcount_cyclic_garbage_head")
def op_gc_rawrefcount_cyclic_garbage_remove(self, *args):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -498,6 +498,7 @@
'gc_rawrefcount_from_obj': LLOp(sideeffects=False),
'gc_rawrefcount_to_obj': LLOp(sideeffects=False),
'gc_rawrefcount_next_dead': LLOp(),
+ 'gc_rawrefcount_next_cyclic_isolate': LLOp(),
'gc_rawrefcount_cyclic_garbage_head': LLOp(sideeffects=False),
'gc_rawrefcount_cyclic_garbage_remove': LLOp(),
More information about the pypy-commit
mailing list