[pypy-commit] pypy cpyext-gc-cycle: First version of rawrefcount legacy finalizer implementation
stevie_92
pypy.commits at gmail.com
Sun Mar 3 15:21:57 EST 2019
Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r96206:3ab559b9b893
Date: 2019-03-03 21:21 +0100
http://bitbucket.org/pypy/pypy/changeset/3ab559b9b893/
Log: First version of rawrefcount legacy finalizer implementation Test is
still failing, because garbage_pypy is not implemented yet Adapted
interface
diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c
--- a/pypy/module/cpyext/src/object.c
+++ b/pypy/module/cpyext/src/object.c
@@ -53,14 +53,15 @@
_PyPy_finalizer_type(PyGC_Head *gc)
{
PyObject *op = FROM_GC(gc);
- if (!_PyGCHead_FINALIZED(gc) &&
+ if (Py_TYPE(op)->tp_del != NULL) {
+ return 2; // legacy (has priority over modern)
+ } else if (!_PyGCHead_FINALIZED(gc) &&
PyType_HasFeature(Py_TYPE(op), Py_TPFLAGS_HAVE_FINALIZE) &&
Py_TYPE(op)->tp_finalize != NULL) {
- return 1;
+ return 1; // modern
} else {
- return 0;
+ return 0; // no finalizer
}
- // TODO: legacy finalizer (tp_del)
}
void
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -120,15 +120,12 @@
if not py_obj:
break
w_list.append(w_obj)
- last_py_obj = lltype.nullptr(PyObject.TO)
while True:
- w_pyobj = rawrefcount.next_garbage_pyobj(PyObject,
- last_py_obj)
+ w_pyobj = rawrefcount.next_garbage_pyobj(PyObject)
if not w_pyobj:
break
w_obj = from_ref(space, w_pyobj)
w_list.append(w_obj)
- last_py_obj = w_pyobj
space.setattr(space.builtin_modules['gc'],
space.newtext('garbage'), w_list)
rawrefcount.end_garbage()
@@ -330,14 +327,12 @@
if not py_obj:
break
w_list.append(w_obj)
- last_py_obj = lltype.nullptr(PyObject.TO)
while True:
- w_pyobj = rawrefcount.next_garbage_pyobj(PyObject, last_py_obj)
+ w_pyobj = rawrefcount.next_garbage_pyobj(PyObject)
if not w_pyobj:
break
w_obj = from_ref(space, w_pyobj)
w_list.append(w_obj)
- last_py_obj = w_pyobj
space.setattr(space.builtin_modules['gc'], space.newtext('garbage'),
w_list)
rawrefcount.end_garbage()
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3137,18 +3137,10 @@
self.rrc_dealloc_pending = self.AddressStack()
self.rrc_tp_traverse = tp_traverse
self.rrc_pyobj_list = self._pygchdr(pyobj_list)
- self.rrc_pyobj_old_list = \
- lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw', immortal=True)
- self.rrc_pyobj_old_list.c_gc_next = self.rrc_pyobj_old_list
- self.rrc_pyobj_old_list.c_gc_prev = self.rrc_pyobj_old_list
- self.rrc_pyobj_isolate_list = \
- lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw', immortal=True)
- self.rrc_pyobj_isolate_list.c_gc_next = self.rrc_pyobj_isolate_list
- self.rrc_pyobj_isolate_list.c_gc_prev = self.rrc_pyobj_isolate_list
- self.rrc_pyobj_garbage_list = \
- lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw', immortal=True)
- self.rrc_pyobj_garbage_list.c_gc_next = self.rrc_pyobj_garbage_list
- self.rrc_pyobj_garbage_list.c_gc_prev = self.rrc_pyobj_garbage_list
+ self.rrc_pyobj_old_list = self._rrc_gc_list_new()
+ self.rrc_pyobj_isolate_list = self._rrc_gc_list_new()
+ self.rrc_pyobj_dead_list = self._rrc_gc_list_new()
+ self.rrc_pyobj_garbage_list = self._rrc_gc_list_new()
self.rrc_gc_as_pyobj = gc_as_pyobj
self.rrc_pyobj_as_gc = pyobj_as_gc
self.rrc_finalizer_type = finalizer_type
@@ -3224,14 +3216,14 @@
return llmemory.NULL
def rawrefcount_cyclic_garbage_head(self):
- if not self._rrc_gc_list_is_empty(self.rrc_pyobj_garbage_list):
+ if not self._rrc_gc_list_is_empty(self.rrc_pyobj_dead_list):
return llmemory.cast_ptr_to_adr(
- self.rrc_gc_as_pyobj(self.rrc_pyobj_garbage_list.c_gc_next))
+ self.rrc_gc_as_pyobj(self.rrc_pyobj_dead_list.c_gc_next))
else:
return llmemory.NULL
def rawrefcount_cyclic_garbage_remove(self):
- gchdr = self.rrc_pyobj_garbage_list.c_gc_next
+ gchdr = self.rrc_pyobj_dead_list.c_gc_next
# remove from old list
next = gchdr.c_gc_next
next.c_gc_prev = gchdr.c_gc_prev
@@ -3254,25 +3246,40 @@
self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT
def rawrefcount_next_garbage_pypy(self):
- # return the next pypy object which is only reachable from garbage
- # pyobjects, probably need two more colors for this. one for marking
- # so that they stay alive during sweep, one for marking, so they do not
- # get returned here again
- return lltype.nullptr(llmemory.GCREF.TO)
-
- def rawrefcount_next_garbage_pyobj(self, curr_pyobj):
- # implement st objects in this list still remain in the set of
- # all pyobjs, because references could still change and cause them
- # to live again. also keep in mind, that state will create references
- # to pyobjs in this list and might increment the refcount.
- return llmemory.NULL
+ # We assume that next_garbage_pypy is always called before
+ # next_garbage_pyobj. As pypy objects can only be in garbage, if there
+ # is at least one pyobj in garbage, we can use this optimization.
+ if self._rrc_gc_list_is_empty(self.rrc_pyobj_garbage_list):
+ return lltype.nullptr(llmemory.GCREF.TO)
+ else:
+ # TODO: return the next pypy object which is marked with GCFLAG_GARBAGE and
+ # remove the flag from this object. We can safely assume that objects
+ # do not move, as this can only happen to old objects.
+ return lltype.nullptr(llmemory.GCREF.TO)
+
+ def rawrefcount_next_garbage_pyobj(self):
+ if self._rrc_gc_list_is_empty(self.rrc_pyobj_garbage_list):
+ return llmemory.NULL
+ else:
+ # rrc_pyobj_garbage_list is not a real list, it just points to
+ # the first (c_gc_next) and last (c_gc_prev) pyobject in the list
+ # of live objects that are garbage, so just fix the references
+ list = self.rrc_pyobj_garbage_list
+ gchdr = list.c_gc_next
+ if list.c_gc_prev == gchdr:
+ list.c_gc_next = list # reached end of list, reset it
+ else:
+ list.c_gc_next = gchdr.c_gc_next # move pointer foward
+ return llmemory.cast_ptr_to_adr(self.rrc_gc_as_pyobj(gchdr))
def rrc_invoke_callback(self):
if self.rrc_enabled and (self.rrc_dealloc_pending.non_empty() or
- self.rrc_pyobj_isolate_list.c_gc_next <>
- self.rrc_pyobj_isolate_list or
- self.rrc_pyobj_garbage_list.c_gc_next <>
- self.rrc_pyobj_garbage_list):
+ not self._rrc_gc_list_is_empty(
+ self.rrc_pyobj_isolate_list) or
+ not self._rrc_gc_list_is_empty(
+ self.rrc_pyobj_dead_list) or
+ not self._rrc_gc_list_is_empty(
+ self.rrc_pyobj_garbage_list)):
self.rrc_dealloc_trigger_callback()
def rrc_minor_collection_trace(self):
@@ -3392,7 +3399,10 @@
self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_list)
# mark all objects reachable from rawrefcounted roots
self._rrc_mark_rawrefcount()
- self._rrc_mark_garbage() # handle legacy finalizers
+ self.rrc_state = self.RAWREFCOUNT_STATE_MARKING
+ if self._rrc_find_garbage(): # handle legacy finalizers
+ self._rrc_mark_garbage()
+ self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT
use_cylicrc = not self._rrc_find_finalizer() # modern finalizers
else:
use_cylicrc = False # don't sweep any objects in cyclic isolates
@@ -3450,7 +3460,7 @@
if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT:
if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list):
self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
- self.rrc_pyobj_garbage_list)
+ self.rrc_pyobj_dead_list)
def _rrc_major_free(self, pyobject, surviving_list, surviving_dict):
# The pyobject survives if the corresponding obj survives.
@@ -3553,11 +3563,65 @@
# now all rawrefcounted objects, which are alive, have a cyclic
# refcount > 0 or are marked
+ def _rrc_find_garbage(self):
+ found_garbage = False
+ gchdr = self.rrc_pyobj_old_list.c_gc_next
+ while gchdr <> self.rrc_pyobj_old_list:
+ next_old = gchdr.c_gc_next
+ garbage = self.rrc_finalizer_type(gchdr) == \
+ self.RAWREFCOUNT_FINALIZER_LEGACY
+ if garbage:
+ self._rrc_move_to_garbage(gchdr)
+ found_garbage = True
+ gchdr = next_old
+ return found_garbage
+
def _rrc_mark_garbage(self):
- self.rrc_state = self.RAWREFCOUNT_STATE_MARKING
- # TODO: move all pyobjs to sublist of pyobj_list and reset cyclic refcount
- # TODO: mark all pypy-objects with special flag and mark them visited
- self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT
+ found_garbage = True
+ #
+ while found_garbage:
+ found_garbage = False
+ gchdr = self.rrc_pyobj_old_list.c_gc_next
+ while gchdr <> self.rrc_pyobj_old_list:
+ next_old = gchdr.c_gc_next
+ alive = (gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT) > 0
+ pyobj = self.rrc_gc_as_pyobj(gchdr)
+ if pyobj.c_ob_pypy_link <> 0:
+ intobj = pyobj.c_ob_pypy_link
+ obj = llmemory.cast_int_to_adr(intobj)
+ if not alive and self.header(obj).tid & (
+ GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS):
+ # add fake refcount, to mark it as live
+ gchdr.c_gc_refs += 1 << self.RAWREFCOUNT_REFS_SHIFT
+ alive = True
+ if alive:
+ self._rrc_move_to_garbage(gchdr)
+ found_garbage = True
+ gchdr = next_old
+
+ def _rrc_move_to_garbage(self, gchdr):
+ pyobj = self.rrc_gc_as_pyobj(gchdr)
+ # remove from old list
+ next = gchdr.c_gc_next
+ next.c_gc_prev = gchdr.c_gc_prev
+ gchdr.c_gc_prev.c_gc_next = next
+ # add to beginning of pyobj_list
+ self._rrc_gc_list_add(self.rrc_pyobj_list, gchdr)
+ # set as new beginning (and optionally end) of
+ # pyobj_garbage_list (not a real list, just pointers to
+ # begin and end)
+ if self._rrc_gc_list_is_empty(self.rrc_pyobj_garbage_list):
+ self.rrc_pyobj_garbage_list.c_gc_prev = gchdr
+ self.rrc_pyobj_garbage_list.c_gc_next = gchdr
+ # mark referenced objects alive (so objects in the old list
+ # will be detected as garbage, as they should have a cyclic
+ # refcount of zero or an unmarked linked pypy object)
+ self._rrc_traverse(pyobj, 1)
+ if pyobj.c_ob_pypy_link <> 0:
+ intobj = pyobj.c_ob_pypy_link
+ obj = llmemory.cast_int_to_adr(intobj)
+ self.objects_to_trace.append(obj)
+ self.visit_all_objects()
def _rrc_check_finalizer(self):
# Check, if the cyclic isolate from the last collection cycle
@@ -3576,7 +3640,7 @@
self.rrc_pyobj_list)
else:
self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
- self.rrc_pyobj_garbage_list)
+ self.rrc_pyobj_dead_list)
def _rrc_find_finalizer(self):
found_finalizer = False
@@ -3619,6 +3683,11 @@
else:
self.rrc_tp_traverse(pyobj, self._rrc_visit_action, None)
+ def _rrc_gc_list_new(self):
+ list = lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw', immortal=True)
+ self._rrc_gc_list_init(list)
+ return list
+
def _rrc_gc_list_init(self, pygclist):
pygclist.c_gc_next = pygclist
pygclist.c_gc_prev = pygclist
diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -51,7 +51,7 @@
if pyobj in self.pyobjs and \
self.pyobj_finalizer.has_key(self.pyobjs.index(pyobj)):
# TODO: improve test, so that NONE is returned, if finalizer
- # has already been called
+ # has already been called (only for modern)
return self.pyobj_finalizer[self.pyobjs.index(pyobj)]
else:
return RAWREFCOUNT_FINALIZER_NONE
@@ -472,7 +472,7 @@
rooted = attr['rooted'] == "y" if 'rooted' in attr else False
ext_refcnt = int(attr['ext_refcnt']) if 'ext_refcnt' in attr else 0
finalizer = attr['finalizer'] if 'finalizer' in attr else None
- if finalizer <> None:
+ if finalizer == "modern":
finalizers = True
resurrect = attr['resurrect'] if 'resurrect' in attr else None
delete = attr['delete'] if 'delete' in attr else None
@@ -632,12 +632,10 @@
while next_garbage <> lltype.nullptr(llmemory.GCREF.TO):
garbage_pypy.append(next_garbage)
next_garbage = self.gc.rawrefcount_next_garbage_pypy()
- last_pyobj = llmemory.NULL
- next = self.gc.rawrefcount_next_garbage_pyobj(last_pyobj)
+ next = self.gc.rawrefcount_next_garbage_pyobj()
while next <> llmemory.NULL:
garbage_pyobj.append(next)
- next = self.gc.rawrefcount_next_garbage_pyobj(last_pyobj)
- last_pyobj = next
+ next = self.gc.rawrefcount_next_garbage_pyobj()
self.gc.rawrefcount_end_garbage()
# do a collection to find cyclic isolates and clean them, if there are
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -528,8 +528,8 @@
GCClass.rawrefcount_next_garbage_pypy, [s_gc],
s_gcref, inline = True)
self.rawrefcount_next_garbage_pyobj_ptr = getfn(
- GCClass.rawrefcount_next_garbage_pyobj, [s_gc, SomeAddress()],
- SomeAddress(), inline = True)
+ GCClass.rawrefcount_next_garbage_pyobj, [s_gc], SomeAddress(),
+ inline = True)
if GCClass.can_usually_pin_objects:
self.pin_ptr = getfn(GCClass.pin,
@@ -1453,12 +1453,9 @@
resultvar=hop.spaceop.result)
def gct_gc_rawrefcount_next_garbage_pyobj(self, hop):
- [v_pyobject] = hop.spaceop.args
- assert v_pyobject.concretetype == llmemory.Address
assert hop.spaceop.result.concretetype == llmemory.Address
hop.genop("direct_call",
- [self.rawrefcount_next_garbage_pyobj_ptr, self.c_const_gc,
- v_pyobject],
+ [self.rawrefcount_next_garbage_pyobj_ptr, self.c_const_gc],
resultvar=hop.spaceop.result)
def _set_into_gc_array_part(self, op):
diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py
--- a/rpython/rlib/rawrefcount.py
+++ b/rpython/rlib/rawrefcount.py
@@ -158,7 +158,7 @@
return None
@not_rpython
-def next_garbage_pyobj(OB_PTR_TYPE, curr_pyobj):
+def next_garbage_pyobj(OB_PTR_TYPE):
return lltype.nullptr(OB_PTR_TYPE.TO)
@not_rpython
@@ -383,7 +383,8 @@
return _spec_p(hop, v_p)
class Entry(ExtRegistryEntry):
- _about_ = (next_dead, cyclic_garbage_head, next_cyclic_isolate)
+ _about_ = (next_dead, cyclic_garbage_head, next_cyclic_isolate,
+ next_garbage_pyobj)
def compute_result_annotation(self, s_OB_PTR_TYPE):
from rpython.rtyper.llannotation import lltype_to_annotation
@@ -397,28 +398,13 @@
name = 'gc_rawrefcount_cyclic_garbage_head'
elif self.instance is next_cyclic_isolate:
name = 'gc_rawrefcount_next_cyclic_isolate'
+ elif self.instance is next_garbage_pyobj:
+ name = 'gc_rawrefcount_next_garbage_pyobj'
hop.exception_cannot_occur()
v_ob = hop.genop(name, [], resulttype = llmemory.Address)
return _spec_ob(hop, v_ob)
class Entry(ExtRegistryEntry):
- _about_ = next_garbage_pyobj
-
- def compute_result_annotation(self, s_OB_PTR_TYPE, s_ob):
- from rpython.rtyper.llannotation import lltype_to_annotation, SomePtr
- assert s_OB_PTR_TYPE.is_constant()
- assert isinstance(s_ob, SomePtr)
- return lltype_to_annotation(s_OB_PTR_TYPE.const)
-
- def specialize_call(self, hop):
- hop.exception_cannot_occur()
- v_ob = hop.inputarg(hop.args_r[1], arg=1)
- v_ob_res = hop.genop('gc_rawrefcount_next_garbage_pyobj',
- [_unspec_ob(hop, v_ob)],
- resulttype = llmemory.Address)
- return _spec_ob(hop, v_ob_res)
-
-class Entry(ExtRegistryEntry):
_about_ = next_garbage_pypy
def compute_result_annotation(self, s_Class):
More information about the pypy-commit
mailing list