[pypy-commit] pypy cpyext-gc-cycle: Adapted tests in gc/rawrefcount to new cycle deletion
stevie_92
pypy.commits at gmail.com
Fri Jan 18 15:08:41 EST 2019
Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-cycle
Changeset: r95672:6e15b053de37
Date: 2019-01-18 21:06 +0100
http://bitbucket.org/pypy/pypy/changeset/6e15b053de37/
Log: Adapted tests in gc/rawrefcount to new cycle deletion
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -214,13 +214,27 @@
def _rawrefcount_perform(space):
- from pypy.module.cpyext.pyobject import PyObject, decref
+ from pypy.module.cpyext.pyobject import PyObject, incref, decref
while True:
py_obj = rawrefcount.next_dead(PyObject)
if not py_obj:
break
decref(space, py_obj)
+ while True:
+ py_obj = rawrefcount.cyclic_garbage_head(PyObject)
+ if not py_obj:
+ break
+
+ pyobj = rffi.cast(PyObject, py_obj)
+ if pyobj.c_ob_type and pyobj.c_ob_type.c_tp_clear:
+ incref(space, py_obj)
+ pyobj.c_ob_type.c_tp_clear(pyobj)
+ decref(space, py_obj)
+
+ if py_obj == rawrefcount.cyclic_garbage_head(PyObject):
+ rawrefcount.cyclic_garbage_remove()
+
class PyObjDeallocAction(executioncontext.AsyncAction):
"""An action that invokes _Py_Dealloc() on the dying PyObjects.
"""
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3031,6 +3031,12 @@
self.rrc_dealloc_pending = self.AddressStack()
self.rrc_tp_traverse = tp_traverse
self.rrc_pyobj_list = self._pygchdr(pyobj_list)
+ self.rrc_pyobj_old_list = \
+ lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw', immortal=True)
+ self.rrc_pyobj_garbage_list = \
+ lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw', immortal=True)
+ self.rrc_pyobj_garbage_list.c_gc_next = self.rrc_pyobj_garbage_list
+ self.rrc_pyobj_garbage_list.c_gc_prev = self.rrc_pyobj_garbage_list
self.rrc_gc_as_pyobj = gc_as_pyobj
self.rrc_pyobj_as_gc = pyobj_as_gc
self.rrc_enabled = True
@@ -3096,6 +3102,26 @@
return self.rrc_dealloc_pending.pop()
return llmemory.NULL
+ def rawrefcount_cyclic_garbage_head(self):
+ if self.rrc_pyobj_garbage_list.c_gc_next <> \
+ self.rrc_pyobj_garbage_list:
+ return llmemory.cast_ptr_to_adr(
+ self.rrc_gc_as_pyobj(self.rrc_pyobj_garbage_list.c_gc_next))
+ else:
+ return llmemory.NULL
+
+ def rawrefcount_cyclic_garbage_remove(self):
+ gchdr = self.rrc_pyobj_garbage_list.c_gc_next
+ # remove from old list
+ next = gchdr.c_gc_next
+ next.c_gc_prev = gchdr.c_gc_prev
+ gchdr.c_gc_prev.c_gc_next = next
+ # add to new list, may die later
+ next = self.rrc_pyobj_list.c_gc_next
+ self.rrc_pyobj_list.c_gc_next = gchdr
+ gchdr.c_gc_prev = self.rrc_pyobj_list
+ gchdr.c_gc_next = next
+ next.c_gc_prev = gchdr
def rrc_invoke_callback(self):
if self.rrc_enabled and self.rrc_dealloc_pending.non_empty():
@@ -3194,19 +3220,16 @@
rc -= REFCNT_FROM_PYPY
self._pyobj(pyobject).c_ob_pypy_link = 0
if rc == 0:
- if not major: # we do it later in major collections
- self.rrc_dealloc_pending.append(pyobject)
- # an object with refcnt == 0 cannot stay around waiting
- # for its deallocator to be called. Some code (lxml)
- # expects that tp_dealloc is called immediately when
- # the refcnt drops to 0. If it isn't, we get some
- # uncleared raw pointer that can still be used to access
- # the object; but (PyObject *)raw_pointer is then bogus
- # because after a Py_INCREF()/Py_DECREF() on it, its
- # tp_dealloc is also called!
- rc = 1
- else:
- rc = REFCNT_FROM_PYPY
+ self.rrc_dealloc_pending.append(pyobject)
+ # an object with refcnt == 0 cannot stay around waiting
+ # for its deallocator to be called. Some code (lxml)
+ # expects that tp_dealloc is called immediately when
+ # the refcnt drops to 0. If it isn't, we get some
+ # uncleared raw pointer that can still be used to access
+ # the object; but (PyObject *)raw_pointer is then bogus
+ # because after a Py_INCREF()/Py_DECREF() on it, its
+ # tp_dealloc is also called!
+ rc = 1
self._pyobj(pyobject).c_ob_refcnt = rc
_rrc_free._always_inline_ = True
@@ -3225,7 +3248,9 @@
# TODO: * move reachable cpython objects back to pyobj_list
# TODO: * mark all reachable objects as potentially uncollectable
- # TODO: handle weakrefs for unreachable objects
+ # TODO: handle weakrefs for unreachable objects and create
+ # TODO: a list of callbacks, which has to be called after the
+ # TODO: the GC runs
# TODO: call tp_finalize for unreachable objects
# TODO: (could resurrect objects, so we have to do it now)
@@ -3249,8 +3274,6 @@
self.visit_all_objects()
def rrc_major_collection_free(self):
- from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
- #
ll_assert(self.rrc_p_dict_nurs.length() == 0, "p_dict_nurs not empty 2")
length_estimate = self.rrc_p_dict.length()
self.rrc_p_dict.delete()
@@ -3270,26 +3293,17 @@
self.rrc_o_list_old.delete()
self.rrc_o_list_old = new_o_list
- # TODO: === DO THIS LATER OUTSIDE OF GC ===
- # TODO: (like dealloc_pending.append, but only pass reference to
- # TODO: unreachable object list, not for each object. free
- # TODO: this list afterwards)
- # TODO: call tp_clear (wrapped between inc- and decref) instead of
- # TODO: free, to break cycles for unreachable objects
-
- # TODO: === REMOVE THE CODE BELOW ===
- # TODO: this exists only to please the current tests, but fails
- # TODO: if the pending deallocations are executed properly
- pygchdr = self.rrc_pyobj_old_list.c_gc_next
- while pygchdr <> self.rrc_pyobj_old_list:
- assert pygchdr.c_gc_refs == 0
- pyobj = self.rrc_gc_as_pyobj(pygchdr)
- if pyobj.c_ob_refcnt == REFCNT_FROM_PYPY:
- pyobj.c_ob_refcnt = 0
- pyobj.c_ob_refcnt += 1
- self.rrc_dealloc_pending.append(llmemory.cast_ptr_to_adr(pyobj))
- pygchdr = pygchdr.c_gc_next
- lltype.free(self.rrc_pyobj_old_list, flavor='raw')
+ # merge old_list into garbage_list and clear old_list
+ if self.rrc_pyobj_old_list.c_gc_next <> self.rrc_pyobj_old_list:
+ next = self.rrc_pyobj_garbage_list.c_gc_next
+ next_old = self.rrc_pyobj_old_list.c_gc_next
+ prev_old = self.rrc_pyobj_old_list.c_gc_prev
+ self.rrc_pyobj_garbage_list.c_gc_next = next_old
+ next_old.c_gc_prev = self.rrc_pyobj_garbage_list
+ prev_old.c_gc_next = next
+ next.c_gc_prev = prev_old
+ self.rrc_pyobj_old_list.c_gc_next = self.rrc_pyobj_old_list
+ self.rrc_pyobj_old_list.c_gc_prev = self.rrc_pyobj_old_list
def _rrc_major_free(self, pyobject, surviving_list, surviving_dict):
# The pyobject survives if the corresponding obj survives.
@@ -3343,7 +3357,6 @@
gchdr.c_gc_refs += 1
def _rrc_mark_rawrefcount(self):
- self.rrc_pyobj_old_list = lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw')
if self.rrc_pyobj_list.c_gc_next == self.rrc_pyobj_list:
self.rrc_pyobj_old_list.c_gc_next = self.rrc_pyobj_old_list
self.rrc_pyobj_old_list.c_gc_prev = self.rrc_pyobj_old_list
diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -461,7 +461,8 @@
dest = nodes[e.get_destination()]
if source.info.type == "C" or dest.info.type == "C":
self._rawrefcount_addref(source.r, dest.r)
- dest.info.ext_refcnt += 1
+ if source.info.alive:
+ dest.info.ext_refcnt += 1
elif source.info.type == "P" or dest.info.type == "P":
if llmemory.cast_ptr_to_adr(source.p.next) == llmemory.NULL:
source.p.next = dest.p
@@ -489,14 +490,47 @@
# do collection
self.gc.collect()
- # simply free all pending deallocations, we don't care about the
- # side effects for now...
+ def decref_children(pyobj):
+ self.gc.rrc_tp_traverse(pyobj, decref, None)
+ def decref(pyobj, ignore):
+ pyobj.c_ob_refcnt -= 1
+ if pyobj.c_ob_refcnt == 0:
+ gchdr = self.gc.rrc_pyobj_as_gc(pyobj)
+ next = gchdr.c_gc_next
+ next.c_gc_prev = gchdr.c_gc_prev
+ gchdr.c_gc_prev.c_gc_next = next
+ decref_children(pyobj)
+ self.pyobjs[self.pyobjs.index(pyobj)] = \
+ lltype.nullptr(PYOBJ_HDR_PTR.TO)
+ lltype.free(pyobj, flavor='raw')
+
next_dead = self.gc.rawrefcount_next_dead()
- while next_dead <> llmemory.NULL:
+ while next_dead <> llmemory.NULL:
pyobj = llmemory.cast_adr_to_ptr(next_dead, self.gc.PYOBJ_HDR_PTR)
- lltype.free(pyobj, flavor='raw')
+ decref(pyobj, None)
next_dead = self.gc.rawrefcount_next_dead()
+ # free cyclic structures
+ next_dead = self.gc.rawrefcount_cyclic_garbage_head()
+ while next_dead <> llmemory.NULL:
+ pyobj = llmemory.cast_adr_to_ptr(next_dead, self.gc.PYOBJ_HDR_PTR)
+ pyobj.c_ob_refcnt += 1
+
+ def free(pyobj_to, pyobj_from):
+ refs = self.pyobj_refs[self.pyobjs.index(pyobj_from)]
+ refs.remove(pyobj_to)
+ decref(pyobj_to, None)
+ self.gc.rrc_tp_traverse(pyobj, free, pyobj)
+
+ decref(pyobj, None)
+
+ curr = llmemory.cast_adr_to_int(next_dead)
+ next_dead = self.gc.rawrefcount_cyclic_garbage_head()
+
+ if llmemory.cast_adr_to_int(next_dead) == curr:
+ self.gc.rawrefcount_cyclic_garbage_remove()
+ next_dead = self.gc.rawrefcount_cyclic_garbage_head()
+
# check livelihood of objects, according to graph
for name in nodes:
n = nodes[name]
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -501,6 +501,12 @@
self.rawrefcount_next_dead_ptr = getfn(
GCClass.rawrefcount_next_dead, [s_gc], SomeAddress(),
inline = True)
+ self.rawrefcount_cyclic_garbage_head_ptr = getfn(
+ GCClass.rawrefcount_cyclic_garbage_head, [s_gc], SomeAddress(),
+ inline = True)
+ self.rawrefcount_cyclic_garbage_remove_ptr = getfn(
+ GCClass.rawrefcount_cyclic_garbage_remove, [s_gc],
+ annmodel.s_None, inline = True)
if GCClass.can_usually_pin_objects:
self.pin_ptr = getfn(GCClass.pin,
@@ -1369,6 +1375,16 @@
[self.rawrefcount_next_dead_ptr, self.c_const_gc],
resultvar=hop.spaceop.result)
+ def gct_gc_rawrefcount_cyclic_garbage_head(self, hop):
+ assert hop.spaceop.result.concretetype == llmemory.Address
+ hop.genop("direct_call",
+ [self.rawrefcount_cyclic_garbage_head_ptr, self.c_const_gc],
+ resultvar=hop.spaceop.result)
+
+ def gct_gc_rawrefcount_cyclic_garbage_remove(self, hop):
+ hop.genop("direct_call",
+ [self.rawrefcount_cyclic_garbage_remove_ptr, self.c_const_gc])
+
def _set_into_gc_array_part(self, op):
if op.opname == 'setarrayitem':
return op.args[1]
diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py
--- a/rpython/rlib/rawrefcount.py
+++ b/rpython/rlib/rawrefcount.py
@@ -131,6 +131,16 @@
return ob
@not_rpython
+def cyclic_garbage_head(OB_PTR_TYPE):
+ # TODO
+ return lltype.nullptr(OB_PTR_TYPE.TO)
+
+ at not_rpython
+def cyclic_garbage_remove():
+ # TODO
+ pass
+
+ at not_rpython
def _collect(track_allocation=True):
"""for tests only. Emulates a GC collection.
Will invoke dealloc_trigger_callback() once if there are objects
@@ -335,20 +345,32 @@
return _spec_p(hop, v_p)
class Entry(ExtRegistryEntry):
- _about_ = next_dead
+ _about_ = (next_dead, cyclic_garbage_head)
def compute_result_annotation(self, s_OB_PTR_TYPE):
- from rpython.annotator import model as annmodel
from rpython.rtyper.llannotation import lltype_to_annotation
assert s_OB_PTR_TYPE.is_constant()
return lltype_to_annotation(s_OB_PTR_TYPE.const)
def specialize_call(self, hop):
+ if self.instance is next_dead:
+ name = 'gc_rawrefcount_next_dead'
+ elif self.instance is cyclic_garbage_head:
+ name = 'gc_rawrefcount_cyclic_garbage_head'
hop.exception_cannot_occur()
- v_ob = hop.genop('gc_rawrefcount_next_dead', [],
- resulttype = llmemory.Address)
+ v_ob = hop.genop(name, [], resulttype = llmemory.Address)
return _spec_ob(hop, v_ob)
+class Entry(ExtRegistryEntry):
+ _about_ = cyclic_garbage_remove
+
+ def compute_result_annotation(self):
+ pass
+
+ def specialize_call(self, hop):
+ hop.exception_cannot_occur()
+ hop.genop('gc_rawrefcount_cyclic_garbage_remove', [])
+
src_dir = py.path.local(__file__).dirpath() / 'src'
boehm_eci = ExternalCompilationInfo(
post_include_bits = [(src_dir / 'boehm-rawrefcount.h').read()],
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -972,6 +972,11 @@
def op_gc_rawrefcount_next_dead(self, *args):
raise NotImplementedError("gc_rawrefcount_next_dead")
+ def op_gc_rawrefcount_cyclic_garbage_head(self, *args):
+ raise NotImplementedError("gc_rawrefcount_cyclic_garbage_head")
+ def op_gc_rawrefcount_cyclic_garbage_remove(self, *args):
+ raise NotImplementedError("gc_rawrefcount_cyclic_garbage_remove")
+
def op_do_malloc_fixedsize(self):
raise NotImplementedError("do_malloc_fixedsize")
def op_do_malloc_fixedsize_clear(self):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -169,6 +169,7 @@
# This list corresponds to the operations implemented by the LLInterpreter.
# ***** Run test_lloperation after changes. *****
+# ***** Run test_lloperation after changes. *****
LL_OPERATIONS = {
'direct_call': LLOp(canraise=(Exception,)),
@@ -490,13 +491,15 @@
'gc_fq_register' : LLOp(),
'gc_ignore_finalizer' : LLOp(canrun=True),
- 'gc_rawrefcount_init': LLOp(),
- 'gc_rawrefcount_create_link_pypy': LLOp(),
- 'gc_rawrefcount_create_link_pyobj': LLOp(),
- 'gc_rawrefcount_mark_deallocating': LLOp(),
- 'gc_rawrefcount_from_obj': LLOp(sideeffects=False),
- 'gc_rawrefcount_to_obj': LLOp(sideeffects=False),
- 'gc_rawrefcount_next_dead': LLOp(),
+ 'gc_rawrefcount_init': LLOp(),
+ 'gc_rawrefcount_create_link_pypy': LLOp(),
+ 'gc_rawrefcount_create_link_pyobj': LLOp(),
+ 'gc_rawrefcount_mark_deallocating': LLOp(),
+ 'gc_rawrefcount_from_obj': LLOp(sideeffects=False),
+ 'gc_rawrefcount_to_obj': LLOp(sideeffects=False),
+ 'gc_rawrefcount_next_dead': LLOp(),
+ 'gc_rawrefcount_cyclic_garbage_head': LLOp(sideeffects=False),
+ 'gc_rawrefcount_cyclic_garbage_remove': LLOp(),
'gc_move_out_of_nursery': LLOp(),
@@ -582,7 +585,6 @@
# __________ instrumentation _________
'instrument_count': LLOp(),
}
-# ***** Run test_lloperation after changes. *****
# ____________________________________________________________
# Post-processing
More information about the pypy-commit
mailing list