[pypy-svn] r77002 - in pypy/branch/gc-module/pypy: module/gc rlib rpython/memory/gc rpython/memory/gctransform translator/c/test
arigo at codespeak.net
arigo at codespeak.net
Fri Sep 10 17:07:19 CEST 2010
Author: arigo
Date: Fri Sep 10 17:07:16 2010
New Revision: 77002
Added:
pypy/branch/gc-module/pypy/rpython/memory/gc/inspect.py (contents, props changed)
Modified:
pypy/branch/gc-module/pypy/module/gc/__init__.py
pypy/branch/gc-module/pypy/module/gc/referents.py
pypy/branch/gc-module/pypy/rlib/rgc.py
pypy/branch/gc-module/pypy/rpython/memory/gc/base.py
pypy/branch/gc-module/pypy/rpython/memory/gc/markcompact.py
pypy/branch/gc-module/pypy/rpython/memory/gc/semispace.py
pypy/branch/gc-module/pypy/rpython/memory/gctransform/framework.py
pypy/branch/gc-module/pypy/translator/c/test/test_newgc.py
Log:
Move the get_rpy_*() helpers in their own file, inspect.py,
instead of polluting the namespace of the GC class.
Add dump_rpy_heap(), which dumps all objects in the heap
to the given file descriptor, in the format recognized by
svn/arigo/hack/pypy-hack/heapstats/.
Modified: pypy/branch/gc-module/pypy/module/gc/__init__.py
==============================================================================
--- pypy/branch/gc-module/pypy/module/gc/__init__.py (original)
+++ pypy/branch/gc-module/pypy/module/gc/__init__.py Fri Sep 10 17:07:16 2010
@@ -25,6 +25,7 @@
'get_objects': 'referents.get_objects',
'get_referents': 'referents.get_referents',
'get_referrers': 'referents.get_referrers',
+ 'dump_rpy_heap': 'referents.dump_rpy_heap',
'GcRef': 'referents.W_GcRef',
})
MixedModule.__init__(self, space, w_name)
Modified: pypy/branch/gc-module/pypy/module/gc/referents.py
==============================================================================
--- pypy/branch/gc-module/pypy/module/gc/referents.py (original)
+++ pypy/branch/gc-module/pypy/module/gc/referents.py Fri Sep 10 17:07:16 2010
@@ -147,3 +147,7 @@
pending_w += referents_w
return space.newlist(result_w.keys())
get_referrers.unwrap_spec = [ObjSpace, 'args_w']
+
+def dump_rpy_heap(space, fd):
+ rgc.dump_rpy_heap(fd)
+dump_rpy_heap.unwrap_spec = [ObjSpace, int]
Modified: pypy/branch/gc-module/pypy/rlib/rgc.py
==============================================================================
--- pypy/branch/gc-module/pypy/rlib/rgc.py (original)
+++ pypy/branch/gc-module/pypy/rlib/rgc.py Fri Sep 10 17:07:16 2010
@@ -368,6 +368,10 @@
else:
return id(gcref._x)
+def dump_rpy_heap(fd):
+ "NOT_RPYTHON"
+ raise NotImplementedError
+
NULL_GCREF = lltype.nullptr(llmemory.GCREF.TO)
class _GcRef(object):
@@ -509,3 +513,12 @@
vtable = classrepr.getvtable()
assert lltype.typeOf(vtable) == rclass.CLASSTYPE
return Constant(vtable, concretetype=rclass.CLASSTYPE)
+
+class Entry(ExtRegistryEntry):
+ _about_ = dump_rpy_heap
+ def compute_result_annotation(self, s_fd):
+ from pypy.annotation.model import s_None
+ return s_None
+ def specialize_call(self, hop):
+ vlist = hop.inputargs(lltype.Signed)
+ return hop.genop('gc_dump_rpy_heap', vlist, resulttype = hop.r_result)
Modified: pypy/branch/gc-module/pypy/rpython/memory/gc/base.py
==============================================================================
--- pypy/branch/gc-module/pypy/rpython/memory/gc/base.py (original)
+++ pypy/branch/gc-module/pypy/rpython/memory/gc/base.py Fri Sep 10 17:07:16 2010
@@ -103,6 +103,9 @@
def get_size(self, obj):
return self._get_size_for_typeid(obj, self.get_type_id(obj))
+ def get_size_incl_hash(self, obj):
+ return self.get_size(obj)
+
def malloc(self, typeid, length=0, zero=False):
"""For testing. The interface used by the gctransformer is
the four malloc_[fixed,var]size[_clear]() functions.
@@ -257,99 +260,6 @@
finally:
self.finalizer_lock_count -= 1
- # ---------- implementation of pypy.rlib.rgc.get_rpy_roots() ----------
-
- def _counting_rpy_root(self, root):
- self._count_rpy += 1
-
- def _do_count_rpy_roots(self):
- self._count_rpy = 0
- self.root_walker.walk_roots(
- GCBase._counting_rpy_root,
- GCBase._counting_rpy_root,
- GCBase._counting_rpy_root)
- return self._count_rpy
-
- def _append_rpy_root(self, root):
- # Can use the gc list, but should not allocate!
- # It is essential that the list is not resizable!
- lst = self._list_rpy
- index = self._count_rpy
- if index >= len(lst):
- raise ValueError
- self._count_rpy = index + 1
- lst[index] = llmemory.cast_adr_to_ptr(root.address[0], llmemory.GCREF)
-
- def _do_append_rpy_roots(self, lst):
- self._count_rpy = 0
- self._list_rpy = lst
- self.root_walker.walk_roots(
- GCBase._append_rpy_root,
- GCBase._append_rpy_root,
- GCBase._append_rpy_root)
- self._list_rpy = None
-
- def get_rpy_roots(self):
- count = self._do_count_rpy_roots()
- extra = 16
- while True:
- result = [lltype.nullptr(llmemory.GCREF.TO)] * (count + extra)
- try:
- self._do_append_rpy_roots(result)
- except ValueError:
- extra *= 3
- else:
- return result
-
- # ---------- implementation of pypy.rlib.rgc.get_rpy_referents() ----------
-
- def _count_rpy_referent(self, pointer, _):
- self._count_rpy += 1
-
- def _do_count_rpy_referents(self, gcref):
- self._count_rpy = 0
- self.trace(llmemory.cast_ptr_to_adr(gcref),
- self._count_rpy_referent, None)
- return self._count_rpy
-
- def _append_rpy_referent(self, pointer, _):
- # Can use the gc list, but should not allocate!
- # It is essential that the list is not resizable!
- lst = self._list_rpy
- index = self._count_rpy
- if index >= len(lst):
- raise ValueError
- self._count_rpy = index + 1
- lst[index] = llmemory.cast_adr_to_ptr(pointer.address[0],
- llmemory.GCREF)
-
- def _do_append_rpy_referents(self, gcref, lst):
- self._count_rpy = 0
- self._list_rpy = lst
- self.trace(llmemory.cast_ptr_to_adr(gcref),
- self._append_rpy_referent, None)
-
- def get_rpy_referents(self, gcref):
- count = self._do_count_rpy_referents(gcref)
- result = [lltype.nullptr(llmemory.GCREF.TO)] * count
- self._do_append_rpy_referents(gcref, result)
- return result
-
- # ----------
-
- def get_rpy_memory_usage(self, gcref):
- # overridden in semispace.py and markcompact.py to also count the hash
- return self.get_size(llmemory.cast_ptr_to_adr(gcref))
-
- def get_rpy_type_index(self, gcref):
- from pypy.rlib.rarithmetic import intmask
- typeid = self.get_type_id(llmemory.cast_ptr_to_adr(gcref))
- return self.get_member_index(typeid)
-
- def is_rpy_instance(self, gcref):
- typeid = self.get_type_id(llmemory.cast_ptr_to_adr(gcref))
- return self.is_rpython_class(typeid)
-
class MovingGCBase(GCBase):
moving_gc = True
Added: pypy/branch/gc-module/pypy/rpython/memory/gc/inspect.py
==============================================================================
--- (empty file)
+++ pypy/branch/gc-module/pypy/rpython/memory/gc/inspect.py Fri Sep 10 17:07:16 2010
@@ -0,0 +1,188 @@
+"""
+Utility RPython functions to inspect objects in the GC.
+"""
+from pypy.rpython.lltypesystem import lltype, llmemory, rffi
+from pypy.rlib.objectmodel import free_non_gc_object
+from pypy.rpython.module.ll_os import underscore_on_windows
+from pypy.rlib import rposix
+
+from pypy.rpython.memory.support import AddressDict, get_address_stack
+
+
+# ---------- implementation of pypy.rlib.rgc.get_rpy_roots() ----------
+
+def _counting_rpy_root(gc, root):
+ gc._count_rpy += 1
+
+def _do_count_rpy_roots(gc):
+ gc._count_rpy = 0
+ gc.root_walker.walk_roots(
+ _counting_rpy_root,
+ _counting_rpy_root,
+ _counting_rpy_root)
+ return gc._count_rpy
+
+def _append_rpy_root(gc, root):
+ # Can use the gc list, but should not allocate!
+ # It is essential that the list is not resizable!
+ lst = gc._list_rpy
+ index = gc._count_rpy
+ if index >= len(lst):
+ raise ValueError
+ gc._count_rpy = index + 1
+ lst[index] = llmemory.cast_adr_to_ptr(root.address[0], llmemory.GCREF)
+
+def _do_append_rpy_roots(gc, lst):
+ gc._count_rpy = 0
+ gc._list_rpy = lst
+ gc.root_walker.walk_roots(
+ _append_rpy_root,
+ _append_rpy_root,
+ _append_rpy_root)
+ gc._list_rpy = None
+
+def get_rpy_roots(gc):
+ count = _do_count_rpy_roots(gc)
+ extra = 16
+ while True:
+ result = [lltype.nullptr(llmemory.GCREF.TO)] * (count + extra)
+ try:
+ _do_append_rpy_roots(gc, result)
+ except ValueError:
+ extra *= 3
+ else:
+ return result
+
+# ---------- implementation of pypy.rlib.rgc.get_rpy_referents() ----------
+
+def _count_rpy_referent(pointer, gc):
+ gc._count_rpy += 1
+
+def _do_count_rpy_referents(gc, gcref):
+ gc._count_rpy = 0
+ gc.trace(llmemory.cast_ptr_to_adr(gcref), _count_rpy_referent, gc)
+ return gc._count_rpy
+
+def _append_rpy_referent(pointer, gc):
+ # Can use the gc list, but should not allocate!
+ # It is essential that the list is not resizable!
+ lst = gc._list_rpy
+ index = gc._count_rpy
+ if index >= len(lst):
+ raise ValueError
+ gc._count_rpy = index + 1
+ lst[index] = llmemory.cast_adr_to_ptr(pointer.address[0],
+ llmemory.GCREF)
+
+def _do_append_rpy_referents(gc, gcref, lst):
+ gc._count_rpy = 0
+ gc._list_rpy = lst
+ gc.trace(llmemory.cast_ptr_to_adr(gcref), _append_rpy_referent, gc)
+
+def get_rpy_referents(gc, gcref):
+ count = _do_count_rpy_referents(gc, gcref)
+ result = [lltype.nullptr(llmemory.GCREF.TO)] * count
+ _do_append_rpy_referents(gc, gcref, result)
+ return result
+
+# ----------
+
+def get_rpy_memory_usage(gc, gcref):
+ return gc.get_size_incl_hash(llmemory.cast_ptr_to_adr(gcref))
+
+def get_rpy_type_index(gc, gcref):
+ typeid = gc.get_type_id(llmemory.cast_ptr_to_adr(gcref))
+ return gc.get_member_index(typeid)
+
+def is_rpy_instance(gc, gcref):
+ typeid = gc.get_type_id(llmemory.cast_ptr_to_adr(gcref))
+ return gc.is_rpython_class(typeid)
+
+# ----------
+
+raw_os_write = rffi.llexternal(underscore_on_windows+'write',
+ [rffi.INT, llmemory.Address, rffi.SIZE_T],
+ rffi.SIZE_T,
+ sandboxsafe=True, _nowrapper=True)
+
+AddressStack = get_address_stack()
+
+class HeapDumper:
+ _alloc_flavor_ = "raw"
+ BUFSIZE = 8192 # words
+
+ def __init__(self, gc, fd):
+ self.gc = gc
+ self.fd = rffi.cast(rffi.INT, fd)
+ self.writebuffer = lltype.malloc(rffi.LONGP.TO, self.BUFSIZE,
+ flavor='raw')
+ self.buf_count = 0
+ self.seen = AddressDict()
+ self.pending = AddressStack()
+
+ def delete(self):
+ self.seen.delete()
+ self.pending.delete()
+ lltype.free(self.writebuffer, flavor='raw')
+ free_non_gc_object(self)
+
+ def flush(self):
+ if self.buf_count > 0:
+ bytes = self.buf_count * rffi.sizeof(rffi.LONG)
+ count = raw_os_write(self.fd,
+ rffi.cast(llmemory.Address, self.writebuffer),
+ rffi.cast(rffi.SIZE_T, bytes))
+ if rffi.cast(lltype.Signed, count) != bytes:
+ raise OSError(rposix.get_errno(), "raw_os_write failed")
+ self.buf_count = 0
+ flush._dont_inline_ = True
+
+ def write(self, value):
+ x = self.buf_count
+ self.writebuffer[x] = value
+ x += 1
+ self.buf_count = x
+ if x == self.BUFSIZE:
+ self.flush()
+ write._always_inline_ = True
+
+ def writeobj(self, obj):
+ gc = self.gc
+ typeid = gc.get_type_id(obj)
+ self.write(llmemory.cast_adr_to_int(obj))
+ self.write(gc.get_member_index(typeid))
+ self.write(gc.get_size_incl_hash(obj))
+ gc.trace(obj, self._writeref, None)
+ self.write(-1)
+
+ def _writeref(self, pointer, _):
+ obj = pointer.address[0]
+ self.write(llmemory.cast_adr_to_int(obj))
+ self.add(obj)
+
+ def add(self, obj):
+ if not self.seen.contains(obj):
+ self.seen.setitem(obj, obj)
+ self.pending.append(obj)
+
+ def add_roots(self):
+ self.gc._heap_dumper = self
+ self.gc.root_walker.walk_roots(
+ _hd_add_root,
+ _hd_add_root,
+ _hd_add_root)
+ self.gc._heap_dumper = None
+
+ def walk(self):
+ while self.pending.non_empty():
+ self.writeobj(self.pending.pop())
+
+def _hd_add_root(gc, root):
+ gc._heap_dumper.add(root.address[0])
+
+def dump_rpy_heap(gc, fd):
+ heapdumper = HeapDumper(gc, fd)
+ heapdumper.add_roots()
+ heapdumper.walk()
+ heapdumper.flush()
+ heapdumper.delete()
Modified: pypy/branch/gc-module/pypy/rpython/memory/gc/markcompact.py
==============================================================================
--- pypy/branch/gc-module/pypy/rpython/memory/gc/markcompact.py (original)
+++ pypy/branch/gc-module/pypy/rpython/memory/gc/markcompact.py Fri Sep 10 17:07:16 2010
@@ -674,8 +674,7 @@
return llmemory.cast_adr_to_int(obj) # not in an arena...
return adr - self.space
- def get_rpy_memory_usage(self, gcref):
- obj = llmemory.cast_ptr_to_adr(gcref)
+ def get_size_incl_hash(self, obj):
size = self.get_size(obj)
hdr = self.header(obj)
if hdr.tid & GCFLAG_HASHFIELD:
Modified: pypy/branch/gc-module/pypy/rpython/memory/gc/semispace.py
==============================================================================
--- pypy/branch/gc-module/pypy/rpython/memory/gc/semispace.py (original)
+++ pypy/branch/gc-module/pypy/rpython/memory/gc/semispace.py Fri Sep 10 17:07:16 2010
@@ -331,9 +331,6 @@
size += llmemory.sizeof(lltype.Signed)
return size
- def get_rpy_memory_usage(self, gcref):
- return self.get_size_incl_hash(llmemory.cast_ptr_to_adr(gcref))
-
def scan_copied(self, scan):
while scan < self.free:
curr = scan + self.size_gc_header()
Modified: pypy/branch/gc-module/pypy/rpython/memory/gctransform/framework.py
==============================================================================
--- pypy/branch/gc-module/pypy/rpython/memory/gctransform/framework.py (original)
+++ pypy/branch/gc-module/pypy/rpython/memory/gctransform/framework.py Fri Sep 10 17:07:16 2010
@@ -139,6 +139,8 @@
def __init__(self, translator):
from pypy.rpython.memory.gc.base import choose_gc_from_config
from pypy.rpython.memory.gc.base import ARRAY_TYPEID_MAP
+ from pypy.rpython.memory.gc import inspect
+
super(FrameworkGCTransformer, self).__init__(translator, inline=True)
if hasattr(self, 'GC_PARAMS'):
# for tests: the GC choice can be specified as class attributes
@@ -388,27 +390,30 @@
else:
self.id_ptr = None
- self.get_rpy_roots_ptr = getfn(GCClass.get_rpy_roots.im_func,
+ self.get_rpy_roots_ptr = getfn(inspect.get_rpy_roots,
[s_gc],
rgc.s_list_of_gcrefs(),
minimal_transform=False)
- self.get_rpy_referents_ptr = getfn(GCClass.get_rpy_referents.im_func,
+ self.get_rpy_referents_ptr = getfn(inspect.get_rpy_referents,
[s_gc, s_gcref],
rgc.s_list_of_gcrefs(),
minimal_transform=False)
- self.get_rpy_memory_usage_ptr = getfn(
- GCClass.get_rpy_memory_usage.im_func,
- [s_gc, s_gcref],
- annmodel.SomeInteger(),
- minimal_transform=False)
- self.get_rpy_type_index_ptr = getfn(GCClass.get_rpy_type_index.im_func,
+ self.get_rpy_memory_usage_ptr = getfn(inspect.get_rpy_memory_usage,
+ [s_gc, s_gcref],
+ annmodel.SomeInteger(),
+ minimal_transform=False)
+ self.get_rpy_type_index_ptr = getfn(inspect.get_rpy_type_index,
[s_gc, s_gcref],
annmodel.SomeInteger(),
minimal_transform=False)
- self.is_rpy_instance_ptr = getfn(GCClass.is_rpy_instance.im_func,
+ self.is_rpy_instance_ptr = getfn(inspect.is_rpy_instance,
[s_gc, s_gcref],
annmodel.SomeBool(),
minimal_transform=False)
+ self.dump_rpy_heap_ptr = getfn(inspect.dump_rpy_heap,
+ [s_gc, annmodel.SomeInteger()],
+ annmodel.s_None,
+ minimal_transform=False)
self.set_max_heap_size_ptr = getfn(GCClass.set_max_heap_size.im_func,
[s_gc,
@@ -944,6 +949,14 @@
resultvar=hop.spaceop.result)
self.pop_roots(hop, livevars)
+ def gct_gc_dump_rpy_heap(self, hop):
+ livevars = self.push_roots(hop)
+ [v_fd] = hop.spaceop.args
+ hop.genop("direct_call",
+ [self.dump_rpy_heap_ptr, self.c_const_gc, v_fd],
+ resultvar=hop.spaceop.result)
+ self.pop_roots(hop, livevars)
+
def gct_malloc_nonmovable_varsize(self, hop):
TYPE = hop.spaceop.result.concretetype
if self.gcdata.gc.can_malloc_nonmovable():
Modified: pypy/branch/gc-module/pypy/translator/c/test/test_newgc.py
==============================================================================
--- pypy/branch/gc-module/pypy/translator/c/test/test_newgc.py (original)
+++ pypy/branch/gc-module/pypy/translator/c/test/test_newgc.py Fri Sep 10 17:07:16 2010
@@ -1063,6 +1063,31 @@
def test_get_rpy_type_index(self):
self.run("get_rpy_type_index")
+ filename_dump = str(udir.join('test_dump_rpy_heap'))
+ def define_dump_rpy_heap(self):
+ U = lltype.GcStruct('U', ('x', lltype.Signed))
+ S = lltype.GcStruct('S', ('u', lltype.Ptr(U)))
+ A = lltype.GcArray(lltype.Ptr(S))
+ filename = self.filename_dump
+
+ def fn():
+ s = lltype.malloc(S)
+ s.u = lltype.malloc(U)
+ a = lltype.malloc(A, 1000)
+ s2 = lltype.malloc(S)
+ #
+ fd = os.open(filename, os.O_WRONLY | os.O_CREAT, 0666)
+ rgc.dump_rpy_heap(fd)
+ os.close(fd)
+ return 0
+
+ return fn
+
+ def test_dump_rpy_heap(self):
+ self.run("dump_rpy_heap")
+ assert os.path.exists(self.filename_dump)
+ assert os.path.getsize(self.filename_dump) > 0 # minimal test
+
class TestSemiSpaceGC(TestUsingFramework, snippet.SemiSpaceGCTestDefines):
gcpolicy = "semispace"
More information about the Pypy-commit
mailing list