[pypy-commit] pypy memory-accounting: * Review all the places that add memory pressure outside of numpy and cpyext
fijal
pypy.commits at gmail.com
Fri Sep 29 11:20:01 EDT 2017
Author: fijal
Branch: memory-accounting
Changeset: r92506:f8fd62a0f087
Date: 2017-09-29 17:19 +0200
http://bitbucket.org/pypy/pypy/changeset/f8fd62a0f087/
Log: * Review all the places that add memory pressure outside of numpy
and cpyext
* Add a test
* Fixes about cast_pointer in the presence of subclasses
* Write down the app-level interface to that
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -35,7 +35,7 @@
except ImportError:
assert '__pypy__' not in sys.builtin_module_names
newlist_hint = lambda sizehint: []
- add_memory_pressure = lambda size: None
+ add_memory_pressure = lambda size, obj: None
if sys.version_info[0] >= 3:
StandardError = Exception
@@ -153,9 +153,10 @@
factory = Connection if not factory else factory
# an sqlite3 db seems to be around 100 KiB at least (doesn't matter if
# backed by :memory: or a file)
- add_memory_pressure(100 * 1024)
- return factory(database, timeout, detect_types, isolation_level,
+ res = factory(database, timeout, detect_types, isolation_level,
check_same_thread, factory, cached_statements)
+ add_memory_pressure(100 * 1024, res)
+ return res
def _unicode_text_factory(x):
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -142,11 +142,14 @@
space.newbool(debug))
@unwrap_spec(estimate=int)
-def add_memory_pressure(estimate):
+def add_memory_pressure(space, estimate, w_obj=None):
""" Add memory pressure of estimate bytes. Useful when calling a C function
that internally allocates a big chunk of memory. This instructs the GC to
garbage collect sooner than it would otherwise."""
- rgc.add_memory_pressure(estimate)
+ if space.is_none(w_obj):
+ rgc.add_memory_pressure(estimate)
+ else:
+ rgc.add_memory_pressure(estimate, w_obj)
@unwrap_spec(w_frame=PyFrame)
def locals_to_fast(space, w_frame):
diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py
--- a/pypy/module/_cffi_backend/allocator.py
+++ b/pypy/module/_cffi_backend/allocator.py
@@ -21,13 +21,13 @@
if self.w_alloc is None:
if self.should_clear_after_alloc:
ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
- flavor='raw', zero=True,
- add_memory_pressure=True)
+ flavor='raw', zero=True)
else:
ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
- flavor='raw', zero=False,
- add_memory_pressure=True)
- return cdataobj.W_CDataNewStd(space, ptr, ctype, length)
+ flavor='raw', zero=False)
+ w_res = cdataobj.W_CDataNewStd(space, ptr, ctype, length)
+ rgc.add_memory_pressure(datasize, w_res)
+ return w_res
else:
w_raw_cdata = space.call_function(self.w_alloc,
space.newint(datasize))
@@ -53,7 +53,7 @@
if self.w_free is not None:
res.w_free = self.w_free
res.register_finalizer(space)
- rgc.add_memory_pressure(datasize)
+ rgc.add_memory_pressure(datasize, res)
return res
@unwrap_spec(w_init=WrappedDefault(None))
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -447,7 +447,7 @@
with self as ptr:
w_res = W_CDataGCP(space, ptr, self.ctype, self, w_destructor)
if size != 0:
- rgc.add_memory_pressure(size)
+ rgc.add_memory_pressure(size, w_res)
return w_res
def unpack(self, length):
diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -61,7 +61,8 @@
ctx = ropenssl.EVP_MD_CTX_new()
if ctx is None:
raise MemoryError
- rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size)
+ rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size,
+ self)
try:
if copy_from:
if not ropenssl.EVP_MD_CTX_copy(ctx, copy_from):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1315,8 +1315,8 @@
if not ctx:
raise ssl_error(space, "failed to allocate SSL context")
- rgc.add_memory_pressure(10 * 1024)
self = space.allocate_instance(_SSLContext, w_subtype)
+ rgc.add_memory_pressure(10 * 1024, self)
self.ctx = ctx
self.check_hostname = False
self.register_finalizer(space)
diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py
--- a/pypy/module/gc/__init__.py
+++ b/pypy/module/gc/__init__.py
@@ -28,6 +28,7 @@
'get_objects': 'referents.get_objects',
'get_referents': 'referents.get_referents',
'get_referrers': 'referents.get_referrers',
+ 'get_stats': 'referents.get_stats',
'_dump_rpy_heap': 'referents._dump_rpy_heap',
'get_typeids_z': 'referents.get_typeids_z',
'get_typeids_list': 'referents.get_typeids_list',
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -1,6 +1,6 @@
from rpython.rlib import rgc
from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.typedef import TypeDef, interp_attrproperty
from pypy.interpreter.gateway import unwrap_spec
from pypy.interpreter.error import oefmt, wrap_oserror
from rpython.rlib.objectmodel import we_are_translated
@@ -170,3 +170,15 @@
l = rgc.get_typeids_list()
list_w = [space.newint(l[i]) for i in range(len(l))]
return space.newlist(list_w)
+
+class W_GcStats(W_Root):
+ def __init__(self):
+ self.total_memory_pressure = rgc.get_stats(rgc.TOTAL_MEMORY_PRESSURE)
+
+W_GcStats.typedef = TypeDef("GcStats",
+ total_memory_pressure=interp_attrproperty("total_memory_pressure",
+ cls=W_GcStats, wrapfn="newint"),
+)
+
+def get_stats(space):
+ return W_GcStats()
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -840,11 +840,11 @@
# Currently this is just the size of the pointer and some estimated bytes.
# The struct isn't actually defined in expat.h - it is in xmlparse.c
# XXX: find a good estimate of the XML_ParserStruct
- rgc.add_memory_pressure(XML_Parser_SIZE + 300)
if not xmlparser:
raise oefmt(space.w_RuntimeError, "XML_ParserCreate failed")
parser = W_XMLParserType(space, xmlparser, w_intern)
+ rgc.add_memory_pressure(XML_Parser_SIZE + 300, parser)
XML_SetUnknownEncodingHandler(
parser.itself, UnknownEncodingHandlerData_callback,
rffi.cast(rffi.VOIDP, parser.id))
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -675,6 +675,8 @@
def _ll_1_gc_add_memory_pressure(num):
llop.gc_add_memory_pressure(lltype.Void, num)
+ def _ll_2_gc_add_memory_pressure(num, obj):
+ llop.gc_add_memory_pressure(lltype.Void, num, obj)
def setup_extra_builtin(rtyper, oopspec_name, nb_args, extra=None):
diff --git a/rpython/memory/gc/inspector.py b/rpython/memory/gc/inspector.py
--- a/rpython/memory/gc/inspector.py
+++ b/rpython/memory/gc/inspector.py
@@ -2,6 +2,7 @@
Utility RPython functions to inspect objects in the GC.
"""
from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, llgroup
+from rpython.rtyper.lltypesystem.lloperation import llop
from rpython.rlib.objectmodel import free_non_gc_object
from rpython.rlib import rposix, rgc, jit
@@ -187,6 +188,11 @@
ofs = gc.get_memory_pressure_ofs(typeid)
val = (obj + ofs).signed[0]
self.count += val
+ gc.trace(obj, self._ref, None)
+
+ def _ref(self, pointer, _):
+ obj = pointer.address[0]
+ self.add(obj)
class HeapDumper(BaseWalker):
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -839,12 +839,21 @@
gct_fv_gc_malloc_varsize = gct_fv_gc_malloc
def gct_gc_add_memory_pressure(self, hop):
+ def _find_correct_type(TP):
+ T = TP.TO
+ while 'special_memory_pressure' not in T._flds:
+ T = T._flds['super']
+ return T
+
if hasattr(self, 'raw_malloc_memory_pressure_ptr'):
op = hop.spaceop
size = op.args[0]
if len(op.args) == 2:
v_fld = rmodel.inputconst(lltype.Void, "special_memory_pressure")
- hop.genop("bare_setfield", [op.args[1], v_fld, size])
+ T = _find_correct_type(op.args[1].concretetype)
+ v_inst = hop.genop("cast_pointer", [op.args[1]],
+ resulttype=lltype.Ptr(T))
+ hop.genop("bare_setfield", [v_inst, v_fld, size])
v_adr = hop.genop("cast_ptr_to_adr", [op.args[1]],
resulttype=llmemory.Address)
else:
diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -85,7 +85,11 @@
def allocate_lock():
- return Lock(allocate_ll_lock())
+ # Add some memory pressure for the size of the lock because it is an
+ # Opaque object
+ lock = Lock(allocate_ll_lock())
+ rgc.add_memory_pressure(TLOCKP_SIZE, lock)
+ return lock
@specialize.arg(0)
def ll_start_new_thread(func):
@@ -242,9 +246,6 @@
if rffi.cast(lltype.Signed, res) <= 0:
lltype.free(ll_lock, flavor='raw', track_allocation=False)
raise error("out of resources")
- # Add some memory pressure for the size of the lock because it is an
- # Opaque object
- rgc.add_memory_pressure(TLOCKP_SIZE)
return ll_lock
def free_ll_lock(ll_lock):
diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py
--- a/rpython/rlib/rzlib.py
+++ b/rpython/rlib/rzlib.py
@@ -269,7 +269,6 @@
compress data.
"""
stream = lltype.malloc(z_stream, flavor='raw', zero=True)
- rgc.add_memory_pressure(rffi.sizeof(z_stream))
err = _deflateInit2(stream, level, method, wbits, memLevel, strategy)
if err == Z_OK:
if zdict is not None:
@@ -304,7 +303,6 @@
decompress data.
"""
stream = lltype.malloc(z_stream, flavor='raw', zero=True)
- rgc.add_memory_pressure(rffi.sizeof(z_stream))
err = _inflateInit2(stream, wbits)
if err == Z_OK:
if zdict is not None and wbits < 0:
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1672,6 +1672,38 @@
class TestIncrementalMiniMarkGC(TestMiniMarkGC):
gcpolicy = "incminimark"
+ def define_total_memory_pressure(cls):
+ class A(object):
+ def __init__(self):
+ rgc.add_memory_pressure(10, self)
+
+ def __del__(self):
+ pass
+
+ class B(A):
+ def __init__(self):
+ rgc.add_memory_pressure(10, self)
+
+ class C(A):
+ pass
+
+ class Glob(object):
+ pass
+ glob = Glob()
+
+ def f():
+ glob.l = [None] * 3
+ for i in range(10000):
+ glob.l[i % 3] = A()
+ glob.l[(i + 1) % 3] = B()
+ glob.l[(i + 2) % 3] = C()
+ return rgc.get_stats(rgc.TOTAL_MEMORY_PRESSURE)
+ return f
+
+ def test_total_memory_pressure(self):
+ res = self.run("total_memory_pressure")
+ assert res == 30 # total reachable is 3
+
def define_random_pin(self):
class A:
foo = None
More information about the pypy-commit
mailing list