[pypy-svn] r76926 - in pypy/branch/jit-bounds: . lib_pypy/_ctypes pypy/annotation pypy/annotation/test pypy/bin pypy/interpreter pypy/jit/backend/llsupport pypy/jit/codewriter pypy/jit/metainterp/optimizeopt pypy/module/__builtin__ pypy/module/__builtin__/test pypy/module/_socket/test pypy/module/array/benchmark pypy/module/array/test pypy/module/cpyext pypy/module/posix/test pypy/objspace/std pypy/rlib pypy/rpython pypy/rpython/lltypesystem pypy/rpython/lltypesystem/test pypy/rpython/memory pypy/rpython/memory/gc pypy/rpython/memory/gc/test pypy/rpython/memory/test pypy/rpython/module pypy/rpython/ootypesystem pypy/rpython/test pypy/translator pypy/translator/backendopt/test pypy/translator/c pypy/translator/c/gcc pypy/translator/c/test
hakanardo at codespeak.net
hakanardo at codespeak.net
Tue Sep 7 19:22:37 CEST 2010
Author: hakanardo
Date: Tue Sep 7 19:22:31 2010
New Revision: 76926
Modified:
pypy/branch/jit-bounds/ (props changed)
pypy/branch/jit-bounds/lib_pypy/_ctypes/array.py
pypy/branch/jit-bounds/lib_pypy/_ctypes/function.py
pypy/branch/jit-bounds/lib_pypy/_ctypes/primitive.py
pypy/branch/jit-bounds/pypy/annotation/binaryop.py
pypy/branch/jit-bounds/pypy/annotation/bookkeeper.py
pypy/branch/jit-bounds/pypy/annotation/builtin.py
pypy/branch/jit-bounds/pypy/annotation/classdef.py
pypy/branch/jit-bounds/pypy/annotation/model.py
pypy/branch/jit-bounds/pypy/annotation/test/test_annrpython.py
pypy/branch/jit-bounds/pypy/bin/py.py
pypy/branch/jit-bounds/pypy/interpreter/argument.py
pypy/branch/jit-bounds/pypy/interpreter/baseobjspace.py
pypy/branch/jit-bounds/pypy/interpreter/pycode.py
pypy/branch/jit-bounds/pypy/jit/backend/llsupport/gc.py
pypy/branch/jit-bounds/pypy/jit/codewriter/jtransform.py
pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt/optimizer.py (props changed)
pypy/branch/jit-bounds/pypy/module/__builtin__/interp_classobj.py
pypy/branch/jit-bounds/pypy/module/__builtin__/test/test_classobj.py
pypy/branch/jit-bounds/pypy/module/_socket/test/test_sock_app.py
pypy/branch/jit-bounds/pypy/module/array/benchmark/Makefile (props changed)
pypy/branch/jit-bounds/pypy/module/array/benchmark/intimg.c (props changed)
pypy/branch/jit-bounds/pypy/module/array/benchmark/intimgtst.c (props changed)
pypy/branch/jit-bounds/pypy/module/array/benchmark/intimgtst.py (props changed)
pypy/branch/jit-bounds/pypy/module/array/benchmark/loop.c (props changed)
pypy/branch/jit-bounds/pypy/module/array/benchmark/sum.c (props changed)
pypy/branch/jit-bounds/pypy/module/array/benchmark/sumtst.c (props changed)
pypy/branch/jit-bounds/pypy/module/array/benchmark/sumtst.py (props changed)
pypy/branch/jit-bounds/pypy/module/array/test/test_array_old.py (props changed)
pypy/branch/jit-bounds/pypy/module/cpyext/api.py
pypy/branch/jit-bounds/pypy/module/posix/test/test_posix2.py
pypy/branch/jit-bounds/pypy/objspace/std/objspace.py
pypy/branch/jit-bounds/pypy/objspace/std/tupleobject.py
pypy/branch/jit-bounds/pypy/rlib/rarithmetic.py
pypy/branch/jit-bounds/pypy/rlib/rsocket.py
pypy/branch/jit-bounds/pypy/rlib/rweakref.py
pypy/branch/jit-bounds/pypy/rpython/lltypesystem/llgroup.py
pypy/branch/jit-bounds/pypy/rpython/lltypesystem/llmemory.py
pypy/branch/jit-bounds/pypy/rpython/lltypesystem/lloperation.py
pypy/branch/jit-bounds/pypy/rpython/lltypesystem/lltype.py
pypy/branch/jit-bounds/pypy/rpython/lltypesystem/opimpl.py
pypy/branch/jit-bounds/pypy/rpython/lltypesystem/rffi.py
pypy/branch/jit-bounds/pypy/rpython/lltypesystem/test/test_llgroup.py
pypy/branch/jit-bounds/pypy/rpython/lltypesystem/test/test_lloperation.py
pypy/branch/jit-bounds/pypy/rpython/lltypesystem/test/test_lltype.py
pypy/branch/jit-bounds/pypy/rpython/memory/gc/base.py
pypy/branch/jit-bounds/pypy/rpython/memory/gc/generation.py
pypy/branch/jit-bounds/pypy/rpython/memory/gc/markcompact.py
pypy/branch/jit-bounds/pypy/rpython/memory/gc/test/test_direct.py
pypy/branch/jit-bounds/pypy/rpython/memory/gctypelayout.py
pypy/branch/jit-bounds/pypy/rpython/memory/gcwrapper.py
pypy/branch/jit-bounds/pypy/rpython/memory/test/test_gc.py
pypy/branch/jit-bounds/pypy/rpython/memory/test/test_gctypelayout.py
pypy/branch/jit-bounds/pypy/rpython/memory/test/test_transformed_gc.py
pypy/branch/jit-bounds/pypy/rpython/module/ll_time.py
pypy/branch/jit-bounds/pypy/rpython/ootypesystem/ootype.py
pypy/branch/jit-bounds/pypy/rpython/ootypesystem/rclass.py
pypy/branch/jit-bounds/pypy/rpython/rbuiltin.py
pypy/branch/jit-bounds/pypy/rpython/rclass.py
pypy/branch/jit-bounds/pypy/rpython/test/test_rclass.py
pypy/branch/jit-bounds/pypy/translator/backendopt/test/test_constfold.py
pypy/branch/jit-bounds/pypy/translator/c/database.py
pypy/branch/jit-bounds/pypy/translator/c/gc.py
pypy/branch/jit-bounds/pypy/translator/c/gcc/trackgcroot.py
pypy/branch/jit-bounds/pypy/translator/c/node.py
pypy/branch/jit-bounds/pypy/translator/c/test/test_newgc.py
pypy/branch/jit-bounds/pypy/translator/exceptiontransform.py
Log:
svn merge -r76816:r76924 svn+ssh://hakanardo@codespeak.net/svn/pypy/trunk
Modified: pypy/branch/jit-bounds/lib_pypy/_ctypes/array.py
==============================================================================
--- pypy/branch/jit-bounds/lib_pypy/_ctypes/array.py (original)
+++ pypy/branch/jit-bounds/lib_pypy/_ctypes/array.py Tue Sep 7 19:22:31 2010
@@ -75,7 +75,7 @@
def _CData_output(self, resarray, base=None, index=-1):
# this seems to be a string if we're array of char, surprise!
- from ctypes import c_char, c_wchar, c_char_p, c_wchar_p
+ from ctypes import c_char, c_wchar
if self._type_ is c_char:
return _rawffi.charp2string(resarray.buffer, self._length_)
if self._type_ is c_wchar:
Modified: pypy/branch/jit-bounds/lib_pypy/_ctypes/function.py
==============================================================================
--- pypy/branch/jit-bounds/lib_pypy/_ctypes/function.py (original)
+++ pypy/branch/jit-bounds/lib_pypy/_ctypes/function.py Tue Sep 7 19:22:31 2010
@@ -60,7 +60,6 @@
return self._restype_
def _setrestype(self, restype):
self._ptr = None
- from ctypes import c_char_p
if restype is int:
from ctypes import c_int
restype = c_int
@@ -214,9 +213,7 @@
@staticmethod
def _guess_argtypes(args):
- from _ctypes import _CData
from ctypes import c_char_p, c_wchar_p, c_void_p, c_int
- from ctypes import Array, Structure
res = []
for arg in args:
if hasattr(arg, '_as_parameter_'):
Modified: pypy/branch/jit-bounds/lib_pypy/_ctypes/primitive.py
==============================================================================
--- pypy/branch/jit-bounds/lib_pypy/_ctypes/primitive.py (original)
+++ pypy/branch/jit-bounds/lib_pypy/_ctypes/primitive.py Tue Sep 7 19:22:31 2010
@@ -57,7 +57,6 @@
pyobj_container = GlobalPyobjContainer()
def generic_xxx_p_from_param(cls, value):
- from _ctypes import Array, _Pointer
if value is None:
return cls(None)
if isinstance(value, basestring):
@@ -119,8 +118,6 @@
result._ffiarray = ffiarray
if tp == 'z':
# c_char_p
- from _ctypes import Array, _Pointer
-
def _getvalue(self):
addr = self._buffer[0]
if addr == 0:
@@ -143,7 +140,7 @@
result.value = property(_getvalue, _setvalue)
elif tp == 'Z':
# c_wchar_p
- from _ctypes import Array, _Pointer, _wstring_at
+ from _ctypes import _wstring_at
def _getvalue(self):
addr = self._buffer[0]
if addr == 0:
Modified: pypy/branch/jit-bounds/pypy/annotation/binaryop.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/annotation/binaryop.py (original)
+++ pypy/branch/jit-bounds/pypy/annotation/binaryop.py Tue Sep 7 19:22:31 2010
@@ -924,10 +924,10 @@
class __extend__(pairtype(SomeAddress, SomeAddress)):
def union((s_addr1, s_addr2)):
- return SomeAddress(is_null=s_addr1.is_null and s_addr2.is_null)
+ return SomeAddress()
def sub((s_addr1, s_addr2)):
- if s_addr1.is_null and s_addr2.is_null:
+ if s_addr1.is_null_address() and s_addr2.is_null_address():
return getbookkeeper().immutablevalue(0)
return SomeInteger()
@@ -953,10 +953,10 @@
class __extend__(pairtype(SomeAddress, SomeInteger)):
def add((s_addr, s_int)):
- return SomeAddress(is_null=False)
+ return SomeAddress()
def sub((s_addr, s_int)):
- return SomeAddress(is_null=False)
+ return SomeAddress()
class __extend__(pairtype(SomeAddress, SomeImpossibleValue)):
# need to override this specifically to hide the 'raise UnionError'
Modified: pypy/branch/jit-bounds/pypy/annotation/bookkeeper.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/annotation/bookkeeper.py (original)
+++ pypy/branch/jit-bounds/pypy/annotation/bookkeeper.py Tue Sep 7 19:22:31 2010
@@ -338,8 +338,12 @@
result = SomeBool()
elif tp is int:
result = SomeInteger(nonneg = x>=0)
- elif tp is long and 0 <= x <= (sys.maxint * 2 + 1):
- result = SomeInteger(unsigned = True)
+ elif tp is long:
+ if -sys.maxint-1 <= x <= sys.maxint:
+ x = int(x)
+ result = SomeInteger(nonneg = x>=0)
+ else:
+ raise Exception("seeing a prebuilt long (value %s)" % hex(x))
elif issubclass(tp, str): # py.lib uses annotated str subclasses
if len(x) == 1:
result = SomeChar()
@@ -431,7 +435,7 @@
elif isinstance(x, lltype._ptr):
result = SomePtr(lltype.typeOf(x))
elif isinstance(x, llmemory.fakeaddress):
- result = SomeAddress(is_null=not x)
+ result = SomeAddress()
elif isinstance(x, ootype._static_meth):
result = SomeOOStaticMeth(ootype.typeOf(x))
elif isinstance(x, ootype._class):
Modified: pypy/branch/jit-bounds/pypy/annotation/builtin.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/annotation/builtin.py (original)
+++ pypy/branch/jit-bounds/pypy/annotation/builtin.py Tue Sep 7 19:22:31 2010
@@ -92,6 +92,8 @@
return s_obj.is_true()
def builtin_int(s_obj, s_base=None):
+ if isinstance(s_obj, SomeInteger):
+ assert not s_obj.unsigned, "instead of int(r_uint(x)), use intmask(r_uint(x))"
assert (s_base is None or isinstance(s_base, SomeInteger)
and s_obj.knowntype == str), "only int(v|string) or int(string,int) expected"
if s_base is not None:
@@ -694,18 +696,14 @@
def raw_free(s_addr):
assert isinstance(s_addr, SomeAddress)
- assert not s_addr.is_null
def raw_memclear(s_addr, s_int):
assert isinstance(s_addr, SomeAddress)
- assert not s_addr.is_null
assert isinstance(s_int, SomeInteger)
def raw_memcopy(s_addr1, s_addr2, s_int):
assert isinstance(s_addr1, SomeAddress)
- assert not s_addr1.is_null
assert isinstance(s_addr2, SomeAddress)
- assert not s_addr2.is_null
assert isinstance(s_int, SomeInteger) #XXX add noneg...?
BUILTIN_ANALYZERS[llmemory.raw_malloc] = raw_malloc
Modified: pypy/branch/jit-bounds/pypy/annotation/classdef.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/annotation/classdef.py (original)
+++ pypy/branch/jit-bounds/pypy/annotation/classdef.py Tue Sep 7 19:22:31 2010
@@ -276,6 +276,8 @@
# create the Attribute and do the generalization asked for
newattr = Attribute(attr, self.bookkeeper)
if s_value:
+ if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
+ import pdb; pdb.set_trace()
newattr.s_value = s_value
# keep all subattributes' values
Modified: pypy/branch/jit-bounds/pypy/annotation/model.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/annotation/model.py (original)
+++ pypy/branch/jit-bounds/pypy/annotation/model.py Tue Sep 7 19:22:31 2010
@@ -500,12 +500,13 @@
class SomeAddress(SomeObject):
immutable = True
- def __init__(self, is_null=False):
- self.is_null = is_null
def can_be_none(self):
return False
+ def is_null_address(self):
+ return self.is_immutable_constant() and not self.const
+
# The following class is used to annotate the intermediate value that
# appears in expressions of the form:
# addr.signed[offset] and addr.signed[offset] = value
Modified: pypy/branch/jit-bounds/pypy/annotation/test/test_annrpython.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/annotation/test/test_annrpython.py (original)
+++ pypy/branch/jit-bounds/pypy/annotation/test/test_annrpython.py Tue Sep 7 19:22:31 2010
@@ -767,7 +767,6 @@
assert s.classdef is a.bookkeeper.getuniqueclassdef(IndexError) # KeyError ignored because l is a list
def test_overrides(self):
- import sys
excs = []
def record_exc(e):
"""NOT_RPYTHON"""
@@ -869,8 +868,27 @@
def f():
return large_constant
a = self.RPythonAnnotator()
+ py.test.raises(Exception, a.build_types, f, [])
+ # if you want to get a r_uint, you have to be explicit about it
+
+ def test_prebuilt_long_that_is_not_too_long(self):
+ small_constant = 12L
+ def f():
+ return small_constant
+ a = self.RPythonAnnotator()
s = a.build_types(f, [])
- assert s.knowntype == r_uint
+ assert s.const == 12
+ assert s.nonneg
+ assert not s.unsigned
+ #
+ small_constant = -23L
+ def f():
+ return small_constant
+ a = self.RPythonAnnotator()
+ s = a.build_types(f, [])
+ assert s.const == -23
+ assert not s.nonneg
+ assert not s.unsigned
def test_pbc_getattr(self):
class C:
@@ -1386,7 +1404,6 @@
assert isinstance(a.binding(ev), annmodel.SomeInstance) and a.binding(ev).classdef == a.bookkeeper.getuniqueclassdef(Exception)
def test_sys_attrs(self):
- import sys
def f():
return sys.argv[0]
a = self.RPythonAnnotator()
Modified: pypy/branch/jit-bounds/pypy/bin/py.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/bin/py.py (original)
+++ pypy/branch/jit-bounds/pypy/bin/py.py Tue Sep 7 19:22:31 2010
@@ -76,6 +76,8 @@
config.objspace.suggest(allworkingmodules=False)
if config.objspace.allworkingmodules:
pypyoption.enable_allworkingmodules(config)
+ if config.objspace.usemodules.thread:
+ config.translation.thread = True
# create the object space
Modified: pypy/branch/jit-bounds/pypy/interpreter/argument.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/interpreter/argument.py (original)
+++ pypy/branch/jit-bounds/pypy/interpreter/argument.py Tue Sep 7 19:22:31 2010
@@ -52,11 +52,15 @@
self.argnames, self.varargname, self.kwargname)
def __eq__(self, other):
+ if not isinstance(other, Signature):
+ return NotImplemented
return (self.argnames == other.argnames and
self.varargname == other.varargname and
self.kwargname == other.kwargname)
def __ne__(self, other):
+ if not isinstance(other, Signature):
+ return NotImplemented
return not self == other
Modified: pypy/branch/jit-bounds/pypy/interpreter/baseobjspace.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/interpreter/baseobjspace.py (original)
+++ pypy/branch/jit-bounds/pypy/interpreter/baseobjspace.py Tue Sep 7 19:22:31 2010
@@ -288,6 +288,7 @@
self.timer.stop("startup " + modname)
def finish(self):
+ self.wait_for_thread_shutdown()
w_exitfunc = self.sys.getdictvalue(self, 'exitfunc')
if w_exitfunc is not None:
self.call_function(w_exitfunc)
@@ -305,6 +306,23 @@
for s in self.FrameClass._space_op_types:
print s
+ def wait_for_thread_shutdown(self):
+ """Wait until threading._shutdown() completes, provided the threading
+ module was imported in the first place. The shutdown routine will
+ wait until all non-daemon 'threading' threads have completed."""
+ if not self.config.translation.thread:
+ return
+
+ w_modules = self.sys.get('modules')
+ w_mod = self.finditem_str(w_modules, 'threading')
+ if w_mod is None:
+ return
+
+ try:
+ self.call_method(w_mod, "_shutdown")
+ except OperationError, e:
+ e.write_unraisable(self, "threading._shutdown()")
+
def reportbytecodecounts(self):
os.write(2, "Starting bytecode report.\n")
fd = os.open('bytecode.txt', os.O_CREAT|os.O_WRONLY|os.O_TRUNC, 0644)
Modified: pypy/branch/jit-bounds/pypy/interpreter/pycode.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/interpreter/pycode.py (original)
+++ pypy/branch/jit-bounds/pypy/interpreter/pycode.py Tue Sep 7 19:22:31 2010
@@ -4,7 +4,7 @@
The bytecode interpreter itself is implemented by the PyFrame class.
"""
-import dis, imp, struct, types, new
+import dis, imp, struct, types, new, sys
from pypy.interpreter import eval
from pypy.interpreter.argument import Signature
@@ -118,7 +118,8 @@
self._compute_flatcall()
def _freeze_(self):
- if self.magic == cpython_magic:
+ if (self.magic == cpython_magic and
+ '__pypy__' not in sys.builtin_module_names):
raise Exception("CPython host codes should not be rendered")
return False
Modified: pypy/branch/jit-bounds/pypy/jit/backend/llsupport/gc.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/jit/backend/llsupport/gc.py (original)
+++ pypy/branch/jit-bounds/pypy/jit/backend/llsupport/gc.py Tue Sep 7 19:22:31 2010
@@ -328,7 +328,7 @@
DEBUG = False # forced to True by x86/test/test_zrpy_gc.py
def __init__(self, gcdescr, translator, rtyper, llop1=llop):
- from pypy.rpython.memory.gctypelayout import _check_typeid
+ from pypy.rpython.memory.gctypelayout import check_typeid
from pypy.rpython.memory.gcheader import GCHeaderBuilder
from pypy.rpython.memory.gctransform import framework
GcLLDescription.__init__(self, gcdescr, translator, rtyper)
@@ -375,7 +375,7 @@
def malloc_basic(size, tid):
type_id = llop.extract_ushort(llgroup.HALFWORD, tid)
has_finalizer = bool(tid & (1<<llgroup.HALFSHIFT))
- _check_typeid(type_id)
+ check_typeid(type_id)
try:
res = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
type_id, size, True,
@@ -395,7 +395,7 @@
#
def malloc_array(itemsize, tid, num_elem):
type_id = llop.extract_ushort(llgroup.HALFWORD, tid)
- _check_typeid(type_id)
+ check_typeid(type_id)
try:
return llop1.do_malloc_varsize_clear(
llmemory.GCREF,
@@ -482,7 +482,7 @@
def init_size_descr(self, S, descr):
type_id = self.layoutbuilder.get_type_id(S)
- assert not self.layoutbuilder.is_weakref(type_id)
+ assert not self.layoutbuilder.is_weakref_type(S)
has_finalizer = bool(self.layoutbuilder.has_finalizer(S))
flags = int(has_finalizer) << llgroup.HALFSHIFT
descr.tid = llop.combine_ushort(lltype.Signed, type_id, flags)
Modified: pypy/branch/jit-bounds/pypy/jit/codewriter/jtransform.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/jit/codewriter/jtransform.py (original)
+++ pypy/branch/jit-bounds/pypy/jit/codewriter/jtransform.py Tue Sep 7 19:22:31 2010
@@ -511,14 +511,11 @@
arraydescr)
return []
# check for deepfrozen structures that force constant-folding
- hints = v_inst.concretetype.TO._hints
- accessor = hints.get("immutable_fields")
- if accessor and c_fieldname.value in accessor.fields:
+ immut = v_inst.concretetype.TO._immutable_field(c_fieldname.value)
+ if immut:
pure = '_pure'
- if accessor.fields[c_fieldname.value] == "[*]":
+ if immut == "[*]":
self.immutable_arrays[op.result] = True
- elif hints.get('immutable'):
- pure = '_pure'
else:
pure = ''
argname = getattr(v_inst.concretetype.TO, '_gckind', 'gc')
Modified: pypy/branch/jit-bounds/pypy/module/__builtin__/interp_classobj.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/module/__builtin__/interp_classobj.py (original)
+++ pypy/branch/jit-bounds/pypy/module/__builtin__/interp_classobj.py Tue Sep 7 19:22:31 2010
@@ -201,7 +201,7 @@
w_inst = W_InstanceObjectWithDel(space, self)
else:
w_inst = W_InstanceObject(space, self)
- w_init = w_inst.getattr(space, space.wrap('__init__'), False)
+ w_init = w_inst.getattr_from_class(space, space.wrap('__init__'))
if w_init is not None:
w_result = space.call_args(w_init, __args__)
if not space.is_w(w_result, space.w_None):
@@ -337,25 +337,44 @@
space.wrap("__class__ must be set to a class"))
self.w_class = w_class
-
- def getattr(self, space, w_name, exc=True):
- w_result = space.finditem(self.w_dict, w_name)
- if w_result is not None:
- return w_result
+ def getattr_from_class(self, space, w_name):
+ # Look up w_name in the class dict, and call its __get__.
+ # This method ignores the instance dict and the __getattr__.
+ # Returns None if not found.
w_value = self.w_class.lookup(space, w_name)
if w_value is None:
- if exc:
- raise operationerrfmt(
- space.w_AttributeError,
- "%s instance has no attribute '%s'",
- self.w_class.name, space.str_w(w_name))
- else:
- return None
+ return None
w_descr_get = space.lookup(w_value, '__get__')
if w_descr_get is None:
return w_value
return space.call_function(w_descr_get, w_value, self, self.w_class)
+ def getattr(self, space, w_name, exc=True):
+ # Normal getattr rules: look up w_name in the instance dict,
+ # in the class dict, and then via a call to __getatttr__.
+ w_result = space.finditem(self.w_dict, w_name)
+ if w_result is not None:
+ return w_result
+ w_result = self.getattr_from_class(space, w_name)
+ if w_result is not None:
+ return w_result
+ w_meth = self.getattr_from_class(space, space.wrap('__getattr__'))
+ if w_meth is not None:
+ try:
+ return space.call_function(w_meth, w_name)
+ except OperationError, e:
+ if not exc and e.match(space, space.w_AttributeError):
+ return None # eat the AttributeError
+ raise
+ # not found at all
+ if exc:
+ raise operationerrfmt(
+ space.w_AttributeError,
+ "%s instance has no attribute '%s'",
+ self.w_class.name, space.str_w(w_name))
+ else:
+ return None
+
def descr_getattribute(self, space, w_attr):
name = space.str_w(w_attr)
if len(name) >= 8 and name[0] == '_':
@@ -363,19 +382,11 @@
return self.w_dict
elif name == "__class__":
return self.w_class
- try:
- return self.getattr(space, w_attr)
- except OperationError, e:
- if not e.match(space, space.w_AttributeError):
- raise
- w_meth = self.getattr(space, space.wrap('__getattr__'), False)
- if w_meth is not None:
- return space.call_function(w_meth, w_attr)
- raise
+ return self.getattr(space, w_attr)
def descr_setattr(self, space, w_name, w_value):
name = unwrap_attr(space, w_name)
- w_meth = self.getattr(space, space.wrap('__setattr__'), False)
+ w_meth = self.getattr_from_class(space, space.wrap('__setattr__'))
if name and name[0] == "_":
if name == '__dict__':
self.setdict(space, w_value)
@@ -405,7 +416,7 @@
# use setclass to raise the error
self.setclass(space, None)
return
- w_meth = self.getattr(space, space.wrap('__delattr__'), False)
+ w_meth = self.getattr_from_class(space, space.wrap('__delattr__'))
if w_meth is not None:
space.call_function(w_meth, w_name)
else:
@@ -658,7 +669,10 @@
def descr_del(self, space):
# Note that this is called from executioncontext.UserDelAction
# via the space.userdel() method.
- w_func = self.getattr(space, space.wrap('__del__'), False)
+ w_name = space.wrap('__del__')
+ w_func = space.finditem(self.w_dict, w_name)
+ if w_func is None:
+ w_func = self.getattr_from_class(space, w_name)
if w_func is not None:
space.call_function(w_func)
Modified: pypy/branch/jit-bounds/pypy/module/__builtin__/test/test_classobj.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/module/__builtin__/test/test_classobj.py (original)
+++ pypy/branch/jit-bounds/pypy/module/__builtin__/test/test_classobj.py Tue Sep 7 19:22:31 2010
@@ -767,6 +767,168 @@
finally:
warnings.simplefilter('default', RuntimeWarning)
+ def test_special_method_via_getattr(self):
+ class A:
+ def __getattr__(self, attr):
+ print 'A getattr:', attr
+ def callable(*args):
+ print 'A called:', attr + repr(args)
+ return attr + repr(args)
+ return callable
+ class B:
+ def __getattr__(self, attr):
+ print 'B getattr:', attr
+ def callable(*args):
+ print 'B called:', attr, args
+ self.called = attr, args
+ if attr == '__coerce__':
+ return self, args[0]
+ return 42
+ return callable
+ a = A()
+ a.instancevalue = 42 # does not go via __getattr__('__setattr__')
+ a.__getattr__ = "hi there, ignore me, I'm in a"
+ a.__setattr__ = "hi there, ignore me, I'm in a too"
+ assert a.instancevalue == 42
+ A.classvalue = 123
+ assert a.classvalue == 123
+ assert a.foobar(5) == 'foobar(5,)'
+ assert a.__dict__ == {'instancevalue': 42,
+ '__getattr__': a.__getattr__,
+ '__setattr__': a.__setattr__}
+ assert a.__class__ is A
+ # This follows the Python 2.5 rules, more precisely.
+ # It is still valid in Python 2.7 too.
+ assert repr(a) == '__repr__()'
+ assert str(a) == '__str__()'
+ assert unicode(a) == u'__unicode__()'
+ b = B()
+ b.__getattr__ = "hi there, ignore me, I'm in b"
+ b.__setattr__ = "hi there, ignore me, I'm in b too"
+ assert 'called' not in b.__dict__ # and not e.g. ('__init__', ())
+ assert len(b) == 42
+ assert b.called == ('__len__', ())
+ assert a[5] == '__getitem__(5,)'
+ b[6] = 7
+ assert b.called == ('__setitem__', (6, 7))
+ del b[8]
+ assert b.called == ('__delitem__', (8,))
+ #
+ class C:
+ def __getattr__(self, name):
+ if name == '__iter__':
+ return lambda: iter([3, 33, 333])
+ raise AttributeError
+ assert list(iter(C())) == [3, 33, 333]
+ #
+ class C:
+ def __getattr__(self, name):
+ if name == '__getitem__':
+ return lambda n: [3, 33, 333][n]
+ raise AttributeError
+ assert list(iter(C())) == [3, 33, 333]
+ #
+ assert a[:6] == '__getslice__(0, 6)'
+ b[3:5] = 7
+ assert b.called == ('__setslice__', (3, 5, 7))
+ del b[:-1000]
+ assert b.called == ('__delslice__', (0, -958)) # adds len(b)...
+ assert a(5) == '__call__(5,)'
+ raises(TypeError, bool, a) # "should return an int"
+ assert not not b
+ #
+ class C:
+ def __getattr__(self, name):
+ if name == '__nonzero__':
+ return lambda: False
+ raise AttributeError
+ assert not C()
+ #
+ class C:
+ def __getattr__(self, name):
+ if name == '__len__':
+ return lambda: 0
+ raise AttributeError
+ assert not C()
+ #
+ #assert cmp(b, 43) == 0 # because __eq__(43) returns 42, so True...
+ # ... I will leave this case as XXX implement me
+ assert hash(b) == 42
+ assert range(100, 200)[b] == 142
+ assert "foo" in b
+ #
+ class C:
+ def __iter__(self):
+ return self
+ def __getattr__(self, name):
+ if name == 'next':
+ return lambda: 'the next item'
+ raise AttributeError
+ for x in C():
+ assert x == 'the next item'
+ break
+ #
+ # XXX a really corner case: '__del__'
+ #
+ import operator
+ op_by_name = {"neg": operator.neg,
+ "pos": operator.pos,
+ "abs": abs,
+ "invert": operator.invert,
+ "int": int,
+ "long": long}
+ for opname, opfunc in op_by_name.items():
+ assert opfunc(b) == 42
+ assert b.called == ("__" + opname + "__", ())
+ assert oct(a) == '__oct__()'
+ assert hex(a) == '__hex__()'
+ #
+ class C:
+ def __getattr__(self, name):
+ return lambda: 5.5
+ raises(TypeError, float, b)
+ assert float(C()) == 5.5
+ #
+ op_by_name = {'eq': operator.eq,
+ 'ne': operator.ne,
+ 'gt': operator.gt,
+ 'lt': operator.lt,
+ 'ge': operator.ge,
+ 'le': operator.le,
+ 'imod': operator.imod,
+ 'iand': operator.iand,
+ 'ipow': operator.ipow,
+ 'itruediv': operator.itruediv,
+ 'ilshift': operator.ilshift,
+ 'ixor': operator.ixor,
+ 'irshift': operator.irshift,
+ 'ifloordiv': operator.ifloordiv,
+ 'idiv': operator.idiv,
+ 'isub': operator.isub,
+ 'imul': operator.imul,
+ 'iadd': operator.iadd,
+ 'ior': operator.ior,
+ 'or': operator.or_,
+ 'and': operator.and_,
+ 'xor': operator.xor,
+ 'lshift': operator.lshift,
+ 'rshift': operator.rshift,
+ 'add': operator.add,
+ 'sub': operator.sub,
+ 'mul': operator.mul,
+ 'div': operator.div,
+ 'mod': operator.mod,
+ 'divmod': divmod,
+ 'floordiv': operator.floordiv,
+ 'truediv': operator.truediv}
+ for opname, opfunc in op_by_name.items():
+ assert opfunc(b, 5) == 42
+ assert b.called == ("__" + opname + "__", (5,))
+ x, y = coerce(b, 5)
+ assert x is b
+ assert y == 5
+
+
class AppTestOldStyleSharing(AppTestOldstyle):
def setup_class(cls):
cls.space = gettestobjspace(**{"objspace.std.withsharingdict": True})
Modified: pypy/branch/jit-bounds/pypy/module/_socket/test/test_sock_app.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/module/_socket/test/test_sock_app.py (original)
+++ pypy/branch/jit-bounds/pypy/module/_socket/test/test_sock_app.py Tue Sep 7 19:22:31 2010
@@ -221,6 +221,21 @@
"(_socket, host, port): return _socket.getaddrinfo(host, port)")
assert space.unwrap(w_l) == info
+def test_unknown_addr_as_object():
+ from pypy.rlib import rsocket
+ from pypy.rpython.lltypesystem import lltype, rffi
+
+ c_addr = lltype.malloc(rsocket._c.sockaddr, flavor='raw')
+ c_addr.c_sa_data[0] = 'c'
+ rffi.setintfield(c_addr, 'c_sa_family', 15)
+ # XXX what size to pass here? for the purpose of this test it has
+ # to be short enough so we have some data, 1 sounds good enough
+ # + sizeof USHORT
+ w_obj = rsocket.Address(c_addr, 1 + 2).as_object(space)
+ assert space.is_true(space.isinstance(w_obj, space.w_tuple))
+ assert space.int_w(space.getitem(w_obj, space.wrap(0))) == 15
+ assert space.str_w(space.getitem(w_obj, space.wrap(1))) == 'c'
+
def test_getnameinfo():
host = "127.0.0.1"
port = 25
@@ -440,7 +455,6 @@
s2 = s.dup()
assert s.fileno() != s2.fileno()
assert s.getsockname() == s2.getsockname()
-
def test_buffer_or_unicode(self):
# Test that send/sendall/sendto accept a buffer or a unicode as arg
Modified: pypy/branch/jit-bounds/pypy/module/cpyext/api.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/module/cpyext/api.py (original)
+++ pypy/branch/jit-bounds/pypy/module/cpyext/api.py Tue Sep 7 19:22:31 2010
@@ -62,11 +62,15 @@
VA_LIST_P = rffi.VOIDP # rffi.COpaquePtr('va_list')
CONST_STRING = lltype.Ptr(lltype.Array(lltype.Char,
- hints={'nolength': True}))
+ hints={'nolength': True}),
+ use_cache=False)
CONST_WSTRING = lltype.Ptr(lltype.Array(lltype.UniChar,
- hints={'nolength': True}))
+ hints={'nolength': True}),
+ use_cache=False)
assert CONST_STRING is not rffi.CCHARP
+assert CONST_STRING == rffi.CCHARP
assert CONST_WSTRING is not rffi.CWCHARP
+assert CONST_WSTRING == rffi.CWCHARP
# FILE* interface
FILEP = rffi.COpaquePtr('FILE')
Modified: pypy/branch/jit-bounds/pypy/module/posix/test/test_posix2.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/module/posix/test/test_posix2.py (original)
+++ pypy/branch/jit-bounds/pypy/module/posix/test/test_posix2.py Tue Sep 7 19:22:31 2010
@@ -324,15 +324,12 @@
if hasattr(__import__(os.name), "openpty"):
def test_openpty(self):
os = self.posix
- master_fd, slave_fd = self.posix.openpty()
- try:
- assert isinstance(master_fd, int)
- assert isinstance(slave_fd, int)
- os.write(slave_fd, 'x')
- assert os.read(master_fd, 1) == 'x'
- finally:
- os.close(master_fd)
- os.close(slave_fd)
+ master_fd, slave_fd = os.openpty()
+ assert isinstance(master_fd, int)
+ assert isinstance(slave_fd, int)
+ os.write(slave_fd, 'x\n')
+ data = os.read(master_fd, 100)
+ assert data.startswith('x')
if hasattr(__import__(os.name), "execv"):
Modified: pypy/branch/jit-bounds/pypy/objspace/std/objspace.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/objspace/std/objspace.py (original)
+++ pypy/branch/jit-bounds/pypy/objspace/std/objspace.py Tue Sep 7 19:22:31 2010
@@ -405,8 +405,8 @@
def getattr(self, w_obj, w_name):
if not self.config.objspace.std.getattributeshortcut:
return DescrOperation.getattr(self, w_obj, w_name)
-
# an optional shortcut for performance
+
w_type = self.type(w_obj)
w_descr = w_type.getattribute_if_not_from_object()
if w_descr is not None:
Modified: pypy/branch/jit-bounds/pypy/objspace/std/tupleobject.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/objspace/std/tupleobject.py (original)
+++ pypy/branch/jit-bounds/pypy/objspace/std/tupleobject.py Tue Sep 7 19:22:31 2010
@@ -10,7 +10,7 @@
class W_TupleObject(W_Object):
from pypy.objspace.std.tupletype import tuple_typedef as typedef
- _immutable_ = True
+ _immutable_fields_ = ['wrappeditems[*]']
def __init__(w_self, wrappeditems):
make_sure_not_resized(wrappeditems)
Modified: pypy/branch/jit-bounds/pypy/rlib/rarithmetic.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rlib/rarithmetic.py (original)
+++ pypy/branch/jit-bounds/pypy/rlib/rarithmetic.py Tue Sep 7 19:22:31 2010
@@ -74,7 +74,7 @@
def widen(n):
from pypy.rpython.lltypesystem import lltype
if _should_widen_type(lltype.typeOf(n)):
- return int(n)
+ return intmask(n)
else:
return n
widen._annspecialcase_ = 'specialize:argtype(0)'
Modified: pypy/branch/jit-bounds/pypy/rlib/rsocket.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rlib/rsocket.py (original)
+++ pypy/branch/jit-bounds/pypy/rlib/rsocket.py Tue Sep 7 19:22:31 2010
@@ -100,8 +100,6 @@
def lock(self, TYPE=_c.sockaddr):
"""Return self.addr_p, cast as a pointer to TYPE. Must call unlock()!
"""
- if not (self.minlen <= self.addrlen <= self.maxlen):
- raise RSocketError("invalid address")
return rffi.cast(lltype.Ptr(TYPE), self.addr_p)
lock._annspecialcase_ = 'specialize:ll'
Modified: pypy/branch/jit-bounds/pypy/rlib/rweakref.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rlib/rweakref.py (original)
+++ pypy/branch/jit-bounds/pypy/rlib/rweakref.py Tue Sep 7 19:22:31 2010
@@ -78,7 +78,7 @@
return self.__class__,
def method_get(self, s_key):
- assert isinstance(s_key, annmodel.SomeString)
+ assert annmodel.SomeString(can_be_None=True).contains(s_key)
return annmodel.SomeInstance(self.valueclassdef, can_be_None=True)
def method_set(self, s_key, s_value):
Modified: pypy/branch/jit-bounds/pypy/rpython/lltypesystem/llgroup.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/lltypesystem/llgroup.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/lltypesystem/llgroup.py Tue Sep 7 19:22:31 2010
@@ -99,6 +99,7 @@
'&~0xFFFF' or with a direct masking like '&0x10000' (resp. on 64-bit
platform, with '&~0xFFFFFFFF' or '&0x100000000').
"""
+ __slots__ = ['lowpart', 'rest']
MASK = (1<<HALFSHIFT)-1 # 0xFFFF or 0xFFFFFFFF
def annotation(self):
@@ -135,6 +136,10 @@
assert (other & CombinedSymbolic.MASK) == 0
return CombinedSymbolic(self.lowpart, self.rest - other)
+ def __rshift__(self, other):
+ assert other >= HALFSHIFT
+ return self.rest >> other
+
def __eq__(self, other):
if (isinstance(other, CombinedSymbolic) and
self.lowpart is other.lowpart):
Modified: pypy/branch/jit-bounds/pypy/rpython/lltypesystem/llmemory.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/lltypesystem/llmemory.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/lltypesystem/llmemory.py Tue Sep 7 19:22:31 2010
@@ -361,19 +361,27 @@
# ____________________________________________________________
+def _sizeof_none(TYPE):
+ assert not TYPE._is_varsize()
+ return ItemOffset(TYPE)
+_sizeof_none._annspecialcase_ = 'specialize:memo'
+
+def _sizeof_int(TYPE, n):
+ "NOT_RPYTHON"
+ if isinstance(TYPE, lltype.Struct):
+ return FieldOffset(TYPE, TYPE._arrayfld) + \
+ itemoffsetof(TYPE._flds[TYPE._arrayfld], n)
+ else:
+ raise Exception("don't know how to take the size of a %r"%TYPE)
+
def sizeof(TYPE, n=None):
if n is None:
- assert not TYPE._is_varsize()
- return ItemOffset(TYPE)
+ return _sizeof_none(TYPE)
+ elif isinstance(TYPE, lltype.Array):
+ return itemoffsetof(TYPE) + _sizeof_none(TYPE.OF) * n
else:
- if isinstance(TYPE, lltype.Array):
- return itemoffsetof(TYPE, n)
- elif isinstance(TYPE, lltype.Struct):
- return FieldOffset(TYPE, TYPE._arrayfld) + \
- itemoffsetof(TYPE._flds[TYPE._arrayfld], n)
- else:
- raise Exception("don't know how to take the size of a %r"%TYPE)
-sizeof._annspecialcase_ = 'specialize:memo' # only for n == None
+ return _sizeof_int(TYPE, n)
+sizeof._annspecialcase_ = 'specialize:arg(0)'
def offsetof(TYPE, fldname):
assert fldname in TYPE._flds
@@ -389,6 +397,7 @@
# -------------------------------------------------------------
class fakeaddress(object):
+ __slots__ = ['ptr']
# NOTE: the 'ptr' in the addresses must be normalized.
# Use cast_ptr_to_adr() instead of directly fakeaddress() if unsure.
def __init__(self, ptr):
@@ -530,7 +539,6 @@
pass
NULL = fakeaddress(None)
-NULL.intaddress = 0 # this is to make memory.lladdress more happy
Address = lltype.Primitive("Address", NULL)
# GCREF is similar to Address but it is GC-aware
Modified: pypy/branch/jit-bounds/pypy/rpython/lltypesystem/lloperation.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/lltypesystem/lloperation.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/lltypesystem/lloperation.py Tue Sep 7 19:22:31 2010
@@ -85,16 +85,20 @@
fold = roproperty(get_fold_impl)
def is_pure(self, args_v):
- return (self.canfold or # canfold => pure operation
- self is llop.debug_assert or # debug_assert is pure enough
- # reading from immutable
- (self in (llop.getfield, llop.getarrayitem) and
- args_v[0].concretetype.TO._hints.get('immutable')) or
- (self is llop.getfield and # reading from immutable_field
- 'immutable_fields' in args_v[0].concretetype.TO._hints and
- args_v[1].value in args_v[0].concretetype.TO
- ._hints['immutable_fields'].fields))
- # XXX: what about ootype immutable arrays?
+ if self.canfold: # canfold => pure operation
+ return True
+ if self is llop.debug_assert: # debug_assert is pure enough
+ return True
+ # reading from immutable (lltype)
+ if self is llop.getfield or self is llop.getarrayitem:
+ field = getattr(args_v[1], 'value', None)
+ return args_v[0].concretetype.TO._immutable_field(field)
+ # reading from immutable (ootype) (xxx what about arrays?)
+ if self is llop.oogetfield:
+ field = getattr(args_v[1], 'value', None)
+ return args_v[0].concretetype._immutable_field(field)
+ # default
+ return False
def __repr__(self):
return '<LLOp %s>' % (getattr(self, 'opname', '?'),)
Modified: pypy/branch/jit-bounds/pypy/rpython/lltypesystem/lltype.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/lltypesystem/lltype.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/lltypesystem/lltype.py Tue Sep 7 19:22:31 2010
@@ -37,7 +37,7 @@
return '<Uninitialized %r>'%(self.TYPE,)
-def saferecursive(func, defl):
+def saferecursive(func, defl, TLS=TLS):
def safe(*args):
try:
seeing = TLS.seeing
@@ -54,7 +54,7 @@
return safe
#safe_equal = saferecursive(operator.eq, True)
-def safe_equal(x, y):
+def safe_equal(x, y, TLS=TLS):
# a specialized version for performance
try:
seeing = TLS.seeing_eq
@@ -97,7 +97,7 @@
raise TypeError
return value
- def __hash__(self):
+ def __hash__(self, TLS=TLS):
# cannot use saferecursive() -- see test_lltype.test_hash().
# NB. the __cached_hash should neither be used nor updated
# if we enter with hash_level > 0, because the computed
@@ -297,6 +297,15 @@
n = 1
return _struct(self, n, initialization='example')
+ def _immutable_field(self, field):
+ if 'immutable_fields' in self._hints:
+ try:
+ s = self._hints['immutable_fields'].fields[field]
+ return s or True
+ except KeyError:
+ pass
+ return self._hints.get('immutable', False)
+
class RttiStruct(Struct):
_runtime_type_info = None
@@ -391,6 +400,9 @@
def _container_example(self):
return _array(self, 1, initialization='example')
+ def _immutable_field(self, index=None):
+ return self._hints.get('immutable', False)
+
class GcArray(Array):
_gckind = 'gc'
def _inline_is_varsize(self, last):
@@ -401,6 +413,19 @@
# behaves more or less like a Struct with fields item0, item1, ...
# but also supports __getitem__(), __setitem__(), __len__().
+ _cache = weakref.WeakValueDictionary() # cache the length-1 FixedSizeArrays
+ def __new__(cls, OF, length, **kwds):
+ if length == 1 and not kwds:
+ try:
+ obj = FixedSizeArray._cache[OF]
+ except KeyError:
+ obj = FixedSizeArray._cache[OF] = Struct.__new__(cls)
+ except TypeError:
+ obj = Struct.__new__(cls)
+ else:
+ obj = Struct.__new__(cls)
+ return obj
+
def __init__(self, OF, length, **kwds):
fields = [('item%d' % i, OF) for i in range(length)]
super(FixedSizeArray, self).__init__('array%d' % length, *fields,
@@ -610,11 +635,22 @@
class Ptr(LowLevelType):
__name__ = property(lambda self: '%sPtr' % self.TO.__name__)
- def __init__(self, TO):
+ _cache = weakref.WeakValueDictionary() # cache the Ptrs
+ def __new__(cls, TO, use_cache=True):
if not isinstance(TO, ContainerType):
raise TypeError, ("can only point to a Container type, "
"not to %s" % (TO,))
- self.TO = TO
+ if not use_cache:
+ obj = LowLevelType.__new__(cls)
+ else:
+ try:
+ return Ptr._cache[TO]
+ except KeyError:
+ obj = Ptr._cache[TO] = LowLevelType.__new__(cls)
+ except TypeError:
+ obj = LowLevelType.__new__(cls)
+ obj.TO = TO
+ return obj
def _needsgc(self):
# XXX deprecated interface
Modified: pypy/branch/jit-bounds/pypy/rpython/lltypesystem/opimpl.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/lltypesystem/opimpl.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/lltypesystem/opimpl.py Tue Sep 7 19:22:31 2010
@@ -150,12 +150,7 @@
# we can constant-fold this if the innermost structure from which we
# read the final field is immutable.
T = lltype.typeOf(innermostcontainer).TO
- if T._hints.get('immutable'):
- pass
- elif ('immutable_fields' in T._hints and
- offsets[-1] in T._hints['immutable_fields'].fields):
- pass
- else:
+ if not T._immutable_field(offsets[-1]):
raise TypeError("cannot fold getinteriorfield on mutable struct")
assert not isinstance(ob, lltype._interior_ptr)
return ob
@@ -197,6 +192,18 @@
assert isinstance(y, int)
return intmask(x - y)
+def op_int_ge(x, y):
+ # special case for 'AddressOffset >= 0'
+ assert isinstance(x, (int, llmemory.AddressOffset))
+ assert isinstance(y, int)
+ return x >= y
+
+def op_int_lt(x, y):
+ # special case for 'AddressOffset < 0'
+ assert isinstance(x, (int, llmemory.AddressOffset))
+ assert isinstance(y, int)
+ return x < y
+
def op_int_between(a, b, c):
assert lltype.typeOf(a) is lltype.Signed
assert lltype.typeOf(b) is lltype.Signed
@@ -222,6 +229,13 @@
assert isinstance(y, (int, llmemory.AddressOffset))
return intmask(x * y)
+def op_int_rshift(x, y):
+ if not isinstance(x, int):
+ from pypy.rpython.lltypesystem import llgroup
+ assert isinstance(x, llgroup.CombinedSymbolic)
+ assert isinstance(y, int)
+ return x >> y
+
def op_int_floordiv(x, y):
assert isinstance(x, (int, llmemory.AddressOffset))
assert isinstance(y, (int, llmemory.AddressOffset))
@@ -418,19 +432,15 @@
def op_getfield(p, name):
checkptr(p)
TYPE = lltype.typeOf(p).TO
- if TYPE._hints.get('immutable'):
- pass
- elif ('immutable_fields' in TYPE._hints and
- name in TYPE._hints['immutable_fields'].fields):
- pass
- else:
+ if not TYPE._immutable_field(name):
raise TypeError("cannot fold getfield on mutable struct")
return getattr(p, name)
def op_getarrayitem(p, index):
checkptr(p)
- if not lltype.typeOf(p).TO._hints.get('immutable'):
- raise TypeError("cannot fold getfield on mutable array")
+ ARRAY = lltype.typeOf(p).TO
+ if not ARRAY._immutable_field(index):
+ raise TypeError("cannot fold getarrayitem on mutable array")
return p[index]
def _normalize(x):
Modified: pypy/branch/jit-bounds/pypy/rpython/lltypesystem/rffi.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/lltypesystem/rffi.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/lltypesystem/rffi.py Tue Sep 7 19:22:31 2010
@@ -593,9 +593,12 @@
""" str -> char*
"""
array = lltype.malloc(TYPEP.TO, len(s) + 1, flavor='raw')
- for i in range(len(s)):
+ i = len(s)
+ array[i] = lastchar
+ i -= 1
+ while i >= 0:
array[i] = s[i]
- array[len(s)] = lastchar
+ i -= 1
return array
str2charp._annenforceargs_ = [strtype]
Modified: pypy/branch/jit-bounds/pypy/rpython/lltypesystem/test/test_llgroup.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/lltypesystem/test/test_llgroup.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/lltypesystem/test/test_llgroup.py Tue Sep 7 19:22:31 2010
@@ -105,6 +105,8 @@
assert p == test.p1b
assert cslist[0] & ~MASK == 0x45 << HALFSHIFT
assert cslist[1] & ~MASK == 0x41 << HALFSHIFT
+ assert cslist[0] >> HALFSHIFT == 0x45
+ assert cslist[1] >> (HALFSHIFT+1) == 0x41 >> 1
#
return 42
return f
Modified: pypy/branch/jit-bounds/pypy/rpython/lltypesystem/test/test_lloperation.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/lltypesystem/test/test_lloperation.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/lltypesystem/test/test_lloperation.py Tue Sep 7 19:22:31 2010
@@ -88,7 +88,7 @@
accessor = rclass.FieldListAccessor()
S3 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed),
hints={'immutable_fields': accessor})
- accessor.initialize(S3, ['x'])
+ accessor.initialize(S3, {'x': ''})
v_s3 = Variable()
v_s3.concretetype = lltype.Ptr(S3)
assert not llop.setfield.is_pure([v_s3, Constant('x'), Variable()])
@@ -103,7 +103,7 @@
accessor = rclass.FieldListAccessor()
S3 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed),
hints={'immutable_fields': accessor})
- accessor.initialize(S3, ['x'])
+ accessor.initialize(S3, {'x': ''})
#
s1 = lltype.malloc(S1); s1.x = 45
py.test.raises(TypeError, llop.getfield, lltype.Signed, s1, 'x')
Modified: pypy/branch/jit-bounds/pypy/rpython/lltypesystem/test/test_lltype.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/lltypesystem/test/test_lltype.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/lltypesystem/test/test_lltype.py Tue Sep 7 19:22:31 2010
@@ -781,6 +781,28 @@
p = cast_opaque_ptr(llmemory.GCREF, a)
assert hash1 == identityhash(p)
+def test_immutable_hint():
+ S = GcStruct('S', ('x', lltype.Signed))
+ assert S._immutable_field('x') == False
+ #
+ S = GcStruct('S', ('x', lltype.Signed), hints={'immutable': True})
+ assert S._immutable_field('x') == True
+ #
+ class FieldListAccessor(object):
+ def __init__(self, fields):
+ self.fields = fields
+ S = GcStruct('S', ('x', lltype.Signed),
+ hints={'immutable_fields': FieldListAccessor({'x':''})})
+ assert S._immutable_field('x') == True
+ #
+ class FieldListAccessor(object):
+ def __init__(self, fields):
+ self.fields = fields
+ S = GcStruct('S', ('x', lltype.Signed),
+ hints={'immutable_fields': FieldListAccessor({'x':'[*]'})})
+ assert S._immutable_field('x') == '[*]'
+
+
class TestTrackAllocation:
def setup_method(self, func):
start_tracking_allocations()
Modified: pypy/branch/jit-bounds/pypy/rpython/memory/gc/base.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/memory/gc/base.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/memory/gc/base.py Tue Sep 7 19:22:31 2010
@@ -86,8 +86,7 @@
addr -= self.gcheaderbuilder.size_gc_header
return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
- def get_size(self, obj):
- typeid = self.get_type_id(obj)
+ def _get_size_for_typeid(self, obj, typeid):
size = self.fixed_size(typeid)
if self.is_varsize(typeid):
lenaddr = obj + self.varsize_offset_to_length(typeid)
@@ -99,6 +98,9 @@
# gctypelayout.encode_type_shape()
return size
+ def get_size(self, obj):
+ return self._get_size_for_typeid(obj, self.get_type_id(obj))
+
def malloc(self, typeid, length=0, zero=False):
"""For testing. The interface used by the gctransformer is
the four malloc_[fixed,var]size[_clear]() functions.
@@ -218,7 +220,6 @@
pending = self._debug_pending
while pending.non_empty():
obj = pending.pop()
- self.debug_check_object(obj)
self.trace(obj, self._debug_callback2, None)
self._debug_seen.delete()
self._debug_pending.delete()
@@ -227,6 +228,7 @@
seen = self._debug_seen
if not seen.contains(obj):
seen.add(obj)
+ self.debug_check_object(obj)
self._debug_pending.append(obj)
def _debug_callback(self, root):
obj = root.address[0]
@@ -348,3 +350,23 @@
globals(), locals(), [classname])
GCClass = getattr(module, classname)
return GCClass, GCClass.TRANSLATION_PARAMS
+
+def read_from_env(varname):
+ import os
+ value = os.environ.get(varname)
+ if value:
+ realvalue = value[:-1]
+ if value[-1] in 'kK':
+ factor = 1024
+ elif value[-1] in 'mM':
+ factor = 1024*1024
+ elif value[-1] in 'gG':
+ factor = 1024*1024*1024
+ else:
+ factor = 1
+ realvalue = value
+ try:
+ return int(float(realvalue) * factor)
+ except ValueError:
+ pass
+ return -1
Modified: pypy/branch/jit-bounds/pypy/rpython/memory/gc/generation.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/memory/gc/generation.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/memory/gc/generation.py Tue Sep 7 19:22:31 2010
@@ -2,6 +2,7 @@
from pypy.rpython.memory.gc.semispace import SemiSpaceGC
from pypy.rpython.memory.gc.semispace import GCFLAG_EXTERNAL, GCFLAG_FORWARDED
from pypy.rpython.memory.gc.semispace import GC_HASH_TAKEN_ADDR
+from pypy.rpython.memory.gc.base import read_from_env
from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage
from pypy.rpython.lltypesystem import lltype, llmemory, llarena
from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE
@@ -625,18 +626,7 @@
import os
def nursery_size_from_env():
- value = os.environ.get('PYPY_GENERATIONGC_NURSERY')
- if value:
- if value[-1] in 'kK':
- factor = 1024
- value = value[:-1]
- else:
- factor = 1
- try:
- return int(value) * factor
- except ValueError:
- pass
- return -1
+ return read_from_env('PYPY_GENERATIONGC_NURSERY')
def best_nursery_size_for_L2cache(L2cache):
# Heuristically, the best nursery size to choose is about half
Modified: pypy/branch/jit-bounds/pypy/rpython/memory/gc/markcompact.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/memory/gc/markcompact.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/memory/gc/markcompact.py Tue Sep 7 19:22:31 2010
@@ -1,27 +1,17 @@
-
-import time
-
from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup
-from pypy.rpython.memory.gc.base import MovingGCBase
+from pypy.rpython.memory.gc.base import MovingGCBase, read_from_env
from pypy.rlib.debug import ll_assert, have_debug_prints
from pypy.rlib.debug import debug_print, debug_start, debug_stop
from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE
from pypy.rpython.memory.support import get_address_stack, get_address_deque
from pypy.rpython.memory.support import AddressDict
from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage
-from pypy.rlib.rarithmetic import ovfcheck
+from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask
from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.objectmodel import we_are_translated
+from pypy.rlib.objectmodel import we_are_translated, running_on_llinterp
from pypy.rpython.lltypesystem import rffi
from pypy.rpython.memory.gcheader import GCHeaderBuilder
-first_gcflag = 1 << 16
-GCFLAG_MARKBIT = first_gcflag << 0
-GCFLAG_HASHTAKEN = first_gcflag << 1 # someone already asked for the hash
-GCFLAG_HASHFIELD = first_gcflag << 2 # we have an extra hash field
-
-memoryError = MemoryError()
-
# Mark'n'compact garbage collector
#
# main point of this GC is to save as much memory as possible
@@ -34,41 +24,44 @@
# this gc works more or less like semispace, but has some essential
# differencies. The main difference is that we have separate phases of
# marking and assigning pointers, hence order of objects is preserved.
-# This means we can reuse the same space if it did not grow enough.
-# More importantly, in case we need to resize space we can copy it bit by
-# bit, hence avoiding double memory consumption at peak times
+# This means we can reuse the same space, overwriting it as we collect.
-# so the algorithm itself is performed in 3 stages (module weakrefs and
-# finalizers)
+# so the algorithm itself is performed in 3 stages (modulo weakrefs and
+# finalizers):
# 1. We mark alive objects
# 2. We walk all objects and assign forward pointers in the same order,
# also updating all references
-# 3. We compact the space by moving. In case we move to the same space,
-# we use arena_new_view trick, which looks like new space to tests,
-# but compiles to the same pointer. Also we use raw_memmove in case
-# objects overlap.
-
-# Exact algorithm for space resizing: we keep allocated more space than needed
-# (2x, can be even more), but it's full of zeroes. After each collection,
-# we bump next_collect_after which is a marker where to start each collection.
-# It should be exponential (but less than 2) from the size occupied by objects
+# 3. We compact the space by moving. We use 'arena_new_view' trick, which
+# looks like new space to tests, but compiles to the same pointer.
+# Also we use raw_memmove in case the object overlaps with its destination.
+
+# After each collection, we bump 'next_collect_after' which is a marker
+# where to start each collection. It should be exponential (but less
+# than 2) from the size occupied by objects so far.
# field optimization - we don't need forward pointer and flags at the same
-# time. Instead we copy list of tids when we know how many objects are alive
-# and store forward pointer there.
+# time. Instead we copy the TIDs in a list when we know how many objects are
+# alive, and store the forward pointer in the old object header.
+first_gcflag_bit = LONG_BIT//2
+first_gcflag = 1 << first_gcflag_bit
+GCFLAG_HASHTAKEN = first_gcflag << 0 # someone already asked for the hash
+GCFLAG_HASHFIELD = first_gcflag << 1 # we have an extra hash field
+# note that only the first 2 bits are preserved during a collection!
+GCFLAG_MARKBIT = intmask(first_gcflag << (LONG_BIT//2-1))
+assert GCFLAG_MARKBIT < 0 # should be 0x80000000
+
+GCFLAG_SAVED_HASHTAKEN = GCFLAG_HASHTAKEN >> first_gcflag_bit
+GCFLAG_SAVED_HASHFIELD = GCFLAG_HASHFIELD >> first_gcflag_bit
-# in case we need to grow space, we use
-# current_space_size * FREE_SPACE_MULTIPLIER / FREE_SPACE_DIVIDER + needed
-FREE_SPACE_MULTIPLIER = 3
-FREE_SPACE_DIVIDER = 2
-FREE_SPACE_ADD = 256
-# XXX adjust
-GC_CLEARANCE = 32*1024
TID_TYPE = llgroup.HALFWORD
BYTES_PER_TID = rffi.sizeof(TID_TYPE)
+TID_BACKUP = rffi.CArray(TID_TYPE)
+
+def translated_to_c():
+ return we_are_translated() and not running_on_llinterp
class MarkCompactGC(MovingGCBase):
@@ -77,37 +70,63 @@
withhash_flag_is_in_field = 'tid', GCFLAG_HASHFIELD
# ^^^ all prebuilt objects have GCFLAG_HASHTAKEN, but only some have
# GCFLAG_HASHFIELD (and then they are one word longer).
- TID_BACKUP = lltype.Array(TID_TYPE, hints={'nolength':True})
- WEAKREF_OFFSETS = lltype.Array(lltype.Signed)
+ # The default space size is 1.9375 GB, i.e. almost 2 GB, allocated as
+ # a big mmap. The process does not actually consume that space until
+ # needed, of course.
+ TRANSLATION_PARAMS = {'space_size': int((1 + 15.0/16)*1024*1024*1024),
+ 'min_next_collect_after': 16*1024*1024} # 16MB
- TRANSLATION_PARAMS = {'space_size': 8*1024*1024} # XXX adjust
-
- malloc_zero_filled = True
+ malloc_zero_filled = False
inline_simple_malloc = True
inline_simple_malloc_varsize = True
- first_unused_gcflag = first_gcflag << 3
- total_collection_time = 0.0
- total_collection_count = 0
-
- def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096):
- import py; py.test.skip("Disabled for now, sorry")
- self.param_space_size = space_size
+ #total_collection_time = 0.0
+ #total_collection_count = 0
+
+ free = NULL
+ next_collect_after = -1
+
+ def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096,
+ min_next_collect_after=128):
MovingGCBase.__init__(self, config, chunk_size)
+ self.space_size = space_size
+ self.min_next_collect_after = min_next_collect_after
- def setup(self):
- self.space_size = self.param_space_size
- self.next_collect_after = self.param_space_size/2 # whatever...
+ def next_collection(self, used_space, num_objects_so_far, requested_size):
+ used_space += BYTES_PER_TID * num_objects_so_far
+ ll_assert(used_space <= self.space_size,
+ "used_space + num_objects_so_far overflow")
+ try:
+ next = (used_space // 3) * 2 + requested_size
+ except OverflowError:
+ next = self.space_size
+ if next < self.min_next_collect_after:
+ next = self.min_next_collect_after
+ if next > self.space_size - used_space:
+ next = self.space_size - used_space
+ # The value we return guarantees that used_space + next <= space_size,
+ # with 'BYTES_PER_TID*num_objects_so_far' included in used_space.
+ # Normally, the value we return should also be at least requested_size
+ # unless we are out of memory.
+ return next
- self.program_start_time = time.time()
- self.space = llarena.arena_malloc(self.space_size, True)
- ll_assert(bool(self.space), "couldn't allocate arena")
+ def setup(self):
+ envsize = read_from_env('PYPY_MARKCOMPACTGC_MAX')
+ if envsize >= 4096:
+ self.space_size = envsize & ~4095
+ mincollect = read_from_env('PYPY_MARKCOMPACTGC_MIN')
+ if mincollect >= 4096:
+ self.min_next_collect_after = mincollect
+
+ #self.program_start_time = time.time()
+ self.space = llarena.arena_malloc(self.space_size, False)
+ if not self.space:
+ raise CannotAllocateGCArena
self.free = self.space
- self.top_of_space = self.space + self.next_collect_after
MovingGCBase.setup(self)
self.objects_with_finalizers = self.AddressDeque()
- self.objects_with_weakrefs = self.AddressStack()
- self.tid_backup = lltype.nullptr(self.TID_BACKUP)
+ self.tid_backup = lltype.nullptr(TID_BACKUP)
+ self.next_collect_after = self.next_collection(0, 0, 0)
def init_gc_object(self, addr, typeid16, flags=0):
hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
@@ -115,216 +134,204 @@
def init_gc_object_immortal(self, addr, typeid16, flags=0):
hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
- flags |= GCFLAG_HASHTAKEN
+ flags |= GCFLAG_HASHTAKEN | GCFLAG_MARKBIT
+ # All prebuilt GC objects have the GCFLAG_MARKBIT always set.
+ # That's convenient to make the GC always think that they
+ # survive the current collection.
hdr.tid = self.combine(typeid16, flags)
- # XXX we can store forward_ptr to itself, if we fix C backend
- # so that get_forwarding_address(obj) returns
- # obj itself if obj is a prebuilt object
- def malloc_fixedsize_clear(self, typeid16, size, can_collect,
- has_finalizer=False, contains_weakptr=False):
- size_gc_header = self.gcheaderbuilder.size_gc_header
- totalsize = size_gc_header + size
- result = self.free
- if raw_malloc_usage(totalsize) > self.top_of_space - result:
- result = self.obtain_free_space(totalsize)
+ def _get_memory(self, totalsize):
+ # also counts the space that will be needed during the following
+ # collection to store the TID
+ requested_size = raw_malloc_usage(totalsize) + BYTES_PER_TID
+ self.next_collect_after -= requested_size
+ if self.next_collect_after < 0:
+ result = self.obtain_free_space(requested_size)
+ else:
+ result = self.free
+ self.free += totalsize
llarena.arena_reserve(result, totalsize)
+ return result
+ _get_memory._always_inline_ = True
+
+ def _get_totalsize_var(self, nonvarsize, itemsize, length):
+ try:
+ varsize = ovfcheck(itemsize * length)
+ except OverflowError:
+ raise MemoryError
+ # Careful to detect overflows. The following works even if varsize
+ # is almost equal to sys.maxint; morever, self.space_size is known
+ # to be at least 4095 bytes smaller than sys.maxint, so this function
+ # always raises instead of returning an integer >= sys.maxint-4095.
+ if (raw_malloc_usage(varsize) > self.space_size -
+ raw_malloc_usage(nonvarsize)):
+ raise MemoryError
+ return llarena.round_up_for_allocation(nonvarsize + varsize)
+ _get_totalsize_var._always_inline_ = True
+
+ def _setup_object(self, result, typeid16, has_finalizer):
+ size_gc_header = self.gcheaderbuilder.size_gc_header
self.init_gc_object(result, typeid16)
- self.free += totalsize
if has_finalizer:
self.objects_with_finalizers.append(result + size_gc_header)
- if contains_weakptr:
- self.objects_with_weakrefs.append(result + size_gc_header)
return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF)
-
+ _setup_object._always_inline_ = True
+
+ def malloc_fixedsize(self, typeid16, size, can_collect,
+ has_finalizer=False, contains_weakptr=False):
+ size_gc_header = self.gcheaderbuilder.size_gc_header
+ totalsize = size_gc_header + size
+ result = self._get_memory(totalsize)
+ return self._setup_object(result, typeid16, has_finalizer)
+
+ def malloc_fixedsize_clear(self, typeid16, size, can_collect,
+ has_finalizer=False, contains_weakptr=False):
+ size_gc_header = self.gcheaderbuilder.size_gc_header
+ totalsize = size_gc_header + size
+ result = self._get_memory(totalsize)
+ llmemory.raw_memclear(result, totalsize)
+ return self._setup_object(result, typeid16, has_finalizer)
+
def malloc_varsize_clear(self, typeid16, length, size, itemsize,
offset_to_length, can_collect):
size_gc_header = self.gcheaderbuilder.size_gc_header
nonvarsize = size_gc_header + size
- try:
- varsize = ovfcheck(itemsize * length)
- totalsize = ovfcheck(nonvarsize + varsize)
- except OverflowError:
- raise memoryError
- result = self.free
- if raw_malloc_usage(totalsize) > self.top_of_space - result:
- result = self.obtain_free_space(totalsize)
- llarena.arena_reserve(result, totalsize)
- self.init_gc_object(result, typeid16)
+ totalsize = self._get_totalsize_var(nonvarsize, itemsize, length)
+ result = self._get_memory(totalsize)
+ llmemory.raw_memclear(result, totalsize)
(result + size_gc_header + offset_to_length).signed[0] = length
- self.free = result + llarena.round_up_for_allocation(totalsize)
- return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF)
+ return self._setup_object(result, typeid16, False)
- def obtain_free_space(self, totalsize):
- # a bit of tweaking to maximize the performance and minimize the
- # amount of code in an inlined version of malloc_fixedsize_clear()
- if not self.try_obtain_free_space(totalsize):
- raise memoryError
+ def obtain_free_space(self, requested_size):
+ if self.free == NULL:
+ return self._emergency_initial_block(requested_size)
+ while True:
+ executed_some_finalizers = self.markcompactcollect(requested_size)
+ self.next_collect_after -= requested_size
+ if self.next_collect_after >= 0:
+ break # ok
+ else:
+ if executed_some_finalizers:
+ pass # try again to do a collection
+ else:
+ raise MemoryError
return self.free
obtain_free_space._dont_inline_ = True
- def try_obtain_free_space(self, needed):
- needed = raw_malloc_usage(needed)
- while 1:
- self.markcompactcollect(needed)
- missing = needed - (self.top_of_space - self.free)
- if missing < 0:
- return True
-
- def new_space_size(self, occupied, needed):
- res = (occupied * FREE_SPACE_MULTIPLIER /
- FREE_SPACE_DIVIDER + FREE_SPACE_ADD + needed)
- # align it to 4096, which is somewhat around page size
- return ((res/4096) + 1) * 4096
-
- def double_space_size(self, minimal_size):
- while self.space_size <= minimal_size:
- self.space_size *= 2
- toaddr = llarena.arena_malloc(self.space_size, True)
- return toaddr
-
- def compute_alive_objects(self):
- fromaddr = self.space
- addraftercollect = self.space
- num = 1
- while fromaddr < self.free:
- size_gc_header = self.gcheaderbuilder.size_gc_header
- tid = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR)).tid
- obj = fromaddr + size_gc_header
- objsize = self.get_size(obj)
- objtotalsize = size_gc_header + objsize
- if self.marked(obj):
- copy_has_hash_field = ((tid & GCFLAG_HASHFIELD) != 0 or
- ((tid & GCFLAG_HASHTAKEN) != 0 and
- addraftercollect < fromaddr))
- addraftercollect += raw_malloc_usage(objtotalsize)
- if copy_has_hash_field:
- addraftercollect += llmemory.sizeof(lltype.Signed)
- num += 1
- fromaddr += objtotalsize
- if tid & GCFLAG_HASHFIELD:
- fromaddr += llmemory.sizeof(lltype.Signed)
- ll_assert(addraftercollect <= fromaddr,
- "markcompactcollect() is trying to increase memory usage")
- self.totalsize_of_objs = addraftercollect - self.space
- return num
+ def _emergency_initial_block(self, requested_size):
+ # xxx before the GC is fully setup, we might get there. Hopefully
+ # we will only allocate a couple of strings, e.g. in read_from_env().
+ # Just allocate them raw and leak them.
+ debug_start("gc-initial-block")
+ debug_print("leaking", requested_size, "bytes")
+ debug_stop("gc-initial-block")
+ return llmemory.raw_malloc(requested_size)
def collect(self, gen=0):
self.markcompactcollect()
-
- def markcompactcollect(self, needed=0):
- start_time = self.debug_collect_start()
+
+ def markcompactcollect(self, requested_size=0):
+ self.debug_collect_start(requested_size)
self.debug_check_consistency()
- self.to_see = self.AddressStack()
- self.mark_roots_recursively()
- if (self.objects_with_finalizers.non_empty() or
- self.run_finalizers.non_empty()):
- self.mark_objects_with_finalizers()
- self._trace_and_mark()
+ #
+ # Mark alive objects
+ #
+ self.to_see = self.AddressDeque()
+ self.trace_from_roots()
self.to_see.delete()
- num_of_alive_objs = self.compute_alive_objects()
- size_of_alive_objs = self.totalsize_of_objs
- totalsize = self.new_space_size(size_of_alive_objs, needed +
- num_of_alive_objs * BYTES_PER_TID)
- tid_backup_size = (llmemory.sizeof(self.TID_BACKUP, 0) +
- llmemory.sizeof(TID_TYPE) * num_of_alive_objs)
- used_space_now = self.next_collect_after + raw_malloc_usage(tid_backup_size)
- if totalsize >= self.space_size or used_space_now >= self.space_size:
- toaddr = self.double_space_size(totalsize)
- llarena.arena_reserve(toaddr + size_of_alive_objs, tid_backup_size)
- self.tid_backup = llmemory.cast_adr_to_ptr(
- toaddr + size_of_alive_objs,
- lltype.Ptr(self.TID_BACKUP))
- resizing = True
- else:
- toaddr = llarena.arena_new_view(self.space)
- llarena.arena_reserve(self.top_of_space, tid_backup_size)
- self.tid_backup = llmemory.cast_adr_to_ptr(
- self.top_of_space,
- lltype.Ptr(self.TID_BACKUP))
- resizing = False
- self.next_collect_after = totalsize
- weakref_offsets = self.collect_weakref_offsets()
- finaladdr = self.update_forward_pointers(toaddr, num_of_alive_objs)
+ #
+ # Prepare new views on the same memory
+ #
+ toaddr = llarena.arena_new_view(self.space)
+ maxnum = self.space_size - (self.free - self.space)
+ maxnum /= BYTES_PER_TID
+ llarena.arena_reserve(self.free, llmemory.sizeof(TID_BACKUP, maxnum))
+ self.tid_backup = llmemory.cast_adr_to_ptr(self.free,
+ lltype.Ptr(TID_BACKUP))
+ #
+ # Walk all objects and assign forward pointers in the same order,
+ # also updating all references
+ #
+ self.update_forward_pointers(toaddr, maxnum)
if (self.run_finalizers.non_empty() or
self.objects_with_finalizers.non_empty()):
self.update_run_finalizers()
- if self.objects_with_weakrefs.non_empty():
- self.invalidate_weakrefs(weakref_offsets)
+
self.update_objects_with_id()
- self.compact(resizing)
- if not resizing:
- size = toaddr + self.space_size - finaladdr
- llarena.arena_reset(finaladdr, size, True)
- else:
- if we_are_translated():
- # because we free stuff already in raw_memmove, we
- # would get double free here. Let's free it anyway
- llarena.arena_free(self.space)
- llarena.arena_reset(toaddr + size_of_alive_objs, tid_backup_size,
- True)
- self.space = toaddr
- self.free = finaladdr
- self.top_of_space = toaddr + self.next_collect_after
+ self.compact()
+ #
+ self.tid_backup = lltype.nullptr(TID_BACKUP)
+ self.free = self.finaladdr
+ self.next_collect_after = self.next_collection(self.finaladdr - toaddr,
+ self.num_alive_objs,
+ requested_size)
+ #
+ if not translated_to_c():
+ remaining_size = (toaddr + self.space_size) - self.finaladdr
+ llarena.arena_reset(self.finaladdr, remaining_size, False)
+ llarena.arena_free(self.space)
+ self.space = toaddr
+ #
self.debug_check_consistency()
- self.tid_backup = lltype.nullptr(self.TID_BACKUP)
+ self.debug_collect_finish()
+ if self.next_collect_after < 0:
+ raise MemoryError
+ #
if self.run_finalizers.non_empty():
self.execute_finalizers()
- self.debug_collect_finish(start_time)
-
- def collect_weakref_offsets(self):
- weakrefs = self.objects_with_weakrefs
- new_weakrefs = self.AddressStack()
- weakref_offsets = lltype.malloc(self.WEAKREF_OFFSETS,
- weakrefs.length(), flavor='raw')
- i = 0
- while weakrefs.non_empty():
- obj = weakrefs.pop()
- offset = self.weakpointer_offset(self.get_type_id(obj))
- weakref_offsets[i] = offset
- new_weakrefs.append(obj)
- i += 1
- self.objects_with_weakrefs = new_weakrefs
- weakrefs.delete()
- return weakref_offsets
+ return True # executed some finalizers
+ else:
+ return False # no finalizer executed
- def debug_collect_start(self):
- if have_debug_prints():
+ def debug_collect_start(self, requested_size):
+ if 1:# have_debug_prints():
debug_start("gc-collect")
debug_print()
- debug_print(".----------- Full collection ------------------")
- start_time = time.time()
- return start_time
- return -1
-
- def debug_collect_finish(self, start_time):
- if start_time != -1:
- end_time = time.time()
- elapsed_time = end_time - start_time
- self.total_collection_time += elapsed_time
- self.total_collection_count += 1
- total_program_time = end_time - self.program_start_time
- ct = self.total_collection_time
- cc = self.total_collection_count
- debug_print("| number of collections so far ",
- cc)
- debug_print("| total collections per second: ",
- cc / total_program_time)
- debug_print("| total time in markcompact-collect: ",
- ct, "seconds")
- debug_print("| percentage collection<->total time:",
- ct * 100.0 / total_program_time, "%")
+ debug_print(".----------- Full collection -------------------")
+ debug_print("| requested size:",
+ requested_size)
+ #start_time = time.time()
+ #return start_time
+ #return -1
+
+ def debug_collect_finish(self):
+ if 1:# start_time != -1:
+ #end_time = time.time()
+ #elapsed_time = end_time - start_time
+ #self.total_collection_time += elapsed_time
+ #self.total_collection_count += 1
+ #total_program_time = end_time - self.program_start_time
+ #ct = self.total_collection_time
+ #cc = self.total_collection_count
+ #debug_print("| number of collections so far ",
+ # cc)
+ debug_print("| total space size ",
+ self.space_size)
+ debug_print("| number of objects alive ",
+ self.num_alive_objs)
+ debug_print("| used space size ",
+ self.free - self.space)
+ debug_print("| next collection after ",
+ self.next_collect_after)
+ #debug_print("| total collections per second: ",
+ # cc / total_program_time)
+ #debug_print("| total time in markcompact-collect: ",
+ # ct, "seconds")
+ #debug_print("| percentage collection<->total time:",
+ # ct * 100.0 / total_program_time, "%")
debug_print("`----------------------------------------------")
debug_stop("gc-collect")
def update_run_finalizers(self):
- run_finalizers = self.AddressDeque()
- while self.run_finalizers.non_empty():
- obj = self.run_finalizers.popleft()
- run_finalizers.append(self.get_forwarding_address(obj))
- self.run_finalizers.delete()
- self.run_finalizers = run_finalizers
+ if self.run_finalizers.non_empty(): # uncommon case
+ run_finalizers = self.AddressDeque()
+ while self.run_finalizers.non_empty():
+ obj = self.run_finalizers.popleft()
+ run_finalizers.append(self.get_forwarding_address(obj))
+ self.run_finalizers.delete()
+ self.run_finalizers = run_finalizers
+ #
objects_with_finalizers = self.AddressDeque()
while self.objects_with_finalizers.non_empty():
obj = self.objects_with_finalizers.popleft()
@@ -353,90 +360,156 @@
tid = self.header(addr).tid
return llop.extract_ushort(llgroup.HALFWORD, tid)
- def mark_roots_recursively(self):
+ def trace_from_roots(self):
self.root_walker.walk_roots(
- MarkCompactGC._mark_root_recursively, # stack roots
- MarkCompactGC._mark_root_recursively, # static in prebuilt non-gc structures
- MarkCompactGC._mark_root_recursively) # static in prebuilt gc objects
+ MarkCompactGC._mark_root, # stack roots
+ MarkCompactGC._mark_root, # static in prebuilt non-gc structures
+ MarkCompactGC._mark_root) # static in prebuilt gc objects
+ if (self.objects_with_finalizers.non_empty() or
+ self.run_finalizers.non_empty()):
+ self.trace_from_objects_with_finalizers()
self._trace_and_mark()
def _trace_and_mark(self):
- # XXX depth-first tracing... it can consume a lot of rawmalloced
- # memory for very long stacks in some cases
while self.to_see.non_empty():
- obj = self.to_see.pop()
+ obj = self.to_see.popleft()
self.trace(obj, self._mark_obj, None)
def _mark_obj(self, pointer, ignored):
- obj = pointer.address[0]
- if self.marked(obj):
- return
- self.mark(obj)
- self.to_see.append(obj)
+ self.mark(pointer.address[0])
- def _mark_root_recursively(self, root):
+ def _mark_root(self, root):
self.mark(root.address[0])
- self.to_see.append(root.address[0])
def mark(self, obj):
- self.header(obj).tid |= GCFLAG_MARKBIT
+ if not self.marked(obj):
+ self.header(obj).tid |= GCFLAG_MARKBIT
+ self.to_see.append(obj)
def marked(self, obj):
- return self.header(obj).tid & GCFLAG_MARKBIT
+ # should work both if tid contains a CombinedSymbolic (for dying
+ # objects, at this point), or a plain integer.
+ return MovingGCBase.header(self, obj).tid & GCFLAG_MARKBIT
+
+ def toaddr_smaller_than_fromaddr(self, toaddr, fromaddr):
+ if translated_to_c():
+ return toaddr < fromaddr
+ else:
+ # convert the addresses to integers, because they are
+ # theoretically not from the same arena
+ return toaddr - self.base_forwarding_addr < fromaddr - self.space
- def update_forward_pointers(self, toaddr, num_of_alive_objs):
- self.base_forwarding_addr = toaddr
+ def update_forward_pointers(self, toaddr, maxnum):
+ self.base_forwarding_addr = base_forwarding_addr = toaddr
fromaddr = self.space
size_gc_header = self.gcheaderbuilder.size_gc_header
- i = 0
+ num = 0
while fromaddr < self.free:
hdr = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR))
obj = fromaddr + size_gc_header
- objsize = self.get_size(obj)
- totalsize = size_gc_header + objsize
- if not self.marked(obj):
- self.set_null_forwarding_address(obj, i)
- else:
- llarena.arena_reserve(toaddr, totalsize)
- self.set_forwarding_address(obj, toaddr, i)
- toaddr += totalsize
- i += 1
- fromaddr += totalsize
+ # compute the original object size, including the
+ # optional hash field
+ basesize = size_gc_header + self.get_size(obj)
+ totalsrcsize = basesize
+ if hdr.tid & GCFLAG_HASHFIELD: # already a hash field, copy it too
+ totalsrcsize += llmemory.sizeof(lltype.Signed)
+ #
+ if self.marked(obj):
+ # the object is marked as suriving. Compute the new object
+ # size
+ totaldstsize = totalsrcsize
+ if (hdr.tid & (GCFLAG_HASHTAKEN|GCFLAG_HASHFIELD) ==
+ GCFLAG_HASHTAKEN):
+ # grow a new hash field -- with the exception: if
+ # the object actually doesn't move, don't
+ # (otherwise, we get a bogus toaddr > fromaddr)
+ if self.toaddr_smaller_than_fromaddr(toaddr, fromaddr):
+ totaldstsize += llmemory.sizeof(lltype.Signed)
+ #
+ if not translated_to_c():
+ llarena.arena_reserve(toaddr, basesize)
+ if (raw_malloc_usage(totaldstsize) >
+ raw_malloc_usage(basesize)):
+ llarena.arena_reserve(toaddr + basesize,
+ llmemory.sizeof(lltype.Signed))
+ #
+ # save the field hdr.tid in the array tid_backup
+ ll_assert(num < maxnum, "overflow of the tid_backup table")
+ self.tid_backup[num] = self.get_type_id(obj)
+ num += 1
+ # compute forward_offset, the offset to the future copy
+ # of this object
+ forward_offset = toaddr - base_forwarding_addr
+ # copy the first two gc flags in forward_offset
+ ll_assert(forward_offset & 3 == 0, "misalignment!")
+ forward_offset |= (hdr.tid >> first_gcflag_bit) & 3
+ hdr.tid = forward_offset | GCFLAG_MARKBIT
+ ll_assert(self.marked(obj), "re-marking object failed!")
+ # done
+ toaddr += totaldstsize
+ #
+ fromaddr += totalsrcsize
+ if not translated_to_c():
+ assert toaddr - base_forwarding_addr <= fromaddr - self.space
+ self.num_alive_objs = num
+ self.finaladdr = toaddr
# now update references
self.root_walker.walk_roots(
- MarkCompactGC._update_root, # stack roots
- MarkCompactGC._update_root, # static in prebuilt non-gc structures
- MarkCompactGC._update_root) # static in prebuilt gc objects
+ MarkCompactGC._update_ref, # stack roots
+ MarkCompactGC._update_ref, # static in prebuilt non-gc structures
+ MarkCompactGC._update_ref) # static in prebuilt gc objects
+ self.walk_marked_objects(MarkCompactGC.trace_and_update_ref)
+
+ def walk_marked_objects(self, callback):
+ num = 0
+ size_gc_header = self.gcheaderbuilder.size_gc_header
fromaddr = self.space
- i = 0
+ toaddr = self.base_forwarding_addr
while fromaddr < self.free:
hdr = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR))
obj = fromaddr + size_gc_header
- objsize = self.get_size_from_backup(obj, i)
- totalsize = size_gc_header + objsize
- if not self.surviving(obj):
- pass
+ survives = self.marked(obj)
+ if survives:
+ typeid = self.get_typeid_from_backup(num)
+ num += 1
else:
- self.trace_with_backup(obj, self._update_ref, i)
- fromaddr += totalsize
- i += 1
- return toaddr
+ typeid = self.get_type_id(obj)
+ baseobjsize = self._get_size_for_typeid(obj, typeid)
+ basesize = size_gc_header + baseobjsize
+ totalsrcsize = basesize
+ #
+ if survives:
+ grow_hash_field = False
+ if hdr.tid & GCFLAG_SAVED_HASHFIELD:
+ totalsrcsize += llmemory.sizeof(lltype.Signed)
+ totaldstsize = totalsrcsize
+ if (hdr.tid & (GCFLAG_SAVED_HASHTAKEN|GCFLAG_SAVED_HASHFIELD)
+ == GCFLAG_SAVED_HASHTAKEN):
+ if self.toaddr_smaller_than_fromaddr(toaddr, fromaddr):
+ grow_hash_field = True
+ totaldstsize += llmemory.sizeof(lltype.Signed)
+ callback(self, obj, typeid, basesize, toaddr, grow_hash_field)
+ toaddr += totaldstsize
+ else:
+ if hdr.tid & GCFLAG_HASHFIELD:
+ totalsrcsize += llmemory.sizeof(lltype.Signed)
+ #
+ fromaddr += totalsrcsize
+ walk_marked_objects._annspecialcase_ = 'specialize:arg(1)'
- def trace_with_backup(self, obj, callback, arg):
+ def trace_and_update_ref(self, obj, typeid, _1, _2, _3):
"""Enumerate the locations inside the given obj that can contain
GC pointers. For each such location, callback(pointer, arg) is
called, where 'pointer' is an address inside the object.
Typically, 'callback' is a bound method and 'arg' can be None.
"""
- typeid = self.get_typeid_from_backup(arg)
if self.is_gcarrayofgcptr(typeid):
# a performance shortcut for GcArray(gcptr)
length = (obj + llmemory.gcarrayofptr_lengthoffset).signed[0]
item = obj + llmemory.gcarrayofptr_itemsoffset
while length > 0:
- if self.points_to_valid_gc_object(item):
- callback(item, arg)
+ self._update_ref(item)
item += llmemory.gcarrayofptr_singleitemoffset
length -= 1
return
@@ -444,8 +517,7 @@
i = 0
while i < len(offsets):
item = obj + offsets[i]
- if self.points_to_valid_gc_object(item):
- callback(item, arg)
+ self._update_ref(item)
i += 1
if self.has_gcptr_in_varsize(typeid):
item = obj + self.varsize_offset_to_variable_part(typeid)
@@ -456,171 +528,122 @@
j = 0
while j < len(offsets):
itemobj = item + offsets[j]
- if self.points_to_valid_gc_object(itemobj):
- callback(itemobj, arg)
+ self._update_ref(itemobj)
j += 1
item += itemlength
length -= 1
- trace_with_backup._annspecialcase_ = 'specialize:arg(2)'
-
- def _update_root(self, pointer):
- if pointer.address[0] != NULL:
- pointer.address[0] = self.get_forwarding_address(pointer.address[0])
-
- def _update_ref(self, pointer, ignore):
- if pointer.address[0] != NULL:
- pointer.address[0] = self.get_forwarding_address(pointer.address[0])
+ else:
+ weakofs = self.weakpointer_offset(typeid)
+ if weakofs >= 0:
+ self._update_weakref(obj + weakofs)
+
+ def _update_ref(self, pointer):
+ if self.points_to_valid_gc_object(pointer):
+ pointer.address[0] = self.get_forwarding_address(
+ pointer.address[0])
+
+ def _update_weakref(self, pointer):
+ # either update the weak pointer's destination, or
+ # if it dies, write a NULL
+ if self.points_to_valid_gc_object(pointer):
+ if self.marked(pointer.address[0]):
+ pointer.address[0] = self.get_forwarding_address(
+ pointer.address[0])
+ else:
+ pointer.address[0] = NULL
def _is_external(self, obj):
- return not (self.space <= obj < self.top_of_space)
+ return not (self.space <= obj < self.free)
def get_forwarding_address(self, obj):
if self._is_external(obj):
return obj
return self.get_header_forwarded_addr(obj)
- def set_null_forwarding_address(self, obj, num):
- self.backup_typeid(num, obj)
- hdr = self.header(obj)
- hdr.tid = -1 # make the object forwarded to NULL
-
- def set_forwarding_address(self, obj, newobjhdr, num):
- self.backup_typeid(num, obj)
- forward_offset = newobjhdr - self.base_forwarding_addr
- hdr = self.header(obj)
- hdr.tid = forward_offset # make the object forwarded to newobj
-
- def restore_normal_header(self, obj, num):
- # Reverse of set_forwarding_address().
- typeid16 = self.get_typeid_from_backup(num)
- hdr = self.header_forwarded(obj)
- hdr.tid = self.combine(typeid16, 0) # restore the normal header
-
def get_header_forwarded_addr(self, obj):
- return (self.base_forwarding_addr +
- self.header_forwarded(obj).tid +
- self.gcheaderbuilder.size_gc_header)
+ tid = self.header_forwarded(obj).tid
+ ll_assert(tid & GCFLAG_MARKBIT != 0, "dying object is not forwarded")
+ GCFLAG_MASK = ~(GCFLAG_MARKBIT | 3)
+ res = (self.base_forwarding_addr + (tid & GCFLAG_MASK) +
+ self.gcheaderbuilder.size_gc_header)
+ ll_assert(res < self.finaladdr, "forwarded address >= self.finaladdr")
+ return res
def surviving(self, obj):
- return self._is_external(obj) or self.header_forwarded(obj).tid != -1
-
- def backup_typeid(self, num, obj):
- self.tid_backup[num] = self.get_type_id(obj)
+ return self.marked(obj)
def get_typeid_from_backup(self, num):
return self.tid_backup[num]
- def get_size_from_backup(self, obj, num):
- typeid = self.get_typeid_from_backup(num)
- size = self.fixed_size(typeid)
- if self.is_varsize(typeid):
- lenaddr = obj + self.varsize_offset_to_length(typeid)
- length = lenaddr.signed[0]
- size += length * self.varsize_item_sizes(typeid)
- size = llarena.round_up_for_allocation(size)
- # XXX maybe we should parametrize round_up_for_allocation()
- # per GC; if we do, we also need to fix the call in
- # gctypelayout.encode_type_shape()
- return size
+ def compact(self):
+ self.walk_marked_objects(MarkCompactGC.copy_and_compact)
- def compact(self, resizing):
- fromaddr = self.space
- size_gc_header = self.gcheaderbuilder.size_gc_header
- start = fromaddr
- end = fromaddr
- num = 0
- while fromaddr < self.free:
- obj = fromaddr + size_gc_header
- objsize = self.get_size_from_backup(obj, num)
- totalsize = size_gc_header + objsize
- if not self.surviving(obj):
- # this object dies. Following line is a noop in C,
- # we clear it to make debugging easier
- llarena.arena_reset(fromaddr, totalsize, False)
- else:
- if resizing:
- end = fromaddr
- forward_obj = self.get_header_forwarded_addr(obj)
- self.restore_normal_header(obj, num)
- if obj != forward_obj:
- #llop.debug_print(lltype.Void, "Copying from to",
- # fromaddr, forward_ptr, totalsize)
- llmemory.raw_memmove(fromaddr,
- forward_obj - size_gc_header,
- totalsize)
- if resizing and end - start > GC_CLEARANCE:
- diff = end - start
- #llop.debug_print(lltype.Void, "Cleaning", start, diff)
- diff = (diff / GC_CLEARANCE) * GC_CLEARANCE
- #llop.debug_print(lltype.Void, "Cleaning", start, diff)
- end = start + diff
- if we_are_translated():
- # XXX wuaaaaa.... those objects are freed incorrectly
- # here in case of test_gc
- llarena.arena_reset(start, diff, True)
- start += diff
- num += 1
- fromaddr += totalsize
+ def copy_and_compact(self, obj, typeid, basesize, toaddr, grow_hash_field):
+ # 'basesize' is the size without any hash field
+ # restore the normal header
+ hdr = self.header_forwarded(obj)
+ gcflags = hdr.tid & 3
+ if grow_hash_field:
+ gcflags |= GCFLAG_SAVED_HASHFIELD
+ hashvalue = self.get_identityhash_from_addr(obj)
+ elif gcflags & GCFLAG_SAVED_HASHFIELD:
+ fromaddr = llarena.getfakearenaaddress(obj)
+ fromaddr -= self.gcheaderbuilder.size_gc_header
+ hashvalue = (fromaddr + basesize).signed[0]
+ else:
+ hashvalue = 0 # not used
+ #
+ hdr.tid = self.combine(typeid, gcflags << first_gcflag_bit)
+ #
+ fromaddr = obj - self.gcheaderbuilder.size_gc_header
+ if translated_to_c():
+ llmemory.raw_memmove(fromaddr, toaddr, basesize)
+ else:
+ llmemory.raw_memcopy(fromaddr, toaddr, basesize)
+ #
+ if gcflags & GCFLAG_SAVED_HASHFIELD:
+ (toaddr + basesize).signed[0] = hashvalue
def debug_check_object(self, obj):
- # not sure what to check here
- pass
-
- def mark_objects_with_finalizers(self):
+ type_id = self.get_type_id(obj)
+ self.has_gcptr_in_varsize(type_id) # checks that the type_id is valid
+ #
+ tid = self.header(obj).tid
+ if self._is_external(obj):
+ # All external objects have GCFLAG_MARKBIT and GCFLAG_HASHTAKEN
+ # set.
+ assert tid & GCFLAG_MARKBIT
+ assert tid & GCFLAG_HASHTAKEN
+ else:
+ # Non-external objects have GCFLAG_MARKBIT that should not be set
+ # at the very start or at the very end of a collection -- only
+ # temporarily during the collection.
+ assert tid & GCFLAG_MARKBIT == 0
+
+ def trace_from_objects_with_finalizers(self):
+ if self.run_finalizers.non_empty(): # uncommon case
+ new_run_finalizers = self.AddressDeque()
+ while self.run_finalizers.non_empty():
+ x = self.run_finalizers.popleft()
+ self.mark(x)
+ new_run_finalizers.append(x)
+ self.run_finalizers.delete()
+ self.run_finalizers = new_run_finalizers
+ #
+ # xxx we get to run the finalizers in a random order
+ self._trace_and_mark()
new_with_finalizers = self.AddressDeque()
- run_finalizers = self.run_finalizers
- new_run_finalizers = self.AddressDeque()
- while run_finalizers.non_empty():
- x = run_finalizers.popleft()
- self.mark(x)
- self.to_see.append(x)
- new_run_finalizers.append(x)
- run_finalizers.delete()
- self.run_finalizers = new_run_finalizers
while self.objects_with_finalizers.non_empty():
x = self.objects_with_finalizers.popleft()
if self.marked(x):
new_with_finalizers.append(x)
else:
- new_run_finalizers.append(x)
+ self.run_finalizers.append(x)
self.mark(x)
- self.to_see.append(x)
+ self._trace_and_mark()
self.objects_with_finalizers.delete()
self.objects_with_finalizers = new_with_finalizers
- def invalidate_weakrefs(self, weakref_offsets):
- # walk over list of objects that contain weakrefs
- # if the object it references survives then update the weakref
- # otherwise invalidate the weakref
- new_with_weakref = self.AddressStack()
- i = 0
- while self.objects_with_weakrefs.non_empty():
- obj = self.objects_with_weakrefs.pop()
- if not self.surviving(obj):
- continue # weakref itself dies
- newobj = self.get_forwarding_address(obj)
- offset = weakref_offsets[i]
- pointing_to = (obj + offset).address[0]
- # XXX I think that pointing_to cannot be NULL here
- if pointing_to:
- if self.surviving(pointing_to):
- (obj + offset).address[0] = self.get_forwarding_address(
- pointing_to)
- new_with_weakref.append(newobj)
- else:
- (obj + offset).address[0] = NULL
- i += 1
- self.objects_with_weakrefs.delete()
- self.objects_with_weakrefs = new_with_weakref
- lltype.free(weakref_offsets, flavor='raw')
-
- def get_size_incl_hash(self, obj):
- size = self.get_size(obj)
- hdr = self.header(obj)
- if hdr.tid & GCFLAG_HASHFIELD:
- size += llmemory.sizeof(lltype.Signed)
- return size
-
def identityhash(self, gcobj):
# Unlike SemiSpaceGC.identityhash(), this function does not have
# to care about reducing top_of_space. The reason is as
@@ -635,8 +658,23 @@
hdr = self.header(obj)
#
if hdr.tid & GCFLAG_HASHFIELD: # the hash is in a field at the end
- obj += self.get_size(obj)
+ obj = llarena.getfakearenaaddress(obj) + self.get_size(obj)
return obj.signed[0]
#
hdr.tid |= GCFLAG_HASHTAKEN
- return llmemory.cast_adr_to_int(obj) # direct case
+ return self.get_identityhash_from_addr(obj)
+
+ def get_identityhash_from_addr(self, obj):
+ if translated_to_c():
+ return llmemory.cast_adr_to_int(obj) # direct case
+ else:
+ try:
+ adr = llarena.getfakearenaaddress(obj) # -> arena address
+ except RuntimeError:
+ return llmemory.cast_adr_to_int(obj) # not in an arena...
+ return adr - self.space
+
+# ____________________________________________________________
+
+class CannotAllocateGCArena(Exception):
+ pass
Modified: pypy/branch/jit-bounds/pypy/rpython/memory/gc/test/test_direct.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/memory/gc/test/test_direct.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/memory/gc/test/test_direct.py Tue Sep 7 19:22:31 2010
@@ -67,7 +67,10 @@
from pypy.config.pypyoption import get_pypy_config
config = get_pypy_config(translating=True).translation
self.stackroots = []
- self.gc = self.GCClass(config, **self.GC_PARAMS)
+ GC_PARAMS = self.GC_PARAMS.copy()
+ if hasattr(meth, 'GC_PARAMS'):
+ GC_PARAMS.update(meth.GC_PARAMS)
+ self.gc = self.GCClass(config, **GC_PARAMS)
self.gc.DEBUG = True
self.rootwalker = DirectRootWalker(self)
self.gc.set_root_walker(self.rootwalker)
@@ -96,7 +99,7 @@
p[index] = newvalue
def malloc(self, TYPE, n=None):
- addr = self.gc.malloc(self.get_type_id(TYPE), n)
+ addr = self.gc.malloc(self.get_type_id(TYPE), n, zero=True)
return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE))
def test_simple(self):
@@ -311,7 +314,18 @@
print hash
assert isinstance(hash, (int, long))
assert hash == self.gc.identityhash(p_const)
-
+ # (5) p is actually moving (for the markcompact gc)
+ p0 = self.malloc(S)
+ self.stackroots.append(p0)
+ p = self.malloc(S)
+ self.stackroots.append(p)
+ hash = self.gc.identityhash(p)
+ self.stackroots.pop(-2)
+ self.gc.collect() # p0 goes away, p shifts left
+ assert hash == self.gc.identityhash(self.stackroots[-1])
+ self.gc.collect()
+ assert hash == self.gc.identityhash(self.stackroots[-1])
+ self.stackroots.pop()
class TestSemiSpaceGC(DirectGCTest):
from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass
@@ -431,3 +445,14 @@
class TestMarkCompactGC(DirectGCTest):
from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass
+ def test_many_objects(self):
+ DirectGCTest.test_many_objects(self)
+ test_many_objects.GC_PARAMS = {'space_size': 3 * 1024 * WORD}
+
+ def test_varsized_from_stack(self):
+ DirectGCTest.test_varsized_from_stack(self)
+ test_varsized_from_stack.GC_PARAMS = {'space_size': 2 * 1024 * WORD}
+
+ def test_varsized_from_prebuilt_gc(self):
+ DirectGCTest.test_varsized_from_prebuilt_gc(self)
+ test_varsized_from_prebuilt_gc.GC_PARAMS = {'space_size': 3 * 1024 * WORD}
Modified: pypy/branch/jit-bounds/pypy/rpython/memory/gctypelayout.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/memory/gctypelayout.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/memory/gctypelayout.py Tue Sep 7 19:22:31 2010
@@ -44,16 +44,18 @@
self.type_info_group_ptr = type_info_group._as_ptr()
def get(self, typeid):
- _check_typeid(typeid)
- return llop.get_group_member(GCData.TYPE_INFO_PTR,
- self.type_info_group_ptr,
- typeid)
+ res = llop.get_group_member(GCData.TYPE_INFO_PTR,
+ self.type_info_group_ptr,
+ typeid)
+ _check_valid_type_info(res)
+ return res
def get_varsize(self, typeid):
- _check_typeid(typeid)
- return llop.get_group_member(GCData.VARSIZE_TYPE_INFO_PTR,
- self.type_info_group_ptr,
- typeid)
+ res = llop.get_group_member(GCData.VARSIZE_TYPE_INFO_PTR,
+ self.type_info_group_ptr,
+ typeid)
+ _check_valid_type_info_varsize(res)
+ return res
def q_is_varsize(self, typeid):
infobits = self.get(typeid).infobits
@@ -115,13 +117,24 @@
# the lowest 16bits are used to store group member index
-T_MEMBER_INDEX = 0xffff
+T_MEMBER_INDEX = 0xffff
T_IS_VARSIZE = 0x10000
T_HAS_GCPTR_IN_VARSIZE = 0x20000
T_IS_GCARRAY_OF_GCPTR = 0x40000
T_IS_WEAKREF = 0x80000
+T_KEY_MASK = 0xFF000000
+T_KEY_VALUE = 0x7A000000 # bug detection only
-def _check_typeid(typeid):
+def _check_valid_type_info(p):
+ ll_assert(p.infobits & T_KEY_MASK == T_KEY_VALUE, "invalid type_id")
+
+def _check_valid_type_info_varsize(p):
+ ll_assert(p.header.infobits & (T_KEY_MASK | T_IS_VARSIZE) ==
+ (T_KEY_VALUE | T_IS_VARSIZE),
+ "invalid varsize type_id")
+
+def check_typeid(typeid):
+ # xxx does not perform a full check of validity, just checks for nonzero
ll_assert(llop.is_group_member_nonzero(lltype.Bool, typeid),
"invalid type_id")
@@ -165,9 +178,9 @@
infobits |= T_HAS_GCPTR_IN_VARSIZE
varinfo.varofstoptrs = builder.offsets2table(offsets, ARRAY.OF)
varinfo.varitemsize = llmemory.sizeof(ARRAY.OF)
- if TYPE == WEAKREF:
+ if builder.is_weakref_type(TYPE):
infobits |= T_IS_WEAKREF
- info.infobits = infobits
+ info.infobits = infobits | T_KEY_VALUE
# ____________________________________________________________
@@ -250,17 +263,21 @@
_, TYPE = TYPE._first_struct()
def get_info(self, type_id):
- return llop.get_group_member(GCData.TYPE_INFO_PTR,
- self.type_info_group._as_ptr(),
- type_id)
+ res = llop.get_group_member(GCData.TYPE_INFO_PTR,
+ self.type_info_group._as_ptr(),
+ type_id)
+ _check_valid_type_info(res)
+ return res
def get_info_varsize(self, type_id):
- return llop.get_group_member(GCData.VARSIZE_TYPE_INFO_PTR,
- self.type_info_group._as_ptr(),
- type_id)
+ res = llop.get_group_member(GCData.VARSIZE_TYPE_INFO_PTR,
+ self.type_info_group._as_ptr(),
+ type_id)
+ _check_valid_type_info_varsize(res)
+ return res
- def is_weakref(self, type_id):
- return self.get_info(type_id).infobits & T_IS_WEAKREF
+ def is_weakref_type(self, TYPE):
+ return TYPE == WEAKREF
def encode_type_shapes_now(self):
if not self.can_encode_type_shape:
Modified: pypy/branch/jit-bounds/pypy/rpython/memory/gcwrapper.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/memory/gcwrapper.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/memory/gcwrapper.py Tue Sep 7 19:22:31 2010
@@ -119,6 +119,9 @@
else:
return True
+ def pyobjectptr(self, klass):
+ raise NotImplementedError(klass)
+
# ____________________________________________________________
class LLInterpRootWalker:
Modified: pypy/branch/jit-bounds/pypy/rpython/memory/test/test_gc.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/memory/test/test_gc.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/memory/test/test_gc.py Tue Sep 7 19:22:31 2010
@@ -639,12 +639,14 @@
class TestMarkCompactGC(TestSemiSpaceGC):
from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass
+ GC_PARAMS = {'space_size': 65536+16384}
+ GC_CAN_SHRINK_ARRAY = False
def test_finalizer_order(self):
py.test.skip("Not implemented yet")
-
-class TestMarkCompactGCGrowing(TestMarkCompactGC):
- GC_PARAMS = {'space_size': 16*WORD}
+ def test_writebarrier_before_copy(self):
+ py.test.skip("Not relevant, and crashes because llarena does not "
+ "support empty GcStructs")
class TestHybridGC(TestGenerationalGC):
from pypy.rpython.memory.gc.hybrid import HybridGC as GCClass
Modified: pypy/branch/jit-bounds/pypy/rpython/memory/test/test_gctypelayout.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/memory/test/test_gctypelayout.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/memory/test/test_gctypelayout.py Tue Sep 7 19:22:31 2010
@@ -101,7 +101,7 @@
accessor = rclass.FieldListAccessor()
S3 = lltype.GcStruct('S', ('x', PT), ('y', PT),
hints={'immutable_fields': accessor})
- accessor.initialize(S3, ['x'])
+ accessor.initialize(S3, {'x': ''})
#
s1 = lltype.malloc(S1)
adr = llmemory.cast_ptr_to_adr(s1)
Modified: pypy/branch/jit-bounds/pypy/rpython/memory/test/test_transformed_gc.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/memory/test/test_transformed_gc.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/memory/test/test_transformed_gc.py Tue Sep 7 19:22:31 2010
@@ -1138,15 +1138,16 @@
class TestMarkCompactGC(GenericMovingGCTests):
gcname = 'markcompact'
- def setup_class(cls):
- py.test.skip("Disabled for now, sorry")
-
class gcpolicy(gc.FrameworkGcPolicy):
class transformerclass(framework.FrameworkGCTransformer):
from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass
- GC_PARAMS = {'space_size': 512*WORD}
+ GC_PARAMS = {'space_size': 4096*WORD}
root_stack_depth = 200
+ def test_writebarrier_before_copy(self):
+ py.test.skip("Not relevant, and crashes because llarena does not "
+ "support empty GcStructs")
+
class TestGenerationGC(GenericMovingGCTests):
gcname = "generation"
GC_CAN_SHRINK_ARRAY = True
@@ -1536,3 +1537,12 @@
GC_PARAMS = {'space_size': 512*WORD,
'nursery_size': 32*WORD}
root_stack_depth = 200
+
+class TestMarkCompactTaggedpointerGC(TaggedPointerGCTests):
+ gcname = 'markcompact'
+
+ class gcpolicy(gc.FrameworkGcPolicy):
+ class transformerclass(framework.FrameworkGCTransformer):
+ from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass
+ GC_PARAMS = {'space_size': 4096*WORD}
+ root_stack_depth = 200
Modified: pypy/branch/jit-bounds/pypy/rpython/module/ll_time.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/module/ll_time.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/module/ll_time.py Tue Sep 7 19:22:31 2010
@@ -9,6 +9,7 @@
from pypy.rpython.lltypesystem import lltype
from pypy.rpython.extfunc import BaseLazyRegistering, registering, extdef
from pypy.rlib import rposix
+from pypy.rlib.rarithmetic import intmask
from pypy.translator.tool.cbuild import ExternalCompilationInfo
if sys.platform == 'win32':
@@ -119,7 +120,8 @@
if self.HAVE_FTIME:
t = lltype.malloc(self.TIMEB, flavor='raw')
c_ftime(t)
- result = float(int(t.c_time)) + float(int(t.c_millitm)) * 0.001
+ result = (float(intmask(t.c_time)) +
+ float(intmask(t.c_millitm)) * 0.001)
lltype.free(t, flavor='raw')
return result
return float(c_time(void))
Modified: pypy/branch/jit-bounds/pypy/rpython/ootypesystem/ootype.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/ootypesystem/ootype.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/ootypesystem/ootype.py Tue Sep 7 19:22:31 2010
@@ -267,6 +267,14 @@
return self._fields_with_default[:]
return self._superclass._get_fields_with_default() + self._fields_with_default
+ def _immutable_field(self, field):
+ if 'immutable_fields' in self._hints:
+ try:
+ s = self._hints['immutable_fields'].fields[field]
+ return s or True
+ except KeyError:
+ pass
+ return self._hints.get('immutable', False)
class SpecializableType(OOType):
Modified: pypy/branch/jit-bounds/pypy/rpython/ootypesystem/rclass.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/ootypesystem/rclass.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/ootypesystem/rclass.py Tue Sep 7 19:22:31 2010
@@ -194,6 +194,7 @@
self.lowleveltype._hints.update(hints)
if self.classdef is None:
+ self.fields = {}
self.allfields = {}
self.allmethods = {}
self.allclassattributes = {}
@@ -210,6 +211,7 @@
allclassattributes = {}
fields = {}
+ nonmangledfields = []
fielddefaults = {}
if llfields:
@@ -224,6 +226,7 @@
allfields[mangled] = repr
oot = repr.lowleveltype
fields[mangled] = oot
+ nonmangledfields.append(name)
try:
value = self.classdef.classdesc.read_attribute(name)
fielddefaults[mangled] = repr.convert_desc_or_const(value)
@@ -294,6 +297,7 @@
if not attrdef.s_value.is_constant():
classattributes[mangled] = attrdef.s_value, value
+ self.fields = nonmangledfields
self.allfields = allfields
self.allmethods = allmethods
self.allclassattributes = allclassattributes
Modified: pypy/branch/jit-bounds/pypy/rpython/rbuiltin.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/rbuiltin.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/rbuiltin.py Tue Sep 7 19:22:31 2010
@@ -542,16 +542,25 @@
return hop.genop('raw_malloc_usage', [v_size], resulttype=lltype.Signed)
def rtype_raw_free(hop):
+ s_addr = hop.args_s[0]
+ if s_addr.is_null_address():
+ raise TyperError("raw_free(x) where x is the constant NULL")
v_addr, = hop.inputargs(llmemory.Address)
hop.exception_cannot_occur()
return hop.genop('raw_free', [v_addr])
def rtype_raw_memcopy(hop):
+ for s_addr in hop.args_s[:2]:
+ if s_addr.is_null_address():
+ raise TyperError("raw_memcopy() with a constant NULL")
v_list = hop.inputargs(llmemory.Address, llmemory.Address, lltype.Signed)
hop.exception_cannot_occur()
return hop.genop('raw_memcopy', v_list)
def rtype_raw_memclear(hop):
+ s_addr = hop.args_s[0]
+ if s_addr.is_null_address():
+ raise TyperError("raw_memclear(x, n) where x is the constant NULL")
v_list = hop.inputargs(llmemory.Address, lltype.Signed)
return hop.genop('raw_memclear', v_list)
Modified: pypy/branch/jit-bounds/pypy/rpython/rclass.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/rclass.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/rclass.py Tue Sep 7 19:22:31 2010
@@ -9,6 +9,7 @@
class FieldListAccessor(object):
def initialize(self, TYPE, fields):
+ assert type(fields) is dict
self.TYPE = TYPE
self.fields = fields
@@ -18,6 +19,10 @@
def _freeze_(self):
return True
+class ImmutableConflictError(Exception):
+ """Raised when the _immutable_ or _immutable_fields_ hints are
+ not consistent across a class hierarchy."""
+
def getclassrepr(rtyper, classdef):
try:
@@ -153,7 +158,7 @@
pass
def _check_for_immutable_hints(self, hints):
- if '_immutable_' in self.classdef.classdesc.classdict:
+ if self.classdef.classdesc.lookup('_immutable_') is not None:
hints = hints.copy()
hints['immutable'] = True
self.immutable_field_list = [] # unless overwritten below
@@ -182,16 +187,20 @@
return 'InstanceR %s' % (clsname,)
def _setup_repr_final(self):
+ self._setup_immutable_field_list()
+ self._check_for_immutable_conflicts()
+
+ def _setup_immutable_field_list(self):
hints = self.object_type._hints
if "immutable_fields" in hints:
accessor = hints["immutable_fields"]
- immutable_fields = {}
- rbase = self
- while rbase.classdef is not None:
- immutable_fields.update(
- dict.fromkeys(rbase.immutable_field_list))
- rbase = rbase.rbase
- self._parse_field_list(immutable_fields, accessor)
+ if not hasattr(accessor, 'fields'):
+ immutable_fields = []
+ rbase = self
+ while rbase.classdef is not None:
+ immutable_fields += rbase.immutable_field_list
+ rbase = rbase.rbase
+ self._parse_field_list(immutable_fields, accessor)
def _parse_field_list(self, fields, accessor):
with_suffix = {}
@@ -209,6 +218,36 @@
accessor.initialize(self.object_type, with_suffix)
return with_suffix
+ def _check_for_immutable_conflicts(self):
+ # check for conflicts, i.e. a field that is defined normally as
+ # mutable in some parent class but that is now declared immutable
+ from pypy.rpython.lltypesystem.lltype import Void
+ is_self_immutable = "immutable" in self.object_type._hints
+ base = self
+ while base.classdef is not None:
+ base = base.rbase
+ for fieldname in base.fields:
+ try:
+ mangled, r = base._get_field(fieldname)
+ except KeyError:
+ continue
+ if r.lowleveltype == Void:
+ continue
+ base._setup_immutable_field_list()
+ if base.object_type._immutable_field(mangled):
+ continue
+ # 'fieldname' is a mutable, non-Void field in the parent
+ if is_self_immutable:
+ raise ImmutableConflictError(
+ "class %r has _immutable_=True, but parent class %r "
+ "defines (at least) the mutable field %r" % (
+ self, base, fieldname))
+ if fieldname in self.immutable_field_list:
+ raise ImmutableConflictError(
+ "field %r is defined mutable in class %r, but "
+ "listed in _immutable_fields_ in subclass %r" % (
+ fieldname, base, self))
+
def new_instance(self, llops, classcallhop=None):
raise NotImplementedError
Modified: pypy/branch/jit-bounds/pypy/rpython/test/test_rclass.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/rpython/test/test_rclass.py (original)
+++ pypy/branch/jit-bounds/pypy/rpython/test/test_rclass.py Tue Sep 7 19:22:31 2010
@@ -796,27 +796,92 @@
assert accessor.fields == {"inst_y" : ""} or \
accessor.fields == {"oy" : ""} # for ootype
- def test_immutable_inheritance(self):
- class I(object):
- def __init__(self, v):
- self.v = v
-
- class J(I):
+ def test_immutable_forbidden_inheritance_1(self):
+ from pypy.rpython.rclass import ImmutableConflictError
+ class A(object):
+ pass
+ class B(A):
+ _immutable_fields_ = ['v']
+ def f():
+ A().v = 123
+ B() # crash: class B says 'v' is immutable,
+ # but it is defined on parent class A
+ py.test.raises(ImmutableConflictError, self.gengraph, f, [])
+
+ def test_immutable_forbidden_inheritance_2(self):
+ from pypy.rpython.rclass import ImmutableConflictError
+ class A(object):
+ pass
+ class B(A):
+ _immutable_ = True
+ def f():
+ A().v = 123
+ B() # crash: class B has _immutable_ = True
+ # but class A defines 'v' to be mutable
+ py.test.raises(ImmutableConflictError, self.gengraph, f, [])
+
+ def test_immutable_ok_inheritance_2(self):
+ from pypy.jit.metainterp.typesystem import deref
+ class A(object):
+ _immutable_fields_ = ['v']
+ class B(A):
+ _immutable_ = True
+ def f():
+ A().v = 123
+ B().w = 456
+ return B()
+ t, typer, graph = self.gengraph(f, [])
+ B_TYPE = deref(graph.getreturnvar().concretetype)
+ assert B_TYPE._hints["immutable"]
+ try:
+ A_TYPE = B_TYPE.super
+ except AttributeError:
+ A_TYPE = B_TYPE._superclass # for ootype
+ accessor = A_TYPE._hints["immutable_fields"]
+ assert accessor.fields == {"inst_v" : ""} or \
+ accessor.fields == {"ov" : ""} # for ootype
+
+ def test_immutable_subclass_1(self):
+ from pypy.jit.metainterp.typesystem import deref
+ class A(object):
+ _immutable_ = True
+ class B(A):
+ pass
+ def f():
+ B().v = 123
+ return B()
+ t, typer, graph = self.gengraph(f, [])
+ B_TYPE = deref(graph.getreturnvar().concretetype)
+ assert B_TYPE._hints["immutable"] # inherited from A
+
+ def test_immutable_subclass_2(self):
+ from pypy.jit.metainterp.typesystem import deref
+ class A(object):
+ pass
+ class B(A):
_immutable_ = True
- def __init__(self, v, w):
- self.w = w
- I.__init__(self, v)
-
- j = J(3, 4)
- def f():
- j.v = j.v * 1 # make the annotator think it is mutated
- j.w = j.w * 1 # make the annotator think it is mutated
- return j.v + j.w
-
- t, typer, graph = self.gengraph(f, [], backendopt=True)
- f_summary = summary(graph)
- assert f_summary == {"setfield": 2} or \
- f_summary == {"oosetfield": 2} # for ootype
+ def f():
+ B().v = 123
+ return B()
+ t, typer, graph = self.gengraph(f, [])
+ B_TYPE = deref(graph.getreturnvar().concretetype)
+ assert B_TYPE._hints["immutable"]
+
+ def test_immutable_subclass_void(self):
+ from pypy.jit.metainterp.typesystem import deref
+ class A(object):
+ pass
+ class B(A):
+ _immutable_ = True
+ def myfunc():
+ pass
+ def f():
+ A().f = myfunc # it's ok to add Void attributes to A
+ B().v = 123 # even though only B is declared _immutable_
+ return B()
+ t, typer, graph = self.gengraph(f, [])
+ B_TYPE = deref(graph.getreturnvar().concretetype)
+ assert B_TYPE._hints["immutable"]
class TestLLtype(BaseTestRclass, LLRtypeMixin):
Modified: pypy/branch/jit-bounds/pypy/translator/backendopt/test/test_constfold.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/translator/backendopt/test/test_constfold.py (original)
+++ pypy/branch/jit-bounds/pypy/translator/backendopt/test/test_constfold.py Tue Sep 7 19:22:31 2010
@@ -49,7 +49,7 @@
accessor = rclass.FieldListAccessor()
S2 = lltype.GcStruct('S2', ('x', lltype.Signed),
hints={'immutable_fields': accessor})
- accessor.initialize(S2, ['x'])
+ accessor.initialize(S2, {'x': ''})
test_simple(S2)
Modified: pypy/branch/jit-bounds/pypy/translator/c/database.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/translator/c/database.py (original)
+++ pypy/branch/jit-bounds/pypy/translator/c/database.py Tue Sep 7 19:22:31 2010
@@ -213,7 +213,7 @@
forcename = self.idelayedfunctionnames[obj][0]
node = self.getcontainernode(container,
forcename=forcename)
- assert node.ptrname == forcename
+ assert node.getptrname() == forcename
return forcename
# /hack hack hack
@@ -222,7 +222,7 @@
return '((%s) %d)' % (cdecl(self.gettype(T), ''),
obj._obj)
node = self.getcontainernode(container)
- return node.ptrname
+ return node.getptrname()
else:
return '((%s) NULL)' % (cdecl(self.gettype(T), ''), )
else:
Modified: pypy/branch/jit-bounds/pypy/translator/c/gc.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/translator/c/gc.py (original)
+++ pypy/branch/jit-bounds/pypy/translator/c/gc.py Tue Sep 7 19:22:31 2010
@@ -172,7 +172,9 @@
defnode = db.gettypedefnode(obj.about)
self.implementationtypename = 'void (@)(void *)'
self.name = defnode.gcinfo.static_deallocator
- self.ptrname = '((void (*)(void *)) %s)' % (self.name,)
+
+ def getptrname(self):
+ return '((void (*)(void *)) %s)' % (self.name,)
def enum_dependencies(self):
return []
@@ -266,7 +268,9 @@
defnode = db.gettypedefnode(obj.about)
self.implementationtypename = self.typename
self.name = self.db.namespace.uniquename('g_rtti_v_'+ defnode.barename)
- self.ptrname = '(&%s)' % (self.name,)
+
+ def getptrname(self):
+ return '(&%s)' % (self.name,)
def enum_dependencies(self):
return []
Modified: pypy/branch/jit-bounds/pypy/translator/c/gcc/trackgcroot.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/translator/c/gcc/trackgcroot.py (original)
+++ pypy/branch/jit-bounds/pypy/translator/c/gcc/trackgcroot.py Tue Sep 7 19:22:31 2010
@@ -375,7 +375,7 @@
IGNORE_OPS_WITH_PREFIXES = dict.fromkeys([
'cmp', 'test', 'set', 'sahf', 'lahf', 'cltd', 'cld', 'std',
- 'rep', 'movs', 'lods', 'stos', 'scas', 'cwtl', 'prefetch',
+ 'rep', 'movs', 'lods', 'stos', 'scas', 'cwtl', 'cwde', 'prefetch',
# floating-point operations cannot produce GC pointers
'f',
'cvt', 'ucomi', 'comi', 'subs', 'subp' , 'adds', 'addp', 'xorp',
Modified: pypy/branch/jit-bounds/pypy/translator/c/node.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/translator/c/node.py (original)
+++ pypy/branch/jit-bounds/pypy/translator/c/node.py Tue Sep 7 19:22:31 2010
@@ -77,6 +77,8 @@
if db.gcpolicy.need_no_typeptr():
assert self.fieldnames == ('typeptr',)
self.fieldnames = ()
+ #
+ self.fulltypename = '%s %s @' % (self.typetag, self.name)
def setup(self):
# this computes self.fields
@@ -119,7 +121,7 @@
gcinfo = defaultproperty(computegcinfo)
def gettype(self):
- return '%s %s @' % (self.typetag, self.name)
+ return self.fulltypename
def c_struct_field_name(self, name):
# occasionally overridden in __init__():
@@ -211,6 +213,8 @@
self.name) = db.namespace.uniquename(basename, with_number=with_number,
bare=True)
self.dependencies = {}
+ self.fulltypename = '%s %s @' % (self.typetag, self.name)
+ self.fullptrtypename = '%s %s *@' % (self.typetag, self.name)
def setup(self):
if hasattr(self, 'itemtypename'):
@@ -236,10 +240,10 @@
gcinfo = defaultproperty(computegcinfo)
def gettype(self):
- return '%s %s @' % (self.typetag, self.name)
+ return self.fulltypename
def getptrtype(self):
- return '%s %s *@' % (self.typetag, self.name)
+ return self.fullptrtypename
def access_expr(self, baseexpr, index):
return '%s.items[%s]' % (baseexpr, index)
@@ -336,16 +340,19 @@
if ARRAY._hints.get("render_as_void"):
contained_type = Void
self.itemtypename = db.gettype(contained_type, who_asks=self)
+ self.fulltypename = self.itemtypename.replace('@', '(@)[%d]' %
+ (self.varlength,))
+ self.fullptrtypename = self.itemtypename.replace('@', '*@')
def setup(self):
"""Array loops are forbidden by ForwardReference.become() because
there is no way to declare them in C."""
def gettype(self):
- return self.itemtypename.replace('@', '(@)[%d]' % (self.varlength,))
+ return self.fulltypename
def getptrtype(self):
- return self.itemtypename.replace('@', '*@')
+ return self.fullptrtypename
def access_expr(self, baseexpr, index):
return '%s[%d]' % (baseexpr, index)
@@ -383,17 +390,19 @@
self.LLTYPE = FIXEDARRAY
self.dependencies = {}
self.itemtypename = db.gettype(FIXEDARRAY.OF, who_asks=self)
+ self.fulltypename = self.itemtypename.replace('@', '(@)[%d]' %
+ FIXEDARRAY.length)
+ self.fullptrtypename = self.itemtypename.replace('@', '*@')
def setup(self):
"""Loops are forbidden by ForwardReference.become() because
there is no way to declare them in C."""
def gettype(self):
- FIXEDARRAY = self.FIXEDARRAY
- return self.itemtypename.replace('@', '(@)[%d]' % FIXEDARRAY.length)
+ return self.fulltypename
def getptrtype(self):
- return self.itemtypename.replace('@', '*@')
+ return self.fullptrtypename
def access_expr(self, baseexpr, index, dummy=False):
if not isinstance(index, int):
@@ -469,7 +478,7 @@
if USESLOTS: # keep the number of slots down!
__slots__ = """db obj
typename implementationtypename
- name ptrname
+ name
globalcontainer""".split()
eci_name = '_compilation_info'
@@ -494,7 +503,9 @@
if self.typename != self.implementationtypename:
if db.gettypedefnode(T).extra_union_for_varlength:
self.name += '.b'
- self.ptrname = '(&%s)' % self.name
+
+ def getptrname(self):
+ return '(&%s)' % self.name
def getTYPE(self):
return typeOf(self.obj)
@@ -667,10 +678,10 @@
if USESLOTS:
__slots__ = ()
- def __init__(self, db, T, obj):
- ContainerNode.__init__(self, db, T, obj)
- if barebonearray(T):
- self.ptrname = self.name
+ def getptrname(self):
+ if barebonearray(self.getTYPE()):
+ return self.name
+ return ContainerNode.getptrname(self)
def basename(self):
return 'array'
@@ -728,10 +739,10 @@
if USESLOTS:
__slots__ = ()
- def __init__(self, db, T, obj):
- ContainerNode.__init__(self, db, T, obj)
- if not isinstance(obj, _subarray): # XXX hackish
- self.ptrname = self.name
+ def getptrname(self):
+ if not isinstance(self.obj, _subarray): # XXX hackish
+ return self.name
+ return ContainerNode.getptrname(self)
def basename(self):
T = self.getTYPE()
@@ -812,7 +823,9 @@
self.make_funcgens()
#self.dependencies = {}
self.typename = db.gettype(T) #, who_asks=self)
- self.ptrname = self.name
+
+ def getptrname(self):
+ return self.name
def make_funcgens(self):
self.funcgens = select_function_code_generators(self.obj, self.db, self.name)
@@ -958,7 +971,7 @@
def startupcode(self):
T = self.getTYPE()
- args = [self.ptrname]
+ args = [self.getptrname()]
# XXX how to make this code more generic?
if T.tag == 'ThreadLock':
lock = self.obj.externalobj
@@ -990,13 +1003,15 @@
self.obj = obj
value = obj.value
self.name = self._python_c_name(value)
- self.ptrname = self.name
self.exported_name = self.name
# a list of expressions giving places where this constant PyObject
# must be copied. Normally just in the global variable of the same
# name, but see also StructNode.initializationexpr() :-(
self.where_to_copy_me = []
+ def getptrname(self):
+ return self.name
+
def _python_c_name(self, value):
# just some minimal cases: None and builtin exceptions
if value is None:
Modified: pypy/branch/jit-bounds/pypy/translator/c/test/test_newgc.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/translator/c/test/test_newgc.py (original)
+++ pypy/branch/jit-bounds/pypy/translator/c/test/test_newgc.py Tue Sep 7 19:22:31 2010
@@ -67,9 +67,8 @@
if not fullname.startswith('define'):
continue
keyword = conftest.option.keyword
- if keyword:
- if keyword.startswith('test_'):
- keyword = keyword[len('test_'):]
+ if keyword.startswith('test_'):
+ keyword = keyword[len('test_'):]
if keyword not in fullname:
continue
prefix, name = fullname.split('_', 1)
@@ -1072,21 +1071,66 @@
should_be_moving = True
GC_CAN_SHRINK_ARRAY = False
- def setup_class(cls):
- py.test.skip("Disabled for now")
-
def test_gc_set_max_heap_size(self):
py.test.skip("not implemented")
+ def test_gc_heap_stats(self):
+ py.test.skip("not implemented")
+
def test_finalizer_order(self):
py.test.skip("not implemented")
+ def define_adding_a_hash(cls):
+ from pypy.rlib.objectmodel import compute_identity_hash
+ S1 = lltype.GcStruct('S1', ('x', lltype.Signed))
+ S2 = lltype.GcStruct('S2', ('p1', lltype.Ptr(S1)),
+ ('p2', lltype.Ptr(S1)),
+ ('p3', lltype.Ptr(S1)),
+ ('p4', lltype.Ptr(S1)),
+ ('p5', lltype.Ptr(S1)),
+ ('p6', lltype.Ptr(S1)),
+ ('p7', lltype.Ptr(S1)),
+ ('p8', lltype.Ptr(S1)),
+ ('p9', lltype.Ptr(S1)))
+ def g():
+ lltype.malloc(S1) # forgotten, will be shifted over
+ s2 = lltype.malloc(S2) # a big object, overlaps its old position
+ s2.p1 = lltype.malloc(S1); s2.p1.x = 1010
+ s2.p2 = lltype.malloc(S1); s2.p2.x = 1020
+ s2.p3 = lltype.malloc(S1); s2.p3.x = 1030
+ s2.p4 = lltype.malloc(S1); s2.p4.x = 1040
+ s2.p5 = lltype.malloc(S1); s2.p5.x = 1050
+ s2.p6 = lltype.malloc(S1); s2.p6.x = 1060
+ s2.p7 = lltype.malloc(S1); s2.p7.x = 1070
+ s2.p8 = lltype.malloc(S1); s2.p8.x = 1080
+ s2.p9 = lltype.malloc(S1); s2.p9.x = 1090
+ return s2
+ def f():
+ rgc.collect()
+ s2 = g()
+ h2 = compute_identity_hash(s2)
+ rgc.collect() # shift s2 to the left, but add a hash field
+ assert s2.p1.x == 1010
+ assert s2.p2.x == 1020
+ assert s2.p3.x == 1030
+ assert s2.p4.x == 1040
+ assert s2.p5.x == 1050
+ assert s2.p6.x == 1060
+ assert s2.p7.x == 1070
+ assert s2.p8.x == 1080
+ assert s2.p9.x == 1090
+ return h2 - compute_identity_hash(s2)
+ return f
+
+ def test_adding_a_hash(self):
+ res = self.run("adding_a_hash")
+ assert res == 0
+
# ____________________________________________________________________
-class TestHybridTaggedPointers(TestHybridGC):
+class TaggedPointersTest(object):
taggedpointers = True
-
def define_tagged(cls):
class Unrelated(object):
pass
@@ -1129,3 +1173,10 @@
__slots__ = 'smallint'
def meth(self, x):
return self.smallint + x + 3
+
+
+class TestHybridTaggedPointers(TaggedPointersTest, TestHybridGC):
+ pass
+
+class TestMarkCompactGCMostCompact(TaggedPointersTest, TestMarkCompactGC):
+ removetypeptr = True
Modified: pypy/branch/jit-bounds/pypy/translator/exceptiontransform.py
==============================================================================
--- pypy/branch/jit-bounds/pypy/translator/exceptiontransform.py (original)
+++ pypy/branch/jit-bounds/pypy/translator/exceptiontransform.py Tue Sep 7 19:22:31 2010
@@ -197,7 +197,7 @@
for graph in self.translator.graphs:
self.create_exception_handling(graph)
- def create_exception_handling(self, graph, always_exc_clear=False):
+ def create_exception_handling(self, graph):
"""After an exception in a direct_call (or indirect_call), that is not caught
by an explicit
except statement, we need to reraise the exception. So after this
@@ -212,7 +212,6 @@
self.raise_analyzer.analyze_direct_call(graph)
graph.exceptiontransformed = self.exc_data_ptr
- self.always_exc_clear = always_exc_clear
join_blocks(graph)
# collect the blocks before changing them
n_need_exc_matching_blocks = 0
@@ -455,13 +454,18 @@
block.recloseblock(l0, l)
insert_zeroing_op = False
- # XXX this is not right. it also inserts zero_gc_pointers_inside
- # XXX on a path that malloc_nonmovable returns null, but does not raise
- # XXX which might end up with a segfault. But we don't have such gc now
- if spaceop.opname == 'malloc' or spaceop.opname == 'malloc_nonmovable':
+ if spaceop.opname == 'malloc':
flavor = spaceop.args[1].value['flavor']
if flavor == 'gc':
insert_zeroing_op = True
+ elif spaceop.opname == 'malloc_nonmovable':
+ # xxx we cannot insert zero_gc_pointers_inside after
+ # malloc_nonmovable, because it can return null. For now
+ # we simply always force the zero=True flag on
+ # malloc_nonmovable.
+ c_flags = spaceop.args[1]
+ c_flags.value = c_flags.value.copy()
+ spaceop.args[1].value['zero'] = True
if insert_zeroing_op:
if normalafterblock is None:
@@ -479,16 +483,6 @@
[v_result_after],
varoftype(lltype.Void)))
- if self.always_exc_clear:
- # insert code that clears the exception even in the non-exceptional
- # case... this is a hint for the JIT, but pointless otherwise
- if normalafterblock is None:
- normalafterblock = insert_empty_block(None, l0)
- llops = rtyper.LowLevelOpList(None)
- self.gen_setfield('exc_value', self.c_null_evalue, llops)
- self.gen_setfield('exc_type', self.c_null_etype, llops)
- normalafterblock.operations[:0] = llops
-
class LLTypeExceptionTransformer(BaseExceptionTransformer):
More information about the Pypy-commit
mailing list