[pypy-commit] pypy remove-objspace-options: use mapdict for all the subclassing
cfbolz
pypy.commits at gmail.com
Sat Apr 23 04:42:24 EDT 2016
Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: remove-objspace-options
Changeset: r83834:311cb478ad96
Date: 2016-04-23 11:41 +0300
http://bitbucket.org/pypy/pypy/changeset/311cb478ad96/
Log: use mapdict for all the subclassing
replace a huge mess by a different kind of (smaller) mess
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -98,175 +98,51 @@
# reason is that it is missing a place to store the __dict__, the slots,
# the weakref lifeline, and it typically has no interp-level __del__.
# So we create a few interp-level subclasses of W_XxxObject, which add
-# some combination of features.
-#
-# We don't build 2**4 == 16 subclasses for all combinations of requested
-# features, but limit ourselves to 6, chosen a bit arbitrarily based on
-# typical usage (case 1 is the most common kind of app-level subclasses;
-# case 2 is the memory-saving kind defined with __slots__).
-#
-# +----------------------------------------------------------------+
-# | NOTE: if withmapdict is enabled, the following doesn't apply! |
-# | Map dicts can flexibly allow any slots/__dict__/__weakref__ to |
-# | show up only when needed. In particular there is no way with |
-# | mapdict to prevent some objects from being weakrefable. |
-# +----------------------------------------------------------------+
-#
-# dict slots del weakrefable
-#
-# 1. Y N N Y UserDictWeakref
-# 2. N Y N N UserSlots
-# 3. Y Y N Y UserDictWeakrefSlots
-# 4. N Y N Y UserSlotsWeakref
-# 5. Y Y Y Y UserDictWeakrefSlotsDel
-# 6. N Y Y Y UserSlotsWeakrefDel
-#
-# Note that if the app-level explicitly requests no dict, we should not
-# provide one, otherwise storing random attributes on the app-level
-# instance would unexpectedly work. We don't care too much, though, if
-# an object is weakrefable when it shouldn't really be. It's important
-# that it has a __del__ only if absolutely needed, as this kills the
-# performance of the GCs.
-#
-# Interp-level inheritance is like this:
-#
-# W_XxxObject base
-# / \
-# 1 2
-# / \
-# 3 4
-# / \
-# 5 6
+# some combination of features. This is done using mapdict.
-def get_unique_interplevel_subclass(config, cls, hasdict, wants_slots,
- needsdel=False, weakrefable=False):
+# we need two subclasses of the app-level type, one to add mapdict, and then one
+# to add del to not slow down the GC.
+
+def get_unique_interplevel_subclass(config, cls, needsdel=False):
"NOT_RPYTHON: initialization-time only"
if hasattr(cls, '__del__') and getattr(cls, "handle_del_manually", False):
needsdel = False
assert cls.typedef.acceptable_as_base_class
- key = config, cls, hasdict, wants_slots, needsdel, weakrefable
+ key = config, cls, needsdel
try:
return _subclass_cache[key]
except KeyError:
- subcls = _getusercls(config, cls, hasdict, wants_slots, needsdel,
- weakrefable)
+ # XXX can save a class if cls already has a __del__
+ if needsdel:
+ cls = get_unique_interplevel_subclass(config, cls, False)
+ subcls = _getusercls(config, cls, needsdel)
assert key not in _subclass_cache
_subclass_cache[key] = subcls
return subcls
get_unique_interplevel_subclass._annspecialcase_ = "specialize:memo"
_subclass_cache = {}
-def _getusercls(config, cls, wants_dict, wants_slots, wants_del, weakrefable):
+def _getusercls(config, cls, wants_del, reallywantdict=False):
+ from rpython.rlib import objectmodel
+ from pypy.objspace.std.mapdict import (BaseUserClassMapdict,
+ MapdictDictSupport, MapdictWeakrefSupport,
+ _make_storage_mixin_size_n)
typedef = cls.typedef
- if wants_dict and typedef.hasdict:
- wants_dict = False
- if not typedef.hasdict:
- # mapdict only works if the type does not already have a dict
- if wants_del:
- parentcls = get_unique_interplevel_subclass(config, cls, True, True,
- False, True)
- return _usersubclswithfeature(config, parentcls, "del")
- return _usersubclswithfeature(config, cls, "user", "dict", "weakref", "slots")
- # Forest of if's - see the comment above.
+ name = cls.__name__ + "User"
+
+ mixins_needed = [BaseUserClassMapdict, _make_storage_mixin_size_n()]
+ if reallywantdict or not typedef.hasdict:
+ # the type has no dict, mapdict to provide the dict
+ mixins_needed.append(MapdictDictSupport)
+ name += "Dict"
+ if not typedef.weakrefable:
+ # the type does not support weakrefs yet, mapdict to provide weakref
+ # support
+ mixins_needed.append(MapdictWeakrefSupport)
+ name += "Weakrefable"
if wants_del:
- if wants_dict:
- # case 5. Parent class is 3.
- parentcls = get_unique_interplevel_subclass(config, cls, True, True,
- False, True)
- else:
- # case 6. Parent class is 4.
- parentcls = get_unique_interplevel_subclass(config, cls, False, True,
- False, True)
- return _usersubclswithfeature(config, parentcls, "del")
- elif wants_dict:
- if wants_slots:
- # case 3. Parent class is 1.
- parentcls = get_unique_interplevel_subclass(config, cls, True, False,
- False, True)
- return _usersubclswithfeature(config, parentcls, "slots")
- else:
- # case 1 (we need to add weakrefable unless it's already in 'cls')
- if not typedef.weakrefable:
- return _usersubclswithfeature(config, cls, "user", "dict", "weakref")
- else:
- return _usersubclswithfeature(config, cls, "user", "dict")
- else:
- if weakrefable and not typedef.weakrefable:
- # case 4. Parent class is 2.
- parentcls = get_unique_interplevel_subclass(config, cls, False, True,
- False, False)
- return _usersubclswithfeature(config, parentcls, "weakref")
- else:
- # case 2 (if the base is already weakrefable, case 2 == case 4)
- return _usersubclswithfeature(config, cls, "user", "slots")
-
-def _usersubclswithfeature(config, parentcls, *features):
- key = config, parentcls, features
- try:
- return _usersubclswithfeature_cache[key]
- except KeyError:
- subcls = _builduserclswithfeature(config, parentcls, *features)
- _usersubclswithfeature_cache[key] = subcls
- return subcls
-_usersubclswithfeature_cache = {}
-_allusersubcls_cache = {}
-
-def _builduserclswithfeature(config, supercls, *features):
- "NOT_RPYTHON: initialization-time only"
- name = supercls.__name__
- name += ''.join([name.capitalize() for name in features])
- body = {}
- #print '..........', name, '(', supercls.__name__, ')'
-
- def add(Proto):
- for key, value in Proto.__dict__.items():
- if (not key.startswith('__') and not key.startswith('_mixin_')
- or key == '__del__'):
- if hasattr(value, "func_name"):
- value = func_with_new_name(value, value.func_name)
- body[key] = value
-
- if "dict" in features:
- from pypy.objspace.std.mapdict import BaseMapdictObject, ObjectMixin
- add(BaseMapdictObject)
- add(ObjectMixin)
- body["user_overridden_class"] = True
- features = ()
-
- if "user" in features: # generic feature needed by all subcls
-
- class Proto(object):
- user_overridden_class = True
-
- def getclass(self, space):
- return promote(self.w__class__)
-
- def setclass(self, space, w_subtype):
- # only used by descr_set___class__
- self.w__class__ = w_subtype
-
- def user_setup(self, space, w_subtype):
- self.space = space
- self.w__class__ = w_subtype
- self.user_setup_slots(w_subtype.layout.nslots)
-
- def user_setup_slots(self, nslots):
- assert nslots == 0
- add(Proto)
-
- if "weakref" in features:
- class Proto(object):
- _lifeline_ = None
- def getweakref(self):
- return self._lifeline_
- def setweakref(self, space, weakreflifeline):
- self._lifeline_ = weakreflifeline
- def delweakref(self):
- self._lifeline_ = None
- add(Proto)
-
- if "del" in features:
- parent_destructor = getattr(supercls, '__del__', None)
+ name += "Del"
+ parent_destructor = getattr(cls, '__del__', None)
def call_parent_del(self):
assert isinstance(self, subcls)
parent_destructor(self)
@@ -281,39 +157,15 @@
if parent_destructor is not None:
self.enqueue_for_destruction(self.space, call_parent_del,
'internal destructor of ')
- add(Proto)
+ mixins_needed.append(Proto)
- if "slots" in features:
- class Proto(object):
- slots_w = []
- def user_setup_slots(self, nslots):
- if nslots > 0:
- self.slots_w = [None] * nslots
- def setslotvalue(self, index, w_value):
- self.slots_w[index] = w_value
- def delslotvalue(self, index):
- if self.slots_w[index] is None:
- return False
- self.slots_w[index] = None
- return True
- def getslotvalue(self, index):
- return self.slots_w[index]
- add(Proto)
-
- subcls = type(name, (supercls,), body)
- _allusersubcls_cache[subcls] = True
+ class subcls(cls):
+ user_overridden_class = True
+ for base in mixins_needed:
+ objectmodel.import_from_mixin(base)
+ subcls.__name__ = name
return subcls
-# a couple of helpers for the Proto classes above, factored out to reduce
-# the translated code size
-def check_new_dictionary(space, w_dict):
- if not space.isinstance_w(w_dict, space.w_dict):
- raise OperationError(space.w_TypeError,
- space.wrap("setting dictionary to a non-dict"))
- from pypy.objspace.std import dictmultiobject
- assert isinstance(w_dict, dictmultiobject.W_DictMultiObject)
- return w_dict
-check_new_dictionary._dont_inline_ = True
# ____________________________________________________________
diff --git a/pypy/module/__builtin__/interp_classobj.py b/pypy/module/__builtin__/interp_classobj.py
--- a/pypy/module/__builtin__/interp_classobj.py
+++ b/pypy/module/__builtin__/interp_classobj.py
@@ -185,12 +185,11 @@
class Cache:
def __init__(self, space):
- from pypy.interpreter.typedef import _usersubclswithfeature
- # evil
- self.cls_without_del = _usersubclswithfeature(
- space.config, W_InstanceObject, "dict", "weakref")
- self.cls_with_del = _usersubclswithfeature(
- space.config, self.cls_without_del, "del")
+ from pypy.interpreter.typedef import _getusercls
+ self.cls_without_del = _getusercls(
+ space.config, W_InstanceObject, False, reallywantdict=True)
+ self.cls_with_del = _getusercls(
+ space.config, W_InstanceObject, True, reallywantdict=True)
def class_descr_call(space, w_self, __args__):
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -450,12 +450,19 @@
INVALID = 2
SLOTS_STARTING_FROM = 3
+# a little bit of a mess of mixin classes that implement various pieces of
+# objspace user object functionality in terms of mapdict
-class BaseMapdictObject:
- _mixin_ = True
+class BaseUserClassMapdict:
+ # everything that's needed to use mapdict for a user subclass at all.
+ # This immediately makes slots possible.
- def _init_empty(self, map):
- raise NotImplementedError("abstract base class")
+ # assumes presence of _init_empty, _mapdict_read_storage,
+ # _mapdict_write_storage, _mapdict_storage_length,
+ # _set_mapdict_storage_and_map
+
+ # _____________________________________________
+ # methods needed for mapdict
def _become(self, new_obj):
self._set_mapdict_storage_and_map(new_obj.storage, new_obj.map)
@@ -464,49 +471,11 @@
return jit.promote(self.map)
def _set_mapdict_map(self, map):
self.map = map
+
# _____________________________________________
# objspace interface
- def getdictvalue(self, space, attrname):
- return self._get_mapdict_map().read(self, attrname, DICT)
-
- def setdictvalue(self, space, attrname, w_value):
- return self._get_mapdict_map().write(self, attrname, DICT, w_value)
-
- def deldictvalue(self, space, attrname):
- new_obj = self._get_mapdict_map().delete(self, attrname, DICT)
- if new_obj is None:
- return False
- self._become(new_obj)
- return True
-
- def getdict(self, space):
- w_dict = self._get_mapdict_map().read(self, "dict", SPECIAL)
- if w_dict is not None:
- assert isinstance(w_dict, W_DictMultiObject)
- return w_dict
-
- strategy = space.fromcache(MapDictStrategy)
- storage = strategy.erase(self)
- w_dict = W_DictObject(space, strategy, storage)
- flag = self._get_mapdict_map().write(self, "dict", SPECIAL, w_dict)
- assert flag
- return w_dict
-
- def setdict(self, space, w_dict):
- from pypy.interpreter.typedef import check_new_dictionary
- w_dict = check_new_dictionary(space, w_dict)
- w_olddict = self.getdict(space)
- assert isinstance(w_dict, W_DictMultiObject)
- # The old dict has got 'self' as dstorage, but we are about to
- # change self's ("dict", SPECIAL) attribute to point to the
- # new dict. If the old dict was using the MapDictStrategy, we
- # have to force it now: otherwise it would remain an empty
- # shell that continues to delegate to 'self'.
- if type(w_olddict.get_strategy()) is MapDictStrategy:
- w_olddict.get_strategy().switch_to_object_strategy(w_olddict)
- flag = self._get_mapdict_map().write(self, "dict", SPECIAL, w_dict)
- assert flag
+ # class access
def getclass(self, space):
return self._get_mapdict_map().terminator.w_cls
@@ -519,9 +488,13 @@
from pypy.module.__builtin__.interp_classobj import W_InstanceObject
self.space = space
assert (not self.typedef.hasdict or
+ isinstance(w_subtype.terminator, NoDictTerminator) or
self.typedef is W_InstanceObject.typedef)
self._init_empty(w_subtype.terminator)
+
+ # methods needed for slots
+
def getslotvalue(self, slotindex):
index = SLOTS_STARTING_FROM + slotindex
return self._get_mapdict_map().read(self, "slot", index)
@@ -538,7 +511,9 @@
self._become(new_obj)
return True
- # used by _weakref implemenation
+
+class MapdictWeakrefSupport(object):
+ # stuff used by the _weakref implementation
def getweakref(self):
from pypy.module._weakref.interp__weakref import WeakrefLifeline
@@ -559,8 +534,69 @@
self._get_mapdict_map().write(self, "weakref", SPECIAL, None)
delweakref._cannot_really_call_random_things_ = True
-class ObjectMixin(object):
- _mixin_ = True
+
+class MapdictDictSupport(object):
+
+ # objspace interface for dictionary operations
+
+ def getdictvalue(self, space, attrname):
+ return self._get_mapdict_map().read(self, attrname, DICT)
+
+ def setdictvalue(self, space, attrname, w_value):
+ return self._get_mapdict_map().write(self, attrname, DICT, w_value)
+
+ def deldictvalue(self, space, attrname):
+ new_obj = self._get_mapdict_map().delete(self, attrname, DICT)
+ if new_obj is None:
+ return False
+ self._become(new_obj)
+ return True
+
+ def getdict(self, space):
+ return _obj_getdict(self, space)
+
+ def setdict(self, space, w_dict):
+ _obj_setdict(self, space, w_dict)
+
+# a couple of helpers for the classes above, factored out to reduce
+# the translated code size
+
+ at objectmodel.dont_inline
+def _obj_getdict(self, space):
+ assert isinstance(self._get_mapdict_map().terminator, DictTerminator)
+ w_dict = self._get_mapdict_map().read(self, "dict", SPECIAL)
+ if w_dict is not None:
+ assert isinstance(w_dict, W_DictMultiObject)
+ return w_dict
+
+ strategy = space.fromcache(MapDictStrategy)
+ storage = strategy.erase(self)
+ w_dict = W_DictObject(space, strategy, storage)
+ flag = self._get_mapdict_map().write(self, "dict", SPECIAL, w_dict)
+ assert flag
+ return w_dict
+
+ at objectmodel.dont_inline
+def _obj_setdict(self, space, w_dict):
+ from pypy.objspace.std import dictmultiobject
+ assert isinstance(self._get_mapdict_map().terminator, DictTerminator)
+ if not space.isinstance_w(w_dict, space.w_dict):
+ raise OperationError(space.w_TypeError,
+ space.wrap("setting dictionary to a non-dict"))
+ assert isinstance(w_dict, dictmultiobject.W_DictMultiObject)
+ w_olddict = self.getdict(space)
+ assert isinstance(w_dict, W_DictMultiObject)
+ # The old dict has got 'self' as dstorage, but we are about to
+ # change self's ("dict", SPECIAL) attribute to point to the
+ # new dict. If the old dict was using the MapDictStrategy, we
+ # have to force it now: otherwise it would remain an empty
+ # shell that continues to delegate to 'self'.
+ if type(w_olddict.get_strategy()) is MapDictStrategy:
+ w_olddict.get_strategy().switch_to_object_strategy(w_olddict)
+ flag = self._get_mapdict_map().write(self, "dict", SPECIAL, w_dict)
+ assert flag
+
+class MapdictStorageMixin(object):
def _init_empty(self, map):
from rpython.rlib.debug import make_sure_not_resized
self.map = map
@@ -579,50 +615,21 @@
self.storage = storage
self.map = map
-class Object(ObjectMixin, BaseMapdictObject, W_Root):
+class ObjectWithoutDict(MapdictStorageMixin, BaseUserClassMapdict, MapdictWeakrefSupport, W_Root):
pass # mainly for tests
-def get_subclass_of_correct_size(space, cls, w_type):
- map = w_type.terminator
- classes = memo_get_subclass_of_correct_size(space, cls)
- if SUBCLASSES_MIN_FIELDS == SUBCLASSES_MAX_FIELDS:
- return classes[0]
- size = map.size_estimate()
- debug.check_nonneg(size)
- if size < len(classes):
- return classes[size]
- else:
- return classes[len(classes)-1]
-get_subclass_of_correct_size._annspecialcase_ = "specialize:arg(1)"
+class Object(MapdictStorageMixin, BaseUserClassMapdict, MapdictDictSupport, MapdictWeakrefSupport, W_Root):
+ pass # mainly for tests
-SUBCLASSES_MIN_FIELDS = 5 # XXX tweak these numbers
-SUBCLASSES_MAX_FIELDS = 5
+SUBCLASSES_NUM_FIELDS = 5
-def memo_get_subclass_of_correct_size(space, supercls):
- key = space, supercls
- try:
- return _subclass_cache[key]
- except KeyError:
- assert not hasattr(supercls, "__del__")
- result = []
- for i in range(SUBCLASSES_MIN_FIELDS, SUBCLASSES_MAX_FIELDS+1):
- result.append(_make_subclass_size_n(supercls, i))
- for i in range(SUBCLASSES_MIN_FIELDS):
- result.insert(0, result[0])
- if SUBCLASSES_MIN_FIELDS == SUBCLASSES_MAX_FIELDS:
- assert len(set(result)) == 1
- _subclass_cache[key] = result
- return result
-memo_get_subclass_of_correct_size._annspecialcase_ = "specialize:memo"
-_subclass_cache = {}
-
-def _make_subclass_size_n(supercls, n):
+def _make_storage_mixin_size_n(n=SUBCLASSES_NUM_FIELDS):
from rpython.rlib import unroll
rangen = unroll.unrolling_iterable(range(n))
nmin1 = n - 1
rangenmin1 = unroll.unrolling_iterable(range(nmin1))
valnmin1 = "_value%s" % nmin1
- class subcls(BaseMapdictObject, supercls):
+ class subcls(object):
def _init_empty(self, map):
for i in rangenmin1:
setattr(self, "_value%s" % i, None)
@@ -690,7 +697,7 @@
erased = erase_list(storage_list)
setattr(self, "_value%s" % nmin1, erased)
- subcls.__name__ = supercls.__name__ + "Size%s" % n
+ subcls.__name__ = "Size%s" % n
return subcls
# ____________________________________________________________
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -356,14 +356,8 @@
if cls.typedef.applevel_subclasses_base is not None:
cls = cls.typedef.applevel_subclasses_base
#
- if cls is W_ObjectObject and not w_subtype.needsdel:
- from pypy.objspace.std.mapdict import get_subclass_of_correct_size
- subcls = get_subclass_of_correct_size(self, cls, w_subtype)
- else:
- subcls = get_unique_interplevel_subclass(
- self.config, cls, w_subtype.hasdict,
- w_subtype.layout.nslots != 0,
- w_subtype.needsdel, w_subtype.weakrefable)
+ subcls = get_unique_interplevel_subclass(
+ self.config, cls, w_subtype.needsdel)
instance = instantiate(subcls)
assert isinstance(instance, cls)
instance.user_setup(self, w_subtype)
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -13,7 +13,7 @@
class Class(object):
def __init__(self, hasdict=True):
- self.hasdict = True
+ self.hasdict = hasdict
if hasdict:
self.terminator = DictTerminator(space, self)
else:
@@ -22,10 +22,17 @@
def instantiate(self, sp=None):
if sp is None:
sp = space
- result = Object()
+ if self.hasdict:
+ result = Object()
+ else:
+ result = ObjectWithoutDict()
result.user_setup(sp, self)
return result
+class ObjectWithoutDict(ObjectWithoutDict):
+ class typedef:
+ hasdict = False
+
class Object(Object):
class typedef:
hasdict = False
@@ -429,6 +436,9 @@
assert obj.getslotvalue(b) == 60
assert obj.storage == [50, 60]
assert not obj.setdictvalue(space, "a", 70)
+ assert obj.getdict(space) is None
+ assert obj.getdictvalue(space, "a") is None
+
def test_getdict():
cls = Class()
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -177,7 +177,13 @@
# itself changes
w_self._version_tag = VersionTag()
from pypy.objspace.std.mapdict import DictTerminator, NoDictTerminator
- if w_self.hasdict:
+ # if the typedef has a dict, then the rpython-class does all the dict
+ # management, which means from the point of view of mapdict there is no
+ # dict. However, W_InstanceObjects are an exception to this
+ from pypy.module.__builtin__.interp_classobj import W_InstanceObject
+ typedef = w_self.layout.typedef
+ if (w_self.hasdict and not typedef.hasdict or
+ typedef is W_InstanceObject.typedef):
w_self.terminator = DictTerminator(space, w_self)
else:
w_self.terminator = NoDictTerminator(space, w_self)
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -211,6 +211,12 @@
func._always_inline_ = True
return func
+def dont_inline(func):
+ """ mark the function as never-to-be-inlined by the RPython optimizations
+ (not the JIT!), no matter its size."""
+ func._dont_inline_ = True
+ return func
+
# ____________________________________________________________
More information about the pypy-commit
mailing list