[pypy-commit] pypy rdict-experiments-2: Give up and use old rdict for all other places that happen to use rdict details
fijal
noreply at buildbot.pypy.org
Mon Jan 7 01:52:36 CET 2013
Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-2
Changeset: r59832:4927a203f8b8
Date: 2013-01-07 02:52 +0200
http://bitbucket.org/pypy/pypy/changeset/4927a203f8b8/
Log: Give up and use old rdict for all other places that happen to use
rdict details
diff --git a/pypy/rlib/_rweakkeydict.py b/pypy/rlib/_rweakkeydict.py
--- a/pypy/rlib/_rweakkeydict.py
+++ b/pypy/rlib/_rweakkeydict.py
@@ -1,13 +1,13 @@
from pypy.objspace.flow.model import Constant
-from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rdict
+from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rdict_old as rdict
+from pypy.rpython.lltypesystem.llmemory import weakref_create, weakref_deref
from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.lltypesystem.llmemory import weakref_create, weakref_deref
from pypy.rpython.rclass import getinstancerepr
from pypy.rpython.rmodel import Repr
from pypy.rlib.rweakref import RWeakKeyDictionary
from pypy.rlib import jit
-from pypy.rlib.debug import ll_assert
from pypy.rlib.objectmodel import compute_identity_hash
+from pypy.rlib.objectmodel import we_are_translated
# Warning: this implementation of RWeakKeyDictionary is not exactly
@@ -25,7 +25,7 @@
def convert_const(self, weakdict):
if not isinstance(weakdict, RWeakKeyDictionary):
- raise TypeError("expected an RWeakKeyDictionary: %r" % (
+ raise TyperError("expected an RWeakKeyDictionary: %r" % (
weakdict,))
try:
key = Constant(weakdict)
@@ -33,7 +33,7 @@
except KeyError:
self.setup()
if weakdict.length() != 0:
- raise TypeError("got a non-empty prebuilt RWeakKeyDictionary")
+ raise TyperError("got a non-empty prebuilt RWeakKeyDictionary")
l_dict = ll_new_weakdict()
self.dict_cache[key] = l_dict
return l_dict
@@ -83,10 +83,8 @@
h = 0
return '<%x>' % (h,)
-def ll_valid(entries, index):
- if index < 0:
- return False
- key = entries[index].key
+def ll_valid(entries, i):
+ key = entries[i].key
if not key:
return False
elif weakref_deref(rclass.OBJECTPTR, key):
@@ -95,16 +93,19 @@
# The entry might be a dead weakref still holding a strong
# reference to the value; for this case, we clear the old
# value from the entry, if any.
- entries[index].value = NULLVALUE
+ entries[i].value = NULLVALUE
return False
+def ll_everused(entries, i):
+ return bool(entries[i].key)
+
entrymeths = {
'allocate': lltype.typeMethod(rdict._ll_malloc_entries),
+ 'delete': rdict._ll_free_entries,
+ 'valid': ll_valid,
+ 'everused': ll_everused,
'hash': rdict.ll_hash_from_cache,
'no_direct_compare': True,
- 'clear_key': lambda : llmemory.dead_wref,
- 'clear_value': lambda : lltype.nullptr(rclass.OBJECTPTR.TO),
- 'valid': ll_valid,
}
WEAKDICTENTRYARRAY = lltype.GcArray(WEAKDICTENTRY,
adtmeths=entrymeths,
@@ -114,30 +115,22 @@
@jit.dont_look_inside
def ll_new_weakdict():
d = lltype.malloc(WEAKDICT)
- d.entries = WEAKDICT.entries.TO.allocate(rdict.DICT_ITEMS_INITSIZE)
- d.indexes = WEAKDICT.indexes.TO.allocate(rdict.DICT_INITSIZE)
+ d.entries = WEAKDICT.entries.TO.allocate(rdict.DICT_INITSIZE)
d.num_items = 0
- d.resize_counter = rdict.DICT_ITEMS_INITSIZE
+ d.resize_counter = rdict.DICT_INITSIZE * 2
return d
@jit.dont_look_inside
def ll_get(d, llkey):
hash = compute_identity_hash(llkey)
- #llop.debug_print(lltype.Void, "computed key", ll_debugrepr(llkey),
- # hex(hash))
i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
- index = d.indexes[i]
- if index < 0:
- #llop.debug_print(lltype.Void, i, 'get', hex(hash), "null")
- return NULLVALUE
- #llop.debug_print(lltype.Void, i, "getting", index)
#llop.debug_print(lltype.Void, i, 'get', hex(hash),
- # ll_debugrepr(d.entries[index].key),
- # ll_debugrepr(d.entries[index].value))
+ # ll_debugrepr(d.entries[i].key),
+ # ll_debugrepr(d.entries[i].value))
# NB. ll_valid() above was just called at least on entry i, so if
# it is an invalid entry with a dead weakref, the value was reset
# to NULLVALUE.
- return d.entries[index].value
+ return d.entries[i].value
@jit.dont_look_inside
def ll_set(d, llkey, llvalue):
@@ -151,20 +144,15 @@
hash = compute_identity_hash(llkey)
keyref = weakref_create(llkey) # GC effects here, before the rest
i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
- index = d.indexes[i]
- everused = index != rdict.FREE
- if index < 0:
- index = d.num_items
- d.indexes[i] = index
- d.num_items += 1
- d.entries[index].key = keyref
- d.entries[index].value = llvalue
- d.entries[index].f_hash = hash
- #llop.debug_print(lltype.Void, i, 'stored', index, d.num_items, hex(hash),
+ everused = d.entries.everused(i)
+ d.entries[i].key = keyref
+ d.entries[i].value = llvalue
+ d.entries[i].f_hash = hash
+ #llop.debug_print(lltype.Void, i, 'stored', hex(hash),
# ll_debugrepr(llkey),
# ll_debugrepr(llvalue))
if not everused:
- d.resize_counter -= 1
+ d.resize_counter -= 3
if d.resize_counter <= 0:
#llop.debug_print(lltype.Void, 'RESIZE')
ll_weakdict_resize(d)
@@ -173,55 +161,26 @@
def ll_set_null(d, llkey):
hash = compute_identity_hash(llkey)
i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
- index = d.indexes[i]
- if d.entries.valid(index):
- d.entries[index].key = llmemory.dead_wref
- d.entries[index].value = NULLVALUE
- #llop.debug_print(lltype.Void, i, index, 'zero')
+ if d.entries.everused(i):
+ # If the entry was ever used, clean up its key and value.
+ # We don't store a NULL value, but a dead weakref, because
+ # the entry must still be marked as everused().
+ d.entries[i].key = llmemory.dead_wref
+ d.entries[i].value = NULLVALUE
+ #llop.debug_print(lltype.Void, i, 'zero')
+
+def ll_update_num_items(d):
+ entries = d.entries
+ num_items = 0
+ for i in range(len(entries)):
+ if entries.valid(i):
+ num_items += 1
+ d.num_items = num_items
def ll_weakdict_resize(d):
- #llop.debug_print(lltype.Void, "weakdict resize")
- old_entries = d.entries
- old_indexes = d.indexes
- old_size = len(old_indexes)
- # make a 'new_size' estimate and shrink it if there are many
- # deleted entry markers. See CPython for why it is a good idea to
- # quadruple the dictionary size as long as it's not too big.
- # count the number of valid entries
- i = 0
- num_items = 0
- while i < d.num_items:
- if old_entries.valid(i):
- num_items += 1
- i += 1
- if num_items > 50000: new_estimate = (num_items + 1) * 2
- else: new_estimate = (num_items + 1) * 4
- new_size = rdict.DICT_INITSIZE
- while new_size <= new_estimate:
- new_size *= 2
- #
- new_item_size = new_size // 3 * 2 + 1
- d.entries = lltype.typeOf(old_entries).TO.allocate(new_item_size)
- d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(new_size)
- i = 0
- indexes = d.indexes
- j = 0
- while i < old_size:
- index = old_indexes[i]
- if old_entries.valid(index):
- hash = old_entries.hash(index)
- lookup_i = rdict.ll_dict_lookup_clean(d, hash)
- indexes[lookup_i] = j
- #llop.debug_print(lltype.Void, "inserting", hex(hash), i,
- # "to", lookup_i, index, "=>", j)
- #llop.debug_print(lltype.Void, hex(old_entries[index].f_hash))
- d.entries[j].key = old_entries[index].key
- d.entries[j].value = old_entries[index].value
- d.entries[j].f_hash = old_entries[index].f_hash
- j += 1
- i += 1
- d.num_items = j
- d.resize_counter = new_item_size - j
+ # first set num_items to its correct, up-to-date value
+ ll_update_num_items(d)
+ rdict.ll_dict_resize(d)
def ll_keyeq(d, weakkey1, realkey2):
# only called by ll_dict_lookup() with the first arg coming from an
@@ -229,14 +188,12 @@
if not weakkey1:
assert bool(realkey2)
return False
- realkey1 = weakref_deref(rclass.OBJECTPTR, weakkey1)
- #llop.debug_print(lltype.Void, "comparison", realkey1, realkey2)
- return realkey1 == realkey2
+ return weakref_deref(rclass.OBJECTPTR, weakkey1) == realkey2
@jit.dont_look_inside
def ll_length(d):
# xxx slow, but it's only for debugging
- d.resize()
+ ll_update_num_items(d)
#llop.debug_print(lltype.Void, 'length:', d.num_items)
return d.num_items
@@ -245,15 +202,10 @@
'll_set': ll_set,
'keyeq': ll_keyeq,
'paranoia': False,
- 'resize': ll_weakdict_resize,
}
-INDEXESARRAY = lltype.GcArray(lltype.Signed,
- adtmeths={'allocate' : lltype.typeMethod(rdict._ll_malloc_indexes)})
-
WEAKDICT = lltype.GcStruct("weakkeydict",
("num_items", lltype.Signed),
("resize_counter", lltype.Signed),
- ("indexes", lltype.Ptr(INDEXESARRAY)),
("entries", lltype.Ptr(WEAKDICTENTRYARRAY)),
adtmeths=dictmeths)
diff --git a/pypy/rlib/_rweakvaldict.py b/pypy/rlib/_rweakvaldict.py
--- a/pypy/rlib/_rweakvaldict.py
+++ b/pypy/rlib/_rweakvaldict.py
@@ -1,5 +1,5 @@
from pypy.objspace.flow.model import Constant
-from pypy.rpython.lltypesystem import lltype, llmemory, rstr, rclass, rdict
+from pypy.rpython.lltypesystem import lltype, llmemory, rstr, rclass, rdict_old as rdict
from pypy.rpython.lltypesystem.llmemory import weakref_create, weakref_deref
from pypy.rpython.lltypesystem.lloperation import llop
from pypy.rpython.rclass import getinstancerepr
@@ -18,10 +18,8 @@
self.ll_keyhash = r_key.get_ll_hash_function()
ll_keyeq = lltype.staticAdtMethod(r_key.get_ll_eq_function())
- def ll_valid(entries, index):
- if index < 0:
- return False
- value = entries[index].value
+ def ll_valid(entries, i):
+ value = entries[i].value
return bool(value) and bool(weakref_deref(rclass.OBJECTPTR, value))
def ll_everused(entries, i):
@@ -32,6 +30,7 @@
entrymeths = {
'allocate': lltype.typeMethod(rdict._ll_malloc_entries),
+ 'delete': rdict._ll_free_entries,
'valid': ll_valid,
'everused': ll_everused,
'hash': ll_hash,
@@ -42,9 +41,6 @@
WEAKDICTENTRYARRAY = lltype.GcArray(WEAKDICTENTRY,
adtmeths=entrymeths,
hints={'weakarray': 'value'})
-
- WEAKINDEXESARRAY = lltype.GcArray(lltype.Signed,
- adtmeths={'allocate': lltype.typeMethod(rdict._ll_malloc_indexes)})
# NB. the 'hints' is not used so far ^^^
dictmeths = {
@@ -52,14 +48,12 @@
'll_set': self.ll_set,
'keyeq': ll_keyeq,
'paranoia': False,
- 'resize': self.ll_weakdict_resize,
}
self.WEAKDICT = lltype.GcStruct(
"weakvaldict",
("num_items", lltype.Signed),
("resize_counter", lltype.Signed),
- ('indexes', lltype.Ptr(WEAKINDEXESARRAY)),
("entries", lltype.Ptr(WEAKDICTENTRYARRAY)),
adtmeths=dictmeths)
@@ -68,7 +62,7 @@
def convert_const(self, weakdict):
if not isinstance(weakdict, RWeakValueDictionary):
- raise TypeError("expected an RWeakValueDictionary: %r" % (
+ raise TyperError("expected an RWeakValueDictionary: %r" % (
weakdict,))
try:
key = Constant(weakdict)
@@ -111,10 +105,9 @@
@jit.dont_look_inside
def ll_new_weakdict(self):
d = lltype.malloc(self.WEAKDICT)
- d.entries = self.WEAKDICT.entries.TO.allocate(rdict.DICT_ITEMS_INITSIZE)
- d.indexes = self.WEAKDICT.indexes.TO.allocate(rdict.DICT_INITSIZE)
+ d.entries = self.WEAKDICT.entries.TO.allocate(rdict.DICT_INITSIZE)
d.num_items = 0
- d.resize_counter = rdict.DICT_ITEMS_INITSIZE
+ d.resize_counter = rdict.DICT_INITSIZE * 2
return d
@jit.dont_look_inside
@@ -122,11 +115,8 @@
hash = self.ll_keyhash(llkey)
i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
#llop.debug_print(lltype.Void, i, 'get')
- index = d.indexes[i]
- if index >= 0:
- valueref = d.entries[index].value
- if not valueref:
- return lltype.nullptr(rclass.OBJECTPTR.TO)
+ valueref = d.entries[i].value
+ if valueref:
return weakref_deref(rclass.OBJECTPTR, valueref)
else:
return lltype.nullptr(rclass.OBJECTPTR.TO)
@@ -137,26 +127,18 @@
self.ll_set_nonnull(d, llkey, llvalue)
else:
self.ll_set_null(d, llkey)
-
+
@jit.dont_look_inside
def ll_set_nonnull(self, d, llkey, llvalue):
hash = self.ll_keyhash(llkey)
valueref = weakref_create(llvalue) # GC effects here, before the rest
i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
- index = d.indexes[i]
- everused = index != rdict.FREE
- if index < 0:
- index = d.num_items
- d.indexes[i] = index
- d.num_items += 1
- d.entries[index].key = llkey
- d.entries[index].value = valueref
- llop.debug_print(lltype.Void, "set nonnull", i, index)
- #llop.debug_print(lltype.Void, i, 'stored', index, d.num_items, hex(hash),
- # ll_debugrepr(llkey),
- # ll_debugrepr(llvalue))
+ everused = d.entries.everused(i)
+ d.entries[i].key = llkey
+ d.entries[i].value = valueref
+ #llop.debug_print(lltype.Void, i, 'stored')
if not everused:
- d.resize_counter -= 1
+ d.resize_counter -= 3
if d.resize_counter <= 0:
#llop.debug_print(lltype.Void, 'RESIZE')
self.ll_weakdict_resize(d)
@@ -165,60 +147,26 @@
def ll_set_null(self, d, llkey):
hash = self.ll_keyhash(llkey)
i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
- index = d.indexes[i]
- if d.entries.valid(index):
+ if d.entries.everused(i):
# If the entry was ever used, clean up its key and value.
# We don't store a NULL value, but a dead weakref, because
# the entry must still be marked as everused().
- d.entries[index].value = llmemory.dead_wref
+ d.entries[i].value = llmemory.dead_wref
if isinstance(self.r_key.lowleveltype, lltype.Ptr):
- d.entries[index].key = self.r_key.convert_const(None)
+ d.entries[i].key = self.r_key.convert_const(None)
else:
- d.entries[index].key = self.r_key.convert_const(0)
+ d.entries[i].key = self.r_key.convert_const(0)
#llop.debug_print(lltype.Void, i, 'zero')
def ll_weakdict_resize(self, d):
- #llop.debug_print(lltype.Void, "weakdict resize")
- old_entries = d.entries
- old_indexes = d.indexes
- old_size = len(old_indexes)
- # make a 'new_size' estimate and shrink it if there are many
- # deleted entry markers. See CPython for why it is a good idea to
- # quadruple the dictionary size as long as it's not too big.
- # count the number of valid entries
- i = 0
+ # first set num_items to its correct, up-to-date value
+ entries = d.entries
num_items = 0
- while i < d.num_items:
- if old_entries.valid(i):
+ for i in range(len(entries)):
+ if entries.valid(i):
num_items += 1
- i += 1
- if num_items > 50000: new_estimate = (num_items + 1) * 2
- else: new_estimate = (num_items + 1) * 4
- new_size = rdict.DICT_INITSIZE
- while new_size <= new_estimate:
- new_size *= 2
- #
- new_item_size = new_size // 3 * 2 + 1
- d.entries = lltype.typeOf(old_entries).TO.allocate(new_item_size)
- d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(new_size)
- i = 0
- indexes = d.indexes
- j = 0
- while i < old_size:
- index = old_indexes[i]
- if old_entries.valid(index):
- hash = old_entries.hash(index)
- lookup_i = rdict.ll_dict_lookup_clean(d, hash)
- indexes[lookup_i] = j
- #llop.debug_print(lltype.Void, "inserting", hex(hash), i,
- # "to", lookup_i, index, "=>", j)
- #llop.debug_print(lltype.Void, hex(old_entries[index].f_hash))
- d.entries[j].key = old_entries[index].key
- d.entries[j].value = old_entries[index].value
- j += 1
- i += 1
- d.num_items = j
- d.resize_counter = new_item_size - j
+ d.num_items = num_items
+ rdict.ll_dict_resize(d)
def specialize_make_weakdict(hop):
hop.exception_cannot_occur()
diff --git a/pypy/rlib/test/test_rweakkeydict.py b/pypy/rlib/test/test_rweakkeydict.py
--- a/pypy/rlib/test/test_rweakkeydict.py
+++ b/pypy/rlib/test/test_rweakkeydict.py
@@ -36,25 +36,19 @@
d = prebuilt
if d is None:
d = RWeakKeyDictionary(KX, VX)
- llop.debug_print(lltype.Void, "XXX 1")
k1, k3, v1, v2, v3 = g(d)
rgc.collect(); rgc.collect()
- llop.debug_print(lltype.Void, "XXX 2")
assert d.get(k1) is v1
assert d.get(k3) is v3
assert d.get(k1) is not v2
assert d.get(k3) is not v2
- llop.debug_print(lltype.Void, "XXX 3")
assert d.length() == 2
- llop.debug_print(lltype.Void, "XXX 4")
d.set(k1, None)
assert d.get(k1) is None
assert d.get(k3) is v3
assert d.length() == 1
- llop.debug_print(lltype.Void, "XXX 5")
# resizing should also work
lots_of_keys = [KX() for i in range(loop)]
- llop.debug_print(lltype.Void, "XXX 6")
for k in lots_of_keys:
d.set(k, v1)
for k in lots_of_keys:
@@ -63,26 +57,19 @@
assert d.get(k3) is v3
assert d.length() == loop + 1
# a subclass
- llop.debug_print(lltype.Void, "XXX 7")
ky = KY()
vy = VY()
d.set(ky, vy)
assert d.get(ky) is vy
assert d.length() == loop + 2
# deleting by storing Nones
- llop.debug_print(lltype.Void, "XXX 8")
for k in lots_of_keys:
d.set(k, None)
- llop.debug_print(lltype.Void, "XXX 9")
for k in lots_of_keys:
assert d.get(k) is None
- llop.debug_print(lltype.Void, "XXX 10")
assert d.get(k1) is None
- llop.debug_print(lltype.Void, "XXX 11")
assert d.get(k3) is v3
- llop.debug_print(lltype.Void, "XXX 12")
assert d.get(ky) is vy
- llop.debug_print(lltype.Void, "XXX 13")
assert d.length() == 2
return f
diff --git a/pypy/rpython/lltypesystem/rdict_old.py b/pypy/rpython/lltypesystem/rdict_old.py
new file mode 100644
--- /dev/null
+++ b/pypy/rpython/lltypesystem/rdict_old.py
@@ -0,0 +1,927 @@
+from pypy.tool.pairtype import pairtype
+from pypy.objspace.flow.model import Constant
+from pypy.rpython.rdict import (AbstractDictRepr, AbstractDictIteratorRepr,
+ rtype_newdict)
+from pypy.rpython.lltypesystem import lltype
+from pypy.rlib import objectmodel, jit
+from pypy.rlib.debug import ll_assert
+from pypy.rlib.rarithmetic import r_uint, intmask, LONG_BIT
+from pypy.rpython import rmodel
+from pypy.rpython.error import TyperError
+
+
+HIGHEST_BIT = intmask(1 << (LONG_BIT - 1))
+MASK = intmask(HIGHEST_BIT - 1)
+
+# ____________________________________________________________
+#
+# generic implementation of RPython dictionary, with parametric DICTKEY and
+# DICTVALUE types.
+#
+# XXX for immutable dicts, the array should be inlined and
+# resize_counter and everused are not needed.
+#
+# struct dictentry {
+# DICTKEY key;
+# bool f_valid; # (optional) the entry is filled
+# bool f_everused; # (optional) the entry is or has ever been filled
+# DICTVALUE value;
+# int f_hash; # (optional) key hash, if hard to recompute
+# }
+#
+# struct dicttable {
+# int num_items;
+# int resize_counter;
+# Array *entries;
+# (Function DICTKEY, DICTKEY -> bool) *fnkeyeq;
+# (Function DICTKEY -> int) *fnkeyhash;
+# }
+#
+#
+
+class DictRepr(AbstractDictRepr):
+
+ def __init__(self, rtyper, key_repr, value_repr, dictkey, dictvalue,
+ custom_eq_hash=None, force_non_null=False):
+ self.rtyper = rtyper
+ self.DICT = lltype.GcForwardReference()
+ self.lowleveltype = lltype.Ptr(self.DICT)
+ self.custom_eq_hash = custom_eq_hash is not None
+ if not isinstance(key_repr, rmodel.Repr): # not computed yet, done by setup()
+ assert callable(key_repr)
+ self._key_repr_computer = key_repr
+ else:
+ self.external_key_repr, self.key_repr = self.pickkeyrepr(key_repr)
+ if not isinstance(value_repr, rmodel.Repr): # not computed yet, done by setup()
+ assert callable(value_repr)
+ self._value_repr_computer = value_repr
+ else:
+ self.external_value_repr, self.value_repr = self.pickrepr(value_repr)
+ self.dictkey = dictkey
+ self.dictvalue = dictvalue
+ self.dict_cache = {}
+ self._custom_eq_hash_repr = custom_eq_hash
+ self.force_non_null = force_non_null
+ # setup() needs to be called to finish this initialization
+
+ def _externalvsinternal(self, rtyper, item_repr):
+ return rmodel.externalvsinternal(self.rtyper, item_repr)
+
+ def _setup_repr(self):
+ if 'key_repr' not in self.__dict__:
+ key_repr = self._key_repr_computer()
+ self.external_key_repr, self.key_repr = self.pickkeyrepr(key_repr)
+ if 'value_repr' not in self.__dict__:
+ self.external_value_repr, self.value_repr = self.pickrepr(self._value_repr_computer())
+ if isinstance(self.DICT, lltype.GcForwardReference):
+ self.DICTKEY = self.key_repr.lowleveltype
+ self.DICTVALUE = self.value_repr.lowleveltype
+
+ # compute the shape of the DICTENTRY structure
+ entryfields = []
+ entrymeths = {
+ 'allocate': lltype.typeMethod(_ll_malloc_entries),
+ 'delete': _ll_free_entries,
+ 'must_clear_key': (isinstance(self.DICTKEY, lltype.Ptr)
+ and self.DICTKEY._needsgc()),
+ 'must_clear_value': (isinstance(self.DICTVALUE, lltype.Ptr)
+ and self.DICTVALUE._needsgc()),
+ }
+
+ # * the key
+ entryfields.append(("key", self.DICTKEY))
+
+ # * if NULL is not a valid ll value for the key or the value
+ # field of the entry, it can be used as a marker for
+ # never-used entries. Otherwise, we need an explicit flag.
+ s_key = self.dictkey.s_value
+ s_value = self.dictvalue.s_value
+ nullkeymarker = not self.key_repr.can_ll_be_null(s_key)
+ nullvaluemarker = not self.value_repr.can_ll_be_null(s_value)
+ if self.force_non_null:
+ if not nullkeymarker:
+ rmodel.warning("%s can be null, but forcing non-null in dict key" % s_key)
+ nullkeymarker = True
+ if not nullvaluemarker:
+ rmodel.warning("%s can be null, but forcing non-null in dict value" % s_value)
+ nullvaluemarker = True
+ dummykeyobj = self.key_repr.get_ll_dummyval_obj(self.rtyper,
+ s_key)
+ dummyvalueobj = self.value_repr.get_ll_dummyval_obj(self.rtyper,
+ s_value)
+
+ # * the state of the entry - trying to encode it as dummy objects
+ if nullkeymarker and dummykeyobj:
+ # all the state can be encoded in the key
+ entrymeths['everused'] = ll_everused_from_key
+ entrymeths['dummy_obj'] = dummykeyobj
+ entrymeths['valid'] = ll_valid_from_key
+ entrymeths['mark_deleted'] = ll_mark_deleted_in_key
+ # the key is overwritten by 'dummy' when the entry is deleted
+ entrymeths['must_clear_key'] = False
+
+ elif nullvaluemarker and dummyvalueobj:
+ # all the state can be encoded in the value
+ entrymeths['everused'] = ll_everused_from_value
+ entrymeths['dummy_obj'] = dummyvalueobj
+ entrymeths['valid'] = ll_valid_from_value
+ entrymeths['mark_deleted'] = ll_mark_deleted_in_value
+ # value is overwritten by 'dummy' when entry is deleted
+ entrymeths['must_clear_value'] = False
+
+ else:
+ # we need a flag to know if the entry was ever used
+ # (we cannot use a NULL as a marker for this, because
+ # the key and value will be reset to NULL to clear their
+ # reference)
+ entryfields.append(("f_everused", lltype.Bool))
+ entrymeths['everused'] = ll_everused_from_flag
+
+ # can we still rely on a dummy obj to mark deleted entries?
+ if dummykeyobj:
+ entrymeths['dummy_obj'] = dummykeyobj
+ entrymeths['valid'] = ll_valid_from_key
+ entrymeths['mark_deleted'] = ll_mark_deleted_in_key
+ # key is overwritten by 'dummy' when entry is deleted
+ entrymeths['must_clear_key'] = False
+ elif dummyvalueobj:
+ entrymeths['dummy_obj'] = dummyvalueobj
+ entrymeths['valid'] = ll_valid_from_value
+ entrymeths['mark_deleted'] = ll_mark_deleted_in_value
+ # value is overwritten by 'dummy' when entry is deleted
+ entrymeths['must_clear_value'] = False
+ else:
+ entryfields.append(("f_valid", lltype.Bool))
+ entrymeths['valid'] = ll_valid_from_flag
+ entrymeths['mark_deleted'] = ll_mark_deleted_in_flag
+
+ # * the value
+ entryfields.append(("value", self.DICTVALUE))
+
+ # * the hash, if needed
+ if self.custom_eq_hash:
+ fasthashfn = None
+ else:
+ fasthashfn = self.key_repr.get_ll_fasthash_function()
+ if fasthashfn is None:
+ entryfields.append(("f_hash", lltype.Signed))
+ entrymeths['hash'] = ll_hash_from_cache
+ else:
+ entrymeths['hash'] = ll_hash_recomputed
+ entrymeths['fasthashfn'] = fasthashfn
+
+ # Build the lltype data structures
+ self.DICTENTRY = lltype.Struct("dictentry", *entryfields)
+ self.DICTENTRYARRAY = lltype.GcArray(self.DICTENTRY,
+ adtmeths=entrymeths)
+ fields = [ ("num_items", lltype.Signed),
+ ("resize_counter", lltype.Signed),
+ ("entries", lltype.Ptr(self.DICTENTRYARRAY)) ]
+ if self.custom_eq_hash:
+ self.r_rdict_eqfn, self.r_rdict_hashfn = self._custom_eq_hash_repr()
+ fields.extend([ ("fnkeyeq", self.r_rdict_eqfn.lowleveltype),
+ ("fnkeyhash", self.r_rdict_hashfn.lowleveltype) ])
+ adtmeths = {
+ 'keyhash': ll_keyhash_custom,
+ 'keyeq': ll_keyeq_custom,
+ 'r_rdict_eqfn': self.r_rdict_eqfn,
+ 'r_rdict_hashfn': self.r_rdict_hashfn,
+ 'paranoia': True,
+ }
+ else:
+ # figure out which functions must be used to hash and compare
+ ll_keyhash = self.key_repr.get_ll_hash_function()
+ ll_keyeq = self.key_repr.get_ll_eq_function() # can be None
+ ll_keyhash = lltype.staticAdtMethod(ll_keyhash)
+ if ll_keyeq is not None:
+ ll_keyeq = lltype.staticAdtMethod(ll_keyeq)
+ adtmeths = {
+ 'keyhash': ll_keyhash,
+ 'keyeq': ll_keyeq,
+ 'paranoia': False,
+ }
+ adtmeths['KEY'] = self.DICTKEY
+ adtmeths['VALUE'] = self.DICTVALUE
+ adtmeths['allocate'] = lltype.typeMethod(_ll_malloc_dict)
+ self.DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths,
+ *fields))
+
+
+ def convert_const(self, dictobj):
+ from pypy.rpython.lltypesystem import llmemory
+ # get object from bound dict methods
+ #dictobj = getattr(dictobj, '__self__', dictobj)
+ if dictobj is None:
+ return lltype.nullptr(self.DICT)
+ if not isinstance(dictobj, (dict, objectmodel.r_dict)):
+ raise TypeError("expected a dict: %r" % (dictobj,))
+ try:
+ key = Constant(dictobj)
+ return self.dict_cache[key]
+ except KeyError:
+ self.setup()
+ l_dict = ll_newdict_size(self.DICT, len(dictobj))
+ self.dict_cache[key] = l_dict
+ r_key = self.key_repr
+ if r_key.lowleveltype == llmemory.Address:
+ raise TypeError("No prebuilt dicts of address keys")
+ r_value = self.value_repr
+ if isinstance(dictobj, objectmodel.r_dict):
+ if self.r_rdict_eqfn.lowleveltype != lltype.Void:
+ l_fn = self.r_rdict_eqfn.convert_const(dictobj.key_eq)
+ l_dict.fnkeyeq = l_fn
+ if self.r_rdict_hashfn.lowleveltype != lltype.Void:
+ l_fn = self.r_rdict_hashfn.convert_const(dictobj.key_hash)
+ l_dict.fnkeyhash = l_fn
+
+ for dictkeycontainer, dictvalue in dictobj._dict.items():
+ llkey = r_key.convert_const(dictkeycontainer.key)
+ llvalue = r_value.convert_const(dictvalue)
+ ll_dict_insertclean(l_dict, llkey, llvalue,
+ dictkeycontainer.hash)
+ return l_dict
+
+ else:
+ for dictkey, dictvalue in dictobj.items():
+ llkey = r_key.convert_const(dictkey)
+ llvalue = r_value.convert_const(dictvalue)
+ ll_dict_insertclean(l_dict, llkey, llvalue,
+ l_dict.keyhash(llkey))
+ return l_dict
+
+ def rtype_len(self, hop):
+ v_dict, = hop.inputargs(self)
+ return hop.gendirectcall(ll_dict_len, v_dict)
+
+ def rtype_is_true(self, hop):
+ v_dict, = hop.inputargs(self)
+ return hop.gendirectcall(ll_dict_is_true, v_dict)
+
+ def make_iterator_repr(self, *variant):
+ return DictIteratorRepr(self, *variant)
+
+ def rtype_method_get(self, hop):
+ v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
+ self.value_repr)
+ hop.exception_cannot_occur()
+ v_res = hop.gendirectcall(ll_get, v_dict, v_key, v_default)
+ return self.recast_value(hop.llops, v_res)
+
+ def rtype_method_setdefault(self, hop):
+ v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
+ self.value_repr)
+ hop.exception_cannot_occur()
+ v_res = hop.gendirectcall(ll_setdefault, v_dict, v_key, v_default)
+ return self.recast_value(hop.llops, v_res)
+
+ def rtype_method_copy(self, hop):
+ v_dict, = hop.inputargs(self)
+ hop.exception_cannot_occur()
+ return hop.gendirectcall(ll_copy, v_dict)
+
+ def rtype_method_update(self, hop):
+ v_dic1, v_dic2 = hop.inputargs(self, self)
+ hop.exception_cannot_occur()
+ return hop.gendirectcall(ll_update, v_dic1, v_dic2)
+
+ def _rtype_method_kvi(self, hop, ll_func):
+ v_dic, = hop.inputargs(self)
+ r_list = hop.r_result
+ cLIST = hop.inputconst(lltype.Void, r_list.lowleveltype.TO)
+ hop.exception_cannot_occur()
+ return hop.gendirectcall(ll_func, cLIST, v_dic)
+
+ def rtype_method_keys(self, hop):
+ return self._rtype_method_kvi(hop, ll_dict_keys)
+
+ def rtype_method_values(self, hop):
+ return self._rtype_method_kvi(hop, ll_dict_values)
+
+ def rtype_method_items(self, hop):
+ return self._rtype_method_kvi(hop, ll_dict_items)
+
+ def rtype_method_iterkeys(self, hop):
+ hop.exception_cannot_occur()
+ return DictIteratorRepr(self, "keys").newiter(hop)
+
+ def rtype_method_itervalues(self, hop):
+ hop.exception_cannot_occur()
+ return DictIteratorRepr(self, "values").newiter(hop)
+
+ def rtype_method_iteritems(self, hop):
+ hop.exception_cannot_occur()
+ return DictIteratorRepr(self, "items").newiter(hop)
+
+ def rtype_method_clear(self, hop):
+ v_dict, = hop.inputargs(self)
+ hop.exception_cannot_occur()
+ return hop.gendirectcall(ll_clear, v_dict)
+
+ def rtype_method_popitem(self, hop):
+ v_dict, = hop.inputargs(self)
+ r_tuple = hop.r_result
+ cTUPLE = hop.inputconst(lltype.Void, r_tuple.lowleveltype)
+ hop.exception_is_here()
+ return hop.gendirectcall(ll_popitem, cTUPLE, v_dict)
+
+ def rtype_method_pop(self, hop):
+ if hop.nb_args == 2:
+ v_args = hop.inputargs(self, self.key_repr)
+ target = ll_pop
+ elif hop.nb_args == 3:
+ v_args = hop.inputargs(self, self.key_repr, self.value_repr)
+ target = ll_pop_default
+ hop.exception_is_here()
+ v_res = hop.gendirectcall(target, *v_args)
+ return self.recast_value(hop.llops, v_res)
+
+class __extend__(pairtype(DictRepr, rmodel.Repr)):
+
+ def rtype_getitem((r_dict, r_key), hop):
+ v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr)
+ if not r_dict.custom_eq_hash:
+ hop.has_implicit_exception(KeyError) # record that we know about it
+ hop.exception_is_here()
+ v_res = hop.gendirectcall(ll_dict_getitem, v_dict, v_key)
+ return r_dict.recast_value(hop.llops, v_res)
+
+ def rtype_delitem((r_dict, r_key), hop):
+ v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr)
+ if not r_dict.custom_eq_hash:
+ hop.has_implicit_exception(KeyError) # record that we know about it
+ hop.exception_is_here()
+ return hop.gendirectcall(ll_dict_delitem, v_dict, v_key)
+
+ def rtype_setitem((r_dict, r_key), hop):
+ v_dict, v_key, v_value = hop.inputargs(r_dict, r_dict.key_repr, r_dict.value_repr)
+ if r_dict.custom_eq_hash:
+ hop.exception_is_here()
+ else:
+ hop.exception_cannot_occur()
+ hop.gendirectcall(ll_dict_setitem, v_dict, v_key, v_value)
+
+ def rtype_contains((r_dict, r_key), hop):
+ v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr)
+ hop.exception_is_here()
+ return hop.gendirectcall(ll_contains, v_dict, v_key)
+
+class __extend__(pairtype(DictRepr, DictRepr)):
+ def convert_from_to((r_dict1, r_dict2), v, llops):
+ # check that we don't convert from Dicts with
+ # different key/value types
+ if r_dict1.dictkey is None or r_dict2.dictkey is None:
+ return NotImplemented
+ if r_dict1.dictkey is not r_dict2.dictkey:
+ return NotImplemented
+ if r_dict1.dictvalue is None or r_dict2.dictvalue is None:
+ return NotImplemented
+ if r_dict1.dictvalue is not r_dict2.dictvalue:
+ return NotImplemented
+ return v
+
+# ____________________________________________________________
+#
+# Low-level methods. These can be run for testing, but are meant to
+# be direct_call'ed from rtyped flow graphs, which means that they will
+# get flowed and annotated, mostly with SomePtr.
+
+def ll_everused_from_flag(entries, i):
+ return entries[i].f_everused
+
+def ll_everused_from_key(entries, i):
+ return bool(entries[i].key)
+
+def ll_everused_from_value(entries, i):
+ return bool(entries[i].value)
+
+def ll_valid_from_flag(entries, i):
+ return entries[i].f_valid
+
+def ll_mark_deleted_in_flag(entries, i):
+ entries[i].f_valid = False
+
+def ll_valid_from_key(entries, i):
+ ENTRIES = lltype.typeOf(entries).TO
+ dummy = ENTRIES.dummy_obj.ll_dummy_value
+ return entries.everused(i) and entries[i].key != dummy
+
+def ll_mark_deleted_in_key(entries, i):
+ ENTRIES = lltype.typeOf(entries).TO
+ dummy = ENTRIES.dummy_obj.ll_dummy_value
+ entries[i].key = dummy
+
+def ll_valid_from_value(entries, i):
+ ENTRIES = lltype.typeOf(entries).TO
+ dummy = ENTRIES.dummy_obj.ll_dummy_value
+ return entries.everused(i) and entries[i].value != dummy
+
+def ll_mark_deleted_in_value(entries, i):
+ ENTRIES = lltype.typeOf(entries).TO
+ dummy = ENTRIES.dummy_obj.ll_dummy_value
+ entries[i].value = dummy
+
+def ll_hash_from_cache(entries, i):
+ return entries[i].f_hash
+
+def ll_hash_recomputed(entries, i):
+ ENTRIES = lltype.typeOf(entries).TO
+ return ENTRIES.fasthashfn(entries[i].key)
+
+def ll_get_value(d, i):
+ return d.entries[i].value
+
+def ll_keyhash_custom(d, key):
+ DICT = lltype.typeOf(d).TO
+ return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
+
+def ll_keyeq_custom(d, key1, key2):
+ DICT = lltype.typeOf(d).TO
+ return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
+
+def ll_dict_len(d):
+ return d.num_items
+
+def ll_dict_is_true(d):
+ # check if a dict is True, allowing for None
+ return bool(d) and d.num_items != 0
+
+def ll_dict_getitem(d, key):
+ i = ll_dict_lookup(d, key, d.keyhash(key))
+ if not i & HIGHEST_BIT:
+ return ll_get_value(d, i)
+ else:
+ raise KeyError
+
+def ll_dict_setitem(d, key, value):
+ hash = d.keyhash(key)
+ i = ll_dict_lookup(d, key, hash)
+ return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
+
+# It may be safe to look inside always, it has a few branches though, and their
+# frequencies needs to be investigated.
+ at jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
+def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
+ valid = (i & HIGHEST_BIT) == 0
+ i = i & MASK
+ ENTRY = lltype.typeOf(d.entries).TO.OF
+ entry = d.entries[i]
+ if not d.entries.everused(i):
+ # a new entry that was never used before
+ ll_assert(not valid, "valid but not everused")
+ rc = d.resize_counter - 3
+ if rc <= 0: # if needed, resize the dict -- before the insertion
+ ll_dict_resize(d)
+ i = ll_dict_lookup_clean(d, hash) # then redo the lookup for 'key'
+ entry = d.entries[i]
+ rc = d.resize_counter - 3
+ ll_assert(rc > 0, "ll_dict_resize failed?")
+ d.resize_counter = rc
+ if hasattr(ENTRY, 'f_everused'): entry.f_everused = True
+ entry.value = value
+ else:
+ # override an existing or deleted entry
+ entry.value = value
+ if valid:
+ return
+ entry.key = key
+ if hasattr(ENTRY, 'f_hash'): entry.f_hash = hash
+ if hasattr(ENTRY, 'f_valid'): entry.f_valid = True
+ d.num_items += 1
+
+def ll_dict_insertclean(d, key, value, hash):
+ # Internal routine used by ll_dict_resize() to insert an item which is
+ # known to be absent from the dict. This routine also assumes that
+ # the dict contains no deleted entries. This routine has the advantage
+ # of never calling d.keyhash() and d.keyeq(), so it cannot call back
+ # to user code. ll_dict_insertclean() doesn't resize the dict, either.
+ i = ll_dict_lookup_clean(d, hash)
+ ENTRY = lltype.typeOf(d.entries).TO.OF
+ entry = d.entries[i]
+ entry.value = value
+ entry.key = key
+ if hasattr(ENTRY, 'f_hash'): entry.f_hash = hash
+ if hasattr(ENTRY, 'f_valid'): entry.f_valid = True
+ if hasattr(ENTRY, 'f_everused'): entry.f_everused = True
+ d.num_items += 1
+ d.resize_counter -= 3
+
+def ll_dict_delitem(d, key):
+ i = ll_dict_lookup(d, key, d.keyhash(key))
+ if i & HIGHEST_BIT:
+ raise KeyError
+ _ll_dict_del(d, i)
+
+ at jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i))
+def _ll_dict_del(d, i):
+ d.entries.mark_deleted(i)
+ d.num_items -= 1
+ # clear the key and the value if they are GC pointers
+ ENTRIES = lltype.typeOf(d.entries).TO
+ ENTRY = ENTRIES.OF
+ entry = d.entries[i]
+ if ENTRIES.must_clear_key:
+ entry.key = lltype.nullptr(ENTRY.key.TO)
+ if ENTRIES.must_clear_value:
+ entry.value = lltype.nullptr(ENTRY.value.TO)
+ #
+ # The rest is commented out: like CPython we no longer shrink the
+ # dictionary here. It may shrink later if we try to append a number
+ # of new items to it. Unsure if this behavior was designed in
+ # CPython or is accidental. A design reason would be that if you
+ # delete all items in a dictionary (e.g. with a series of
+ # popitem()), then CPython avoids shrinking the table several times.
+ #num_entries = len(d.entries)
+ #if num_entries > DICT_INITSIZE and d.num_items <= num_entries / 4:
+ # ll_dict_resize(d)
+ # A previous xxx: move the size checking and resize into a single
+ # call which is opaque to the JIT when the dict isn't virtual, to
+ # avoid extra branches.
+
+def ll_dict_resize(d):
+ old_entries = d.entries
+ old_size = len(old_entries)
+ # make a 'new_size' estimate and shrink it if there are many
+ # deleted entry markers. See CPython for why it is a good idea to
+ # quadruple the dictionary size as long as it's not too big.
+ num_items = d.num_items + 1
+ if num_items > 50000: new_estimate = num_items * 2
+ else: new_estimate = num_items * 4
+ new_size = DICT_INITSIZE
+ while new_size <= new_estimate:
+ new_size *= 2
+ #
+ d.entries = lltype.typeOf(old_entries).TO.allocate(new_size)
+ d.num_items = 0
+ d.resize_counter = new_size * 2
+ i = 0
+ while i < old_size:
+ if old_entries.valid(i):
+ hash = old_entries.hash(i)
+ entry = old_entries[i]
+ ll_dict_insertclean(d, entry.key, entry.value, hash)
+ i += 1
+ old_entries.delete()
+ll_dict_resize.oopspec = 'dict.resize(d)'
+
+# ------- a port of CPython's dictobject.c's lookdict implementation -------
+PERTURB_SHIFT = 5
+
+ at jit.look_inside_iff(lambda d, key, hash: jit.isvirtual(d) and jit.isconstant(key))
+def ll_dict_lookup(d, key, hash):
+ entries = d.entries
+ ENTRIES = lltype.typeOf(entries).TO
+ direct_compare = not hasattr(ENTRIES, 'no_direct_compare')
+ mask = len(entries) - 1
+ i = hash & mask
+ # do the first try before any looping
+ if entries.valid(i):
+ checkingkey = entries[i].key
+ if direct_compare and checkingkey == key:
+ return i # found the entry
+ if d.keyeq is not None and entries.hash(i) == hash:
+ # correct hash, maybe the key is e.g. a different pointer to
+ # an equal object
+ found = d.keyeq(checkingkey, key)
+ if d.paranoia:
+ if (entries != d.entries or
+ not entries.valid(i) or entries[i].key != checkingkey):
+ # the compare did major nasty stuff to the dict: start over
+ return ll_dict_lookup(d, key, hash)
+ if found:
+ return i # found the entry
+ freeslot = -1
+ elif entries.everused(i):
+ freeslot = i
+ else:
+ return i | HIGHEST_BIT # pristine entry -- lookup failed
+
+ # In the loop, a deleted entry (everused and not valid) is by far
+ # (factor of 100s) the least likely outcome, so test for that last.
+ perturb = r_uint(hash)
+ while 1:
+ # compute the next index using unsigned arithmetic
+ i = r_uint(i)
+ i = (i << 2) + i + perturb + 1
+ i = intmask(i) & mask
+ # keep 'i' as a signed number here, to consistently pass signed
+ # arguments to the small helper methods.
+ if not entries.everused(i):
+ if freeslot == -1:
+ freeslot = i
+ return freeslot | HIGHEST_BIT
+ elif entries.valid(i):
+ checkingkey = entries[i].key
+ if direct_compare and checkingkey == key:
+ return i
+ if d.keyeq is not None and entries.hash(i) == hash:
+ # correct hash, maybe the key is e.g. a different pointer to
+ # an equal object
+ found = d.keyeq(checkingkey, key)
+ if d.paranoia:
+ if (entries != d.entries or
+ not entries.valid(i) or entries[i].key != checkingkey):
+ # the compare did major nasty stuff to the dict:
+ # start over
+ return ll_dict_lookup(d, key, hash)
+ if found:
+ return i # found the entry
+ elif freeslot == -1:
+ freeslot = i
+ perturb >>= PERTURB_SHIFT
+
+def ll_dict_lookup_clean(d, hash):
+ # a simplified version of ll_dict_lookup() which assumes that the
+ # key is new, and the dictionary doesn't contain deleted entries.
+ # It only finds the next free slot for the given hash.
+ entries = d.entries
+ mask = len(entries) - 1
+ i = hash & mask
+ perturb = r_uint(hash)
+ while entries.everused(i):
+ i = r_uint(i)
+ i = (i << 2) + i + perturb + 1
+ i = intmask(i) & mask
+ perturb >>= PERTURB_SHIFT
+ return i
+
+# ____________________________________________________________
+#
+# Irregular operations.
+
+DICT_INITSIZE = 8
+
+def ll_newdict(DICT):
+ d = DICT.allocate()
+ d.entries = DICT.entries.TO.allocate(DICT_INITSIZE)
+ d.num_items = 0
+ d.resize_counter = DICT_INITSIZE * 2
+ return d
+
+def ll_newdict_size(DICT, length_estimate):
+ length_estimate = (length_estimate // 2) * 3
+ n = DICT_INITSIZE
+ while n < length_estimate:
+ n *= 2
+ d = DICT.allocate()
+ d.entries = DICT.entries.TO.allocate(n)
+ d.num_items = 0
+ d.resize_counter = n * 2
+ return d
+
+# pypy.rpython.memory.lldict uses a dict based on Struct and Array
+# instead of GcStruct and GcArray, which is done by using different
+# 'allocate' and 'delete' adtmethod implementations than the ones below
+def _ll_malloc_dict(DICT):
+ return lltype.malloc(DICT)
+def _ll_malloc_entries(ENTRIES, n):
+ return lltype.malloc(ENTRIES, n, zero=True)
+def _ll_free_entries(entries):
+ pass
+
+
+def rtype_r_dict(hop, i_force_non_null=None):
+ r_dict = hop.r_result
+ if not r_dict.custom_eq_hash:
+ raise TyperError("r_dict() call does not return an r_dict instance")
+ v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
+ v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
+ if i_force_non_null is not None:
+ assert i_force_non_null == 2
+ hop.inputarg(lltype.Void, arg=2)
+ cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
+ hop.exception_cannot_occur()
+ v_result = hop.gendirectcall(ll_newdict, cDICT)
+ if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
+ cname = hop.inputconst(lltype.Void, 'fnkeyeq')
+ hop.genop('setfield', [v_result, cname, v_eqfn])
+ if r_dict.r_rdict_hashfn.lowleveltype != lltype.Void:
+ cname = hop.inputconst(lltype.Void, 'fnkeyhash')
+ hop.genop('setfield', [v_result, cname, v_hashfn])
+ return v_result
+
+# ____________________________________________________________
+#
+# Iteration.
+
+class DictIteratorRepr(AbstractDictIteratorRepr):
+
+ def __init__(self, r_dict, variant="keys"):
+ self.r_dict = r_dict
+ self.variant = variant
+ self.lowleveltype = lltype.Ptr(lltype.GcStruct('dictiter',
+ ('dict', r_dict.lowleveltype),
+ ('index', lltype.Signed)))
+ self.ll_dictiter = ll_dictiter
+ self.ll_dictnext = ll_dictnext_group[variant]
+
+
+def ll_dictiter(ITERPTR, d):
+ iter = lltype.malloc(ITERPTR.TO)
+ iter.dict = d
+ iter.index = 0
+ return iter
+
+def _make_ll_dictnext(kind):
+ # make three versions of the following function: keys, values, items
+ @jit.look_inside_iff(lambda RETURNTYPE, iter: jit.isvirtual(iter)
+ and (iter.dict is None or
+ jit.isvirtual(iter.dict)))
+ @jit.oopspec("dictiter.next%s(iter)" % kind)
+ def ll_dictnext(RETURNTYPE, iter):
+ # note that RETURNTYPE is None for keys and values
+ dict = iter.dict
+ if dict:
+ entries = dict.entries
+ index = iter.index
+ entries_len = len(entries)
+ while index < entries_len:
+ entry = entries[index]
+ is_valid = entries.valid(index)
+ index = index + 1
+ if is_valid:
+ iter.index = index
+ if RETURNTYPE is lltype.Void:
+ return None
+ elif kind == 'items':
+ r = lltype.malloc(RETURNTYPE.TO)
+ r.item0 = recast(RETURNTYPE.TO.item0, entry.key)
+ r.item1 = recast(RETURNTYPE.TO.item1, entry.value)
+ return r
+ elif kind == 'keys':
+ return entry.key
+ elif kind == 'values':
+ return entry.value
+ # clear the reference to the dict and prevent restarts
+ iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO)
+ raise StopIteration
+ return ll_dictnext
+
+ll_dictnext_group = {'keys' : _make_ll_dictnext('keys'),
+ 'values': _make_ll_dictnext('values'),
+ 'items' : _make_ll_dictnext('items')}
+
+# _____________________________________________________________
+# methods
+
+def ll_get(dict, key, default):
+ i = ll_dict_lookup(dict, key, dict.keyhash(key))
+ if not i & HIGHEST_BIT:
+ return ll_get_value(dict, i)
+ else:
+ return default
+
+def ll_setdefault(dict, key, default):
+ hash = dict.keyhash(key)
+ i = ll_dict_lookup(dict, key, hash)
+ if not i & HIGHEST_BIT:
+ return ll_get_value(dict, i)
+ else:
+ _ll_dict_setitem_lookup_done(dict, key, default, hash, i)
+ return default
+
+def ll_copy(dict):
+ DICT = lltype.typeOf(dict).TO
+ dictsize = len(dict.entries)
+ d = DICT.allocate()
+ d.entries = DICT.entries.TO.allocate(dictsize)
+ d.num_items = dict.num_items
+ d.resize_counter = dict.resize_counter
+ if hasattr(DICT, 'fnkeyeq'): d.fnkeyeq = dict.fnkeyeq
+ if hasattr(DICT, 'fnkeyhash'): d.fnkeyhash = dict.fnkeyhash
+ i = 0
+ while i < dictsize:
+ d_entry = d.entries[i]
+ entry = dict.entries[i]
+ ENTRY = lltype.typeOf(d.entries).TO.OF
+ d_entry.key = entry.key
+ if hasattr(ENTRY, 'f_valid'): d_entry.f_valid = entry.f_valid
+ if hasattr(ENTRY, 'f_everused'): d_entry.f_everused = entry.f_everused
+ d_entry.value = entry.value
+ if hasattr(ENTRY, 'f_hash'): d_entry.f_hash = entry.f_hash
+ i += 1
+ return d
+ll_copy.oopspec = 'dict.copy(dict)'
+
+def ll_clear(d):
+ if (len(d.entries) == DICT_INITSIZE and
+ d.resize_counter == DICT_INITSIZE * 2):
+ return
+ old_entries = d.entries
+ d.entries = lltype.typeOf(old_entries).TO.allocate(DICT_INITSIZE)
+ d.num_items = 0
+ d.resize_counter = DICT_INITSIZE * 2
+ old_entries.delete()
+ll_clear.oopspec = 'dict.clear(d)'
+
+def ll_update(dic1, dic2):
+ entries = dic2.entries
+ d2len = len(entries)
+ i = 0
+ while i < d2len:
+ if entries.valid(i):
+ entry = entries[i]
+ hash = entries.hash(i)
+ key = entry.key
+ j = ll_dict_lookup(dic1, key, hash)
+ _ll_dict_setitem_lookup_done(dic1, key, entry.value, hash, j)
+ i += 1
+ll_update.oopspec = 'dict.update(dic1, dic2)'
+
+# this is an implementation of keys(), values() and items()
+# in a single function.
+# note that by specialization on func, three different
+# and very efficient functions are created.
+
+def recast(P, v):
+ if isinstance(P, lltype.Ptr):
+ return lltype.cast_pointer(P, v)
+ else:
+ return v
+
+def _make_ll_keys_values_items(kind):
+ def ll_kvi(LIST, dic):
+ res = LIST.ll_newlist(dic.num_items)
+ entries = dic.entries
+ dlen = len(entries)
+ items = res.ll_items()
+ i = 0
+ p = 0
+ while i < dlen:
+ if entries.valid(i):
+ ELEM = lltype.typeOf(items).TO.OF
+ if ELEM is not lltype.Void:
+ entry = entries[i]
+ if kind == 'items':
+ r = lltype.malloc(ELEM.TO)
+ r.item0 = recast(ELEM.TO.item0, entry.key)
+ r.item1 = recast(ELEM.TO.item1, entry.value)
+ items[p] = r
+ elif kind == 'keys':
+ items[p] = recast(ELEM, entry.key)
+ elif kind == 'values':
+ items[p] = recast(ELEM, entry.value)
+ p += 1
+ i += 1
+ assert p == res.ll_length()
+ return res
+ ll_kvi.oopspec = 'dict.%s(dic)' % kind
+ return ll_kvi
+
+ll_dict_keys = _make_ll_keys_values_items('keys')
+ll_dict_values = _make_ll_keys_values_items('values')
+ll_dict_items = _make_ll_keys_values_items('items')
+
+def ll_contains(d, key):
+ i = ll_dict_lookup(d, key, d.keyhash(key))
+ return not i & HIGHEST_BIT
+
+POPITEMINDEX = lltype.Struct('PopItemIndex', ('nextindex', lltype.Signed))
+global_popitem_index = lltype.malloc(POPITEMINDEX, zero=True, immortal=True)
+
+def _ll_getnextitem(dic):
+ entries = dic.entries
+ ENTRY = lltype.typeOf(entries).TO.OF
+ dmask = len(entries) - 1
+ if hasattr(ENTRY, 'f_hash'):
+ if entries.valid(0):
+ return 0
+ base = entries[0].f_hash
+ else:
+ base = global_popitem_index.nextindex
+ counter = 0
+ while counter <= dmask:
+ i = (base + counter) & dmask
+ counter += 1
+ if entries.valid(i):
+ break
+ else:
+ raise KeyError
+ if hasattr(ENTRY, 'f_hash'):
+ entries[0].f_hash = base + counter
+ else:
+ global_popitem_index.nextindex = base + counter
+ return i
+
+def ll_popitem(ELEM, dic):
+ i = _ll_getnextitem(dic)
+ entry = dic.entries[i]
+ r = lltype.malloc(ELEM.TO)
+ r.item0 = recast(ELEM.TO.item0, entry.key)
+ r.item1 = recast(ELEM.TO.item1, entry.value)
+ _ll_dict_del(dic, i)
+ return r
+
+def ll_pop(dic, key):
+ i = ll_dict_lookup(dic, key, dic.keyhash(key))
+ if not i & HIGHEST_BIT:
+ value = ll_get_value(dic, i)
+ _ll_dict_del(dic, i)
+ return value
+ else:
+ raise KeyError
+
+def ll_pop_default(dic, key, dfl):
+ try:
+ return ll_pop(dic, key)
+ except KeyError:
+ return dfl
diff --git a/pypy/rpython/memory/lldict.py b/pypy/rpython/memory/lldict.py
--- a/pypy/rpython/memory/lldict.py
+++ b/pypy/rpython/memory/lldict.py
@@ -1,5 +1,5 @@
from pypy.rpython.lltypesystem import lltype, llmemory
-from pypy.rpython.lltypesystem import rdict
+from pypy.rpython.lltypesystem import rdict_old as rdict
from pypy.rlib.objectmodel import we_are_translated
from pypy.rpython.memory.support import mangle_hash
More information about the pypy-commit
mailing list