[pypy-svn] r70425 - in pypy/branch/change-celldict2/pypy: interpreter module/pypyjit/test objspace/std objspace/std/test
cfbolz at codespeak.net
cfbolz at codespeak.net
Thu Jan 7 14:34:36 CET 2010
Author: cfbolz
Date: Thu Jan 7 14:34:36 2010
New Revision: 70425
Modified:
pypy/branch/change-celldict2/pypy/interpreter/pycode.py
pypy/branch/change-celldict2/pypy/module/pypyjit/test/test_pypy_c.py
pypy/branch/change-celldict2/pypy/objspace/std/celldict.py
pypy/branch/change-celldict2/pypy/objspace/std/objspace.py
pypy/branch/change-celldict2/pypy/objspace/std/test/test_celldict.py
Log:
merge in changes from change-celldict
Modified: pypy/branch/change-celldict2/pypy/interpreter/pycode.py
==============================================================================
--- pypy/branch/change-celldict2/pypy/interpreter/pycode.py (original)
+++ pypy/branch/change-celldict2/pypy/interpreter/pycode.py Thu Jan 7 14:34:36 2010
@@ -117,9 +117,6 @@
self._compute_flatcall()
- if self.space.config.objspace.std.withcelldict:
- from pypy.objspace.std.celldict import init_code
- init_code(self)
def _init_flags(self):
co_code = self.co_code
Modified: pypy/branch/change-celldict2/pypy/module/pypyjit/test/test_pypy_c.py
==============================================================================
--- pypy/branch/change-celldict2/pypy/module/pypyjit/test/test_pypy_c.py (original)
+++ pypy/branch/change-celldict2/pypy/module/pypyjit/test/test_pypy_c.py Thu Jan 7 14:34:36 2010
@@ -210,21 +210,31 @@
def test_simple_call(self):
self.run_source('''
+ OFFSET = 0
def f(i):
- return i + 1
+ return i + 1 + OFFSET
def main(n):
i = 0
- while i < n:
+ while i < n+OFFSET:
i = f(f(i))
return i
- ''', 76,
+ ''', 96,
([20], 20),
([31], 32))
ops = self.get_by_bytecode("LOAD_GLOBAL")
- assert len(ops) == 2
- assert ops[0].get_opnames() == ["getfield_gc", "getarrayitem_gc",
+ assert len(ops) == 5
+ assert ops[0].get_opnames() == ["getfield_gc", "guard_value",
+ "getfield_gc", "guard_isnull",
"getfield_gc", "guard_nonnull_class"]
- assert not ops[1] # second LOAD_GLOBAL folded away
+ # the second getfield on the same globals is quicker
+ assert ops[1].get_opnames() == ["getfield_gc", "guard_nonnull_class"]
+ assert not ops[2] # second LOAD_GLOBAL of the same name folded away
+ # LOAD_GLOBAL of the same name but in different function partially
+ # folded away
+ # XXX could be improved
+ assert ops[3].get_opnames() == ["guard_value",
+ "getfield_gc", "guard_isnull"]
+ assert not ops[4]
ops = self.get_by_bytecode("CALL_FUNCTION")
assert len(ops) == 2
for bytecode in ops:
@@ -281,7 +291,7 @@
while i < n:
i = f(f(i), j=1)
return i
- ''', 98,
+ ''', 100,
([20], 20),
([31], 32))
ops = self.get_by_bytecode("CALL_FUNCTION")
@@ -305,7 +315,7 @@
a.x = 2
i = i + a.x
return i
- ''', 63,
+ ''', 65,
([20], 20),
([31], 32))
Modified: pypy/branch/change-celldict2/pypy/objspace/std/celldict.py
==============================================================================
--- pypy/branch/change-celldict2/pypy/objspace/std/celldict.py (original)
+++ pypy/branch/change-celldict2/pypy/objspace/std/celldict.py Thu Jan 7 14:34:36 2010
@@ -1,4 +1,8 @@
-from pypy.interpreter.pycode import CO_CONTAINSGLOBALS
+""" A very simple cell dict implementation. The dictionary maps keys to cell.
+This ensures that the function (dict, key) -> cell is pure. By itself, this
+optimization is not helping at all, but in conjunction with the JIT it can
+speed up global lookups a lot."""
+
from pypy.objspace.std.dictmultiobject import IteratorImplementation
from pypy.objspace.std.dictmultiobject import W_DictMultiObject, _is_sane_hash
from pypy.rlib import jit
@@ -19,31 +23,22 @@
def __init__(self, space):
self.space = space
self.content = {}
- self.unshadowed_builtins = {}
- def getcell(self, key, make_new=True):
+ def getcell(self, key, makenew):
+ if makenew or jit.we_are_jitted():
+ # when we are jitting, we always go through the pure function
+ # below, to ensure that we have no residual dict lookup
+ return self._getcell_makenew(key)
+ return self.content.get(key, None)
+
+ @jit.purefunction_promote
+ def _getcell_makenew(self, key):
res = self.content.get(key, None)
if res is not None:
return res
- if not make_new:
- return None
result = self.content[key] = ModuleCell()
return result
- def add_unshadowed_builtin(self, name, builtin_impl):
- assert isinstance(builtin_impl, ModuleDictImplementation)
- self.unshadowed_builtins[name] = builtin_impl
-
- def invalidate_unshadowed_builtin(self, name):
- impl = self.unshadowed_builtins[name]
- try:
- cell = impl.content[name]
- except KeyError:
- pass
- else:
- w_value = cell.invalidate()
- cell = impl.content[name] = ModuleCell(w_value)
-
def impl_setitem(self, w_key, w_value):
space = self.space
if space.is_w(space.type(w_key), space.w_str):
@@ -52,11 +47,7 @@
self._as_rdict().setitem(w_key, w_value)
def impl_setitem_str(self, name, w_value, shadows_type=True):
- self.getcell(name).w_value = w_value
-
- if name in self.unshadowed_builtins:
- self.invalidate_unshadowed_builtin(name)
- del self.unshadowed_builtins[name]
+ self.getcell(name, True).w_value = w_value
def impl_delitem(self, w_key):
space = self.space
@@ -64,17 +55,25 @@
if space.is_w(w_key_type, space.w_str):
key = space.str_w(w_key)
cell = self.getcell(key, False)
- if cell is None:
+ if cell is None or cell.w_value is None:
raise KeyError
+ # note that we don't remove the cell from self.content, to make
+ # sure that a key that was found at any point in the dict, still
+ # maps to the same cell later (even if this cell no longer
+ # represents a key)
cell.invalidate()
- del self.content[key]
elif _is_sane_hash(space, w_key_type):
raise KeyError
else:
self._as_rdict().delitem(w_key)
def impl_length(self):
- return len(self.content)
+ # inefficient, but do we care?
+ res = 0
+ for cell in self.content.itervalues():
+ if cell.w_value is not None:
+ res += 1
+ return res
def impl_getitem(self, w_lookup):
space = self.space
@@ -91,6 +90,7 @@
res = self.getcell(lookup, False)
if res is None:
return None
+ # note that even if the res.w_value is None, the next line is fine
return res.w_value
def impl_iter(self):
@@ -98,39 +98,34 @@
def impl_keys(self):
space = self.space
- return [space.wrap(key) for key in self.content.iterkeys()]
+ return [space.wrap(key) for key, cell in self.content.iteritems()
+ if cell.w_value is not None]
def impl_values(self):
- return [cell.w_value for cell in self.content.itervalues()]
+ return [cell.w_value for cell in self.content.itervalues()
+ if cell.w_value is not None]
def impl_items(self):
space = self.space
return [space.newtuple([space.wrap(key), cell.w_value])
- for (key, cell) in self.content.iteritems()]
+ for (key, cell) in self.content.iteritems()
+ if cell.w_value is not None]
def impl_clear(self):
- # inefficient, but who cares
for k, cell in self.content.iteritems():
cell.invalidate()
- for k in self.unshadowed_builtins:
- self.invalidate_unshadowed_builtin(k)
- self.content.clear()
- self.unshadowed_builtins.clear()
-
def _as_rdict(self):
r_dict_content = self.initialize_as_rdict()
for k, cell in self.content.iteritems():
- r_dict_content[self.space.wrap(k)] = cell.w_value
+ if cell.w_value is not None:
+ r_dict_content[self.space.wrap(k)] = cell.w_value
cell.invalidate()
- for k in self.unshadowed_builtins:
- self.invalidate_unshadowed_builtin(k)
self._clear_fields()
return self
def _clear_fields(self):
self.content = None
- self.unshadowed_builtins = None
class ModuleDictIteratorImplementation(IteratorImplementation):
def __init__(self, space, dictimplementation):
@@ -138,99 +133,8 @@
self.iterator = dictimplementation.content.iteritems()
def next_entry(self):
- # note that this 'for' loop only runs once, at most
for key, cell in self.iterator:
- return (self.space.wrap(key), cell.w_value)
+ if cell.w_value is not None:
+ return (self.space.wrap(key), cell.w_value)
else:
return None, None
-
-
-class State(object):
- def __init__(self, space):
- self.space = space
- self.invalidcell = ModuleCell()
- self.always_invalid_cache = []
- self.neverused_dictcontent = {}
-
-class GlobalCacheHolder(object):
- def __init__(self, space):
- self.cache = None
- state = space.fromcache(State)
- self.dictcontent = state.neverused_dictcontent
-
- def getcache(self, space, code, w_globals):
- if type(w_globals) is ModuleDictImplementation:
- content = w_globals.content
- else:
- content = None
- if self.dictcontent is content:
- return self.cache
- return self.getcache_slow(space, code, w_globals, content)
- getcache._always_inline_ = True
-
- def getcache_slow(self, space, code, w_globals, content):
- state = space.fromcache(State)
- if content is None:
- cache = state.always_invalid_cache
- if len(code.co_names_w) > len(cache):
- cache = [state.invalidcell] * len(code.co_names_w)
- state.always_invalid_cache = cache
- else:
- cache = [state.invalidcell] * len(code.co_names_w)
- self.cache = cache
- self.dictcontent = content
- return cache
- getcache_slow._dont_inline_ = True
-
-def init_code(code):
- if code.co_flags & CO_CONTAINSGLOBALS:
- code.globalcacheholder = GlobalCacheHolder(code.space)
- else:
- code.globalcacheholder = None
-
-
-def get_global_cache(space, code, w_globals):
- from pypy.interpreter.pycode import PyCode
- assert isinstance(code, PyCode)
- holder = code.globalcacheholder
- if holder is not None:
- return holder.getcache(space, code, w_globals)
- return None
-
-def getimplementation(w_dict):
- if type(w_dict) is ModuleDictImplementation and w_dict.r_dict_content is None:
- return w_dict
- else:
- return None
-
-def LOAD_GLOBAL(f, nameindex, *ignored):
- cell = f.cache_for_globals[nameindex]
- w_value = cell.w_value
- if w_value is None:
- # slow path
- w_value = load_global_fill_cache(f, nameindex)
- f.pushvalue(w_value)
-LOAD_GLOBAL._always_inline_ = True
-
-def find_cell_from_dict(implementation, name):
- if implementation is not None:
- return implementation.getcell(name, False)
- return None
-
- at jit.dont_look_inside
-def load_global_fill_cache(f, nameindex):
- name = f.space.str_w(f.getname_w(nameindex))
- implementation = getimplementation(f.w_globals)
- if implementation is not None:
- cell = implementation.getcell(name, False)
- if cell is None:
- builtin_impl = getimplementation(f.get_builtin().getdict())
- cell = find_cell_from_dict(builtin_impl, name)
- if cell is not None:
- implementation.add_unshadowed_builtin(name, builtin_impl)
-
- if cell is not None:
- f.cache_for_globals[nameindex] = cell
- return cell.w_value
- return f._load_global(f.getname_u(nameindex))
-load_global_fill_cache._dont_inline_ = True
Modified: pypy/branch/change-celldict2/pypy/objspace/std/objspace.py
==============================================================================
--- pypy/branch/change-celldict2/pypy/objspace/std/objspace.py (original)
+++ pypy/branch/change-celldict2/pypy/objspace/std/objspace.py Thu Jan 7 14:34:36 2010
@@ -71,16 +71,8 @@
# Import all the object types and implementations
self.model = StdTypeModel(self.config)
- from pypy.objspace.std.celldict import get_global_cache
class StdObjSpaceFrame(pyframe.PyFrame):
- if self.config.objspace.std.withcelldict:
- def __init__(self, space, code, w_globals, closure):
- pyframe.PyFrame.__init__(self, space, code, w_globals, closure)
- self.cache_for_globals = get_global_cache(space, code, w_globals)
-
- from pypy.objspace.std.celldict import LOAD_GLOBAL
-
if self.config.objspace.std.optimized_int_add:
if self.config.objspace.std.withsmallint:
def BINARY_ADD(f, oparg, *ignored):
Modified: pypy/branch/change-celldict2/pypy/objspace/std/test/test_celldict.py
==============================================================================
--- pypy/branch/change-celldict2/pypy/objspace/std/test/test_celldict.py (original)
+++ pypy/branch/change-celldict2/pypy/objspace/std/test/test_celldict.py Thu Jan 7 14:34:36 2010
@@ -1,262 +1,31 @@
import py
from pypy.conftest import gettestobjspace, option
-from pypy.objspace.std.celldict import get_global_cache, ModuleCell, ModuleDictImplementation
+from pypy.objspace.std.celldict import ModuleCell, ModuleDictImplementation
+from pypy.objspace.std.test.test_dictmultiobject import FakeSpace
from pypy.interpreter import gateway
-# this file tests mostly the effects of caching global lookup. The dict
-# implementation itself is tested in test_dictmultiobject.py
-
-
-class AppTestCellDict(object):
- def setup_class(cls):
- if option.runappdirect:
- py.test.skip("not appdirect tests")
- cls.space = gettestobjspace(**{"objspace.std.withcelldict": True})
- cls.w_impl_used = cls.space.appexec([], """():
- import __pypy__
- def impl_used(obj):
- assert "ModuleDictImplementation" in __pypy__.internal_repr(obj)
- return impl_used
- """)
- def is_in_cache(space, w_code, w_globals, w_name):
- name = space.str_w(w_name)
- cache = get_global_cache(space, w_code, w_globals)
- index = [space.str_w(w_n) for w_n in w_code.co_names_w].index(name)
- return space.wrap(cache[index].w_value is not None)
- is_in_cache = gateway.interp2app(is_in_cache)
- cls.w_is_in_cache = cls.space.wrap(is_in_cache)
- stored_builtins = []
- def rescue_builtins(space):
- w_dict = space.builtin.getdict()
- content = {}
- for key, cell in w_dict.content.iteritems():
- newcell = ModuleCell()
- newcell.w_value = cell.w_value
- content[key] = newcell
- stored_builtins.append(content)
- rescue_builtins = gateway.interp2app(rescue_builtins)
- cls.w_rescue_builtins = cls.space.wrap(rescue_builtins)
- def restore_builtins(space):
- w_dict = space.builtin.getdict()
- assert isinstance(w_dict, ModuleDictImplementation)
- w_dict.content = stored_builtins.pop()
- w_dict.fallback = None
- restore_builtins = gateway.interp2app(restore_builtins)
- cls.w_restore_builtins = cls.space.wrap(restore_builtins)
-
- def test_same_code_in_different_modules(self):
- import sys
- mod1 = type(sys)("abc")
- self.impl_used(mod1.__dict__)
- glob1 = mod1.__dict__
- mod2 = type(sys)("abc")
- self.impl_used(mod2.__dict__)
- glob2 = mod2.__dict__
- def f():
- return x + 1
- code = f.func_code
- f1 = type(f)(code, glob1)
- mod1.x = 1
- assert not self.is_in_cache(code, glob1, "x")
- assert f1() == 2
- assert self.is_in_cache(code, glob1, "x")
- assert f1() == 2
- assert self.is_in_cache(code, glob1, "x")
- mod1.x = 2
- assert f1() == 3
- assert self.is_in_cache(code, glob1, "x")
- assert f1() == 3
- assert self.is_in_cache(code, glob1, "x")
- f2 = type(f)(code, glob2)
- mod2.x = 5
- assert not self.is_in_cache(code, glob2, "x")
- assert f2() == 6
- assert self.is_in_cache(code, glob2, "x")
- assert f2() == 6
- assert self.is_in_cache(code, glob2, "x")
- mod2.x = 7
- assert f2() == 8
- assert self.is_in_cache(code, glob2, "x")
- assert f2() == 8
- assert self.is_in_cache(code, glob2, "x")
-
- def test_override_builtins(self):
- import sys, __builtin__
- mod1 = type(sys)("abc")
- glob1 = mod1.__dict__
- self.impl_used(mod1.__dict__)
- def f():
- return len(x)
- code = f.func_code
- f1 = type(f)(f.func_code, glob1)
- mod1.x = []
- assert not self.is_in_cache(code, glob1, "len")
- assert not self.is_in_cache(code, glob1, "x")
- assert f1() == 0
- assert self.is_in_cache(code, glob1, "len")
- assert self.is_in_cache(code, glob1, "x")
- assert f1() == 0
- mod1.x.append(1)
- assert f1() == 1
- assert self.is_in_cache(code, glob1, "len")
- assert self.is_in_cache(code, glob1, "x")
- mod1.len = lambda x: 15
- assert not self.is_in_cache(code, glob1, "len")
- mod1.x.append(1)
- assert f1() == 15
- assert self.is_in_cache(code, glob1, "len")
- assert f1() == 15
- assert self.is_in_cache(code, glob1, "len")
- del mod1.len
- mod1.x.append(1)
- assert not self.is_in_cache(code, glob1, "len")
- assert f1() == 3
- assert self.is_in_cache(code, glob1, "len")
- assert f1() == 3
- assert self.is_in_cache(code, glob1, "len")
- orig_len = __builtins__.len
- try:
- __builtins__.len = lambda x: 12
- mod1.x.append(1)
- assert self.is_in_cache(code, glob1, "len")
- assert f1() == 12
- assert self.is_in_cache(code, glob1, "len")
- assert f1() == 12
- assert self.is_in_cache(code, glob1, "len")
- finally:
- __builtins__.len = orig_len
-
- def test_override_builtins2(self):
- import sys, __builtin__
- mod1 = type(sys)("abc")
- glob1 = mod1.__dict__
- self.impl_used(mod1.__dict__)
- def f():
- return l(x)
- code = f.func_code
- f1 = type(f)(f.func_code, glob1)
- mod1.x = []
- __builtin__.l = len
- try:
- assert not self.is_in_cache(code, glob1, "l")
- assert not self.is_in_cache(code, glob1, "x")
- assert f1() == 0
- assert self.is_in_cache(code, glob1, "l")
- assert self.is_in_cache(code, glob1, "x")
- assert f1() == 0
- mod1.x.append(1)
- assert f1() == 1
- assert self.is_in_cache(code, glob1, "l")
- assert self.is_in_cache(code, glob1, "x")
- del __builtin__.l
- mod1.l = len
- mod1.x.append(1)
- assert not self.is_in_cache(code, glob1, "l")
- assert f1() == 2
- assert self.is_in_cache(code, glob1, "l")
- assert self.is_in_cache(code, glob1, "x")
- finally:
- if hasattr(__builtins__, "l"):
- del __builtins__.l
-
- def test_generator(self):
- import sys, __builtin__
- mod1 = type(sys)("abc")
- glob1 = mod1.__dict__
- self.impl_used(mod1.__dict__)
- def f():
- yield 1
- yield x
- yield len(x)
- code = f.func_code
- f1 = type(f)(f.func_code, glob1)
- mod1.x = []
- gen = f1()
- assert not self.is_in_cache(code, glob1, "len")
- assert not self.is_in_cache(code, glob1, "x")
- v = gen.next()
- assert v == 1
- assert not self.is_in_cache(code, glob1, "len")
- assert not self.is_in_cache(code, glob1, "x")
- v = gen.next()
- assert v is mod1.x
- assert not self.is_in_cache(code, glob1, "len")
- assert self.is_in_cache(code, glob1, "x")
- v = gen.next()
- assert v == 0
- assert self.is_in_cache(code, glob1, "len")
- assert self.is_in_cache(code, glob1, "x")
-
- def test_degenerate_to_rdict(self):
- import sys
- mod1 = type(sys)("abc")
- self.impl_used(mod1.__dict__)
- glob1 = mod1.__dict__
- def f():
- return x + 1
- code = f.func_code
- f1 = type(f)(code, glob1)
- mod1.x = 1
- assert not self.is_in_cache(code, glob1, "x")
- assert f1() == 2
- assert self.is_in_cache(code, glob1, "x")
- glob1[1] = 2
- assert not self.is_in_cache(code, glob1, "x")
- assert f1() == 2
- assert not self.is_in_cache(code, glob1, "x")
-
- def test_degenerate_builtin_to_rdict(self):
- import sys, __builtin__
- mod1 = type(sys)("abc")
- self.impl_used(mod1.__dict__)
- glob1 = mod1.__dict__
- def f():
- return len(x)
- code = f.func_code
- f1 = type(f)(code, glob1)
- mod1.x = [1, 2]
- assert not self.is_in_cache(code, glob1, "x")
- assert not self.is_in_cache(code, glob1, "len")
- assert f1() == 2
- assert self.is_in_cache(code, glob1, "x")
- assert self.is_in_cache(code, glob1, "len")
- self.rescue_builtins()
- try:
- __builtin__.__dict__[1] = 2
- assert not self.is_in_cache(code, glob1, "len")
- assert f1() == 2
- assert not self.is_in_cache(code, glob1, "len")
- finally:
- self.restore_builtins()
-
- def test_mapping_as_locals(self):
- import sys
- if sys.version_info < (2,5) or not hasattr(sys, 'pypy_objspaceclass'):
- skip("need CPython 2.5 or PyPy for non-dictionaries in exec statements")
- class M(object):
- def __getitem__(self, key):
- return key
- def __setitem__(self, key, value):
- self.result[key] = value
- m = M()
- m.result = {}
- exec "x=m" in {}, m
- assert m.result == {'x': 'm'}
- exec "y=n" in m # NOTE: this doesn't work in CPython 2.4
- assert m.result == {'x': 'm', 'y': 'n'}
-
- def test_subclass_of_dict_as_locals(self):
- import sys
- if sys.version_info < (2,5) or not hasattr(sys, 'pypy_objspaceclass'):
- skip("need CPython 2.5 or PyPy for non-dictionaries in exec statements")
- class M(dict):
- def __getitem__(self, key):
- return key
- def __setitem__(self, key, value):
- dict.__setitem__(self, key, value)
- m = M()
- exec "x=m" in {}, m
- assert m == {'x': 'm'}
- exec "y=n" in m # NOTE: this doesn't work in CPython 2.4
- assert m == {'x': 'm', 'y': 'n'}
+space = FakeSpace()
+class TestCellDict(object):
+ def test_basic_property(self):
+ d = ModuleDictImplementation(space)
+ d.setitem("a", 1)
+ assert d.getcell("a", False) is d.getcell("a", False)
+ acell = d.getcell("a", False)
+ d.setitem("b", 2)
+ assert d.getcell("b", False) is d.getcell("b", False)
+ assert d.getcell("c", True) is d.getcell("c", True)
+
+ assert d.getitem("a") == 1
+ assert d.getitem("b") == 2
+
+ d.delitem("a")
+ py.test.raises(KeyError, d.delitem, "a")
+ assert d.getitem("a") is None
+ assert d.getcell("a", False) is acell
+ assert d.length() == 1
+
+ d.clear()
+ assert d.getitem("a") is None
+ assert d.getcell("a", False) is acell
+ assert d.length() == 1
More information about the Pypy-commit
mailing list