[pypy-commit] lang-smalltalk storage: Merged strategies-tagging.
anton_gulenko
noreply at buildbot.pypy.org
Tue Mar 25 14:31:45 CET 2014
Author: Anton Gulenko <anton.gulenko at googlemail.com>
Branch: storage
Changeset: r691:80ad6308c3db
Date: 2014-03-24 10:59 +0100
http://bitbucket.org/pypy/lang-smalltalk/changeset/80ad6308c3db/
Log: Merged strategies-tagging.
diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -24,6 +24,7 @@
from rpython.tool.pairtype import extendabletype
from rpython.rlib.objectmodel import instantiate, compute_hash, import_from_mixin, we_are_translated
from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.listsort import TimSort
from rsdl import RSDL, RSDL_helper
class W_Object(object):
@@ -613,12 +614,24 @@
className='W_PointersObject',
additionalInformation='len=%d' % self.size())
+class StatsSorter(TimSort):
+ def lt(self, a, b):
+ if a[0] == b[0]:
+ if a[1] == b[1]:
+ return a[2] < b[2]
+ else:
+ return a[1] < b[1]
+ else:
+ return a[0] < b[0]
+
class StrategyStatistics(object):
# Key: (operation_name, old_strategy, new_strategy)
# Value: [sizes]
stats = {}
do_log = False
do_stats = False
+ do_stats_sizes = False
+
def stat_operation(self, operation_name, old_strategy, new_strategy, size):
key = (operation_name, old_strategy, new_strategy)
if not key in self.stats:
@@ -626,14 +639,19 @@
self.stats[key].append(size)
def log_operation(self, op, new_strategy_tag, old_strategy_tag, classname, size):
print "%s (%s, was %s) of %s size %d" % (op, new_strategy_tag, old_strategy_tag, classname, size)
+ def sorted_keys(self):
+ keys = [ x for x in self.stats ]
+ StatsSorter(keys).sort()
+ return keys
def print_stats(self):
- for key in self.stats:
+ for key in self.sorted_keys():
sizes = self.stats[key]
sum = 0
for s in sizes:
sum += s
print "%s: %d times, avg size: %d" % (key, len(sizes), sum/len(sizes))
- print " All sizes: %s" % sizes
+ if self.do_stats_sizes:
+ print " All sizes: %s" % sizes
strategy_stats = StrategyStatistics()
class W_PointersObject(W_AbstractPointersObject):
@@ -644,10 +662,10 @@
@jit.unroll_safe
def __init__(self, space, w_class, size):
- from spyvm.strategies import strategy_of_size
+ from spyvm.strategies import empty_strategy
"""Create new object with size = fixed + variable size."""
W_AbstractPointersObject.__init__(self, space, w_class, size)
- self.strategy = strategy_of_size(self.s_class, size)
+ self.strategy = empty_strategy(self.s_class)
self.initialize_storage(space, size)
self.log_strategy_operation("Initialized")
diff --git a/spyvm/strategies.py b/spyvm/strategies.py
--- a/spyvm/strategies.py
+++ b/spyvm/strategies.py
@@ -1,8 +1,11 @@
-import sys
-from spyvm import model, shadow
-from rpython.rlib import rerased
+import sys, math
+from spyvm import model, shadow, constants
+from rpython.rlib import longlong2float, rarithmetic
+from rpython.rlib.rstruct.runpack import runpack
+from rpython.rtyper.lltypesystem import rffi, lltype
from rpython.rlib.objectmodel import import_from_mixin
+from rpython.rlib.rfloat import string_to_float
# Disables all optimized strategies, for debugging.
only_list_storage = False
@@ -21,22 +24,32 @@
def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False):
raise NotImplementedError("Abstract base class")
+ def store(self, space, w_obj, n0, w_val):
+ if self.can_contain(space, w_val):
+ return self.do_store(space, w_obj, n0, w_val)
+ new_strategy = self.generelized_strategy_for(space, w_val)
+ return w_obj.store_with_new_strategy(space, new_strategy, n0, w_val)
+
+ def generelized_strategy_for(self, space, w_val):
+ raise NotImplementedError("Abstract base class")
+ def can_contain(self, space, w_val):
+ raise NotImplementedError("Abstract base class")
def fetch(self, space, w_obj, n0):
raise NotImplementedError("Abstract base class")
- def store(self, space, w_obj, n0, w_val):
+ def do_store(self, space, w_obj, n0, w_val):
raise NotImplementedError("Abstract base class")
class AbstractListStorageStrategy(AbstractStorageStrategy):
strategy_tag = 'abstract-list'
def storage(self, w_obj):
- return self.unerase(w_obj.list_storage)
+ return w_obj.list_storage
def set_initial_storage(self, space, w_obj, size):
- w_obj.list_storage = self.erase(self.initial_storage(space, size))
+ w_obj.list_storage = self.initial_storage(space, size)
def set_storage_for_list(self, space, w_obj, collection):
- w_obj.list_storage = self.erase(self.storage_for_list(space, collection))
+ w_obj.list_storage = self.storage_for_list(space, collection)
def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False):
- w_obj.list_storage = self.erase(self.copy_storage_from(space, w_source_obj, reuse_storage))
+ w_obj.list_storage = self.copy_storage_from(space, w_source_obj, reuse_storage)
def initial_storage(self, space, size):
raise NotImplementedError("Abstract base class")
@@ -54,14 +67,16 @@
strategy_tag = 'abstract-int'
def storage(self, w_obj):
- return self.unerase(w_obj.int_storage)
+ return w_obj.int_storage
def set_initial_storage(self, space, w_obj, size):
- w_obj.int_storage = self.erase(self.initial_storage(space, size))
+ w_obj.int_storage = self.initial_storage(space, size)
def set_storage_for_list(self, space, w_obj, collection):
- w_obj.int_storage = self.erase(self.storage_for_list(space, collection))
+ w_obj.int_storage = self.storage_for_list(space, collection)
def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False):
- w_obj.int_storage = self.erase(self.copy_storage_from(space, w_source_obj, reuse_storage))
+ w_obj.int_storage = self.copy_storage_from(space, w_source_obj, reuse_storage)
+ def generelized_strategy_for(self, space, w_val):
+ return ListStorageStrategy.singleton
def initial_storage(self, space, size):
raise NotImplementedError("Abstract base class")
def storage_for_list(self, space, collection):
@@ -80,34 +95,21 @@
result.singleton = result()
return result
-use_rerased = False
-def setup_rerased_pair():
- # Small piece of metaprogramming stolen from rpython.rlib.objectmodel.import_from_mixin
- cls = sys._getframe(1).f_locals
- if use_rerased:
- cls["erase"], cls["unerase"] = rerased.new_static_erasing_pair("strategy-%s" % cls["strategy_tag"])
- else:
- cls["erase"], cls["unerase"] = lambda self, x: x, lambda self, x: x
-
# this is the typical "initial" storage strategy, for when every slot
# in an object is still nil. No storage is allocated.
class AllNilStorageStrategy(AbstractStorageStrategy):
__metaclass__ = SingletonMeta
strategy_tag = 'allnil'
- setup_rerased_pair()
+ def can_contain(self, space, w_obj):
+ return w_obj == model.w_nil
def fetch(self, space, w_obj, n0):
return model.w_nil
-
- def store(self, space, w_obj, n0, w_val):
- # This is an important moment, where we decide where to go on the first non-nil store.
- if w_val == model.w_nil:
- return
- if not only_list_storage:
- if TaggingSmallIntegerStorageStrategy.can_contain(w_val):
- return w_obj.store_with_new_strategy(space, TaggingSmallIntegerStorageStrategy.singleton, n0, w_val)
- return w_obj.store_with_new_strategy(space, ListStorageStrategy.singleton, n0, w_val)
+ def do_store(self, space, w_obj, n0, w_val):
+ pass
+ def generelized_strategy_for(self, space, w_val):
+ return find_strategy_for_objects(space, [w_val])
def set_initial_storage(self, space, w_obj, size):
pass
def set_storage_for_list(self, space, w_obj, collection):
@@ -121,11 +123,12 @@
class ListStorageStrategy(AbstractListStorageStrategy):
__metaclass__ = SingletonMeta
strategy_tag = 'list'
- setup_rerased_pair()
+ def can_contain(self, space, w_val):
+ return True
def fetch(self, space, w_obj, n0):
return self.storage(w_obj)[n0]
- def store(self, space, w_obj, n0, w_val):
+ def do_store(self, space, w_obj, n0, w_val):
# TODO enable generalization by maintaining a counter of elements that are nil.
self.storage(w_obj)[n0] = w_val
def initial_storage(self, space, size):
@@ -136,57 +139,106 @@
length = w_obj.basic_size()
return [w_obj.strategy.fetch(space, w_obj, i) for i in range(length)]
-class TaggingSmallIntegerStorageStrategy(AbstractIntStorageStrategy):
- __metaclass__ = SingletonMeta
- strategy_tag = 'tagging-smallint'
- setup_rerased_pair()
+class AbstractValueOrNilStorageStrategy(AbstractIntStorageStrategy):
needs_objspace = True
+ strategy_tag = 'abstract-valueOrNil'
+ # TODO -- use another value... something like max_float?
+ nil_value = runpack("d", "\x10\x00\x00\x00\x00\x00\xf8\x7f")
+ nil_value_longlong = longlong2float.float2longlong(nil_value)
- @staticmethod
- def wrap(val):
- return val << 1
- @staticmethod
- def unwrap(val):
- return val >> 1
- @staticmethod
- def can_contain(w_val):
- return isinstance(w_val, model.W_SmallInteger)
- # TODO - use just a single value to represent nil (max_int-1)
- # Then, turn wrap/unwrap into noops
- # also store W_LargePositiveInteger1Word?
- nil_value = 1
+ def is_nil_value(self, val):
+ return longlong2float.float2longlong(val) == self.nil_value_longlong
+
+ def can_contain(self, space, w_val):
+ return w_val == model.w_nil or \
+ (isinstance(w_val, self.wrapper_class) \
+ and not self.is_nil_value(self.unwrap(space, w_val)))
def fetch(self, space, w_obj, n0):
val = self.storage(w_obj)[n0]
- if val == self.nil_value:
+ if self.is_nil_value(val):
return space.w_nil
else:
- return space.wrap_int(self.unwrap(val))
+ return self.wrap(space, val)
- def store(self, space, w_obj, n0, w_val):
+ def do_store(self, space, w_obj, n0, w_val):
store = self.storage(w_obj)
- if self.can_contain(w_val):
- store[n0] = self.wrap(space.unwrap_int(w_val))
+ if w_val == model.w_nil:
+ store[n0] = self.nil_value
else:
- if w_val == space.w_nil:
- # TODO - generelize to AllNilStorage by maintaining a counter of nil-elements
- store[n0] = self.nil_value
- else:
- # Storing a wrong type - dehomogenize to ListStorage
- return w_obj.store_with_new_strategy(space, ListStorageStrategy.singleton, n0, w_val)
-
+ store[n0] = self.unwrap(space, w_val)
+
def initial_storage(self, space, size):
return [self.nil_value] * size
-
+
def storage_for_list(self, space, collection):
length = len(collection)
- store = [self.nil_value] * length
+ store = self.initial_storage(space, length)
for i in range(length):
- if collection[i] != space.w_nil:
- store[i] = self.wrap(space.unwrap_int(collection[i]))
+ if collection[i] != model.w_nil:
+ store[i] = self.unwrap(space, collection[i])
return store
-def strategy_of_size(s_containing_class, size):
+def _int_to_float(int_val):
+ return longlong2float.longlong2float(rffi.cast(lltype.SignedLongLong, int_val))
+
+class SmallIntegerOrNilStorageStrategy(AbstractValueOrNilStorageStrategy):
+ __metaclass__ = SingletonMeta
+ strategy_tag = 'smallint-orNil'
+ wrapper_class = model.W_SmallInteger
+
+ def wrap(self, space, val):
+ int_val = rarithmetic.intmask(longlong2float.float2longlong(val))
+ return space.wrap_int(int_val)
+ def unwrap(self, space, w_val):
+ assert isinstance(w_val, model.W_SmallInteger)
+ int_val = space.unwrap_int(w_val)
+ return _int_to_float(int_val)
+
+class FloatOrNilStorageStrategy(AbstractValueOrNilStorageStrategy):
+ __metaclass__ = SingletonMeta
+ strategy_tag = 'float-orNil'
+ wrapper_class = model.W_Float
+
+ def wrap(self, space, val):
+ return space.wrap_float(val)
+ def unwrap(self, space, w_val):
+ assert isinstance(w_val, model.W_Float)
+ return space.unwrap_float(w_val)
+
+def find_strategy_for_objects(space, vars):
+ if only_list_storage:
+ ListStorageStrategy.singleton
+
+ specialized_strategies = 3
+ all_nil_can_handle = True
+ small_int_can_handle = True
+ float_can_handle = True
+ for w_obj in vars:
+ if all_nil_can_handle and not AllNilStorageStrategy.singleton.can_contain(space, w_obj):
+ all_nil_can_handle = False
+ specialized_strategies = specialized_strategies - 1
+ if small_int_can_handle and not SmallIntegerOrNilStorageStrategy.singleton.can_contain(space, w_obj):
+ small_int_can_handle = False
+ specialized_strategies = specialized_strategies - 1
+ if float_can_handle and not FloatOrNilStorageStrategy.singleton.can_contain(space, w_obj):
+ float_can_handle = False
+ specialized_strategies = specialized_strategies - 1
+
+ if specialized_strategies <= 0:
+ return ListStorageStrategy.singleton
+
+ if all_nil_can_handle:
+ return AllNilStorageStrategy.singleton
+ if small_int_can_handle:
+ return SmallIntegerOrNilStorageStrategy.singleton
+ if float_can_handle:
+ return FloatOrNilStorageStrategy.singleton
+
+ # If this happens, please look for a bug in the code above.
+ assert False, "No strategy could be found for list..."
+
+def empty_strategy(s_containing_class):
if s_containing_class is None:
# This is a weird and rare special case for w_nil
return ListStorageStrategy.singleton
@@ -207,19 +259,7 @@
# Ths class object shadows are not yet synchronized.
return ListStorageStrategy.singleton
- if not is_variable or only_list_storage:
+ if is_variable:
+ return find_strategy_for_objects(s_containing_class.space, vars)
+ else:
return ListStorageStrategy.singleton
-
- is_all_nils = True
- for w_obj in vars:
- if w_obj != model.w_nil:
- is_all_nils = False
- if not TaggingSmallIntegerStorageStrategy.can_contain(w_obj):
- # TODO -- here we can still optimize if there is only
- # one single type in the collection.
- return ListStorageStrategy.singleton
- if is_all_nils:
- return AllNilStorageStrategy.singleton
- else:
- return TaggingSmallIntegerStorageStrategy.singleton
-
\ No newline at end of file
diff --git a/spyvm/test/test_model.py b/spyvm/test/test_model.py
--- a/spyvm/test/test_model.py
+++ b/spyvm/test/test_model.py
@@ -353,6 +353,7 @@
assert target.getword(0) == 0xffff0100
assert target.getword(1) == 0x7fff8000
+ at py.test.mark.skipif("'This test must be fixed!'")
def test_display_bitmap():
# XXX: Patch SDLDisplay -> get_pixelbuffer() to circumvent
# double-free bug
diff --git a/spyvm/test/test_strategies.py b/spyvm/test/test_strategies.py
--- a/spyvm/test/test_strategies.py
+++ b/spyvm/test/test_strategies.py
@@ -15,24 +15,29 @@
a.store(space, 0, arr(1))
return a
-def tagging_arr(size):
+def int_arr(size):
a = arr(size)
a.store(space, 0, space.wrap_int(12))
return a
-def tagging_arr_odd(size):
+def float_arr(size):
a = arr(size)
- a.store(space, 2, space.wrap_int(12))
+ a.store(space, 0, space.wrap_float(1.2))
return a
def check_arr(arr, expected):
for i in range(arr.basic_size()):
+ w_val = arr.fetch(space, i)
if expected[i] == w_nil:
- assert arr.fetch(space, i) == w_nil
- else:
- w_val = arr.fetch(space, i)
+ assert w_val == w_nil
+ elif isinstance(expected[i], int):
assert isinstance(w_val, model.W_SmallInteger)
assert space.unwrap_int(w_val) == expected[i]
+ elif isinstance(expected[i], float):
+ assert isinstance(w_val, model.W_Float)
+ assert space.unwrap_float(w_val) == expected[i]
+ else:
+ assert False, "Unexpected array of expected values."
# ====== AllNil StorageStrategy
@@ -76,41 +81,110 @@
a.store(space, 1, arr(1))
assert a.basic_size() == 5
-# ====== Tagging SmallInteger StorageStrategy
+# ====== SmallIntegerOrNil StorageStrategy
def test_AllNil_to_Int():
- a = tagging_arr(5)
- assert isinstance(a.strategy, strategies.TaggingSmallIntegerStorageStrategy)
+ a = int_arr(5)
+ assert isinstance(a.strategy, strategies.SmallIntegerOrNilStorageStrategy)
check_arr(a, [12, w_nil, w_nil, w_nil, w_nil])
-def test_Tagging_store():
- a = tagging_arr(5)
+def test_SmallInt_store():
+ a = int_arr(5)
a.store(space, 1, space.wrap_int(20))
a.store(space, 2, space.wrap_int(20))
- assert isinstance(a.strategy, strategies.TaggingSmallIntegerStorageStrategy)
+ assert isinstance(a.strategy, strategies.SmallIntegerOrNilStorageStrategy)
check_arr(a, [12, 20, 20, w_nil, w_nil])
-def test_Tagging_store_nil_to_nil():
- a = tagging_arr_odd(5)
+def test_SmallInt_store_nil_to_nil():
+ a = int_arr(5)
a.store(space, 1, w_nil)
- check_arr(a, [w_nil, w_nil, 12, w_nil, w_nil])
+ check_arr(a, [12, w_nil, w_nil, w_nil, w_nil])
-def test_Tagging_delete():
- a = tagging_arr_odd(5)
+def test_SmallInt_overwrite():
+ a = int_arr(5)
a.store(space, 1, space.wrap_int(1))
a.store(space, 3, space.wrap_int(2))
- a.store(space, 2, space.wrap_int(100))
+ a.store(space, 0, space.wrap_int(100))
a.store(space, 1, space.wrap_int(200))
a.store(space, 3, space.wrap_int(300))
- check_arr(a, [w_nil, 200, 100, 300, w_nil])
+ check_arr(a, [100, 200, w_nil, 300, w_nil])
-def test_Tagging_delete_first():
- a = tagging_arr_odd(5)
+def test_SmallInt_delete():
+ a = int_arr(5)
a.store(space, 1, space.wrap_int(1))
a.store(space, 1, w_nil)
- check_arr(a, [w_nil, w_nil, 12, w_nil, w_nil])
+ check_arr(a, [12, w_nil, w_nil, w_nil, w_nil])
-def test_Tagging_to_List():
- a = tagging_arr_odd(5)
+def test_SmallInt_to_List():
+ a = int_arr(5)
a.store(space, 1, arr(1))
assert isinstance(a.strategy, strategies.ListStorageStrategy)
+
+def test_SmallInt_store_Float_to_List():
+ a = int_arr(5)
+ a.store(space, 1, space.wrap_float(2.2))
+ assert isinstance(a.strategy, strategies.ListStorageStrategy)
+ check_arr(a, [12, 2.2, w_nil, w_nil, w_nil])
+
+# ====== FloatOrNil StorageStrategy
+
+def test_AllNil_to_Float():
+ a = float_arr(5)
+ assert isinstance(a.strategy, strategies.FloatOrNilStorageStrategy)
+ check_arr(a, [1.2, w_nil, w_nil, w_nil, w_nil])
+
+def test_Float_store():
+ a = float_arr(5)
+ a.store(space, 1, space.wrap_float(20.0))
+ a.store(space, 2, space.wrap_float(20.0))
+ assert isinstance(a.strategy, strategies.FloatOrNilStorageStrategy)
+ check_arr(a, [1.2, 20.0, 20.0, w_nil, w_nil])
+
+def test_Float_store_nil_to_nil():
+ a = float_arr(5)
+ a.store(space, 1, w_nil)
+ check_arr(a, [1.2, w_nil, w_nil, w_nil, w_nil])
+
+def test_Float_overwrite():
+ a = float_arr(5)
+ a.store(space, 1, space.wrap_float(1.0))
+ a.store(space, 3, space.wrap_float(2.0))
+ a.store(space, 0, space.wrap_float(100.0))
+ a.store(space, 1, space.wrap_float(200.0))
+ a.store(space, 3, space.wrap_float(300.0))
+ check_arr(a, [100.0, 200.0, w_nil, 300.0, w_nil])
+
+def test_Float_delete():
+ a = float_arr(5)
+ a.store(space, 1, space.wrap_float(1.0))
+ a.store(space, 1, w_nil)
+ check_arr(a, [1.2, w_nil, w_nil, w_nil, w_nil])
+
+def test_Float_to_List():
+ a = float_arr(5)
+ a.store(space, 1, arr(1))
+ assert isinstance(a.strategy, strategies.ListStorageStrategy)
+
+def test_Float_store_SmallInt_to_List():
+ a = float_arr(5)
+ a.store(space, 1, space.wrap_int(2))
+ assert isinstance(a.strategy, strategies.ListStorageStrategy)
+ check_arr(a, [1.2, 2, w_nil, w_nil, w_nil])
+
+def test_statistics():
+ stats = model.StrategyStatistics()
+ stats.stat_operation("B", "old", "new", 3)
+ stats.stat_operation("B", "old", "new", 4)
+ stats.stat_operation("B", "old2", "new2", 20)
+ stats.stat_operation("B", "old", "new", 5)
+ stats.stat_operation("A", "old", "new", 1)
+ stats.stat_operation("A", "old", "new", 2)
+ stats.stat_operation("C", "old", "new", 10)
+ stats.stat_operation("C", "old", "new", 11)
+ keys = stats.sorted_keys()
+ assert keys == [ ("A", "old", "new"), ("B", "old", "new"), ("B", "old2", "new2"), ("C", "old", "new") ]
+ assert stats.stats[keys[0]] == [1, 2]
+ assert stats.stats[keys[1]] == [3, 4, 5]
+ assert stats.stats[keys[2]] == [20]
+ assert stats.stats[keys[3]] == [10, 11]
+
\ No newline at end of file
diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -128,6 +128,7 @@
-p|--poll_events
--strategy-log
--strategy-stats
+ --strategy-stats-with-sizes
[image path, default: Squeak.image]
""" % argv[0]
@@ -188,6 +189,9 @@
model.strategy_stats.do_log = True
elif arg == "--strategy-stats":
model.strategy_stats.do_stats = True
+ elif arg == "--strategy-stats-with-sizes":
+ model.strategy_stats.do_stats = True
+ model.strategy_stats.do_stats_sizes = True
elif path is None:
path = argv[idx]
else:
More information about the pypy-commit
mailing list