[pypy-svn] r58617 - in pypy/dist/pypy/translator/backendopt: . test
arigo at codespeak.net
arigo at codespeak.net
Mon Oct 6 10:54:44 CEST 2008
Author: arigo
Date: Mon Oct 6 10:54:43 2008
New Revision: 58617
Added:
pypy/dist/pypy/translator/backendopt/mallocv.py (contents, props changed)
pypy/dist/pypy/translator/backendopt/test/test_mallocv.py
- copied, changed from r58615, pypy/dist/pypy/translator/backendopt/test/test_malloc.py
Log:
In-progress. Another approach to malloc removal which is
abstract-interpretation-based.
Added: pypy/dist/pypy/translator/backendopt/mallocv.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/translator/backendopt/mallocv.py Mon Oct 6 10:54:43 2008
@@ -0,0 +1,236 @@
+from pypy.objspace.flow.model import Variable, Constant, Block, Link
+from pypy.objspace.flow.model import SpaceOperation
+from pypy.translator.backendopt.support import log
+
+
+# ____________________________________________________________
+
+
+class CannotVirtualize(Exception):
+ pass
+
+
+class MallocVirtualizer(object):
+
+ def __init__(self, graphs):
+ self.graphs = list(graphs)
+ self.cache = BlockSpecCache()
+ self.count_virtualized = 0
+
+ def report_result(self):
+ log.mallocv('removed %d mallocs so far' % (self.count_virtualized,))
+ return self.count_virtualized
+
+ def find_all_mallocs(self, graph):
+ result = []
+ for block in graph.iterblocks():
+ for op in block.operations:
+ if op.opname == 'malloc':
+ result.append((block, op))
+ return result
+
+ def remove_mallocs_once(self, verbose=False):
+ prev = self.count_virtualized
+ for graph in self.graphs:
+ all_mallocs = self.find_all_mallocs(graph)
+ for block, op in all_mallocs:
+ try:
+ self.try_remove_malloc(block, op)
+ if verbose:
+ log.mallocv('%s removed' % (op.result,))
+ break # don't continue on this graph, which was mutated
+ except CannotVirtualize, e:
+ if verbose:
+ log.mallocv('%s failed: %s' % (op.result, e))
+ progress = self.report_result() - prev
+ return progress
+
+ def try_remove_malloc(self, block, op):
+ mallocspec = MallocSpecializer(self, op.result.concretetype.TO)
+ mallocspec.remove_malloc(block, op.result)
+ mallocspec.propagate_specializations()
+ # if we read this point without a CannotVirtualize, success
+ mallocspec.commit()
+ self.count_virtualized += 1
+
+
+class BlockSpecCache(object):
+
+ def __init__(self, fallback=None):
+ self.specialized_blocks = {}
+ self.block_keys = {}
+ self.fallback = fallback
+
+ def lookup_spec_block(self, block, extra_key):
+ key = self.block_keys.get(block, frozenset())
+ key = key.union([extra_key])
+ try:
+ return self.specialized_blocks[key]
+ except KeyError:
+ if self.fallback is None:
+ return None
+ else:
+ return self.fallback.specialized_blocks.get(key)
+
+ def remember_spec_block(self, block, extra_key, specblock):
+ key = self.block_keys.get(block, frozenset())
+ key = key.union([extra_key])
+ self.specialized_blocks[key] = specblock
+ self.block_keys[specblock] = key
+
+ def push_changes(self):
+ self.fallback.specialized_blocks.update(self.specialized_blocks)
+ self.fallback.block_keys.update(self.block_keys)
+
+
+class MallocSpecializer(object):
+
+ def __init__(self, mallocv, MALLOCTYPE):
+ self.mallocv = mallocv
+ self.MALLOCTYPE = MALLOCTYPE
+ self.names_and_types = []
+ self.name2index = {}
+ self.initialize_type(MALLOCTYPE)
+ self.pending_specializations = []
+ self.cache = BlockSpecCache(fallback=mallocv.cache)
+
+ def initialize_type(self, TYPE):
+ fieldnames = TYPE._names
+ firstname, FIRSTTYPE = TYPE._first_struct()
+ if FIRSTTYPE is not None:
+ self.initialize_type(FIRSTTYPE)
+ fieldnames = fieldnames[1:]
+ for name in fieldnames:
+ FIELDTYPE = TYPE._flds[name]
+ self.name2index[name] = len(self.names_and_types)
+ self.names_and_types.append((name, FIELDTYPE))
+
+ def remove_malloc(self, block, v_result):
+ self.startblock = block
+ spec = BlockSpecializer(self, v_result)
+ spec.initialize_renamings(block.inputargs)
+ self.newinputargs = spec.expand_vars(block.inputargs)
+ self.newoperations = spec.specialize_operations(block.operations)
+ self.newexits = self.follow_exits(block, spec)
+
+ def follow_exits(self, block, spec):
+ newlinks = []
+ for link in block.exits:
+ targetblock = link.target
+ for v1, v2 in zip(link.args, link.target.inputargs):
+ if v1 in spec.curvars:
+ targetblock = self.get_specialized_block(targetblock, v2)
+ newlink = Link(spec.expand_vars(link.args), targetblock)
+ newlinks.append(newlink)
+ return newlinks
+
+ def get_specialized_block(self, block, v):
+ specblock = self.cache.lookup_spec_block(block, (self.MALLOCTYPE, v))
+ if specblock is None:
+ if block.operations == ():
+ raise CannotVirtualize("return or except block")
+ spec = BlockSpecializer(self, v)
+ spec.make_expanded_vars()
+ spec.initialize_renamings(block.inputargs)
+ specblock = Block(spec.expand_vars(block.inputargs))
+ self.pending_specializations.append((spec, block, specblock))
+ self.cache.remember_spec_block(block, (self.MALLOCTYPE, v),
+ specblock)
+ return specblock
+
+ def propagate_specializations(self):
+ while self.pending_specializations:
+ spec, block, specblock = self.pending_specializations.pop()
+ specblock.operations = spec.specialize_operations(block.operations)
+ specblock.closeblock(*self.follow_exits(block, spec))
+
+ def commit(self):
+ self.startblock.inputargs = self.newinputargs
+ self.startblock.operations = self.newoperations
+ self.startblock.recloseblock(*self.newexits)
+ self.cache.push_changes()
+
+
+class BlockSpecializer(object):
+
+ def __init__(self, mallocspec, v):
+ self.mallocspec = mallocspec
+ self.curvars = set([v])
+
+ def make_expanded_vars(self):
+ self.expanded_v = []
+ for name, FIELDTYPE in self.mallocspec.names_and_types:
+ v = Variable(name)
+ v.concretetype = FIELDTYPE
+ self.expanded_v.append(v)
+
+ def make_expanded_zero_constants(self):
+ self.expanded_v = []
+ for name, FIELDTYPE in self.mallocspec.names_and_types:
+ c = Constant(FIELDTYPE._defl())
+ c.concretetype = FIELDTYPE
+ self.expanded_v.append(c)
+
+ def rename_nonvirtual(self, v, where=None):
+ if v in self.curvars:
+ raise CannotVirtualize(where)
+ [v2] = self.renamings[v]
+ return v2
+
+ def expand_vars(self, vars):
+ result_v = []
+ for v in vars:
+ result_v += self.renamings[v]
+ return result_v
+
+ def initialize_renamings(self, inputargs):
+ self.renamings = {}
+ for v in inputargs:
+ if v in self.curvars:
+ self.renamings[v] = self.expanded_v
+ else:
+ v2 = Variable(v)
+ v2.concretetype = v.concretetype
+ self.renamings[v] = [v2]
+
+ def specialize_operations(self, operations):
+ newoperations = []
+ for op in operations:
+ meth = getattr(self, 'handle_op_' + op.opname,
+ self.handle_default)
+ newoperations += meth(op)
+ return newoperations
+
+ def handle_default(self, op):
+ newargs = [self.rename_nonvirtual(v, op) for v in op.args]
+ newresult = Variable(op.result)
+ newresult.concretetype = op.result.concretetype
+ self.renamings[op.result] = [newresult]
+ return [SpaceOperation(op.opname, newargs, newresult)]
+
+ def handle_op_getfield(self, op):
+ if op.args[0] in self.curvars:
+ fieldname = op.args[1].value
+ index = self.mallocspec.name2index[fieldname]
+ v_field = self.expanded_v[index]
+ self.renamings[op.result] = [v_field]
+ return []
+ else:
+ return self.handle_default(op)
+
+ def handle_op_setfield(self, op):
+ if op.args[0] in self.curvars:
+ fieldname = op.args[1].value
+ index = self.mallocspec.name2index[fieldname]
+ self.expanded_v[index] = self.rename_nonvirtual(op.args[2], op)
+ return []
+ else:
+ return self.handle_default(op)
+
+ def handle_op_malloc(self, op):
+ if op.result in self.curvars:
+ self.make_expanded_zero_constants()
+ self.renamings[op.result] = self.expanded_v
+ return []
+ else:
+ return self.handle_default(op)
Copied: pypy/dist/pypy/translator/backendopt/test/test_mallocv.py (from r58615, pypy/dist/pypy/translator/backendopt/test/test_malloc.py)
==============================================================================
--- pypy/dist/pypy/translator/backendopt/test/test_malloc.py (original)
+++ pypy/dist/pypy/translator/backendopt/test/test_mallocv.py Mon Oct 6 10:54:43 2008
@@ -1,10 +1,11 @@
import py
-from pypy.translator.backendopt.malloc import LLTypeMallocRemover, OOTypeMallocRemover
+from pypy.translator.backendopt.mallocv import MallocVirtualizer
from pypy.translator.backendopt.inline import inline_function
from pypy.translator.backendopt.all import backend_optimizations
from pypy.translator.translator import TranslationContext, graphof
from pypy.translator import simplify
from pypy.objspace.flow.model import checkgraph, flatten, Block, mkentrymap
+from pypy.objspace.flow.model import summary
from pypy.rpython.llinterp import LLInterpreter
from pypy.rpython.lltypesystem import lltype, llmemory
from pypy.rpython.ootypesystem import ootype
@@ -20,16 +21,16 @@
py.test.skip(msg)
def check_malloc_removed(cls, graph):
- remover = cls.MallocRemover()
+ #remover = cls.MallocRemover()
checkgraph(graph)
count1 = count2 = 0
for node in flatten(graph):
if isinstance(node, Block):
for op in node.operations:
- if op.opname == cls.MallocRemover.MALLOC_OP:
+ if op.opname == 'malloc': #cls.MallocRemover.MALLOC_OP:
S = op.args[0].value
- if not remover.union_wrapper(S): # union wrappers are fine
- count1 += 1
+ #if not remover.union_wrapper(S): # union wrappers are fine
+ count1 += 1
if op.opname in ('direct_call', 'indirect_call'):
count2 += 1
assert count1 == 0 # number of mallocs left
@@ -37,7 +38,6 @@
check_malloc_removed = classmethod(check_malloc_removed)
def check(self, fn, signature, args, expected_result, must_be_removed=True):
- remover = self.MallocRemover()
t = TranslationContext()
t.buildannotator().build_types(fn, signature)
t.buildrtyper(type_system=self.type_system).specialize()
@@ -46,9 +46,11 @@
t.view()
# to detect missing keepalives and broken intermediate graphs,
# we do the loop ourselves instead of calling remove_simple_mallocs()
+ maxiter = 100
+ mallocv = MallocVirtualizer(t.graphs)
while True:
- progress = remover.remove_mallocs_once(graph)
- simplify.transform_dead_op_vars_in_blocks(list(graph.iterblocks()))
+ progress = mallocv.remove_mallocs_once(verbose=True)
+ #simplify.transform_dead_op_vars_in_blocks(list(graph.iterblocks()))
if progress and option.view:
t.view()
if expected_result is not Ellipsis:
@@ -69,9 +71,12 @@
t = x-y, x+y
s, d = t
return s*d
- self.check(fn1, [int, int], [15, 10], 125)
+ graph = self.check(fn1, [int, int], [15, 10], 125)
+ insns = summary(graph)
+ assert insns['int_mul'] == 1
def test_fn2(self):
+ py.test.skip("redo me")
class T:
pass
def fn2(x, y):
@@ -85,12 +90,14 @@
self.check(fn2, [int, int], [-6, 7], -13)
def test_fn3(self):
+ py.test.skip("redo me")
def fn3(x):
a, ((b, c), d, e) = x+1, ((x+2, x+3), x+4, x+5)
return a+b+c+d+e
self.check(fn3, [int], [10], 65)
def test_fn4(self):
+ py.test.skip("redo me")
class A:
pass
class B(A):
@@ -104,6 +111,7 @@
self.check(fn4, [int], [42], 42)
def test_fn5(self):
+ py.test.skip("redo me")
class A:
attr = 666
class B(A):
@@ -114,6 +122,7 @@
self.check(fn5, [], [], 42)
def test_aliasing(self):
+ py.test.skip("redo me")
class A:
pass
def fn6(n):
@@ -133,9 +142,10 @@
class TestLLTypeMallocRemoval(BaseMallocRemovalTest):
type_system = 'lltype'
- MallocRemover = LLTypeMallocRemover
+ #MallocRemover = LLTypeMallocRemover
def test_with_keepalive(self):
+ py.test.skip("redo me")
from pypy.rlib.objectmodel import keepalive_until_here
def fn1(x, y):
if x > 0:
@@ -148,6 +158,7 @@
self.check(fn1, [int, int], [15, 10], 125)
def test_dont_remove_with__del__(self):
+ py.test.skip("redo me")
import os
delcalls = [0]
class A(object):
@@ -177,6 +188,7 @@
assert op.opname == "malloc"
def test_add_keepalives(self):
+ py.test.skip("redo me")
class A:
pass
SMALL = lltype.Struct('SMALL', ('x', lltype.Signed))
@@ -194,6 +206,7 @@
self.check(fn7, [int], [10], 55, must_be_removed=False)
def test_getsubstruct(self):
+ py.test.skip("redo me")
py.test.skip("fails because of the interior structure changes")
SMALL = lltype.Struct('SMALL', ('x', lltype.Signed))
BIG = lltype.GcStruct('BIG', ('z', lltype.Signed), ('s', SMALL))
@@ -207,6 +220,7 @@
self.check(fn, [int, int], [100, 58], 42)
def test_fixedsizearray(self):
+ py.test.skip("redo me")
py.test.skip("fails because of the interior structure changes")
A = lltype.FixedSizeArray(lltype.Signed, 3)
S = lltype.GcStruct('S', ('a', A))
@@ -221,6 +235,7 @@
self.check(fn, [int, int], [100, 42], 58)
def test_wrapper_cannot_be_removed(self):
+ py.test.skip("redo me")
SMALL = lltype.OpaqueType('SMALL')
BIG = lltype.GcStruct('BIG', ('z', lltype.Signed), ('s', SMALL))
@@ -233,6 +248,7 @@
self.check(fn, [], [], None, must_be_removed=False)
def test_direct_fieldptr(self):
+ py.test.skip("redo me")
S = lltype.GcStruct('S', ('x', lltype.Signed))
def fn():
@@ -244,6 +260,7 @@
self.check(fn, [], [], 11)
def test_direct_fieldptr_2(self):
+ py.test.skip("redo me")
T = lltype.GcStruct('T', ('z', lltype.Signed))
S = lltype.GcStruct('S', ('t', T),
('x', lltype.Signed),
@@ -261,6 +278,7 @@
self.check(fn, [], [], 42)
def test_getarraysubstruct(self):
+ py.test.skip("redo me")
py.test.skip("fails because of the interior structure changes")
U = lltype.Struct('U', ('n', lltype.Signed))
for length in [1, 2]:
@@ -274,6 +292,7 @@
self.check(fn, [], [], 12)
def test_ptr_nonzero(self):
+ py.test.skip("redo me")
S = lltype.GcStruct('S')
def fn():
s = lltype.malloc(S)
@@ -281,6 +300,7 @@
self.check(fn, [], [], True)
def test_substruct_not_accessed(self):
+ py.test.skip("redo me")
SMALL = lltype.Struct('SMALL', ('x', lltype.Signed))
BIG = lltype.GcStruct('BIG', ('z', lltype.Signed), ('s', SMALL))
def fn():
@@ -291,6 +311,7 @@
self.check(fn, [], [], 12)
def test_union(self):
+ py.test.skip("redo me")
py.test.skip("fails because of the interior structure changes")
UNION = lltype.Struct('UNION', ('a', lltype.Signed), ('b', lltype.Signed),
hints = {'union': True})
@@ -303,6 +324,7 @@
self.check(fn, [], [], Ellipsis)
def test_keep_all_keepalives(self):
+ py.test.skip("redo me")
SIZE = llmemory.sizeof(lltype.Signed)
PARRAY = lltype.Ptr(lltype.FixedSizeArray(lltype.Signed, 1))
class A:
@@ -335,6 +357,7 @@
assert link.prevblock.operations[-1].opname == 'keepalive'
def test_nested_struct(self):
+ py.test.skip("redo me")
S = lltype.GcStruct("S", ('x', lltype.Signed))
T = lltype.GcStruct("T", ('s', S))
def f(x):
@@ -346,6 +369,7 @@
graph = self.check(f, [int], [42], 2 * 42)
def test_interior_ptr(self):
+ py.test.skip("redo me")
py.test.skip("fails")
S = lltype.Struct("S", ('x', lltype.Signed))
T = lltype.GcStruct("T", ('s', S))
@@ -356,6 +380,7 @@
graph = self.check(f, [int], [42], 42)
def test_interior_ptr_with_index(self):
+ py.test.skip("redo me")
S = lltype.Struct("S", ('x', lltype.Signed))
T = lltype.GcArray(S)
def f(x):
@@ -365,6 +390,7 @@
graph = self.check(f, [int], [42], 42)
def test_interior_ptr_with_field_and_index(self):
+ py.test.skip("redo me")
S = lltype.Struct("S", ('x', lltype.Signed))
T = lltype.GcStruct("T", ('items', lltype.Array(S)))
def f(x):
@@ -374,6 +400,7 @@
graph = self.check(f, [int], [42], 42)
def test_interior_ptr_with_index_and_field(self):
+ py.test.skip("redo me")
S = lltype.Struct("S", ('x', lltype.Signed))
T = lltype.Struct("T", ('s', S))
U = lltype.GcArray(T)
@@ -384,9 +411,9 @@
graph = self.check(f, [int], [42], 42)
-class TestOOTypeMallocRemoval(BaseMallocRemovalTest):
+class DISABLED_TestOOTypeMallocRemoval(BaseMallocRemovalTest):
type_system = 'ootype'
- MallocRemover = OOTypeMallocRemover
+ #MallocRemover = OOTypeMallocRemover
def test_oononnull(self):
FOO = ootype.Instance('Foo', ootype.ROOT)
More information about the Pypy-commit
mailing list