[pypy-svn] r79094 - in pypy/branch/jit-free/pypy: jit/metainterp jit/metainterp/test rlib
arigo at codespeak.net
arigo at codespeak.net
Mon Nov 15 13:39:56 CET 2010
Author: arigo
Date: Mon Nov 15 13:39:54 2010
New Revision: 79094
Added:
pypy/branch/jit-free/pypy/jit/metainterp/memmgr.py
pypy/branch/jit-free/pypy/jit/metainterp/test/test_memmgr.py
Modified:
pypy/branch/jit-free/pypy/jit/metainterp/blackhole.py
pypy/branch/jit-free/pypy/jit/metainterp/compile.py
pypy/branch/jit-free/pypy/jit/metainterp/history.py
pypy/branch/jit-free/pypy/jit/metainterp/pyjitpl.py
pypy/branch/jit-free/pypy/jit/metainterp/resume.py
pypy/branch/jit-free/pypy/jit/metainterp/test/test_compile.py
pypy/branch/jit-free/pypy/jit/metainterp/test/test_optimizeopt.py
pypy/branch/jit-free/pypy/jit/metainterp/test/test_warmspot.py
pypy/branch/jit-free/pypy/jit/metainterp/warmspot.py
pypy/branch/jit-free/pypy/jit/metainterp/warmstate.py
pypy/branch/jit-free/pypy/rlib/jit.py
Log:
In-progress: implement the new memmgr.py.
Contains some kind of clean-up: now the ResumeDescrs contain,
instead of 'original_greenkey', just 'original_loop_token'
that points to the loop token corresponding to the root of
the loop in which the guard is. Move 'greenkey' there too.
Not done: see comment in test_memmgr.py.
Modified: pypy/branch/jit-free/pypy/jit/metainterp/blackhole.py
==============================================================================
--- pypy/branch/jit-free/pypy/jit/metainterp/blackhole.py (original)
+++ pypy/branch/jit-free/pypy/jit/metainterp/blackhole.py Mon Nov 15 13:39:54 2010
@@ -1362,14 +1362,12 @@
# We will continue to loop in _run_forever() from the parent level.
return blackholeinterp, lle
-def resume_in_blackhole(metainterp_sd, jitdriver_sd, resumedescr,
- all_virtuals=None):
+def resume_in_blackhole(metainterp_sd, resumedescr, all_virtuals=None):
from pypy.jit.metainterp.resume import blackhole_from_resumedata
debug_start('jit-blackhole')
metainterp_sd.profiler.start_blackhole()
blackholeinterp = blackhole_from_resumedata(
metainterp_sd.blackholeinterpbuilder,
- jitdriver_sd,
resumedescr,
all_virtuals)
current_exc = blackholeinterp._prepare_resume_from_failure(
Modified: pypy/branch/jit-free/pypy/jit/metainterp/compile.py
==============================================================================
--- pypy/branch/jit-free/pypy/jit/metainterp/compile.py (original)
+++ pypy/branch/jit-free/pypy/jit/metainterp/compile.py Mon Nov 15 13:39:54 2010
@@ -42,10 +42,11 @@
name = metainterp.staticdata.stats.name_for_new_loop()
return TreeLoop(name)
-def make_loop_token(nb_args, jitdriver_sd):
+def make_loop_token(nb_args, jitdriver_sd, greenkey):
loop_token = LoopToken()
loop_token.specnodes = [prebuiltNotSpecNode] * nb_args
loop_token.outermost_jitdriver_sd = jitdriver_sd
+ loop_token.outermost_greenkey = greenkey
return loop_token
# ____________________________________________________________
@@ -56,7 +57,6 @@
"""
history = metainterp.history
loop = create_empty_loop(metainterp)
- loop.greenkey = greenkey
loop.inputargs = history.inputargs
for box in loop.inputargs:
assert isinstance(box, Box)
@@ -65,7 +65,7 @@
loop.operations = [h_ops[i].clone() for i in range(start, len(h_ops))]
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
- loop_token = make_loop_token(len(loop.inputargs), jitdriver_sd)
+ loop_token = make_loop_token(len(loop.inputargs), jitdriver_sd, greenkey)
loop.token = loop_token
loop.operations[-1].setdescr(loop_token) # patch the target of the JUMP
try:
@@ -98,6 +98,9 @@
loop_token = loop.token
loop_token.number = n = globaldata.loopnumbering
globaldata.loopnumbering += 1
+ desc = metainterp_sd.warmrunnerdesc
+ if desc is not None: # for tests
+ desc.memory_manager.record_loop(loop_token)
metainterp_sd.logger_ops.log_loop(loop.inputargs, loop.operations, n, type)
if not we_are_translated():
@@ -142,32 +145,32 @@
pass
class DoneWithThisFrameDescrVoid(_DoneWithThisFrameDescr):
- def handle_fail(self, metainterp_sd, jitdriver_sd):
- assert jitdriver_sd.result_type == history.VOID
+ def handle_fail(self, metainterp_sd):
+ #assert jitdriver_sd.result_type == history.VOID
raise metainterp_sd.DoneWithThisFrameVoid()
class DoneWithThisFrameDescrInt(_DoneWithThisFrameDescr):
- def handle_fail(self, metainterp_sd, jitdriver_sd):
- assert jitdriver_sd.result_type == history.INT
+ def handle_fail(self, metainterp_sd):
+ #assert jitdriver_sd.result_type == history.INT
result = metainterp_sd.cpu.get_latest_value_int(0)
raise metainterp_sd.DoneWithThisFrameInt(result)
class DoneWithThisFrameDescrRef(_DoneWithThisFrameDescr):
- def handle_fail(self, metainterp_sd, jitdriver_sd):
- assert jitdriver_sd.result_type == history.REF
+ def handle_fail(self, metainterp_sd):
+ #assert jitdriver_sd.result_type == history.REF
cpu = metainterp_sd.cpu
result = cpu.get_latest_value_ref(0)
cpu.clear_latest_values(1)
raise metainterp_sd.DoneWithThisFrameRef(cpu, result)
class DoneWithThisFrameDescrFloat(_DoneWithThisFrameDescr):
- def handle_fail(self, metainterp_sd, jitdriver_sd):
- assert jitdriver_sd.result_type == history.FLOAT
+ def handle_fail(self, metainterp_sd):
+ #assert jitdriver_sd.result_type == history.FLOAT
result = metainterp_sd.cpu.get_latest_value_float(0)
raise metainterp_sd.DoneWithThisFrameFloat(result)
class ExitFrameWithExceptionDescrRef(_DoneWithThisFrameDescr):
- def handle_fail(self, metainterp_sd, jitdriver_sd):
+ def handle_fail(self, metainterp_sd):
cpu = metainterp_sd.cpu
value = cpu.get_latest_value_ref(0)
cpu.clear_latest_values(1)
@@ -209,8 +212,9 @@
}
class ResumeDescr(AbstractFailDescr):
- def __init__(self, original_greenkey):
- self.original_greenkey = original_greenkey
+ def __init__(self, original_loop_token):
+ assert isinstance(original_loop_token, history.LoopToken)
+ self.original_loop_token = original_loop_token
class ResumeGuardDescr(ResumeDescr):
_counter = 0 # if < 0, there is one counter per value;
@@ -229,10 +233,6 @@
CNT_FLOAT = -0x60000000
CNT_MASK = 0x1FFFFFFF
- def __init__(self, metainterp_sd, original_greenkey):
- ResumeDescr.__init__(self, original_greenkey)
- self.metainterp_sd = metainterp_sd
-
def store_final_boxes(self, guard_op, boxes):
guard_op.setfailargs(boxes)
self.guard_opnum = guard_op.getopnum()
@@ -257,26 +257,29 @@
# a negative value
self._counter = cnt | i
- def handle_fail(self, metainterp_sd, jitdriver_sd):
- if self.must_compile(metainterp_sd, jitdriver_sd):
- return self._trace_and_compile_from_bridge(metainterp_sd,
- jitdriver_sd)
+ def handle_fail(self, metainterp_sd):
+ if self.must_compile(metainterp_sd):
+ return self._trace_and_compile_from_bridge(metainterp_sd)
else:
from pypy.jit.metainterp.blackhole import resume_in_blackhole
- resume_in_blackhole(metainterp_sd, jitdriver_sd, self)
+ resume_in_blackhole(metainterp_sd, self)
assert 0, "unreachable"
- def _trace_and_compile_from_bridge(self, metainterp_sd, jitdriver_sd):
- # 'jitdriver_sd' corresponds to the outermost one, i.e. the one
- # of the jit_merge_point where we started the loop, even if the
- # loop itself may contain temporarily recursion into other
- # jitdrivers.
+ def _trace_and_compile_from_bridge(self, metainterp_sd):
from pypy.jit.metainterp.pyjitpl import MetaInterp
- metainterp = MetaInterp(metainterp_sd, jitdriver_sd)
+ metainterp = MetaInterp(metainterp_sd, self.get_jitdriver_sd())
return metainterp.handle_guard_failure(self)
_trace_and_compile_from_bridge._dont_inline_ = True
- def must_compile(self, metainterp_sd, jitdriver_sd):
+ def get_jitdriver_sd(self):
+ # Returns the jitdriver_sd that corresponds to the outermost
+ # level, i.e. the level of the jit_merge_point where we started
+ # the loop, even if the loop itself may contain some recursion
+ # into other jitdrivers.
+ return self.original_loop_token.outermost_jitdriver_sd
+
+ def must_compile(self, metainterp_sd):
+ jitdriver_sd = self.get_jitdriver_sd()
trace_eagerness = jitdriver_sd.warmstate.trace_eagerness
if self._counter >= 0:
self._counter += 1
@@ -334,17 +337,18 @@
res.rd_pendingfields = self.rd_pendingfields
def _clone_if_mutable(self):
- res = ResumeGuardDescr(self.metainterp_sd, self.original_greenkey)
+ res = ResumeGuardDescr(self.original_loop_token)
self.copy_all_attrbutes_into(res)
return res
class ResumeGuardForcedDescr(ResumeGuardDescr):
- def __init__(self, metainterp_sd, original_greenkey, jitdriver_sd):
- ResumeGuardDescr.__init__(self, metainterp_sd, original_greenkey)
+ def __init__(self, metainterp_sd, original_loop_token, jitdriver_sd):
+ ResumeGuardDescr.__init__(self, original_loop_token)
+ self.metainterp_sd = metainterp_sd
self.jitdriver_sd = jitdriver_sd
- def handle_fail(self, metainterp_sd, jitdriver_sd):
+ def handle_fail(self, metainterp_sd):
# Failures of a GUARD_NOT_FORCED are never compiled, but
# always just blackholed. First fish for the data saved when
# the virtualrefs and virtualizable have been forced by
@@ -354,8 +358,7 @@
all_virtuals = self.fetch_data(token)
if all_virtuals is None:
all_virtuals = []
- assert jitdriver_sd is self.jitdriver_sd
- resume_in_blackhole(metainterp_sd, jitdriver_sd, self, all_virtuals)
+ resume_in_blackhole(metainterp_sd, self, all_virtuals)
assert 0, "unreachable"
@staticmethod
@@ -408,7 +411,7 @@
def _clone_if_mutable(self):
res = ResumeGuardForcedDescr(self.metainterp_sd,
- self.original_greenkey,
+ self.original_loop_token,
self.jitdriver_sd)
self.copy_all_attrbutes_into(res)
return res
@@ -475,8 +478,11 @@
class ResumeFromInterpDescr(ResumeDescr):
- def __init__(self, original_greenkey, redkey):
- ResumeDescr.__init__(self, original_greenkey)
+ def __init__(self, metainterp, greenkey, redkey):
+ original_loop_token = make_loop_token(len(redkey),
+ metainterp.jitdriver_sd,
+ greenkey)
+ ResumeDescr.__init__(self, original_loop_token)
self.redkey = redkey
def compile_and_attach(self, metainterp, new_loop):
@@ -487,18 +493,18 @@
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
metainterp.history.inputargs = self.redkey
- new_loop_token = make_loop_token(len(self.redkey), jitdriver_sd)
- new_loop.greenkey = self.original_greenkey
+ new_loop_token = self.original_loop_token
+ original_greenkey = new_loop_token.outermost_greenkey
new_loop.inputargs = self.redkey
new_loop.token = new_loop_token
send_loop_to_backend(metainterp_sd, new_loop, "entry bridge")
# send the new_loop to warmspot.py, to be called directly the next time
jitdriver_sd.warmstate.attach_unoptimized_bridge_from_interp(
- self.original_greenkey,
+ original_greenkey,
new_loop_token)
# store the new loop in compiled_merge_points too
old_loop_tokens = metainterp.get_compiled_merge_points(
- self.original_greenkey)
+ original_greenkey)
# it always goes at the end of the list, as it is the most
# general loop token
old_loop_tokens.append(new_loop_token)
@@ -555,7 +561,7 @@
# ____________________________________________________________
class PropagateExceptionDescr(AbstractFailDescr):
- def handle_fail(self, metainterp_sd, jitdriver_sd):
+ def handle_fail(self, metainterp_sd):
cpu = metainterp_sd.cpu
exception = cpu.grab_exc_value()
raise metainterp_sd.ExitFrameWithExceptionRef(cpu, exception)
@@ -569,7 +575,7 @@
"""
# 'redboxes' is only used to know the types of red arguments.
inputargs = [box.clonebox() for box in redboxes]
- loop_token = make_loop_token(len(inputargs), jitdriver_sd)
+ loop_token = make_loop_token(len(inputargs), jitdriver_sd, greenboxes)
# 'nb_red_args' might be smaller than len(redboxes),
# because it doesn't include the virtualizable boxes.
nb_red_args = jitdriver_sd.num_red_args
Modified: pypy/branch/jit-free/pypy/jit/metainterp/history.py
==============================================================================
--- pypy/branch/jit-free/pypy/jit/metainterp/history.py (original)
+++ pypy/branch/jit-free/pypy/jit/metainterp/history.py Mon Nov 15 13:39:54 2010
@@ -4,7 +4,7 @@
from pypy.rpython.ootypesystem import ootype
from pypy.rlib.objectmodel import we_are_translated, r_dict, Symbolic
from pypy.rlib.objectmodel import compute_hash, compute_unique_id
-from pypy.rlib.rarithmetic import intmask
+from pypy.rlib.rarithmetic import intmask, r_longlong
from pypy.tool.uid import uid
from pypy.conftest import option
@@ -178,8 +178,9 @@
class AbstractFailDescr(AbstractDescr):
index = -1
+ original_loop_token = None
- def handle_fail(self, metainterp_sd, jitdriver_sd):
+ def handle_fail(self, metainterp_sd):
raise NotImplementedError
def compile_and_attach(self, metainterp, new_loop):
raise NotImplementedError
@@ -727,13 +728,21 @@
generated assembler.
"""
terminating = False # see TerminatingLoopToken in compile.py
+ has_been_freed = False
outermost_jitdriver_sd = None
+ outermost_greenkey = None
# specnodes = ...
# and more data specified by the backend when the loop is compiled
- number = 0
def __init__(self, number=0):
self.number = number
+ # See get_fail_descr_number() in backend/model.py: this growing
+ # list gives the 'descr_number' of all fail descrs that belong to
+ # this loop or to a bridge attached to it.
+ self.faildescr_indices = []
+ # For memory management of assembled loops
+ self.contains_jumps_to = {} # set of other LoopTokens
+ self.generation = r_longlong(0)
def repr_of_descr(self):
return '<Loop%d>' % self.number
Added: pypy/branch/jit-free/pypy/jit/metainterp/memmgr.py
==============================================================================
--- (empty file)
+++ pypy/branch/jit-free/pypy/jit/metainterp/memmgr.py Mon Nov 15 13:39:54 2010
@@ -0,0 +1,106 @@
+import math
+from pypy.rlib.rarithmetic import r_longlong
+from pypy.rlib.objectmodel import we_are_translated
+
+#
+# Logic to decide which loops are old and not used any more.
+#
+# Idea: We use the notion of a global 'current generation' which
+# is, in practice, the total number of loops and bridges produced
+# so far. When executing a loop:
+# (1) we set 'generation' to -1
+# (2) we execute it
+# (3) we set 'generation' to the latest generation
+# (with a bit extra work to handle nested calls and to guarantee
+# that 'generation' is always < 0 on a loop that is currently
+# executing).
+#
+# A loop is said "old" if its generation is >= 0 but much smaller
+# than the current generation. If a loop L is old, and if all
+# other loops from which we can reach L through
+# 'contains_jumps_to' are also old, then we can free L.
+#
+
+class MemoryManager(object):
+
+ def __init__(self, cpu):
+ self.cpu = cpu
+ self.check_frequency = -1
+ # NB. use of r_longlong to be extremely far on the safe side:
+ # this is increasing by one after each loop or bridge is
+ # compiled, and it must not overflow. If the backend implements
+ # complete freeing in cpu.free_loop_and_bridges(), then it may
+ # be possible to get arbitrary many of them just by waiting long
+ # enough. But in this day and age, you'd still never have the
+ # patience of waiting for a slowly-increasing 64-bit number to
+ # overflow :-)
+ self.current_generation = r_longlong(0)
+ self.next_check = r_longlong(-1)
+ self.looptokens = []
+
+ def set_max_age(self, max_age, check_frequency=0):
+ if max_age <= 0:
+ self.next_check = r_longlong(-1)
+ else:
+ self.max_age = max_age
+ if check_frequency <= 0:
+ check_frequency = int(math.sqrt(max_age))
+ self.check_frequency = check_frequency
+ self.next_check = self.current_generation + 1
+
+ def next_generation(self):
+ self.current_generation += 1
+ if self.current_generation == self.next_check:
+ self._free_old_loops_now()
+ self.next_check = self.current_generation + self.check_frequency
+
+ def enter_loop(self, looptoken):
+ if not we_are_translated():
+ assert looptoken in self.looptokens
+ assert not looptoken.has_been_freed
+ if looptoken.generation >= 0:
+ looptoken.generation = -1
+ else:
+ looptoken.generation -= 1 # nested enter_loop()
+
+ def leave_loop(self, looptoken):
+ assert looptoken.generation < 0
+ if looptoken.generation == -1:
+ looptoken.generation = self.current_generation
+ else:
+ looptoken.generation += 1 # nested leave_loop()
+
+ def record_loop(self, looptoken):
+ looptoken.generation = self.current_generation
+ self.looptokens.append(looptoken)
+
+ def _free_old_loops_now(self):
+ #
+ # Initialize '_is_young' on all loop tokens
+ max_generation = self.current_generation - self.max_age
+ youngloops = []
+ for looptoken in self.looptokens:
+ if 0 <= looptoken.generation < max_generation:
+ looptoken._is_young = False # but may be turned to True later
+ else:
+ looptoken._is_young = True
+ youngloops.append(looptoken)
+ #
+ # Propagate forward the knowledge of "is a young loop"
+ while len(youngloops) > 0:
+ looptoken = youngloops.pop()
+ for jumptargettok in looptoken.contains_jumps_to:
+ if not jumptargettok._is_young:
+ jumptargettok._is_young = True
+ youngloops.append(jumptargettok)
+ #
+ # Now free all looptokens that still have _is_young == False.
+ i = 0
+ while i < len(self.looptokens):
+ looptoken = self.looptokens[i]
+ if looptoken._is_young:
+ i += 1
+ else:
+ self.looptokens[i] = self.looptokens[-1]
+ del self.looptokens[-1]
+ self.cpu.free_loop_and_bridges(looptoken)
Modified: pypy/branch/jit-free/pypy/jit/metainterp/pyjitpl.py
==============================================================================
--- pypy/branch/jit-free/pypy/jit/metainterp/pyjitpl.py (original)
+++ pypy/branch/jit-free/pypy/jit/metainterp/pyjitpl.py Mon Nov 15 13:39:54 2010
@@ -1048,14 +1048,13 @@
else:
moreargs = list(extraargs)
metainterp_sd = metainterp.staticdata
- original_greenkey = metainterp.resumekey.original_greenkey
+ original_loop_token = metainterp.resumekey.original_loop_token
if opnum == rop.GUARD_NOT_FORCED:
- resumedescr = compile.ResumeGuardForcedDescr(metainterp_sd,
- original_greenkey,
- metainterp.jitdriver_sd)
+ resumedescr = compile.ResumeGuardForcedDescr(metainterp.staticdata,
+ original_loop_token,
+ metainterp.jitdriver_sd)
else:
- resumedescr = compile.ResumeGuardDescr(metainterp_sd,
- original_greenkey)
+ resumedescr = compile.ResumeGuardDescr(original_loop_token)
guard_op = metainterp.history.record(opnum, moreargs, None,
descr=resumedescr)
virtualizable_boxes = None
@@ -1370,6 +1369,9 @@
self.portal_trace_positions = []
self.free_frames_list = []
self.last_exc_value_box = None
+ # Increase here the generation recorded by the memory manager.
+ if self.staticdata.warmrunnerdesc is not None: # for tests
+ self.staticdata.warmrunnerdesc.memory_manager.next_generation()
def perform_call(self, jitcode, boxes, greenkey=None):
# causes the metainterp to enter the given subfunction
@@ -1637,7 +1639,7 @@
num_green_args = self.jitdriver_sd.num_green_args
original_greenkey = original_boxes[:num_green_args]
redkey = original_boxes[num_green_args:]
- self.resumekey = compile.ResumeFromInterpDescr(original_greenkey,
+ self.resumekey = compile.ResumeFromInterpDescr(self, original_greenkey,
redkey)
self.seen_loop_header_for_jdindex = -1
try:
@@ -1660,7 +1662,7 @@
debug_stop('jit-tracing')
def _handle_guard_failure(self, key):
- original_greenkey = key.original_greenkey
+ original_greenkey = key.original_loop_token.outermost_greenkey
# notice that here we just put the greenkey
# use -1 to mark that we will have to give up
# because we cannot reconstruct the beginning of the proper loop
Modified: pypy/branch/jit-free/pypy/jit/metainterp/resume.py
==============================================================================
--- pypy/branch/jit-free/pypy/jit/metainterp/resume.py (original)
+++ pypy/branch/jit-free/pypy/jit/metainterp/resume.py Mon Nov 15 13:39:54 2010
@@ -915,10 +915,11 @@
# ---------- when resuming for blackholing, get direct values ----------
-def blackhole_from_resumedata(blackholeinterpbuilder, jitdriver_sd, storage,
+def blackhole_from_resumedata(blackholeinterpbuilder, storage,
all_virtuals=None):
resumereader = ResumeDataDirectReader(blackholeinterpbuilder.metainterp_sd,
storage, all_virtuals)
+ jitdriver_sd = storage.get_jitdriver_sd()
vinfo = jitdriver_sd.virtualizable_info
ginfo = jitdriver_sd.greenfield_info
vrefinfo = blackholeinterpbuilder.metainterp_sd.virtualref_info
Modified: pypy/branch/jit-free/pypy/jit/metainterp/test/test_compile.py
==============================================================================
--- pypy/branch/jit-free/pypy/jit/metainterp/test/test_compile.py (original)
+++ pypy/branch/jit-free/pypy/jit/metainterp/test/test_compile.py Mon Nov 15 13:39:54 2010
@@ -5,6 +5,7 @@
from pypy.jit.metainterp.compile import ResumeGuardDescr
from pypy.jit.metainterp.compile import ResumeGuardCountersInt
from pypy.jit.metainterp.compile import compile_tmp_callback
+from pypy.jit.metainterp.compile import send_loop_to_backend
from pypy.jit.metainterp import optimize, jitprof, typesystem, compile
from pypy.jit.metainterp.test.test_optimizefindnode import LLtypeMixin
from pypy.jit.tool.oparser import parse
@@ -207,15 +208,62 @@
class ExitFrameWithExceptionRef(Exception):
pass
FakeMetaInterpSD.cpu = cpu
- class FakeJitDriverSD:
- pass
cpu.set_future_value_int(0, -156)
cpu.set_future_value_int(1, -178)
cpu.set_future_value_int(2, -190)
fail_descr = cpu.execute_token(loop_token)
try:
- fail_descr.handle_fail(FakeMetaInterpSD(), FakeJitDriverSD())
+ fail_descr.handle_fail(FakeMetaInterpSD())
except FakeMetaInterpSD.ExitFrameWithExceptionRef, e:
assert lltype.cast_opaque_ptr(lltype.Ptr(EXC), e.args[1]) == llexc
else:
assert 0, "should have raised"
+
+
+def test_send_loop_to_backend():
+ class FakeMetaInterpSD:
+ class globaldata:
+ loopnumbering = 17
+ class warmrunnerdesc:
+ class memory_manager:
+ @staticmethod
+ def record_loop(token):
+ assert token is FakeLoop.token
+ token._recorded = True
+ class logger_ops:
+ @staticmethod
+ def log_loop(*args, **kwds):
+ pass
+ class profiler:
+ @staticmethod
+ def start_backend():
+ pass
+ @staticmethod
+ def end_backend():
+ pass
+ class cpu:
+ @staticmethod
+ def compile_loop(inputargs, operations, token):
+ assert inputargs is FakeLoop.inputargs
+ assert operations is FakeLoop.operations
+ assert token is FakeLoop.token
+ token._compiled = True
+ class stats:
+ @staticmethod
+ def add_new_loop(loop):
+ pass
+ def log(self, text):
+ pass
+ class FakeLoop:
+ inputargs = []
+ operations = []
+ class token:
+ number = 0
+ _recorded = _compiled = False
+ def check_consistency(self):
+ pass
+ send_loop_to_backend(FakeMetaInterpSD(), FakeLoop(), "entry bridge")
+ assert FakeMetaInterpSD.globaldata.loopnumbering == 18
+ assert FakeLoop.token.number == 17
+ assert FakeLoop.token._recorded is True
+ assert FakeLoop.token._compiled is True
Added: pypy/branch/jit-free/pypy/jit/metainterp/test/test_memmgr.py
==============================================================================
--- (empty file)
+++ pypy/branch/jit-free/pypy/jit/metainterp/test/test_memmgr.py Mon Nov 15 13:39:54 2010
@@ -0,0 +1,158 @@
+from pypy.jit.metainterp.memmgr import MemoryManager
+from pypy.jit.metainterp.history import LoopToken
+
+##missing:
+
+## in _free_old_loops_now(), remove looptoken from everywhere
+## or mark it as freed
+
+## contains_jumps_to needs to be filled
+
+
+class FakeCPU:
+ def free_loop_and_bridges(self, looptoken):
+ looptoken.has_been_freed = True
+cpu = FakeCPU()
+
+
+class TestMemoryManager:
+
+ def test_disabled(self):
+ memmgr = MemoryManager(cpu)
+ memmgr.set_max_age(0)
+ tokens = [LoopToken() for i in range(10)]
+ for token in tokens:
+ memmgr.record_loop(token)
+ memmgr.next_generation()
+ for token in tokens:
+ assert not token.has_been_freed
+
+ def test_basic(self):
+ memmgr = MemoryManager(cpu)
+ memmgr.set_max_age(3, 1)
+ tokens = [LoopToken() for i in range(10)]
+ for token in tokens:
+ memmgr.record_loop(token)
+ memmgr.next_generation()
+ for i in range(len(tokens)):
+ assert tokens[i].has_been_freed == (i < 7)
+
+ def test_basic_2(self):
+ memmgr = MemoryManager(cpu)
+ memmgr.set_max_age(3, 1)
+ token = LoopToken()
+ memmgr.record_loop(token)
+ for i in range(10):
+ memmgr.next_generation()
+ assert token.has_been_freed == (i >= 3)
+
+ def test_enter_loop_1(self):
+ memmgr = MemoryManager(cpu)
+ memmgr.set_max_age(3, 1)
+ tokens = [LoopToken() for i in range(10)]
+ for i in range(len(tokens)):
+ print 'record tokens[%d]' % i
+ memmgr.record_loop(tokens[i])
+ memmgr.next_generation()
+ for j in range(0, i, 2):
+ assert not tokens[j].has_been_freed
+ print 'enter and leave tokens[%d]' % j
+ memmgr.enter_loop(tokens[j])
+ memmgr.leave_loop(tokens[j])
+ for i in range(len(tokens)):
+ assert tokens[i].has_been_freed == (i < 7 and (i%2) != 0)
+
+ def test_enter_loop_2(self):
+ memmgr = MemoryManager(cpu)
+ memmgr.set_max_age(3, 1)
+ tokens = [LoopToken() for i in range(10)]
+ for i in range(len(tokens)):
+ print 'record tokens[%d]' % i
+ memmgr.record_loop(tokens[i])
+ memmgr.next_generation()
+ for j in range(i-2, i+1):
+ if j >= 0:
+ assert not tokens[j].has_been_freed
+ print 'enter and leave tokens[%d]' % j
+ memmgr.enter_loop(tokens[j])
+ memmgr.leave_loop(tokens[j])
+ for i in range(len(tokens)):
+ assert tokens[i].has_been_freed == (i < 4)
+
+ def test_loop_is_running(self):
+ memmgr = MemoryManager(cpu)
+ memmgr.set_max_age(3, 1)
+ token = LoopToken()
+ memmgr.record_loop(token)
+ memmgr.enter_loop(token)
+ for i in range(10):
+ memmgr.next_generation()
+ assert token.has_been_freed == False
+ memmgr.leave_loop(token)
+ for i in range(10):
+ memmgr.next_generation()
+ assert token.has_been_freed == (i >= 3)
+
+ def test_nested_enter_loop(self):
+ memmgr = MemoryManager(cpu)
+ memmgr.set_max_age(3, 1)
+ token = LoopToken()
+ memmgr.record_loop(token)
+ memmgr.enter_loop(token)
+ # here we recursively end up seeing the same token again
+ memmgr.enter_loop(token)
+ for i in range(10):
+ memmgr.next_generation()
+ assert token.has_been_freed == False
+ memmgr.leave_loop(token)
+ # out of the recursive call, but the loop is still "locked"
+ for i in range(10):
+ memmgr.next_generation()
+ assert token.has_been_freed == False
+ memmgr.leave_loop(token)
+ for i in range(10):
+ memmgr.next_generation()
+ assert token.has_been_freed == (i >= 3)
+
+ def test_contains_jumps_to(self):
+ memmgr = MemoryManager(cpu)
+ memmgr.set_max_age(3, 1)
+ token1 = LoopToken()
+ token2 = LoopToken()
+ token1.contains_jumps_to[token2] = None
+ memmgr.record_loop(token1)
+ memmgr.record_loop(token2)
+ memmgr.enter_loop(token1)
+ for i in range(10):
+ memmgr.next_generation()
+ assert token1.has_been_freed == False
+ assert token2.has_been_freed == False
+ memmgr.leave_loop(token1)
+ for i in range(10):
+ memmgr.next_generation()
+ assert token1.has_been_freed == (i >= 3)
+ assert token2.has_been_freed == (i >= 3)
+
+ def test_contains_jumps_to_2(self):
+ memmgr = MemoryManager(cpu)
+ memmgr.set_max_age(3, 1)
+ token1 = LoopToken()
+ token2 = LoopToken()
+ token3 = LoopToken()
+ token1.contains_jumps_to[token2] = None
+ token2.contains_jumps_to[token3] = None
+ memmgr.record_loop(token1)
+ memmgr.record_loop(token2)
+ memmgr.record_loop(token3)
+ memmgr.enter_loop(token1)
+ for i in range(10):
+ memmgr.next_generation()
+ assert token1.has_been_freed == False
+ assert token2.has_been_freed == False
+ assert token3.has_been_freed == False
+ memmgr.leave_loop(token1)
+ for i in range(10):
+ memmgr.next_generation()
+ assert token1.has_been_freed == (i >= 3)
+ assert token2.has_been_freed == (i >= 3)
+ assert token3.has_been_freed == (i >= 3)
Modified: pypy/branch/jit-free/pypy/jit/metainterp/test/test_optimizeopt.py
==============================================================================
--- pypy/branch/jit-free/pypy/jit/metainterp/test/test_optimizeopt.py (original)
+++ pypy/branch/jit-free/pypy/jit/metainterp/test/test_optimizeopt.py Mon Nov 15 13:39:54 2010
@@ -41,7 +41,7 @@
b1 = BoxInt()
opt = optimizeopt.Optimizer(FakeMetaInterpStaticData(LLtypeMixin.cpu),
None)
- fdescr = ResumeGuardDescr(None, None)
+ fdescr = ResumeGuardDescr(history.LoopToken())
op = ResOperation(rop.GUARD_TRUE, ['dummy'], None, descr=fdescr)
# setup rd data
fi0 = resume.FrameInfo(None, "code0", 11)
Modified: pypy/branch/jit-free/pypy/jit/metainterp/test/test_warmspot.py
==============================================================================
--- pypy/branch/jit-free/pypy/jit/metainterp/test/test_warmspot.py (original)
+++ pypy/branch/jit-free/pypy/jit/metainterp/test/test_warmspot.py Mon Nov 15 13:39:54 2010
@@ -382,7 +382,7 @@
def __init__(self, no):
self.no = no
- def handle_fail(self, metainterp_sd, jitdrivers_sd):
+ def handle_fail(self, metainterp_sd):
if self.no == 0:
raise metainterp_sd.warmrunnerdesc.DoneWithThisFrameInt(3)
if self.no == 1:
Modified: pypy/branch/jit-free/pypy/jit/metainterp/warmspot.py
==============================================================================
--- pypy/branch/jit-free/pypy/jit/metainterp/warmspot.py (original)
+++ pypy/branch/jit-free/pypy/jit/metainterp/warmspot.py Mon Nov 15 13:39:54 2010
@@ -12,11 +12,12 @@
from pypy.rlib.unroll import unrolling_iterable
from pypy.rlib.rarithmetic import r_uint, intmask
from pypy.rlib.debug import debug_print, fatalerror
+from pypy.rlib.debug import debug_start, debug_stop
from pypy.rpython.lltypesystem.lloperation import llop
from pypy.translator.simplify import get_funcobj, get_functype
from pypy.translator.unsimplify import call_final_function
-from pypy.jit.metainterp import history, pyjitpl, gc
+from pypy.jit.metainterp import history, pyjitpl, gc, memmgr
from pypy.jit.metainterp.pyjitpl import MetaInterpStaticData, MetaInterp
from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper
from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler
@@ -183,6 +184,7 @@
self.rewrite_set_param()
self.rewrite_force_virtual(vrefinfo)
self.add_finish()
+ self.memory_manager = memmgr.MemoryManager(self.cpu)
self.metainterp_sd.finish_setup(self.codewriter, optimizer=optimizer)
def finish(self):
@@ -711,10 +713,10 @@
vinfo.VTYPEPTR, virtualizableref)
vinfo.reset_vable_token(virtualizable)
try:
- loop_token = fail_descr.handle_fail(self.metainterp_sd, jd)
+ loop_token = fail_descr.handle_fail(self.metainterp_sd)
except JitException, e:
return handle_jitexception(e)
- fail_descr = self.cpu.execute_token(loop_token)
+ fail_descr = self.execute_token(loop_token)
jd._assembler_call_helper = assembler_call_helper # for debugging
jd._assembler_helper_ptr = self.helper_func(
@@ -808,3 +810,15 @@
py.test.skip("rewrite_force_virtual: port it to ootype")
all_graphs = self.translator.graphs
vrefinfo.replace_force_virtual_with_call(all_graphs)
+
+ # ____________________________________________________________
+
+ def execute_token(self, loop_token):
+ self.metainterp_sd.profiler.start_running()
+ debug_start("jit-running")
+ self.memory_manager.enter_loop(loop_token)
+ fail_descr = self.cpu.execute_token(loop_token)
+ self.memory_manager.leave_loop(loop_token)
+ debug_stop("jit-running")
+ self.metainterp_sd.profiler.end_running()
+ return fail_descr
Modified: pypy/branch/jit-free/pypy/jit/metainterp/warmstate.py
==============================================================================
--- pypy/branch/jit-free/pypy/jit/metainterp/warmstate.py (original)
+++ pypy/branch/jit-free/pypy/jit/metainterp/warmstate.py Mon Nov 15 13:39:54 2010
@@ -214,6 +214,10 @@
if self.profiler is not None:
self.profiler.set_printing(value >= DEBUG_PROFILE)
+ def set_param_loop_longevity(self, value):
+ # note: it's a global parameter, not a per-jitdriver one
+ self.warmrunnerdesc.memory_manager.set_max_age(value)
+
def disable_noninlinable_function(self, greenkey):
cell = self.jit_cell_at_key(greenkey)
cell.dont_trace_here = True
@@ -239,7 +243,8 @@
if hasattr(self, 'maybe_compile_and_run'):
return self.maybe_compile_and_run
- metainterp_sd = self.warmrunnerdesc.metainterp_sd
+ warmrunnerdesc = self.warmrunnerdesc
+ metainterp_sd = warmrunnerdesc.metainterp_sd
jitdriver_sd = self.jitdriver_sd
vinfo = jitdriver_sd.virtualizable_info
index_of_virtualizable = jitdriver_sd.index_of_virtualizable
@@ -304,16 +309,11 @@
# ---------- execute assembler ----------
while True: # until interrupted by an exception
- metainterp_sd.profiler.start_running()
- debug_start("jit-running")
- fail_descr = metainterp_sd.cpu.execute_token(loop_token)
- debug_stop("jit-running")
- metainterp_sd.profiler.end_running()
+ fail_descr = warmrunnerdesc.execute_token(loop_token)
if vinfo is not None:
vinfo.reset_vable_token(virtualizable)
- loop_token = fail_descr.handle_fail(metainterp_sd,
- jitdriver_sd)
-
+ loop_token = fail_descr.handle_fail(metainterp_sd)
+
maybe_compile_and_run._dont_inline_ = True
self.maybe_compile_and_run = maybe_compile_and_run
return maybe_compile_and_run
Modified: pypy/branch/jit-free/pypy/rlib/jit.py
==============================================================================
--- pypy/branch/jit-free/pypy/rlib/jit.py (original)
+++ pypy/branch/jit-free/pypy/rlib/jit.py Mon Nov 15 13:39:54 2010
@@ -271,6 +271,7 @@
'inlining': False,
'optimizer': OPTIMIZER_FULL,
'debug' : DEBUG_STEPS,
+ 'loop_longevity': 1000,
}
unroll_parameters = unrolling_iterable(PARAMETERS.keys())
More information about the Pypy-commit
mailing list