[pypy-commit] pypy concurrent-marksweep: hg merge default
arigo
noreply at buildbot.pypy.org
Sun Jan 1 13:15:34 CET 2012
Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r50971:01950ba0f81a
Date: 2012-01-01 13:15 +0100
http://bitbucket.org/pypy/pypy/changeset/01950ba0f81a/
Log: hg merge default
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -27,7 +27,7 @@
DEALINGS IN THE SOFTWARE.
-PyPy Copyright holders 2003-2011
+PyPy Copyright holders 2003-2012
-----------------------------------
Except when otherwise stated (look for LICENSE files or information at
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -619,7 +619,8 @@
self.descr_reqcls,
args)
except Exception, e:
- raise self.handle_exception(space, e)
+ self.handle_exception(space, e)
+ w_result = None
if w_result is None:
w_result = space.w_None
return w_result
@@ -655,7 +656,8 @@
self.descr_reqcls,
args)
except Exception, e:
- raise self.handle_exception(space, e)
+ self.handle_exception(space, e)
+ w_result = None
if w_result is None:
w_result = space.w_None
return w_result
@@ -674,7 +676,8 @@
self.descr_reqcls,
args.prepend(w_obj))
except Exception, e:
- raise self.handle_exception(space, e)
+ self.handle_exception(space, e)
+ w_result = None
if w_result is None:
w_result = space.w_None
return w_result
@@ -690,7 +693,8 @@
raise OperationError(space.w_SystemError,
space.wrap("unexpected DescrMismatch error"))
except Exception, e:
- raise self.handle_exception(space, e)
+ self.handle_exception(space, e)
+ w_result = None
if w_result is None:
w_result = space.w_None
return w_result
@@ -708,7 +712,8 @@
self.descr_reqcls,
Arguments(space, [w1]))
except Exception, e:
- raise self.handle_exception(space, e)
+ self.handle_exception(space, e)
+ w_result = None
if w_result is None:
w_result = space.w_None
return w_result
@@ -726,7 +731,8 @@
self.descr_reqcls,
Arguments(space, [w1, w2]))
except Exception, e:
- raise self.handle_exception(space, e)
+ self.handle_exception(space, e)
+ w_result = None
if w_result is None:
w_result = space.w_None
return w_result
@@ -744,7 +750,8 @@
self.descr_reqcls,
Arguments(space, [w1, w2, w3]))
except Exception, e:
- raise self.handle_exception(space, e)
+ self.handle_exception(space, e)
+ w_result = None
if w_result is None:
w_result = space.w_None
return w_result
@@ -763,7 +770,8 @@
Arguments(space,
[w1, w2, w3, w4]))
except Exception, e:
- raise self.handle_exception(space, e)
+ self.handle_exception(space, e)
+ w_result = None
if w_result is None:
w_result = space.w_None
return w_result
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -59,7 +59,8 @@
self.is_guard_not_invalidated = is_guard_not_invalidated
DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed),
- ('bridge', lltype.Signed), # 0 or 1
+ ('type', lltype.Char), # 'b'ridge, 'l'abel or
+ # 'e'ntry point
('number', lltype.Signed))
class Assembler386(object):
@@ -150,10 +151,12 @@
debug_start('jit-backend-counts')
for i in range(len(self.loop_run_counters)):
struct = self.loop_run_counters[i]
- if not struct.bridge:
+ if struct.type == 'l':
prefix = 'TargetToken(%d)' % struct.number
+ elif struct.type == 'b':
+ prefix = 'bridge ' + str(struct.number)
else:
- prefix = 'bridge ' + str(struct.number)
+ prefix = 'entry ' + str(struct.number)
debug_print(prefix + ':' + str(struct.i))
debug_stop('jit-backend-counts')
@@ -425,7 +428,7 @@
self.setup(looptoken)
if log:
operations = self._inject_debugging_code(looptoken, operations,
- False, looptoken.number)
+ 'e', looptoken.number)
regalloc = RegAlloc(self, self.cpu.translate_support_code)
#
@@ -492,7 +495,7 @@
self.setup(original_loop_token)
if log:
operations = self._inject_debugging_code(faildescr, operations,
- True, descr_number)
+ 'b', descr_number)
arglocs = self.rebuild_faillocs_from_descr(failure_recovery)
if not we_are_translated():
@@ -599,15 +602,15 @@
return self.mc.materialize(self.cpu.asmmemmgr, allblocks,
self.cpu.gc_ll_descr.gcrootmap)
- def _register_counter(self, bridge, number, token):
+ def _register_counter(self, tp, number, token):
# YYY very minor leak -- we need the counters to stay alive
# forever, just because we want to report them at the end
# of the process
struct = lltype.malloc(DEBUG_COUNTER, flavor='raw',
track_allocation=False)
struct.i = 0
- struct.bridge = int(bridge)
- if bridge:
+ struct.type = tp
+ if tp == 'b' or tp == 'e':
struct.number = number
else:
assert token
@@ -657,8 +660,8 @@
targettoken._x86_loop_code += rawstart
self.target_tokens_currently_compiling = None
- def _append_debugging_code(self, operations, bridge, number, token):
- counter = self._register_counter(bridge, number, token)
+ def _append_debugging_code(self, operations, tp, number, token):
+ counter = self._register_counter(tp, number, token)
c_adr = ConstInt(rffi.cast(lltype.Signed, counter))
box = BoxInt()
box2 = BoxInt()
@@ -670,7 +673,7 @@
operations.extend(ops)
@specialize.argtype(1)
- def _inject_debugging_code(self, looptoken, operations, bridge, number):
+ def _inject_debugging_code(self, looptoken, operations, tp, number):
if self._debug:
# before doing anything, let's increase a counter
s = 0
@@ -679,13 +682,12 @@
looptoken._x86_debug_checksum = s
newoperations = []
- if bridge:
- self._append_debugging_code(newoperations, bridge, number,
- None)
+ self._append_debugging_code(newoperations, tp, number,
+ None)
for op in operations:
newoperations.append(op)
if op.getopnum() == rop.LABEL:
- self._append_debugging_code(newoperations, bridge, number,
+ self._append_debugging_code(newoperations, 'l', number,
op.getdescr())
operations = newoperations
return operations
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -546,13 +546,16 @@
struct = self.cpu.assembler.loop_run_counters[0]
assert struct.i == 1
struct = self.cpu.assembler.loop_run_counters[1]
+ assert struct.i == 1
+ struct = self.cpu.assembler.loop_run_counters[2]
assert struct.i == 9
self.cpu.finish_once()
finally:
debug._log = None
+ l0 = ('debug_print', 'entry -1:1')
l1 = ('debug_print', preambletoken.repr_of_descr() + ':1')
l2 = ('debug_print', targettoken.repr_of_descr() + ':9')
- assert ('jit-backend-counts', [l1, l2]) in dlog
+ assert ('jit-backend-counts', [l0, l1, l2]) in dlog
def test_debugger_checksum(self):
loop = """
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -162,7 +162,6 @@
_ll_4_list_setslice = rlist.ll_listsetslice
_ll_2_list_delslice_startonly = rlist.ll_listdelslice_startonly
_ll_3_list_delslice_startstop = rlist.ll_listdelslice_startstop
-_ll_1_list_list2fixed = lltypesystem_rlist.ll_list2fixed
_ll_2_list_inplace_mul = rlist.ll_inplace_mul
_ll_2_list_getitem_foldable = _ll_2_list_getitem
diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py
--- a/pypy/jit/metainterp/compile.py
+++ b/pypy/jit/metainterp/compile.py
@@ -112,33 +112,26 @@
"""
from pypy.jit.metainterp.optimizeopt import optimize_trace
- history = metainterp.history
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
+ history = metainterp.history
- if False:
- part = partial_trace
- assert False
- procedur_token = metainterp.get_procedure_token(greenkey)
- assert procedure_token
- all_target_tokens = []
- else:
- jitcell_token = make_jitcell_token(jitdriver_sd)
- part = create_empty_loop(metainterp)
- part.inputargs = inputargs[:]
- h_ops = history.operations
- part.resume_at_jump_descr = resume_at_jump_descr
- part.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(jitcell_token))] + \
- [h_ops[i].clone() for i in range(start, len(h_ops))] + \
- [ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)]
+ jitcell_token = make_jitcell_token(jitdriver_sd)
+ part = create_empty_loop(metainterp)
+ part.inputargs = inputargs[:]
+ h_ops = history.operations
+ part.resume_at_jump_descr = resume_at_jump_descr
+ part.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(jitcell_token))] + \
+ [h_ops[i].clone() for i in range(start, len(h_ops))] + \
+ [ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)]
- try:
- optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts)
- except InvalidLoop:
- return None
- target_token = part.operations[0].getdescr()
- assert isinstance(target_token, TargetToken)
- all_target_tokens = [target_token]
+ try:
+ optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts)
+ except InvalidLoop:
+ return None
+ target_token = part.operations[0].getdescr()
+ assert isinstance(target_token, TargetToken)
+ all_target_tokens = [target_token]
loop = create_empty_loop(metainterp)
loop.inputargs = part.inputargs
@@ -176,10 +169,10 @@
loop.original_jitcell_token = jitcell_token
for label in all_target_tokens:
assert isinstance(label, TargetToken)
- label.original_jitcell_token = jitcell_token
if label.virtual_state and label.short_preamble:
metainterp_sd.logger_ops.log_short_preamble([], label.short_preamble)
jitcell_token.target_tokens = all_target_tokens
+ propagate_original_jitcell_token(loop)
send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop")
record_loop_or_bridge(metainterp_sd, loop)
return all_target_tokens[0]
@@ -247,11 +240,11 @@
for box in loop.inputargs:
assert isinstance(box, Box)
- target_token = loop.operations[-1].getdescr()
+ target_token = loop.operations[-1].getdescr()
resumekey.compile_and_attach(metainterp, loop)
+
target_token = label.getdescr()
assert isinstance(target_token, TargetToken)
- target_token.original_jitcell_token = loop.original_jitcell_token
record_loop_or_bridge(metainterp_sd, loop)
return target_token
@@ -288,6 +281,15 @@
assert i == len(inputargs)
loop.operations = extra_ops + loop.operations
+def propagate_original_jitcell_token(trace):
+ for op in trace.operations:
+ if op.getopnum() == rop.LABEL:
+ token = op.getdescr()
+ assert isinstance(token, TargetToken)
+ assert token.original_jitcell_token is None
+ token.original_jitcell_token = trace.original_jitcell_token
+
+
def send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, type):
vinfo = jitdriver_sd.virtualizable_info
if vinfo is not None:
@@ -319,7 +321,10 @@
metainterp_sd.stats.compiled()
metainterp_sd.log("compiled new " + type)
#
- metainterp_sd.logger_ops.log_loop(loop.inputargs, loop.operations, n, type, ops_offset)
+ loopname = jitdriver_sd.warmstate.get_location_str(greenkey)
+ metainterp_sd.logger_ops.log_loop(loop.inputargs, loop.operations, n,
+ type, ops_offset,
+ name=loopname)
#
if metainterp_sd.warmrunnerdesc is not None: # for tests
metainterp_sd.warmrunnerdesc.memory_manager.keep_loop_alive(original_jitcell_token)
@@ -558,6 +563,7 @@
inputargs = metainterp.history.inputargs
if not we_are_translated():
self._debug_suboperations = new_loop.operations
+ propagate_original_jitcell_token(new_loop)
send_bridge_to_backend(metainterp.jitdriver_sd, metainterp.staticdata,
self, inputargs, new_loop.operations,
new_loop.original_jitcell_token)
@@ -744,6 +750,7 @@
jitdriver_sd = metainterp.jitdriver_sd
redargs = new_loop.inputargs
new_loop.original_jitcell_token = jitcell_token = make_jitcell_token(jitdriver_sd)
+ propagate_original_jitcell_token(new_loop)
send_loop_to_backend(self.original_greenkey, metainterp.jitdriver_sd,
metainterp_sd, new_loop, "entry bridge")
# send the new_loop to warmspot.py, to be called directly the next time
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -79,9 +79,9 @@
opnum == rop.COPYSTRCONTENT or
opnum == rop.COPYUNICODECONTENT):
return
- if rop._OVF_FIRST <= opnum <= rop._OVF_LAST:
- return
- if rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST:
+ if (rop._OVF_FIRST <= opnum <= rop._OVF_LAST or
+ rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST or
+ rop._GUARD_FIRST <= opnum <= rop._GUARD_LAST):
return
if opnum == rop.CALL or opnum == rop.CALL_LOOPINVARIANT:
effectinfo = descr.get_extra_info()
diff --git a/pypy/jit/metainterp/logger.py b/pypy/jit/metainterp/logger.py
--- a/pypy/jit/metainterp/logger.py
+++ b/pypy/jit/metainterp/logger.py
@@ -13,14 +13,14 @@
self.metainterp_sd = metainterp_sd
self.guard_number = guard_number
- def log_loop(self, inputargs, operations, number=0, type=None, ops_offset=None):
+ def log_loop(self, inputargs, operations, number=0, type=None, ops_offset=None, name=''):
if type is None:
debug_start("jit-log-noopt-loop")
logops = self._log_operations(inputargs, operations, ops_offset)
debug_stop("jit-log-noopt-loop")
else:
debug_start("jit-log-opt-loop")
- debug_print("# Loop", number, ":", type,
+ debug_print("# Loop", number, '(%s)' % name , ":", type,
"with", len(operations), "ops")
logops = self._log_operations(inputargs, operations, ops_offset)
debug_stop("jit-log-opt-loop")
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py b/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py
@@ -386,6 +386,17 @@
"""
self.optimize_loop(ops, expected)
+ def test_virtual_as_field_of_forced_box(self):
+ ops = """
+ [p0]
+ pv1 = new_with_vtable(ConstClass(node_vtable))
+ label(pv1, p0)
+ pv2 = new_with_vtable(ConstClass(node_vtable))
+ setfield_gc(pv2, pv1, descr=valuedescr)
+ jump(pv1, pv2)
+ """
+ with raises(InvalidLoop):
+ self.optimize_loop(ops, ops)
class OptRenameStrlen(Optimization):
def propagate_forward(self, op):
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -409,7 +409,13 @@
if self.level == LEVEL_CONSTANT:
return
assert 0 <= self.position_in_notvirtuals
- boxes[self.position_in_notvirtuals] = value.force_box(optimizer)
+ if optimizer:
+ box = value.force_box(optimizer)
+ else:
+ if value.is_virtual():
+ raise BadVirtualState
+ box = value.get_key_box()
+ boxes[self.position_in_notvirtuals] = box
def _enum(self, virtual_state):
if self.level == LEVEL_CONSTANT:
@@ -471,8 +477,14 @@
optimizer = optimizer.optearlyforce
assert len(values) == len(self.state)
inputargs = [None] * len(self.notvirtuals)
+
+ # We try twice. The first time around we allow boxes to be forced
+ # which might change the virtual state if the box appear in more
+ # than one place among the inputargs.
for i in range(len(values)):
self.state[i].enum_forced_boxes(inputargs, values[i], optimizer)
+ for i in range(len(values)):
+ self.state[i].enum_forced_boxes(inputargs, values[i], None)
if keyboxes:
for i in range(len(values)):
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -976,10 +976,13 @@
self.verify_green_args(jitdriver_sd, greenboxes)
self.debug_merge_point(jitdriver_sd, jdindex, self.metainterp.in_recursion,
greenboxes)
-
+
if self.metainterp.seen_loop_header_for_jdindex < 0:
- if not jitdriver_sd.no_loop_header or not any_operation:
+ if not any_operation:
return
+ if self.metainterp.in_recursion or not self.metainterp.get_procedure_token(greenboxes, True):
+ if not jitdriver_sd.no_loop_header:
+ return
# automatically add a loop_header if there is none
self.metainterp.seen_loop_header_for_jdindex = jdindex
#
@@ -2053,9 +2056,15 @@
from pypy.jit.metainterp.resoperation import opname
raise NotImplementedError(opname[opnum])
- def get_procedure_token(self, greenkey):
+ def get_procedure_token(self, greenkey, with_compiled_targets=False):
cell = self.jitdriver_sd.warmstate.jit_cell_at_key(greenkey)
- return cell.get_procedure_token()
+ token = cell.get_procedure_token()
+ if with_compiled_targets:
+ if not token:
+ return None
+ if not token.target_tokens:
+ return None
+ return token
def compile_loop(self, original_boxes, live_arg_boxes, start, resume_at_jump_descr):
num_green_args = self.jitdriver_sd.num_green_args
@@ -2088,11 +2097,9 @@
def compile_trace(self, live_arg_boxes, resume_at_jump_descr):
num_green_args = self.jitdriver_sd.num_green_args
greenkey = live_arg_boxes[:num_green_args]
- target_jitcell_token = self.get_procedure_token(greenkey)
+ target_jitcell_token = self.get_procedure_token(greenkey, True)
if not target_jitcell_token:
return
- if not target_jitcell_token.target_tokens:
- return
self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None,
descr=target_jitcell_token)
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -2697,7 +2697,7 @@
# bridge back to the preamble of the first loop is produced. A guard in
# this bridge is later traced resulting in a failed attempt of retracing
# the second loop.
- self.check_trace_count(8)
+ self.check_trace_count(9)
# FIXME: Add a gloabl retrace counter and test that we are not trying more than 5 times.
diff --git a/pypy/jit/metainterp/test/test_compile.py b/pypy/jit/metainterp/test/test_compile.py
--- a/pypy/jit/metainterp/test/test_compile.py
+++ b/pypy/jit/metainterp/test/test_compile.py
@@ -18,7 +18,7 @@
self.seen.append((inputargs, operations, token))
class FakeLogger(object):
- def log_loop(self, inputargs, operations, number=0, type=None, ops_offset=None):
+ def log_loop(self, inputargs, operations, number=0, type=None, ops_offset=None, name=''):
pass
def repr_of_resop(self, op):
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -255,6 +255,11 @@
assert h.getarrayitem(box1, descr1, index1) is box2
assert h.getarrayitem(box1, descr1, index2) is box4
+ h.invalidate_caches(rop.GUARD_TRUE, None, [])
+ assert h.getfield(box1, descr1) is box2
+ assert h.getarrayitem(box1, descr1, index1) is box2
+ assert h.getarrayitem(box1, descr1, index2) is box4
+
h.invalidate_caches(
rop.CALL_LOOPINVARIANT, FakeCallDescr(FakeEffektinfo.EF_LOOPINVARIANT), [])
diff --git a/pypy/jit/metainterp/test/test_logger.py b/pypy/jit/metainterp/test/test_logger.py
--- a/pypy/jit/metainterp/test/test_logger.py
+++ b/pypy/jit/metainterp/test/test_logger.py
@@ -180,7 +180,7 @@
def test_intro_loop(self):
bare_logger = logger.Logger(self.make_metainterp_sd())
output = capturing(bare_logger.log_loop, [], [], 1, "foo")
- assert output.splitlines()[0] == "# Loop 1 : foo with 0 ops"
+ assert output.splitlines()[0] == "# Loop 1 () : foo with 0 ops"
pure_parse(output)
def test_intro_bridge(self):
diff --git a/pypy/jit/metainterp/test/test_loop.py b/pypy/jit/metainterp/test/test_loop.py
--- a/pypy/jit/metainterp/test/test_loop.py
+++ b/pypy/jit/metainterp/test/test_loop.py
@@ -756,7 +756,7 @@
res = self.meta_interp(interpret, [1])
assert res == interpret(1)
# XXX it's unsure how many loops should be there
- self.check_trace_count(3)
+ self.check_trace_count(2)
def test_path_with_operations_not_from_start(self):
jitdriver = JitDriver(greens = ['k'], reds = ['n', 'z'])
diff --git a/pypy/jit/metainterp/test/test_virtual.py b/pypy/jit/metainterp/test/test_virtual.py
--- a/pypy/jit/metainterp/test/test_virtual.py
+++ b/pypy/jit/metainterp/test/test_virtual.py
@@ -880,7 +880,7 @@
elif op == 'j':
j = Int(0)
elif op == '+':
- sa += i.val * j.val
+ sa += (i.val + 2) * (j.val + 2)
elif op == 'a':
i = Int(i.val + 1)
elif op == 'b':
@@ -902,6 +902,7 @@
assert res == f(10)
self.check_aborted_count(0)
self.check_target_token_count(3)
+ self.check_resops(int_mul=2)
def test_nested_loops_bridge(self):
class Int(object):
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -4,6 +4,7 @@
class PyPyModule(MixedModule):
interpleveldefs = {
'debug_repr': 'interp_extras.debug_repr',
+ 'remove_invalidates': 'interp_extras.remove_invalidates',
}
appleveldefs = {}
diff --git a/pypy/module/micronumpy/interp_extras.py b/pypy/module/micronumpy/interp_extras.py
--- a/pypy/module/micronumpy/interp_extras.py
+++ b/pypy/module/micronumpy/interp_extras.py
@@ -5,3 +5,11 @@
@unwrap_spec(array=BaseArray)
def debug_repr(space, array):
return space.wrap(array.find_sig().debug_repr())
+
+ at unwrap_spec(array=BaseArray)
+def remove_invalidates(space, array):
+ """ Array modification will no longer invalidate any of it's
+ potential children. Use only for performance debugging
+ """
+ del array.invalidates[:]
+ return space.w_None
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1,6 +1,6 @@
from pypy.interpreter.baseobjspace import Wrappable
from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped
+from pypy.interpreter.gateway import interp2app, NoneNotWrapped
from pypy.interpreter.typedef import TypeDef, GetSetProperty
from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature
from pypy.module.micronumpy.strides import calculate_slice_strides
@@ -14,22 +14,26 @@
numpy_driver = jit.JitDriver(
greens=['shapelen', 'sig'],
virtualizables=['frame'],
- reds=['result_size', 'frame', 'ri', 'self', 'result']
+ reds=['result_size', 'frame', 'ri', 'self', 'result'],
+ get_printable_location=signature.new_printable_location('numpy'),
)
all_driver = jit.JitDriver(
greens=['shapelen', 'sig'],
virtualizables=['frame'],
- reds=['frame', 'self', 'dtype']
+ reds=['frame', 'self', 'dtype'],
+ get_printable_location=signature.new_printable_location('all'),
)
any_driver = jit.JitDriver(
greens=['shapelen', 'sig'],
virtualizables=['frame'],
- reds=['frame', 'self', 'dtype']
+ reds=['frame', 'self', 'dtype'],
+ get_printable_location=signature.new_printable_location('any'),
)
slice_driver = jit.JitDriver(
greens=['shapelen', 'sig'],
virtualizables=['frame'],
- reds=['self', 'frame', 'source', 'res_iter']
+ reds=['self', 'frame', 'source', 'res_iter'],
+ get_printable_location=signature.new_printable_location('slice'),
)
def _find_shape_and_elems(space, w_iterable):
@@ -291,7 +295,8 @@
def _reduce_argmax_argmin_impl(op_name):
reduce_driver = jit.JitDriver(
greens=['shapelen', 'sig'],
- reds=['result', 'idx', 'frame', 'self', 'cur_best', 'dtype']
+ reds=['result', 'idx', 'frame', 'self', 'cur_best', 'dtype'],
+ get_printable_location=signature.new_printable_location(op_name),
)
def loop(self):
sig = self.find_sig()
@@ -578,8 +583,8 @@
strides.append(concrete.strides[i])
backstrides.append(concrete.backstrides[i])
shape.append(concrete.shape[i])
- return space.wrap(W_NDimSlice(concrete.start, strides[:],
- backstrides[:], shape[:], concrete))
+ return space.wrap(W_NDimSlice(concrete.start, strides,
+ backstrides, shape, concrete))
def descr_get_flatiter(self, space):
return space.wrap(W_FlatIterator(self))
@@ -820,8 +825,8 @@
if self.order == 'C':
strides.reverse()
backstrides.reverse()
- self.strides = strides[:]
- self.backstrides = backstrides[:]
+ self.strides = strides
+ self.backstrides = backstrides
def array_sig(self, res_shape):
if res_shape is not None and self.shape != res_shape:
@@ -1025,9 +1030,9 @@
strides.reverse()
backstrides.reverse()
new_shape.reverse()
- self.strides = strides[:]
- self.backstrides = backstrides[:]
- self.shape = new_shape[:]
+ self.strides = strides
+ self.backstrides = backstrides
+ self.shape = new_shape
return
new_strides = calc_new_strides(new_shape, self.shape, self.strides)
if new_strides is None:
@@ -1037,7 +1042,7 @@
for nd in range(len(new_shape)):
new_backstrides[nd] = (new_shape[nd] - 1) * new_strides[nd]
self.strides = new_strides[:]
- self.backstrides = new_backstrides[:]
+ self.backstrides = new_backstrides
self.shape = new_shape[:]
class W_NDimArray(ConcreteArray):
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -1,9 +1,10 @@
from pypy.interpreter.baseobjspace import Wrappable
from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.gateway import interp2app
from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
-from pypy.module.micronumpy import interp_boxes, interp_dtype, types
-from pypy.module.micronumpy.signature import ReduceSignature, ScalarSignature, find_sig
+from pypy.module.micronumpy import interp_boxes, interp_dtype
+from pypy.module.micronumpy.signature import ReduceSignature, ScalarSignature,\
+ find_sig, new_printable_location
from pypy.rlib import jit
from pypy.rlib.rarithmetic import LONG_BIT
from pypy.tool.sourcetools import func_with_new_name
@@ -11,7 +12,8 @@
reduce_driver = jit.JitDriver(
greens = ['shapelen', "sig"],
virtualizables = ["frame"],
- reds = ["frame", "self", "dtype", "value", "obj"]
+ reds = ["frame", "self", "dtype", "value", "obj"],
+ get_printable_location=new_printable_location('reduce'),
)
class W_Ufunc(Wrappable):
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -5,6 +5,11 @@
from pypy.module.micronumpy.strides import calculate_slice_strides
from pypy.rlib.jit import hint, unroll_safe, promote
+def new_printable_location(driver_name):
+ def get_printable_location(shapelen, sig):
+ return 'numpy ' + sig.debug_repr() + ' [%d dims,%s]' % (shapelen, driver_name)
+ return get_printable_location
+
def sigeq(one, two):
return one.eq(two)
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -1,4 +1,9 @@
+from pypy.rlib import jit
+
+ at jit.look_inside_iff(lambda shape, start, strides, backstrides, chunks:
+ jit.isconstant(len(chunks))
+)
def calculate_slice_strides(shape, start, strides, backstrides, chunks):
rstrides = []
rbackstrides = []
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -898,6 +898,15 @@
b[0] = 3
assert debug_repr(b) == 'Array'
+ def test_remove_invalidates(self):
+ from numpypy import array
+ from numpypy.pypy import remove_invalidates
+ a = array([1, 2, 3])
+ b = a + a
+ remove_invalidates(a)
+ a[0] = 14
+ assert b[0] == 28
+
def test_virtual_views(self):
from numpypy import arange
a = arange(15)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -537,7 +537,7 @@
builder.append(by)
builder.append_slice(input, upper, len(input))
else:
- # An ok guess for the result size
+ # First compute the exact result size
count = input.count(sub)
if count > maxsplit and maxsplit > 0:
count = maxsplit
@@ -553,21 +553,16 @@
builder = StringBuilder(result_size)
start = 0
sublen = len(sub)
- first = True
while maxsplit != 0:
next = input.find(sub, start)
if next < 0:
break
- if not first:
- builder.append(by)
- first = False
builder.append_slice(input, start, next)
+ builder.append(by)
start = next + sublen
maxsplit -= 1 # NB. if it's already < 0, it stays < 0
- if not first:
- builder.append(by)
builder.append_slice(input, start, len(input))
return space.wrap(builder.build())
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -528,6 +528,9 @@
set_param(driver, name1, int(value))
except ValueError:
raise
+ break
+ else:
+ raise ValueError
set_user_param._annspecialcase_ = 'specialize:arg(0)'
diff --git a/pypy/rlib/rsre/rsre_jit.py b/pypy/rlib/rsre/rsre_jit.py
--- a/pypy/rlib/rsre/rsre_jit.py
+++ b/pypy/rlib/rsre/rsre_jit.py
@@ -22,7 +22,7 @@
info = '%s/%d' % (info, args[debugprint[2]])
else:
info = ''
- return '%s%s %s' % (name, info, s)
+ return 're %s%s %s' % (name, info, s)
#
self.get_printable_location = get_printable_location
diff --git a/pypy/rpython/lltypesystem/rlist.py b/pypy/rpython/lltypesystem/rlist.py
--- a/pypy/rpython/lltypesystem/rlist.py
+++ b/pypy/rpython/lltypesystem/rlist.py
@@ -375,7 +375,6 @@
newitems = malloc(LIST.items.TO, n)
rgc.ll_arraycopy(olditems, newitems, 0, 0, n)
return newitems
-ll_list2fixed.oopspec = 'list.list2fixed(l)'
def ll_list2fixed_exact(l):
ll_assert(l.length == len(l.items), "ll_list2fixed_exact: bad length")
diff --git a/pypy/rpython/test/test_generator.py b/pypy/rpython/test/test_generator.py
--- a/pypy/rpython/test/test_generator.py
+++ b/pypy/rpython/test/test_generator.py
@@ -54,6 +54,26 @@
res = self.interpret(f, [0])
assert res == 42
+ def test_except_block(self):
+ def foo():
+ raise ValueError
+ def g(a, b, c):
+ yield a
+ yield b
+ try:
+ foo()
+ except ValueError:
+ pass
+ yield c
+ def f():
+ gen = g(3, 5, 8)
+ x = gen.next() * 100
+ x += gen.next() * 10
+ x += gen.next()
+ return x
+ res = self.interpret(f, [])
+ assert res == 358
+
class TestLLtype(BaseTestGenerator, LLRtypeMixin):
pass
diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py
--- a/pypy/tool/jitlogparser/parser.py
+++ b/pypy/tool/jitlogparser/parser.py
@@ -24,27 +24,24 @@
self.failargs = failargs
def getarg(self, i):
- return self._getvar(self.args[i])
+ return self.args[i]
def getargs(self):
- return [self._getvar(v) for v in self.args]
+ return self.args[:]
def getres(self):
- return self._getvar(self.res)
+ return self.res
def getdescr(self):
return self.descr
- def _getvar(self, v):
- return v
-
def is_guard(self):
return self._is_guard
def repr(self):
args = self.getargs()
if self.descr is not None:
- args.append('descr=%s' % self.getdescr())
+ args.append('descr=%s' % self.descr)
arglist = ', '.join(args)
if self.res is not None:
return '%s = %s(%s)' % (self.getres(), self.name, arglist)
@@ -53,8 +50,6 @@
def __repr__(self):
return self.repr()
- ## return '<%s (%s)>' % (self.name, ', '.join([repr(a)
- ## for a in self.args]))
class SimpleParser(OpParser):
@@ -146,18 +141,27 @@
is_bytecode = True
inline_level = None
- def __init__(self, operations, storage):
- if operations[0].name == 'debug_merge_point':
- self.inline_level = int(operations[0].args[0])
- m = re.search('<code object ([<>\w]+)\. file \'(.+?)\'\. line (\d+)> #(\d+) (\w+)',
- operations[0].args[1])
- if m is None:
- # a non-code loop, like StrLiteralSearch or something
- self.bytecode_name = operations[0].args[1][1:-1]
- else:
- self.name, self.filename, lineno, bytecode_no, self.bytecode_name = m.groups()
- self.startlineno = int(lineno)
- self.bytecode_no = int(bytecode_no)
+ def parse_code_data(self, arg):
+ m = re.search('<code object ([<>\w]+)[\.,] file \'(.+?)\'[\.,] line (\d+)> #(\d+) (\w+)',
+ arg)
+ if m is None:
+ # a non-code loop, like StrLiteralSearch or something
+ self.bytecode_name = arg
+ else:
+ self.name, self.filename, lineno, bytecode_no, self.bytecode_name = m.groups()
+ self.startlineno = int(lineno)
+ self.bytecode_no = int(bytecode_no)
+
+
+ def __init__(self, operations, storage, loopname):
+ for op in operations:
+ if op.name == 'debug_merge_point':
+ self.inline_level = int(op.args[0])
+ self.parse_code_data(op.args[1][1:-1])
+ break
+ else:
+ self.inline_level = 0
+ self.parse_code_data(loopname)
self.operations = operations
self.storage = storage
self.code = storage.disassemble_code(self.filename, self.startlineno,
@@ -165,7 +169,7 @@
def repr(self):
if self.filename is None:
- return "Unknown"
+ return self.bytecode_name
return "%s, file '%s', line %d" % (self.name, self.filename,
self.startlineno)
@@ -220,7 +224,8 @@
self.storage = storage
@classmethod
- def from_operations(cls, operations, storage, limit=None, inputargs=''):
+ def from_operations(cls, operations, storage, limit=None, inputargs='',
+ loopname=''):
""" Slice given operation list into a chain of TraceForOpcode chunks.
Also detect inlined functions and make them Function
"""
@@ -246,13 +251,13 @@
for op in operations:
if op.name == 'debug_merge_point':
if so_far:
- append_to_res(cls.TraceForOpcode(so_far, storage))
+ append_to_res(cls.TraceForOpcode(so_far, storage, loopname))
if limit:
break
so_far = []
so_far.append(op)
if so_far:
- append_to_res(cls.TraceForOpcode(so_far, storage))
+ append_to_res(cls.TraceForOpcode(so_far, storage, loopname))
# wrap stack back up
if not stack:
# no ops whatsoever
@@ -300,7 +305,7 @@
def repr(self):
if self.filename is None:
- return "Unknown"
+ return self.chunks[0].bytecode_name
return "%s, file '%s', line %d" % (self.name, self.filename,
self.startlineno)
@@ -385,18 +390,27 @@
parser.postprocess(loop, backend_tp=bname,
backend_dump=dump,
dump_start=start_ofs))
- loops.append(loop)
+ loops += split_trace(loop)
return log, loops
def split_trace(trace):
- labels = [i for i, op in enumerate(trace.operations)
- if op.name == 'label']
- labels = [0] + labels + [len(trace.operations) - 1]
+ labels = [0]
+ if trace.comment and 'Guard' in trace.comment:
+ descrs = ['bridge ' + re.search('Guard (\d+)', trace.comment).group(1)]
+ else:
+ descrs = ['entry ' + re.search('Loop (\d+)', trace.comment).group(1)]
+ for i, op in enumerate(trace.operations):
+ if op.name == 'label':
+ labels.append(i)
+ descrs.append(op.descr)
+ labels.append(len(trace.operations) - 1)
parts = []
for i in range(len(labels) - 1):
start, stop = labels[i], labels[i+1]
part = copy(trace)
part.operations = trace.operations[start : stop + 1]
+ part.descr = descrs[i]
+ part.comment = trace.comment
parts.append(part)
return parts
@@ -407,11 +421,7 @@
lines = input[-1].splitlines()
mapping = {}
for loop in loops:
- com = loop.comment
- if 'Loop' in com:
- mapping['loop ' + re.search('Loop (\d+)', com).group(1)] = loop
- else:
- mapping['bridge ' + re.search('Guard (\d+)', com).group(1)] = loop
+ mapping[loop.descr] = loop
for line in lines:
if line:
num, count = line.split(':', 2)
diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py
--- a/pypy/tool/jitlogparser/test/test_parser.py
+++ b/pypy/tool/jitlogparser/test/test_parser.py
@@ -1,6 +1,7 @@
from pypy.tool.jitlogparser.parser import (SimpleParser, TraceForOpcode,
Function, adjust_bridges,
- import_log, split_trace, Op)
+ import_log, split_trace, Op,
+ parse_log_counts)
from pypy.tool.jitlogparser.storage import LoopStorage
import py, sys
@@ -32,23 +33,26 @@
''')
res = Function.from_operations(ops.operations, LoopStorage())
assert len(res.chunks) == 1
- assert res.chunks[0].repr()
+ assert 'SomeRandomStuff' in res.chunks[0].repr()
def test_split():
ops = parse('''
[i0]
+ label()
debug_merge_point(0, "<code object stuff. file '/I/dont/exist.py'. line 200> #10 ADD")
debug_merge_point(0, "<code object stuff. file '/I/dont/exist.py'. line 200> #11 SUB")
i1 = int_add(i0, 1)
debug_merge_point(0, "<code object stuff. file '/I/dont/exist.py'. line 200> #11 SUB")
i2 = int_add(i1, 1)
''')
- res = Function.from_operations(ops.operations, LoopStorage())
- assert len(res.chunks) == 3
+ res = Function.from_operations(ops.operations, LoopStorage(), loopname='<loopname>')
+ assert len(res.chunks) == 4
assert len(res.chunks[0].operations) == 1
- assert len(res.chunks[1].operations) == 2
+ assert len(res.chunks[1].operations) == 1
assert len(res.chunks[2].operations) == 2
- assert res.chunks[2].bytecode_no == 11
+ assert len(res.chunks[3].operations) == 2
+ assert res.chunks[3].bytecode_no == 11
+ assert res.chunks[0].bytecode_name == '<loopname>'
def test_inlined_call():
ops = parse("""
@@ -236,16 +240,46 @@
loop = parse('''
[i7]
i9 = int_lt(i7, 1003)
- label(i9)
+ label(i9, descr=grrr)
guard_true(i9, descr=<Guard2>) []
i13 = getfield_raw(151937600, descr=<SignedFieldDescr pypysig_long_struct.c_value 0>)
- label(i13)
+ label(i13, descr=asb)
i19 = int_lt(i13, 1003)
guard_true(i19, descr=<Guard2>) []
i113 = getfield_raw(151937600, descr=<SignedFieldDescr pypysig_long_struct.c_value 0>)
''')
+ loop.comment = 'Loop 0'
parts = split_trace(loop)
assert len(parts) == 3
assert len(parts[0].operations) == 2
assert len(parts[1].operations) == 4
assert len(parts[2].operations) == 4
+ assert parts[1].descr == 'grrr'
+ assert parts[2].descr == 'asb'
+
+def test_parse_log_counts():
+ loop = parse('''
+ [i7]
+ i9 = int_lt(i7, 1003)
+ label(i9, descr=grrr)
+ guard_true(i9, descr=<Guard2>) []
+ i13 = getfield_raw(151937600, descr=<SignedFieldDescr pypysig_long_struct.c_value 0>)
+ label(i13, descr=asb)
+ i19 = int_lt(i13, 1003)
+ guard_true(i19, descr=<Guard3>) []
+ i113 = getfield_raw(151937600, descr=<SignedFieldDescr pypysig_long_struct.c_value 0>)
+ ''')
+ bridge = parse('''
+ # bridge out of Guard 2 with 1 ops
+ []
+ i0 = int_lt(1, 2)
+ finish(i0)
+ ''')
+ bridge.comment = 'bridge out of Guard 2 with 1 ops'
+ loop.comment = 'Loop 0'
+ loops = split_trace(loop) + split_trace(bridge)
+ input = ['grrr:123\nasb:12\nbridge 2:1234']
+ parse_log_counts(input, loops)
+ assert loops[-1].count == 1234
+ assert loops[1].count == 123
+ assert loops[2].count == 12
diff --git a/pypy/translator/generator.py b/pypy/translator/generator.py
--- a/pypy/translator/generator.py
+++ b/pypy/translator/generator.py
@@ -2,7 +2,7 @@
from pypy.objspace.flow.model import Variable, Constant, FunctionGraph
from pypy.translator.unsimplify import insert_empty_startblock
from pypy.translator.unsimplify import split_block
-from pypy.translator.simplify import eliminate_empty_blocks
+from pypy.translator.simplify import eliminate_empty_blocks, simplify_graph
from pypy.tool.sourcetools import func_with_new_name
from pypy.interpreter.argument import Signature
@@ -64,6 +64,7 @@
def next(self):
entry = self.current
self.current = None
+ assert entry is not None # else, recursive generator invocation
(next_entry, return_value) = func(entry)
self.current = next_entry
return return_value
@@ -91,6 +92,10 @@
block.inputargs = [v_entry1]
def tweak_generator_body_graph(Entry, graph):
+ # First, always run simplify_graph in order to reduce the number of
+ # variables passed around
+ simplify_graph(graph)
+ #
assert graph.startblock.operations[0].opname == 'generator_mark'
graph.startblock.operations.pop(0)
#
@@ -100,12 +105,20 @@
#
mappings = [Entry]
#
+ stopblock = Block([])
+ v0 = Variable(); v1 = Variable()
+ stopblock.operations = [
+ SpaceOperation('simple_call', [Constant(StopIteration)], v0),
+ SpaceOperation('type', [v0], v1),
+ ]
+ stopblock.closeblock(Link([v1, v0], graph.exceptblock))
+ #
for block in list(graph.iterblocks()):
for exit in block.exits:
if exit.target is graph.returnblock:
- exit.args = [Constant(StopIteration),
- Constant(StopIteration())]
- exit.target = graph.exceptblock
+ exit.args = []
+ exit.target = stopblock
+ assert block is not stopblock
for index in range(len(block.operations)-1, -1, -1):
op = block.operations[index]
if op.opname == 'yield':
diff --git a/pypy/translator/sandbox/pypy_interact.py b/pypy/translator/sandbox/pypy_interact.py
--- a/pypy/translator/sandbox/pypy_interact.py
+++ b/pypy/translator/sandbox/pypy_interact.py
@@ -13,7 +13,8 @@
ATM this only works with PyPy translated with Boehm or
the semispace or generation GCs.
--timeout=N limit execution time to N (real-time) seconds.
- --log=FILE log all user input into the FILE
+ --log=FILE log all user input into the FILE.
+ --verbose log all proxied system calls.
Note that you can get readline-like behavior with a tool like 'ledit',
provided you use enough -u options:
@@ -26,18 +27,19 @@
from pypy.translator.sandbox.sandlib import SimpleIOSandboxedProc
from pypy.translator.sandbox.sandlib import VirtualizedSandboxedProc
from pypy.translator.sandbox.vfs import Dir, RealDir, RealFile
-from pypy.tool.lib_pypy import LIB_ROOT
+import pypy
+LIB_ROOT = os.path.dirname(os.path.dirname(pypy.__file__))
class PyPySandboxedProc(VirtualizedSandboxedProc, SimpleIOSandboxedProc):
- debug = True
argv0 = '/bin/pypy-c'
virtual_cwd = '/tmp'
virtual_env = {}
virtual_console_isatty = True
- def __init__(self, executable, arguments, tmpdir=None):
+ def __init__(self, executable, arguments, tmpdir=None, debug=True):
self.executable = executable = os.path.abspath(executable)
self.tmpdir = tmpdir
+ self.debug = debug
super(PyPySandboxedProc, self).__init__([self.argv0] + arguments,
executable=executable)
@@ -67,12 +69,13 @@
if __name__ == '__main__':
from getopt import getopt # and not gnu_getopt!
- options, arguments = getopt(sys.argv[1:], 't:h',
+ options, arguments = getopt(sys.argv[1:], 't:hv',
['tmp=', 'heapsize=', 'timeout=', 'log=',
- 'help'])
+ 'verbose', 'help'])
tmpdir = None
timeout = None
logfile = None
+ debug = False
extraoptions = []
def help():
@@ -104,6 +107,8 @@
timeout = int(value)
elif option == '--log':
logfile = value
+ elif option in ['-v', '--verbose']:
+ debug = True
elif option in ['-h', '--help']:
help()
else:
@@ -113,7 +118,7 @@
help()
sandproc = PyPySandboxedProc(arguments[0], extraoptions + arguments[1:],
- tmpdir=tmpdir)
+ tmpdir=tmpdir, debug=debug)
if timeout is not None:
sandproc.settimeout(timeout, interrupt_main=True)
if logfile is not None:
diff --git a/pypy/translator/sandbox/sandlib.py b/pypy/translator/sandbox/sandlib.py
--- a/pypy/translator/sandbox/sandlib.py
+++ b/pypy/translator/sandbox/sandlib.py
@@ -4,25 +4,29 @@
for the outer process, which can run CPython or PyPy.
"""
-import py
import sys, os, posixpath, errno, stat, time
-from pypy.tool.ansi_print import AnsiLog
import subprocess
from pypy.tool.killsubprocess import killsubprocess
from pypy.translator.sandbox.vfs import UID, GID
+import py
-class MyAnsiLog(AnsiLog):
- KW_TO_COLOR = {
- 'call': ((34,), False),
- 'result': ((34,), False),
- 'exception': ((34,), False),
- 'vpath': ((35,), False),
- 'timeout': ((1, 31), True),
- }
+def create_log():
+ """Make and return a log for the sandbox to use, if needed."""
+ # These imports are local to avoid importing pypy if we don't need to.
+ from pypy.tool.ansi_print import AnsiLog
-log = py.log.Producer("sandlib")
-py.log.setconsumer("sandlib", MyAnsiLog())
+ class MyAnsiLog(AnsiLog):
+ KW_TO_COLOR = {
+ 'call': ((34,), False),
+ 'result': ((34,), False),
+ 'exception': ((34,), False),
+ 'vpath': ((35,), False),
+ 'timeout': ((1, 31), True),
+ }
+ log = py.log.Producer("sandlib")
+ py.log.setconsumer("sandlib", MyAnsiLog())
+ return log
# Note: we use lib_pypy/marshal.py instead of the built-in marshal
# for two reasons. The built-in module could be made to segfault
@@ -30,8 +34,9 @@
# load(). Also, marshal.load(f) blocks with the GIL held when
# f is a pipe with no data immediately avaialble, preventing the
# _waiting_thread to run.
-from pypy.tool.lib_pypy import import_from_lib_pypy
-marshal = import_from_lib_pypy('marshal')
+import pypy
+marshal = py.path.local(pypy.__file__).join('..', '..', 'lib_pypy',
+ 'marshal.py').pyimport()
# Non-marshal result types
RESULTTYPE_STATRESULT = object()
@@ -126,6 +131,7 @@
for the external functions xxx that you want to support.
"""
debug = False
+ log = None
os_level_sandboxing = False # Linux only: /proc/PID/seccomp
def __init__(self, args, executable=None):
@@ -142,6 +148,9 @@
self.currenttimeout = None
self.currentlyidlefrom = None
+ if self.debug:
+ self.log = create_log()
+
def withlock(self, function, *args, **kwds):
lock = self.popenlock
if lock is not None:
@@ -169,7 +178,8 @@
if delay <= 0.0:
break # expired!
time.sleep(min(delay*1.001, 1))
- log.timeout("timeout!")
+ if self.log:
+ self.log.timeout("timeout!")
self.kill()
#if interrupt_main:
# if hasattr(os, 'kill'):
@@ -246,22 +256,22 @@
args = read_message(child_stdout)
except EOFError, e:
break
- if self.debug and not self.is_spam(fnname, *args):
- log.call('%s(%s)' % (fnname,
+ if self.log and not self.is_spam(fnname, *args):
+ self.log.call('%s(%s)' % (fnname,
', '.join([shortrepr(x) for x in args])))
try:
answer, resulttype = self.handle_message(fnname, *args)
except Exception, e:
tb = sys.exc_info()[2]
write_exception(child_stdin, e, tb)
- if self.debug:
+ if self.log:
if str(e):
- log.exception('%s: %s' % (e.__class__.__name__, e))
+ self.log.exception('%s: %s' % (e.__class__.__name__, e))
else:
- log.exception('%s' % (e.__class__.__name__,))
+ self.log.exception('%s' % (e.__class__.__name__,))
else:
- if self.debug and not self.is_spam(fnname, *args):
- log.result(shortrepr(answer))
+ if self.log and not self.is_spam(fnname, *args):
+ self.log.result(shortrepr(answer))
try:
write_message(child_stdin, 0) # error code - 0 for ok
write_message(child_stdin, answer, resulttype)
@@ -440,7 +450,8 @@
node = dirnode.join(name)
else:
node = dirnode
- log.vpath('%r => %r' % (vpath, node))
+ if self.log:
+ self.log.vpath('%r => %r' % (vpath, node))
return node
def do_ll_os__ll_os_stat(self, vpathname):
More information about the pypy-commit
mailing list