[pypy-commit] lang-smalltalk storage: Extracted small library to parse jit traces.
anton_gulenko
noreply at buildbot.pypy.org
Fri Mar 28 22:24:38 CET 2014
Author: Anton Gulenko <anton.gulenko at googlemail.com>
Branch: storage
Changeset: r721:c7f1757381ce
Date: 2014-03-28 22:24 +0100
http://bitbucket.org/pypy/lang-smalltalk/changeset/c7f1757381ce/
Log: Extracted small library to parse jit traces. Used that library for
small script to split a trace file into multiple parts for better
analysis.
diff --git a/spyvm/test/jittest/base.py b/spyvm/test/jittest/base.py
--- a/spyvm/test/jittest/base.py
+++ b/spyvm/test/jittest/base.py
@@ -1,14 +1,9 @@
import subprocess
import os
-
-# TODO:
-from rpython.tool.jitlogparser.parser import SimpleParser, Op
-from rpython.tool.jitlogparser.storage import LoopStorage
-
+from rpython.tool.jitlogparser.parser import Op
from rpython.jit.metainterp.resoperation import opname
from rpython.jit.tool import oparser
-from rpython.tool import logparser
-
+from spyvm.tool import logparser
BasePath = os.path.abspath(
os.path.join(
@@ -21,26 +16,15 @@
class BaseJITTest(object):
def run(self, spy, tmpdir, code):
+ logfile = str(tmpdir.join("x.pypylog"))
proc = subprocess.Popen(
[str(spy), "-r", code.replace("\n", "\r\n"), BenchmarkImage],
cwd=str(tmpdir),
- env={"PYPYLOG": "jit-log-opt:%s" % tmpdir.join("x.pypylog"),
+ env={"PYPYLOG": "jit-log-opt:%s" % logfile,
"SDL_VIDEODRIVER": "dummy"}
)
proc.wait()
- data = logparser.parse_log_file(str(tmpdir.join("x.pypylog")), verbose=False)
- data = logparser.extract_category(data, "jit-log-opt-")
-
- storage = LoopStorage()
- traces = [SimpleParser.parse_from_input(t) for t in data]
- main_loops = storage.reconnect_loops(traces)
- traces_w = []
- for trace in traces:
- if trace in main_loops:
- traces_w.append(Trace(trace))
- else:
- traces_w[len(traces_w) - 1].addbridge(trace)
- return traces_w
+ return logparser.extract_traces(logfile)
def assert_matches(self, trace, expected):
expected_lines = [
@@ -65,7 +49,6 @@
aliases[arg] = arg = expected_arg
assert arg == expected_arg
-
class Parser(oparser.OpParser):
def get_descr(self, poss_descr, allow_invent):
if poss_descr.startswith(("TargetToken", "<Guard")):
@@ -77,46 +60,3 @@
def create_op(self, opnum, args, res, descr):
return Op(opname[opnum].lower(), args, res, descr)
-
-
-class Trace(object):
- def __init__(self, trace):
- self._trace = trace
- self._bridges = []
- self._bridgeops = None
- self._loop = None
-
- def addbridge(self, trace):
- self._bridges.append(trace)
-
- @property
- def bridges(self):
- if self._bridgeops:
- return self._bridgeops
- else:
- self._bridgeops = []
- for bridge in self._bridges:
- self._bridgeops.append([op for op in bridge.operations if not op.name.startswith("debug_")])
- return self._bridgeops
-
- @property
- def loop(self):
- if self._loop:
- return self._loop
- else:
- self._loop = self._parse_loop_from(self._trace)
- return self._loop
-
- def _parse_loop_from(self, trace, label_seen=None):
- _loop = []
- for idx, op in enumerate(self._trace.operations):
- if label_seen and not op.name.startswith("debug_"):
- _loop.append(op)
- if op.name == "label":
- if label_seen is None: # first label
- label_seen = False
- else:
- label_seen = True # second label
- if len(_loop) == 0:
- raise ValueError("Loop body couldn't be found")
- return _loop
diff --git a/spyvm/tool/extract_loops.py b/spyvm/tool/extract_loops.py
new file mode 100644
--- /dev/null
+++ b/spyvm/tool/extract_loops.py
@@ -0,0 +1,31 @@
+import sys, os, shutil
+from spyvm.tool import logparser
+
+def main(argv):
+ if len(argv) != 1:
+ print "Need pypy log-file as parameter."
+ return 1
+ logfile = argv[0]
+ traces = logparser.extract_traces(logfile, remove_main_labels=False)
+
+ tracedir = logfile + "_traces"
+ if os.path.exists(tracedir):
+ shutil.rmtree(tracedir)
+ os.mkdir(tracedir)
+
+ for i, trace in enumerate(traces):
+ basename = os.path.join(tracedir, "loop_" + str(i))
+ print_trace(trace.loop, basename + '_main')
+ print_trace(trace.setup, basename + '_setup')
+ for bridge_num, bridge in enumerate(trace.bridges):
+ print_trace(bridge, basename + "_bridge_" + str(bridge_num))
+
+def print_trace(trace, filename):
+ if trace:
+ file = open(filename, 'w')
+ for t in trace:
+ file.write(str(t))
+ file.write('\n')
+
+if __name__ == "__main__":
+ main(sys.argv[1:])
diff --git a/spyvm/tool/logparser.py b/spyvm/tool/logparser.py
new file mode 100644
--- /dev/null
+++ b/spyvm/tool/logparser.py
@@ -0,0 +1,67 @@
+from rpython.tool import logparser
+from rpython.tool.jitlogparser.parser import SimpleParser, Op
+from rpython.tool.jitlogparser.storage import LoopStorage
+from rpython.jit.tool import oparser
+
+def extract_traces(file, remove_debug=True, remove_main_labels=True, remove_all_labels=False):
+ data = logparser.parse_log_file(file, verbose=False)
+ data = logparser.extract_category(data, "jit-log-opt-")
+
+ storage = LoopStorage()
+ traces = [SimpleParser.parse_from_input(t) for t in data]
+ main_loops = storage.reconnect_loops(traces)
+ traces_w = []
+ for trace in traces:
+ if trace in main_loops:
+ traces_w.append(Trace(trace))
+ else:
+ traces_w[len(traces_w) - 1].addbridge(trace)
+ for trace in traces_w:
+ trace.parse(remove_debug, remove_main_labels, remove_all_labels)
+ return traces_w
+
+class Trace(object):
+ def __init__(self, trace,):
+ self._trace = trace
+ self._bridgetraces = []
+ self.bridges = None
+ self.setup = None
+ self.loop = None
+
+ def addbridge(self, trace):
+ self._bridgetraces.append(trace)
+
+ def parse(self, remove_debug, remove_main_labels, remove_all_labels):
+ self.remove_debug, self.remove_main_labels, self.remove_all_labels = \
+ remove_debug, remove_main_labels, remove_all_labels
+ self.parse_loop()
+ self.parse_bridges()
+
+ def keep_op(self, op):
+ return \
+ (not self.remove_debug or not op.name.startswith("debug_")) and \
+ (not self.remove_all_labels or not op.name == "label")
+
+ def parse_bridges(self):
+ self.bridges = []
+ for bridge in self._bridgetraces:
+ self.bridges.append([op for op in bridge.operations if self.keep_op(op)])
+
+ def parse_loop(self):
+ self.loop = []
+ self.setup = []
+ label_seen = None
+ for idx, op in enumerate(self._trace.operations):
+ if op.name == "label":
+ if label_seen is None: # first label
+ label_seen = False
+ if self.remove_main_labels: continue
+ elif label_seen is False:
+ label_seen = True # second label
+ if self.remove_main_labels: continue
+ if self.keep_op(op):
+ if label_seen:
+ self.loop.append(op)
+ else:
+ self.setup.append(op)
+
\ No newline at end of file
More information about the pypy-commit
mailing list