[pypy-svn] r58993 - in pypy/build/benchmem: . testing

[xoraxax at codespeak.net]

Author: xoraxax
Date: Sat Oct 11 18:24:55 2008
New Revision: 58993

Modified:
   pypy/build/benchmem/runbench.py
   pypy/build/benchmem/testing/test_benchtool.py
Log:
(hpk, xoraxax) Add new startup time benchmark.

Modified: pypy/build/benchmem/runbench.py
==============================================================================
--- pypy/build/benchmem/runbench.py	(original)
+++ pypy/build/benchmem/runbench.py	Sat Oct 11 18:24:55 2008
@@ -55,6 +55,45 @@
         popen.wait()
 
 
+class BenchRunnerBaseTime(BenchRunner):
+    def __init__(self, executable, logpath, options):
+        self.executable = executable
+        self.executable_full_path = py.path.local.sysfind(executable)
+        self.logpath = py.path.local(logpath)
+        self.logstream = self.logpath.open("a")
+        self.numiter = options.basetime_numiter
+
+    def write_benchheader(self):
+        print >>self.logstream, self.SEPBENCH
+        print >>self.logstream, "#benchtype=basetime"
+        print >>self.logstream, "#executable=%s" %(str(self.executable ),)
+        print >>self.logstream
+
+    def timecommand(self, cmd):
+        cmds = "time -p " + cmd
+        self.log("running %r for %i times" % (cmds, self.numiter))
+        alltimes = []
+        for _ in xrange(self.numiter):
+            popen = Popen(cmds, shell=True, stderr=PIPE, close_fds=True)
+            if popen.wait():
+                raise Exception('Error in command ' + cmds)
+            timelines = popen.stderr.read().strip().splitlines()[-3:]
+            times = {}
+            for line in timelines:
+                name, time = line.split(" ", 1)
+                times[name] = float(time)
+            alltimes.append(times)
+        return alltimes
+
+    def run(self):
+        self.write_benchheader()
+        for name, cmdpostfix in (('site', '-c pass'), ('nosite', '-S -c pass'),
+                ('importos', '-S -c "import os"')):
+            cmd = "%s %s " % (self.executable, cmdpostfix)
+            times = self.timecommand(cmd)
+            print >>self.logstream, "%s:%r" % (name, times)
+
+
 class BenchRunnerObjsize(BenchRunner):
     def __init__(self, executable, logpath, options):
         self.executable = executable
@@ -257,6 +296,27 @@
         self.executable = executable
 
 
+class BasetimeResult(Result):
+    benchtype = 'basetime'
+    def __init__(self, timings, executable):
+        self.timings = timings
+        self.executable = executable
+        self.mintimings = []
+        for name, timing in timings:
+            self.mintimings.append((name, min(timing, key=lambda x: x['real'])))
+
+    @classmethod
+    def parse(cls, lnstream, kw):
+        timings = []
+        for lineno, line in lnstream:
+            if line.strip() == BenchRunner.SEPBENCH:
+                break
+            name, data = line.split(":", 1)
+            times = eval(data)
+            timings.append((name, times))
+        return cls(timings, **kw)
+
+
 class Mappings(object):
     HEAP, CODE, DATA = object(), object(), object()
 
@@ -311,7 +371,9 @@
 parser.add_option("-a", "--append", action="store_true", dest="append", default=False, 
                   help="append to logfile")
 parser.add_option("-n", "--numiter", action="store", dest="numiter",
-                  default=100000, help="number of iterations")
+                  default=1000, help="number of iterations")
+parser.add_option("--basetime-numiter", action="store", dest="basetime_numiter",
+                  default=10, help="number of iterations for base startup time")
 
 def getbenchlog(options):
     benchlog = options.benchlog
@@ -340,6 +402,8 @@
         return BenchRunnerObjsize
     if benchtype == "basesize":
         return BenchRunnerBaseSize
+    if benchtype == "basetime":
+        return BenchRunnerBaseTime
 
 if __name__ == '__main__':
     (options, args) = parser.parse_args()

Modified: pypy/build/benchmem/testing/test_benchtool.py
==============================================================================
--- pypy/build/benchmem/testing/test_benchtool.py	(original)
+++ pypy/build/benchmem/testing/test_benchtool.py	Sat Oct 11 18:24:55 2008
@@ -232,3 +232,23 @@
     assert result2.snapshot.heap_private() - result1.snapshot.heap_private() < 21
     report.CheckpointDetails(resultset).run() # shouldnt do anything
 
+def test_startuptime_half_functional():
+    tmpdir = py.test.ensuretemp("test_startuptime")
+    script = py.path.local(runbench.__file__).dirpath("runbench.py")
+    benchlog = tmpdir.join("benchlog")
+    py.process.cmdexec("%s -e python2.5,python2.5 --benchlog=%s basetime" % (script, benchlog))
+
+    resultset = runbench.ResultSet()
+    resultset.parse(benchlog)
+
+    assert len(resultset.results) == 2
+    result1, result2 = resultset.results
+    assert result1.benchtype == 'basetime'
+    assert result1.executable.endswith('python2.5')
+    report.CheckpointDetails(resultset).run() # shouldnt do anything
+
+    for name, timing in result1.mintimings:
+        assert timing['real'] >= timing['user'] + timing['sys'] - 0.02
+        # -0.02 is some skew for rounding problems
+        assert timing['real'] > 0
+