[pypy-commit] pypy default: more cleaning
fijal
noreply at buildbot.pypy.org
Sat Oct 13 00:19:19 CEST 2012
Author: Maciej Fijalkowski <fijall at gmail.com>
Branch:
Changeset: r58088:64820bd14b80
Date: 2012-10-13 00:19 +0200
http://bitbucket.org/pypy/pypy/changeset/64820bd14b80/
Log: more cleaning
diff --git a/pypy/translator/goal/bench-cronjob.py b/pypy/translator/goal/bench-cronjob.py
deleted file mode 100755
--- a/pypy/translator/goal/bench-cronjob.py
+++ /dev/null
@@ -1,141 +0,0 @@
-#! /usr/bin/env python
-
-import os
-homedir = os.getenv('HOME')
-os.environ['PATH'] += ':'+homedir+'/bin'
-
-import autopath
-import py
-import time, os, sys, stat
-
-os.umask(022) # allow everyone to read/execute the produced pypy-c's
-
-
-tmpdir = py.std.tempfile.gettempdir() + '/usession-' + os.environ['USER'] + '/'
-cflags = "-O3"
-lflags = "-lgc -lm -lpthread"
-
-dry_run = False
-
-def run(cmd):
- print 'RUN:', cmd
- sys.stdout.flush()
- result = 0 #OK
- if not dry_run:
- result = os.system(cmd) #note: result is system dependent but works on Linux the way we want
- return result
-
-def update_pypy():
- os.chdir(homedir + '/projects/pypy-trunk')
- run('/usr/local/bin/svn up 2>&1')
-
-def compile(backend):
- try:
- backend, features = backend.split('--', 1)
- featureoptions = ''.join([" --" + f for f in features.split('--') if f[0] != '_'])
- targetoptions = ''.join([" --" + f[1:] for f in features.split('--') if f[0] == '_'])
- except:
- features = ''
- featureoptions = ''
- targetoptions = ''
-
- translateoptions = ''
-
- def normalize(f):
- if f.startswith('_'):
- f = f[1:]
- if f.startswith('profopt'):
- f = 'prof'
- return f
- features = '--'.join([normalize(f) for f in features.split('--')])
-
- os.chdir(homedir + '/projects/pypy-trunk/pypy/translator/goal')
- run('/usr/local/bin/python translate.py --backend=%(backend)s%(featureoptions)s%(translateoptions)s --batch targetpypystandalone.py %(targetoptions)s 2>&1' % locals())
-
- os.chdir(homedir + '/projects/pypy-trunk')
- try:
- revision = '%d' % (py.path.svnwc('.').info().rev,)
- except:
- revision = 'unknown'
- basename = homedir + '/projects/pypy-trunk/pypy/translator/goal/' + 'pypy-' + backend
- realname = basename + '-' + revision
- if features:
- realname += "-" + features
-
- if os.path.exists(basename): #copy executable
- run("mv %s %s" % (basename, realname))
- if backend == 'cli':
- basename_dir = basename + '-data'
- realname_dir = realname + '-data'
- run("mv %s %s" % (basename_dir, realname_dir))
- elif backend == 'jvm':
- basename_jar = basename + '.jar'
- realname_jar = realname + '.jar'
- run("mv %s %s" % (basename_jar, realname_jar))
- #pypy = open(basename, 'rb').read()
- #if len(pypy) > 0:
- # open(realname, 'wb').write(pypy)
- #os.chmod(realname, stat.S_IRWXU)
- #os.unlink(basename)
-
-def get_load():
- g = os.popen('uptime', 'r')
- buf = g.read().strip()
- g.close()
- return buf
-
-def benchmark():
- os.chdir(homedir + '/projects/pypy-trunk/pypy/translator/goal')
- uname = os.popen('uname -a', 'r').read()
- startload = get_load()
-# result = run('/usr/local/bin/withlock /tmp/cpu_cycles_lock /usr/local/bin/python bench-unix.py 2>&1 | tee benchmark.txt' % locals())
- result = run('/usr/local/bin/python bench-unix.py 2>&1 | tee benchmark.txt' % locals())
- endload = get_load()
- if not dry_run and result == 0:
- f = open('benchmark.html', 'w')
- print >> f, "<html><body>"
- print >> f, "<pre>"
- print >> f, "uname -a:", uname
- print >> f, "Benchmark started:", startload
- print >> f, " ended:", endload
- print >> f
- f.write(open('benchmark.txt').read())
- print >> f, "</pre>"
- print >> f, "</body></html>"
- f.close()
-
-def main(backends=[]):
- if backends == []: #_ prefix means target specific option, # prefix to outcomment
- backends = [backend.strip() for backend in """
- c--opt=0--_no-allworkingmodules
- c--stackless--gc=boehm--opt=3--_no-allworkingmodules
- c--gc=boehm--opt=3
- c--thread--gc=hybrid--opt=3--_no-allworkingmodules
- c--gc=semispace--opt=3--_no-allworkingmodules
- c--gc=generation--opt=3--_no-allworkingmodules
- c--gc=hybrid--opt=3--_no-allworkingmodules
- cli--opt=3--_no-allworkingmodules
- jvm--opt=3--_no-allworkingmodules
- jvm--inline-threshold=0--opt=3--_no-allworkingmodules
- """.split('\n') if backend.strip() and not backend.strip().startswith('#')]
- print time.ctime()
- update_pypy()
- for backend in backends:
- try:
- compile(backend)
- except:
- raise
- pass
- benchmark()
- print time.ctime()
- print 80*'-'
-
-if __name__ == '__main__':
- args = sys.argv[1:]
- if args and args[0] == '--benchmark-only':
- benchmark()
- else:
- if args and args[0] == '--dry-run':
- del args[0]
- dry_run = True
- main(args)
diff --git a/pypy/translator/goal/bench-unix.py b/pypy/translator/goal/bench-unix.py
deleted file mode 100644
--- a/pypy/translator/goal/bench-unix.py
+++ /dev/null
@@ -1,171 +0,0 @@
-# benchmarks on a unix machine.
-# to be executed in the goal folder,
-# where a couple of pypy-* files is expected.
-
-import os, sys, time, pickle
-
-PYSTONE_CMD = 'from test import pystone;pystone.main(%s)'
-PYSTONE_PATTERN = 'This machine benchmarks at'
-PYSTONE_ASCENDING_GOOD = True
-
-RICHARDS_CMD = 'from richards import *;main(iterations=%d)'
-RICHARDS_PATTERN = 'Average time per iteration:'
-RICHARDS_ASCENDING_GOOD = False
-
-class BenchmarkResult(object):
-
- def __init__(self, filename, max_results=10):
- self.filename = filename
- self.max_results = max_results
- if os.path.exists(filename):
- f = open(filename, 'r')
- self.n_results = pickle.load(f)
- self.best_result = pickle.load(f)
- f.close()
- # any exception while loading the file is best reported
- # as a crash, instead of as a silent loss of all the
- # data :-/
- else:
- self.n_results = {}
- self.best_result = {}
-
- def is_stable(self, name):
- try:
- return self.n_results[name] >= self.max_results
- except:
- return False
-
- def update(self, name, result, ascending_good):
- try:
- if ascending_good:
- self.best_result[name] = max(self.best_result[name], result)
- else:
- self.best_result[name] = min(self.best_result[name], result)
- except KeyError:
- self.n_results[name] = 0
- self.best_result[name] = result
- self.n_results[name] += 1
-
- f = open(self.filename, 'w')
- pickle.dump(self.n_results , f)
- pickle.dump(self.best_result, f)
- f.close()
-
- def get_best_result(self, name):
- return self.best_result[name]
-
-
-def get_result(txt, pattern):
- for line in txt.split('\n'):
- if line.startswith(pattern):
- break
- else:
- print 'warning: this is no valid output'
- return 99999.0
- return float(line.split()[len(pattern.split())])
-
-def run_cmd(cmd):
- #print "running", cmd
- pipe = os.popen(cmd + ' 2>&1')
- return pipe.read()
-
-def run_pystone(executable='/usr/local/bin/python', n=0):
- argstr = PYSTONE_CMD % (str(n) and n or '')
- txt = run_cmd('"%s" -c "%s"' % (executable, argstr))
- return get_result(txt, PYSTONE_PATTERN)
-
-def run_richards(executable='/usr/local/bin/python', n=5):
- argstr = RICHARDS_CMD % n
- txt = run_cmd('"%s" -c "%s"' % (executable, argstr))
- return get_result(txt, RICHARDS_PATTERN)
-
-def get_executables(): #sorted by revision number (highest first)
- exes = []
- for exe in [os.path.join('.', name) for name in os.listdir('.') if name.startswith('pypy-')]:
- if os.path.isdir(exe) or exe.endswith('.jar'):
- continue
- try:
- exes.append( (exe.split('-')[2], exe) )
- except:
- pass #skip filenames without version number
- exes.sort()
- exes.reverse()
- exes = [s[1] for s in exes]
- return exes
-
-def main():
- benchmark_result = BenchmarkResult('bench-unix.benchmark_result')
-
- print 'date size codesize executable richards pystone'
- sys.stdout.flush()
-
- ref_rich, ref_stone = None, None
-
-# for exe in '/usr/local/bin/python2.5 python2.4 python2.3'.split():
- for exe in 'python2.4 python2.3'.split():
- v = os.popen(exe + ' -c "import sys;print sys.version.split()[0]"').read().strip()
- if not v:
- continue
- r = v + '_richards'
- if not benchmark_result.is_stable(r):
- benchmark_result.update(r, run_richards(exe), RICHARDS_ASCENDING_GOOD)
- rich = benchmark_result.get_best_result(r)
- if not ref_rich:
- ref_rich = rich
-
- p = v + '_pystone'
- if not benchmark_result.is_stable(p):
- benchmark_result.update(p, run_pystone(exe), PYSTONE_ASCENDING_GOOD)
- stone = benchmark_result.get_best_result(p)
- if not ref_stone:
- ref_stone = stone
-
- fmt = '%-26s %8s %8s <a href="microbench-archive/%s.txt">%-60s</a> %6dms (%6.1fx) %6d (%6.1fx)'
- print fmt % (time.ctime(), '-', '-', 'python', 'CPython ' + v, rich, rich / ref_rich, stone, stone / ref_stone)
- sys.stdout.flush()
-
- for exe in get_executables():
- exename = os.path.splitext(exe)[0].lstrip('./')
- ctime = time.ctime( os.path.getmtime(exename) )
-
- #compute microbenchmark results (only once)
- f = '../microbench/archive/%s.txt' % exe
- if not os.path.exists(f) or os.stat(f).st_size < 100:
- os.chdir('../microbench')
- run_cmd('python2.4 ./microbench.py python2.4 "../goal/%s" > "archive/%s.txt"' % (exe, exe))
- os.chdir('../goal')
-
- r = exe + '_richards'
- if not benchmark_result.is_stable(r):
- #continue with our regular benchmarks
- benchmark_result.update(r, run_richards(exe, 1), RICHARDS_ASCENDING_GOOD)
- rich = benchmark_result.get_best_result(r)
-
- p = exe + '_pystone'
- if not benchmark_result.is_stable(p):
- benchmark_result.update(p, run_pystone(exe), PYSTONE_ASCENDING_GOOD)
- stone = benchmark_result.get_best_result(p)
-
- if 'pypy-cli' in exename:
- dirname = exename + '-data'
- codesize = 'N/A'
- try:
- exesize = os.path.getsize(os.path.join(dirname, 'main.exe'))
- except OSError:
- exesize = 'XXX'
- elif 'pypy-jvm' in exename:
- jarname = exename + '.jar'
- codesize = 'N/A'
- try:
- exesize = os.path.getsize(jarname)
- except OSError:
- exesize = 'XXX'
- else:
- codesize = os.popen('size "%s" | tail -n1 | cut -f1'%(exename,)).read().strip()
- exesize = os.path.getsize(exe)
-
- print fmt % (ctime, exesize, codesize, exename, exename, rich, rich / ref_rich, stone, ref_stone / stone)
- sys.stdout.flush()
-
-if __name__ == '__main__':
- main()
diff --git a/pypy/translator/goal/bench-windows.py b/pypy/translator/goal/bench-windows.py
deleted file mode 100644
--- a/pypy/translator/goal/bench-windows.py
+++ /dev/null
@@ -1,186 +0,0 @@
-# benchmarks on a windows machine.
-# to be executed in the goal folder,
-# where a couple of .exe files is expected.
-
-USE_HIGH_PRIORITY = True
-# usage with high priority:
-# the program will try to import subprocess.
-# you can have this with python older than 2.4: copy
-# subprocess into lib and change line 392 to use win32
-
-current_result = """
-executable richards pystone size (MB)
-pypy-c-17439 37413 47.7x 678.4 60.7x 5.65
-pypy-c-17600-lo 26352 33.6x 906.2 45.4x 6.43
-pypy-c-17634-lo 20108 25.7x 1023.5 40.2x 6.42
-pypy-c-17649-lo 22612 28.9x 1042.0 39.5x 6.41
-pypy-c-17674-lo 19248 24.6x 1358.8 30.3x 6.40
-pypy-c-17674 12402 15.8x 1941.4 21.2x 7.37
-pypy-c-17439-lo 29638 37.8x 971.4 42.4x 6.49
-pypy-c-17707 14095 18.0x 2092.7 19.7x 7.37
-pypy-c-17707-lo 19102 24.4x 1354.7 30.4x 6.40
-pypy-c-17707-lo-range 18786 24.0x 2800.8 14.7x 6.40
-pypy-c-17707-range 13980 17.8x 2899.9 14.2x 7.38
-pypy-c-17743 13944 17.8x 2800.3 14.7x 7.30
-pypy-c-17761-samuele 13243 16.9x 2983.3 13.8x 7.69
-pypy-c-17794-ref-crash 41088 52.4x 1084.5 37.9x 14.62
-pypy-c-17950 12888 16.4x 3203.0 12.8x 5.49
-pypy-c-18236 9263 11.8x 3702.8 11.1x 5.12
-python 2.4.1 783 1.0x 41150.3 1.0x 0.96
-
-Defaults are: --gc=boehm
-'lo' indicates --lowmem
-STarting with rev. 18236, gc_pypy.dll is used
-"""
-
-import os, sys, pickle, md5
-try:
- from subprocess import *
-except ImportError:
- Popen = None
-
-PYSTONE_CMD = 'from test import pystone;pystone.main(%s)'
-PYSTONE_PATTERN = 'This machine benchmarks at'
-RICHARDS_CMD = 'from richards import *;main(iterations=%d)'
-RICHARDS_PATTERN = 'Average time per iteration:'
-
-def get_result(txt, pattern):
- for line in txt.split('\n'):
- if line.startswith(pattern):
- break
- else:
- raise ValueError, 'this is no valid output: %r' % txt
- return float(line.split()[len(pattern.split())])
-
-def run_cmd(cmd):
- print "running", cmd
- pipe = os.popen(cmd + ' 2>&1')
- result = pipe.read()
- print "done"
- return result
-
-def run_cmd_subprocess(cmd):
- print "running", cmd
- result = Popen(cmd, stdout=PIPE, creationflags=CREATIONFLAGS
- ).communicate()[0]
- print "done"
- return result
-
-CREATIONFLAGS = 0
-if Popen:
- run_cmd = run_cmd_subprocess
- try:
- import win32con, win32api
- except ImportError:
- pass
- else:
- if USE_HIGH_PRIORITY:
- CREATIONFLAGS = win32con.HIGH_PRIORITY_CLASS
- print "configured to run under high priority"
-
-BENCH_EXECONFIG = '_bench_windows_exe.txt'
-bench_exe = None
-
-def reference(progname):
- global bench_exe
- if not bench_exe:
- if os.path.exists(BENCH_EXECONFIG):
- progname = file(BENCH_EXECONFIG).read().strip()
- print "using %s instead of the system default" % progname
- bench_exe = progname
- return bench_exe
-
-def run_version_size(executable=reference('python'), *args):
- ver, size, dll = run_cmd('%s -c "import sys, os; print sys.version.split()[0], '
- 'os.path.getsize(sys.executable), sys.dllhandle"'
- % executable).split()
- size = int(size)
- try:
- import win32api
- except ImportError:
- pass
- else:
- size += os.path.getsize(win32api.GetModuleFileName(int(dll)))
- return ver, size
-
-def run_pystone(executable=reference('python'), n=0, rpy=False):
- if rpy:
- txt = run_cmd('%s pystone' % executable)
- else:
- argstr = PYSTONE_CMD % (str(n) and n or '')
- txt = run_cmd('%s -c "%s"' % (executable, argstr))
- res = get_result(txt, PYSTONE_PATTERN)
- print res
- return res
-
-def run_richards(executable=reference('python'), n=20, rpy=False):
- if rpy:
- txt = run_cmd('%s richards' % executable)
- else:
- argstr = RICHARDS_CMD % n
- txt = run_cmd('%s -c "%s"' % (executable, argstr))
- res = get_result(txt, RICHARDS_PATTERN)
- print res
- return res
-
-def get_executables():
- exes = [name for name in os.listdir('.') if name.endswith('.exe')]
- exes.sort()
- return exes
-
-STAT_FILE = '_bench_windows.dump'
-def load_stats(statfile=STAT_FILE):
- try:
- dic = pickle.load(file(statfile, 'rb'))
- except IOError:
- dic = {}
- return dic
-
-def save_stats(dic, statfile=STAT_FILE):
- pickle.dump(dic, file(statfile, 'wb'))
-
-HEADLINE = '''\
-executable richards pystone size (MB)'''
-FMT = '''\
-%-27s''' + '%5d %5.1fx' + ' %9.1f %5.1fx %5.3f'
-FMT2 = '''\
-%-27s''' + '%5.3f %5.1f/' + ' %9.1f %5.1f/ %5.3f'
-
-def main():
- print 'getting the richards reference'
- ref_rich = run_richards()
- print 'getting the pystone reference'
- ref_stone = run_pystone()
- resdic = {}
- prior = load_stats()
- for exe in get_executables():
- exename = os.path.splitext(exe)[0]
- mtime = os.path.getmtime(exe)
- size = os.path.getsize(exe)
- rpy = size < 500000
- key = md5.new(file(exe,'rb').read()).digest()
- if key in prior:
- print 'skipped', exename
- resdic[key] = prior[key][:2] + (exename, mtime, size)
- else:
- resdic[key] = (run_richards(exe, 2,rpy), run_pystone(exe, 20000, rpy),
- exename, mtime, size)
- prior[key] = resdic[key] # save result, temporarily
- save_stats(prior)
- save_stats(resdic) # save cleaned result
- res = [ (stone / rich, exe, size, rich, stone)
- for rich, stone, exe, mtime, size in resdic.values()]
- version, size = run_version_size()
- res.append( (ref_stone/ref_rich, 'python %s' % version, size, ref_rich, ref_stone) )
- res.sort()
- print HEADLINE
- for speed2, exe, size, rich, stone in res:
- if speed2 <= ref_stone/ref_rich:
- print FMT % (exe, rich, rich / ref_rich, stone, ref_stone / stone,
- size / float(1024 * 1024))
- else:
- print FMT2 % (exe, rich, ref_rich / rich, stone, stone / ref_stone,
- size / float(1024 * 1024))
-
-if __name__ == '__main__':
- main()
More information about the pypy-commit
mailing list