[pypy-svn] r57354 - in pypy/dist/pypy: config config/test doc doc/config tool/bench/test translator translator/goal
arigo at codespeak.net
arigo at codespeak.net
Sun Aug 17 15:15:53 CEST 2008
Author: arigo
Date: Sun Aug 17 15:15:50 2008
New Revision: 57354
Added:
pypy/dist/pypy/doc/config/objspace.std.multimethods.txt
- copied unchanged from r57352, pypy/branch/opt-option/pypy/doc/config/objspace.std.multimethods.txt
pypy/dist/pypy/doc/config/opt.txt
- copied unchanged from r57352, pypy/branch/opt-option/pypy/doc/config/opt.txt
Removed:
pypy/dist/pypy/doc/config/objspace.std.allopts.txt
pypy/dist/pypy/translator/goal/targetmultiplespaces.py
Modified:
pypy/dist/pypy/config/config.py
pypy/dist/pypy/config/makerestdoc.py
pypy/dist/pypy/config/pypyoption.py
pypy/dist/pypy/config/test/test_config.py
pypy/dist/pypy/config/test/test_pypyoption.py
pypy/dist/pypy/config/translationoption.py
pypy/dist/pypy/doc/config/commandline.txt
pypy/dist/pypy/doc/config/confrest.py
pypy/dist/pypy/doc/config/index.txt
pypy/dist/pypy/doc/garbage_collection.txt
pypy/dist/pypy/doc/getting-started.txt
pypy/dist/pypy/doc/interpreter-optimizations.txt
pypy/dist/pypy/tool/bench/test/test_pypyresult.py
pypy/dist/pypy/translator/driver.py
pypy/dist/pypy/translator/goal/bench-cronjob.py
pypy/dist/pypy/translator/goal/targetprologstandalone.py
pypy/dist/pypy/translator/goal/targetpypystandalone.py
pypy/dist/pypy/translator/goal/translate.py
Log:
Remove the --allopts and --faassen options and replace
them with a global translate.py option '--opt'.
See pypy/doc/config/opt.txt for more information.
Note that --opt is not a regular option, but is implemented
in a pair of functions that can suggest options based on
other information, e.g. the selected backend.
This is a merge from the opt-option branch. Also contains:
* clean up a bit the translation options and their defaults
* updated bench-cronjob.py, tuatara might need updates too
* updated getting-started.txt
* small extension to the config system to support the kind
of 'suggest' used by --opt.
Modified: pypy/dist/pypy/config/config.py
==============================================================================
--- pypy/dist/pypy/config/config.py (original)
+++ pypy/dist/pypy/config/config.py Sun Aug 17 15:15:50 2008
@@ -13,6 +13,9 @@
class ConfigError(Exception):
pass
+class ConflictConfigError(ConfigError):
+ pass
+
class Config(object):
_cfgimpl_frozen = False
@@ -99,11 +102,23 @@
if oldvalue != value and oldowner not in ("default", "suggested"):
if who in ("default", "suggested"):
return
- raise ConfigError('cannot override value to %s for option %s' %
- (value, name))
+ raise ConflictConfigError('cannot override value to %s for '
+ 'option %s' % (value, name))
child.setoption(self, value, who)
self._cfgimpl_value_owners[name] = who
+ def suggest(self, **kwargs):
+ for name, value in kwargs.items():
+ self.suggestoption(name, value)
+
+ def suggestoption(self, name, value):
+ try:
+ self.setoption(name, value, "suggested")
+ except ConflictConfigError:
+ # setting didn't work, but that is fine, since it is
+ # suggested only
+ pass
+
def set(self, **kwargs):
all_paths = [p.split(".") for p in self.getpaths()]
for key, value in kwargs.iteritems():
@@ -248,12 +263,7 @@
for path, reqvalue in self._suggests.get(value, []):
toplevel = config._cfgimpl_get_toplevel()
homeconfig, name = toplevel._cfgimpl_get_home_by_path(path)
- try:
- homeconfig.setoption(name, reqvalue, "suggested")
- except ConfigError:
- # setting didn't work, but that is fine, since it is
- # suggested only
- pass
+ homeconfig.suggestoption(name, reqvalue)
super(ChoiceOption, self).setoption(config, value, who)
def validate(self, value):
@@ -298,12 +308,7 @@
for path, reqvalue in self._suggests:
toplevel = config._cfgimpl_get_toplevel()
homeconfig, name = toplevel._cfgimpl_get_home_by_path(path)
- try:
- homeconfig.setoption(name, reqvalue, "suggested")
- except ConfigError:
- # setting didn't work, but that is fine, since it is
- # suggested
- pass
+ homeconfig.suggestoption(name, reqvalue)
super(BoolOption, self).setoption(config, value, who)
Modified: pypy/dist/pypy/config/makerestdoc.py
==============================================================================
--- pypy/dist/pypy/config/makerestdoc.py (original)
+++ pypy/dist/pypy/config/makerestdoc.py Sun Aug 17 15:15:50 2008
@@ -178,10 +178,12 @@
return "Internal Options"
return ""
-def make_cmdline_overview(descr):
- content = Rest(
- Title("Overview of Command Line Options for '%s'" % (descr._name, ),
- abovechar="=", belowchar="="))
+def make_cmdline_overview(descr, title=True):
+ content = Rest()
+ if title:
+ content.add(
+ Title("Overview of Command Line Options for '%s'" % (descr._name, ),
+ abovechar="=", belowchar="="))
cmdlines = []
config = Config(descr)
for path in config.getpaths(include_groups=False):
Modified: pypy/dist/pypy/config/pypyoption.py
==============================================================================
--- pypy/dist/pypy/config/pypyoption.py (original)
+++ pypy/dist/pypy/config/pypyoption.py Sun Aug 17 15:15:50 2008
@@ -3,7 +3,7 @@
import sys
from pypy.config.config import OptionDescription, BoolOption, IntOption, ArbitraryOption
from pypy.config.config import ChoiceOption, StrOption, to_optparse, Config
-from pypy.config.config import ConfigError
+from pypy.config.config import ConflictConfigError
modulepath = py.magic.autopath().dirpath().dirpath().join("module")
all_modules = [p.basename for p in modulepath.listdir()
@@ -27,7 +27,8 @@
["_socket", "unicodedata", "mmap", "fcntl",
"rctime" , "select", "zipimport", "_lsprof",
"crypt", "signal", "dyngram", "_rawffi", "termios", "zlib",
- "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO"]
+ "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO",
+ "thread"]
))
if sys.platform == "win32":
@@ -74,7 +75,8 @@
"The module %r is disabled\n" % (modname,) +
"because importing %s raised %s\n" % (name, errcls) +
str(e))
- raise ConfigError("--withmod-%s: %s" % (modname, errcls))
+ raise ConflictConfigError("--withmod-%s: %s" % (modname,
+ errcls))
return validator
else:
return None
@@ -165,11 +167,11 @@
BoolOption("withsmallint", "use tagged integers",
default=False,
- requires=[("translation.gc", "boehm")]),
+ requires=[("translation.gc", "boehm"),
+ ("objspace.std.withprebuiltint", False)]),
BoolOption("withprebuiltint", "prebuild commonly used int objects",
- default=False,
- requires=[("objspace.std.withsmallint", False)]),
+ default=False),
IntOption("prebuiltintfrom", "lowest integer which is prebuilt",
default=-5, cmdline="--prebuiltintfrom"),
@@ -304,45 +306,10 @@
"a instrumentation option: before exit, print the types seen by "
"certain simpler bytecodes",
default=False),
-
- BoolOption("allopts",
- "enable all thought-to-be-working optimizations",
- default=False,
- suggests=[("objspace.opcodes.CALL_LIKELY_BUILTIN", True),
- ("objspace.opcodes.CALL_METHOD", True),
- ("translation.withsmallfuncsets", 5),
- ("translation.profopt",
- "-c 'from richards import main;main(); from test import pystone; pystone.main()'"),
- ("objspace.std.withmultidict", True),
-# ("objspace.std.withstrjoin", True),
- ("objspace.std.withshadowtracking", True),
-# ("objspace.std.withstrslice", True),
-# ("objspace.std.withsmallint", True),
- ("objspace.std.withrangelist", True),
- ("objspace.std.withmethodcache", True),
-# ("objspace.std.withfastslice", True),
- ("objspace.std.withprebuiltchar", True),
- ("objspace.std.builtinshortcut", True),
- ("objspace.std.optimized_list_getitem", True),
- ("objspace.std.getattributeshortcut", True),
- ("translation.list_comprehension_operations",True),
- ("translation.backendopt.remove_asserts",True),
- ],
- cmdline="--allopts --faassen", negation=False),
-
-## BoolOption("llvmallopts",
-## "enable all optimizations, and use llvm compiled via C",
-## default=False,
-## requires=[("objspace.std.allopts", True),
-## ("translation.llvm_via_c", True),
-## ("translation.backend", "llvm")],
-## cmdline="--llvm-faassen", negation=False),
+ ChoiceOption("multimethods", "the multimethod implementation to use",
+ ["doubledispatch", "mrd"],
+ default="mrd"),
]),
- #BoolOption("lowmem", "Try to use less memory during translation",
- # default=False, cmdline="--lowmem",
- # requires=[("objspace.geninterp", False)]),
-
-
])
def get_pypy_config(overrides=None, translating=False):
@@ -351,6 +318,51 @@
pypy_optiondescription, overrides=overrides,
translating=translating)
+def set_pypy_opt_level(config, level):
+ """Apply PyPy-specific optimization suggestions on the 'config'.
+ The optimizations depend on the selected level and possibly on the backend.
+ """
+ # warning: during some tests, the type_system and the backend may be
+ # unspecified and we get None. It shouldn't occur in translate.py though.
+ type_system = config.translation.type_system
+ backend = config.translation.backend
+
+ # all the good optimizations for PyPy should be listed here
+ if level in ['2', '3']:
+ config.objspace.opcodes.suggest(CALL_LIKELY_BUILTIN=True)
+ config.objspace.opcodes.suggest(CALL_METHOD=True)
+ config.objspace.std.suggest(withmultidict=True)
+ config.objspace.std.suggest(withshadowtracking=True)
+ config.objspace.std.suggest(withrangelist=True)
+ config.objspace.std.suggest(withmethodcache=True)
+ config.objspace.std.suggest(withprebuiltchar=True)
+ config.objspace.std.suggest(builtinshortcut=True)
+ config.objspace.std.suggest(optimized_list_getitem=True)
+ config.objspace.std.suggest(getattributeshortcut=True)
+
+ # extra costly optimizations only go in level 3
+ if level == '3':
+ config.translation.suggest(profopt=
+ "-c 'from richards import main;main(); "
+ "from test import pystone; pystone.main()'")
+
+ # memory-saving optimizations
+ if level == 'mem':
+ config.objspace.std.suggest(withprebuiltint=True)
+ config.objspace.std.suggest(withrangelist=True)
+ config.objspace.std.suggest(withprebuiltchar=True)
+ config.objspace.std.suggest(withsharingdict=True)
+ # xxx other options? ropes maybe?
+
+ # completely disable geninterp in a level 0 translation
+ if level == '0':
+ config.objspace.suggest(geninterp=False)
+
+ # some optimizations have different effects depending on the typesystem
+ if type_system == 'ootype':
+ config.objspace.std.suggest(multimethods="doubledispatch")
+
+
if __name__ == '__main__':
config = get_pypy_config()
print config.getpaths()
Modified: pypy/dist/pypy/config/test/test_config.py
==============================================================================
--- pypy/dist/pypy/config/test/test_config.py (original)
+++ pypy/dist/pypy/config/test/test_config.py Sun Aug 17 15:15:50 2008
@@ -530,6 +530,15 @@
assert not c.toplevel
+def test_bogus_suggests():
+ descr = OptionDescription("test", '', [
+ BoolOption("toplevel", "", suggests=[("opt", "bogusvalue")]),
+ ChoiceOption("opt", "", ["a", "b", "c"], "a"),
+ ])
+ c = Config(descr)
+ py.test.raises(ConfigError, "c.toplevel = True")
+
+
def test_delattr():
descr = OptionDescription("opt", "", [
OptionDescription("s1", "", [
@@ -549,7 +558,7 @@
def my_validator_2(config):
assert config is c
- raise ConfigError
+ raise ConflictConfigError
descr = OptionDescription("opt", "", [
BoolOption('booloption1', 'option test1', default=False,
Modified: pypy/dist/pypy/config/test/test_pypyoption.py
==============================================================================
--- pypy/dist/pypy/config/test/test_pypyoption.py (original)
+++ pypy/dist/pypy/config/test/test_pypyoption.py Sun Aug 17 15:15:50 2008
@@ -1,6 +1,7 @@
import py
-from pypy.config.pypyoption import get_pypy_config
+from pypy.config.pypyoption import get_pypy_config, set_pypy_opt_level
from pypy.config.config import Config, ConfigError
+from pypy.config.translationoption import set_opt_level
thisdir = py.magic.autopath().dirpath()
@@ -29,10 +30,32 @@
conf.translation.gc = name
assert conf.translation.gctransformer == "framework"
+def test_set_opt_level():
+ conf = get_pypy_config()
+ set_opt_level(conf, '0')
+ assert conf.translation.gc == 'boehm'
+ assert conf.translation.backendopt.none == True
+ conf = get_pypy_config()
+ set_opt_level(conf, '2')
+ assert conf.translation.gc != 'boehm'
+ assert not conf.translation.backendopt.none
+ conf = get_pypy_config()
+ set_opt_level(conf, 'mem')
+ assert conf.translation.gc == 'marksweep'
+ assert not conf.translation.backendopt.none
+
+def test_set_pypy_opt_level():
+ conf = get_pypy_config()
+ set_pypy_opt_level(conf, '2')
+ assert conf.objspace.std.withmultidict
+ conf = get_pypy_config()
+ set_pypy_opt_level(conf, '0')
+ assert not conf.objspace.std.withmultidict
+
def test_rweakref_required():
conf = get_pypy_config()
conf.translation.rweakref = False
- conf.objspace.std.allopts = True
+ set_pypy_opt_level(conf, '3')
assert not conf.objspace.std.withtypeversion
assert not conf.objspace.std.withmethodcache
Modified: pypy/dist/pypy/config/translationoption.py
==============================================================================
--- pypy/dist/pypy/config/translationoption.py (original)
+++ pypy/dist/pypy/config/translationoption.py Sun Aug 17 15:15:50 2008
@@ -2,12 +2,14 @@
import py, os
from pypy.config.config import OptionDescription, BoolOption, IntOption, ArbitraryOption, FloatOption
from pypy.config.config import ChoiceOption, StrOption, to_optparse, Config
+from pypy.config.config import ConfigError
DEFL_INLINE_THRESHOLD = 32.4 # just enough to inline add__Int_Int()
# and just small enough to prevend inlining of some rlist functions.
DEFL_PROF_BASED_INLINE_THRESHOLD = 32.4
DEFL_CLEVER_MALLOC_REMOVAL_INLINE_THRESHOLD = 32.4
+DEFL_LOW_INLINE_THRESHOLD = DEFL_INLINE_THRESHOLD / 2.0
translation_optiondescription = OptionDescription(
"translation", "Translation Options", [
@@ -39,6 +41,8 @@
BoolOption("llvm_via_c", "compile llvm via C",
default=False, cmdline="--llvm-via-c",
requires=[("translation.backend", "llvm")]),
+
+ # gc
ChoiceOption("gc", "Garbage Collection Strategy",
["boehm", "ref", "marksweep", "semispace", "statistics",
"generation", "hybrid", "none"],
@@ -82,18 +86,10 @@
"llvmgc": [("translation.gc", "generation")],
"asmgcc": [("translation.gc", "generation")],
}),
+
+ # other noticeable options
BoolOption("thread", "enable use of threading primitives",
default=False, cmdline="--thread"),
- BoolOption("verbose", "Print extra information", default=False),
- BoolOption("debug", "Record extra annotation information",
- cmdline="-d --debug", default=False),
- BoolOption("insist", "Try hard to go on RTyping", default=False,
- cmdline="--insist"),
- IntOption("withsmallfuncsets",
- "Represent groups of less funtions than this as indices into an array",
- default=0),
- BoolOption("countmallocs", "Count mallocs and frees", default=False,
- cmdline=None),
BoolOption("sandbox", "Produce a fully-sandboxed executable",
default=False, cmdline="--sandbox",
requires=[("translation.thread", False)]),
@@ -101,6 +97,11 @@
default=True),
# misc
+ BoolOption("verbose", "Print extra information", default=False),
+ BoolOption("debug", "Record extra annotation information",
+ cmdline="-d --debug", default=True),
+ BoolOption("insist", "Try hard to go on RTyping", default=False,
+ cmdline="--insist"),
StrOption("cc", "Specify compiler to use for compiling generated C", cmdline="--cc"),
StrOption("profopt", "Specify profile based optimization script",
cmdline="--profopt"),
@@ -108,6 +109,13 @@
default=False, cmdline="--no-profopt", negation=False),
BoolOption("instrument", "internal: turn instrumentation on",
default=False, cmdline=None),
+ BoolOption("countmallocs", "Count mallocs and frees", default=False,
+ cmdline=None),
+ ChoiceOption("fork_before",
+ "(UNIX) Create restartable checkpoint before step",
+ ["annotate", "rtype", "backendopt", "database", "source",
+ "hintannotate", "timeshift"],
+ default=None, cmdline="--fork-before"),
ArbitraryOption("instrumentctl", "internal",
default=None),
@@ -140,11 +148,9 @@
"attempt to pre-allocate the list",
default=False,
cmdline='--listcompr'),
- ChoiceOption("fork_before",
- "(UNIX) Create restartable checkpoint before step",
- ["annotate", "rtype", "backendopt", "database", "source",
- "hintannotate", "timeshift"],
- default=None, cmdline="--fork-before"),
+ IntOption("withsmallfuncsets",
+ "Represent groups of less funtions than this as indices into an array",
+ default=0),
# options for ootype
OptionDescription("ootype", "Object Oriented Typesystem options", [
@@ -273,3 +279,61 @@
value = getattr(existing_config, child._name)
config._cfgimpl_values[child._name] = value
return config
+
+# ____________________________________________________________
+
+OPT_LEVELS = ['0', '1', 'size', 'mem', '2', '3']
+DEFAULT_OPT_LEVEL = '2'
+
+OPT_TABLE_DOC = {
+ '0': 'No optimization. Uses the Boehm GC.',
+ '1': 'Enable a default set of optimizations. Uses the Boehm GC.',
+ 'size': 'Optimize for the size of the executable. Uses the Boehm GC.',
+ 'mem': 'Optimize for run-time memory usage and use a memory-saving GC.',
+ '2': 'Enable most optimizations and use a high-performance GC.',
+ '3': 'Enable all optimizations and use a high-performance GC.',
+ }
+
+OPT_TABLE = {
+ #level: gc backend optimizations...
+ '0': 'boehm nobackendopt',
+ '1': 'boehm lowinline',
+ 'size': 'boehm lowinline remove_asserts',
+ 'mem': 'marksweep lowinline remove_asserts',
+ '2': 'hybrid extraopts',
+ '3': 'hybrid extraopts remove_asserts',
+ }
+
+def set_opt_level(config, level):
+ """Apply optimization suggestions on the 'config'.
+ The optimizations depend on the selected level and possibly on the backend.
+ """
+ # warning: during some tests, the type_system and the backend may be
+ # unspecified and we get None. It shouldn't occur in translate.py though.
+ type_system = config.translation.type_system
+ backend = config.translation.backend
+
+ try:
+ opts = OPT_TABLE[level]
+ except KeyError:
+ raise ConfigError("no such optimization level: %r" % (level,))
+ words = opts.split()
+ gc = words.pop(0)
+
+ # set the GC (only meaningful with lltype)
+ config.translation.suggest(gc=gc)
+
+ # set the backendopts
+ for word in words:
+ if word == 'nobackendopt':
+ config.translation.backendopt.suggest(none=True)
+ elif word == 'lowinline':
+ config.translation.backendopt.suggest(inline_threshold=
+ DEFL_LOW_INLINE_THRESHOLD)
+ elif word == 'remove_asserts':
+ config.translation.backendopt.suggest(remove_asserts=True)
+ elif word == 'extraopts':
+ config.translation.suggest(withsmallfuncsets=5)
+ config.translation.suggest(list_comprehension_operations=True)
+ else:
+ raise ValueError(word)
Modified: pypy/dist/pypy/doc/config/commandline.txt
==============================================================================
--- pypy/dist/pypy/doc/config/commandline.txt (original)
+++ pypy/dist/pypy/doc/config/commandline.txt Sun Aug 17 15:15:50 2008
@@ -1,4 +1,33 @@
-.. intentionally empty, but contains the following comment to fool py.test:
- overview of command line options for objspace
- overview of command line options for translation
+
+.. contents::
+
+
+.. _objspace:
+.. _`overview-of-command-line-options-for-objspace`:
+
+-------------------------------
+PyPy Python interpreter options
+-------------------------------
+
+The following options can be used after ``translate.py
+targetpypystandalone`` or as options to ``py.py``.
+
+.. GENERATE: objspace
+
+
+.. _translation:
+.. _`overview-of-command-line-options-for-translation`:
+
+---------------------------
+General translation options
+---------------------------
+
+The following are options of ``translate.py``. They must be
+given before the ``targetxxx`` on the command line.
+
+* `--opt -O:`__ set the optimization level `[0, 1, size, mem, 2, 3]`
+
+.. __: opt.html
+
+.. GENERATE: translation
Modified: pypy/dist/pypy/doc/config/confrest.py
==============================================================================
--- pypy/dist/pypy/doc/config/confrest.py (original)
+++ pypy/dist/pypy/doc/config/confrest.py Sun Aug 17 15:15:50 2008
@@ -30,12 +30,13 @@
def get_content(self, txtpath, encoding):
if txtpath.basename == "commandline.txt":
- result = [".. contents::"]
- for descr in all_optiondescrs:
- result.append(".. %s_:\n" % (descr._name, ))
- result.append(make_cmdline_overview(descr).text())
- result.append("")
- result.append(txtpath.read())
+ result = []
+ for line in txtpath.read().splitlines():
+ if line.startswith('.. GENERATE:'):
+ start = line[len('.. GENERATE:'):].strip()
+ descr = start_to_descr[start]
+ line = make_cmdline_overview(descr, title=False).text()
+ result.append(line)
return "\n".join(result)
fullpath = txtpath.purebasename
start = fullpath.split(".")[0]
Modified: pypy/dist/pypy/doc/config/index.txt
==============================================================================
--- pypy/dist/pypy/doc/config/index.txt (original)
+++ pypy/dist/pypy/doc/config/index.txt Sun Aug 17 15:15:50 2008
@@ -44,9 +44,9 @@
.. image:: ../image/compat-matrix.png
.. _`configuration`: ../configuration.html
-.. _`objspace options`: commandline.html#overview-of-command-line-options-for-objspace
-.. _`object space options`: commandline.html#overview-of-command-line-options-for-objspace
-.. _`translation options`: commandline.html#overview-of-command-line-options-for-translation
+.. _`objspace options`: commandline.html#objspace
+.. _`object space options`: commandline.html#objspace
+.. _`translation options`: commandline.html#translation
.. _`overview`: commandline.html
.. _`Standard Interpreter Optimizations`: ../interpreter-optimizations.html
.. _`What PyPy can do for your objects`: ../objspace-proxies.html
Modified: pypy/dist/pypy/doc/garbage_collection.txt
==============================================================================
--- pypy/dist/pypy/doc/garbage_collection.txt (original)
+++ pypy/dist/pypy/doc/garbage_collection.txt Sun Aug 17 15:15:50 2008
@@ -26,7 +26,7 @@
For more details, see the `overview of command line options for
translation`_.
-.. _`overview of command line options for translation`: config/commandline.html#overview-of-command-line-options-for-translation
+.. _`overview of command line options for translation`: config/commandline.html#translation
Mark and Sweep
--------------
Modified: pypy/dist/pypy/doc/getting-started.txt
==============================================================================
--- pypy/dist/pypy/doc/getting-started.txt (original)
+++ pypy/dist/pypy/doc/getting-started.txt Sun Aug 17 15:15:50 2008
@@ -546,44 +546,49 @@
.. _`windows document`: windows.html
You can translate the whole of PyPy's Python interpreter to low level C
-code. This is the largest and ultimate example of RPython program that
-our translation toolchain can process. The most standard variant
-nowadays is::
+code. (This is the largest and ultimate example of RPython program that
+our translation toolchain can process.)
- cd pypy/translator/goal
- python translate.py --gc=hybrid --thread targetpypystandalone.py --allworkingmodules --allopts
+1. Install dependencies. You need (these are Debian package names,
+ adapt as needed):
-Dependencies: this will compile all supported built-in modules, some of
-which have external dependencies. On a Debian Linux, for example, you
-need to install the following packages: a full C compiler like gcc;
-``python-dev, python-ctypes``; ``libffi-dev, libz-dev, libbz2-dev,
-libncurses-dev``.
-
-This whole process will take some time and quite a lot of memory (around
-1200 MB on 32 bit machines). It creates an executable ``pypy-c`` in
-the current directory.
-The ``--gc=hybrid`` option means that the ``pypy-c`` will use our own
-exact generational garbage collector implementation, whose performance
-is rather good nowadays. The ``--thread`` option enables the thread
-module, which is still slightly experimental.
-The ``--allopts`` option enables all the
-worthwhile performance optimizations, but slows down the translation
-itself. On Linux 32-bit Intel machines, if you don't need threads, you
-can get some extra speed (and extra translation time) by removing
-``--thread`` and replacing it with ``--gcrootfinder=asmgcc``.
-
-An alternative is to use the `Boehm-Demers-Weiser garbage
-collector`_ instead of our own. For this, use ``--gc=boehm``.
-Be sure to install Boehm before starting the translation (e.g. by running
-``apt-get install libgc-dev`` on Debian or Ubuntu). Translating with Boehm
-is somewhat faster and less memory-hungry than translating with our own GCs.
-
-In any case, as described above, you can find the produced executable under the
-name ``pypy-c``. Type ``pypy-c --help`` to see the options it supports --
-mainly the same basic options as CPython. In addition, ``pypy-c --info``
-prints the translation options that where used to produce this particular
-executable. This executable can be moved around or copied on other machines;
-see Installation_ below.
+ * gcc
+ * ``python-dev``
+ * ``python-ctypes``
+ * ``libffi-dev``
+ * ``libz-dev`` (for the optional ``zlib`` module)
+ * ``libbz2-dev`` (for the optional ``bz2`` module)
+ * ``libncurses-dev`` (for the optional ``_minimal_curses`` module)
+ * ``libgc-dev`` (only when translating with `--opt=0, 1` or `size`)
+
+2. Be warned that translation is time-consuming (30 min to
+ over one hour) and extremely RAM-hungry (kill it if it
+ starts swapping heavily). If you have less than 1.5 GB of
+ RAM (or a slow machine) you might want to pick the
+ `optimization level`_ `1` in the next step. The default
+ level `3` gives much better results, though.
+
+3. Run::
+
+ cd pypy/translator/goal
+ python translate.py --opt=3 targetpypystandalone.py --allworkingmodules
+
+ possibly replacing ``--opt=3`` with ``--opt=1`` or another
+ `optimization level`_ of your choice.
+
+ On Linux 32-bit Intel machines, if you don't need threads, you
+ can get some extra speed (and extra translation time) by adding
+ ``--gcrootfinder=asmgcc`` just after the ``--opt`` option.
+
+.. _`optimization level`: config/opt.html
+
+If everything works correctly this will create an executable
+``pypy-c`` in the current directory. Type ``pypy-c --help``
+to see the options it supports -- mainly the same basic
+options as CPython. In addition, ``pypy-c --info`` prints the
+translation options that where used to produce this particular
+executable. This executable can be moved around or copied on
+other machines; see Installation_ below.
The ``translate.py`` script takes a very large number of options controlling
what to translate and how. See ``translate.py -h``. Some of the more
Modified: pypy/dist/pypy/doc/interpreter-optimizations.txt
==============================================================================
--- pypy/dist/pypy/doc/interpreter-optimizations.txt (original)
+++ pypy/dist/pypy/doc/interpreter-optimizations.txt Sun Aug 17 15:15:50 2008
@@ -384,6 +384,6 @@
.. waffles about ropes
-You can build a pypy with all generally useful optimizations turned on by using
-the :config:`objspace.std.allopts` option. Such a build is between 1.5 and 2.5
-times faster than the default, depending on the benchmark.
+When building pypy, all generally useful optimizations are turned on by default
+unless you explicitly lower the translation optimization level with the
+``--opt`` option.
Modified: pypy/dist/pypy/tool/bench/test/test_pypyresult.py
==============================================================================
--- pypy/dist/pypy/tool/bench/test/test_pypyresult.py (original)
+++ pypy/dist/pypy/tool/bench/test/test_pypyresult.py Sun Aug 17 15:15:50 2008
@@ -12,8 +12,8 @@
return cache[0]
pp = tmpdir.join("testpickle")
f = pp.open("wb")
- pickle.dump({'./pypy-llvm-39474-faassen-c_richards': 5}, f)
- pickle.dump({'./pypy-llvm-39474-faassen-c_richards': 42.0}, f)
+ pickle.dump({'./pypy-llvm-39474-O3-c_richards': 5}, f)
+ pickle.dump({'./pypy-llvm-39474-O3-c_richards': 42.0}, f)
f.close()
cache.append(pp)
return pp
@@ -38,9 +38,9 @@
assert res.besttime == 2.0
def test_BenchResult_pypy():
- res = BenchResult("pypy-llvm-39474-faassen-c_richards",
+ res = BenchResult("pypy-llvm-39474-O3-c_richards",
besttime=2.0, numruns=3)
- assert res.executable == "pypy-llvm-39474-faassen-c"
+ assert res.executable == "pypy-llvm-39474-O3-c"
assert res.revision == 39474
assert res.name == "richards"
assert res.numruns == 3
Modified: pypy/dist/pypy/translator/driver.py
==============================================================================
--- pypy/dist/pypy/translator/driver.py (original)
+++ pypy/dist/pypy/translator/driver.py Sun Aug 17 15:15:50 2008
@@ -15,22 +15,6 @@
log = py.log.Producer("translation")
py.log.setconsumer("translation", ansi_log)
-DEFAULTS = {
- 'translation.gc': 'ref',
- 'translation.cc': None,
- 'translation.profopt': None,
-
- 'translation.thread': False, # influences GC policy
-
- 'translation.stackless': False,
- 'translation.debug': True,
- 'translation.insist': False,
- 'translation.backend': 'c',
- 'translation.fork_before': None,
- 'translation.backendopt.raisingop2direct_call' : False,
- 'translation.backendopt.merge_if_blocks': True,
-}
-
def taskdef(taskfunc, deps, title, new_state=None, expected_states=[],
idemp=False, earlycheck=None):
@@ -93,7 +77,7 @@
if config is None:
from pypy.config.pypyoption import get_pypy_config
- config = get_pypy_config(DEFAULTS, translating=True)
+ config = get_pypy_config(translating=True)
self.config = config
if overrides is not None:
self.config.override(overrides)
Modified: pypy/dist/pypy/translator/goal/bench-cronjob.py
==============================================================================
--- pypy/dist/pypy/translator/goal/bench-cronjob.py (original)
+++ pypy/dist/pypy/translator/goal/bench-cronjob.py Sun Aug 17 15:15:50 2008
@@ -181,15 +181,15 @@
if backends == []: #_ prefix means target specific option, # prefix to outcomment
backends = [backend.strip() for backend in """
c
- c--stackless--_faassen
- c--_faassen--_allworkingmodules
- c--thread--gc=hybrid--_faassen
- c--gc=semispace--_faassen
- c--gc=generation--_faassen
- c--gc=hybrid--_faassen
- cli--_faassen
- jvm--_faassen
- jvm--inline-threshold=0--_faassen
+ c--stackless--_O3
+ c--_O3--_allworkingmodules
+ c--thread--gc=hybrid--_O3
+ c--gc=semispace--_O3
+ c--gc=generation--_O3
+ c--gc=hybrid--_O3
+ cli--_O3
+ jvm--_O3
+ jvm--inline-threshold=0--_O3
""".split('\n') if backend.strip() and not backend.strip().startswith('#')]
print time.ctime()
for backend in backends:
Modified: pypy/dist/pypy/translator/goal/targetprologstandalone.py
==============================================================================
--- pypy/dist/pypy/translator/goal/targetprologstandalone.py (original)
+++ pypy/dist/pypy/translator/goal/targetprologstandalone.py Sun Aug 17 15:15:50 2008
@@ -27,9 +27,7 @@
# _____ Define and setup target ___
-def handle_config(config):
- return
- config.translation.stackless = True
+# XXX this should suggest --stackless somehow
def target(driver, args):
driver.exe_name = 'pyrolog-%(backend)s'
Modified: pypy/dist/pypy/translator/goal/targetpypystandalone.py
==============================================================================
--- pypy/dist/pypy/translator/goal/targetpypystandalone.py (original)
+++ pypy/dist/pypy/translator/goal/targetpypystandalone.py Sun Aug 17 15:15:50 2008
@@ -7,6 +7,7 @@
from pypy.interpreter.error import OperationError
from pypy.translator.goal.ann_override import PyPyAnnotatorPolicy
from pypy.config.config import Config, to_optparse, make_dict, SUPPRESS_USAGE
+from pypy.config.config import ConflictConfigError
from pypy.tool.option import make_objspace
from pypy.translator.goal.nanos import setup_nanos
@@ -90,26 +91,29 @@
parserkwargs={'usage': self.usage})
return parser
- def handle_config(self, config):
+ def handle_config(self, config, translateconfig):
+ self.translateconfig = translateconfig
+ # set up the objspace optimizations based on the --opt argument
+ from pypy.config.pypyoption import set_pypy_opt_level
+ set_pypy_opt_level(config, translateconfig.opt)
+
# as of revision 27081, multimethod.py uses the InstallerVersion1 by default
# because it is much faster both to initialize and run on top of CPython.
# The InstallerVersion2 is optimized for making a translator-friendly
# structure for low level backends. However, InstallerVersion1 is still
# preferable for high level backends, so we patch here.
+
from pypy.objspace.std import multimethod
- if config.translation.type_system == 'lltype':
+ if config.objspace.std.multimethods == 'mrd':
assert multimethod.InstallerVersion1.instance_counter == 0,\
'The wrong Installer version has already been instatiated'
multimethod.Installer = multimethod.InstallerVersion2
- else:
+ elif config.objspace.std.multimethods == 'doubledispatch':
# don't rely on the default, set again here
assert multimethod.InstallerVersion2.instance_counter == 0,\
'The wrong Installer version has already been instatiated'
multimethod.Installer = multimethod.InstallerVersion1
- def handle_translate_config(self, translateconfig):
- self.translateconfig = translateconfig
-
def print_help(self, config):
self.opt_parser(config).print_help()
@@ -131,7 +135,16 @@
if config.translation.thread:
config.objspace.usemodules.thread = True
elif config.objspace.usemodules.thread:
- config.translation.thread = True
+ try:
+ config.translation.thread = True
+ except ConflictConfigError:
+ # If --allworkingmodules is given, we reach this point
+ # if threads cannot be enabled (e.g. they conflict with
+ # something else). In this case, we can try setting the
+ # usemodules.thread option to False again. It will
+ # cleanly fail if that option was set to True by the
+ # command-line directly instead of via --allworkingmodules.
+ config.objspace.usemodules.thread = False
if config.translation.stackless:
config.objspace.usemodules._stackless = True
@@ -184,7 +197,7 @@
def interface(self, ns):
for name in ['take_options', 'handle_config', 'print_help', 'target',
- 'handle_translate_config', 'portal',
+ 'portal',
'get_additional_config_options']:
ns[name] = getattr(self, name)
Modified: pypy/dist/pypy/translator/goal/translate.py
==============================================================================
--- pypy/dist/pypy/translator/goal/translate.py (original)
+++ pypy/dist/pypy/translator/goal/translate.py Sun Aug 17 15:15:50 2008
@@ -12,6 +12,8 @@
ArbitraryOption, StrOption, IntOption, Config, \
ChoiceOption, OptHelpFormatter
from pypy.config.translationoption import get_combined_translation_config
+from pypy.config.translationoption import set_opt_level
+from pypy.config.translationoption import OPT_LEVELS, DEFAULT_OPT_LEVEL
GOALS= [
@@ -46,6 +48,9 @@
translate_optiondescr = OptionDescription("translate", "XXX", [
StrOption("targetspec", "XXX", default='targetpypystandalone',
cmdline=None),
+ ChoiceOption("opt",
+ "optimization level", OPT_LEVELS, default=DEFAULT_OPT_LEVEL,
+ cmdline="--opt -O"),
BoolOption("profile",
"cProfile (to debug the speed of the translation process)",
default=False,
@@ -72,17 +77,7 @@
OVERRIDES = {
'translation.debug': False,
- 'translation.insist': False,
-
- 'translation.gc': 'boehm',
'translation.backend': 'c',
- 'translation.stackless': False,
- 'translation.backendopt.raisingop2direct_call' : False,
- 'translation.backendopt.merge_if_blocks': True,
-
- 'translation.cc': None,
- 'translation.profopt': None,
- 'translation.output': None,
}
import py
@@ -162,13 +157,13 @@
existing_config=config,
translating=True)
+ # apply the optimization level settings
+ set_opt_level(config, translateconfig.opt)
+
# let the target modify or prepare itself
# based on the config
if 'handle_config' in targetspec_dic:
- targetspec_dic['handle_config'](config)
-
- if 'handle_translate_config' in targetspec_dic:
- targetspec_dic['handle_translate_config'](translateconfig)
+ targetspec_dic['handle_config'](config, translateconfig)
if translateconfig.help:
opt_parser.print_help()
More information about the Pypy-commit
mailing list