[pypy-svn] r45089 - in pypy/dist/pypy: config interpreter interpreter/pyparser interpreter/pyparser/test interpreter/test module/recparser module/symbol
justas at codespeak.net
justas at codespeak.net
Sat Jul 14 17:36:00 CEST 2007
Author: justas
Date: Sat Jul 14 17:35:59 2007
New Revision: 45089
Modified:
pypy/dist/pypy/config/pypyoption.py
pypy/dist/pypy/interpreter/pycompiler.py
pypy/dist/pypy/interpreter/pyparser/astbuilder.py
pypy/dist/pypy/interpreter/pyparser/pythonparse.py
pypy/dist/pypy/interpreter/pyparser/pythonutil.py
pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder.py
pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder_future.py
pypy/dist/pypy/interpreter/pyparser/test/test_parser.py
pypy/dist/pypy/interpreter/pyparser/test/test_pytokenizer.py
pypy/dist/pypy/interpreter/pyparser/test/test_samples.py
pypy/dist/pypy/interpreter/test/test_compiler.py
pypy/dist/pypy/module/recparser/compat.py
pypy/dist/pypy/module/recparser/pyparser.py
pypy/dist/pypy/module/symbol/__init__.py
Log:
(Arlo, Justas)
Introduce --pyversion option to select the parser/compiler grammar
Defaults to 2.4, also accepts 2.3 and 2.5a.
Changed a bunch of imports to use the selected grammar.
TODO: pypy/interpreter/pyparser/symbol.py uses a fixed list of symbols, and it shouldn't
Modified: pypy/dist/pypy/config/pypyoption.py
==============================================================================
--- pypy/dist/pypy/config/pypyoption.py (original)
+++ pypy/dist/pypy/config/pypyoption.py Sat Jul 14 17:35:59 2007
@@ -60,6 +60,10 @@
["cpython", "ast"], "ast",
cmdline='--compiler'),
+ ChoiceOption("pyversion", "which grammar to use for app-level code",
+ ["2.3", "2.4", "2.5a"], "2.4",
+ cmdline='--pyversion'),
+
OptionDescription("opcodes", "opcodes to enable in the interpreter", [
BoolOption("CALL_LIKELY_BUILTIN", "emit a special bytecode for likely calls to builtin functions",
default=False,
Modified: pypy/dist/pypy/interpreter/pycompiler.py
==============================================================================
--- pypy/dist/pypy/interpreter/pycompiler.py (original)
+++ pypy/dist/pypy/interpreter/pycompiler.py Sat Jul 14 17:35:59 2007
@@ -5,8 +5,6 @@
from codeop import PyCF_DONT_IMPLY_DEDENT
from pypy.interpreter.error import OperationError
-ENABLE_GRAMMAR_VERSION = "2.4"
-
class AbstractCompiler:
"""Abstract base class for a bytecode compiler."""
@@ -201,13 +199,13 @@
of incomplete inputs (e.g. we shouldn't re-compile from sracth
the whole source after having only added a new '\n')
"""
- def __init__(self, space, grammar_version=ENABLE_GRAMMAR_VERSION):
- from pyparser.pythonparse import PYTHON_PARSER
+ def __init__(self, space, override_version=None):
+
+ from pyparser.pythonparse import make_pyparser
PyCodeCompiler.__init__(self, space)
- self.parser = PYTHON_PARSER
+ self.grammar_version = override_version or space.config.objspace.pyversion
+ self.parser = make_pyparser(self.grammar_version)
self.additional_rules = {}
- self.grammar_version = grammar_version
-
def compile(self, source, filename, mode, flags):
from pyparser.error import SyntaxError
@@ -223,8 +221,7 @@
flags |= stdlib___future__.generators.compiler_flag # always on (2.2 compat)
space = self.space
try:
- builder = AstBuilder(self.parser, space=space,
- grammar_version=self.grammar_version)
+ builder = AstBuilder(self.parser, self.grammar_version, space=space)
for rulename, buildfunc in self.additional_rules.iteritems():
assert isinstance(buildfunc, Function)
builder.user_build_rules[rulename] = buildfunc
Modified: pypy/dist/pypy/interpreter/pyparser/astbuilder.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/astbuilder.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/astbuilder.py Sat Jul 14 17:35:59 2007
@@ -11,7 +11,6 @@
#import pypy.interpreter.pyparser.pytoken as tok
from pypy.interpreter.pyparser.error import SyntaxError
from pypy.interpreter.pyparser.parsestring import parsestr
-from pypy.interpreter.pycompiler import ENABLE_GRAMMAR_VERSION
from pypy.interpreter.gateway import interp2app
from asthelper import *
@@ -1102,8 +1101,7 @@
class AstBuilder(BaseGrammarBuilder):
"""A builder that directly produce the AST"""
- def __init__(self, parser, debug=0, space=None,
- grammar_version=ENABLE_GRAMMAR_VERSION):
+ def __init__(self, parser, grammar_version, debug=0, space=None):
BaseGrammarBuilder.__init__(self, parser, debug)
self.rule_stack = []
self.space = space
Modified: pypy/dist/pypy/interpreter/pyparser/pythonparse.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/pythonparse.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/pythonparse.py Sat Jul 14 17:35:59 2007
@@ -1,6 +1,5 @@
#!/usr/bin/env python
-"""This module loads the python Grammar (2.3 or 2.4) and builds
-the parser for this grammar in the global PYTHON_PARSER
+"""This module loads the python Grammar (2.3, 2.4 or 2.5)
helper functions are provided that use the grammar to parse
using file_input, single_input and eval_input targets
@@ -9,7 +8,6 @@
import os
from pypy.interpreter.error import OperationError, debug_print
from pypy.interpreter import gateway
-from pypy.interpreter.pycompiler import ENABLE_GRAMMAR_VERSION
from pypy.interpreter.pyparser.error import SyntaxError
from pypy.interpreter.pyparser.pythonlexer import Source, match_encoding_declaration
from pypy.interpreter.astcompiler.consts import CO_FUTURE_WITH_STATEMENT
@@ -166,11 +164,11 @@
# recompute first sets
self.build_first_sets()
-def make_pyparser(version=ENABLE_GRAMMAR_VERSION):
+
+def make_pyparser(version):
parser = PythonParser()
return build_parser_for_version(version, parser=parser)
-PYTHON_PARSER = make_pyparser()
def translation_target(grammardef):
parser = PythonParser() # predefined_symbols=symbol.sym_name)
@@ -194,7 +192,7 @@
def grammar_rules( space ):
w_rules = space.newdict()
- parser = make_pyparser()
+ parser = make_pyparser(space.config.objspace.pyversion)
for key, value in parser.rules.iteritems():
space.setitem(w_rules, space.wrap(key), space.wrap(value))
return w_rules
Modified: pypy/dist/pypy/interpreter/pyparser/pythonutil.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/pythonutil.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/pythonutil.py Sat Jul 14 17:35:59 2007
@@ -23,7 +23,7 @@
i = filename.rfind(os.sep) + 1
assert i >= 0
return filename[:i]
-
+
def get_grammar_file(version):
"""returns the python grammar corresponding to our CPython version"""
@@ -66,7 +66,7 @@
return build_parser(gramfile, parser)
-## XXX: the below code should probably go elsewhere
+## XXX: the below code should probably go elsewhere
## convenience functions for computing AST objects using recparser
def ast_from_input(input, mode, transformer, parser):
@@ -88,7 +88,6 @@
def pypy_parse(source, mode='exec', lineno=False):
from pypy.interpreter.pyparser.pythonparse import PythonParser, make_pyparser
- from pypy.interpreter.pyparser.astbuilder import AstBuilder
# parser = build_parser_for_version("2.4", PythonParser())
parser = make_pyparser('stable')
builder = TupleBuilder(parser)
@@ -100,10 +99,10 @@
from pypy.interpreter.pyparser.pythonparse import PythonParser, make_pyparser
from pypy.interpreter.pyparser.astbuilder import AstBuilder
parser = make_pyparser(version)
- builder = AstBuilder(parser, space=space)
+ builder = AstBuilder(parser, version, space=space)
parser.parse_source(source, mode, builder)
return builder.rule_stack[-1]
-
+
## convenience functions around CPython's parser functions
def python_parsefile(filename, lineno=False):
Modified: pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder.py Sat Jul 14 17:35:59 2007
@@ -174,17 +174,17 @@
"\t # hello\n": "Module(None, Stmt([]))",
}
-
# Create parser from Grammar_stable, not current grammar.
stable_parser = pythonparse.make_pyparser('stable')
-python_parser = pythonparse.make_pyparser() # 'native') # 2.5a')
def tuple_parse_expr(expr, target='single'):
t = Transformer("dummyfile")
return ast_from_input(expr, target, t, stable_parser)
def source2ast(source, mode, space=FakeSpace()):
- builder = AstBuilder(space=space, parser=python_parser)
+ version = '2.4'
+ python_parser = pythonparse.make_pyparser(version)
+ builder = AstBuilder(python_parser, version, space=space)
python_parser.parse_source(source, mode, builder)
return builder.rule_stack[-1]
Modified: pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder_future.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder_future.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder_future.py Sat Jul 14 17:35:59 2007
@@ -63,8 +63,7 @@
def setup_method(self, method):
self.builder = astbuilder.AstBuilder(
- self.parser, space=FakeSpaceForFeatureLookup(),
- grammar_version="2.5a")
+ self.parser, "2.5a", space=FakeSpaceForFeatureLookup())
def test_future_rules(self):
assert (self.builder.build_rules['future_import_feature'] is
Modified: pypy/dist/pypy/interpreter/pyparser/test/test_parser.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/test/test_parser.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/test/test_parser.py Sat Jul 14 17:35:59 2007
@@ -78,12 +78,15 @@
class TestFuture(object):
+ _grammar_ver = '2.5a'
+
def setup_class(self):
from pypy.interpreter.pyparser.pythonparse import make_pyparser
- self.parser = make_pyparser('2.5a')
+ self.parser = make_pyparser(self._grammar_ver)
def setup_method(self, method):
- self.builder = MockBuilder(self.parser, space=FakeSpace())
+ self.builder = MockBuilder(self.parser, self._grammar_ver,
+ space=FakeSpace())
def check_parse_mode(self, tst, expected, mode):
self.parser.parse_source(tst, mode, self.builder)
Modified: pypy/dist/pypy/interpreter/pyparser/test/test_pytokenizer.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/test/test_pytokenizer.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/test/test_pytokenizer.py Sat Jul 14 17:35:59 2007
@@ -3,7 +3,7 @@
from pypy.interpreter.pyparser.grammar import Token, GrammarElement
from pypy.interpreter.pyparser.pythonparse import make_pyparser
-P = make_pyparser()
+P = make_pyparser('2.4')
EQUAL = P.tokens['EQUAL']
ENDMARKER = P.tokens['ENDMARKER']
Modified: pypy/dist/pypy/interpreter/pyparser/test/test_samples.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/test/test_samples.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/test/test_samples.py Sat Jul 14 17:35:59 2007
@@ -173,7 +173,7 @@
for snippet in snippets:
yield check_parse_input, snippet, 'single'
-
+
def test_bad_inputs():
inputs = ['x = (', 'x = (\n', 'x = (\n\n']
for inp in inputs:
Modified: pypy/dist/pypy/interpreter/test/test_compiler.py
==============================================================================
--- pypy/dist/pypy/interpreter/test/test_compiler.py (original)
+++ pypy/dist/pypy/interpreter/test/test_compiler.py Sat Jul 14 17:35:59 2007
@@ -570,12 +570,12 @@
class TestPythonAstCompiler(BaseTestCompiler):
def setup_method(self, method):
- self.compiler = PythonAstCompiler(self.space, grammar_version="2.4")
+ self.compiler = PythonAstCompiler(self.space, "2.4")
class TestPythonAstCompiler_25_grammar:
def setup_method(self, method):
- self.compiler = PythonAstCompiler(self.space, grammar_version="2.5a")
+ self.compiler = PythonAstCompiler(self.space, "2.5a")
def test_from_future_import(self):
source = """from __future__ import with_statement
Modified: pypy/dist/pypy/module/recparser/compat.py
==============================================================================
--- pypy/dist/pypy/module/recparser/compat.py (original)
+++ pypy/dist/pypy/module/recparser/compat.py Sat Jul 14 17:35:59 2007
@@ -6,12 +6,20 @@
import symbol # XXX use PYTHON_PARSER.symbols ?
from compiler import transformer, compile as pycompile
-PYTHON_PARSER = make_pyparser()
+_PARSER = None
+
+def get_parser():
+ if not _PARSER:
+ from pypy.config.pypyoption import get_pypy_config
+ config = get_pypy_config(translating=False)
+ _PARSER = make_pyparser(config.objspace.pyversion)
+ return _PARSER
def suite( source ):
strings = [line+'\n' for line in source.split('\n')]
- builder = TupleBuilder(PYTHON_PARSER)
- PYTHON_PARSER.parse_source(source, 'exec', builder)
+ parser = get_parser()
+ builder = TupleBuilder(parser)
+ parser.parse_source(source, 'exec', builder)
nested_tuples = builder.stack[-1].as_tuple()
if builder.source_encoding is not None:
return (symbol.encoding_decl, nested_tuples, builder.source_encoding)
@@ -21,8 +29,9 @@
def expr( source ):
strings = [line+'\n' for line in source.split('\n')]
- builder = TupleBuilder(PYTHON_PARSER)
- PYTHON_PARSER.parse_source(source, 'eval', builder)
+ parser = get_parser()
+ builder = TupleBuilder(parser)
+ parser.parse_source(source, 'eval', builder)
nested_tuples = builder.stack[-1].as_tuple()
if builder.source_encoding is not None:
return (symbol.encoding_decl, nested_tuples, builder.source_encoding)
Modified: pypy/dist/pypy/module/recparser/pyparser.py
==============================================================================
--- pypy/dist/pypy/module/recparser/pyparser.py (original)
+++ pypy/dist/pypy/module/recparser/pyparser.py Sat Jul 14 17:35:59 2007
@@ -14,7 +14,7 @@
from pypy.interpreter.argument import Arguments
# backward compat (temp)
-PYTHON_PARSER = make_pyparser()
+
__all__ = [ "ASTType", "STType", "suite", "expr" ]
@@ -151,10 +151,11 @@
)
def parse_python_source(space, source, mode):
- builder = grammar.BaseGrammarBuilder(debug=False, parser=PYTHON_PARSER)
+ parser = make_pyparser(space.config.objspace.pyversion)
+ builder = grammar.BaseGrammarBuilder(debug=False, parser=parser)
builder.space = space
try:
- PYTHON_PARSER.parse_source(source, mode, builder )
+ parser.parse_source(source, mode, builder)
return builder.stack[-1]
except SyntaxError, e:
raise OperationError(space.w_SyntaxError,
Modified: pypy/dist/pypy/module/symbol/__init__.py
==============================================================================
--- pypy/dist/pypy/module/symbol/__init__.py (original)
+++ pypy/dist/pypy/module/symbol/__init__.py Sat Jul 14 17:35:59 2007
@@ -4,7 +4,7 @@
at run-time.
"""
-from pypy.interpreter.mixedmodule import MixedModule
+from pypy.interpreter.mixedmodule import MixedModule
# Forward imports so they run at startup time
import pypy.interpreter.pyparser.pythonlexer
@@ -16,16 +16,28 @@
appleveldefs = {}
interpleveldefs = {} # see below
+ def __init__(self, space, w_name):
+ MixedModule.__init__(self, space, w_name)
+ _init_symbols(space.config.objspace.pyversion)
+
# Export the values from our custom symbol module.
# Skip negative values (the corresponding symbols are not visible in
# pure Python).
-from pypy.interpreter.pyparser.pythonparse import make_pyparser
-parser = make_pyparser()
-
sym_name = {}
-for name, val in parser.symbols.items():
- if val >= 0:
- Module.interpleveldefs[name] = 'space.wrap(%d)' % val
- sym_name[val] = name
-Module.interpleveldefs['sym_name'] = 'space.wrap(%r)' % (sym_name,)
+
+def _init_symbols(grammar_version):
+ global sym_name
+
+ sym_name = {}
+ from pypy.interpreter.pyparser.pythonparse import make_pyparser
+ parser = make_pyparser(grammar_version)
+
+ for name, val in parser.symbols.items():
+ if val >= 0:
+ Module.interpleveldefs[name] = 'space.wrap(%d)' % val
+ sym_name[val] = name
+ Module.interpleveldefs['sym_name'] = 'space.wrap(%r)' % (sym_name,)
+
+# This is very evil
+_init_symbols('2.4')
More information about the Pypy-commit
mailing list