[pypy-svn] r35754 - in pypy/branch/ast-experiments/pypy: interpreter interpreter/pyparser interpreter/pyparser/test interpreter/stablecompiler module/recparser
adim at codespeak.net
adim at codespeak.net
Thu Dec 14 17:28:11 CET 2006
Author: adim
Date: Thu Dec 14 17:28:05 2006
New Revision: 35754
Added:
pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/expressions.py
Modified:
pypy/branch/ast-experiments/pypy/interpreter/pycompiler.py
pypy/branch/ast-experiments/pypy/interpreter/pyparser/astbuilder.py
pypy/branch/ast-experiments/pypy/interpreter/pyparser/ebnfparse.py
pypy/branch/ast-experiments/pypy/interpreter/pyparser/pythonparse.py
pypy/branch/ast-experiments/pypy/interpreter/pyparser/pythonutil.py
pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/test_astbuilder.py
pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/test_astcompiler.py
pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/test_samples.py
pypy/branch/ast-experiments/pypy/interpreter/stablecompiler/transformer.py
pypy/branch/ast-experiments/pypy/module/recparser/pyparser.py
Log:
small pyparser refactorings
- removed some unused code
- eased a bit python parser creation (build_parser / get_pyparser)
Modified: pypy/branch/ast-experiments/pypy/interpreter/pycompiler.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/interpreter/pycompiler.py (original)
+++ pypy/branch/ast-experiments/pypy/interpreter/pycompiler.py Thu Dec 14 17:28:05 2006
@@ -205,15 +205,15 @@
from pypy.interpreter.astcompiler.pycodegen import InteractiveCodeGenerator
from pypy.interpreter.astcompiler.pycodegen import ExpressionCodeGenerator
from pypy.interpreter.astcompiler.ast import Node
- from pyparser.pythonutil import AstBuilder, PYTHON_PARSER, TARGET_DICT
+ from pyparser.astbuilder import AstBuilder
+ from pyparser.pythonparse import PYTHON_PARSER
from pypy.interpreter.pycode import PyCode
flags |= stdlib___future__.generators.compiler_flag # always on (2.2 compat)
space = self.space
try:
builder = AstBuilder(space=space)
- target_rule = TARGET_DICT[mode]
- PYTHON_PARSER.parse_source(source, target_rule, builder, flags)
+ PYTHON_PARSER.parse_source(source, mode, builder, flags)
ast_tree = builder.rule_stack[-1]
encoding = builder.source_encoding
except SyntaxError, e:
Modified: pypy/branch/ast-experiments/pypy/interpreter/pyparser/astbuilder.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/interpreter/pyparser/astbuilder.py (original)
+++ pypy/branch/ast-experiments/pypy/interpreter/pyparser/astbuilder.py Thu Dec 14 17:28:05 2006
@@ -10,10 +10,6 @@
from pypy.interpreter.pyparser.error import SyntaxError
from pypy.interpreter.pyparser.parsestring import parsestr
-sym = pythonparse.PYTHON_PARSER.symbols
-
-DEBUG_MODE = 0
-
# XXX : use builder.parser instead
sym = pythonparse.PYTHON_PARSER.symbols
rsym = pythonparse.PYTHON_PARSER.symbol_repr
@@ -1698,8 +1694,9 @@
if self.with_enabled:
return
self.with_enabled = True
- self.keywords.update({'with':None, 'as': None})
-
+ # XXX
+ # self.keywords.update({'with':None, 'as': None})
+
def context(self):
return AstBuilderContext(self.rule_stack)
Modified: pypy/branch/ast-experiments/pypy/interpreter/pyparser/ebnfparse.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/interpreter/pyparser/ebnfparse.py (original)
+++ pypy/branch/ast-experiments/pypy/interpreter/pyparser/ebnfparse.py Thu Dec 14 17:28:05 2006
@@ -5,7 +5,6 @@
from ebnflexer import GrammarSource
import ebnfgrammar
from ebnfgrammar import GRAMMAR_GRAMMAR
-from syntaxtree import AbstractSyntaxVisitor
ORDA = ord("A")
@@ -21,14 +20,14 @@
return False
v = ord(name[0])
if not (ORDA <= v <= ORDZ or
- ORDa <= v <= ORDz or v == ORD_ ):
+ ORDa <= v <= ORDz or v == ORD_):
return False
for c in name:
v = ord(c)
if not (ORDA <= v <= ORDZ or
ORDa <= v <= ORDz or
ORD0 <= v <= ORD9 or
- v == ORD_ ):
+ v == ORD_):
return False
return True
@@ -41,21 +40,16 @@
'%', '<<', '//', '\\', '', '\n\\)', '\\(', ';', ':',
'@', '\\[', '\\]', '`', '\\{', '\\}']
-
-
-
TERMINALS = [
'NAME', 'NUMBER', 'STRING', 'NEWLINE', 'ENDMARKER',
'INDENT', 'DEDENT' ]
-## Grammar Visitors ##################################################
# FIXME: parsertools.py ? parser/__init__.py ?
-
class NameToken(Token):
"""A token that is not a keyword"""
- def __init__(self, parser, keywords=None ):
- Token.__init__(self, parser, parser.tokens['NAME'] )
+ def __init__(self, parser, keywords=None):
+ Token.__init__(self, parser, parser.tokens['NAME'])
self.keywords = keywords
def match(self, source, builder, level=0):
@@ -69,7 +63,6 @@
else:
# error unknown or negative integer
"""
-
ctx = source.context()
tk = source.next()
if tk.codename == self.codename:
@@ -81,6 +74,7 @@
source.restore( ctx )
return 0
+
def match_token(self, builder, other):
"""special case of match token for tokens which are really keywords
"""
@@ -97,11 +91,17 @@
return True
+class EBNFBuilderContext(AbstractContext):
+ def __init__(self, stackpos, seqcounts, altcounts):
+ self.stackpos = stackpos
+ self.seqcounts = seqcounts
+ self.altcounts = altcounts
+
class EBNFBuilder(AbstractBuilder):
"""Build a grammar tree"""
- def __init__(self, gram_parser, dest_parser ):
- AbstractBuilder.__init__(self, dest_parser )
+ def __init__(self, gram_parser, dest_parser):
+ AbstractBuilder.__init__(self, dest_parser)
self.gram = gram_parser
self.rule_stack = []
self.seqcounts = [] # number of items in the current sequence
@@ -114,8 +114,17 @@
self.tokens = {}
self.keywords = []
NAME = dest_parser.add_token('NAME')
+ # NAME = dest_parser.tokens['NAME']
self.tokens[NAME] = NameToken(dest_parser, keywords=self.keywords)
+ def context(self):
+ return EBNFBuilderContext(len(self.rule_stack), self.seqcounts, self.altcounts)
+
+ def restore(self, ctx):
+ del self.rule_stack[ctx.stackpos:]
+ self.seqcounts = ctx.seqcounts
+ self.altcounts = ctx.altcounts
+
def new_symbol(self):
"""Allocate and return a new (anonymous) grammar symbol whose
name is based on the current grammar rule being parsed"""
@@ -169,14 +178,6 @@
self.parser.root_rules[codename] = proxy
return proxy
- def context(self):
- """Return an opaque context object"""
- return None
-
- def restore(self, ctx):
- """Accept an opaque context object"""
- assert False, "Not supported"
-
def alternative(self, rule, source):
return True
@@ -190,7 +191,6 @@
def sequence(self, rule, source, elts_number):
_rule = rule.codename
if _rule == self.gram.sequence:
-# print " -sequence", self.curaltcount, self.curseqcount
if self.curseqcount==1:
self.curseqcount = 0
self.curaltcount += 1
@@ -201,7 +201,6 @@
self.curseqcount = 0
self.curaltcount += 1
elif _rule == self.gram.alternative:
-# print " -alternative", self.curaltcount, self.curseqcount
if self.curaltcount == 1:
self.curaltcount = 0
return True
@@ -210,17 +209,14 @@
self.rule_stack.append( new_rule )
self.curaltcount = 0
elif _rule == self.gram.group:
-# print " -group", self.curaltcount, self.curseqcount
self.curseqcount += 1
elif _rule == self.gram.option:
-# print " -option", self.curaltcount, self.curseqcount
# pops the last alternative
rules = self.pop_rules( 1 )
new_rule = self.parser.KleeneStar( self.new_symbol(), _min=0, _max=1, rule=rules[0] )
self.rule_stack.append( new_rule )
self.curseqcount += 1
elif _rule == self.gram.rule:
-# print " -rule", self.curaltcount, self.curseqcount
assert len(self.rule_stack)==1
old_rule = self.rule_stack[0]
del self.rule_stack[0]
@@ -235,7 +231,6 @@
return True
def token(self, name, value, source):
-# print "token", name, value
if name == self.gram.TOK_STRING:
self.handle_TOK_STRING( name, value )
self.curseqcount += 1
@@ -292,35 +287,16 @@
self.rule_stack.append(tok)
-def parse_grammar_text( parser, txt):
- """parses a grammar input
-
- stream : file-like object representing the grammar to parse
- """
- source = GrammarSource( GRAMMAR_GRAMMAR, txt)
- builder = EBNFBuilder(GRAMMAR_GRAMMAR, dest_parser=parser )
- result = GRAMMAR_GRAMMAR.root_rules['grammar'].match(source, builder)
- builder.resolve_rules()
- parser.build_first_sets()
- return parser
-
-def target_parse_grammar_text(txt):
- vis = parse_grammar_text(txt)
- # do nothing
- return None
-
-def main_build():
- from pprint import pprint
- grambuild = parse_grammar(file('data/Grammar2.3'))
- for i,r in enumerate(grambuild.items):
- print "% 3d : %s" % (i, r)
- pprint(grambuild.terminals.keys())
- pprint(grambuild.tokens)
- print "|".join(grambuild.tokens.keys() )
-
-def main_build():
- import sys
- return parse_grammar_text( file(sys.argv[-1]).read() )
-if __name__ == "__main__":
- result = main_build()
+## # XXX: parser method ?
+## def parse_grammar_text(parser, txt):
+## """parses a grammar input
+
+## txt : the grammar definition
+## """
+## source = GrammarSource(GRAMMAR_GRAMMAR, txt)
+## builder = EBNFBuilder(GRAMMAR_GRAMMAR, dest_parser=parser)
+## result = GRAMMAR_GRAMMAR.root_rules['grammar'].match(source, builder)
+## builder.resolve_rules()
+## parser.build_first_sets()
+## return parser
Modified: pypy/branch/ast-experiments/pypy/interpreter/pyparser/pythonparse.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/interpreter/pyparser/pythonparse.py (original)
+++ pypy/branch/ast-experiments/pypy/interpreter/pyparser/pythonparse.py Thu Dec 14 17:28:05 2006
@@ -17,64 +17,18 @@
import pypy.interpreter.pyparser.pytoken as pytoken
import pypy.interpreter.pyparser.ebnfparse as ebnfparse
import pypy.interpreter.pyparser.grammar as grammar
+from pypy.interpreter.pyparser.pythonutil import build_parser_for_version
-try:
- from pypy.interpreter.pyparser import symbol
-except ImportError:
- # for standalone testing
- import symbol
+# try:
+from pypy.interpreter.pyparser import symbol
+# except ImportError:
+# # for standalone testing
+# import symbol
from codeop import PyCF_DONT_IMPLY_DEDENT
-class AlternateGrammarException(Exception):
- pass
-
-class PythonParser(grammar.Parser):
- """Wrapper class for python grammar"""
- def __init__(self):
- grammar.Parser.__init__(self)
- # XXX (adim): this is trunk's keyword management
- # self.with_grammar = None
- # self.keywords = dict.fromkeys(grammar_builder.keywords)
- # # Only when with_statement is enabled
- # self.keywords.pop('with', None)
- # self.keywords.pop('as', None)
-
- def parse_source(self, textsrc, goal, builder, flags=0):
- """Parse a python source according to goal"""
- # Detect source encoding.
- if textsrc[:3] == '\xEF\xBB\xBF':
- textsrc = textsrc[3:]
- enc = 'utf-8'
- else:
- enc = _normalize_encoding(_check_for_encoding(textsrc))
- if enc is not None and enc not in ('utf-8', 'iso-8859-1'):
- textsrc = recode_to_utf8(builder.space, textsrc, enc)
-
- lines = [line + '\n' for line in textsrc.split('\n')]
- builder.source_encoding = enc
- if len(textsrc) and textsrc[-1] == '\n':
- lines.pop()
- flags &= ~PyCF_DONT_IMPLY_DEDENT
- return self.parse_lines(lines, goal, builder, flags)
-
-
- def parse_lines(self, lines, goal, builder, flags=0):
- # XXX (adim): this is trunk's keyword management
- # builder.keywords = self.keywords.copy()
- # if flags & CO_FUTURE_WITH_STATEMENT:
- # builder.enable_with()
- goalnumber = self.symbols[goal]
- target = self.root_rules[goalnumber]
- src = Source(self, lines, flags)
-
- if not target.match(src, builder):
- line, lineno = src.debug()
- # XXX needs better error messages
- raise SyntaxError("invalid syntax", lineno, -1, line)
- # return None
- return builder
+## files encoding management ############################################
_recode_to_utf8 = gateway.applevel(r'''
def _recode_to_utf8(text, encoding):
return unicode(text, encoding).encode("utf-8")
@@ -115,6 +69,7 @@
return _check_line_for_encoding(s[eol + 1:])
return _check_line_for_encoding(s[eol + 1:eol2])
+
def _check_line_for_encoding(line):
"""returns the declared encoding or None"""
i = 0
@@ -125,138 +80,99 @@
return None
return match_encoding_declaration(line[i:])
-PYTHON_VERSION = ".".join([str(i) for i in sys.version_info[:2]])
-def get_grammar_file( version ):
- """returns the python grammar corresponding to our CPython version"""
- if version == "native":
- _ver = PYTHON_VERSION
- elif version == "stable":
- _ver = "_stablecompiler"
- elif version in ("2.3","2.4","2.5a"):
- _ver = version
- return os.path.join( os.path.dirname(__file__), "data", "Grammar" + _ver ), _ver
-# unfortunately the command line options are not parsed yet
-PYTHON_GRAMMAR, PYPY_VERSION = get_grammar_file( Options.version )
+## Python Source Parser ###################################################
+class AlternateGrammarException(Exception):
+ pass
+class PythonParser(grammar.Parser):
+ """Wrapper class for python grammar"""
+ targets = {
+ 'eval' : "eval_input",
+ 'single' : "single_input",
+ 'exec' : "file_input",
+ }
+
+ def __init__(self, predefined_symbols=None):
+ grammar.Parser.__init__(self)
+ pytoken.setup_tokens(self)
+ if predefined_symbols:
+ self.load_symbols(predefined_symbols)
+
+ # XXX (adim): this is trunk's keyword management
+ # self.with_grammar = None
+ # self.keywords = dict.fromkeys(grammar_builder.keywords)
+ # # Only when with_statement is enabled
+ # self.keywords.pop('with', None)
+ # self.keywords.pop('as', None)
+
+ def parse_source(self, textsrc, mode, builder, flags=0):
+ """Parse a python source according to goal"""
+ goal = self.targets[mode]
+ # Detect source encoding.
+ if textsrc[:3] == '\xEF\xBB\xBF':
+ textsrc = textsrc[3:]
+ enc = 'utf-8'
+ else:
+ enc = _normalize_encoding(_check_for_encoding(textsrc))
+ if enc is not None and enc not in ('utf-8', 'iso-8859-1'):
+ textsrc = recode_to_utf8(builder.space, textsrc, enc)
-def load_python_grammar(fname):
- """Loads the grammar using the 'dynamic' rpython parser"""
- _grammar_file = file(fname)
- parser = PYTHON_PARSER
- # populate symbols
- ebnfparse.parse_grammar_text( parser, file(fname).read() )
- return parser
-
-debug_print( "Loading grammar %s" % PYTHON_GRAMMAR )
-PYTHON_PARSER = PythonParser()
-# print "sym_name ===>", symbol.sym_name
-PYTHON_PARSER.load_symbols( symbol.sym_name )
-# debug_print( "Loading grammar %s" % PYTHON_GRAMMAR )
-# PYTHON_PARSER = python_grammar( PYTHON_GRAMMAR)
-#PYTHON_PARSER.with_grammar = python_grammar( PYTHON_GRAMMAR + '_with' )
-
-def reload_grammar(version):
- """helper function to test with pypy different grammars"""
- global PYTHON_GRAMMAR, PYTHON_PARSER, PYPY_VERSION
- PYTHON_GRAMMAR, PYPY_VERSION = get_grammar_file( version )
- debug_print( "Reloading grammar %s" % PYTHON_GRAMMAR )
- PYTHON_PARSER = python_grammar( PYTHON_GRAMMAR )
-
-def parse_file_input(pyf, gram, builder ):
- """Parse a python file"""
- return gram.parse_source( pyf.read(), "file_input", builder )
-
-def parse_single_input(textsrc, gram, builder ):
- """Parse a python single statement"""
- return gram.parse_source( textsrc, "single_input", builder )
-
-def parse_eval_input(textsrc, gram, builder):
- """Parse a python expression"""
- return gram.parse_source( textsrc, "eval_input", builder )
-
-def grammar_rules( space ):
- w_rules = space.newdict()
- for key, value in PYTHON_PARSER.rules.iteritems():
- space.setitem(w_rules, space.wrap(key), space.wrap(value))
- return w_rules
-
-def dot_node( gen, rule_name, rule, symbols, edges, count ):
- from pypy.interpreter.pyparser.grammar import KleeneStar, Sequence, Alternative, Token
- subrule_name = symbols.get( rule.codename, rule.codename )
- label = None
- if not subrule_name.startswith(":"+rule_name):
- node_name = rule_name + "_ext_" + str(count[0])
- count[0]+=1
- label = subrule_name
- gen.emit_node( node_name, shape="parallelogram", label=subrule_name )
- edges.append( (node_name, subrule_name) )
- return node_name
- subrule_name = subrule_name.replace(":","_")
- if isinstance(rule, KleeneStar):
- node = dot_node( gen, rule_name, rule.args[0], symbols, edges, count )
- gen.emit_edge( node, node, label=rule.get_star(), style='solid' )
- return node
- elif isinstance(rule, Sequence):
- gen.enter_subgraph( subrule_name )
- first_node = None
- for n in rule.args:
- node_name = dot_node( gen, rule_name, n, symbols, edges, count )
- if first_node:
- gen.emit_edge( first_node, node_name, style='solid' )
- first_node = node_name
- gen.leave_subgraph()
- return subrule_name
- elif isinstance(rule, Alternative):
- gen.enter_subgraph( subrule_name )
- for n in rule.args:
- node_name = dot_node( gen, rule_name, n, symbols, edges, count )
- gen.leave_subgraph()
- return subrule_name
- elif isinstance(rule, Token):
- node_name = rule_name + "_ext_" + str(count[0])
- count[0]+=1
- gen.emit_node( node_name, shape='box', label=rule.display( 0, symbols ) )
- return node_name
- raise RuntimeError("Unknown node type")
-
-def gen_grammar_dot( name, root_rules, rules, symbols ):
- """Quick hack to output a dot graph of the grammar"""
- from pypy.translator.tool.make_dot import DotGen
- gen = DotGen(name)
- edges = []
- count = [0]
- for r in root_rules:
- rule_name = symbols.get( r.codename, r.codename )
- gen.emit_node( rule_name, shape='hexagon', label=r.display(0,symbols) )
- for rule in r.args:
- node = dot_node( gen, rule_name, rule, symbols, edges, count )
- gen.emit_edge( rule_name, node, style='solid' )
- for left, right in edges:
- gen.emit_edge( left, right, style='solid' )
- gen.generate(target='ps')
-
-
-def parse_grammar(space, w_src):
- """Loads the grammar using the 'dynamic' rpython parser"""
- src = space.str_w( w_src )
- ebnfbuilder = ebnfparse.parse_grammar_text( src )
- ebnfbuilder.resolve_rules()
- grammar.build_first_sets(ebnfbuilder.all_rules)
- return space.wrap( ebnfbuilder.root_rules )
-
-## debug_print( "Loading grammar %s" % PYTHON_GRAMMAR )
-## PYTHON_PARSER = PythonParser()
-## PYTHON_PARSER.load_symbols( symbol.sym_name )
-pytoken.setup_tokens( PYTHON_PARSER )
-load_python_grammar( PYTHON_GRAMMAR )
-#PYTHON_PARSER.with_grammar = python_grammar( PYTHON_GRAMMAR + '_with' )
-
-def make_rule( space, w_rule ):
- rule = space.str_w( w_rule )
-
-if __name__=="__main__":
- symbols = {}
- symbols.update( pytoken.tok_name )
- symbols.update( pysymbol._cpython_symbols.sym_name )
- gen_grammar_dot("grammar", PYTHON_PARSER.rules.values(), PYTHON_PARSER.items, symbols )
+ lines = [line + '\n' for line in textsrc.split('\n')]
+ builder.source_encoding = enc
+ if len(textsrc) and textsrc[-1] == '\n':
+ lines.pop()
+ flags &= ~PyCF_DONT_IMPLY_DEDENT
+ return self.parse_lines(lines, goal, builder, flags)
+
+
+ def parse_lines(self, lines, goal, builder, flags=0):
+ # XXX (adim): this is trunk's keyword management
+ # builder.keywords = self.keywords.copy()
+ # if flags & CO_FUTURE_WITH_STATEMENT:
+ # builder.enable_with()
+ goalnumber = self.symbols[goal]
+ target = self.root_rules[goalnumber]
+ src = Source(self, lines, flags)
+
+ if not target.match(src, builder):
+ line, lineno = src.debug()
+ # XXX needs better error messages
+ raise SyntaxError("invalid syntax", lineno, -1, line)
+ # return None
+ return builder
+
+
+## def eval(self, source, builder=None):
+## if builder is None:
+## builder = self.builder
+## rule = self.root_rules['eval_input']
+## rule.match(source, builder)
+
+
+def get_pyparser_for_version(version):
+ parser = PythonParser(predefined_symbols=symbol.sym_name)
+ return build_parser_for_version(version, parser=parser)
+
+
+# unfortunately the command line options are not parsed yet
+debug_print( "Loading grammar %s" % Options.version )
+# PYTHON_PARSER = get_pyparser_for_version(Options.version)
+PYTHON_PARSER = get_pyparser_for_version( Options.version )
+
+
+## XXX BROKEN
+## def grammar_rules( space ):
+## w_rules = space.newdict()
+## for key, value in PYTHON_PARSER.rules.iteritems():
+## space.setitem(w_rules, space.wrap(key), space.wrap(value))
+## return w_rules
+##
+##
+## def parse_grammar(space, w_src):
+## """Loads the grammar using the 'dynamic' rpython parser"""
+## src = space.str_w( w_src )
+## ebnfbuilder = ebnfparse.parse_grammar_text( src )
+## ebnfbuilder.resolve_rules()
+## grammar.build_first_sets(ebnfbuilder.all_rules)
+## return space.wrap( ebnfbuilder.root_rules )
Modified: pypy/branch/ast-experiments/pypy/interpreter/pyparser/pythonutil.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/interpreter/pyparser/pythonutil.py (original)
+++ pypy/branch/ast-experiments/pypy/interpreter/pyparser/pythonutil.py Thu Dec 14 17:28:05 2006
@@ -1,17 +1,90 @@
-__all__ = ["python_parse", "pypy_parse"]
+"""miscelanneous utility functions
+XXX: svn mv pythonutil.py gramtools.py / parsertools.py
+"""
+
+import sys
+import os
import parser
-import pythonparse
-from tuplebuilder import TupleBuilder
-from astbuilder import AstBuilder
-
-PYTHON_PARSER = pythonparse.PYTHON_PARSER
-TARGET_DICT = {
- 'exec' : "file_input",
- 'eval' : "eval_input",
- 'single' : "single_input",
- }
+from pypy.interpreter.pyparser.grammar import Parser
+from pypy.interpreter.pyparser.pytoken import setup_tokens
+from pypy.interpreter.pyparser.ebnfgrammar import GRAMMAR_GRAMMAR
+from pypy.interpreter.pyparser.ebnflexer import GrammarSource
+from pypy.interpreter.pyparser.ebnfparse import EBNFBuilder
+
+from pypy.interpreter.pyparser.tuplebuilder import TupleBuilder
+
+PYTHON_VERSION = ".".join([str(i) for i in sys.version_info[:2]])
+
+def get_grammar_file(version):
+ """returns the python grammar corresponding to our CPython version"""
+ if version == "native":
+ _ver = PYTHON_VERSION
+ elif version == "stable":
+ _ver = "_stablecompiler"
+ elif version in ("2.3","2.4","2.5a"):
+ _ver = version
+ return os.path.join( os.path.dirname(__file__), "data", "Grammar" + _ver ), _ver
+
+
+def build_parser(gramfile, parser=None):
+ """reads a (EBNF) grammar definition and builds a parser for it"""
+ if parser is None:
+ parser = Parser()
+ setup_tokens(parser)
+ # XXX: clean up object dependencies
+ source = GrammarSource(GRAMMAR_GRAMMAR, file(gramfile).read())
+ builder = EBNFBuilder(GRAMMAR_GRAMMAR, dest_parser=parser)
+ GRAMMAR_GRAMMAR.root_rules['grammar'].match(source, builder)
+ builder.resolve_rules()
+ parser.build_first_sets()
+ return parser
+
+
+def build_parser_for_version(version, parser=None):
+ gramfile, _ = get_grammar_file(version)
+ return build_parser(gramfile, parser)
+
+
+## XXX: the below code should probably go elsewhere
+
+## convenience functions for computing AST objects using recparser
+def ast_from_input(input, mode, transformer, parser):
+ """converts a source input into an AST
+
+ - input : the source to be converted
+ - mode : 'exec', 'eval' or 'single'
+ - transformer : the transfomer instance to use to convert
+ the nested tuples into the AST
+ XXX: transformer could be instantiated here but we don't want
+ here to explicitly import compiler or stablecompiler or
+ etc. This is to be fixed in a clean way
+ """
+ builder = TupleBuilder(parser, lineno=True)
+ parser.parse_source(input, mode, builder)
+ tuples = builder.stack[-1].as_tuple(True)
+ return transformer.compile_node(tuples)
+
+
+def pypy_parse(source, mode='exec', lineno=False):
+ from pypy.interpreter.pyparser.pythonparse import PythonParser, get_pyparser_for_version
+ from pypy.interpreter.pyparser.astbuilder import AstBuilder
+ # parser = build_parser_for_version("2.4", PythonParser())
+ parser = get_pyparser_for_version('2.4')
+ builder = TupleBuilder(parser)
+ parser.parse_source(source, mode, builder)
+ return builder.stack[-1].as_tuple(lineno)
+
+
+def source2ast(source, mode='exec', version='2.4', space=None):
+ from pypy.interpreter.pyparser.pythonparse import PythonParser, get_pyparser_for_version
+ from pypy.interpreter.pyparser.astbuilder import AstBuilder
+ parser = get_pyparser_for_version(version)
+ builder = AstBuilder(parser, space=space)
+ parser.parse_source(source, mode, builder)
+ return builder
+
## convenience functions around CPython's parser functions
def python_parsefile(filename, lineno=False):
@@ -32,7 +105,6 @@
tp = parser.suite(source)
return parser.ast2tuple(tp, line_info=lineno)
-## convenience functions around recparser functions
def pypy_parsefile(filename, lineno=False):
"""parse <filename> using PyPy's parser module and return
a tuple of three elements :
@@ -48,105 +120,3 @@
source = pyf.read()
pyf.close()
return pypy_parse(source, 'exec', lineno)
-
-def internal_pypy_parse(source, mode='exec', lineno=False, flags=0, space=None,
- parser = PYTHON_PARSER):
- """This function has no other role than testing the parser's annotation
-
- annotateme() is basically the same code that pypy_parse(), but with the
- following differences :
-
- - returns a tuplebuilder.StackElement instead of the *real* nested
- tuples (StackElement is only a wrapper class around these tuples)
-
- """
- builder = TupleBuilder(parser, lineno=False)
- if space is not None:
- builder.space = space
- target_rule = TARGET_DICT[mode]
- parser.parse_source(source, target_rule, builder, flags)
- stack_element = builder.stack[-1]
- return (builder.source_encoding, stack_element)
-
-def parse_result_to_nested_tuples(parse_result, lineno=False):
- """NOT_RPYTHON"""
- source_encoding, stack_element = parse_result
- return stack_element.as_tuple(lineno)
-
-def pypy_parse(source, mode='exec', lineno=False, flags=0, parser = PYTHON_PARSER):
- """
- NOT_RPYTHON !
- parse <source> using PyPy's parser module and return
- a tuple of three elements :
- - The encoding declaration symbol or None if there were no encoding
- statement
- - The TupleBuilder's stack top element (instance of
- tuplebuilder.StackElement which is a wrapper of some nested tuples
- like those returned by the CPython's parser)
- - The encoding string or None if there were no encoding statement
- nested tuples
- """
- source_encoding, stack_element = internal_pypy_parse(source, mode, lineno=lineno,
- flags=lineno, parser = parser)
- # convert the stack element into nested tuples (caution, the annotator
- # can't follow this call)
- return parse_result_to_nested_tuples((source_encoding, stack_element), lineno=lineno)
-
-## convenience functions for computing AST objects using recparser
-def ast_from_input(input, mode, transformer, parser = PYTHON_PARSER):
- """converts a source input into an AST
-
- - input : the source to be converted
- - mode : 'exec', 'eval' or 'single'
- - transformer : the transfomer instance to use to convert
- the nested tuples into the AST
- XXX: transformer could be instantiated here but we don't want
- here to explicitly import compiler or stablecompiler or
- etc. This is to be fixed in a clean way
- """
- tuples = pypy_parse(input, mode, True, parser)
- if 'import' in input:
- toto # XXX ? What's the point here ?
- return transformer.compile_node(tuples)
-
-def target_ast_compile(space, input, mode):
- from pypy.interpreter.astcompiler import ast, misc, pycodegen
- builder = AstBuilder(rules=None, debug=0, space=space)
- target = TARGET_DICT[mode]
- PYTHON_PARSER.parse_source(input, target, builder)
- ast_tree = builder.rule_stack[-1]
- misc.set_filename("<?>", ast_tree)
- if mode=="single":
- codegenerator = pycodegen.InteractiveCodeGenerator(space,ast_tree)
- elif mode=="eval":
- codegenerator = pycodegen.ExpressionCodeGenerator(space,ast_tree)
- elif mode=="exec":
- codegenerator = pycodegen.ModuleCodeGenerator(space,ast_tree)
- else:
- raise ValueError("incorrect mode")
- code1 = codegenerator.getCode()
- return code1
-
-
-def internal_pypy_parse_to_ast(source, mode='exec', lineno=False, flags=0):
- builder = AstBuilder()
- target_rule = TARGET_DICT[mode]
- PYTHON_PARSER.parse_source(source, target_rule, builder, flags)
- ast_tree = builder.rule_stack[-1]
- return (builder.source_encoding, ast_tree)
-
-
-if __name__ == "__main__":
- import sys
- if len(sys.argv) < 2:
- print "python parse.py [-d N] test_file.py"
- sys.exit(1)
- if sys.argv[1] == "-d":
- debug_level = int(sys.argv[2])
- test_file = sys.argv[3]
- else:
- test_file = sys.argv[1]
- print "-"*20
- print
- print "pyparse \n", pypy_parsefile(test_file)
- print "parser \n", python_parsefile(test_file)
Added: pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/expressions.py
==============================================================================
--- (empty file)
+++ pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/expressions.py Thu Dec 14 17:28:05 2006
@@ -0,0 +1,504 @@
+"""
+list of tested expressions / suites (used by test_parser and test_astbuilder)
+"""
+
+constants = [
+ "0",
+ "7",
+ "-3",
+ "053",
+ "0x18",
+ "14L",
+ "1.0",
+ "3.9",
+ "-3.6",
+ "1.8e19",
+ "90000000000000",
+ "90000000000000.",
+ "3j"
+ ]
+
+expressions = [
+ "x = a + 1",
+ "x = 1 - a",
+ "x = a * b",
+ "x = a ** 2",
+ "x = a / b",
+ "x = a & b",
+ "x = a | b",
+ "x = a ^ b",
+ "x = a // b",
+ "x = a * b + 1",
+ "x = a + 1 * b",
+ "x = a * b / c",
+ "x = a * (1 + c)",
+ "x, y, z = 1, 2, 3",
+ "x = 'a' 'b' 'c'",
+ "del foo",
+ "del foo[bar]",
+ "del foo.bar",
+ "l[0]",
+ "k[v,]",
+ "m[a,b]",
+ "a.b.c[d]",
+ "file('some.txt').read()",
+ "a[0].read()",
+ "a[1:1].read()",
+ "f('foo')('bar')('spam')",
+ "f('foo')('bar')('spam').read()[0]",
+ "a.b[0][0]",
+ "a.b[0][:]",
+ "a.b[0][::]",
+ "a.b[0][0].pop()[0].push('bar')('baz').spam",
+ "a.b[0].read()[1][2].foo().spam()[0].bar",
+ "a**2",
+ "a**2**2",
+ "a.b[0]**2",
+ "a.b[0].read()[1][2].foo().spam()[0].bar ** 2",
+ "l[start:end] = l2",
+ "l[::] = l2",
+ "a = `s`",
+ "a = `1 + 2 + f(3, 4)`",
+ "[a, b] = c",
+ "(a, b) = c",
+ "[a, (b,c), d] = e",
+ "a, (b, c), d = e",
+ ]
+
+# We do not export the following tests because we would have to implement 2.5
+# features in the stable compiler (other than just building the AST).
+expressions_inbetweenversions = expressions + [
+ "1 if True else 2",
+ "1 if False else 2",
+ ]
+
+funccalls = [
+ "l = func()",
+ "l = func(10)",
+ "l = func(10, 12, a, b=c, *args)",
+ "l = func(10, 12, a, b=c, **kwargs)",
+ "l = func(10, 12, a, b=c, *args, **kwargs)",
+ "l = func(10, 12, a, b=c)",
+ "e = l.pop(3)",
+ "e = k.l.pop(3)",
+ "simplefilter('ignore', category=PendingDeprecationWarning, append=1)",
+ """methodmap = dict(subdirs=phase4,
+ same_files=phase3, diff_files=phase3, funny_files=phase3,
+ common_dirs = phase2, common_files=phase2, common_funny=phase2,
+ common=phase1, left_only=phase1, right_only=phase1,
+ left_list=phase0, right_list=phase0)""",
+ "odata = b2a_qp(data, quotetabs = quotetabs, header = header)",
+ ]
+
+listmakers = [
+ "l = []",
+ "l = [1, 2, 3]",
+ "l = [i for i in range(10)]",
+ "l = [i for i in range(10) if i%2 == 0]",
+ "l = [i for i in range(10) if i%2 == 0 or i%2 == 1]", # <--
+ "l = [i for i in range(10) if i%2 == 0 and i%2 == 1]",
+ "l = [i for j in range(10) for i in range(j)]",
+ "l = [i for j in range(10) for i in range(j) if j%2 == 0]",
+ "l = [i for j in range(10) for i in range(j) if j%2 == 0 and i%2 == 0]",
+ "l = [(a, b) for (a,b,c) in l2]",
+ "l = [{a:b} for (a,b,c) in l2]",
+ "l = [i for j in k if j%2 == 0 if j*2 < 20 for i in j if i%2==0]",
+ ]
+
+genexps = [
+ "l = (i for i in j)",
+ "l = (i for i in j if i%2 == 0)",
+ "l = (i for j in k for i in j)",
+ "l = (i for j in k for i in j if j%2==0)",
+ "l = (i for j in k if j%2 == 0 if j*2 < 20 for i in j if i%2==0)",
+ "l = (i for i in [ j*2 for j in range(10) ] )",
+ "l = [i for i in ( j*2 for j in range(10) ) ]",
+ "l = (i for i in [ j*2 for j in ( k*3 for k in range(10) ) ] )",
+ "l = [i for j in ( j*2 for j in [ k*3 for k in range(10) ] ) ]",
+ "l = f(i for i in j)",
+ ]
+
+
+dictmakers = [
+ "l = {a : b, 'c' : 0}",
+ "l = {}",
+ ]
+
+backtrackings = [
+ "f = lambda x: x+1",
+ "f = lambda x,y: x+y",
+ "f = lambda x,y=1,z=t: x+y",
+ "f = lambda x,y=1,z=t,*args,**kwargs: x+y",
+ "f = lambda x,y=1,z=t,*args: x+y",
+ "f = lambda x,y=1,z=t,**kwargs: x+y",
+ "f = lambda: 1",
+ "f = lambda *args: 1",
+ "f = lambda **kwargs: 1",
+ ]
+
+comparisons = [
+ "a < b",
+ "a > b",
+ "a not in b",
+ "a is not b",
+ "a in b",
+ "a is b",
+ "3 < x < 5",
+ "(3 < x) < 5",
+ "a < b < c < d",
+ "(a < b) < (c < d)",
+ "a < (b < c) < d",
+ ]
+
+multiexpr = [
+ 'a = b; c = d;',
+ 'a = b = c = d',
+ ]
+
+attraccess = [
+ 'a.b = 2',
+ 'x = a.b',
+ ]
+
+slices = [
+ "l[:]",
+ "l[::]",
+ "l[1:2]",
+ "l[1:]",
+ "l[:2]",
+ "l[1::]",
+ "l[:1:]",
+ "l[::1]",
+ "l[1:2:]",
+ "l[:1:2]",
+ "l[1::2]",
+ "l[0:1:2]",
+ "a.b.l[:]",
+ "a.b.l[1:2]",
+ "a.b.l[1:]",
+ "a.b.l[:2]",
+ "a.b.l[0:1:2]",
+ "a[1:2:3, 100]",
+ "a[:2:3, 100]",
+ "a[1::3, 100,]",
+ "a[1:2:, 100]",
+ "a[1:2, 100]",
+ "a[1:, 100,]",
+ "a[:2, 100]",
+ "a[:, 100]",
+ "a[100, 1:2:3,]",
+ "a[100, :2:3]",
+ "a[100, 1::3]",
+ "a[100, 1:2:,]",
+ "a[100, 1:2]",
+ "a[100, 1:]",
+ "a[100, :2,]",
+ "a[100, :]",
+ ]
+
+imports = [
+ 'import os',
+ 'import sys, os',
+ 'import os.path',
+ 'import os.path, sys',
+ 'import sys, os.path as osp',
+ 'import os.path as osp',
+ 'import os.path as osp, sys as _sys',
+ 'import a.b.c.d',
+ 'import a.b.c.d as abcd',
+ 'from os import path',
+ 'from os import path, system',
+ ]
+
+imports_newstyle = [
+ 'from os import path, system',
+ 'from os import path as P, system as S',
+ 'from os import (path as P, system as S,)',
+ 'from os import *',
+ ]
+
+if_stmts = [
+ "if a == 1: a+= 2",
+ """if a == 1:
+ a += 2
+elif a == 2:
+ a += 3
+else:
+ a += 4
+""",
+ "if a and not b == c: pass",
+ "if a and not not not b == c: pass",
+ "if 0: print 'foo'"
+ ]
+
+asserts = [
+ 'assert False',
+ 'assert a == 1',
+ 'assert a == 1 and b == 2',
+ 'assert a == 1 and b == 2, "assertion failed"',
+ ]
+
+execs = [
+ 'exec a',
+ 'exec "a=b+3"',
+ 'exec a in f()',
+ 'exec a in f(), g()',
+ ]
+
+prints = [
+ 'print',
+ 'print a',
+ 'print a,',
+ 'print a, b',
+ 'print a, "b", c',
+ 'print >> err',
+ 'print >> err, "error"',
+ 'print >> err, "error",',
+ 'print >> err, "error", a',
+ ]
+
+globs = [
+ 'global a',
+ 'global a,b,c',
+ ]
+
+raises_ = [ # NB. 'raises' creates a name conflict with py.test magic
+ 'raise',
+ 'raise ValueError',
+ 'raise ValueError("error")',
+ 'raise ValueError, "error"',
+ 'raise ValueError, "error", foo',
+ ]
+
+tryexcepts = [
+ """try:
+ a
+ b
+except:
+ pass
+""",
+ """try:
+ a
+ b
+except NameError:
+ pass
+""",
+ """try:
+ a
+ b
+except NameError, err:
+ pass
+""",
+ """try:
+ a
+ b
+except (NameError, ValueError):
+ pass
+""",
+ """try:
+ a
+ b
+except (NameError, ValueError), err:
+ pass
+""",
+ """try:
+ a
+except NameError, err:
+ pass
+except ValueError, err:
+ pass
+""",
+ """def f():
+ try:
+ a
+ except NameError, err:
+ a = 1
+ b = 2
+ except ValueError, err:
+ a = 2
+ return a
+"""
+ """try:
+ a
+except NameError, err:
+ a = 1
+except ValueError, err:
+ a = 2
+else:
+ a += 3
+""",
+ """try:
+ a
+finally:
+ b
+""",
+ """def f():
+ try:
+ return a
+ finally:
+ a = 3
+ return 1
+""",
+
+ ]
+
+one_stmt_funcdefs = [
+ "def f(): return 1",
+ "def f(x): return x+1",
+ "def f(x,y): return x+y",
+ "def f(x,y=1,z=t): return x+y",
+ "def f(x,y=1,z=t,*args,**kwargs): return x+y",
+ "def f(x,y=1,z=t,*args): return x+y",
+ "def f(x,y=1,z=t,**kwargs): return x+y",
+ "def f(*args): return 1",
+ "def f(**kwargs): return 1",
+ "def f(t=()): pass",
+ "def f(a, b, (c, d), e): pass",
+ "def f(a, b, (c, (d, e), f, (g, h))): pass",
+ "def f(a, b, (c, (d, e), f, (g, h)), i): pass",
+ "def f((a)): pass",
+ ]
+
+one_stmt_classdefs = [
+ "class Pdb(bdb.Bdb, cmd.Cmd): pass",
+ ]
+
+docstrings = [
+ '''def foo(): return 1''',
+ '''class Foo: pass''',
+ '''class Foo: "foo"''',
+ '''def foo():
+ """foo docstring"""
+ return 1
+''',
+ '''def foo():
+ """foo docstring"""
+ a = 1
+ """bar"""
+ return a
+''',
+ '''def foo():
+ """doc"""; print 1
+ a=1
+''',
+ '''"""Docstring""";print 1''',
+ ]
+
+returns = [
+ 'def f(): return',
+ 'def f(): return 1',
+ 'def f(): return a.b',
+ 'def f(): return a',
+ 'def f(): return a,b,c,d',
+ #'return (a,b,c,d)', --- this one makes no sense, as far as I can tell
+ ]
+
+augassigns = [
+ 'a=1;a+=2',
+ 'a=1;a-=2',
+ 'a=1;a*=2',
+ 'a=1;a/=2',
+ 'a=1;a//=2',
+ 'a=1;a%=2',
+ 'a=1;a**=2',
+ 'a=1;a>>=2',
+ 'a=1;a<<=2',
+ 'a=1;a&=2',
+ 'a=1;a^=2',
+ 'a=1;a|=2',
+
+ 'a=A();a.x+=2',
+ 'a=A();a.x-=2',
+ 'a=A();a.x*=2',
+ 'a=A();a.x/=2',
+ 'a=A();a.x//=2',
+ 'a=A();a.x%=2',
+ 'a=A();a.x**=2',
+ 'a=A();a.x>>=2',
+ 'a=A();a.x<<=2',
+ 'a=A();a.x&=2',
+ 'a=A();a.x^=2',
+ 'a=A();a.x|=2',
+
+ 'a=A();a[0]+=2',
+ 'a=A();a[0]-=2',
+ 'a=A();a[0]*=2',
+ 'a=A();a[0]/=2',
+ 'a=A();a[0]//=2',
+ 'a=A();a[0]%=2',
+ 'a=A();a[0]**=2',
+ 'a=A();a[0]>>=2',
+ 'a=A();a[0]<<=2',
+ 'a=A();a[0]&=2',
+ 'a=A();a[0]^=2',
+ 'a=A();a[0]|=2',
+
+ 'a=A();a[0:2]+=2',
+ 'a=A();a[0:2]-=2',
+ 'a=A();a[0:2]*=2',
+ 'a=A();a[0:2]/=2',
+ 'a=A();a[0:2]//=2',
+ 'a=A();a[0:2]%=2',
+ 'a=A();a[0:2]**=2',
+ 'a=A();a[0:2]>>=2',
+ 'a=A();a[0:2]<<=2',
+ 'a=A();a[0:2]&=2',
+ 'a=A();a[0:2]^=2',
+ 'a=A();a[0:2]|=2',
+ ]
+
+PY23_TESTS = [
+ constants,
+ expressions,
+ augassigns,
+ comparisons,
+ funccalls,
+ backtrackings,
+ listmakers, # ERRORS
+ dictmakers,
+ multiexpr,
+ attraccess,
+ slices,
+ imports,
+ execs,
+ prints,
+ globs,
+ raises_,
+
+ ]
+
+OPTIONAL_TESTS = [
+ # expressions_inbetweenversions,
+ genexps,
+ imports_newstyle,
+ asserts,
+ ]
+
+TESTS = PY23_TESTS + OPTIONAL_TESTS
+
+
+## TESTS = [
+## ["l = [i for i in range(10) if i%2 == 0 or i%2 == 1]"],
+## ]
+
+EXEC_INPUTS = [
+ one_stmt_classdefs,
+ one_stmt_funcdefs,
+ if_stmts,
+ tryexcepts,
+ docstrings,
+ returns,
+ ]
+
+SINGLE_INPUTS = [
+ one_stmt_funcdefs,
+ ['\t # hello\n',
+ 'print 6*7',
+ 'if 1: x\n',
+ 'x = 5',
+ 'x = 5 ',
+ '''"""Docstring""";print 1''',
+ '''"Docstring"''',
+ '''"Docstring" "\\x00"''',
+ ]
+]
Modified: pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/test_astbuilder.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/test_astbuilder.py (original)
+++ pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/test_astbuilder.py Thu Dec 14 17:28:05 2006
@@ -1,8 +1,9 @@
import os
+from pypy.tool.option import Options
from pypy.interpreter.pyparser import pythonparse
from pypy.interpreter.pyparser.astbuilder import AstBuilder
-from pypy.interpreter.pyparser.pythonutil import ast_from_input
+from pypy.interpreter.pyparser.pythonutil import ast_from_input, build_parser_for_version
from pypy.interpreter.stablecompiler.transformer import Transformer
import pypy.interpreter.stablecompiler.ast as test_ast
import pypy.interpreter.astcompiler.ast as ast_ast
@@ -13,6 +14,9 @@
from pypy.interpreter.astcompiler import ast
+
+from expressions import TESTS, SINGLE_INPUTS, EXEC_INPUTS
+
def arglist_equal(left,right):
"""needs special case because we handle the argumentlist differently"""
for l,r in zip(left,right):
@@ -142,211 +146,13 @@
return False
return True
-EXPECTED = {}
-
-constants = [
- "0",
- "7",
- "-3",
- "053",
- "0x18",
- "14L",
- "1.0",
- "3.9",
- "-3.6",
- "1.8e19",
- "90000000000000",
- "90000000000000.",
- "3j"
- ]
-
-expressions = [
- "x = a + 1",
- "x = 1 - a",
- "x = a * b",
- "x = a ** 2",
- "x = a / b",
- "x = a & b",
- "x = a | b",
- "x = a ^ b",
- "x = a // b",
- "x = a * b + 1",
- "x = a + 1 * b",
- "x = a * b / c",
- "x = a * (1 + c)",
- "x, y, z = 1, 2, 3",
- "x = 'a' 'b' 'c'",
- "del foo",
- "del foo[bar]",
- "del foo.bar",
- "l[0]",
- "k[v,]",
- "m[a,b]",
- "a.b.c[d]",
- "file('some.txt').read()",
- "a[0].read()",
- "a[1:1].read()",
- "f('foo')('bar')('spam')",
- "f('foo')('bar')('spam').read()[0]",
- "a.b[0][0]",
- "a.b[0][:]",
- "a.b[0][::]",
- "a.b[0][0].pop()[0].push('bar')('baz').spam",
- "a.b[0].read()[1][2].foo().spam()[0].bar",
- "a**2",
- "a**2**2",
- "a.b[0]**2",
- "a.b[0].read()[1][2].foo().spam()[0].bar ** 2",
- "l[start:end] = l2",
- "l[::] = l2",
- "a = `s`",
- "a = `1 + 2 + f(3, 4)`",
- "[a, b] = c",
- "(a, b) = c",
- "[a, (b,c), d] = e",
- "a, (b, c), d = e",
- ]
-
-# We do not export the following tests because we would have to implement 2.5
-# features in the stable compiler (other than just building the AST).
-expressions_inbetweenversions = expressions + [
- "1 if True else 2",
- "1 if False else 2",
- ]
-
-EXPECTED["k[v,]"] = ("Module(None, Stmt([Discard(Subscript(Name('k'), 2, "
- "Tuple([Name('v')])))]))")
-EXPECTED["m[a,b]"] = ("Module(None, Stmt([Discard(Subscript(Name('m'), 2, "
- "Tuple([Name('a'), Name('b')])))]))")
-EXPECTED["1 if True else 2"] = ("Module(None, Stmt([Discard(CondExpr("
- "Name('True'), Const(1), Const(2)))]))")
-EXPECTED["1 if False else 2"] = ("Module(None, Stmt([Discard(CondExpr("
- "Name('False'), Const(1), Const(2)))]))")
-
-funccalls = [
- "l = func()",
- "l = func(10)",
- "l = func(10, 12, a, b=c, *args)",
- "l = func(10, 12, a, b=c, **kwargs)",
- "l = func(10, 12, a, b=c, *args, **kwargs)",
- "l = func(10, 12, a, b=c)",
- "e = l.pop(3)",
- "e = k.l.pop(3)",
- "simplefilter('ignore', category=PendingDeprecationWarning, append=1)",
- """methodmap = dict(subdirs=phase4,
- same_files=phase3, diff_files=phase3, funny_files=phase3,
- common_dirs = phase2, common_files=phase2, common_funny=phase2,
- common=phase1, left_only=phase1, right_only=phase1,
- left_list=phase0, right_list=phase0)""",
- "odata = b2a_qp(data, quotetabs = quotetabs, header = header)",
- ]
-
-listmakers = [
- "l = []",
- "l = [1, 2, 3]",
- "l = [i for i in range(10)]",
- "l = [i for i in range(10) if i%2 == 0]",
- "l = [i for i in range(10) if i%2 == 0 or i%2 == 1]",
- "l = [i for i in range(10) if i%2 == 0 and i%2 == 1]",
- "l = [i for j in range(10) for i in range(j)]",
- "l = [i for j in range(10) for i in range(j) if j%2 == 0]",
- "l = [i for j in range(10) for i in range(j) if j%2 == 0 and i%2 == 0]",
- "l = [(a, b) for (a,b,c) in l2]",
- "l = [{a:b} for (a,b,c) in l2]",
- "l = [i for j in k if j%2 == 0 if j*2 < 20 for i in j if i%2==0]",
- ]
-
-genexps = [
- "l = (i for i in j)",
- "l = (i for i in j if i%2 == 0)",
- "l = (i for j in k for i in j)",
- "l = (i for j in k for i in j if j%2==0)",
- "l = (i for j in k if j%2 == 0 if j*2 < 20 for i in j if i%2==0)",
- "l = (i for i in [ j*2 for j in range(10) ] )",
- "l = [i for i in ( j*2 for j in range(10) ) ]",
- "l = (i for i in [ j*2 for j in ( k*3 for k in range(10) ) ] )",
- "l = [i for j in ( j*2 for j in [ k*3 for k in range(10) ] ) ]",
- "l = f(i for i in j)",
- ]
+EXPECTED = {
+ "k[v,]" : "Module(None, Stmt([Discard(Subscript(Name('k'), 2, Tuple([Name('v')])))]))",
+ "m[a,b]" : "Module(None, Stmt([Discard(Subscript(Name('m'), 2, Tuple([Name('a'), Name('b')])))]))",
+ "1 if True else 2" : "Module(None, Stmt([Discard(CondExpr(Name('True'), Const(1), Const(2)))]))",
+ "1 if False else 2" : "Module(None, Stmt([Discard(CondExpr(Name('False'), Const(1), Const(2)))]))",
-dictmakers = [
- "l = {a : b, 'c' : 0}",
- "l = {}",
- ]
-
-backtrackings = [
- "f = lambda x: x+1",
- "f = lambda x,y: x+y",
- "f = lambda x,y=1,z=t: x+y",
- "f = lambda x,y=1,z=t,*args,**kwargs: x+y",
- "f = lambda x,y=1,z=t,*args: x+y",
- "f = lambda x,y=1,z=t,**kwargs: x+y",
- "f = lambda: 1",
- "f = lambda *args: 1",
- "f = lambda **kwargs: 1",
- ]
-
-comparisons = [
- "a < b",
- "a > b",
- "a not in b",
- "a is not b",
- "a in b",
- "a is b",
- "3 < x < 5",
- "(3 < x) < 5",
- "a < b < c < d",
- "(a < b) < (c < d)",
- "a < (b < c) < d",
- ]
-
-multiexpr = [
- 'a = b; c = d;',
- 'a = b = c = d',
- ]
-
-attraccess = [
- 'a.b = 2',
- 'x = a.b',
- ]
-
-slices = [
- "l[:]",
- "l[::]",
- "l[1:2]",
- "l[1:]",
- "l[:2]",
- "l[1::]",
- "l[:1:]",
- "l[::1]",
- "l[1:2:]",
- "l[:1:2]",
- "l[1::2]",
- "l[0:1:2]",
- "a.b.l[:]",
- "a.b.l[1:2]",
- "a.b.l[1:]",
- "a.b.l[:2]",
- "a.b.l[0:1:2]",
- "a[1:2:3, 100]",
- "a[:2:3, 100]",
- "a[1::3, 100,]",
- "a[1:2:, 100]",
- "a[1:2, 100]",
- "a[1:, 100,]",
- "a[:2, 100]",
- "a[:, 100]",
- "a[100, 1:2:3,]",
- "a[100, :2:3]",
- "a[100, 1::3]",
- "a[100, 1:2:,]",
- "a[100, 1:2]",
- "a[100, 1:]",
- "a[100, :2,]",
- "a[100, :]",
- ]
-EXPECTED.update({
"a[1:2:3, 100]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Sliceobj([Const(1), Const(2), Const(3)]), Const(100)])))]))",
"a[:2:3, 100]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Sliceobj([Const(None), Const(2), Const(3)]), Const(100)])))]))",
"a[1::3, 100,]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Sliceobj([Const(1), Const(None), Const(3)]), Const(100)])))]))",
@@ -363,307 +169,6 @@
"a[100, 1:]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(None)])])))]))",
"a[100, :2,]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(2)])])))]))",
"a[100, :]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(None)])])))]))",
- })
-
-imports = [
- 'import os',
- 'import sys, os',
- 'import os.path',
- 'import os.path, sys',
- 'import sys, os.path as osp',
- 'import os.path as osp',
- 'import os.path as osp, sys as _sys',
- 'import a.b.c.d',
- 'import a.b.c.d as abcd',
- 'from os import path',
- 'from os import path, system',
- ]
-
-imports_newstyle = [
- 'from os import path, system',
- 'from os import path as P, system as S',
- 'from os import (path as P, system as S,)',
- 'from os import *',
- ]
-
-if_stmts = [
- "if a == 1: a+= 2",
- """if a == 1:
- a += 2
-elif a == 2:
- a += 3
-else:
- a += 4
-""",
- "if a and not b == c: pass",
- "if a and not not not b == c: pass",
- "if 0: print 'foo'"
- ]
-
-asserts = [
- 'assert False',
- 'assert a == 1',
- 'assert a == 1 and b == 2',
- 'assert a == 1 and b == 2, "assertion failed"',
- ]
-
-execs = [
- 'exec a',
- 'exec "a=b+3"',
- 'exec a in f()',
- 'exec a in f(), g()',
- ]
-
-prints = [
- 'print',
- 'print a',
- 'print a,',
- 'print a, b',
- 'print a, "b", c',
- 'print >> err',
- 'print >> err, "error"',
- 'print >> err, "error",',
- 'print >> err, "error", a',
- ]
-
-globs = [
- 'global a',
- 'global a,b,c',
- ]
-
-raises_ = [ # NB. 'raises' creates a name conflict with py.test magic
- 'raise',
- 'raise ValueError',
- 'raise ValueError("error")',
- 'raise ValueError, "error"',
- 'raise ValueError, "error", foo',
- ]
-
-tryexcepts = [
- """try:
- a
- b
-except:
- pass
-""",
- """try:
- a
- b
-except NameError:
- pass
-""",
- """try:
- a
- b
-except NameError, err:
- pass
-""",
- """try:
- a
- b
-except (NameError, ValueError):
- pass
-""",
- """try:
- a
- b
-except (NameError, ValueError), err:
- pass
-""",
- """try:
- a
-except NameError, err:
- pass
-except ValueError, err:
- pass
-""",
- """def f():
- try:
- a
- except NameError, err:
- a = 1
- b = 2
- except ValueError, err:
- a = 2
- return a
-"""
- """try:
- a
-except NameError, err:
- a = 1
-except ValueError, err:
- a = 2
-else:
- a += 3
-""",
- """try:
- a
-finally:
- b
-""",
- """def f():
- try:
- return a
- finally:
- a = 3
- return 1
-""",
-
- ]
-
-one_stmt_funcdefs = [
- "def f(): return 1",
- "def f(x): return x+1",
- "def f(x,y): return x+y",
- "def f(x,y=1,z=t): return x+y",
- "def f(x,y=1,z=t,*args,**kwargs): return x+y",
- "def f(x,y=1,z=t,*args): return x+y",
- "def f(x,y=1,z=t,**kwargs): return x+y",
- "def f(*args): return 1",
- "def f(**kwargs): return 1",
- "def f(t=()): pass",
- "def f(a, b, (c, d), e): pass",
- "def f(a, b, (c, (d, e), f, (g, h))): pass",
- "def f(a, b, (c, (d, e), f, (g, h)), i): pass",
- "def f((a)): pass",
- ]
-
-one_stmt_classdefs = [
- "class Pdb(bdb.Bdb, cmd.Cmd): pass",
- ]
-
-docstrings = [
- '''def foo(): return 1''',
- '''class Foo: pass''',
- '''class Foo: "foo"''',
- '''def foo():
- """foo docstring"""
- return 1
-''',
- '''def foo():
- """foo docstring"""
- a = 1
- """bar"""
- return a
-''',
- '''def foo():
- """doc"""; print 1
- a=1
-''',
- '''"""Docstring""";print 1''',
- ]
-
-returns = [
- 'def f(): return',
- 'def f(): return 1',
- 'def f(): return a.b',
- 'def f(): return a',
- 'def f(): return a,b,c,d',
- #'return (a,b,c,d)', --- this one makes no sense, as far as I can tell
- ]
-
-augassigns = [
- 'a=1;a+=2',
- 'a=1;a-=2',
- 'a=1;a*=2',
- 'a=1;a/=2',
- 'a=1;a//=2',
- 'a=1;a%=2',
- 'a=1;a**=2',
- 'a=1;a>>=2',
- 'a=1;a<<=2',
- 'a=1;a&=2',
- 'a=1;a^=2',
- 'a=1;a|=2',
-
- 'a=A();a.x+=2',
- 'a=A();a.x-=2',
- 'a=A();a.x*=2',
- 'a=A();a.x/=2',
- 'a=A();a.x//=2',
- 'a=A();a.x%=2',
- 'a=A();a.x**=2',
- 'a=A();a.x>>=2',
- 'a=A();a.x<<=2',
- 'a=A();a.x&=2',
- 'a=A();a.x^=2',
- 'a=A();a.x|=2',
-
- 'a=A();a[0]+=2',
- 'a=A();a[0]-=2',
- 'a=A();a[0]*=2',
- 'a=A();a[0]/=2',
- 'a=A();a[0]//=2',
- 'a=A();a[0]%=2',
- 'a=A();a[0]**=2',
- 'a=A();a[0]>>=2',
- 'a=A();a[0]<<=2',
- 'a=A();a[0]&=2',
- 'a=A();a[0]^=2',
- 'a=A();a[0]|=2',
-
- 'a=A();a[0:2]+=2',
- 'a=A();a[0:2]-=2',
- 'a=A();a[0:2]*=2',
- 'a=A();a[0:2]/=2',
- 'a=A();a[0:2]//=2',
- 'a=A();a[0:2]%=2',
- 'a=A();a[0:2]**=2',
- 'a=A();a[0:2]>>=2',
- 'a=A();a[0:2]<<=2',
- 'a=A();a[0:2]&=2',
- 'a=A();a[0:2]^=2',
- 'a=A();a[0:2]|=2',
- ]
-
-TESTS = [
- constants,
- expressions_inbetweenversions,
- augassigns,
- comparisons,
- funccalls,
- backtrackings,
- listmakers,
- genexps,
- dictmakers,
- multiexpr,
- attraccess,
- slices,
- imports,
- imports_newstyle,
- asserts,
- execs,
- prints,
- globs,
- raises_,
- ]
-
-EXEC_INPUTS = [
- one_stmt_classdefs,
- one_stmt_funcdefs,
- if_stmts,
- tryexcepts,
- docstrings,
- returns,
- ]
-
-SINGLE_INPUTS = [
- one_stmt_funcdefs,
- ['\t # hello\n',
- 'print 6*7',
- 'if 1: x\n',
- 'x = 5',
- 'x = 5 ',
- '''"""Docstring""";print 1''',
- '''"Docstring"''',
- '''"Docstring" "\\x00"''',
- ]
-]
-
-TARGET_DICT = {
- 'single' : 'single_input',
- 'exec' : 'file_input',
- 'eval' : 'eval_input',
}
@@ -704,39 +209,39 @@
builtin = dict(int=int, long=long, float=float, complex=complex)
-def ast_parse_expr(expr, target='single'):
- target = TARGET_DICT[target]
- builder = AstBuilder(space=FakeSpace())
- pythonparse.PYTHON_PARSER.parse_source(expr, target, builder)
- return builder
-
# Create parser from Grammar_stable, not current grammar.
-stable_grammar, _ = pythonparse.get_grammar_file("stable")
-stable_parser = pythonparse.python_grammar(stable_grammar)
+stable_parser = pythonparse.get_pyparser_for_version('stable')
+python_parser = pythonparse.get_pyparser_for_version(Options.version) # 'native') # 2.5a')
def tuple_parse_expr(expr, target='single'):
t = Transformer("dummyfile")
return ast_from_input(expr, target, t, stable_parser)
-def check_expression(expr, target='single'):
- r1 = ast_parse_expr(expr, target)
+def source2ast(source, mode, space=FakeSpace()):
+ builder = AstBuilder(space=space)
+ python_parser.parse_source(source, mode, builder)
+ return builder.rule_stack[-1]
+
+def check_expression(expr, mode='single'):
+ pypy_ast = source2ast(expr, mode)
try:
- ast = EXPECTED[expr]
+ python_ast = EXPECTED[expr]
except KeyError:
# trust the stablecompiler's Transformer when no explicit result has
# been provided (although trusting it is a foolish thing to do)
- ast = tuple_parse_expr(expr, target)
+ python_ast = tuple_parse_expr(expr, mode)
check_lineno = True
else:
- if isinstance(ast, str):
- ast = eval(ast, ast_ast.__dict__)
+ if isinstance(python_ast, str):
+ python_ast = eval(python_ast, ast_ast.__dict__)
check_lineno = False
print "-" * 30
- print "ORIG :", ast
+ print "ORIG :", python_ast
print
- print "BUILT:", r1.rule_stack[-1]
+ print "BUILT:", pypy_ast
print "-" * 30
- assert nodes_equal(ast, r1.rule_stack[-1], check_lineno), 'failed on %r' % (expr)
+ assert nodes_equal(python_ast, pypy_ast, check_lineno), 'failed on %r' % (expr)
+
def test_basic_astgen():
for family in TESTS:
@@ -748,6 +253,7 @@
for expr in family:
yield check_expression, expr, 'exec'
+
NEW_GRAMMAR_SNIPPETS = [
'snippet_with_1.py',
'snippet_with_2.py',
@@ -809,7 +315,7 @@
for snippet_name in LIBSTUFF:
filepath = os.path.join(os.path.dirname(__file__), '../../../lib', snippet_name)
source = file(filepath).read()
- yield check_expression, source, 'exec'
+ yield check_expression, source, 'exec'
# FIXME: find the sys' attribute that define this
STDLIB_PATH = os.path.dirname(os.__file__)
Modified: pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/test_astcompiler.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/test_astcompiler.py (original)
+++ pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/test_astcompiler.py Thu Dec 14 17:28:05 2006
@@ -1,4 +1,6 @@
+import sys
import os
+
from pypy.interpreter.pyparser import pythonparse
from pypy.interpreter.pyparser.astbuilder import AstBuilder
from pypy.interpreter.pyparser.tuplebuilder import TupleBuilder
@@ -11,92 +13,47 @@
from pypy.interpreter.astcompiler import ast, misc, pycodegen
-from test_astbuilder import expressions, comparisons, funccalls, backtrackings,\
- listmakers, genexps, dictmakers, multiexpr, attraccess, slices, imports,\
- asserts, execs, prints, globs, raises_, imports_newstyle, augassigns, \
- if_stmts, one_stmt_classdefs, one_stmt_funcdefs, tryexcepts, docstrings, \
- returns, SNIPPETS, SINGLE_INPUTS, LIBSTUFF, constants
-
-from test_astbuilder import FakeSpace
-
-
-TESTS = [
- constants,
- expressions,
- augassigns,
- comparisons,
- funccalls,
- backtrackings,
- listmakers,
- dictmakers,
- multiexpr,
- genexps,
- attraccess,
- slices,
- imports,
- execs,
- prints,
- globs,
- raises_,
-# EXEC_INPUTS
- one_stmt_classdefs,
- one_stmt_funcdefs,
- if_stmts,
- tryexcepts,
- docstrings,
- returns,
- ]
+from test_astbuilder import SNIPPETS, LIBSTUFF, FakeSpace, source2ast
+from expressions import PY23_TESTS, EXEC_INPUTS, SINGLE_INPUTS, OPTIONAL_TESTS
+
+TESTS = PY23_TESTS + EXEC_INPUTS
+
-import sys
if sys.version_info[0]==2 and sys.version_info[1]>=4:
# genexps and new style import don't work on python2.3
# TESTS.append(genexps) XXX: 2.4 optimizes bytecode so our comparison doesn't work
- TESTS.append(imports_newstyle)
- # assertions give different bytecode with 2.4 (optimize if __debug__)
- TESTS.append(asserts)
-TARGET_DICT = {
- 'single' : 'single_input',
- 'exec' : 'file_input',
- 'eval' : 'eval_input',
- }
-
-def ast_parse_expr(expr, target='single', space=FakeSpace()):
- target = TARGET_DICT[target]
- builder = AstBuilder(space=space)
- pythonparse.PYTHON_PARSER.parse_source(expr, target, builder)
- return builder.rule_stack[-1]
+ TESTS += OPTIONAL_TESTS
-def compile_with_astcompiler(expr, target='exec', space=FakeSpace()):
- ast = ast_parse_expr(expr, target='exec', space=space) # xxx exec: single not really tested, mumble
+def compile_with_astcompiler(expr, mode='exec', space=FakeSpace()):
+ ast = source2ast(expr, mode, space) # xxx exec: single not really tested, mumble
misc.set_filename('<?>', ast)
- if target == 'exec':
+ if mode == 'exec':
Generator = pycodegen.ModuleCodeGenerator
- elif target == 'single':
+ elif mode == 'single':
Generator = pycodegen.InteractiveCodeGenerator
- elif target == 'eval':
+ elif mode == 'eval':
Generator = pycodegen.ExpressionCodeGenerator
codegen = Generator(space, ast)
rcode = codegen.getCode()
return rcode
# Create parser from Grammar_stable, not current grammar.
-stable_grammar, _ = pythonparse.get_grammar_file("stable")
-stable_parser = pythonparse.python_grammar(stable_grammar)
+stable_parser = pythonparse.get_pyparser_for_version('stable')
-def compile_with_testcompiler(expr, target='exec', space=FakeSpace()):
- target2 = TARGET_DICT['exec'] # xxx exec: single not really tested
- builder = TupleBuilder()
- stable_parser.parse_source(expr, target2, builder)
+def compile_with_testcompiler(expr, mode='exec', space=FakeSpace()):
+ mode2 = 'exec' # xxx exec: single not really tested
+ builder = TupleBuilder(stable_parser)
+ stable_parser.parse_source(expr, mode2, builder)
tuples = builder.stack[-1].as_tuple(True)
from pypy.interpreter.stablecompiler import transformer, pycodegen, misc
ast = transformer.Transformer('<?>').compile_node(tuples)
misc.set_filename('<?>', ast)
- if target == 'exec':
+ if mode == 'exec':
Generator = pycodegen.ModuleCodeGenerator
- elif target == 'single':
+ elif mode == 'single':
Generator = pycodegen.InteractiveCodeGenerator
- elif target == 'eval':
+ elif mode == 'eval':
Generator = pycodegen.ExpressionCodeGenerator
codegen = Generator(ast)
rcode = codegen.getCode()
@@ -150,7 +107,7 @@
tuple(rcode.co_cellvars) )
return code
-def check_compile(expr, target='exec', quiet=False, space=None):
+def check_compile(expr, mode='exec', quiet=False, space=None):
if expr == "k[v,]":
py.test.skip('bug of the reference "stable compiler"')
if not quiet:
@@ -159,31 +116,18 @@
if space is None:
space = std_space
- sc_code = compile_with_testcompiler(expr, target=target)
- ac_code = compile_with_astcompiler(expr, target=target, space=space)
+ sc_code = compile_with_testcompiler(expr, mode=mode)
+ ac_code = compile_with_astcompiler(expr, mode=mode, space=space)
compare_code(ac_code, sc_code, space=space)
-## def check_compile( expr ):
-## space = FakeSpace()
-## ast_tree = ast_parse_expr( expr, target='exec', space=space )
-## misc.set_filename("<?>", ast_tree)
-## print "Compiling:", expr
-## print ast_tree
-## codegenerator = pycodegen.ModuleCodeGenerator(space,ast_tree)
-## rcode = codegenerator.getCode()
-## code1 = to_code( rcode )
-## code2 = ast_compile( expr )
-## compare_code(code1,code2)
def test_compile_argtuple_1():
- #py.test.skip('will be tested when more basic stuff will work')
code = """def f( x, (y,z) ):
print x,y,z
"""
check_compile( code )
def test_compile_argtuple_2():
- #py.test.skip('will be tested when more basic stuff will work')
code = """def f( x, (y,(z,t)) ):
print x,y,z,t
"""
@@ -191,14 +135,12 @@
def test_compile_argtuple_3():
- #py.test.skip('will be tested when more basic stuff will work')
code = """def f( x, (y,(z,(t,u))) ):
print x,y,z,t,u
"""
check_compile( code )
-
def test_basic_astgen():
for family in TESTS:
for expr in family:
Modified: pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/test_samples.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/test_samples.py (original)
+++ pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/test_samples.py Thu Dec 14 17:28:05 2006
@@ -5,14 +5,15 @@
import py.test
+
+from pypy.tool.option import Options
from pypy.interpreter.pyparser.pythonutil import python_parsefile, \
- pypy_parsefile, python_parse, pypy_parse
+ pypy_parsefile, pypy_parse, python_parse, get_grammar_file, PYTHON_VERSION
from pypy.interpreter.pyparser import grammar
from pypy.interpreter.pyparser.pythonlexer import TokenError
-from pypy.interpreter.pyparser.pythonparse import PYTHON_VERSION, PYPY_VERSION
grammar.DEBUG = False
-
+_, PYPY_VERSION = get_grammar_file( Options.version )
# these samples are skipped if the native version of Python does not match
# the version of the grammar we use
GRAMMAR_MISMATCH = PYTHON_VERSION != PYPY_VERSION
Modified: pypy/branch/ast-experiments/pypy/interpreter/stablecompiler/transformer.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/interpreter/stablecompiler/transformer.py (original)
+++ pypy/branch/ast-experiments/pypy/interpreter/stablecompiler/transformer.py Thu Dec 14 17:28:05 2006
@@ -26,9 +26,7 @@
# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
# make sure we import the parser with the correct grammar
-from pypy.interpreter.pyparser import pythonparse
-
-import pypy.interpreter.pyparser.pythonparse as pythonparse
+from pypy.interpreter.pyparser.pythonparse import get_pyparser_for_version
from pypy.interpreter.stablecompiler.ast import *
import parser
@@ -36,15 +34,16 @@
import sys
# Create parser from Grammar_stable, not current grammar.
-stable_grammar, _ = pythonparse.get_grammar_file("stable")
-stable_parser = pythonparse.python_grammar(stable_grammar)
+# stable_grammar, _ = pythonparse.get_grammar_file("stable")
+# stable_parser = pythonparse.python_grammar(stable_grammar)
-sym_name = stable_parser.symbols.sym_name
+stable_parser = get_pyparser_for_version('stable')
class symbol:
pass
-
-for value, name in sym_name.iteritems():
+sym_name = {}
+for name, value in stable_parser.symbols.items():
+ sym_name[value] = name
setattr(symbol, name, value)
# transforming is requiring a lot of recursion depth so make sure we have enough
@@ -58,6 +57,7 @@
from consts import CO_VARARGS, CO_VARKEYWORDS
from consts import OP_ASSIGN, OP_DELETE, OP_APPLY
+
def parseFile(path):
f = open(path, "U")
# XXX The parser API tolerates files without a trailing newline,
@@ -130,6 +130,7 @@
for value, name in sym_name.items():
if hasattr(self, name):
self._dispatch[value] = getattr(self, name)
+
self._dispatch[token.NEWLINE] = self.com_NEWLINE
self._atom_dispatch = {token.LPAR: self.atom_lpar,
token.LSQB: self.atom_lsqb,
@@ -223,7 +224,6 @@
assert isinstance(stmts[0], Discard)
assert isinstance(stmts[0].expr, Const)
del stmts[0]
-
return Module(doc, Stmt(stmts))
def eval_input(self, nodelist):
@@ -608,6 +608,7 @@
return self.com_generator_expression(test, nodelist[1])
return self.testlist(nodelist)
+
def test(self, nodelist):
# test: or_test ['if' or_test 'else' test] | lambdef
if len(nodelist) == 1:
@@ -618,8 +619,10 @@
return self.com_node(nodelist[0])
else:
# Here we implement conditional expressions
- return ast.CondExpr(nodelist[2], nodelist[0], nodelist[4],
- nodelist[1].lineno)
+ # XXX: CPython's nodename is IfExp, not CondExpr
+ return CondExpr(delist[2], nodelist[0], nodelist[4],
+ nodelist[1].lineno)
+
def and_test(self, nodelist):
# not_test ('and' not_test)*
@@ -632,6 +635,9 @@
assert len(nodelist) == 1
return self.com_node(nodelist[0])
+ # XXX
+ # test = old_test
+
def or_test(self, nodelist):
# or_test: and_test ('or' and_test)*
return self.com_binary(Or, nodelist)
@@ -920,7 +926,7 @@
# String together the dotted names and return the string
name = ""
for n in node:
- if type(n) == type(()) and n[0] == 1:
+ if type(n) == type(()) and n[0] == stable_parser.tokens['NAME']:
name = name + n[1] + '.'
return name[:-1]
Modified: pypy/branch/ast-experiments/pypy/module/recparser/pyparser.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/module/recparser/pyparser.py (original)
+++ pypy/branch/ast-experiments/pypy/module/recparser/pyparser.py Thu Dec 14 17:28:05 2006
@@ -8,7 +8,7 @@
from pypy.interpreter.typedef import interp_attrproperty, GetSetProperty
from pypy.interpreter.pycode import PyCode
from pypy.interpreter.pyparser.syntaxtree import TokenNode, SyntaxNode, AbstractSyntaxVisitor
-from pypy.interpreter.pyparser.pythonutil import PYTHON_PARSER
+from pypy.interpreter.pyparser.pythonparse import PYTHON_PARSER
from pypy.interpreter.pyparser.error import SyntaxError
from pypy.interpreter.pyparser import grammar, symbol, pytoken
from pypy.interpreter.argument import Arguments
More information about the Pypy-commit
mailing list