[pypy-svn] r35169 - pypy/dist/pypy/rlib/parsing/test
cfbolz at codespeak.net
cfbolz at codespeak.net
Thu Nov 30 17:02:21 CET 2006
Author: cfbolz
Date: Thu Nov 30 17:02:20 2006
New Revision: 35169
Modified:
pypy/dist/pypy/rlib/parsing/test/pygrammar.txt
pypy/dist/pypy/rlib/parsing/test/test_pythonparse.py
Log:
some more experiments making the grammar nicer
Modified: pypy/dist/pypy/rlib/parsing/test/pygrammar.txt
==============================================================================
--- pypy/dist/pypy/rlib/parsing/test/pygrammar.txt (original)
+++ pypy/dist/pypy/rlib/parsing/test/pygrammar.txt Thu Nov 30 17:02:20 2006
@@ -22,8 +22,8 @@
decorators: decorator+;
funcdef: decorators? ["def"] NAME parameters [":"] suite;
parameters: ["("] >varargslist< [")"] | ["("] [")"];
-varargslist: (parameter ",")* >star_or_starstarargs< |
- parameter ("," parameter)* ","?;
+varargslist: (parameter [","])* >star_or_starstarargs< |
+ parameter ([","] parameter)* [","]?;
parameter: fpdef ("=" test)?;
star_or_starstarargs: starargs [","] starstarargs | starargs | starstarargs;
starargs: ["*"] NAME;
@@ -32,26 +32,29 @@
fplist: fpdef ("," fpdef)* ","?;
stmt: <simple_stmt> | <compound_stmt>;
-simple_stmt: small_stmt ([";"] small_stmt)* [";"]? [NEWLINE];
-small_stmt: <expr_stmt> | <print_stmt> | <del_stmt> | <pass_stmt> |
- <flow_stmt> | <import_stmt> | <global_stmt> | <exec_stmt> |
- <assert_stmt>;
-expr_stmt: testlist augassign testlist | testlist ("=" testlist)*;
+simple_stmt: small_stmt ([";"] small_stmt)+ [";"]? [NEWLINE] |
+ <small_stmt> [";"]? [NEWLINE];
+small_stmt: <expr_stmt> | <print_stmt> | <del_stmt> |
+ <pass_stmt> | <flow_stmt> | <import_stmt> | <global_stmt> |
+ <exec_stmt> | <assert_stmt>;
+expr_stmt: <augassign_stmt> | testlist (["="] testlist)*;
+augassign_stmt: testlist >augassign< testlist;
augassign: "+=" | "-=" | "*=" | "/=" | "%=" | "&=" | "|=" | "^=" | "<<=" |
">>=" | "**=" | "//=";
# For normal assignments, additional restrictions enforced by the interpreter
-print_stmt: "print" ">>" test (("," test)+ ","?)? |
- "print" (test ("," test)* ","?)?;
-del_stmt: "del" exprlist;
-pass_stmt: "pass";
+print_stmt: ["print"] print_target (([","] test)+ [","]?)? |
+ ["print"] (test ([","] test)* [","]?)?;
+print_target: [">>"] test;
+del_stmt: ["del"] exprlist;
+pass_stmt: ["pass"];
flow_stmt: <break_stmt> | <continue_stmt> | <return_stmt> | <raise_stmt> |
<yield_stmt>;
-break_stmt: "break";
-continue_stmt: "continue";
+break_stmt: ["break"];
+continue_stmt: ["continue"];
return_stmt: ["return"] testlist?;
-yield_stmt: "yield" testlist;
-raise_stmt: "raise" (test ("," test ("," test)?)?)?;
+yield_stmt: ["yield"] testlist;
+raise_stmt: ["raise"] (test ("," test ("," test)?)?)?;
import_stmt: import_name | import_from;
import_name: "import" dotted_as_names;
import_from: "from" dotted_name "import" import_what;
@@ -71,12 +74,12 @@
<funcdef> | <classdef>;
if_stmt: "if" test ":" suite ("elif" test ":" suite)* ("else" ":" suite)?;
while_stmt: ["while"] test [":"] suite (["else" ":"] suite)?;
-for_stmt: "for" exprlist "in" testlist ":" suite ("else" ":" suite)?;
+for_stmt: ["for"] exprlist ["in"] testlist [":"] suite (["else" ":"] suite)?;
try_stmt: "try" ":" suite (except_clause ":" suite)+
("else" ":" suite)? | "try" ":" suite "finally" ":" suite;
except_clause: "except" (test ("," test)?)?;
-suite: simple_stmt | [NEWLINE] [INDENT] stmt+ [DEDENT];
+suite: >simple_stmt< | [NEWLINE] [INDENT] stmt+ [DEDENT];
test: and_test "or" test | <and_test> | <lambdef>;
and_test: not_test "and" and_test | <not_test>;
@@ -100,13 +103,19 @@
<factor>;
factor: "+" factor | "-" factor | "~" factor | <power>;
power: atom trailer+ ("**" factor)? | atom "**" factor | <atom>;
-atom: "(" testlist_gexp? ")" | "[" listmaker? "]" |
- "{" dictmaker? "}" | "`" testlist1 "`" | <NAME> | <NUMBER> | STRING+;
-listmaker: test list_for |
- test ("," test)* ","?;
+
+atom: "(" testlist_gexp? ")" | <listliteral> | <dictliteral> |
+ "`" testlist1 "`" | <NAME> | <NUMBER> | STRING+;
+listliteral: ["[" "]"] | "[" <listcomprehension> "]" | ["["] >listmaker< ["]"];
+listcomprehension: test list_for;
+listmaker: test ([","] test)* [","]?;
testlist_gexp: test gen_for |
test ("," test)* ","?;
-lambdef: "lambda" varargslist? ":" test;
+
+dictliteral: ["{" "}"] | ["{"] dictitem ([","] dictitem)* [","]? ["}"];
+dictitem: test [":"] test;
+
+lambdef: ["lambda"] varargslist? [":"] test;
trailer: "(" ")" | "(" arglist ")" | "[" subscriptlist "]" | "." NAME;
subscriptlist: subscript ("," subscript)* ","?;
subscript: "." "." "." | test? ":" test? sliceop? | test;
@@ -114,7 +123,6 @@
exprlist: expr ("," expr)* ","?;
testlist: test ("," test)* ","?;
testlist_safe: test (("," test)+ ","?)?;
-dictmaker: test ":" test ("," test ":" test)* ","?;
classdef: ["class"] NAME (["("] testlist [")"])? [":"] suite;
@@ -122,13 +130,13 @@
arglist_rest: "*" test ("," "**" test)? | "**" test | argument ","?;
argument: (test "=")? test gen_for?;
-list_iter: list_for | list_if;
-list_for: "for" exprlist "in" testlist_safe list_iter?;
-list_if: "if" test list_iter?;
-
-gen_iter: gen_for | gen_if;
-gen_for: "for" exprlist "in" test gen_iter?;
-gen_if: "if" test gen_iter?;
+list_iter: <list_for> | <list_if>;
+list_for: ["for"] exprlist ["in"] testlist_safe list_iter?;
+list_if: ["if"] test list_iter?;
+
+gen_iter: <gen_for> | <gen_if>;
+gen_for: ["for"] exprlist ["in"] test gen_iter?;
+gen_if: ["if"] test gen_iter?;
testlist1: test ("," test)*;
Modified: pypy/dist/pypy/rlib/parsing/test/test_pythonparse.py
==============================================================================
--- pypy/dist/pypy/rlib/parsing/test/test_pythonparse.py (original)
+++ pypy/dist/pypy/rlib/parsing/test/test_pythonparse.py Thu Nov 30 17:02:20 2006
@@ -18,12 +18,12 @@
IGNORE: " ";
NAME: "[a-zA-Z_]*";
NUMBER: "0|[1-9][0-9]*";
-parameters: ["("] >varargslist< [")"] | ["("] [")"];
-varargslist: (fpdef ("=" test)? ",")* star_or_starstarargs |
- fpdef ("=" test)? ("," fpdef ("=" test)?)* ","?;
-star_or_starstarargs: "*" NAME "," "**" NAME | "*" NAME | "**" NAME;
-fpdef: NAME | "(" fplist ")";
-fplist: fpdef ("," fpdef)* ","?;
+parameters: ["("] >varargslist<? [")"];
+varargslist: (fpdef ("=" test)? [","])* star_or_starstarargs |
+ fpdef ("=" test)? ([","] fpdef ("=" test)?)* [","]?;
+star_or_starstarargs: "*" NAME [","] "**" NAME | "*" NAME | "**" NAME;
+fpdef: <NAME> | "(" <fplist> ")";
+fplist: fpdef ([","] fpdef)* [","]?;
test: NUMBER;
""")
parse = make_parse_function(regexs, rules)
@@ -32,6 +32,7 @@
t = parse("(a,b,c,d)").visit(ToAST())[0]
t = parse("(a,b,c,d,)").visit(ToAST())[0]
t = parse("((a, b, c),b,c,d,)").visit(ToAST())[0]
+ t = parse("((a, b, (d, e, (f, g))), b, *args, **kwargs)").visit(ToAST())[0]
t = parse("((a, b, c),b,c,d,*args)").visit(ToAST())[0]
t = parse("((a, b, c),b,c,d,**kwargs)").visit(ToAST())[0]
t = parse("((a, b, c),b,c,d,*args, **args)").visit(ToAST())[0]
@@ -90,6 +91,7 @@
readline = cStringIO.StringIO(source).readline
for token in tokenize.generate_tokens(readline):
typ, s, (row, col), _, line = token
+ row -= 1
pos += len(s)
typ = tokenize.tok_name[typ]
if typ == "ENDMARKER":
@@ -118,9 +120,7 @@
pass
return null - x
""")
- t = t.visit(self.ToAST)
- assert len(t) == 1
- t = t[0]
+ t = self.ToAST.transform(t)
def test_class(self):
t = self.parse("""
@@ -135,7 +135,7 @@
def diagonal(self):
return (self.a ** 2 + self.b ** 2) ** 0.5
""")
- t = t.visit(self.ToAST)[0]
+ t = self.ToAST.transform(t)
def test_while(self):
t = self.parse("""
@@ -151,9 +151,7 @@
return result - 15
return result
""")
- t = t.visit(self.ToAST)
- assert len(t) == 1
- t = t[0]
+ t = self.ToAST.transform(t)
def test_comment(self):
t = self.parse("""
@@ -163,8 +161,77 @@
""")
t = self.ToAST.transform(t)
+ def test_parse_print(self):
+ t = self.parse("""
+print >> f, a, b, c,
+print >> f, a, b
+print >> f
+print
+print 1
+print 1, 2
+print 1, 2,
+""")
+ t = self.ToAST.transform(t)
+
+ def test_assignment(self):
+ t = self.parse("""
+a = 1
+a = b = c
+(a, b) = c
+a += 1
+b //= 3
+""")
+ t = self.ToAST.transform(t)
+
+ def test_lists(self):
+ t = self.parse("""
+l0 = [1, 2, [3, 4, [5, []]]]
+l1 = [i for i in range(10)]
+l1 = [i for i in range(10) if i ** 2 % 3 == 0]
+l2 = [ ]
+l3 = [ ]
+l4 = []
+""")
+ t = self.ToAST.transform(t)
+
+ def test_dicts(self):
+ t = self.parse("""
+{1: 2, 2: {1: 3},}
+{}
+""")
+ t = self.ToAST.transform(t)
+
+ def test_calls(self):
+ #XXX horrible trees
+ t = self.parse("""
+f(a)(b)(c)
+f() ** 2
+f(x) * 2
+f(x, y, z, *args) + 2
+""")
+ t = self.ToAST.transform(t)
+
+ def test_errors(self):
+ source = """
+def f(x):
+ if a:
+ pass
+ else:
+ pass
+ else:
+ pass
+"""
+ excinfo = py.test.raises(ParseError, self.parse, source)
+ error = excinfo.value.errorinformation
+ msg = excinfo.value.nice_error_message("<stdin>", source)
+
def test_parse_this(self):
s = py.magic.autopath().read()
- s = s[s.index("\nclass"):]
t = self.parse(s)
- t = t.visit(self.ToAST)[0]
+ t = self.ToAST.transform(t)
+
+ def test_parsing(self):
+ s = py.magic.autopath().dirpath().dirpath().join("parsing.py").read()
+ t = self.parse(s)
+ t = self.ToAST.transform(t)
+
More information about the Pypy-commit
mailing list