[Python-checkins] r82399 - in python/trunk/Demo/parser: README test_parser.py test_unparse.py texipre.dat unparse.py
mark.dickinson
python-checkins at python.org
Wed Jun 30 16:19:56 CEST 2010
Author: mark.dickinson
Date: Wed Jun 30 16:19:56 2010
New Revision: 82399
Log:
Update Demo/parser directory; backport unparse fixes from py3k.
Removed:
python/trunk/Demo/parser/texipre.dat
Modified:
python/trunk/Demo/parser/README
python/trunk/Demo/parser/test_parser.py
python/trunk/Demo/parser/test_unparse.py
python/trunk/Demo/parser/unparse.py
Modified: python/trunk/Demo/parser/README
==============================================================================
--- python/trunk/Demo/parser/README (original)
+++ python/trunk/Demo/parser/README Wed Jun 30 16:19:56 2010
@@ -6,26 +6,27 @@
Files:
------
- FILES -- list of files associated with the parser module.
+ FILES -- list of files associated with the parser module.
- README -- this file.
+ README -- this file.
- example.py -- module that uses the `parser' module to extract
- information from the parse tree of Python source
- code.
+ docstring.py -- sample source file containing only a module docstring.
- docstring.py -- sample source file containing only a module docstring.
+ example.py -- module that uses the `parser' module to extract
+ information from the parse tree of Python source
+ code.
- simple.py -- sample source containing a "short form" definition.
+ simple.py -- sample source containing a "short form" definition.
- source.py -- sample source code used to demonstrate ability to
- handle nested constructs easily using the functions
- and classes in example.py.
+ source.py -- sample source code used to demonstrate ability to
+ handle nested constructs easily using the functions
+ and classes in example.py.
- test_parser.py program to put the parser module through its paces.
+ test_parser.py program to put the parser module through its paces.
- unparse.py AST (2.5) based example to recreate source code
- from an AST. This is incomplete; contributions
- are welcome.
+ test_unparse.py tests for the unparse module
+
+ unparse.py AST (2.7) based example to recreate source code
+ from an AST.
Enjoy!
Modified: python/trunk/Demo/parser/test_parser.py
==============================================================================
--- python/trunk/Demo/parser/test_parser.py (original)
+++ python/trunk/Demo/parser/test_parser.py Wed Jun 30 16:19:56 2010
@@ -11,19 +11,19 @@
global _numFailed
print '----', fileName,
try:
- ast = parser.suite(t)
- tup = parser.ast2tuple(ast)
- # this discards the first AST; a huge memory savings when running
+ st = parser.suite(t)
+ tup = parser.st2tuple(st)
+ # this discards the first ST; a huge memory savings when running
# against a large source file like Tkinter.py.
- ast = None
- new = parser.tuple2ast(tup)
+ st = None
+ new = parser.tuple2st(tup)
except parser.ParserError, err:
print
print 'parser module raised exception on input file', fileName + ':'
traceback.print_exc()
_numFailed = _numFailed + 1
else:
- if tup != parser.ast2tuple(new):
+ if tup != parser.st2tuple(new):
print
print 'parser module failed on input file', fileName
_numFailed = _numFailed + 1
Modified: python/trunk/Demo/parser/test_unparse.py
==============================================================================
--- python/trunk/Demo/parser/test_unparse.py (original)
+++ python/trunk/Demo/parser/test_unparse.py Wed Jun 30 16:19:56 2010
@@ -1,12 +1,20 @@
import unittest
from test import test_support
-
import cStringIO
+import sys
+import os
+import tokenize
import ast
-import _ast
import unparse
-forelse = """\
+def read_pyfile(filename):
+ """Read and return the contents of a Python source file (as a
+ string), taking into account the file encoding."""
+ with open(filename, "r") as pyfile:
+ source = pyfile.read()
+ return source
+
+for_else = """\
def f():
for x in range(10):
break
@@ -15,7 +23,7 @@
z = 3
"""
-whileelse = """\
+while_else = """\
def g():
while True:
break
@@ -24,16 +32,63 @@
z = 3
"""
-class UnparseTestCase(unittest.TestCase):
- # Tests for specific bugs found in earlier versions of unparse
+relative_import = """\
+from . import fred
+from .. import barney
+from .australia import shrimp as prawns
+"""
+
+class_decorator = """\
+ at f1(arg)
+ at f2
+class Foo: pass
+"""
+
+elif1 = """\
+if cond1:
+ suite1
+elif cond2:
+ suite2
+else:
+ suite3
+"""
+
+elif2 = """\
+if cond1:
+ suite1
+elif cond2:
+ suite2
+"""
+
+try_except_finally = """\
+try:
+ suite1
+except ex1:
+ suite2
+except ex2:
+ suite3
+else:
+ suite4
+finally:
+ suite5
+"""
+
+class ASTTestCase(unittest.TestCase):
+ def assertASTEqual(self, ast1, ast2):
+ dump1 = ast.dump(ast1)
+ dump2 = ast.dump(ast2)
+ self.assertEqual(ast.dump(ast1), ast.dump(ast2))
def check_roundtrip(self, code1, filename="internal"):
- ast1 = compile(code1, filename, "exec", _ast.PyCF_ONLY_AST)
+ ast1 = compile(code1, filename, "exec", ast.PyCF_ONLY_AST)
unparse_buffer = cStringIO.StringIO()
unparse.Unparser(ast1, unparse_buffer)
code2 = unparse_buffer.getvalue()
- ast2 = compile(code2, filename, "exec", _ast.PyCF_ONLY_AST)
- self.assertEqual(ast.dump(ast1), ast.dump(ast2))
+ ast2 = compile(code2, filename, "exec", ast.PyCF_ONLY_AST)
+ self.assertASTEqual(ast1, ast2)
+
+class UnparseTestCase(ASTTestCase):
+ # Tests for specific bugs found in earlier versions of unparse
def test_del_statement(self):
self.check_roundtrip("del x, y, z")
@@ -43,23 +98,116 @@
self.check_roundtrip("13 >> 7")
def test_for_else(self):
- self.check_roundtrip(forelse)
+ self.check_roundtrip(for_else)
def test_while_else(self):
- self.check_roundtrip(whileelse)
+ self.check_roundtrip(while_else)
def test_unary_parens(self):
self.check_roundtrip("(-1)**7")
+ self.check_roundtrip("(-1.)**8")
+ self.check_roundtrip("(-1j)**6")
self.check_roundtrip("not True or False")
self.check_roundtrip("True or not False")
+ def test_integer_parens(self):
+ self.check_roundtrip("3 .__abs__()")
+
+ def test_huge_float(self):
+ self.check_roundtrip("1e1000")
+ self.check_roundtrip("-1e1000")
+ self.check_roundtrip("1e1000j")
+ self.check_roundtrip("-1e1000j")
+
+ def test_min_int(self):
+ self.check_roundtrip(str(-sys.maxint-1))
+ self.check_roundtrip("-(%s)" % (sys.maxint + 1))
+
+ def test_imaginary_literals(self):
+ self.check_roundtrip("7j")
+ self.check_roundtrip("-7j")
+ self.check_roundtrip("-(7j)")
+ self.check_roundtrip("0j")
+ self.check_roundtrip("-0j")
+ self.check_roundtrip("-(0j)")
+
+ def test_negative_zero(self):
+ self.check_roundtrip("-0")
+ self.check_roundtrip("-(0)")
+ self.check_roundtrip("-0b0")
+ self.check_roundtrip("-(0b0)")
+ self.check_roundtrip("-0o0")
+ self.check_roundtrip("-(0o0)")
+ self.check_roundtrip("-0x0")
+ self.check_roundtrip("-(0x0)")
+
+ def test_lambda_parentheses(self):
+ self.check_roundtrip("(lambda: int)()")
+
def test_chained_comparisons(self):
self.check_roundtrip("1 < 4 <= 5")
self.check_roundtrip("a is b is c is not d")
+ def test_function_arguments(self):
+ self.check_roundtrip("def f(): pass")
+ self.check_roundtrip("def f(a): pass")
+ self.check_roundtrip("def f(b = 2): pass")
+ self.check_roundtrip("def f(a, b): pass")
+ self.check_roundtrip("def f(a, b = 2): pass")
+ self.check_roundtrip("def f(a = 5, b = 2): pass")
+ self.check_roundtrip("def f(*args, **kwargs): pass")
+
+ def test_relative_import(self):
+ self.check_roundtrip(relative_import)
+
+ def test_bytes(self):
+ self.check_roundtrip("b'123'")
+
+ def test_set_literal(self):
+ self.check_roundtrip("{'a', 'b', 'c'}")
+
+ def test_set_comprehension(self):
+ self.check_roundtrip("{x for x in range(5)}")
+
+ def test_dict_comprehension(self):
+ self.check_roundtrip("{x: x*x for x in range(10)}")
+
+ def test_class_decorators(self):
+ self.check_roundtrip(class_decorator)
+
+ def test_elifs(self):
+ self.check_roundtrip(elif1)
+ self.check_roundtrip(elif2)
+
+ def test_try_except_finally(self):
+ self.check_roundtrip(try_except_finally)
+
+class DirectoryTestCase(ASTTestCase):
+ """Test roundtrip behaviour on all files in Lib and Lib/test."""
+
+ # test directories, relative to the root of the distribution
+ test_directories = 'Lib', os.path.join('Lib', 'test')
+
+ def test_files(self):
+ # get names of files to test
+ dist_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
+
+ names = []
+ for d in self.test_directories:
+ test_dir = os.path.join(dist_dir, d)
+ for n in os.listdir(test_dir):
+ if n.endswith('.py') and not n.startswith('bad'):
+ names.append(os.path.join(test_dir, n))
+
+ for filename in names:
+ if test_support.verbose:
+ print('Testing %s' % filename)
+ source = read_pyfile(filename)
+ self.check_roundtrip(source)
+
def test_main():
- test_support.run_unittest(UnparseTestCase)
+ test_support.run_unittest(UnparseTestCase, DirectoryTestCase)
if __name__ == '__main__':
test_main()
Deleted: python/trunk/Demo/parser/texipre.dat
==============================================================================
--- python/trunk/Demo/parser/texipre.dat Wed Jun 30 16:19:56 2010
+++ (empty file)
@@ -1,100 +0,0 @@
-\input texinfo @c -*-texinfo-*-
- at c %**start of header
- at setfilename parser.info
- at settitle Python Parser Module Reference
- at setchapternewpage odd
- at footnotestyle end
- at c %**end of header
-
- at ifinfo
-This file describes the interfaces
-published by the optional @code{parser} module and gives examples of
-how they may be used. It contains the same text as the chapter on the
- at code{parser} module in the @cite{Python Library Reference}, but is
-presented as a separate document.
-
-Copyright 1995-1996 by Fred L. Drake, Jr., Reston, Virginia, USA, and
-Virginia Polytechnic Institute and State University, Blacksburg,
-Virginia, USA. Portions of the software copyright 1991-1995 by
-Stichting Mathematisch Centrum, Amsterdam, The Netherlands. Copying is
-permitted under the terms associated with the main Python distribution,
-with the additional restriction that this additional notice be included
-and maintained on all distributed copies.
-
- All Rights Reserved
-
-Permission to use, copy, modify, and distribute this software and its
-documentation for any purpose and without fee is hereby granted,
-provided that the above copyright notice appear in all copies and that
-both that copyright notice and this permission notice appear in
-supporting documentation, and that the names of Fred L. Drake, Jr. and
-Virginia Polytechnic Institute and State University not be used in
-advertising or publicity pertaining to distribution of the software
-without specific, written prior permission.
-
-FRED L. DRAKE, JR. AND VIRGINIA POLYTECHNIC INSTITUTE AND STATE
-UNIVERSITY DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
-INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
-EVENT SHALL FRED L. DRAKE, JR. OR VIRGINIA POLYTECHNIC INSTITUTE AND
-STATE UNIVERSITY BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL
-DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
-PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
-TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-PERFORMANCE OF THIS SOFTWARE.
- at end ifinfo
-
- at titlepage
- at title Python Parser Module Reference
- at author Fred L. Drake, Jr.
-
- at c The following two commands start the copyright page.
- at page
- at vskip 0pt plus 1filll
-Copyright 1995-1996 by Fred L. Drake, Jr., Reston, Virginia, USA, and
-Virginia Polytechnic Institute and State University, Blacksburg,
-Virginia, USA. Portions of the software copyright 1991-1995 by
-Stichting Mathematisch Centrum, Amsterdam, The Netherlands. Copying is
-permitted under the terms associated with the main Python distribution,
-with the additional restriction that this additional notice be included
-and maintained on all distributed copies.
-
- at center All Rights Reserved
-
-Permission to use, copy, modify, and distribute this software and its
-documentation for any purpose and without fee is hereby granted,
-provided that the above copyright notice appear in all copies and that
-both that copyright notice and this permission notice appear in
-supporting documentation, and that the names of Fred L. Drake, Jr. and
-Virginia Polytechnic Institute and State University not be used in
-advertising or publicity pertaining to distribution of the software
-without specific, written prior permission.
-
-FRED L. DRAKE, JR. AND VIRGINIA POLYTECHNIC INSTITUTE AND STATE
-UNIVERSITY DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
-INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
-EVENT SHALL FRED L. DRAKE, JR. OR VIRGINIA POLYTECHNIC INSTITUTE AND
-STATE UNIVERSITY BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL
-DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
-PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
-TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-PERFORMANCE OF THIS SOFTWARE.
- at end titlepage
-
-
- at node Top, Overview, (dir), (dir)
- at top The Python Parser Module
-
- at ifinfo
-This file describes the interfaces
-published by the optional @code{parser} module and gives examples of
-how they may be used. It contains the same text as the chapter on the
- at code{parser} module in the @cite{Python Library Reference}, but is
-presented as a separate document.
-
-This version corresponds to Python version 1.4 (1 Sept. 1996).
-
- at end ifinfo
-
- at c placeholder for the master menu -- patched by texinfo-all-menus-update
- at menu
- at end menu
Modified: python/trunk/Demo/parser/unparse.py
==============================================================================
--- python/trunk/Demo/parser/unparse.py (original)
+++ python/trunk/Demo/parser/unparse.py Wed Jun 30 16:19:56 2010
@@ -1,15 +1,19 @@
"Usage: unparse.py <path to source file>"
import sys
-import _ast
+import ast
import cStringIO
import os
+# Large float and imaginary literals get turned into infinities in the AST.
+# We unparse those infinities to INFSTR.
+INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1)
+
def interleave(inter, f, seq):
"""Call f on each item in seq, calling inter() in between.
"""
seq = iter(seq)
try:
- f(seq.next())
+ f(next(seq))
except StopIteration:
pass
else:
@@ -20,15 +24,16 @@
class Unparser:
"""Methods in this class recursively traverse an AST and
output source code for the abstract syntax; original formatting
- is disregarged. """
+ is disregarded. """
def __init__(self, tree, file = sys.stdout):
"""Unparser(tree, file=sys.stdout) -> None.
Print the source for tree to file."""
self.f = file
+ self.future_imports = []
self._indent = 0
self.dispatch(tree)
- print >>self.f,""
+ self.f.write("")
self.f.flush()
def fill(self, text = ""):
@@ -79,11 +84,16 @@
interleave(lambda: self.write(", "), self.dispatch, t.names)
def _ImportFrom(self, t):
+ # A from __future__ import may affect unparsing, so record it.
+ if t.module and t.module == '__future__':
+ self.future_imports.extend(n.name for n in t.names)
+
self.fill("from ")
- self.write(t.module)
+ self.write("." * t.level)
+ if t.module:
+ self.write(t.module)
self.write(" import ")
interleave(lambda: self.write(", "), self.dispatch, t.names)
- # XXX(jpe) what is level for?
def _Assign(self, t):
self.fill()
@@ -186,10 +196,14 @@
self.leave()
def _TryFinally(self, t):
- self.fill("try")
- self.enter()
- self.dispatch(t.body)
- self.leave()
+ if len(t.body) == 1 and isinstance(t.body[0], ast.TryExcept):
+ # try-except-finally
+ self.dispatch(t.body)
+ else:
+ self.fill("try")
+ self.enter()
+ self.dispatch(t.body)
+ self.leave()
self.fill("finally")
self.enter()
@@ -202,7 +216,7 @@
self.write(" ")
self.dispatch(t.type)
if t.name:
- self.write(", ")
+ self.write(" as ")
self.dispatch(t.name)
self.enter()
self.dispatch(t.body)
@@ -210,6 +224,9 @@
def _ClassDef(self, t):
self.write("\n")
+ for deco in t.decorator_list:
+ self.fill("@")
+ self.dispatch(deco)
self.fill("class "+t.name)
if t.bases:
self.write("(")
@@ -251,9 +268,18 @@
self.fill("if ")
self.dispatch(t.test)
self.enter()
- # XXX elif?
self.dispatch(t.body)
self.leave()
+ # collapse nested ifs into equivalent elifs.
+ while (t.orelse and len(t.orelse) == 1 and
+ isinstance(t.orelse[0], ast.If)):
+ t = t.orelse[0]
+ self.fill("elif ")
+ self.dispatch(t.test)
+ self.enter()
+ self.dispatch(t.body)
+ self.leave()
+ # final else
if t.orelse:
self.fill("else")
self.enter()
@@ -284,7 +310,17 @@
# expr
def _Str(self, tree):
- self.write(repr(tree.s))
+ # if from __future__ import unicode_literals is in effect,
+ # then we want to output string literals using a 'b' prefix
+ # and unicode literals with no prefix.
+ if "unicode_literals" not in self.future_imports:
+ self.write(repr(tree.s))
+ elif isinstance(tree.s, str):
+ self.write("b" + repr(tree.s))
+ elif isinstance(tree.s, unicode):
+ self.write(repr(tree.s).lstrip("u"))
+ else:
+ assert False, "shouldn't get here"
def _Name(self, t):
self.write(t.id)
@@ -295,16 +331,14 @@
self.write("`")
def _Num(self, t):
- # There are no negative numeric literals in Python; however,
- # some optimizations produce a negative Num in the AST. Add
- # parentheses to avoid turning (-1)**2 into -1**2.
- strnum = repr(t.n)
- if strnum.startswith("-"):
+ repr_n = repr(t.n)
+ # Parenthesize negative numbers, to avoid turning (-1)**2 into -1**2.
+ if repr_n.startswith("-"):
self.write("(")
- self.write(strnum)
+ # Substitute overflowing decimal literal for AST infinities.
+ self.write(repr_n.replace("inf", INFSTR))
+ if repr_n.startswith("-"):
self.write(")")
- else:
- self.write(repr(t.n))
def _List(self, t):
self.write("[")
@@ -325,6 +359,22 @@
self.dispatch(gen)
self.write(")")
+ def _SetComp(self, t):
+ self.write("{")
+ self.dispatch(t.elt)
+ for gen in t.generators:
+ self.dispatch(gen)
+ self.write("}")
+
+ def _DictComp(self, t):
+ self.write("{")
+ self.dispatch(t.key)
+ self.write(": ")
+ self.dispatch(t.value)
+ for gen in t.generators:
+ self.dispatch(gen)
+ self.write("}")
+
def _comprehension(self, t):
self.write(" for ")
self.dispatch(t.target)
@@ -343,13 +393,20 @@
self.dispatch(t.orelse)
self.write(")")
+ def _Set(self, t):
+ assert(t.elts) # should be at least one element
+ self.write("{")
+ interleave(lambda: self.write(", "), self.dispatch, t.elts)
+ self.write("}")
+
def _Dict(self, t):
self.write("{")
- def writem((k, v)):
+ def write_pair(pair):
+ (k, v) = pair
self.dispatch(k)
self.write(": ")
self.dispatch(v)
- interleave(lambda: self.write(", "), writem, zip(t.keys, t.values))
+ interleave(lambda: self.write(", "), write_pair, zip(t.keys, t.values))
self.write("}")
def _Tuple(self, t):
@@ -367,7 +424,17 @@
self.write("(")
self.write(self.unop[t.op.__class__.__name__])
self.write(" ")
- self.dispatch(t.operand)
+ # If we're applying unary minus to a number, parenthesize the number.
+ # This is necessary: -2147483648 is different from -(2147483648) on
+ # a 32-bit machine (the first is an int, the second a long), and
+ # -7j is different from -(7j). (The first has real part 0.0, the second
+ # has real part -0.0.)
+ if isinstance(t.op, ast.USub) and isinstance(t.operand, ast.Num):
+ self.write("(")
+ self.dispatch(t.operand)
+ self.write(")")
+ else:
+ self.dispatch(t.operand)
self.write(")")
binop = { "Add":"+", "Sub":"-", "Mult":"*", "Div":"/", "Mod":"%",
@@ -390,7 +457,7 @@
self.dispatch(e)
self.write(")")
- boolops = {_ast.And: 'and', _ast.Or: 'or'}
+ boolops = {ast.And: 'and', ast.Or: 'or'}
def _BoolOp(self, t):
self.write("(")
s = " %s " % self.boolops[t.op.__class__]
@@ -399,6 +466,11 @@
def _Attribute(self,t):
self.dispatch(t.value)
+ # Special case: 3.__abs__() is a syntax error, so if t.value
+ # is an integer literal then we need to either parenthesize
+ # it or add an extra space to get 3 .__abs__().
+ if isinstance(t.value, ast.Num) and isinstance(t.value.n, int):
+ self.write(" ")
self.write(".")
self.write(t.attr)
@@ -455,21 +527,24 @@
# others
def _arguments(self, t):
first = True
- nonDef = len(t.args)-len(t.defaults)
- for a in t.args[0:nonDef]:
- if first:first = False
- else: self.write(", ")
- self.dispatch(a)
- for a,d in zip(t.args[nonDef:], t.defaults):
+ # normal arguments
+ defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults
+ for a,d in zip(t.args, defaults):
if first:first = False
else: self.write(", ")
self.dispatch(a),
- self.write("=")
- self.dispatch(d)
+ if d:
+ self.write("=")
+ self.dispatch(d)
+
+ # varargs
if t.vararg:
if first:first = False
else: self.write(", ")
- self.write("*"+t.vararg)
+ self.write("*")
+ self.write(t.vararg)
+
+ # kwargs
if t.kwarg:
if first:first = False
else: self.write(", ")
@@ -481,10 +556,12 @@
self.dispatch(t.value)
def _Lambda(self, t):
+ self.write("(")
self.write("lambda ")
self.dispatch(t.args)
self.write(": ")
self.dispatch(t.body)
+ self.write(")")
def _alias(self, t):
self.write(t.name)
@@ -492,8 +569,9 @@
self.write(" as "+t.asname)
def roundtrip(filename, output=sys.stdout):
- source = open(filename).read()
- tree = compile(source, filename, "exec", _ast.PyCF_ONLY_AST)
+ with open(filename, "r") as pyfile:
+ source = pyfile.read()
+ tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST)
Unparser(tree, output)
@@ -502,7 +580,7 @@
try:
names = [n for n in os.listdir(a) if n.endswith('.py')]
except OSError:
- print >> sys.stderr, "Directory not readable: %s" % a
+ sys.stderr.write("Directory not readable: %s" % a)
else:
for n in names:
fullname = os.path.join(a, n)
@@ -511,7 +589,7 @@
print 'Testing %s' % fullname
try:
roundtrip(fullname, output)
- except Exception, e:
+ except Exception as e:
print ' Failed to compile, exception is %s' % repr(e)
elif os.path.isdir(fullname):
testdir(fullname)
More information about the Python-checkins
mailing list