[Python-checkins] r56373 - in sandbox/trunk/2to3: fixes/basefix.py fixes/fix_next.py fixes/fix_stringio.py pytree.py refactor.py tests/test_fixers.py tests/test_pytree.py
collin.winter
python-checkins at python.org
Sat Jul 14 20:23:32 CEST 2007
Author: collin.winter
Date: Sat Jul 14 20:23:32 2007
New Revision: 56373
Modified:
sandbox/trunk/2to3/ (props changed)
sandbox/trunk/2to3/fixes/basefix.py
sandbox/trunk/2to3/fixes/fix_next.py
sandbox/trunk/2to3/fixes/fix_stringio.py
sandbox/trunk/2to3/pytree.py
sandbox/trunk/2to3/refactor.py
sandbox/trunk/2to3/tests/test_fixers.py
sandbox/trunk/2to3/tests/test_pytree.py
Log:
Add the ability for fixers to indicate whether they want a pre- or post-order traversal of the AST; change the StringIO and next fixers to take advantage of this (all the delayed-stringification crap is gone).
Modified: sandbox/trunk/2to3/fixes/basefix.py
==============================================================================
--- sandbox/trunk/2to3/fixes/basefix.py (original)
+++ sandbox/trunk/2to3/fixes/basefix.py Sat Jul 14 20:23:32 2007
@@ -34,6 +34,7 @@
logger = None # A logger (set by set_filename)
numbers = itertools.count(1) # For new_name()
used_names = set() # A set of all used NAMEs
+ order = "post" # Does the fixer prefer pre- or post-order traversal
# Shortcut for access to Python grammar symbols
syms = pygram.python_symbols
Modified: sandbox/trunk/2to3/fixes/fix_next.py
==============================================================================
--- sandbox/trunk/2to3/fixes/fix_next.py (original)
+++ sandbox/trunk/2to3/fixes/fix_next.py Sat Jul 14 20:23:32 2007
@@ -6,7 +6,6 @@
# - "with" statement targets aren't checked
# Local imports
-import pytree
from pgen2 import token
from pygram import python_symbols as syms
from fixes import basefix
@@ -15,44 +14,9 @@
bind_warning = "Calls to builtin next() possibly shadowed by global binding"
-class DelayedStrNode(object):
-
- def __init__(self, type, base):
- self.parent = None
- self.shadowed_next = False
- self.base = base
- self.type = type
- self.value = ""
- self.prefix = ""
-
- def __str__(self):
- if self.shadowed_next:
- b = "".join([str(n) for n in self.base])
- return self.prefix + "%s.__next__()" % b
- else:
- b_prefix = prefix = self.base[0].get_prefix()
- self.base[0].set_prefix("")
- b = "".join([str(n) for n in self.base])
- self.base[0].set_prefix(b_prefix)
- return self.prefix + prefix + "next(%s)" % b
-
- def clone(self):
- node = DelayedStrNode(self.type, self.base)
- node.shadowed_next = self.shadowed_next
- node.value = self.value
- node.prefix = self.prefix
- return node
-
- def set_prefix(self, prefix):
- self.prefix = prefix
-
- def get_prefix(self):
- return self.prefix
-
-
class FixNext(basefix.BaseFix):
PATTERN = """
- power< base=any+ trailer< '.' 'next' > trailer< '(' ')' > >
+ power< base=any+ trailer< '.' attr='next' > trailer< '(' ')' > >
|
power< head=any+ trailer< '.' attr='next' > not trailer< '(' ')' > >
|
@@ -68,10 +32,11 @@
mod=file_input< any+ >
"""
+ order = "pre" # Pre-order tree traversal
+
def start_tree(self, tree, filename):
super(FixNext, self).start_tree(tree, filename)
self.shadowed_next = False
- self.delayed = []
def transform(self, node, results):
assert results
@@ -82,9 +47,12 @@
mod = results.get("mod")
if base:
- n = DelayedStrNode(syms.power, base)
- node.replace(n)
- self.delayed.append(n)
+ if self.shadowed_next:
+ attr.replace(Name("__next__", prefix=attr.get_prefix()))
+ else:
+ base = [n.clone() for n in base]
+ base[0].set_prefix("")
+ node.replace(Call(Name("next", prefix=node.get_prefix()), base))
elif name:
n = Name("__next__", prefix=name.get_prefix())
name.replace(n)
@@ -107,12 +75,6 @@
self.warning(n, bind_warning)
self.shadowed_next = True
- def finish_tree(self, tree, filename):
- super(FixNext, self).finish_tree(tree, filename)
- if self.shadowed_next:
- for node in self.delayed:
- node.shadowed_next = True
-
### The following functions help test if node is part of an assignment
### target.
Modified: sandbox/trunk/2to3/fixes/fix_stringio.py
==============================================================================
--- sandbox/trunk/2to3/fixes/fix_stringio.py (original)
+++ sandbox/trunk/2to3/fixes/fix_stringio.py Sat Jul 14 20:23:32 2007
@@ -12,30 +12,10 @@
# Author: Collin Winter
# Local imports
-import patcomp
from fixes import basefix
from fixes.util import Name, attr_chain, any
-class DelayedStrLeaf(object):
- def __init__(self, fixer, leaf):
- self.fixer = fixer
- self.leaf = leaf
- self.parent = None
-
- def __getattr__(self, attr):
- return getattr(self.leaf, attr)
-
- def __str__(self):
- if self.fixer.module_import:
- return self.leaf.get_prefix() + "io"
- else:
- return str(self.leaf)
-
- def clone(self):
- return DelayedStrLeaf(self.fixer, self.leaf)
-
-
class FixStringio(basefix.BaseFix):
PATTERN = """
import_name< 'import' (module='StringIO'
@@ -51,6 +31,8 @@
bare_name='StringIO'
"""
+ order = "pre" # Pre-order tree traversal
+
# Don't match 'StringIO' if it's within another match
def match(self, node):
match = super(FixStringio, self).match
@@ -75,7 +57,5 @@
import_mod.replace(Name("io", prefix=import_mod.get_prefix()))
elif module_name:
module_name.replace(Name("io", prefix=module_name.get_prefix()))
- elif bare_name:
- bare_name.replace(DelayedStrLeaf(self, bare_name))
- else:
- raise RuntimeError("Hmm, shouldn't have gotten here")
+ elif bare_name and self.module_import:
+ bare_name.replace(Name("io", prefix=bare_name.get_prefix()))
Modified: sandbox/trunk/2to3/pytree.py
==============================================================================
--- sandbox/trunk/2to3/pytree.py (original)
+++ sandbox/trunk/2to3/pytree.py Sat Jul 14 20:23:32 2007
@@ -79,6 +79,13 @@
"""
raise NotImplementedError
+ def pre_order(self):
+ """Returns a pre-order iterator for the tree.
+
+ This must be implemented by the concrete subclass.
+ """
+ raise NotImplementedError
+
def set_prefix(self, prefix):
"""Sets the prefix for the node (see Leaf class).
@@ -216,6 +223,13 @@
yield node
yield self
+ def pre_order(self):
+ """Returns a pre-order iterator for the tree."""
+ yield self
+ for child in self.children:
+ for node in child.post_order():
+ yield node
+
def set_prefix(self, prefix):
"""Sets the prefix for the node.
@@ -302,6 +316,10 @@
"""Returns a post-order iterator for the tree."""
yield self
+ def pre_order(self):
+ """Returns a pre-order iterator for the tree."""
+ yield self
+
def set_prefix(self, prefix):
"""Sets the prefix for the node."""
self.changed()
Modified: sandbox/trunk/2to3/refactor.py
==============================================================================
--- sandbox/trunk/2to3/refactor.py (original)
+++ sandbox/trunk/2to3/refactor.py Sat Jul 14 20:23:32 2007
@@ -111,12 +111,19 @@
self.driver = driver.Driver(pygram.python_grammar,
convert=pytree.convert,
logger=self.logger)
- self.fixers = self.get_fixers()
+ self.pre_order, self.post_order = self.get_fixers()
self.files = [] # List of files that were or should be modified
def get_fixers(self):
- """Inspects the options to load the requested patterns and handlers."""
- fixers = []
+ """Inspects the options to load the requested patterns and handlers.
+
+ Returns:
+ (pre_order, post_order), where pre_order is the list of fixers that
+ want a pre-order AST traversal, and post_order is the list that want
+ post-order traversal.
+ """
+ pre_order_fixers = []
+ post_order_fixers = []
fix_names = self.options.fix
if not fix_names or "all" in fix_names:
fix_names = get_all_fix_names()
@@ -142,8 +149,14 @@
continue
if self.options.verbose:
self.log_message("Adding transformation: %s", fix_name)
- fixers.append(fixer)
- return fixers
+
+ if fixer.order == "pre":
+ pre_order_fixers.append(fixer)
+ elif fixer.order == "post":
+ post_order_fixers.append(fixer)
+ else:
+ raise ValueError("Illegal fixer order: %r" % fixer.order)
+ return (pre_order_fixers, post_order_fixers)
def log_error(self, msg, *args, **kwds):
"""Increments error count and log a message."""
@@ -249,11 +262,26 @@
def refactor_tree(self, tree, filename):
"""Refactors a parse tree (modifying the tree in place)."""
- for fixer in self.fixers:
+ changed = False
+ all_fixers = self.pre_order + self.post_order
+ for fixer in all_fixers:
fixer.start_tree(tree, filename)
- changes = 0
- for node in tree.post_order():
- for fixer in self.fixers:
+
+ changed |= self.traverse_by(self.pre_order, tree.pre_order())
+ changed |= self.traverse_by(self.post_order, tree.post_order())
+ if tree.was_changed:
+ changes = True
+
+ for fixer in all_fixers:
+ fixer.finish_tree(tree, filename)
+ return changed
+
+ def traverse_by(self, fixers, traversal):
+ changed = False
+ if not fixers:
+ return changed
+ for node in traversal:
+ for fixer in fixers:
results = fixer.match(node)
if results:
new = fixer.transform(node, results)
@@ -261,12 +289,8 @@
str(new) != str(node)):
node.replace(new)
node = new
- changes += 1
- elif tree.was_changed:
- changes += 1
- for fixer in self.fixers:
- fixer.finish_tree(tree, filename)
- return changes
+ changed = True
+ return changed
def write_file(self, new_text, filename, old_text=None):
"""Writes a string to a file.
Modified: sandbox/trunk/2to3/tests/test_fixers.py
==============================================================================
--- sandbox/trunk/2to3/tests/test_fixers.py (original)
+++ sandbox/trunk/2to3/tests/test_fixers.py Sat Jul 14 20:23:32 2007
@@ -47,7 +47,10 @@
self.logging_stream = StringIO()
sh = logging.StreamHandler(self.logging_stream)
sh.setFormatter(logging.Formatter("%(message)s"))
- self.refactor.fixers = [Fixer(f, sh) for f in self.refactor.fixers]
+ self.refactor.pre_order = [Fixer(f, sh) for f
+ in self.refactor.pre_order]
+ self.refactor.post_order = [Fixer(f, sh) for f
+ in self.refactor.post_order]
def tearDown(self):
self.logging_stream = None
Modified: sandbox/trunk/2to3/tests/test_pytree.py
==============================================================================
--- sandbox/trunk/2to3/tests/test_pytree.py (original)
+++ sandbox/trunk/2to3/tests/test_pytree.py Sat Jul 14 20:23:32 2007
@@ -150,6 +150,18 @@
# XXX
pass
+ def testPostOrder(self):
+ l1 = pytree.Leaf(100, "foo")
+ l2 = pytree.Leaf(100, "bar")
+ n1 = pytree.Node(1000, [l1, l2])
+ self.assertEqual(list(n1.post_order()), [l1, l2, n1])
+
+ def testPreOrder(self):
+ l1 = pytree.Leaf(100, "foo")
+ l2 = pytree.Leaf(100, "bar")
+ n1 = pytree.Node(1000, [l1, l2])
+ self.assertEqual(list(n1.pre_order()), [n1, l1, l2])
+
def testChangedLeaf(self):
l1 = pytree.Leaf(100, "f")
self.failIf(l1.was_changed)
More information about the Python-checkins
mailing list