[Python-checkins] r53004 - sandbox/trunk/2to3/fix_has_key.py sandbox/trunk/2to3/fix_null.py sandbox/trunk/2to3/fix_print.py sandbox/trunk/2to3/pytree.py sandbox/trunk/2to3/refactor.py
guido.van.rossum
python-checkins at python.org
Tue Dec 12 02:17:07 CET 2006
Author: guido.van.rossum
Date: Tue Dec 12 02:17:06 2006
New Revision: 53004
Added:
sandbox/trunk/2to3/fix_null.py (contents, props changed)
sandbox/trunk/2to3/refactor.py (contents, props changed)
Modified:
sandbox/trunk/2to3/fix_has_key.py
sandbox/trunk/2to3/fix_print.py
sandbox/trunk/2to3/pytree.py
Log:
Add a generic refactoring tool.
As a safety measure, this doesn't modify files (yet), but prints
unified diffs instead.
Modified: sandbox/trunk/2to3/fix_has_key.py
==============================================================================
--- sandbox/trunk/2to3/fix_has_key.py (original)
+++ sandbox/trunk/2to3/fix_has_key.py Tue Dec 12 02:17:06 2006
@@ -55,7 +55,7 @@
# Tree matching pattern
pat_compile = patcomp.PatternCompiler().compile_pattern
-p_has_key_top = pat_compile("""
+p_has_key = pat_compile("""
power<
before=any+
trailer< '.' 'has_key' >
@@ -72,7 +72,7 @@
def fix_has_key(node):
results = {}
- if not p_has_key_top.match(node, results):
+ if not p_has_key.match(node, results):
return
prefix = node.get_prefix()
before = results["before"]
Added: sandbox/trunk/2to3/fix_null.py
==============================================================================
--- (empty file)
+++ sandbox/trunk/2to3/fix_null.py Tue Dec 12 02:17:06 2006
@@ -0,0 +1,13 @@
+#!/usr/bin/env python2.5
+# Copyright 2006 Google Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Null refactoring."""
+
+import patcomp
+
+pat_compile = patcomp.PatternCompiler().compile_pattern
+p_null = pat_compile("'?'")
+
+def fix_null(node):
+ pass
Modified: sandbox/trunk/2to3/fix_print.py
==============================================================================
--- sandbox/trunk/2to3/fix_print.py (original)
+++ sandbox/trunk/2to3/fix_print.py Tue Dec 12 02:17:06 2006
@@ -21,6 +21,7 @@
from pgen2 import driver
import pytree
+import patcomp
logging.basicConfig(level=logging.DEBUG)
@@ -59,8 +60,18 @@
visit(child, func)
+pat_compile = patcomp.PatternCompiler().compile_pattern
+p_print = pat_compile("""
+'print' | print_stmt
+""")
+
+
def fix_print(node):
+ if not p_print.match(node):
+ return
if node == pytree.Leaf(token.NAME, "print"):
+ if node.parent.type == syms.print_stmt:
+ return # Will be covered when parent is handled
# Special-case print all by itself
new = pytree.Node(syms.power,
(pytree.Leaf(token.NAME, "Print"),
Modified: sandbox/trunk/2to3/pytree.py
==============================================================================
--- sandbox/trunk/2to3/pytree.py (original)
+++ sandbox/trunk/2to3/pytree.py Tue Dec 12 02:17:06 2006
@@ -64,6 +64,13 @@
"""
raise NotImplementedError
+ def post_order(self):
+ """Returns a post-order iterator for the tree.
+
+ This must be implemented by the concrete subclass.
+ """
+ raise NotImplementedError
+
def set_prefix(self, prefix):
"""Sets the prefix for the node (see Leaf class).
@@ -139,6 +146,13 @@
"""Compares two nodes for equality."""
return (self.type, self.children) == (other.type, other.children)
+ def post_order(self):
+ """Returns a post-order iterator for the tree."""
+ for child in self.children:
+ for node in child.post_order():
+ yield node
+ yield self
+
def set_prefix(self, prefix):
"""Sets the prefix for the node.
@@ -195,6 +209,10 @@
"""Compares two nodes for equality."""
return (self.type, self.value) == (other.type, other.value)
+ def post_order(self):
+ """Returns a post-order iterator for the tree."""
+ yield self
+
def set_prefix(self, prefix):
"""Sets the prefix for the node."""
self.prefix = prefix
Added: sandbox/trunk/2to3/refactor.py
==============================================================================
--- (empty file)
+++ sandbox/trunk/2to3/refactor.py Tue Dec 12 02:17:06 2006
@@ -0,0 +1,212 @@
+#!/usr/bin/env python2.5
+# Copyright 2006 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Refactoring framework.
+
+Used as a main program, this can refactory any number of files and/or
+recursively descend down directories. Imported as a module, this
+provides infrastructure to write your own refactoring tool.
+"""
+
+__author__ = "Guido van Rossum <guido at python.org>"
+
+
+# Python imports
+import os
+import sys
+import optparse
+import tempfile
+
+# Local imports
+import pytree
+import patcomp
+from pgen2 import driver
+
+
+def main(args=None):
+ """Main program.
+
+ Call without arguments to use sys.argv[1:] as the arguments; or
+ call with a list of arguments (excluding sys.argv[0]).
+
+ Returns a suggested exit status (0, 1, 2).
+ """
+ # Set up option parser
+ parser = optparse.OptionParser(usage="refactor.py [options] file|dir ...")
+ parser.add_option("-f", "--fix", action="append", default=[],
+ help="Each FIX specifies a transformation; default all")
+ parser.add_option("-l", "--list-fixes", action="store_true",
+ help="List available transformations")
+ parser.add_option("-v", "--verbose", action="store_true",
+ help="More verbose logging")
+
+ # Parse command line arguments
+ options, args = parser.parse_args(args)
+ if options.list_fixes:
+ print "Available transformations for the -f/--fix option:"
+ for fixname in get_all_fixes():
+ print fixname
+ if not args:
+ return 0
+ if not args:
+ print >>sys.stderr, "At least one file or directory argument required."
+ print >>sys.stderr, "Use --help to show usage."
+ return 2
+
+ # Initialize the refactoring tool
+ rt = RefactoringTool(options)
+
+ # Refactor all files and directories passed as arguments
+ if not rt.errors:
+ rt.refactor_args(args)
+
+ # Return error status (0 if rt.errors is zero)
+ return int(bool(rt.errors))
+
+
+def get_all_fixes():
+ """Return a sorted list of all available fixes."""
+ fixes = []
+ for name in os.listdir(os.path.dirname(__file__)):
+ if name.startswith("fix_") and name.endswith(".py"):
+ fixes.append(name[4:-3])
+ fixes.sort()
+ return fixes
+
+
+class RefactoringTool(object):
+
+ def __init__(self, options):
+ """Initializer.
+
+ The argument is an optparse.Values instance.
+ """
+ self.options = options
+ self.errors = 0
+ self.gr = driver.load_grammar("Grammar.txt")
+ self.dr = dr = driver.Driver(self.gr, convert=pytree.convert)
+ self.pairs = self.get_refactoring_pairs()
+
+ def get_refactoring_pairs(self):
+ """Inspects the options to load the requested patterns and handlers."""
+ pairs = []
+ fixes = self.options.fix
+ if not fixes or "all" in fixes:
+ fixes = get_all_fixes()
+ for fixname in fixes:
+ try:
+ mod = __import__("fix_" + fixname)
+ except (ImportError, AttributeError):
+ self.log_error("Can't find transformation %s", fixname)
+ else:
+ name = "?"
+ try:
+ name = "p_" + fixname
+ pattern = getattr(mod, name)
+ name = "fix_" + fixname
+ handler = getattr(mod, name)
+ except AttributeError:
+ self.log_error("Can't find fix_%s.%s", fixname, name)
+ else:
+ if self.options.verbose:
+ self.log_message("adding transformation: %s", fixname)
+ pairs.append((pattern, handler))
+ return pairs
+
+ def log_error(self, msg, *args):
+ """Increment error count and log a message."""
+ self.errors += 1
+ self.log_message(msg, *args)
+
+ def log_message(self, msg, *args):
+ """Hook to log a message."""
+ if args:
+ msg = msg % args
+ print >>sys.stderr, msg
+
+ def refactor_args(self, args):
+ """Refactor files and directories from an argument list."""
+ for arg in args:
+ if os.path.isdir(arg):
+ self.refactor_dir(arg)
+ else:
+ self.refactor_file(arg)
+
+ def refactor_dir(self, arg):
+ """Descend down a directory and refactor every Python file found.
+
+ Python files are assumed to have a .py extension.
+
+ Files and subdirectories starting with '.' are skipped.
+ """
+ for dirpath, dirnames, filenames in os.walk(arg):
+ if self.options.verbose:
+ self.log_message("Descending into %s", dirpath)
+ for name in filenames:
+ if not name.startswith(".") and name.endswith("py"):
+ fullname = os.path.join(dirpath, name)
+ self.refactor_file(fullname)
+ # Modify dirnames in-place to remove subdirs with leading dots
+ dirnames[:] = [dn for dn in dirnames if not dn.startswith(".")]
+
+ def refactor_file(self, filename):
+ """Refactor a file."""
+ try:
+ f = open(filename)
+ except IOError, err:
+ self.log_error("Can't open %s: %s", filename, err)
+ return
+ try:
+ try:
+ tree = self.dr.parse_file(filename)
+ except Exception, err:
+ self.log_error("Can't parse %s: %s: %s",
+ filename, err.__class__.__name__, err)
+ return
+ if self.options.verbose:
+ self.log_message("Refactoring %s", filename)
+ if self.refactor_tree(tree):
+ self.save_tree(tree, filename)
+ finally:
+ f.close()
+
+ def refactor_tree(self, tree):
+ changes = 0
+ for node in tree.post_order():
+ for pattern, handler in self.pairs:
+ if pattern.match(node):
+ # XXX Change handler API to return a replacement node
+ handler(node)
+ changes += 1
+ return changes
+
+ def save_tree(self, tree, filename):
+ """Save a (presumably modified) tree to a file.
+
+ Skip the saving if no changes were made.
+
+ XXX For now, don't save, just print a unified diff.
+ """
+ tfn = tempfile.mktemp()
+ f = open(tfn, "w")
+ try:
+ try:
+ f.write(str(tree))
+ finally:
+ f.close()
+ sts = os.system("diff -u %s %s" % (filename, tfn))
+ if sts == 0:
+ pass # No changes
+ elif sts == 1<<8:
+ # XXX Actually save it
+ pass
+ else:
+ self.log_error("diff %s returned exit (%s,%s)",
+ filename, sts>>8, sts&0xFF)
+ finally:
+ os.remove(tfn)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
More information about the Python-checkins
mailing list