[Python-checkins] cpython (merge 3.2 -> default): Issue #13930: Adds ability for 2to3 to write its output to a different
gregory.p.smith
python-checkins at python.org
Mon Feb 13 01:00:29 CET 2012
http://hg.python.org/cpython/rev/4b791e513c2c
changeset: 74897:4b791e513c2c
parent: 74893:5d0f7b275fe9
parent: 74895:ceea9ebfe003
user: Gregory P. Smith <greg at krypto.org>
date: Sun Feb 12 15:56:49 2012 -0800
summary:
Issue #13930: Adds ability for 2to3 to write its output to a different
directory tree instead of overwriting the input files. Adds three command
line options: -o/--output-dir, -W/--write-unchanged-files and --add-suffix.
files:
Doc/library/2to3.rst | 32 +++++
Lib/lib2to3/main.py | 93 +++++++++++++++-
Lib/lib2to3/refactor.py | 20 ++-
Lib/lib2to3/tests/test_main.py | 100 ++++++++++++++++-
Lib/lib2to3/tests/test_refactor.py | 36 +++++-
Misc/NEWS | 9 +
6 files changed, 275 insertions(+), 15 deletions(-)
diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst
--- a/Doc/library/2to3.rst
+++ b/Doc/library/2to3.rst
@@ -94,6 +94,38 @@
:option:`-p` to run fixers on code that already has had its print statements
converted.
+The :option:`-o` or :option:`--output-dir` option allows specification of an
+alternate directory for processed output files to be written to. The
+:option:`-n` flag is required when using this as backup files do not make sense
+when not overwriting the input files.
+
+.. versionadded:: 3.2.3
+ The :option:`-o` option was added.
+
+The :option:`-W` or :option:`--write-unchanged-files` flag tells 2to3 to always
+write output files even if no changes were required to the file. This is most
+useful with :option:`-o` so that an entire Python source tree is copied with
+translation from one directory to another.
+This option implies the :option:`-w` flag as it would not make sense otherwise.
+
+.. versionadded:: 3.2.3
+ The :option:`-W` flag was added.
+
+The :option:`--add-suffix` option specifies a string to append to all output
+filenames. The :option:`-n` flag is required when specifying this as backups
+are not necessary when writing to different filenames. Example::
+
+ $ 2to3 -n -W --add-suffix=3 example.py
+
+Will cause a converted file named ``example.py3`` to be written.
+
+.. versionadded:: 3.2.3
+ The :option:`--add-suffix` option was added.
+
+To translate an entire project from one directory tree to another use::
+
+ $ 2to3 --output-dir=python3-version/mycode -W -n python2-version/mycode
+
.. _2to3-fixers:
diff --git a/Lib/lib2to3/main.py b/Lib/lib2to3/main.py
--- a/Lib/lib2to3/main.py
+++ b/Lib/lib2to3/main.py
@@ -25,12 +25,41 @@
class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
"""
+ A refactoring tool that can avoid overwriting its input files.
Prints output to stdout.
+
+ Output files can optionally be written to a different directory and or
+ have an extra file suffix appended to their name for use in situations
+ where you do not want to replace the input files.
"""
- def __init__(self, fixers, options, explicit, nobackups, show_diffs):
+ def __init__(self, fixers, options, explicit, nobackups, show_diffs,
+ input_base_dir='', output_dir='', append_suffix=''):
+ """
+ Args:
+ fixers: A list of fixers to import.
+ options: A dict with RefactoringTool configuration.
+ explicit: A list of fixers to run even if they are explicit.
+ nobackups: If true no backup '.bak' files will be created for those
+ files that are being refactored.
+ show_diffs: Should diffs of the refactoring be printed to stdout?
+ input_base_dir: The base directory for all input files. This class
+ will strip this path prefix off of filenames before substituting
+ it with output_dir. Only meaningful if output_dir is supplied.
+ All files processed by refactor() must start with this path.
+ output_dir: If supplied, all converted files will be written into
+ this directory tree instead of input_base_dir.
+ append_suffix: If supplied, all files output by this tool will have
+ this appended to their filename. Useful for changing .py to
+ .py3 for example by passing append_suffix='3'.
+ """
self.nobackups = nobackups
self.show_diffs = show_diffs
+ if input_base_dir and not input_base_dir.endswith(os.sep):
+ input_base_dir += os.sep
+ self._input_base_dir = input_base_dir
+ self._output_dir = output_dir
+ self._append_suffix = append_suffix
super(StdoutRefactoringTool, self).__init__(fixers, options, explicit)
def log_error(self, msg, *args, **kwargs):
@@ -38,6 +67,23 @@
self.logger.error(msg, *args, **kwargs)
def write_file(self, new_text, filename, old_text, encoding):
+ orig_filename = filename
+ if self._output_dir:
+ if filename.startswith(self._input_base_dir):
+ filename = os.path.join(self._output_dir,
+ filename[len(self._input_base_dir):])
+ else:
+ raise ValueError('filename %s does not start with the '
+ 'input_base_dir %s' % (
+ filename, self._input_base_dir))
+ if self._append_suffix:
+ filename += self._append_suffix
+ if orig_filename != filename:
+ output_dir = os.path.dirname(filename)
+ if not os.path.isdir(output_dir):
+ os.makedirs(output_dir)
+ self.log_message('Writing converted %s to %s.', orig_filename,
+ filename)
if not self.nobackups:
# Make backup
backup = filename + ".bak"
@@ -55,6 +101,9 @@
write(new_text, filename, old_text, encoding)
if not self.nobackups:
shutil.copymode(backup, filename)
+ if orig_filename != filename:
+ # Preserve the file mode in the new output directory.
+ shutil.copymode(orig_filename, filename)
def print_output(self, old, new, filename, equal):
if equal:
@@ -113,11 +162,33 @@
help="Write back modified files")
parser.add_option("-n", "--nobackups", action="store_true", default=False,
help="Don't write backups for modified files")
+ parser.add_option("-o", "--output-dir", action="store", type="str",
+ default="", help="Put output files in this directory "
+ "instead of overwriting the input files. Requires -n.")
+ parser.add_option("-W", "--write-unchanged-files", action="store_true",
+ help="Also write files even if no changes were required"
+ " (useful with --output-dir); implies -w.")
+ parser.add_option("--add-suffix", action="store", type="str", default="",
+ help="Append this string to all output filenames."
+ " Requires -n if non-empty. "
+ "ex: --add-suffix='3' will generate .py3 files.")
# Parse command line arguments
refactor_stdin = False
flags = {}
options, args = parser.parse_args(args)
+ if options.write_unchanged_files:
+ flags["write_unchanged_files"] = True
+ if not options.write:
+ warn("--write-unchanged-files/-W implies -w.")
+ options.write = True
+ # If we allowed these, the original files would be renamed to backup names
+ # but not replaced.
+ if options.output_dir and not options.nobackups:
+ parser.error("Can't use --output-dir/-o without -n.")
+ if options.add_suffix and not options.nobackups:
+ parser.error("Can't use --add-suffix without -n.")
+
if not options.write and options.no_diffs:
warn("not writing files and not printing diffs; that's not very useful")
if not options.write and options.nobackups:
@@ -143,6 +214,7 @@
# Set up logging handler
level = logging.DEBUG if options.verbose else logging.INFO
logging.basicConfig(format='%(name)s: %(message)s', level=level)
+ logger = logging.getLogger('lib2to3.main')
# Initialize the refactoring tool
avail_fixes = set(refactor.get_fixers_from_package(fixer_pkg))
@@ -159,8 +231,23 @@
else:
requested = avail_fixes.union(explicit)
fixer_names = requested.difference(unwanted_fixes)
- rt = StdoutRefactoringTool(sorted(fixer_names), flags, sorted(explicit),
- options.nobackups, not options.no_diffs)
+ input_base_dir = os.path.commonprefix(args)
+ if (input_base_dir and not input_base_dir.endswith(os.sep)
+ and not os.path.isdir(input_base_dir)):
+ # One or more similar names were passed, their directory is the base.
+ # os.path.commonprefix() is ignorant of path elements, this corrects
+ # for that weird API.
+ input_base_dir = os.path.dirname(input_base_dir)
+ if options.output_dir:
+ input_base_dir = input_base_dir.rstrip(os.sep)
+ logger.info('Output in %r will mirror the input directory %r layout.',
+ options.output_dir, input_base_dir)
+ rt = StdoutRefactoringTool(
+ sorted(fixer_names), flags, sorted(explicit),
+ options.nobackups, not options.no_diffs,
+ input_base_dir=input_base_dir,
+ output_dir=options.output_dir,
+ append_suffix=options.add_suffix)
# Refactor all files and directories passed as arguments
if not rt.errors:
diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py
--- a/Lib/lib2to3/refactor.py
+++ b/Lib/lib2to3/refactor.py
@@ -173,7 +173,8 @@
class RefactoringTool(object):
- _default_options = {"print_function" : False}
+ _default_options = {"print_function" : False,
+ "write_unchanged_files" : False}
CLASS_PREFIX = "Fix" # The prefix for fixer classes
FILE_PREFIX = "fix_" # The prefix for modules with a fixer within
@@ -195,6 +196,10 @@
self.grammar = pygram.python_grammar_no_print_statement
else:
self.grammar = pygram.python_grammar
+ # When this is True, the refactor*() methods will call write_file() for
+ # files processed even if they were not changed during refactoring. If
+ # and only if the refactor method's write parameter was True.
+ self.write_unchanged_files = self.options.get("write_unchanged_files")
self.errors = []
self.logger = logging.getLogger("RefactoringTool")
self.fixer_log = []
@@ -341,13 +346,13 @@
if doctests_only:
self.log_debug("Refactoring doctests in %s", filename)
output = self.refactor_docstring(input, filename)
- if output != input:
+ if self.write_unchanged_files or output != input:
self.processed_file(output, filename, input, write, encoding)
else:
self.log_debug("No doctest changes in %s", filename)
else:
tree = self.refactor_string(input, filename)
- if tree and tree.was_changed:
+ if self.write_unchanged_files or (tree and tree.was_changed):
# The [:-1] is to take off the \n we added earlier
self.processed_file(str(tree)[:-1], filename,
write=write, encoding=encoding)
@@ -386,13 +391,13 @@
if doctests_only:
self.log_debug("Refactoring doctests in stdin")
output = self.refactor_docstring(input, "<stdin>")
- if output != input:
+ if self.write_unchanged_files or output != input:
self.processed_file(output, "<stdin>", input)
else:
self.log_debug("No doctest changes in stdin")
else:
tree = self.refactor_string(input, "<stdin>")
- if tree and tree.was_changed:
+ if self.write_unchanged_files or (tree and tree.was_changed):
self.processed_file(str(tree), "<stdin>", input)
else:
self.log_debug("No changes in stdin")
@@ -502,7 +507,7 @@
def processed_file(self, new_text, filename, old_text=None, write=False,
encoding=None):
"""
- Called when a file has been refactored, and there are changes.
+ Called when a file has been refactored and there may be changes.
"""
self.files.append(filename)
if old_text is None:
@@ -513,7 +518,8 @@
self.print_output(old_text, new_text, filename, equal)
if equal:
self.log_debug("No changes to %s", filename)
- return
+ if not self.write_unchanged_files:
+ return
if write:
self.write_file(new_text, filename, old_text, encoding)
else:
diff --git a/Lib/lib2to3/tests/test_main.py b/Lib/lib2to3/tests/test_main.py
--- a/Lib/lib2to3/tests/test_main.py
+++ b/Lib/lib2to3/tests/test_main.py
@@ -1,18 +1,30 @@
# -*- coding: utf-8 -*-
+import codecs
+import io
+import logging
+import os
+import shutil
import sys
-import codecs
-import logging
-import io
+import tempfile
import unittest
from lib2to3 import main
+TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
+PY2_TEST_MODULE = os.path.join(TEST_DATA_DIR, "py2_test_grammar.py")
+
+
class TestMain(unittest.TestCase):
+ def setUp(self):
+ self.temp_dir = None # tearDown() will rmtree this directory if set.
+
def tearDown(self):
# Clean up logging configuration down by main.
del logging.root.handlers[:]
+ if self.temp_dir:
+ shutil.rmtree(self.temp_dir)
def run_2to3_capture(self, args, in_capture, out_capture, err_capture):
save_stdin = sys.stdin
@@ -39,3 +51,85 @@
self.assertTrue("-print 'nothing'" in output)
self.assertTrue("WARNING: couldn't encode <stdin>'s diff for "
"your terminal" in err.getvalue())
+
+ def setup_test_source_trees(self):
+ """Setup a test source tree and output destination tree."""
+ self.temp_dir = tempfile.mkdtemp() # tearDown() cleans this up.
+ self.py2_src_dir = os.path.join(self.temp_dir, "python2_project")
+ self.py3_dest_dir = os.path.join(self.temp_dir, "python3_project")
+ os.mkdir(self.py2_src_dir)
+ os.mkdir(self.py3_dest_dir)
+ # Turn it into a package with a few files.
+ self.setup_files = []
+ open(os.path.join(self.py2_src_dir, "__init__.py"), "w").close()
+ self.setup_files.append("__init__.py")
+ shutil.copy(PY2_TEST_MODULE, self.py2_src_dir)
+ self.setup_files.append(os.path.basename(PY2_TEST_MODULE))
+ self.trivial_py2_file = os.path.join(self.py2_src_dir, "trivial.py")
+ self.init_py2_file = os.path.join(self.py2_src_dir, "__init__.py")
+ with open(self.trivial_py2_file, "w") as trivial:
+ trivial.write("print 'I need a simple conversion.'")
+ self.setup_files.append("trivial.py")
+
+ def test_filename_changing_on_output_single_dir(self):
+ """2to3 a single directory with a new output dir and suffix."""
+ self.setup_test_source_trees()
+ out = io.StringIO()
+ err = io.StringIO()
+ suffix = "TEST"
+ ret = self.run_2to3_capture(
+ ["-n", "--add-suffix", suffix, "--write-unchanged-files",
+ "--no-diffs", "--output-dir",
+ self.py3_dest_dir, self.py2_src_dir],
+ io.StringIO(""), out, err)
+ self.assertEqual(ret, 0)
+ stderr = err.getvalue()
+ self.assertIn(" implies -w.", stderr)
+ self.assertIn(
+ "Output in %r will mirror the input directory %r layout" % (
+ self.py3_dest_dir, self.py2_src_dir), stderr)
+ self.assertEqual(set(name+suffix for name in self.setup_files),
+ set(os.listdir(self.py3_dest_dir)))
+ for name in self.setup_files:
+ self.assertIn("Writing converted %s to %s" % (
+ os.path.join(self.py2_src_dir, name),
+ os.path.join(self.py3_dest_dir, name+suffix)), stderr)
+ self.assertRegexpMatches(stderr, r"No changes to .*/__init__\.py")
+ self.assertNotRegex(stderr, r"No changes to .*/trivial\.py")
+
+ def test_filename_changing_on_output_two_files(self):
+ """2to3 two files in one directory with a new output dir."""
+ self.setup_test_source_trees()
+ err = io.StringIO()
+ py2_files = [self.trivial_py2_file, self.init_py2_file]
+ expected_files = set(os.path.basename(name) for name in py2_files)
+ ret = self.run_2to3_capture(
+ ["-n", "-w", "--write-unchanged-files",
+ "--no-diffs", "--output-dir", self.py3_dest_dir] + py2_files,
+ io.StringIO(""), io.StringIO(), err)
+ self.assertEqual(ret, 0)
+ stderr = err.getvalue()
+ self.assertIn(
+ "Output in %r will mirror the input directory %r layout" % (
+ self.py3_dest_dir, self.py2_src_dir), stderr)
+ self.assertEqual(expected_files, set(os.listdir(self.py3_dest_dir)))
+
+ def test_filename_changing_on_output_single_file(self):
+ """2to3 a single file with a new output dir."""
+ self.setup_test_source_trees()
+ err = io.StringIO()
+ ret = self.run_2to3_capture(
+ ["-n", "-w", "--no-diffs", "--output-dir", self.py3_dest_dir,
+ self.trivial_py2_file],
+ io.StringIO(""), io.StringIO(), err)
+ self.assertEqual(ret, 0)
+ stderr = err.getvalue()
+ self.assertIn(
+ "Output in %r will mirror the input directory %r layout" % (
+ self.py3_dest_dir, self.py2_src_dir), stderr)
+ self.assertEqual(set([os.path.basename(self.trivial_py2_file)]),
+ set(os.listdir(self.py3_dest_dir)))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/lib2to3/tests/test_refactor.py b/Lib/lib2to3/tests/test_refactor.py
--- a/Lib/lib2to3/tests/test_refactor.py
+++ b/Lib/lib2to3/tests/test_refactor.py
@@ -53,6 +53,12 @@
self.assertTrue(rt.driver.grammar is
pygram.python_grammar_no_print_statement)
+ def test_write_unchanged_files_option(self):
+ rt = self.rt()
+ self.assertFalse(rt.write_unchanged_files)
+ rt = self.rt({"write_unchanged_files" : True})
+ self.assertTrue(rt.write_unchanged_files)
+
def test_fixer_loading_helpers(self):
contents = ["explicit", "first", "last", "parrot", "preorder"]
non_prefixed = refactor.get_all_fix_names("myfixes")
@@ -176,7 +182,9 @@
"<stdin>", False]
self.assertEqual(results, expected)
- def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS):
+ def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS,
+ options=None, mock_log_debug=None,
+ actually_write=True):
tmpdir = tempfile.mkdtemp(prefix="2to3-test_refactor")
self.addCleanup(shutil.rmtree, tmpdir)
# make a copy of the tested file that we can write to
@@ -189,11 +197,15 @@
return fp.read()
old_contents = read_file()
- rt = self.rt(fixers=fixers)
+ rt = self.rt(fixers=fixers, options=options)
+ if mock_log_debug:
+ rt.log_debug = mock_log_debug
rt.refactor_file(test_file)
self.assertEqual(old_contents, read_file())
+ if not actually_write:
+ return
rt.refactor_file(test_file, True)
new_contents = read_file()
self.assertNotEqual(old_contents, new_contents)
@@ -203,6 +215,26 @@
test_file = os.path.join(FIXER_DIR, "parrot_example.py")
self.check_file_refactoring(test_file, _DEFAULT_FIXERS)
+ def test_refactor_file_write_unchanged_file(self):
+ test_file = os.path.join(FIXER_DIR, "parrot_example.py")
+ debug_messages = []
+ def recording_log_debug(msg, *args):
+ debug_messages.append(msg % args)
+ self.check_file_refactoring(test_file, fixers=(),
+ options={"write_unchanged_files": True},
+ mock_log_debug=recording_log_debug,
+ actually_write=False)
+ # Testing that it logged this message when write=False was passed is
+ # sufficient to see that it did not bail early after "No changes".
+ message_regex = r"Not writing changes to .*%s%s" % (
+ os.sep, os.path.basename(test_file))
+ for message in debug_messages:
+ if "Not writing changes" in message:
+ self.assertRegexpMatches(message, message_regex)
+ break
+ else:
+ self.fail("%r not matched in %r" % (message_regex, debug_messages))
+
def test_refactor_dir(self):
def check(structure, expected):
def mock_refactor_file(self, f, *args):
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -466,6 +466,10 @@
Library
-------
+- Issue #13930: lib2to3 now supports writing converted output files to another
+ directory tree as well as copying unchanged files and altering the file
+ suffix.
+
- Issue #9750: Fix sqlite3.Connection.iterdump on tables and fields
with a name that is a keyword or contains quotes. Patch by Marko
Kohtala.
@@ -1927,6 +1931,11 @@
Tools/Demos
-----------
+- Issue #13930: 2to3 is now able to write its converted output files to another
+ directory tree as well as copying unchanged files and altering the file
+ suffix. See its new -o, -W and --add-suffix options. This makes it more
+ useful in many automated code translation workflows.
+
- Issue #13628: python-gdb.py is now able to retrieve more frames in the Python
traceback if Python is optimized.
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list