Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Issue #13930: Adds ability for 2to3 to write its output to a different 

directory tree instead of overwriting the input files.  Adds three command
line options: -o/--output-dir, -W/--write-unchanged-files and --add-suffix.
This commit is contained in:
Gregory P. Smith 2012-02-12 15:56:49 -08:00
parent c7551a16c7 58f23ffb21
commit db66310e8f
6 changed files with 274 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
Doc/library/2to3.rst
View File

@ -94,6 +94,38 @@ change can also be enabled manually with the :option:`-p` flag. Use
:option:`-p` to run fixers on code that already has had its print statements
converted.
The :option:`-o` or :option:`--output-dir` option allows specification of an
alternate directory for processed output files to be written to. The
:option:`-n` flag is required when using this as backup files do not make sense
when not overwriting the input files.
.. versionadded:: 3.2.3
The :option:`-o` option was added.
The :option:`-W` or :option:`--write-unchanged-files` flag tells 2to3 to always
write output files even if no changes were required to the file. This is most
useful with :option:`-o` so that an entire Python source tree is copied with
translation from one directory to another.
This option implies the :option:`-w` flag as it would not make sense otherwise.
.. versionadded:: 3.2.3
The :option:`-W` flag was added.
The :option:`--add-suffix` option specifies a string to append to all output
filenames. The :option:`-n` flag is required when specifying this as backups
are not necessary when writing to different filenames. Example::
$ 2to3 -n -W --add-suffix=3 example.py
Will cause a converted file named ``example.py3`` to be written.
.. versionadded:: 3.2.3
The :option:`--add-suffix` option was added.
To translate an entire project from one directory tree to another use::
$ 2to3 --output-dir=python3-version/mycode -W -n python2-version/mycode
.. _2to3-fixers:

93
Lib/lib2to3/main.py
View File

@ -25,12 +25,41 @@ def diff_texts(a, b, filename):
class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
"""
A refactoring tool that can avoid overwriting its input files.
Prints output to stdout.
Output files can optionally be written to a different directory and or
have an extra file suffix appended to their name for use in situations
where you do not want to replace the input files.
"""
def __init__(self, fixers, options, explicit, nobackups, show_diffs):
def __init__(self, fixers, options, explicit, nobackups, show_diffs,
input_base_dir='', output_dir='', append_suffix=''):
"""
Args:
fixers: A list of fixers to import.
options: A dict with RefactoringTool configuration.
explicit: A list of fixers to run even if they are explicit.
nobackups: If true no backup '.bak' files will be created for those
files that are being refactored.
show_diffs: Should diffs of the refactoring be printed to stdout?
input_base_dir: The base directory for all input files. This class
will strip this path prefix off of filenames before substituting
it with output_dir. Only meaningful if output_dir is supplied.
All files processed by refactor() must start with this path.
output_dir: If supplied, all converted files will be written into
this directory tree instead of input_base_dir.
append_suffix: If supplied, all files output by this tool will have
this appended to their filename. Useful for changing .py to
.py3 for example by passing append_suffix='3'.
"""
self.nobackups = nobackups
self.show_diffs = show_diffs
if input_base_dir and not input_base_dir.endswith(os.sep):
input_base_dir += os.sep
self._input_base_dir = input_base_dir
self._output_dir = output_dir
self._append_suffix = append_suffix
super(StdoutRefactoringTool, self).__init__(fixers, options, explicit)
def log_error(self, msg, *args, **kwargs):
@ -38,6 +67,23 @@ class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
self.logger.error(msg, *args, **kwargs)
def write_file(self, new_text, filename, old_text, encoding):
orig_filename = filename
if self._output_dir:
if filename.startswith(self._input_base_dir):
filename = os.path.join(self._output_dir,
filename[len(self._input_base_dir):])
else:
raise ValueError('filename %s does not start with the '
'input_base_dir %s' % (
filename, self._input_base_dir))
if self._append_suffix:
filename += self._append_suffix
if orig_filename != filename:
output_dir = os.path.dirname(filename)
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
self.log_message('Writing converted %s to %s.', orig_filename,
filename)
if not self.nobackups:
# Make backup
backup = filename + ".bak"
@ -55,6 +101,9 @@ class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
write(new_text, filename, old_text, encoding)
if not self.nobackups:
shutil.copymode(backup, filename)
if orig_filename != filename:
# Preserve the file mode in the new output directory.
shutil.copymode(orig_filename, filename)
def print_output(self, old, new, filename, equal):
if equal:
@ -113,11 +162,33 @@ def main(fixer_pkg, args=None):
help="Write back modified files")
parser.add_option("-n", "--nobackups", action="store_true", default=False,
help="Don't write backups for modified files")
parser.add_option("-o", "--output-dir", action="store", type="str",
default="", help="Put output files in this directory "
"instead of overwriting the input files. Requires -n.")
parser.add_option("-W", "--write-unchanged-files", action="store_true",
help="Also write files even if no changes were required"
" (useful with --output-dir); implies -w.")
parser.add_option("--add-suffix", action="store", type="str", default="",
help="Append this string to all output filenames."
" Requires -n if non-empty. "
"ex: --add-suffix='3' will generate .py3 files.")
# Parse command line arguments
refactor_stdin = False
flags = {}
options, args = parser.parse_args(args)
if options.write_unchanged_files:
flags["write_unchanged_files"] = True
if not options.write:
warn("--write-unchanged-files/-W implies -w.")
options.write = True
# If we allowed these, the original files would be renamed to backup names
# but not replaced.
if options.output_dir and not options.nobackups:
parser.error("Can't use --output-dir/-o without -n.")
if options.add_suffix and not options.nobackups:
parser.error("Can't use --add-suffix without -n.")
if not options.write and options.no_diffs:
warn("not writing files and not printing diffs; that's not very useful")
if not options.write and options.nobackups:
@ -143,6 +214,7 @@ def main(fixer_pkg, args=None):
# Set up logging handler
level = logging.DEBUG if options.verbose else logging.INFO
logging.basicConfig(format='%(name)s: %(message)s', level=level)
logger = logging.getLogger('lib2to3.main')
# Initialize the refactoring tool
avail_fixes = set(refactor.get_fixers_from_package(fixer_pkg))
@ -159,8 +231,23 @@ def main(fixer_pkg, args=None):
else:
requested = avail_fixes.union(explicit)
fixer_names = requested.difference(unwanted_fixes)
rt = StdoutRefactoringTool(sorted(fixer_names), flags, sorted(explicit),
options.nobackups, not options.no_diffs)
input_base_dir = os.path.commonprefix(args)
if (input_base_dir and not input_base_dir.endswith(os.sep)
and not os.path.isdir(input_base_dir)):
# One or more similar names were passed, their directory is the base.
# os.path.commonprefix() is ignorant of path elements, this corrects
# for that weird API.
input_base_dir = os.path.dirname(input_base_dir)
if options.output_dir:
input_base_dir = input_base_dir.rstrip(os.sep)
logger.info('Output in %r will mirror the input directory %r layout.',
options.output_dir, input_base_dir)
rt = StdoutRefactoringTool(
sorted(fixer_names), flags, sorted(explicit),
options.nobackups, not options.no_diffs,
input_base_dir=input_base_dir,
output_dir=options.output_dir,
append_suffix=options.add_suffix)
# Refactor all files and directories passed as arguments
if not rt.errors:

20
Lib/lib2to3/refactor.py
View File

@ -173,7 +173,8 @@ class FixerError(Exception):
class RefactoringTool(object):
_default_options = {"print_function" : False}
_default_options = {"print_function" : False,
"write_unchanged_files" : False}
CLASS_PREFIX = "Fix" # The prefix for fixer classes
FILE_PREFIX = "fix_" # The prefix for modules with a fixer within
@ -195,6 +196,10 @@ class RefactoringTool(object):
self.grammar = pygram.python_grammar_no_print_statement
else:
self.grammar = pygram.python_grammar
# When this is True, the refactor*() methods will call write_file() for
# files processed even if they were not changed during refactoring. If
# and only if the refactor method's write parameter was True.
self.write_unchanged_files = self.options.get("write_unchanged_files")
self.errors = []
self.logger = logging.getLogger("RefactoringTool")
self.fixer_log = []
@ -341,13 +346,13 @@ class RefactoringTool(object):
if doctests_only:
self.log_debug("Refactoring doctests in %s", filename)
output = self.refactor_docstring(input, filename)
if output != input:
if self.write_unchanged_files or output != input:
self.processed_file(output, filename, input, write, encoding)
else:
self.log_debug("No doctest changes in %s", filename)
else:
tree = self.refactor_string(input, filename)
if tree and tree.was_changed:
if self.write_unchanged_files or (tree and tree.was_changed):
# The [:-1] is to take off the \n we added earlier
self.processed_file(str(tree)[:-1], filename,
write=write, encoding=encoding)
@ -386,13 +391,13 @@ class RefactoringTool(object):
if doctests_only:
self.log_debug("Refactoring doctests in stdin")
output = self.refactor_docstring(input, "<stdin>")
if output != input:
if self.write_unchanged_files or output != input:
self.processed_file(output, "<stdin>", input)
else:
self.log_debug("No doctest changes in stdin")
else:
tree = self.refactor_string(input, "<stdin>")
if tree and tree.was_changed:
if self.write_unchanged_files or (tree and tree.was_changed):
self.processed_file(str(tree), "<stdin>", input)
else:
self.log_debug("No changes in stdin")
@ -502,7 +507,7 @@ class RefactoringTool(object):
def processed_file(self, new_text, filename, old_text=None, write=False,
encoding=None):
"""
Called when a file has been refactored, and there are changes.
Called when a file has been refactored and there may be changes.
"""
self.files.append(filename)
if old_text is None:
@ -513,7 +518,8 @@ class RefactoringTool(object):
self.print_output(old_text, new_text, filename, equal)
if equal:
self.log_debug("No changes to %s", filename)
return
if not self.write_unchanged_files:
return
if write:
self.write_file(new_text, filename, old_text, encoding)
else:

98
Lib/lib2to3/tests/test_main.py
View File

@ -1,18 +1,30 @@
# -*- coding: utf-8 -*-
import sys
import codecs
import logging
import io
import logging
import os
import shutil
import sys
import tempfile
import unittest
from lib2to3 import main
TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
PY2_TEST_MODULE = os.path.join(TEST_DATA_DIR, "py2_test_grammar.py")
class TestMain(unittest.TestCase):
def setUp(self):
self.temp_dir = None # tearDown() will rmtree this directory if set.
def tearDown(self):
# Clean up logging configuration down by main.
del logging.root.handlers[:]
if self.temp_dir:
shutil.rmtree(self.temp_dir)
def run_2to3_capture(self, args, in_capture, out_capture, err_capture):
save_stdin = sys.stdin
@ -39,3 +51,85 @@ class TestMain(unittest.TestCase):
self.assertTrue("-print 'nothing'" in output)
self.assertTrue("WARNING: couldn't encode <stdin>'s diff for "
"your terminal" in err.getvalue())
def setup_test_source_trees(self):
"""Setup a test source tree and output destination tree."""
self.temp_dir = tempfile.mkdtemp() # tearDown() cleans this up.
self.py2_src_dir = os.path.join(self.temp_dir, "python2_project")
self.py3_dest_dir = os.path.join(self.temp_dir, "python3_project")
os.mkdir(self.py2_src_dir)
os.mkdir(self.py3_dest_dir)
# Turn it into a package with a few files.
self.setup_files = []
open(os.path.join(self.py2_src_dir, "__init__.py"), "w").close()
self.setup_files.append("__init__.py")
shutil.copy(PY2_TEST_MODULE, self.py2_src_dir)
self.setup_files.append(os.path.basename(PY2_TEST_MODULE))
self.trivial_py2_file = os.path.join(self.py2_src_dir, "trivial.py")
self.init_py2_file = os.path.join(self.py2_src_dir, "__init__.py")
with open(self.trivial_py2_file, "w") as trivial:
trivial.write("print 'I need a simple conversion.'")
self.setup_files.append("trivial.py")
def test_filename_changing_on_output_single_dir(self):
"""2to3 a single directory with a new output dir and suffix."""
self.setup_test_source_trees()
out = io.StringIO()
err = io.StringIO()
suffix = "TEST"
ret = self.run_2to3_capture(
["-n", "--add-suffix", suffix, "--write-unchanged-files",
"--no-diffs", "--output-dir",
self.py3_dest_dir, self.py2_src_dir],
io.StringIO(""), out, err)
self.assertEqual(ret, 0)
stderr = err.getvalue()
self.assertIn(" implies -w.", stderr)
self.assertIn(
"Output in %r will mirror the input directory %r layout" % (
self.py3_dest_dir, self.py2_src_dir), stderr)
self.assertEqual(set(name+suffix for name in self.setup_files),
set(os.listdir(self.py3_dest_dir)))
for name in self.setup_files:
self.assertIn("Writing converted %s to %s" % (
os.path.join(self.py2_src_dir, name),
os.path.join(self.py3_dest_dir, name+suffix)), stderr)
self.assertRegexpMatches(stderr, r"No changes to .*/__init__\.py")
self.assertNotRegex(stderr, r"No changes to .*/trivial\.py")
def test_filename_changing_on_output_two_files(self):
"""2to3 two files in one directory with a new output dir."""
self.setup_test_source_trees()
err = io.StringIO()
py2_files = [self.trivial_py2_file, self.init_py2_file]
expected_files = set(os.path.basename(name) for name in py2_files)
ret = self.run_2to3_capture(
["-n", "-w", "--write-unchanged-files",
"--no-diffs", "--output-dir", self.py3_dest_dir] + py2_files,
io.StringIO(""), io.StringIO(), err)
self.assertEqual(ret, 0)
stderr = err.getvalue()
self.assertIn(
"Output in %r will mirror the input directory %r layout" % (
self.py3_dest_dir, self.py2_src_dir), stderr)
self.assertEqual(expected_files, set(os.listdir(self.py3_dest_dir)))
def test_filename_changing_on_output_single_file(self):
"""2to3 a single file with a new output dir."""
self.setup_test_source_trees()
err = io.StringIO()
ret = self.run_2to3_capture(
["-n", "-w", "--no-diffs", "--output-dir", self.py3_dest_dir,
self.trivial_py2_file],
io.StringIO(""), io.StringIO(), err)
self.assertEqual(ret, 0)
stderr = err.getvalue()
self.assertIn(
"Output in %r will mirror the input directory %r layout" % (
self.py3_dest_dir, self.py2_src_dir), stderr)
self.assertEqual(set([os.path.basename(self.trivial_py2_file)]),
set(os.listdir(self.py3_dest_dir)))
if __name__ == '__main__':
unittest.main()

36
Lib/lib2to3/tests/test_refactor.py
View File

@ -53,6 +53,12 @@ class TestRefactoringTool(unittest.TestCase):
self.assertTrue(rt.driver.grammar is
pygram.python_grammar_no_print_statement)
def test_write_unchanged_files_option(self):
rt = self.rt()
self.assertFalse(rt.write_unchanged_files)
rt = self.rt({"write_unchanged_files" : True})
self.assertTrue(rt.write_unchanged_files)
def test_fixer_loading_helpers(self):
contents = ["explicit", "first", "last", "parrot", "preorder"]
non_prefixed = refactor.get_all_fix_names("myfixes")
@ -176,7 +182,9 @@ from __future__ import print_function"""
"<stdin>", False]
self.assertEqual(results, expected)
def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS):
def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS,
options=None, mock_log_debug=None,
actually_write=True):
tmpdir = tempfile.mkdtemp(prefix="2to3-test_refactor")
self.addCleanup(shutil.rmtree, tmpdir)
# make a copy of the tested file that we can write to
@ -189,11 +197,15 @@ from __future__ import print_function"""
return fp.read()
old_contents = read_file()
rt = self.rt(fixers=fixers)
rt = self.rt(fixers=fixers, options=options)
if mock_log_debug:
rt.log_debug = mock_log_debug
rt.refactor_file(test_file)
self.assertEqual(old_contents, read_file())
if not actually_write:
return
rt.refactor_file(test_file, True)
new_contents = read_file()
self.assertNotEqual(old_contents, new_contents)
@ -203,6 +215,26 @@ from __future__ import print_function"""
test_file = os.path.join(FIXER_DIR, "parrot_example.py")
self.check_file_refactoring(test_file, _DEFAULT_FIXERS)
def test_refactor_file_write_unchanged_file(self):
test_file = os.path.join(FIXER_DIR, "parrot_example.py")
debug_messages = []
def recording_log_debug(msg, *args):
debug_messages.append(msg % args)
self.check_file_refactoring(test_file, fixers=(),
options={"write_unchanged_files": True},
mock_log_debug=recording_log_debug,
actually_write=False)
# Testing that it logged this message when write=False was passed is
# sufficient to see that it did not bail early after "No changes".
message_regex = r"Not writing changes to .*%s%s" % (
os.sep, os.path.basename(test_file))
for message in debug_messages:
if "Not writing changes" in message:
self.assertRegexpMatches(message, message_regex)
break
else:
self.fail("%r not matched in %r" % (message_regex, debug_messages))
def test_refactor_dir(self):
def check(structure, expected):
def mock_refactor_file(self, f, *args):

9
Misc/NEWS
View File

@ -466,6 +466,10 @@ Core and Builtins
Library
-------
- Issue #13930: lib2to3 now supports writing converted output files to another
directory tree as well as copying unchanged files and altering the file
suffix.
- Issue #9750: Fix sqlite3.Connection.iterdump on tables and fields
with a name that is a keyword or contains quotes. Patch by Marko
Kohtala.
@ -1927,6 +1931,11 @@ IDLE
Tools/Demos
-----------
- Issue #13930: 2to3 is now able to write its converted output files to another
directory tree as well as copying unchanged files and altering the file
suffix. See its new -o, -W and --add-suffix options. This makes it more
useful in many automated code translation workflows.
- Issue #13628: python-gdb.py is now able to retrieve more frames in the Python
traceback if Python is optimized.