diff -r 813a34170ac5 -r a6cd0518495e Lib/lib2to3/main.py --- a/Lib/lib2to3/main.py Fri Feb 03 18:23:05 2012 +0000 +++ b/Lib/lib2to3/main.py Thu Feb 02 23:33:45 2012 -0800 @@ -77,6 +77,61 @@ (filename,)) return + +class FilenameChangingStdoutRefactoringTool(StdoutRefactoringTool): + """A refactoring tool that can avoid overwriting its input files. + + Output files can optionally be written to a different directory and or + have an extra file suffix appended to their name for use in situations + where you do not want to replace the input files. + """ + + def __init__(self, fixers, options, explicit, nobackups, show_diffs, + input_base_dir='', output_dir='', append_suffix=''): + """Create a refactoring tool that can avoid overwriting its input files. + + Args: + fixers: See the super class. + options: See the super class. + explicit: See the super class. + nobackups: See the super class. + show_diffs: See the super class. + input_base_dir: The base directory for all input files. This class + will strip this path prefix off of filenames before substituting + it with output_dir. Only meaningful if output_dir is supplied. + output_dir: If supplied, all converted files will be written into + this directory tree instead of input_base_dir. + append_suffix: If supplied, all files output by this tool will have + this appended to their filename. Useful for changing .py to + .py3 for example by passing append_suffix='3'. + """ + if input_base_dir and input_base_dir[-1] != os.sep: + input_base_dir += os.sep + self._input_base_dir = input_base_dir + self._output_dir = output_dir + self._append_suffix = append_suffix + super(FilenameChangingStdoutRefactoringTool, self).__init__( + fixers, options, explicit, nobackups, show_diffs) + + def write_file(self, new_text, filename, old_text, encoding): + """Writes new_text; to a new path and directory if configured to.""" + orig_filename = filename + if self._output_dir: + if filename.startswith(self._input_base_dir): + filename = os.path.join(self._output_dir, + filename[len(self._input_base_dir):]) + if self._append_suffix: + filename += self._append_suffix + if orig_filename != filename: + output_dir = os.path.dirname(filename) + if not os.path.isdir(output_dir): + os.makedirs(output_dir) + self.log_message('Writing converted %s to %s.', orig_filename, filename) + super(FilenameChangingStdoutRefactoringTool, self).write_file( + new_text, filename, old_text, encoding) + shutil.copymode(orig_filename, filename) + + def warn(msg): print("WARNING: %s" % (msg,), file=sys.stderr) @@ -113,11 +168,33 @@ help="Write back modified files") parser.add_option("-n", "--nobackups", action="store_true", default=False, help="Don't write backups for modified files") + parser.add_option("-o", "--output-dir", action="store", type="str", + default="", help="Put output files in this directory " + "instead of overwriting the input files. Requires -n.") + parser.add_option("-W", "--write-unchanged-files", action="store_true", + help="Also write files even if no changes were required" + " (useful with --output-dir); implies -w and -n.") + parser.add_option("--add-suffix", action="store", type="str", default="", + help="Append this string to all output filenames." + " Requires -n if non-empty. " + "ex: --add-suffix='3' will generate .py3 files.") # Parse command line arguments refactor_stdin = False flags = {} options, args = parser.parse_args(args) + if options.write_unchanged_files: + flags["write_unchanged_files"] = True + if not options.write: + warn("--write-unchanged-files/-W implies -w.") + options.write = True + # If we allowed these, the original files would be renamed to backup names + # but not replaced. + if options.output_dir and not options.nobackups: + parser.error("Can't use --output-dir/-o without -n.") + if options.add_suffix and not options.nobackups: + parser.error("Can't use --add-suffix without -n.") + if not options.write and options.no_diffs: warn("not writing files and not printing diffs; that's not very useful") if not options.write and options.nobackups: @@ -143,6 +220,7 @@ # Set up logging handler level = logging.DEBUG if options.verbose else logging.INFO logging.basicConfig(format='%(name)s: %(message)s', level=level) + logger = logging.getLogger('lib2to3.main') # Initialize the refactoring tool avail_fixes = set(refactor.get_fixers_from_package(fixer_pkg)) @@ -159,8 +237,21 @@ else: requested = avail_fixes.union(explicit) fixer_names = requested.difference(unwanted_fixes) - rt = StdoutRefactoringTool(sorted(fixer_names), flags, sorted(explicit), - options.nobackups, not options.no_diffs) + input_base_dir = os.path.commonprefix(args) + if (input_base_dir and not input_base_dir.endswith(os.sep) + and not os.path.isdir(input_base_dir)): + # One or more similar names were passed, their directory is the base. + input_base_dir = os.path.dirname(input_base_dir) + if options.output_dir: + input_base_dir = input_base_dir.rstrip(os.sep) + logger.info('Output in %r will mirror the input directory %r layout.', + options.output_dir, input_base_dir) + rt = FilenameChangingStdoutRefactoringTool( + sorted(fixer_names), flags, sorted(explicit), + options.nobackups, not options.no_diffs, + input_base_dir=input_base_dir, + output_dir=options.output_dir, + append_suffix=options.add_suffix) # Refactor all files and directories passed as arguments if not rt.errors: diff -r 813a34170ac5 -r a6cd0518495e Lib/lib2to3/refactor.py --- a/Lib/lib2to3/refactor.py Fri Feb 03 18:23:05 2012 +0000 +++ b/Lib/lib2to3/refactor.py Thu Feb 02 23:33:45 2012 -0800 @@ -173,7 +173,8 @@ class RefactoringTool(object): - _default_options = {"print_function" : False} + _default_options = {"print_function" : False, + "write_unchanged_files" : False} CLASS_PREFIX = "Fix" # The prefix for fixer classes FILE_PREFIX = "fix_" # The prefix for modules with a fixer within @@ -195,6 +196,10 @@ self.grammar = pygram.python_grammar_no_print_statement else: self.grammar = pygram.python_grammar + # When this is True, the refactor*() methods will call write_file() for + # files processed even if they were not changed during refactoring. If + # and only if the refactor method's write parameter was True. + self.write_unchanged_files = self.options.get("write_unchanged_files") self.errors = [] self.logger = logging.getLogger("RefactoringTool") self.fixer_log = [] @@ -341,13 +346,13 @@ if doctests_only: self.log_debug("Refactoring doctests in %s", filename) output = self.refactor_docstring(input, filename) - if output != input: + if output != input or self.write_unchanged_files: self.processed_file(output, filename, input, write, encoding) else: self.log_debug("No doctest changes in %s", filename) else: tree = self.refactor_string(input, filename) - if tree and tree.was_changed: + if tree and tree.was_changed or self.write_unchanged_files: # The [:-1] is to take off the \n we added earlier self.processed_file(str(tree)[:-1], filename, write=write, encoding=encoding) @@ -386,13 +391,13 @@ if doctests_only: self.log_debug("Refactoring doctests in stdin") output = self.refactor_docstring(input, "") - if output != input: + if output != input or self.write_unchanged_files: self.processed_file(output, "", input) else: self.log_debug("No doctest changes in stdin") else: tree = self.refactor_string(input, "") - if tree and tree.was_changed: + if tree and tree.was_changed or self.write_unchanged_files: self.processed_file(str(tree), "", input) else: self.log_debug("No changes in stdin") @@ -502,7 +507,7 @@ def processed_file(self, new_text, filename, old_text=None, write=False, encoding=None): """ - Called when a file has been refactored, and there are changes. + Called when a file has been refactored and there may be changes. """ self.files.append(filename) if old_text is None: @@ -513,7 +518,8 @@ self.print_output(old_text, new_text, filename, equal) if equal: self.log_debug("No changes to %s", filename) - return + if not self.write_unchanged_files: + return if write: self.write_file(new_text, filename, old_text, encoding) else: diff -r 813a34170ac5 -r a6cd0518495e Lib/lib2to3/tests/test_main.py --- a/Lib/lib2to3/tests/test_main.py Fri Feb 03 18:23:05 2012 +0000 +++ b/Lib/lib2to3/tests/test_main.py Thu Feb 02 23:33:45 2012 -0800 @@ -1,18 +1,39 @@ # -*- coding: utf-8 -*- +import codecs +import io +import logging +import os +import shutil import sys -import codecs -import logging -import io +import tempfile import unittest from lib2to3 import main +TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data") +PY2_TEST_MODULE = os.path.join(TEST_DATA_DIR, "py2_test_grammar.py") + + class TestMain(unittest.TestCase): + if not hasattr(unittest.TestCase, 'assertNotRegex'): + # This method was only introduced in 3.2. + def assertNotRegex(self, text, regexp, msg=None): + import re + if not hasattr(regexp, 'search'): + regexp = re.compile(regexp) + if regexp.search(text): + self.fail("regexp %s MATCHED text %r" % (regexp.pattern, text)) + + def setUp(self): + self.temp_dir = None # tearDown() will rmtree this dir up if set. + def tearDown(self): # Clean up logging configuration down by main. del logging.root.handlers[:] + if self.temp_dir: + shutil.rmtree(self.temp_dir) def run_2to3_capture(self, args, in_capture, out_capture, err_capture): save_stdin = sys.stdin @@ -39,3 +60,86 @@ self.assertTrue("-print 'nothing'" in output) self.assertTrue("WARNING: couldn't encode 's diff for " "your terminal" in err.getvalue()) + + def setup_test_source_trees(self): + """Setup a test source tree and output destination tree.""" + self.temp_dir = tempfile.mkdtemp() # tearDown() cleans this up. + self.py2_src_dir = os.path.join(self.temp_dir, "python2_project") + self.py3_dest_dir = os.path.join(self.temp_dir, "python3_project") + os.mkdir(self.py2_src_dir) + os.mkdir(self.py3_dest_dir) + # Turn it into a package with a few files. + self.setup_files = [] + open(os.path.join(self.py2_src_dir, "__init__.py"), "w").close() + self.setup_files.append("__init__.py") + shutil.copy(PY2_TEST_MODULE, self.py2_src_dir) + self.setup_files.append(os.path.basename(PY2_TEST_MODULE)) + self.trivial_py2_file = os.path.join(self.py2_src_dir, "trivial.py") + with open(self.trivial_py2_file, "w") as trivial: + trivial.write("print 'I need a simple conversion.'") + self.setup_files.append("trivial.py") + + def test_filename_changing_on_output_single_dir(self): + """2to3 a single directory with a new output dir and suffix.""" + self.setup_test_source_trees() + out = io.StringIO() + err = io.StringIO() + suffix = "TEST" + ret = self.run_2to3_capture( + ["-n", "--add-suffix", suffix, "--write-unchanged-files", + "--no-diffs", "--output-dir", + self.py3_dest_dir, self.py2_src_dir], + io.StringIO(""), out, err) + self.assertEqual(ret, 0) + stderr = err.getvalue() + self.assertIn(" implies -w.", stderr) + self.assertIn( + "Output in {!r} will mirror the input directory {!r} layout" + .format(self.py3_dest_dir, self.py2_src_dir), stderr) + self.assertEqual(set(name+suffix for name in self.setup_files), + set(os.listdir(self.py3_dest_dir))) + for name in self.setup_files: + self.assertIn("Writing converted {!s} to {!s}".format( + os.path.join(self.py2_src_dir, name), + os.path.join(self.py3_dest_dir, name+suffix)), stderr) + self.assertRegexpMatches(stderr, r"No changes to .*/__init__\.py") + self.assertNotRegex(stderr, r"No changes to .*/trivial\.py") + + def test_filename_changing_on_output_two_files(self): + """2to3 two files in one directory with a new output dir.""" + self.setup_test_source_trees() + err = io.StringIO() + ret = self.run_2to3_capture( + ["-n", "-w", "--write-unchanged-files", + "--no-diffs", "--output-dir", self.py3_dest_dir, + os.path.join(self.py2_src_dir, self.setup_files[0]), + os.path.join(self.py2_src_dir, self.setup_files[1])], + io.StringIO(""), io.StringIO(), err) + self.assertEqual(ret, 0) + stderr = err.getvalue() + self.assertIn( + "Output in {!r} will mirror the input directory {!r} layout" + .format(self.py3_dest_dir, self.py2_src_dir), stderr) + self.assertEqual(set(self.setup_files[:2]), + set(os.listdir(self.py3_dest_dir))) + + def test_filename_changing_on_output_single_file(self): + """2to3 a single file with a new output dir.""" + self.setup_test_source_trees() + err = io.StringIO() + ret = self.run_2to3_capture( + ["-n", "-w", "--no-diffs", "--output-dir", self.py3_dest_dir, + self.trivial_py2_file], + io.StringIO(""), io.StringIO(), err) + self.assertEqual(ret, 0) + stderr = err.getvalue() + self.assertIn( + "Output in {!r} will mirror the input directory {!r} layout" + .format(self.py3_dest_dir, self.py2_src_dir), stderr) + self.assertEqual(set([os.path.basename(self.trivial_py2_file)]), + set(os.listdir(self.py3_dest_dir))) + + + +if __name__ == '__main__': + unittest.main() diff -r 813a34170ac5 -r a6cd0518495e Lib/lib2to3/tests/test_refactor.py --- a/Lib/lib2to3/tests/test_refactor.py Fri Feb 03 18:23:05 2012 +0000 +++ b/Lib/lib2to3/tests/test_refactor.py Thu Feb 02 23:33:45 2012 -0800 @@ -53,6 +53,12 @@ self.assertTrue(rt.driver.grammar is pygram.python_grammar_no_print_statement) + def test_write_unchanged_files_option(self): + rt = self.rt() + self.assertFalse(rt.write_unchanged_files) + rt = self.rt({"write_unchanged_files" : True}) + self.assertTrue(rt.write_unchanged_files) + def test_fixer_loading_helpers(self): contents = ["explicit", "first", "last", "parrot", "preorder"] non_prefixed = refactor.get_all_fix_names("myfixes") @@ -176,16 +182,22 @@ "", False] self.assertEqual(results, expected) - def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS): + def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS, + options=None, mock_log_debug=None, + actually_write=True): def read_file(): with open(test_file, "rb") as fp: return fp.read() old_contents = read_file() - rt = self.rt(fixers=fixers) + rt = self.rt(fixers=fixers, options=options) + if mock_log_debug: + rt.log_debug = mock_log_debug rt.refactor_file(test_file) self.assertEqual(old_contents, read_file()) + if not actually_write: + return try: rt.refactor_file(test_file, True) new_contents = read_file() @@ -199,6 +211,19 @@ test_file = os.path.join(FIXER_DIR, "parrot_example.py") self.check_file_refactoring(test_file, _DEFAULT_FIXERS) + def test_refactor_file_write_unchanged_file(self): + test_file = os.path.join(FIXER_DIR, "parrot_example.py") + debug_messages = [] + def recording_log_debug(msg, *args): + debug_messages.append(msg % args) + self.check_file_refactoring(test_file, fixers=(), + options={"write_unchanged_files": True}, + mock_log_debug=recording_log_debug, + actually_write=False) + # Testing that it logged this message when write=False was passed is + # sufficient to see that it did not bail early with "No changes". + self.assertIn("Not writing changes to %s" % test_file, debug_messages) + def test_refactor_dir(self): def check(structure, expected): def mock_refactor_file(self, f, *args): diff -r 813a34170ac5 -r a6cd0518495e Lib/logging/__init__.py --- a/Lib/logging/__init__.py Fri Feb 03 18:23:05 2012 +0000 +++ b/Lib/logging/__init__.py Thu Feb 02 23:33:45 2012 -0800 @@ -721,7 +721,7 @@ You could, however, replace this with a custom handler if you wish. The record which was being processed is passed in to this method. """ - if raiseExceptions: + if raiseExceptions and sys.stderr: # see issue 13807 ei = sys.exc_info() try: traceback.print_exception(ei[0], ei[1], ei[2], None, sys.stderr)