Index: Doc/library/2to3.rst =================================================================== --- Doc/library/2to3.rst (revision 77336) +++ Doc/library/2to3.rst (working copy) @@ -76,7 +76,8 @@ Sometimes 2to3 will find a place in your source code that needs to be changed, but 2to3 cannot fix automatically. In this case, 2to3 will print a warning beneath the diff for a file. You should address the warning in order to have -compliant 3.x code. +compliant 3.x code. Using the :option:`-c` flag will write the warning +as a comment above the line which caused the warning. 2to3 can also refactor doctests. To enable this mode, use the :option:`-d` flag. Note that *only* doctests will be refactored. This also doesn't require Index: Lib/lib2to3/pytree.py =================================================================== --- Lib/lib2to3/pytree.py (revision 77336) +++ Lib/lib2to3/pytree.py (working copy) @@ -180,6 +180,35 @@ self.parent = None return i + def get_previous_sibling(self): + """Return the node immediately preceding the invocant in their + parent's children list. If the invocant does not have a previous + sibling, return None.""" + if self.parent is None: + return None + + # Can't use index(); we need to test by identity + for i, sibling in enumerate(self.parent.children): + if sibling is self: + # The first child has no previous sibling. Special-case this + # because i-1 would return -1, which is a valid index. + if i == 0: + return None + try: + return self.parent.children[i-1] + except IndexError: + return None + + def get_previous_in_tree(self): + """Return the node immediately preceding the invocant, following + reverse pre-order traversal.""" + sib = self.get_previous_sibling() + if sib is not None: + while sib.children: + sib = sib.children[-1] + return sib + return self.parent + @property def next_sibling(self): """ Index: Lib/lib2to3/fixer_util.py =================================================================== --- Lib/lib2to3/fixer_util.py (revision 77336) +++ Lib/lib2to3/fixer_util.py (working copy) @@ -127,7 +127,40 @@ imp = Node(syms.import_from, children) return imp +def insert_comment(node, comment_text): + """Inserts a Python comment on the line above the given Node or Leaf, + taking indendation into account.""" + # Calculate the indentation level of the given node. Do this by traversing + # backwards (previous sibling(s), then parent) until either a token.NEWLINE + # or token.INDENT is encountered. When this loop is done, `indent` will + # hold a string representing the exact indentation, and `node` will be the + # node immediately after the newline -- the one before which the comment + # should be added. + + indent = '' + while node.parent is not None: + prev = node.get_previous_in_tree() + if prev.type == token.INDENT: + indent = prev.value + node = prev + break + elif prev.type == token.NEWLINE: + break + node = prev + + # Drill down into the Node until we have a Leaf. + while isinstance(node, Node): + node = node.children[0] + + # Comments are represented in the AST using the 'prefix' attribute + # of a Leaf, so we add comments by changing the 'prefix' attribute. + # Note that we're including the previous node.prefix in here, in case + # this Leaf already had a comment on it. In that case, our new comment + # will be added *after* it. + node.prefix = '%s%s# %s\n' % (indent, node.prefix, comment_text) + + ########################################################### ### Determine whether a node represents a given literal ########################################################### Index: Lib/lib2to3/tests/test_util.py =================================================================== --- Lib/lib2to3/tests/test_util.py (revision 77336) +++ Lib/lib2to3/tests/test_util.py (working copy) @@ -59,6 +59,117 @@ self.assertFalse(self.is_list("[]+[]")) +class Test_insert_comment(support.TestCase): + def test_simple(self): + """ + Input Output + ----- ------ + + foo # test + foo + """ + node = parse('foo') + fixer_util.insert_comment(node.children[0].children[0], 'test') + self.assertEqual(str(node), "# test\nfoo\n\n") + + def test_simple2(self): + """ + Input Output + ----- ------ + + foo # test + foo + """ + node = parse('foo') + fixer_util.insert_comment(node, 'test') + self.assertEqual(str(node), "# test\nfoo\n\n") + + def test_indent1(self): + """ + Input Output + ----- ------ + + if foo: if foo: + bar() # test + bar() + """ + node = parse('if foo:\n bar()') + expected = 'if foo:\n # test\n bar()\n\n' + leaf = node.children[0].children[3].children[2] + self.assertEqual(str(leaf), 'bar()\n') + fixer_util.insert_comment(leaf, 'test') + self.assertEqual(str(node), expected) + + def test_indent2(self): + """ + Input Output + ----- ------ + + if foo: if foo: + if bar: if bar: + baz() # test + baz() + """ + node = parse('if foo:\n if bar:\n baz()') + expected = 'if foo:\n if bar:\n # test\n baz()\n\n' + leaf = node.children[0].children[3].children[2].children[3].children[2] + self.assertEqual(str(leaf), 'baz()\n') + fixer_util.insert_comment(leaf, 'test') + self.assertEqual(str(node), expected) + + def test_indent3(self): + """ + Input Output + ----- ------ + + if foo: if foo: + bar() bar() + else: else: + baz() # test + baz() + """ + node = parse('if foo:\n bar()\nelse:\n baz()') + expected = 'if foo:\n bar()\nelse:\n # test\n baz()\n\n' + leaf = node.children[0].children[6].children[2] + self.assertEqual(str(leaf), 'baz()\n') + fixer_util.insert_comment(leaf, 'test') + self.assertEqual(str(node), expected) + + def test_indent4(self): + """ + Input Output + ----- ------ + + if foo: if foo: + bar() bar() + else: # test + baz() else: + baz() + """ + node = parse('if foo:\n bar()\nelse:\n baz()') + expected = 'if foo:\n bar()\n# test\nelse:\n baz()\n\n' + leaf = node.children[0].children[4] + self.assertEqual(str(leaf), 'else') + fixer_util.insert_comment(leaf, 'test') + self.assertEqual(str(node), expected) + + def test_indent5(self): + """ + Input Output + ----- ------ + + # legacycomment # legacycomment + foo # test + foo + """ + node = parse('# legacycomment\nfoo') + expected = '# legacycomment\n# test\nfoo\n\n' + leaf = node.children[0].children[0] + self.assertEqual(str(leaf), '# legacycomment\nfoo') + fixer_util.insert_comment(leaf, 'test') + self.assertEqual(str(node), expected) + + class Test_Attr(MacroTestCase): def test(self): call = parse("foo()", strip_levels=2) Index: Lib/lib2to3/tests/test_pytree.py =================================================================== --- Lib/lib2to3/tests/test_pytree.py (revision 77336) +++ Lib/lib2to3/tests/test_pytree.py (working copy) @@ -296,6 +296,15 @@ # I don't care what it raises, so long as it's an exception self.assertRaises(Exception, n1.insert_child, 0, list) + def testNodePreviousSibling(self): + n1 = pytree.Node(1000, []) + n2 = pytree.Node(1000, []) + p1 = pytree.Node(1000, [n1, n2]) + + self.assertTrue(n2.get_previous_sibling() is n1) + self.assertEqual(n1.get_previous_sibling(), None) + self.assertEqual(p1.get_previous_sibling(), None) + def test_node_append_child(self): n1 = pytree.Node(1000, []) @@ -321,6 +330,15 @@ self.assertEqual(n2.next_sibling, None) self.assertEqual(p1.next_sibling, None) + def testLeafPreviousSibling(self): + l1 = pytree.Leaf(100, "a") + l2 = pytree.Leaf(100, "b") + p1 = pytree.Node(1000, [l1, l2]) + + self.assertTrue(l2.get_previous_sibling() is l1) + self.assertEqual(l1.get_previous_sibling(), None) + self.assertEqual(p1.get_previous_sibling(), None) + def test_leaf_next_sibling(self): l1 = pytree.Leaf(100, "a") l2 = pytree.Leaf(100, "b") @@ -330,6 +348,47 @@ self.assertEqual(l2.next_sibling, None) self.assertEqual(p1.next_sibling, None) + def testGetPreviousInTree(self): + # Create the following tree: + # + # P1 + # / \ + # N1 N2 + # / \ / \ + # N3 N4 N5 N6 + # / / | / \ + # N7 N8 N9 N10 N11 + # | + # N12 + n7 = pytree.Node(1000, []) + n8 = pytree.Node(1000, []) + n9 = pytree.Node(1000, []) + n5 = pytree.Node(1000, []) + n12 = pytree.Node(1000, []) + n11 = pytree.Node(1000, []) + n3 = pytree.Node(1000, [n7]) + n4 = pytree.Node(1000, [n8, n9]) + n10 = pytree.Node(1000, [n12]) + n6 = pytree.Node(1000, [n10, n11]) + n1 = pytree.Node(1000, [n3, n4]) + n2 = pytree.Node(1000, [n5, n6]) + p1 = pytree.Node(1000, [n1, n2]) + + self.assertEqual(p1.get_previous_in_tree(), None) + self.assertTrue(n1.get_previous_in_tree() is p1) + self.assertTrue(n2.get_previous_in_tree() is n9) + self.assertTrue(n3.get_previous_in_tree() is n1) + self.assertTrue(n4.get_previous_in_tree() is n7) + self.assertTrue(n5.get_previous_in_tree() is n2) + self.assertTrue(n6.get_previous_in_tree() is n5) + self.assertTrue(n7.get_previous_in_tree() is n3) + self.assertTrue(n8.get_previous_in_tree() is n4) + self.assertTrue(n9.get_previous_in_tree() is n8) + self.assertTrue(n10.get_previous_in_tree() is n6) + self.assertTrue(n11.get_previous_in_tree() is n12) + self.assertTrue(n12.get_previous_in_tree() is n10) + + def test_node_prev_sibling(self): n1 = pytree.Node(1000, []) n2 = pytree.Node(1000, []) Index: Lib/lib2to3/main.py =================================================================== --- Lib/lib2to3/main.py (revision 77336) +++ Lib/lib2to3/main.py (working copy) @@ -105,6 +105,8 @@ help="Write back modified files") parser.add_option("-n", "--nobackups", action="store_true", default=False, help="Don't write backups for modified files.") + parser.add_option("-c", "--comments", action="store_true", + help="Write warnings as comments in source") # Parse command line arguments refactor_stdin = False @@ -131,6 +133,8 @@ return 2 if options.print_function: flags["print_function"] = True + if options.comments: + flags["comments"] = True # Set up logging handler level = logging.DEBUG if options.verbose else logging.INFO Index: Lib/lib2to3/refactor.py =================================================================== --- Lib/lib2to3/refactor.py (revision 77336) +++ Lib/lib2to3/refactor.py (working copy) @@ -168,7 +168,7 @@ class RefactoringTool(object): - _default_options = {"print_function" : False} + _default_options = {"print_function" : False, "comments" : False} CLASS_PREFIX = "Fix" # The prefix for fixer classes FILE_PREFIX = "fix_" # The prefix for modules with a fixer within Index: Lib/lib2to3/fixer_base.py =================================================================== --- Lib/lib2to3/fixer_base.py (revision 77336) +++ Lib/lib2to3/fixer_base.py (working copy) @@ -10,7 +10,7 @@ # Local imports from .patcomp import PatternCompiler from . import pygram -from .fixer_util import does_tree_import +from .fixer_util import does_tree_import, insert_comment class BaseFix(object): @@ -113,6 +113,17 @@ self.log.append("### In file %s ###" % self.filename) self.log.append(message) + def generate_log_message(self, node, detail, reason=None): + lineno = node.get_lineno() + for_output = node.clone() + for_output.prefix = "" + msg = "Line %d: %s: %s" + self.log_message(msg % (lineno, detail, for_output)) + if reason: + self.log_message(reason) + if self.options["comments"]: + insert_comment(node, "2to3: %s" % reason) + def cannot_convert(self, node, reason=None): """Warn the user that a given chunk of code is not valid Python 3, but that it cannot be converted automatically. @@ -120,13 +131,7 @@ First argument is the top-level node for the code in question. Optional second argument is why it can't be converted. """ - lineno = node.get_lineno() - for_output = node.clone() - for_output.prefix = "" - msg = "Line %d: could not convert: %s" - self.log_message(msg % (lineno, for_output)) - if reason: - self.log_message(reason) + self.generate_log_message(node, "could not convert", reason) def warning(self, node, reason): """Used for warning the user about possible uncertainty in the @@ -135,8 +140,7 @@ First argument is the top-level node for the code in question. Optional second argument is why it can't be converted. """ - lineno = node.get_lineno() - self.log_message("Line %d: %s" % (lineno, reason)) + self.generate_log_message(node, "uncertain conversion", reason) def start_tree(self, tree, filename): """Some fixers need to maintain tree-wide state.