tokenize the source to manage Pdb breakpoints

Lib/test/test_pdb.py

 # A test suite for pdb; not very comprehensive at the moment.
 
 import imp
+import bdb
 import pdb
 import sys
 import unittest
 import subprocess
 import textwrap
+import time
+try:
+    import threading
+except ImportError:
+    threading = None
 
 from test import support
 # This little helper class is essential for testing pdb under doctest.
 from test.test_doctest import _FakeInput
 
 
 class PdbTestInput(object):
     """Context manager that makes testing Pdb in doctests easier."""
 
     def __init__(self, input):
@@ skipping 246 matching lines @@
     Breakpoint 4 at <doctest test.test_pdb.test_pdb_breakpoint_commands[0]>:5
     (Pdb) continue
     2
     Deleted breakpoint 4 at <doctest test.test_pdb.test_pdb_breakpoint_commands[0]>:5
     > <doctest test.test_pdb.test_pdb_breakpoint_commands[0]>(5)test_function()
     -> print(3)
     (Pdb) break
     (Pdb) continue
     3
     4
+    """
+
+
+def test_issue_14792():
+    """
+    >>> def foo():
+    ...     x = 1
+    ...     x = 2
+
+    >>> def test_function():
+    ...     import pdb; pdb.Pdb(nosigint=True).set_trace()
+    ...     foo()
+
+    >>> with PdbTestInput([
+    ...     'step',
+    ...     'step',
+    ...     'break foo',
+    ...     'continue',
+    ... ]):
+    ...    test_function()
+    > <doctest test.test_pdb.test_issue_14792[1]>(3)test_function()
+    -> foo()
+    (Pdb) step
+    --Call--
+    > <doctest test.test_pdb.test_issue_14792[0]>(1)foo()
+    -> def foo():
+    (Pdb) step
+    > <doctest test.test_pdb.test_issue_14792[0]>(2)foo()
+    -> x = 1
+    (Pdb) break foo
+    Breakpoint 1 at <doctest test.test_pdb.test_issue_14792[0]>:2
+    (Pdb) continue
     """
 
 
 def do_nothing():
     pass
 
 def do_something():
     print(42)
 
 def test_list_commands():
@@ skipping 303 matching lines @@
     (Pdb) continue
 
     >>> with PdbTestInput(['x', 'continue']):
     ...     x=0
     ...     pdb_invoke('runeval', compile('x+1', '<string>', 'eval'))
     > <string>(1)<module>()->None
     (Pdb) x
     1
     (Pdb) continue
     """
+
+
+def normalize(result, filename='', strip_bp_lnum=False):
+    """Normalize a test result."""
+    lines = []
+    for line in result.splitlines():
+        while line.startswith('(Pdb) ') or line.startswith('(com) '):
+            line = line[6:]
+        words = line.split()
+        line = []
+        # Replace tabs with spaces
+        for word in words:
+            if filename:
+                idx = word.find(filename)
+                # Remove the filename prefix
+                if idx > 0:
+                    word = word[idx:]
+                if idx >=0 and strip_bp_lnum:
+                    idx = word.find(':')
+                    # Remove the ':' separator and breakpoint line number
+                    if idx > 0:
+                        word = word[:idx]
+            line.append(word)
+        line = ' '.join(line)
+        lines.append(line.strip())
+    return '\n'.join(lines)
 
 
 class PdbTestCase(unittest.TestCase):
 
-    def run_pdb(self, script, commands):
+    def run_pdb(self, script, commands, filename):
         """Run 'script' lines with pdb and the pdb 'commands'."""
-        filename = 'main.py'
         with open(filename, 'w') as f:
             f.write(textwrap.dedent(script))
         self.addCleanup(support.unlink, filename)
         cmd = [sys.executable, '-m', 'pdb', filename]
         stdout = stderr = None
         with subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                    stdin=subprocess.PIPE,
-                                   stderr=subprocess.STDOUT,
+                                   stderr=subprocess.PIPE,
                                    ) as proc:
             stdout, stderr = proc.communicate(str.encode(commands))
         stdout = stdout and bytes.decode(stdout)
         stderr = stderr and bytes.decode(stderr)
         return stdout, stderr
 
     def test_issue7964(self):
         # open the file as binary so we can force \r\n newline
         with open(support.TESTFN, 'wb') as f:
             f.write(b'print("testing my pdb")\r\n')
@@ skipping 32 matching lines @@
             step
             quit
         """
         bar = """
             def bar():
                 pass
         """
         with open('bar.py', 'w') as f:
             f.write(textwrap.dedent(bar))
         self.addCleanup(support.unlink, 'bar.py')
-        stdout, stderr = self.run_pdb(script, commands)
+        stdout, stderr = self.run_pdb(script, commands, 'main.py')
         self.assertTrue(
             any('main.py(5)foo()->None' in l for l in stdout.splitlines()),
             'Fail to step into the caller after a return')
 
+    def test_bdb_module_parser(self):
+        script = """
+            '''Module documentation.'''
+            X = None
+
+            def bar(a, b):
+                # comment
+                pass # comment
+                pass
+                # comment
+
+            def main(x=None,
+                     y=None):
+                '''Function documentation.'''
+                def nested_def():
+                    pass
+
+                bar(x,
+                    y)
+                # comment
+                pass
+                bar(
+                    x, y # comment
+                    )
+                class NestedClass:
+                    def n_foo(self):
+                        pass
+                pass
+
+                return bar(
+                    x, y
+                    )
+
+            class C:
+                '''Class documentation.'''
+                x = None
+                y = None
+
+                class D:
+                    def d_foo(self):
+                        pass
+                    class E:
+                        def e_foo(self):
+                            pass
+
+                def c_bar(self):
+                    pass
+
+                c_bar2 = c_bar
+                c_bar3 = c_bar
+
+            def foo():
+                pass
+
+            if __name__ == '__main__':
+                result = main(
+                            1, 2
+                            )
+                print(result)
+
+        """
+        expected = """
+            Functions:
+            [('bar', 5),
+             ('main', 11),
+             ('C.D.d_foo', 39),
+             ('C.D.E.e_foo', 42),
+             ('C.c_bar', 45),
+             ('foo', 51)]
+            Functions and classes firstlineno:
+            [(5, 5),
+             (11, 11),
+             (14, 11),
+             (24, 11),
+             (25, 11),
+             (33, 0),
+             (38, 0),
+             (39, 39),
+             (41, 0),
+             (42, 42),
+             (45, 45),
+             (51, 51)]
+            Last line: 59
+            Module lines:
+            [(3, 3, 1, 0),
+             (33, 36, 1, 0),
+             (38, 38, 1, 0),
+             (41, 41, 1, 0),
+             (48, 49, 1, 0),
+             (54, 54, 1, 0),
+             (55, 57, 2, 0),
+             (58, 58, 1, 0)]
+            Functions lines:
+            [(5, 5, 0, 5), (7, 8, 1, 5)]
+            [(11, 12, 0, 11),
+             (14, 15, 1, 11),
+             (17, 18, 2, 11),
+             (20, 20, 1, 11),
+             (21, 23, 2, 11),
+             (24, 27, 1, 11),
+             (29, 31, 2, 11)]
+            [(39, 39, 0, 39), (40, 40, 1, 39)]
+            [(42, 42, 0, 42), (43, 43, 1, 42)]
+            [(45, 45, 0, 45), (46, 46, 1, 45)]
+            [(51, 51, 0, 51), (52, 52, 1, 51)]
+        """
+        filename = 'main.py'
+        with open(filename, 'w') as f:
+            f.write(textwrap.dedent(script))
+        self.addCleanup(support.unlink, filename)
+        result = str(bdb.ModuleSource(filename))
+        expected = textwrap.dedent(expected)
+        self.assertTrue(result in expected,
+            '\n\nExpected:\n{}\nGot:\n{}\n'
+            'Fail to parse correctly a module.'.format(expected, result))
+
+    def test_breakpoints_set_on_non_statement(self):
+        script = """
+            def foo():
+
+                # comment
+                x = 1
+
+                x = len(
+                        'foo'
+                        )
+
+            foo()
+        """
+        commands = """
+            break 3
+            break 4
+            break 8
+            continue
+            break
+            continue
+            quit
+        """
+        expected = """
+            > main.py(2)<module>()
+            -> def foo():
+            Breakpoint 1 at main.py:3
+            Breakpoint 2 at main.py:4
+            Breakpoint 3 at main.py:8
+            > main.py(5)foo()
+            -> x = 1
+            Num Type Disp Enb Where
+            1 breakpoint keep yes at main.py:3
+            breakpoint already hit 1 time
+            2 breakpoint keep yes at main.py:4
+            breakpoint already hit 1 time
+            3 breakpoint keep yes at main.py:8
+            > main.py(7)foo()
+            -> x = len(
+        """
+        filename = 'main.py'
+        stdout, stderr = self.run_pdb(script, commands, filename)
+        stdout = normalize(stdout, filename)
+        expected = normalize(expected)
+        self.assertTrue(stdout in expected,
+            '\n\nExpected:\n{}\nGot:\n{}\n'
+            'Fail to stop at breakpoint set at empty line, comment line or'
+            ' multi-line statement.'.format(expected, stdout))
+
+    def test_issue14789(self):
+        script = """
+            def bar(a):
+                x = 1
+
+            bar(10)
+            bar(20)
+        """
+        commands = """
+            break bar
+            commands 1
+            print a
+            end
+            ignore 1 1
+            break bar
+            commands 2
+            print a + 1
+            end
+            ignore 2 1
+            continue
+            break
+            quit
+        """
+        expected = """
+            > main.py(2)<module>()
+            -> def bar(a):
+            Breakpoint 1 at main.py:3
+            Will ignore next 1 crossing of breakpoint 1.
+            Breakpoint 2 at main.py:3
+            Will ignore next 1 crossing of breakpoint 2.
+            20
+            21
+            > main.py(3)bar()
+            -> x = 1
+            Num Type         Disp Enb   Where
+            1 breakpoint keep yes at main.py:3
+                    breakpoint already hit 2 times
+            2 breakpoint keep yes at main.py:3
+                    breakpoint already hit 2 times
+        """
+        filename = 'main.py'
+        stdout, stderr = self.run_pdb(script, commands, filename)
+        stdout = normalize(stdout, filename)
+        expected = normalize(expected)
+        self.assertTrue(stdout in expected,
+            '\n\nExpected:\n{}\nGot:\n{}\n'
+            'Fail to handle two breakpoints set on the same line.'.format(
+                expected, stdout))
+
+    def test_issue14795(self):
+        script = """
+            class C:
+                def foo(self):
+                    pass
+        """
+        commands = """
+            break C.foo
+            break bar.bar
+            quit
+        """
+        bar = """
+            def bar():
+                pass
+        """
+        expected = """
+            > main.py(2)<module>()
+            -> class C:
+            Breakpoint 1 at main.py:4
+            Breakpoint 2 at bar.py:3
+        """
+        with open('bar.py', 'w') as f:
+            f.write(textwrap.dedent(bar))
+        self.addCleanup(support.unlink, 'bar.py')
+        filename = 'main.py'
+        stdout, stderr = self.run_pdb(script, commands, filename)
+        stdout = normalize(normalize(stdout, filename), 'bar.py')
+        expected = normalize(expected)
+        self.assertTrue(stdout in expected,
+            '\n\nExpected:\n{}\nGot:\n{}\n'
+            'Fail to set a breakpoint in a method, or in a function whose'
+            ' module is not yet imported.'.format(expected, stdout))
+
+    def test_issue_14808(self):
+        script = """
+            def foo():
+                pass
+
+            def bar():
+                pass
+
+            foo()
+            bar()
+        """
+        commands = """
+            break 2
+            break bar
+            continue
+            continue
+            break
+            quit
+        """
+        expected = """
+            > main.py(2)<module>()
+            -> def foo():
+            Breakpoint 1 at main.py:2
+            Breakpoint 2 at main.py:6
+            > main.py(3)foo()
+            -> pass
+            > main.py(6)bar()
+            -> pass
+            Num Type Disp Enb Where
+            1 breakpoint keep yes at main.py:2
+            breakpoint already hit 1 time
+            2 breakpoint keep yes at main.py:6
+            breakpoint already hit 1 time
+        """
+        filename = 'main.py'
+        stdout, stderr = self.run_pdb(script, commands, filename)
+        stdout = normalize(stdout, filename)
+        expected = normalize(expected)
+        self.assertTrue(stdout in expected,
+            '\n\nExpected:\n{}\nGot:\n{}\n'
+            'Fail to stop at breakpoint set at function definition line'
+            ' or at function name.'.format(expected, stdout))
+
+    @unittest.skipIf(not threading, 'the source file is changed in a thread')
+    def test_issue_14912(self):
+        script = """
+            import bar
+
+            def foo():
+                bar.bar()
+
+            foo()
+        """
+        commands = """
+            break bar.bar
+            continue
+            list
+            import time; time.sleep(2)
+            restart
+            continue
+            list
+            quit
+        """
+        bar = """
+            def bar():
+                x = 1
+                x = 2
+        """
+        bar_2 = """
+            def bar():
+                # comment
+                # comment
+                x = 1
+                x = 2
+        """
+        expected = """
+            > main.py(2)<module>()
+            -> import bar
+            Breakpoint 1 at bar.py:3
+            > bar.py(3)bar()
+            -> x = 1
+            1
+            2 def bar():
+            3 B-> x = 1
+            4 x = 2
+            [EOF]
+            Restarting main.py with arguments:
+            main.py
+            > main.py(2)<module>()
+            -> import bar
+            > bar.py(5)bar()
+            -> x = 1
+            1
+            2 def bar():
+            3 B # comment
+            4 # comment
+            5 -> x = 1
+            6 x = 2
+            [EOF]
+        """
+        with open('bar.py', 'w') as f:
+            f.write(textwrap.dedent(bar))
+        self.addCleanup(support.unlink, 'bar.py')
+        filename = 'main.py'
+        with open(filename, 'w') as f:
+            f.write(textwrap.dedent(script))
+        self.addCleanup(support.unlink, filename)
+        cmd = [sys.executable, '-m', 'pdb', filename]
+        stdout = stderr = None
+        with subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                                   stdin=subprocess.PIPE,
+                                   stderr=subprocess.PIPE,
+                                   ) as proc:
+            def update_source():
+                # Wait for the Pdb subprocess to be fully started.
+                time.sleep(1)
+                with open('bar.py', 'w') as f:
+                    f.write(textwrap.dedent(bar_2))
+            threading.Thread(target=update_source).start()
+            stdout, stderr = proc.communicate(str.encode(commands))
+        stdout = stdout and bytes.decode(stdout)
+        stderr = stderr and bytes.decode(stderr)
+        stdout = normalize(normalize(stdout, filename), 'bar.py')
+        expected = normalize(expected, 'bar.py')
+        self.assertTrue(stdout in expected,
+            '\n\nExpected:\n{}\nGot:\n{}\n'
+            'Fail to restart a new session after the source of an imported'
+            ' module has been modified.'.format(expected, stdout))
+
+    def test_set_breakpoint_by_function_name(self):
+        script = """
+            class C:
+                c_foo = 1
+
+            class D:
+                def d_foo(self):
+                    pass
+
+            def foo():
+                pass
+
+            not_a_function = 1
+            foo()
+        """
+        commands = """
+            break C
+            break C.c_foo
+            break D.d_foo
+            break foo
+            break not_a_function
+            break len
+            break logging.handlers.SocketHandler.close
+            continue
+            break C
+            break C.c_foo
+            break D.d_foo
+            break foo
+            break not_a_function
+            quit
+        """
+        expected = """
+            > main.py(2)<module>()
+            -> class C:
+            *** Bad name: "C".
+            *** Bad name: "C.c_foo".
+            Breakpoint 1 at main.py:7
+            Breakpoint 2 at main.py:10
+            *** Bad name: "not_a_function".
+            *** Cannot set a breakpoint at "len"
+            Breakpoint 3 at handlers.py:<LINE_NUMBER>
+            > main.py(10)foo()
+            -> pass
+            *** Bad function name: "C".
+            *** Cannot set a breakpoint at "C.c_foo"
+            Breakpoint 4 at main.py:7
+            Breakpoint 5 at main.py:10
+            *** Cannot set a breakpoint at "not_a_function"
+        """
+        filename = 'main.py'
+        stdout, stderr = self.run_pdb(script, commands, filename)
+        stdout = normalize(normalize(stdout, 'handlers.py', strip_bp_lnum=True), filename)
+        expected = normalize(expected, 'handlers.py', strip_bp_lnum=True)
+        self.assertTrue(stdout in expected,
+            '\n\nExpected:\n{}\nGot:\n{}\n'
+            'Fail to handle a breakpoint set by function name.'.format(expected, stdout))
+
+    def test_breakpoint_set_by_line_and_funcname(self):
+        script = """
+            def foo():
+                pass
+
+            foo()
+        """
+        commands = """
+            break foo
+            break 2
+            continue
+            break
+            quit
+        """
+        expected = """
+            > main.py(2)<module>()
+            -> def foo():
+            Breakpoint 1 at main.py:3
+            Breakpoint 2 at main.py:2
+            > main.py(3)foo()
+            -> pass
+            Num Type Disp Enb Where
+            1 breakpoint keep yes at main.py:3
+            breakpoint already hit 1 time
+            2 breakpoint keep yes at main.py:2
+            breakpoint already hit 1 time
+        """
+        filename = 'main.py'
+        stdout, stderr = self.run_pdb(script, commands, filename)
+        stdout = normalize(stdout, filename)
+        expected = normalize(expected)
+        self.assertTrue(stdout in expected,
+            '\n\nExpected:\n{}\nGot:\n{}\n'
+            'Failed hits for breakpoints set by line number and by function'
+            ' name.'.format(expected, stdout))
+
     def tearDown(self):
         support.unlink(support.TESTFN)
 
 
 def test_main():
     from test import test_pdb
     support.run_doctest(test_pdb, verbosity=True)
     support.run_unittest(PdbTestCase)
 
 
 if __name__ == '__main__':
     test_main()