diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -143,6 +143,8 @@ import unittest import warnings from inspect import isabstract +import ast +import contextlib try: import threading @@ -322,6 +324,14 @@ group.add_argument('-F', '--forever', action='store_true', help='run the specified tests in a loop, until an ' 'error happens') + safe_eval = lambda x: eval(x) if x else None + range_help = ('run the {0} whose numbers are in RANGE, run and list all' + ' the {0} and their number when RANGE is an empty sequence') + group.add_argument('-X', '--test_range', metavar='RANGE', type=safe_eval, + help=range_help.format('tests')) + group.add_argument('-Y', '--subtest_range', metavar='RANGE', + type=safe_eval, + help=range_help.format('subtests')) parser.add_argument('args', nargs=argparse.REMAINDER, help=argparse.SUPPRESS) @@ -361,7 +371,7 @@ findleaks=False, use_resources=None, trace=False, coverdir='coverage', runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, random_seed=None, use_mp=None, verbose3=False, forever=False, - header=False, failfast=False, match_tests=None) + header=False, failfast=False, match_tests=None, testrange=None) for k, v in kwargs.items(): if not hasattr(ns, k): raise TypeError('%r is an invalid keyword argument ' @@ -373,6 +383,12 @@ parser = _create_parser() parser.parse_args(args=args, namespace=ns) + if ns.test_range is not None: + ns.testrange = TestRange(test_range=ns.test_range) + if ns.subtest_range is not None: + ns.testrange.subtest_range = ns.subtest_range + elif ns.subtest_range is not None: + ns.testrange = TestRange(subtest_range=ns.subtest_range) if ns.single and ns.fromfile: parser.error("-s and -f don't go together!") if ns.use_mp and ns.trace: @@ -386,6 +402,10 @@ if ns.quiet: ns.verbose = 0 + if (ns.verbose or ns.verbose2) and ns.test_range is not None: + parser.error("cannot use -X with either -v or -w!") + if (ns.verbose or ns.verbose2) and ns.subtest_range is not None: + parser.error("cannot use -Y with either -v or -w!") if ns.timeout is not None: if hasattr(faulthandler, 'dump_traceback_later'): if ns.timeout <= 0: @@ -442,7 +462,8 @@ use_resources=ns.use_resources, output_on_failure=ns.verbose3, timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests)) + match_tests=ns.match_tests, + testrange=ns.testrange)) # Running the child from the same working directory as regrtest's original # invocation ensures that TEMPDIR for the child is the same when # sysconfig.is_python_build() is true. See issue 15300. @@ -641,6 +662,7 @@ test_times = [] support.verbose = ns.verbose # Tell tests to be moderately quiet support.use_resources = ns.use_resources + support.testrange = ns.testrange save_modules = sys.modules.keys() def accumulate_result(test, result): @@ -757,7 +779,7 @@ else: try: result = runtest(test, ns.verbose, ns.quiet, - ns.huntrleaks, + ns.huntrleaks, ns.testrange, output_on_failure=ns.verbose3, timeout=ns.timeout, failfast=ns.failfast, match_tests=ns.match_tests) @@ -910,7 +932,7 @@ atexit.register(restore_stdout) def runtest(test, verbose, quiet, - huntrleaks=False, use_resources=None, + huntrleaks=False, testrange=None, use_resources=None, output_on_failure=False, failfast=False, match_tests=None, timeout=None): """Run a single test. @@ -925,6 +947,8 @@ timeout -- dump the traceback and exit if a test takes more than timeout seconds failfast, match_tests -- See regrtest command-line flags for these. + testrange -- run the tests and their subtests whose numbers are found + in this TestRange instance Returns the tuple result, test_time, where result is one of the constants: INTERRUPTED KeyboardInterrupt when run under -j @@ -964,7 +988,7 @@ sys.stdout = stream sys.stderr = stream result = runtest_inner(test, verbose, quiet, huntrleaks, - display_failure=False) + testrange, display_failure=False) if result[0] == FAILED: output = stream.getvalue() orig_stderr.write(output) @@ -975,7 +999,7 @@ else: support.verbose = verbose # Tell tests to be moderately quiet result = runtest_inner(test, verbose, quiet, huntrleaks, - display_failure=not verbose) + testrange, display_failure=not verbose) return result finally: if use_timeout: @@ -1253,9 +1277,114 @@ file=sys.stderr) return False +class TestRange: + """A range of tests and subtests.""" + + def __init__(self, test_range=None, subtest_range=None): + self.test_range = test_range + self.subtest_range = subtest_range + self.test_count = 0 + + def reset(self): + self.test_count = 0 + return self + +class TestRangeTestCase(unittest.TestCase): + """A TestCase that runs only tests found in a TestRange.""" + + def __init__(self, methodName='runTest'): + super().__init__(methodName) + self.subtest_count = 0 + + def run(self, result): + testrange = result.testrange + testrange.test_count += 1 + trange = testrange.test_range + if trange is not None: + if trange and testrange.test_count not in trange: + return + print('Test# %d: %s' % + (testrange.test_count, self._testMethodName)) + return super().run(result) + + @contextlib.contextmanager + def subTest(self, msg=None, **params): + self.subtest_count += 1 + srange = self._outcome.result.testrange.subtest_range + if srange is not None: + if srange and self.subtest_count not in srange: + self.skipTest('Sub test number not in range.') + pfx = 'subTest# %d: %s,' % (self.subtest_count, msg) + print(pfx, str(params)[:79-len(pfx)]) + yield super().subTest(msg, **params) + +class IgnoreSkipTest(ast.NodeTransformer): + """Ignore a SkipTest exception raised by a subTest context manager.""" + + def visit_With(self, node): + # Visit nested with statements. + self.generic_visit(node) + + # When this is a subTest context manager, + for item in node.items: + try: + func = item.context_expr.func + if func.value.id == 'self' and func.attr == 'subTest': + break + except AttributeError: + pass + else: + return node + + # enclose the context manager in a try/except clause. + handler = ast.copy_location( + ast.ExceptHandler( + type=ast.Attribute( + value=ast.Name(id='unittest', ctx=ast.Load()), + attr='SkipTest', ctx=ast.Load()), + name=None, + body=[ast.Pass()] + ), node) + ast.fix_missing_locations(handler) + # Preserve the assumption that line numbers increase monotonically. + first = last = node.lineno + for n in ast.walk(node): + if hasattr(n, 'lineno'): + last = max(last, n.lineno) + ast.increment_lineno(handler, n=last-first+1) + + return ast.copy_location( + ast.Try( + body=[node], + handlers=[handler], + orelse=[], + finalbody=[] + ), node) + +def import_module(name, testrange): + if testrange: + unittest.TestCase = TestRangeTestCase + the_module = importlib.import_module(name) + + if testrange and testrange.subtest_range: + import types + import linecache + + filename = the_module.__file__ + node = ast.parse(''.join(linecache.getlines(filename)), filename) + # Transform the ast. + node = IgnoreSkipTest().visit(node) + code = compile(node, filename, 'exec') + + the_module = types.ModuleType(the_module.__name__) + the_module.__file__ = filename + the_module.__dict__['unittest'] = unittest + exec(code, the_module.__dict__) + sys.modules[name] = the_module + return the_module def runtest_inner(test, verbose, quiet, - huntrleaks=False, display_failure=True): + huntrleaks=False, testrange=None, display_failure=True): support.unload(test) test_time = 0.0 @@ -1268,7 +1397,7 @@ abstest = 'test.' + test with saved_test_environment(test, verbose, quiet) as environment: start_time = time.time() - the_module = importlib.import_module(abstest) + the_module = import_module(abstest, testrange) # If the test has a test_main, that will run the appropriate # tests. If not, use normal unittest test loading. test_runner = getattr(the_module, "test_main", None) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -274,6 +274,7 @@ real_max_memuse = 0 failfast = False match_tests = None +testrange = None # _original_stdout is meant to hold stdout at the time regrtest began. # This may be "the real" stdout, or IDLE's emulation of stdout, or whatever. @@ -1644,8 +1645,13 @@ # unittest integration. class BasicTestRunner: + def __init__(self, testrange=None): + self.testrange = testrange + def run(self, test): result = unittest.TestResult() + if self.testrange: + result.testrange = self.testrange.reset() test(result) return result @@ -1745,7 +1751,7 @@ runner = unittest.TextTestRunner(sys.stdout, verbosity=2, failfast=failfast) else: - runner = BasicTestRunner() + runner = BasicTestRunner(testrange) result = runner.run(suite) if not result.wasSuccessful():