diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -70,6 +70,33 @@ -n/--nowindows -- suppress error message boxes on Windows -F/--forever -- run the specified tests in a loop, until an error happens +Running doctests + + The script can find and run doctests in both modules (module doctests) and + documentation files (file doctests). + + For doctest discovery, the script attempts to find all doctests in the + standard library and, if --docdir is specified, all doctests in the + documentation directory. + + The script assigns module doctests names of the form "mdoc:" + (e.g. "mdoc:ipaddress"), and it assigns file doctests names of the form + "fdoc:", where is the path to the file relative to + the documentation directory (e.g. "fdoc:howto/ipaddress.rst"). + + The doctest-specific options are as follows: + + --docdir DIR + -- the path to the documentation directory (i.e. the directory + containing *.rst files). Defaults to None, which means no + documentation directory. + --docall -- run all doctests. With this option, the script runs all + discovered doctests (subject to the --docunsafe option), + in addition to any tests that would run in the absence + of the --docall option. + --docunsafe -- run the doctests selected, whether or not they have been + marked "doctest safe". Otherwise, the script runs only + those docttests marked as doctest safe. Additional Option Details: @@ -169,7 +196,10 @@ import importlib import builtins +import collections +import doctest import faulthandler +import fnmatch import getopt import io import json @@ -248,6 +278,15 @@ TEMPDIR = os.path.abspath(tempfile.gettempdir()) + +class DocTestOptions(object): + """Encapsulates doctest-specific options.""" + def __init__(self): + self.docall = False + self.docdir = None + self.docsafe = True + + def usage(msg): print(msg, file=sys.stderr) print("Use --help for usage", file=sys.stderr) @@ -306,7 +345,7 @@ 'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=', 'multiprocess=', 'coverage', 'slaveargs=', 'forever', 'debug', 'start=', 'nowindows', 'header', 'testdir=', 'timeout=', 'wait', - 'failfast', 'match']) + 'failfast', 'match', 'docdir=', 'docall', 'docunsafe']) except getopt.error as msg: usage(msg) @@ -318,6 +357,7 @@ debug = False start = None timeout = None + doc_options = DocTestOptions() for o, a in opts: if o in ('-h', '--help'): print(__doc__) @@ -449,6 +489,13 @@ print() # Force a newline (just in case) print(json.dumps(result)) sys.exit(0) + elif o == '--docall': + doc_options.docall = True + elif o == '--docdir': + # Interpret the directory relative to the original CWD. + doc_options.docdir = os.path.join(support.SAVEDCWD, a) + elif o == '--docunsafe': + doc_options.docsafe = False elif o == '--testdir': # CWD is replaced with a temporary dir before calling main(), so we # join it with the saved CWD so it ends up where the user expects. @@ -542,14 +589,44 @@ print("== ", os.getcwd()) print("Testing with flags:", sys.flags) - # if testdir is set, then we are not running the python tests suite, so - # don't add default tests to be executed or skipped (pass empty values) + all_doctests = find_doctests(docdir=doc_options.docdir) + if testdir: + # If testdir is set, then we are not running the python test suite, + # so don't add default tests to be executed or skipped (pass empty + # values). alltests = findtests(testdir, list(), set()) else: alltests = findtests(testdir, stdtests, nottests) - selected = tests or args or alltests + # User-provided test strings. + tests = tests or args + + if tests: + # Then use the test strings to filter alltests. + if not doc_options.docall: + # Then we need to select/filter among doctests, too. Otherwise, + # we need to run all doctests, and we can simply add all doctests + # after filtering. + alltests += all_doctests + # Use an ordered dictionary to keep the user-provided ordering. + selected = collections.OrderedDict() + for test_pattern in tests: + test_matches = fnmatch.filter(alltests, test_pattern) + for test_match in test_matches: + selected[test_match] = True + selected = list(selected.keys()) + else: + selected = alltests + + if doc_options.docall: + selected.extend(all_doctests) + + # Keep the user-provided test ordering. + if not tests: + selected.sort() + + # TODO: make sure single behaves the same as before. if single: selected = selected[:1] try: @@ -713,7 +790,8 @@ result = runtest(test, verbose, quiet, huntrleaks, debug, output_on_failure=verbose3, timeout=timeout, failfast=failfast, - match_tests=match_tests) + match_tests=match_tests, + doc_options=doc_options) accumulate_result(test, result) except KeyboardInterrupt: interrupted = True @@ -784,7 +862,8 @@ sys.stdout.flush() try: verbose = True - ok = runtest(test, True, quiet, huntrleaks, debug, timeout=timeout) + ok = runtest(test, True, quiet, huntrleaks, debug, + timeout=timeout, doc_options=doc_options) except KeyboardInterrupt: # print a newline separate from the ^C print() @@ -827,6 +906,96 @@ # set of tests that we don't want to be executed when using regrtest NOTTESTS = set() +def path_split(path): + """Split the path into parts using os.path.split().""" + parts = [] + while True: + path, tail = os.path.split(path) + parts.append(tail) + if not path: + break + parts.reverse() + return parts + +def find_files(top_dir, include_file=None, include_dir=None): + """Return the relative paths to files inside top_dir.""" + # Include all files and directories by default. + if include_file is None: + include_file = lambda path: True + if include_dir is None: + include_dir = lambda path: True + + def get_paths(rel_dir, names, include): + paths = [] + for name in names: + # Call normpath() to collapse the leading "./", for example. + path = os.path.normpath(os.path.join(rel_dir, name)) + if not include(path): + continue + paths.append(path) + return paths + + paths = [] + if top_dir is None: + return paths + for (dirpath, dirnames, filenames) in os.walk(top_dir): + rel_dir = os.path.relpath(dirpath, top_dir) + # For example, to collapse the leading "./". + rel_dir = os.path.normpath(rel_dir) + if not include_dir(rel_dir): + continue + paths.extend(get_paths(rel_dir, filenames, include_file)) + paths.sort() + return paths + +def find_doc_paths(docdir=None): + """Return a list of relative paths to all documentation files.""" + include_file = lambda path: path.endswith(".rst") + return find_files(docdir, include_file=include_file) + +def find_nontest_mod_paths(): + """Return a list of relative paths to non-test Library modules.""" + def include_dir(path): + parts = path_split(path) + if "test" in parts or "tests" in parts: + return False + basename = os.path.basename(path) + if basename == "__pycache__": + return False + return True + + def include_file(path): + filename = os.path.basename(path) + if filename == "__main__.py": + return False + mod, ext = os.path.splitext(filename) + return ext == ".py" + + libdir = findlibdir() + return find_files(libdir, include_file=include_file, + include_dir=include_dir) + +def find_doctests(docdir=None): + """Return a list of test names for all doctests.""" + tests = [] + if docdir is not None: + doc_paths = find_doc_paths(docdir=docdir) + # Add file doctests. + tests.extend(["fdoc:" + path for path in doc_paths]) + + mod_paths = find_nontest_mod_paths() + for path in mod_paths: + mod_base, ext = os.path.splitext(path) + parts = path_split(mod_base) + if parts[-1] == "__init__": + parts.pop() + mod_name = ".".join(parts) + # Add a module doctest. + tests.append("mdoc:" + mod_name) + + tests.sort() + return tests + def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): """Return a list of all applicable test modules.""" testdir = findtestdir(testdir) @@ -878,7 +1047,7 @@ def runtest(test, verbose, quiet, huntrleaks=False, debug=False, use_resources=None, output_on_failure=False, failfast=False, match_tests=None, - timeout=None): + timeout=None, doc_options=None): """Run a single test. test -- the name of the test @@ -890,6 +1059,7 @@ output_on_failure -- if true, display test output on failure timeout -- dump the traceback and exit if a test takes more than timeout seconds + doc_options -- a DocTestOptions instance. Returns one of the test result constants: INTERRUPTED KeyboardInterrupt when run under -j @@ -929,7 +1099,8 @@ sys.stdout = stream sys.stderr = stream result = runtest_inner(test, verbose, quiet, huntrleaks, - debug, display_failure=False) + debug, display_failure=False, + doc_options=doc_options) if result[0] == FAILED: output = stream.getvalue() orig_stderr.write(output) @@ -940,7 +1111,8 @@ else: support.verbose = verbose # Tell tests to be moderately quiet result = runtest_inner(test, verbose, quiet, huntrleaks, debug, - display_failure=not verbose) + display_failure=not verbose, + doc_options=doc_options) return result finally: if use_timeout: @@ -1195,33 +1367,95 @@ file=sys.stderr) return False +def make_test_runner(test_suite, warn_no_tests=True): + return lambda: support.run_unittest(test_suite, + warn_no_tests=warn_no_tests) + +def load_file_doc_test_runner(path, docdir=None): + """Return a test runner for doctests in the given doc file. + + """ + if docdir is not None: + path = os.path.join(docdir, path) + try: + suite = doctest.DocFileSuite(path, module_relative=False) + except FileNotFoundError: + raise + except Exception as err: + raise err.__class__("Error creating DocFileSuite from: %s" % + repr(path)) from err + return make_test_runner(suite) + +def load_module_doc_test_runner(mod_name, doc_options=None): + """Return a test runner for doctests in the given module. + + """ + if doc_options is None: + doc_options = DocTestOptions() + mod = __import__(mod_name, globals(), locals(), []) + suite = unittest.TestSuite() + # Add tests only if it is safe to do so. + if not doc_options.docsafe or getattr(mod, "_doctest_safe", False): + # We pass a test_finder to prevent a ValueError from being thrown + # in case the module has no docstrings. See also issue #14649. + finder = doctest.DocTestFinder(exclude_empty=False) + suite = doctest.DocTestSuite(mod, test_finder=finder) + # Only warn if the doctest was explicitly provided. + warn_no_tests = not doc_options.docall + return make_test_runner(suite, warn_no_tests=warn_no_tests) + +def load_test_module_test_runner(test): + """Return a callable test runner for the given test. + + The argument *test* is the name of the test module. + + """ + if test.startswith('test.'): + abstest = test + else: + # Always import it from the test package + abstest = 'test.' + test + the_package = __import__(abstest, globals(), locals(), []) + the_module = getattr(the_package, test) + # If the test has a test_main, that will run the appropriate + # tests. If not, use normal unittest test loading. + test_runner = getattr(the_module, "test_main", None) + if test_runner is None: + tests = unittest.TestLoader().loadTestsFromModule(the_module) + test_runner = make_test_runner(tests) + + return test_runner + +def load_test_runner(test, doc_options=None): + """Return a callable test runner for the given test. + + The argument *test* is the name of the test module. + + """ + fprefix, mprefix = "fdoc:", "mdoc:" + if test.startswith(fprefix): + runner = load_file_doc_test_runner(test[len(fprefix):], docdir=doc_options.docdir) + elif test.startswith(mprefix): + runner = load_module_doc_test_runner(test[len(mprefix):], + doc_options=doc_options) + else: + runner = load_test_module_test_runner(test) + return runner def runtest_inner(test, verbose, quiet, - huntrleaks=False, debug=False, display_failure=True): + huntrleaks=False, debug=False, display_failure=True, + doc_options=None): support.unload(test) test_time = 0.0 refleak = False # True if the test leaked references. try: - if test.startswith('test.'): - abstest = test - else: - # Always import it from the test package - abstest = 'test.' + test with saved_test_environment(test, verbose, quiet) as environment: start_time = time.time() - the_package = __import__(abstest, globals(), locals(), []) - the_module = getattr(the_package, test) - # If the test has a test_main, that will run the appropriate - # tests. If not, use normal unittest test loading. - test_runner = getattr(the_module, "test_main", None) - if test_runner is None: - tests = unittest.TestLoader().loadTestsFromModule(the_module) - test_runner = lambda: support.run_unittest(tests) + test_runner = load_test_runner(test, doc_options=doc_options) test_runner() if huntrleaks: - refleak = dash_R(the_module, test, test_runner, - huntrleaks) + refleak = dash_R(None, test, test_runner, huntrleaks) test_time = time.time() - start_time except support.ResourceDenied as msg: if not quiet: @@ -1440,6 +1674,11 @@ # int cache x = list(range(-5, 257)) +def findlibdir(): + """Return the path to the source Lib directory.""" + # We can choose any Python module in the standard library. + return os.path.dirname(sysconfig.__file__) + def findtestdir(path=None): return path or os.path.dirname(__file__) or os.curdir diff --git a/Lib/test/support.py b/Lib/test/support.py --- a/Lib/test/support.py +++ b/Lib/test/support.py @@ -1486,7 +1486,7 @@ raise TestFailed(err) -def run_unittest(*classes): +def run_unittest(*classes, warn_no_tests=True): """Run tests from unittest.TestCase-derived classes.""" valid_types = (unittest.TestSuite, unittest.TestCase) suite = unittest.TestSuite() @@ -1508,6 +1508,8 @@ return True return False _filter_suite(suite, case_pred) + if warn_no_tests and suite.countTestCases() == 0: + warnings.warn("TestSuite has no tests") _run_suite(suite)