diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -62,6 +62,33 @@ -n/--nowindows -- suppress error message boxes on Windows -F/--forever -- run the specified tests in a loop, until an error happens +Running doctests + + The script can find and run doctests in both modules (module doctests) and + documentation files (file doctests). + + For doctest discovery, the script attempts to find all doctests in the + standard library and, if --docdir is specified, all doctests in the + documentation directory. + + The script assigns module doctests names of the form "mdoc:" + (e.g. "mdoc:ipaddress"), and it assigns file doctests names of the form + "fdoc:", where is the path to the file relative to + the documentation directory (e.g. "fdoc:howto/ipaddress.rst"). + + The doctest-specific options are as follows: + + --docdir DIR + -- the path to the documentation directory (i.e. the directory + containing *.rst files). Defaults to None, which means no + documentation directory. + --docall -- run all doctests. With this option, the script runs all + discovered doctests (subject to the --docunsafe option), + in addition to any tests that would run in the absence + of the --docall option. + --docunsafe -- run the doctests selected, whether or not they have been + marked "doctest safe". Otherwise, the script runs only + those docttests marked as doctest safe. Additional Option Details: @@ -156,7 +183,10 @@ """ import builtins +import collections +import doctest import errno +import fnmatch import getopt import io import json @@ -233,6 +263,15 @@ TEMPDIR = os.path.abspath(tempfile.gettempdir()) + +class DocTestOptions(object): + """Encapsulates doctest-specific options.""" + def __init__(self): + self.docall = False + self.docdir = None + self.docsafe = True + + def usage(msg): print(msg, file=sys.stderr) print("Use --help for usage", file=sys.stderr) @@ -278,7 +317,8 @@ 'use=', 'threshold=', 'coverdir=', 'nocoverdir', 'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=', 'multiprocess=', 'coverage', 'slaveargs=', 'forever', 'debug', - 'start=', 'nowindows', 'header', 'failfast', 'match']) + 'start=', 'nowindows', 'header', 'failfast', 'match', + 'docdir=', 'docall', 'docunsafe']) except getopt.error as msg: usage(msg) @@ -289,6 +329,7 @@ use_resources = [] debug = False start = None + doc_options = DocTestOptions() for o, a in opts: if o in ('-h', '--help'): print(__doc__) @@ -403,6 +444,13 @@ print() # Force a newline (just in case) print(json.dumps(result)) sys.exit(0) + elif o == '--docall': + doc_options.docall = True + elif o == '--docdir': + # Interpret the directory relative to the original CWD. + doc_options.docdir = os.path.join(support.SAVEDCWD, a) + elif o == '--docunsafe': + doc_options.docsafe = False else: print(("No handler for option {}. Please report this as a bug " "at http://bugs.python.org.").format(o), file=sys.stderr) @@ -481,8 +529,37 @@ print("== ", os.getcwd()) print("Testing with flags:", sys.flags) + all_doctests = find_doctests(docdir=doc_options.docdir) alltests = findtests(testdir, stdtests, nottests) - selected = tests or args or alltests + + # User-provided test strings. + tests = tests or args + + if tests: + # Then use the test strings to filter alltests. + if not doc_options.docall: + # Then we need to select/filter among doctests, too. Otherwise, + # we need to run all doctests, and we can simply add all doctests + # after filtering. + alltests += all_doctests + # Use an ordered dictionary to keep the user-provided ordering. + selected = collections.OrderedDict() + for test_pattern in tests: + test_matches = fnmatch.filter(alltests, test_pattern) + for test_match in test_matches: + selected[test_match] = True + selected = list(selected.keys()) + else: + selected = alltests + + if doc_options.docall: + selected.extend(all_doctests) + + # Keep the user-provided test ordering. + if not tests: + selected.sort() + + # TODO: make sure single behaves the same as before. if single: selected = selected[:1] try: @@ -638,7 +715,8 @@ try: result = runtest(test, verbose, quiet, huntrleaks, debug, output_on_failure=verbose3, - failfast=failfast, match_tests=match_tests) + failfast=failfast, match_tests=match_tests, + doc_options=doc_options) accumulate_result(test, result) except KeyboardInterrupt: interrupted = True @@ -709,7 +787,8 @@ sys.stdout.flush() try: verbose = True - ok = runtest(test, True, quiet, huntrleaks, debug) + ok = runtest(test, True, quiet, huntrleaks, debug, + doc_options=doc_options) except KeyboardInterrupt: # print a newline separate from the ^C print() @@ -751,6 +830,96 @@ 'test_future2', } +def path_split(path): + """Split the path into parts using os.path.split().""" + parts = [] + while True: + path, tail = os.path.split(path) + parts.append(tail) + if not path: + break + parts.reverse() + return parts + +def find_files(top_dir, include_file=None, include_dir=None): + """Return the relative paths to files inside top_dir.""" + # Include all files and directories by default. + if include_file is None: + include_file = lambda path: True + if include_dir is None: + include_dir = lambda path: True + + def get_paths(rel_dir, names, include): + paths = [] + for name in names: + # Call normpath() to collapse the leading "./", for example. + path = os.path.normpath(os.path.join(rel_dir, name)) + if not include(path): + continue + paths.append(path) + return paths + + paths = [] + if top_dir is None: + return paths + for (dirpath, dirnames, filenames) in os.walk(top_dir): + rel_dir = os.path.relpath(dirpath, top_dir) + # For example, to collapse the leading "./". + rel_dir = os.path.normpath(rel_dir) + if not include_dir(rel_dir): + continue + paths.extend(get_paths(rel_dir, filenames, include_file)) + paths.sort() + return paths + +def find_doc_paths(docdir=None): + """Return a list of relative paths to all documentation files.""" + include_file = lambda path: path.endswith(".rst") + return find_files(docdir, include_file=include_file) + +def find_nontest_mod_paths(): + """Return a list of relative paths to non-test Library modules.""" + def include_dir(path): + parts = path_split(path) + if "test" in parts or "tests" in parts: + return False + basename = os.path.basename(path) + if basename == "__pycache__": + return False + return True + + def include_file(path): + filename = os.path.basename(path) + if filename == "__main__.py": + return False + mod, ext = os.path.splitext(filename) + return ext == ".py" + + libdir = findlibdir() + return find_files(libdir, include_file=include_file, + include_dir=include_dir) + +def find_doctests(docdir=None): + """Return a list of test names for all doctests.""" + tests = [] + if docdir is not None: + doc_paths = find_doc_paths(docdir=docdir) + # Add file doctests. + tests.extend(["fdoc:" + path for path in doc_paths]) + + mod_paths = find_nontest_mod_paths() + for path in mod_paths: + mod_base, ext = os.path.splitext(path) + parts = path_split(mod_base) + if parts[-1] == "__init__": + parts.pop() + mod_name = ".".join(parts) + # Add a module doctest. + tests.append("mdoc:" + mod_name) + + tests.sort() + return tests + def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): """Return a list of all applicable test modules.""" testdir = findtestdir(testdir) @@ -804,7 +973,8 @@ def runtest(test, verbose, quiet, huntrleaks=False, debug=False, use_resources=None, - output_on_failure=False, failfast=False, match_tests=None): + output_on_failure=False, failfast=False, match_tests=None, + doc_options=None): """Run a single test. test -- the name of the test @@ -814,6 +984,7 @@ huntrleaks -- run multiple times to test for leaks; requires a debug build; a triple corresponding to -R's three arguments output_on_failure -- if true, display test output on failure + doc_options -- a DocTestOptions instance. Returns one of the test result constants: INTERRUPTED KeyboardInterrupt when run under -j @@ -850,7 +1021,8 @@ sys.stdout = stream sys.stderr = stream result = runtest_inner(test, verbose, quiet, huntrleaks, - debug, display_failure=False) + debug, display_failure=False, + doc_options=doc_options) if result[0] == FAILED: output = stream.getvalue() orig_stderr.write(output) @@ -861,7 +1033,8 @@ else: support.verbose = verbose # Tell tests to be moderately quiet result = runtest_inner(test, verbose, quiet, huntrleaks, debug, - display_failure=not verbose) + display_failure=not verbose, + doc_options=doc_options) return result finally: cleanup_test_droppings(test, verbose) @@ -1091,32 +1264,95 @@ file=sys.stderr) return False +def make_test_runner(test_suite, warn_no_tests=True): + return lambda: support.run_unittest(test_suite, + warn_no_tests=warn_no_tests) + +def load_file_doc_test_runner(path, docdir=None): + """Return a test runner for doctests in the given doc file. + + """ + if docdir is not None: + path = os.path.join(docdir, path) + try: + suite = doctest.DocFileSuite(path, module_relative=False) + except FileNotFoundError: + raise + except Exception as err: + raise err.__class__("Error creating DocFileSuite from: %s" % + repr(path)) from err + return make_test_runner(suite) + +def load_module_doc_test_runner(mod_name, doc_options=None): + """Return a test runner for doctests in the given module. + + """ + if doc_options is None: + doc_options = DocTestOptions() + mod = __import__(mod_name, globals(), locals(), []) + suite = unittest.TestSuite() + # Add tests only if it is safe to do so. + if not doc_options.docsafe or getattr(mod, "_doctest_safe", False): + # We pass a test_finder to prevent a ValueError from being thrown + # in case the module has no docstrings. See also issue #14649. + finder = doctest.DocTestFinder(exclude_empty=False) + suite = doctest.DocTestSuite(mod, test_finder=finder) + # Only warn if the doctest was explicitly provided. + warn_no_tests = not doc_options.docall + return make_test_runner(suite, warn_no_tests=warn_no_tests) + +def load_test_module_test_runner(test): + """Return a callable test runner for the given test. + + The argument *test* is the name of the test module. + + """ + if test.startswith('test.'): + abstest = test + else: + # Always import it from the test package + abstest = 'test.' + test + the_package = __import__(abstest, globals(), locals(), []) + the_module = getattr(the_package, test) + # If the test has a test_main, that will run the appropriate + # tests. If not, use normal unittest test loading. + test_runner = getattr(the_module, "test_main", None) + if test_runner is None: + tests = unittest.TestLoader().loadTestsFromModule(the_module) + test_runner = make_test_runner(tests) + + return test_runner + +def load_test_runner(test, doc_options=None): + """Return a callable test runner for the given test. + + The argument *test* is the name of the test module. + + """ + fprefix, mprefix = "fdoc:", "mdoc:" + if test.startswith(fprefix): + runner = load_file_doc_test_runner(test[len(fprefix):], docdir=doc_options.docdir) + elif test.startswith(mprefix): + runner = load_module_doc_test_runner(test[len(mprefix):], + doc_options=doc_options) + else: + runner = load_test_module_test_runner(test) + return runner def runtest_inner(test, verbose, quiet, - huntrleaks=False, debug=False, display_failure=True): + huntrleaks=False, debug=False, display_failure=True, + doc_options=None): support.unload(test) test_time = 0.0 refleak = False # True if the test leaked references. try: - if test.startswith('test.'): - abstest = test - else: - # Always import it from the test package - abstest = 'test.' + test with saved_test_environment(test, verbose, quiet) as environment: start_time = time.time() - the_package = __import__(abstest, globals(), locals(), []) - the_module = getattr(the_package, test) - # Old tests run to completion simply as a side-effect of - # being imported. For tests based on unittest or doctest, - # explicitly invoke their test_main() function (if it exists). - indirect_test = getattr(the_module, "test_main", None) - if indirect_test is not None: - indirect_test() + test_runner = load_test_runner(test, doc_options=doc_options) + test_runner() if huntrleaks: - refleak = dash_R(the_module, test, indirect_test, - huntrleaks) + refleak = dash_R(None, test, test_runner, huntrleaks) test_time = time.time() - start_time except support.ResourceDenied as msg: if not quiet: @@ -1329,6 +1565,11 @@ for i in range(256): s[i:i+1] +def findlibdir(): + """Return the path to the source Lib directory.""" + # We can choose any Python module in the standard library. + return os.path.dirname(sysconfig.__file__) + def findtestdir(path=None): return path or os.path.dirname(__file__) or os.curdir diff --git a/Lib/test/support.py b/Lib/test/support.py --- a/Lib/test/support.py +++ b/Lib/test/support.py @@ -1375,7 +1375,7 @@ raise TestFailed(err) -def run_unittest(*classes): +def run_unittest(*classes, warn_no_tests=True): """Run tests from unittest.TestCase-derived classes.""" valid_types = (unittest.TestSuite, unittest.TestCase) suite = unittest.TestSuite() @@ -1397,6 +1397,8 @@ return True return False _filter_suite(suite, case_pred) + if warn_no_tests and suite.countTestCases() == 0: + warnings.warn("TestSuite has no tests") _run_suite(suite)