diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -1,11 +1,104 @@ #! /usr/bin/env python3 """ -Usage: +Script to run Python regression tests. + +Run this script with -h or --help for documentation. + +""" + +# We import importlib *ASAP* in order to test #15386 +import importlib + +import argparse +import builtins +import faulthandler +import getopt +import io +import json +import logging +import os +import platform +import random +import re +import shutil +import signal +import sys +import sysconfig +import tempfile +import time +import traceback +import unittest +import warnings +from inspect import isabstract + +try: + import threading +except ImportError: + threading = None +try: + import multiprocessing.process +except ImportError: + multiprocessing = None + + +# Some times __path__ and __file__ are not absolute (e.g. while running from +# Lib/) and, if we change the CWD to run the tests in a temporary dir, some +# imports might fail. This affects only the modules imported before os.chdir(). +# These modules are searched first in sys.path[0] (so '' -- the CWD) and if +# they are found in the CWD their __file__ and __path__ will be relative (this +# happens before the chdir). All the modules imported after the chdir, are +# not found in the CWD, and since the other paths in sys.path[1:] are absolute +# (site.py absolutize them), the __file__ and __path__ will be absolute too. +# Therefore it is necessary to absolutize manually the __file__ and __path__ of +# the packages to prevent later imports to fail when the CWD is different. +for module in sys.modules.values(): + if hasattr(module, '__path__'): + module.__path__ = [os.path.abspath(path) for path in module.__path__] + if hasattr(module, '__file__'): + module.__file__ = os.path.abspath(module.__file__) + + +# MacOSX (a.k.a. Darwin) has a default stack size that is too small +# for deeply recursive regular expressions. We see this as crashes in +# the Python test suite when running test_re.py and test_sre.py. The +# fix is to set the stack limit to 2048. +# This approach may also be useful for other Unixy platforms that +# suffer from small default stack limits. +if sys.platform == 'darwin': + try: + import resource + except ImportError: + pass + else: + soft, hard = resource.getrlimit(resource.RLIMIT_STACK) + newsoft = min(hard, max(soft, 1024*2048)) + resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) + +# Test result constants. +PASSED = 1 +FAILED = 0 +ENV_CHANGED = -1 +SKIPPED = -2 +RESOURCE_DENIED = -3 +INTERRUPTED = -4 +CHILD_ERROR = -5 # error in a child process + +from test import support + +RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', + 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') + +TEMPDIR = os.path.abspath(tempfile.gettempdir()) + +USAGE = """ python -m test [options] [test_name1 [test_name2 ...]] python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] +""" +DESCRIPTION = """\ +Run Python regression tests. If no arguments or options are provided, finds all files matching the pattern "test_*" in the Lib/test subdirectory and runs @@ -15,63 +108,10 @@ command line: python -E -Wd -m test [options] [test_name1 ...] +""" - -Options: - --h/--help -- print this text and exit ---timeout TIMEOUT - -- dump the traceback and exit if a test takes more - than TIMEOUT seconds; disabled if TIMEOUT is negative - or equals to zero ---wait -- wait for user input, e.g., allow a debugger to be attached - -Verbosity - --v/--verbose -- run tests in verbose mode with output to stdout --w/--verbose2 -- re-run failed tests in verbose mode --W/--verbose3 -- display test output on failure --d/--debug -- print traceback for failed tests --q/--quiet -- no output unless one or more tests fail --o/--slow -- print the slowest 10 tests - --header -- print header with interpreter info - -Selecting tests - --r/--random -- randomize test execution order (see below) - --randseed -- pass a random seed to reproduce a previous random run --f/--fromfile -- read names of tests to run from a file (see below) --x/--exclude -- arguments are tests to *exclude* --s/--single -- single step through a set of tests (see below) --m/--match PAT -- match test cases and methods with glob pattern PAT --G/--failfast -- fail as soon as a test fails (only with -v or -W) --u/--use RES1,RES2,... - -- specify which special resource intensive tests to run --M/--memlimit LIMIT - -- run very large memory-consuming tests - --testdir DIR - -- execute test files in the specified directory (instead - of the Python stdlib test suite) - -Special runs - --l/--findleaks -- if GC is available detect tests that leak memory --L/--runleaks -- run the leaks(1) command just before exit --R/--huntrleaks RUNCOUNTS - -- search for reference leaks (needs debug build, v. slow) --j/--multiprocess PROCESSES - -- run PROCESSES processes at once --T/--coverage -- turn on code coverage tracing using the trace module --D/--coverdir DIRECTORY - -- Directory where coverage files are put --N/--nocoverdir -- Put coverage files alongside modules --t/--threshold THRESHOLD - -- call gc.set_threshold(THRESHOLD) --n/--nowindows -- suppress error message boxes on Windows --F/--forever -- run the specified tests in a loop, until an error happens - - -Additional Option Details: +EPILOG = """\ +Additional option details: -r randomizes test execution order. You can use --randseed=int to provide a int seed value for the randomizer; this is useful for reproducing troublesome @@ -165,93 +205,170 @@ option '-uall,-gui'. """ -# We import importlib *ASAP* in order to test #15386 -import importlib +def usage(msg, sys_exit=None, file=None): + """Display a usage error message and exit.""" + if sys_exit is None: + sys_exit = sys.exit + if file is None: + file = sys.stderr + print(msg, file=file) + print("Use -h or --help for usage", file=file) + sys_exit(2) -import builtins -import faulthandler -import getopt -import io -import json -import logging -import os -import platform -import random -import re -import shutil -import signal -import sys -import sysconfig -import tempfile -import time -import traceback -import unittest -import warnings -from inspect import isabstract +def _create_parser_class(sys_exit=None, stderr=None, stdout=None): + if sys_exit is None: + sys_exit = sys.exit + if stderr is None: + stderr = sys.stderr + if stdout is None: + stdout = sys.stdout + class ArgParser(argparse.ArgumentParser): -try: - import threading -except ImportError: - threading = None -try: - import multiprocessing.process -except ImportError: - multiprocessing = None + def print_help(self, file=None): + """This overrides ArgumentParser.print_help().""" + return super().print_help(file=stdout) + def exit(self, status=0, message=None): + """This overrides ArgumentParser.exit().""" + if message is not None: + print(message, file=stderr) + sys_exit(status) -# Some times __path__ and __file__ are not absolute (e.g. while running from -# Lib/) and, if we change the CWD to run the tests in a temporary dir, some -# imports might fail. This affects only the modules imported before os.chdir(). -# These modules are searched first in sys.path[0] (so '' -- the CWD) and if -# they are found in the CWD their __file__ and __path__ will be relative (this -# happens before the chdir). All the modules imported after the chdir, are -# not found in the CWD, and since the other paths in sys.path[1:] are absolute -# (site.py absolutize them), the __file__ and __path__ will be absolute too. -# Therefore it is necessary to absolutize manually the __file__ and __path__ of -# the packages to prevent later imports to fail when the CWD is different. -for module in sys.modules.values(): - if hasattr(module, '__path__'): - module.__path__ = [os.path.abspath(path) for path in module.__path__] - if hasattr(module, '__file__'): - module.__file__ = os.path.abspath(module.__file__) + def error(self, message): + """This overrides ArgumentParser.error().""" + usage(message, sys_exit=sys_exit, file=stderr) + return ArgParser -# MacOSX (a.k.a. Darwin) has a default stack size that is too small -# for deeply recursive regular expressions. We see this as crashes in -# the Python test suite when running test_re.py and test_sre.py. The -# fix is to set the stack limit to 2048. -# This approach may also be useful for other Unixy platforms that -# suffer from small default stack limits. -if sys.platform == 'darwin': - try: - import resource - except ImportError: - pass - else: - soft, hard = resource.getrlimit(resource.RLIMIT_STACK) - newsoft = min(hard, max(soft, 1024*2048)) - resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) +def _create_parser(parser_class): + """Create an ArgumentParser instance from an ArgumentParser subclass.""" + parser = parser_class(usage=USAGE, + description=DESCRIPTION, + epilog=EPILOG, + add_help=False, + formatter_class=argparse.RawDescriptionHelpFormatter) -# Test result constants. -PASSED = 1 -FAILED = 0 -ENV_CHANGED = -1 -SKIPPED = -2 -RESOURCE_DENIED = -3 -INTERRUPTED = -4 -CHILD_ERROR = -5 # error in a child process + # Arguments with this clause added to its help are described further in + # the epilog's "Additional option details" section. + more_details = ' See the section at bottom for more details.' -from test import support + group = parser.add_argument_group('General options') + # We add help explicitly to control what argument group it renders under. + group.add_argument('-h', '--help', action='help', + help='show this help message and exit') + group.add_argument('--timeout', metavar='TIMEOUT', + help='dump the traceback and exit if a test takes ' + 'more than TIMEOUT seconds; disabled if TIMEOUT ' + 'is negative or equals to zero') + group.add_argument('--wait', action='store_true', help='wait for user ' + 'input, e.g., allow a debugger to be attached') + group.add_argument('--slaveargs', metavar='ARGS') + group.add_argument('-S', '--start', metavar='START', help='the name of ' + 'the test at which to start.' + more_details) -RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', - 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') + group = parser.add_argument_group('Verbosity') + group.add_argument('-v', '--verbose', action='store_true', + help='run tests in verbose mode with output to stdout') + group.add_argument('-w', '--verbose2', action='store_true', + help='re-run failed tests in verbose mode') + group.add_argument('-W', '--verbose3', action='store_true', + help='display test output on failure') + group.add_argument('-d', '--debug', action='store_true', + help='print traceback for failed tests') + group.add_argument('-q', '--quiet', action='store_true', + help='no output unless one or more tests fail') + group.add_argument('-o', '--slow', action='store_true', + help='print the slowest 10 tests') + group.add_argument('--header', action='store_true', + help='print header with interpreter info') -TEMPDIR = os.path.abspath(tempfile.gettempdir()) + group = parser.add_argument_group('Selecting tests') + group.add_argument('-r', '--randomize', action='store_true', + help='randomize test execution order.' + more_details) + group.add_argument('--randseed', metavar='SEED', help='pass a random seed ' + 'to reproduce a previous random run') + group.add_argument('-f', '--fromfile', metavar='FILE', help='read names ' + 'of tests to run from a file.' + more_details) + group.add_argument('-x', '--exclude', action='store_true', + help='arguments are tests to *exclude*') + group.add_argument('-s', '--single', action='store_true', help='single ' + 'step through a set of tests.' + more_details) + group.add_argument('-m', '--match', metavar='PAT', help='match test cases ' + 'and methods with glob pattern PAT') + group.add_argument('-G', '--failfast', action='store_true', help='fail as ' + 'soon as a test fails (only with -v or -W)') + group.add_argument('-u', '--use', metavar='RES1,RES2,...', help='specify ' + 'which special resource intensive tests to run.' + + more_details) + group.add_argument('-M', '--memlimit', metavar='LIMIT', help='run very ' + 'large memory-consuming tests.' + more_details) + group.add_argument('--testdir', metavar='DIR', + help='execute test files in the specified directory ' + '(instead of the Python stdlib test suite)') -def usage(msg): - print(msg, file=sys.stderr) - print("Use --help for usage", file=sys.stderr) - sys.exit(2) + group = parser.add_argument_group('Special runs') + group.add_argument('-l', '--findleaks', action='store_true', help='if GC ' + 'is available detect tests that leak memory') + group.add_argument('-L', '--runleaks', action='store_true', + help='run the leaks(1) command just before exit.' + + more_details) + group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', + help='search for reference leaks (needs debug build, ' + 'very slow).' + more_details) + group.add_argument('-j', '--multiprocess', metavar='PROCESSES', + help='run PROCESSES processes at once') + group.add_argument('-T', '--coverage', action='store_true', help='turn on ' + 'code coverage tracing using the trace module') + group.add_argument('-D', '--coverdir', metavar='DIR', + help='Directory where coverage files are put') + group.add_argument('-N', '--nocoverdir', action='store_true', + help='Put coverage files alongside modules') + group.add_argument('-t', '--threshold', metavar='THRESHOLD', + help='call gc.set_threshold(THRESHOLD)') + group.add_argument('-n', '--nowindows', action='store_true', + help='suppress error message boxes on Windows') + group.add_argument('-F', '--forever', action='store_true', + help='run the specified tests in a loop, until an ' + 'error happens') + + parser.add_argument('args', nargs=argparse.REMAINDER, + help=argparse.SUPPRESS) + + return parser + +def _convert_namespace_to_getopt(ns): + """Convert an argparse.Namespace object to a getopt-style (opts, args).""" + opts = [] + args_dict = vars(ns) + for key in sorted(args_dict.keys()): + if key == 'args': + continue + val = args_dict[key] + # Don't "continue" if the empty string was provided as a value. + if val is None or val is False: + continue + if val is True: + val = '' + opts.append(('--' + key, val)) + return opts, ns.args + +# This function has a getopt-style return value because regrtest.main() +# was originally written using getopt. +# TODO: switch this to return an argparse.Namespace instance. +def _parse_args(args=None, parser_class=None): + """Parse arguments, and return a getopt-style (opts, args). + + This method mimics the return value of getopt.getopt(). In addition, + the (option, value) pairs in opts are sorted by option and use the long + option string. + """ + if args is None: + args = sys.argv[1:] + if parser_class is None: + parser_class = _create_parser_class() + parser = _create_parser(parser_class) + ns = parser.parse_args(args=args) + return _convert_namespace_to_getopt(ns) def main(tests=None, testdir=None, verbose=0, quiet=False, @@ -298,17 +415,8 @@ replace_stdout() support.record_original_stdout(sys.stdout) - try: - opts, args = getopt.getopt(sys.argv[1:], 'hvqxsoS:rf:lu:t:TD:NLR:FdwWM:nj:Gm:', - ['help', 'verbose', 'verbose2', 'verbose3', 'quiet', - 'exclude', 'single', 'slow', 'random', 'fromfile', 'findleaks', - 'use=', 'threshold=', 'coverdir=', 'nocoverdir', - 'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=', - 'multiprocess=', 'coverage', 'slaveargs=', 'forever', 'debug', - 'start=', 'nowindows', 'header', 'testdir=', 'timeout=', 'wait', - 'failfast', 'match']) - except getopt.error as msg: - usage(msg) + + opts, args = _parse_args() # Defaults if random_seed is None: @@ -319,10 +427,7 @@ start = None timeout = None for o, a in opts: - if o in ('-h', '--help'): - print(__doc__) - return - elif o in ('-v', '--verbose'): + if o in ('-v', '--verbose'): verbose += 1 elif o in ('-w', '--verbose2'): verbose2 = True diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py new file mode 100644 --- /dev/null +++ b/Lib/test/test_regrtest.py @@ -0,0 +1,126 @@ +""" +Tests of regrtest.py. + +""" + +from contextlib import contextmanager +import io +import sys +import unittest +from test import regrtest +from test import support + + +class TestExit(Exception): + pass + +class ParseArgsTestCase(unittest.TestCase): + + def _parse_args(self, args): + def sys_exit(data): + raise TestExit(data) + parsed, stderr, stdout = None, io.StringIO(), io.StringIO() + cls = regrtest._create_parser_class(sys_exit=sys_exit, stderr=stderr, + stdout=stdout) + exit_args = None + try: + parsed = regrtest._parse_args(args=args, parser_class=cls) + except TestExit as err: + exit_args = err.args + return parsed, stderr.getvalue(), stdout.getvalue(), exit_args + + def _assert_parse(self, args, expected): + """Check that valid args are parsed correctly.""" + parsed, stderr, stdout, exit_args = self._parse_args(args) + self.assertEqual(parsed, expected) + self.assertEqual(stderr, '') + self.assertEqual(stdout, '') + + def _assert_help(self, args): + """Check the help option.""" + parsed, stderr, stdout, exit_args = self._parse_args(args) + self.assertEqual(exit_args, (0,)) + self.assertEqual(stderr, '') + self.assertTrue(stdout.startswith('usage: ')) + + def _assert_usage_error(self, args, message): + """Check that invalid args raise and display the right error.""" + parsed, stderr, stdout, exit_args = self._parse_args(args) + self.assertEqual(exit_args, (2,)) + self.assertEqual(stderr, + '{0}\nUse -h or --help for usage\n'.format(message)) + self.assertEqual(stdout, '') + + def test_help(self): + self._assert_help(['-h']) + self._assert_help(['--help']) + + def test_unrecognized_argument(self): + self._assert_usage_error(['--xxx'], 'unrecognized arguments: --xxx') + + def test_value_not_provided(self): + self._assert_usage_error(['--start'], + 'argument -S/--start: expected one argument') + + def test_short_option(self): + args = ['-q'] + expected = ([('--quiet', '')], []) + self._assert_parse(args, expected) + + def test_long_option(self): + args = ['--quiet'] + expected = ([('--quiet', '')], []) + self._assert_parse(args, expected) + + def test_long_option__partial(self): + args = ['--qui'] + expected = ([('--quiet', '')], []) + self._assert_parse(args, expected) + + def test_two_options(self): + args = ['--quiet', '--exclude'] + expected = ([('--exclude', ''), ('--quiet', '')], []) + self._assert_parse(args, expected) + + def test_option_with_value(self): + args = ['--start', 'foo'] + expected = ([('--start', 'foo')], []) + self._assert_parse(args, expected) + + def test_option_with_empty_string_value(self): + args = ['--start', ''] + expected = ([('--start', '')], []) + self._assert_parse(args, expected) + + def test_arg(self): + args = ['foo'] + expected = ([], ['foo']) + self._assert_parse(args, expected) + + def test_option_and_arg(self): + args = ['-q', 'foo'] + expected = ([('--quiet', '')], ['foo']) + self._assert_parse(args, expected) + + def test_fromfile(self): + args = ['--fromfile', 'file'] + expected = ([('--fromfile', 'file')], []) + self._assert_parse(args, expected) + + def test_match(self): + args = ['--match', 'pattern'] + expected = ([('--match', 'pattern')], []) + self._assert_parse(args, expected) + + def test_randomize(self): + args = ['--randomize'] + expected = ([('--randomize', '')], []) + self._assert_parse(args, expected) + + +def test_main(): + tests = [ParseArgsTestCase] + support.run_unittest(*tests) + +if __name__ == '__main__': + test_main()