diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -1,168 +1,10 @@ #! /usr/bin/env python3 """ -Usage: +Script to run Python regression tests. -python -m test [options] [test_name1 [test_name2 ...]] -python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] +Pass -h or --help for documentation. - -If no arguments or options are provided, finds all files matching -the pattern "test_*" in the Lib/test subdirectory and runs -them in alphabetical order (but see -M and -u, below, for exceptions). - -For more rigorous testing, it is useful to use the following -command line: - -python -E -Wd -m test [options] [test_name1 ...] - - -Options: - --h/--help -- print this text and exit ---timeout TIMEOUT - -- dump the traceback and exit if a test takes more - than TIMEOUT seconds; disabled if TIMEOUT is negative - or equals to zero ---wait -- wait for user input, e.g., allow a debugger to be attached - -Verbosity - --v/--verbose -- run tests in verbose mode with output to stdout --w/--verbose2 -- re-run failed tests in verbose mode --W/--verbose3 -- display test output on failure --d/--debug -- print traceback for failed tests --q/--quiet -- no output unless one or more tests fail --o/--slow -- print the slowest 10 tests - --header -- print header with interpreter info - -Selecting tests - --r/--random -- randomize test execution order (see below) - --randseed -- pass a random seed to reproduce a previous random run --f/--fromfile -- read names of tests to run from a file (see below) --x/--exclude -- arguments are tests to *exclude* --s/--single -- single step through a set of tests (see below) --m/--match PAT -- match test cases and methods with glob pattern PAT --G/--failfast -- fail as soon as a test fails (only with -v or -W) --u/--use RES1,RES2,... - -- specify which special resource intensive tests to run --M/--memlimit LIMIT - -- run very large memory-consuming tests - --testdir DIR - -- execute test files in the specified directory (instead - of the Python stdlib test suite) - -Special runs - --l/--findleaks -- if GC is available detect tests that leak memory --L/--runleaks -- run the leaks(1) command just before exit --R/--huntrleaks RUNCOUNTS - -- search for reference leaks (needs debug build, v. slow) --j/--multiprocess PROCESSES - -- run PROCESSES processes at once --T/--coverage -- turn on code coverage tracing using the trace module --D/--coverdir DIRECTORY - -- Directory where coverage files are put --N/--nocoverdir -- Put coverage files alongside modules --t/--threshold THRESHOLD - -- call gc.set_threshold(THRESHOLD) --n/--nowindows -- suppress error message boxes on Windows --F/--forever -- run the specified tests in a loop, until an error happens - - -Additional Option Details: - --r randomizes test execution order. You can use --randseed=int to provide a -int seed value for the randomizer; this is useful for reproducing troublesome -test orders. - --s On the first invocation of regrtest using -s, the first test file found -or the first test file given on the command line is run, and the name of -the next test is recorded in a file named pynexttest. If run from the -Python build directory, pynexttest is located in the 'build' subdirectory, -otherwise it is located in tempfile.gettempdir(). On subsequent runs, -the test in pynexttest is run, and the next test is written to pynexttest. -When the last test has been run, pynexttest is deleted. In this way it -is possible to single step through the test files. This is useful when -doing memory analysis on the Python interpreter, which process tends to -consume too many resources to run the full regression test non-stop. - --S is used to continue running tests after an aborted run. It will -maintain the order a standard run (ie, this assumes -r is not used). -This is useful after the tests have prematurely stopped for some external -reason and you want to start running from where you left off rather -than starting from the beginning. - --f reads the names of tests from the file given as f's argument, one -or more test names per line. Whitespace is ignored. Blank lines and -lines beginning with '#' are ignored. This is especially useful for -whittling down failures involving interactions among tests. - --L causes the leaks(1) command to be run just before exit if it exists. -leaks(1) is available on Mac OS X and presumably on some other -FreeBSD-derived systems. - --R runs each test several times and examines sys.gettotalrefcount() to -see if the test appears to be leaking references. The argument should -be of the form stab:run:fname where 'stab' is the number of times the -test is run to let gettotalrefcount settle down, 'run' is the number -of times further it is run and 'fname' is the name of the file the -reports are written to. These parameters all have defaults (5, 4 and -"reflog.txt" respectively), and the minimal invocation is '-R :'. - --M runs tests that require an exorbitant amount of memory. These tests -typically try to ascertain containers keep working when containing more than -2 billion objects, which only works on 64-bit systems. There are also some -tests that try to exhaust the address space of the process, which only makes -sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, -which is a string in the form of '2.5Gb', determines howmuch memory the -tests will limit themselves to (but they may go slightly over.) The number -shouldn't be more memory than the machine has (including swap memory). You -should also keep in mind that swap memory is generally much, much slower -than RAM, and setting memlimit to all available RAM or higher will heavily -tax the machine. On the other hand, it is no use running these tests with a -limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect -to use more than memlimit memory will be skipped. The big-memory tests -generally run very, very long. - --u is used to specify which special resource intensive tests to run, -such as those requiring large file support or network connectivity. -The argument is a comma-separated list of words indicating the -resources to test. Currently only the following are defined: - - all - Enable all special resources. - - none - Disable all special resources (this is the default). - - audio - Tests that use the audio device. (There are known - cases of broken audio drivers that can crash Python or - even the Linux kernel.) - - curses - Tests that use curses and will modify the terminal's - state and output modes. - - largefile - It is okay to run some test that may create huge - files. These tests can take a long time and may - consume >2GB of disk space temporarily. - - network - It is okay to run tests that use external network - resource, e.g. testing SSL support for sockets. - - decimal - Test the decimal module against a large suite that - verifies compliance with standards. - - cpu - Used for certain CPU-heavy tests. - - subprocess Run all tests for the subprocess module. - - urlfetch - It is okay to download files required on testing. - - gui - Run tests that require a running GUI. - -To enable all resources except one, use '-uall,-'. For -example, to run all the tests except for the gui tests, give the -option '-uall,-gui'. """ # We import importlib *ASAP* in order to test #15386 @@ -170,7 +12,6 @@ import builtins import faulthandler -import getopt import io import json import logging @@ -188,6 +29,7 @@ import unittest import warnings from inspect import isabstract +from test import regrtestlib try: import threading @@ -248,10 +90,7 @@ TEMPDIR = os.path.abspath(tempfile.gettempdir()) -def usage(msg): - print(msg, file=sys.stderr) - print("Use --help for usage", file=sys.stderr) - sys.exit(2) +usage = regrtestlib.usage def main(tests=None, testdir=None, verbose=0, quiet=False, @@ -298,17 +137,8 @@ replace_stdout() support.record_original_stdout(sys.stdout) - try: - opts, args = getopt.getopt(sys.argv[1:], 'hvqxsoS:rf:lu:t:TD:NLR:FdwWM:nj:Gm:', - ['help', 'verbose', 'verbose2', 'verbose3', 'quiet', - 'exclude', 'single', 'slow', 'random', 'fromfile', 'findleaks', - 'use=', 'threshold=', 'coverdir=', 'nocoverdir', - 'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=', - 'multiprocess=', 'coverage', 'slaveargs=', 'forever', 'debug', - 'start=', 'nowindows', 'header', 'testdir=', 'timeout=', 'wait', - 'failfast', 'match']) - except getopt.error as msg: - usage(msg) + + opts, args = regrtestlib.parse_args() # Defaults if random_seed is None: @@ -319,10 +149,7 @@ start = None timeout = None for o, a in opts: - if o in ('-h', '--help'): - print(__doc__) - return - elif o in ('-v', '--verbose'): + if o in ('-v', '--verbose'): verbose += 1 elif o in ('-w', '--verbose2'): verbose2 = True diff --git a/Lib/test/regrtestlib.py b/Lib/test/regrtestlib.py new file mode 100644 --- /dev/null +++ b/Lib/test/regrtestlib.py @@ -0,0 +1,275 @@ +""" +Unit-tested code to support regrtest. + +""" + +import argparse +import getopt +import sys + + +USAGE = """ + +python -m test [options] [test_name1 [test_name2 ...]] +python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] +""" + +DESCRIPTION = """\ +Run Python regression tests. + +If no arguments or options are provided, finds all files matching +the pattern "test_*" in the Lib/test subdirectory and runs +them in alphabetical order (but see -M and -u, below, for exceptions). + +For more rigorous testing, it is useful to use the following +command line: + +python -E -Wd -m test [options] [test_name1 ...] +""" + +EPILOG = """\ +Additional option details: + +-r randomizes test execution order. You can use --randseed=int to provide a +int seed value for the randomizer; this is useful for reproducing troublesome +test orders. + +-s On the first invocation of regrtest using -s, the first test file found +or the first test file given on the command line is run, and the name of +the next test is recorded in a file named pynexttest. If run from the +Python build directory, pynexttest is located in the 'build' subdirectory, +otherwise it is located in tempfile.gettempdir(). On subsequent runs, +the test in pynexttest is run, and the next test is written to pynexttest. +When the last test has been run, pynexttest is deleted. In this way it +is possible to single step through the test files. This is useful when +doing memory analysis on the Python interpreter, which process tends to +consume too many resources to run the full regression test non-stop. + +-S is used to continue running tests after an aborted run. It will +maintain the order a standard run (ie, this assumes -r is not used). +This is useful after the tests have prematurely stopped for some external +reason and you want to start running from where you left off rather +than starting from the beginning. + +-f reads the names of tests from the file given as f's argument, one +or more test names per line. Whitespace is ignored. Blank lines and +lines beginning with '#' are ignored. This is especially useful for +whittling down failures involving interactions among tests. + +-L causes the leaks(1) command to be run just before exit if it exists. +leaks(1) is available on Mac OS X and presumably on some other +FreeBSD-derived systems. + +-R runs each test several times and examines sys.gettotalrefcount() to +see if the test appears to be leaking references. The argument should +be of the form stab:run:fname where 'stab' is the number of times the +test is run to let gettotalrefcount settle down, 'run' is the number +of times further it is run and 'fname' is the name of the file the +reports are written to. These parameters all have defaults (5, 4 and +"reflog.txt" respectively), and the minimal invocation is '-R :'. + +-M runs tests that require an exorbitant amount of memory. These tests +typically try to ascertain containers keep working when containing more than +2 billion objects, which only works on 64-bit systems. There are also some +tests that try to exhaust the address space of the process, which only makes +sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, +which is a string in the form of '2.5Gb', determines howmuch memory the +tests will limit themselves to (but they may go slightly over.) The number +shouldn't be more memory than the machine has (including swap memory). You +should also keep in mind that swap memory is generally much, much slower +than RAM, and setting memlimit to all available RAM or higher will heavily +tax the machine. On the other hand, it is no use running these tests with a +limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect +to use more than memlimit memory will be skipped. The big-memory tests +generally run very, very long. + +-u is used to specify which special resource intensive tests to run, +such as those requiring large file support or network connectivity. +The argument is a comma-separated list of words indicating the +resources to test. Currently only the following are defined: + + all - Enable all special resources. + + none - Disable all special resources (this is the default). + + audio - Tests that use the audio device. (There are known + cases of broken audio drivers that can crash Python or + even the Linux kernel.) + + curses - Tests that use curses and will modify the terminal's + state and output modes. + + largefile - It is okay to run some test that may create huge + files. These tests can take a long time and may + consume >2GB of disk space temporarily. + + network - It is okay to run tests that use external network + resource, e.g. testing SSL support for sockets. + + decimal - Test the decimal module against a large suite that + verifies compliance with standards. + + cpu - Used for certain CPU-heavy tests. + + subprocess Run all tests for the subprocess module. + + urlfetch - It is okay to download files required on testing. + + gui - Run tests that require a running GUI. + +To enable all resources except one, use '-uall,-'. For +example, to run all the tests except for the gui tests, give the +option '-uall,-gui'. +""" + + +def usage(msg, sys_exit=None): + if sys_exit is None: + sys_exit = sys.exit + print(msg, file=sys.stderr) + print("Use --help for usage", file=sys.stderr) + sys_exit(2) + + +# This function has a getopt-style return value because regrtest.main() +# was originally written using getopt. This is temporary until regrtest is +# fully refactored to use an argparse.Namespace return value. +def parse_args(sys_exit=None): + """Parse arguments, and return a getopt-style (opts, args). + + This method mimics the return value of getopt.getopt(). In addition, + the (option, value) pairs in opts are sorted by option and use the long + option string. + """ + if sys_exit is None: + sys_exit = sys.exit + parser = _create_parser(sys_exit) + pargs = parser.parse_args(sys.argv[1:]) + return _convert_pargs_to_getopt(pargs) + + +def _convert_pargs_to_getopt(pargs): + """Convert an argparse.Namespace instance to a getopt (opts, args).""" + args = pargs.args + opts = [] + args_dict = vars(pargs) + for key in sorted(args_dict.keys()): + if key in ('args', ): + continue + val = args_dict[key] + # Don't "continue" if the empty string was provided as a value. + if val is None or val is False: + continue + if val is True: + val = '' + opts.append(('--' + key, val)) + return opts, args + + +def _create_parser_class(sys_exit): + class ArgParser(argparse.ArgumentParser): + + def exit(self, status=0, message=None): + """This overrides ArgumentParser.exit().""" + if message is not None: + print(message, file=sys.stderr) + sys_exit(status) + + def error(self, message): + """This overrides ArgumentParser.error().""" + usage(message, sys_exit=sys_exit) + + return ArgParser + + +def _create_parser(sys_exit): + ArgParser = _create_parser_class(sys_exit) + parser = ArgParser(usage=USAGE, + description=DESCRIPTION, + epilog=EPILOG, + add_help=True, + formatter_class=argparse.RawDescriptionHelpFormatter) + + # Arguments with this clause added to its help are described further in + # the epilog's "Additional option details" section. + more_details = ' See the section at bottom for more details.' + + parser.add_argument('--timeout', metavar='TIMEOUT', + help='dump the traceback and exit if a test takes ' + 'more than TIMEOUT seconds; disabled if TIMEOUT ' + 'is negative or equals to zero') + parser.add_argument('--wait', action='store_true', help='wait for user ' + 'input, e.g., allow a debugger to be attached') + parser.add_argument('--slaveargs', metavar='ARGS') + parser.add_argument('-S', '--start', metavar='START', help='the name of ' + 'the test at which to start.' + more_details) + + group = parser.add_argument_group('Verbosity') + group.add_argument('-v', '--verbose', action='store_true', + help='run tests in verbose mode with output to stdout') + group.add_argument('-w', '--verbose2', action='store_true', + help='re-run failed tests in verbose mode') + group.add_argument('-W', '--verbose3', action='store_true', + help='display test output on failure') + group.add_argument('-d', '--debug', action='store_true', + help='print traceback for failed tests') + group.add_argument('-q', '--quiet', action='store_true', + help='no output unless one or more tests fail') + group.add_argument('-o', '--slow', action='store_true', + help='print the slowest 10 tests') + group.add_argument('--header', action='store_true', + help='print header with interpreter info') + + group = parser.add_argument_group('Selecting tests') + group.add_argument('-r', '--randomize', action='store_true', + help='randomize test execution order.' + more_details) + group.add_argument('--randseed', metavar='SEED', help='pass a random seed ' + 'to reproduce a previous random run') + group.add_argument('-f', '--fromfile', metavar='FILE', help='read names ' + 'of tests to run from a file.' + more_details) + group.add_argument('-x', '--exclude', action='store_true', + help='arguments are tests to *exclude*') + group.add_argument('-s', '--single', action='store_true', help='single ' + 'step through a set of tests.' + more_details) + group.add_argument('-m', '--match', metavar='PAT', help='match test cases ' + 'and methods with glob pattern PAT') + group.add_argument('-G', '--failfast', action='store_true', help='fail as ' + 'soon as a test fails (only with -v or -W)') + group.add_argument('-u', '--use', metavar='RES1,RES2,...', help='specify ' + 'which special resource intensive tests to run.' + + more_details) + group.add_argument('-M', '--memlimit', metavar='LIMIT', help='run very ' + 'large memory-consuming tests.' + more_details) + group.add_argument('--testdir', metavar='DIR', + help='execute test files in the specified directory ' + '(instead of the Python stdlib test suite)') + + group = parser.add_argument_group('Special runs') + group.add_argument('-l', '--findleaks', action='store_true', help='if GC ' + 'is available detect tests that leak memory') + group.add_argument('-L', '--runleaks', action='store_true', + help='run the leaks(1) command just before exit.' + + more_details) + group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', + help='search for reference leaks (needs debug build, ' + 'very slow).' + more_details) + group.add_argument('-j', '--multiprocess', metavar='PROCESSES', + help='run PROCESSES processes at once') + group.add_argument('-T', '--coverage', action='store_true', help='turn on ' + 'code coverage tracing using the trace module') + group.add_argument('-D', '--coverdir', metavar='DIR', + help='Directory where coverage files are put') + group.add_argument('-N', '--nocoverdir', action='store_true', + help='Put coverage files alongside modules') + group.add_argument('-t', '--threshold', metavar='THRESHOLD', + help='call gc.set_threshold(THRESHOLD)') + group.add_argument('-n', '--nowindows', action='store_true', + help='suppress error message boxes on Windows') + group.add_argument('-F', '--forever', action='store_true', + help='run the specified tests in a loop, until an ' + 'error happens') + + parser.add_argument('args', nargs=argparse.REMAINDER, + help=argparse.SUPPRESS) + + return parser diff --git a/Lib/test/test_regrtestlib.py b/Lib/test/test_regrtestlib.py new file mode 100644 --- /dev/null +++ b/Lib/test/test_regrtestlib.py @@ -0,0 +1,123 @@ +""" +Tests for regrtestlib.py. + +""" + +from contextlib import contextmanager +import sys +import unittest +from test import regrtestlib +from test import support + + +@contextmanager +def temp_sys_argv(argv): + """Context manager that temporarily changes sys.argv.""" + saved_argv = sys.argv + sys.argv = argv + try: + yield + finally: + sys.argv = saved_argv + + +class TestException(Exception): + pass + + +class ParseArgsTestCase(unittest.TestCase): + + def _parse_args(self, args): + argv = ['prog'] + args + def sys_exit(data): + raise TestException(data) + with temp_sys_argv(argv): + opts, args = regrtestlib.parse_args(sys_exit=sys_exit) + return opts, args + + def _assert_parse(self, args, expected): + """Check that valid args are parsed correctly.""" + with support.captured_stderr() as stream: + parsed = self._parse_args(args) + self.assertEqual(parsed, expected) + self.assertEqual(stream.getvalue(), '') + + def _assert_bad_args(self, args, message): + """Check that invalid args raise and display the right error.""" + with self.assertRaises(TestException) as cm: + with support.captured_stderr() as stream: + self._parse_args(args) + err = cm.exception + self.assertEqual(err.args, (2,)) + self.assertEqual(stream.getvalue(), + '{0}\nUse --help for usage\n'.format(message)) + + def test_unrecognized_argument(self): + self._assert_bad_args(['--xxx'], 'unrecognized arguments: --xxx') + + def test_value_not_provided(self): + self._assert_bad_args(['--start'], + 'argument -S/--start: expected one argument') + + def test_short_option(self): + args = ['-q'] + expected = ([('--quiet', '')], []) + self._assert_parse(args, expected) + + def test_long_option(self): + args = ['--quiet'] + expected = ([('--quiet', '')], []) + self._assert_parse(args, expected) + + def test_long_option__partial(self): + args = ['--qui'] + expected = ([('--quiet', '')], []) + self._assert_parse(args, expected) + + def test_two_options(self): + args = ['--quiet', '--exclude'] + expected = ([('--exclude', ''), ('--quiet', '')], []) + self._assert_parse(args, expected) + + def test_option_with_value(self): + args = ['--start', 'foo'] + expected = ([('--start', 'foo')], []) + self._assert_parse(args, expected) + + def test_option_with_empty_string_value(self): + args = ['--start', ''] + expected = ([('--start', '')], []) + self._assert_parse(args, expected) + + def test_arg(self): + args = ['foo'] + expected = ([], ['foo']) + self._assert_parse(args, expected) + + def test_option_and_arg(self): + args = ['-q', 'foo'] + expected = ([('--quiet', '')], ['foo']) + self._assert_parse(args, expected) + + def test_fromfile(self): + args = ['--fromfile', 'file'] + expected = ([('--fromfile', 'file')], []) + self._assert_parse(args, expected) + + def test_match(self): + args = ['--match', 'pattern'] + expected = ([('--match', 'pattern')], []) + self._assert_parse(args, expected) + + def test_randomize(self): + args = ['--randomize'] + expected = ([('--randomize', '')], []) + self._assert_parse(args, expected) + + +def test_main(): + tests = [ParseArgsTestCase] + support.run_unittest(*tests) + +if __name__ == '__main__': + test_main()