diff -r 4d9b31b2a360 Lib/test/test_regrtest.py --- a/Lib/test/test_regrtest.py Mon Sep 28 06:30:43 2015 +0000 +++ b/Lib/test/test_regrtest.py Mon Sep 28 09:29:29 2015 +0200 @@ -6,13 +6,23 @@ import argparse import faulthandler import getopt import os.path +import re +import sys +import textwrap import unittest from test import libregrtest from test import support +from test.support import script_helper + + +ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') +ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) + class ParseArgsTestCase(unittest.TestCase): - - """Test regrtest's argument parsing.""" + """ + Test regrtest's argument parsing, function _parse_args(). + """ def checkError(self, args, msg): with support.captured_stderr() as err, self.assertRaises(SystemExit): @@ -272,5 +282,230 @@ class ParseArgsTestCase(unittest.TestCas self.assertEqual(ns.args, ['foo']) +class BaseTestCase(unittest.TestCase): + TESTCASE_TEMPLATE = textwrap.dedent(""" + import unittest + + {module} + + + class NoopTestCase(unittest.TestCase): + def test_noop(self): + pass + + + if __name__ == '__main__': + unittest.main() + """) + + def create_test(self, name, module=''): + name = 'test_regrtest_%s' % name + content = self.TESTCASE_TEMPLATE.format(module=module) + + testdir = os.path.join(ROOT_DIR, 'Lib', 'test') + path = os.path.join(testdir, name + '.py') + + self.addCleanup(support.unlink, path) + # Use 'x' mode to ensure that we do not override existing tests + with open(path, 'x', encoding='utf-8') as fp: + fp.write(content) + return name + + def regex_search(self, regex, output): + match = re.search(regex, output, re.MULTILINE) + if not match: + self.fail("%r not found in %r" % (regex, output)) + return match + + def check_line(self, output, regex): + regex = re.compile(r'^' + regex, re.MULTILINE) + self.assertRegex(output, regex) + + def parse_executed_tests(self, output): + parser = re.finditer(r'^\[[0-9]+/[0-9]+\] (test_[a-z0-9_]+)$', + output, + re.MULTILINE) + return set(match.group(1) for match in parser) + + def check_executed_tests(self, output, tests, skipped=None): + executed = self.parse_executed_tests(output) + self.assertEqual(executed, set(tests), output) + ntest = len(tests) + if skipped: + if isinstance(skipped, str): + skipped = [skipped] + nskipped = len(skipped) + + plural = 's' if nskipped != 1 else '' + names = ' '.join(sorted(skipped)) + expected = (r'%s test%s skipped:\n %s$' + % (nskipped, plural, names)) + self.check_line(output, expected) + + ok = ntest - nskipped + if ok: + self.check_line(output, r'%s test OK\.$' % ok) + else: + self.check_line(output, r'All %s tests OK\.$' % ntest) + + def parse_random_seed(self, output): + match = self.regex_search(r'Using random seed ([0-9]+)', output) + randseed = int(match.group(1)) + self.assertTrue(0 <= randseed <= 10000000, randseed) + return randseed + + +class ProgramsTestCase(BaseTestCase): + """ + Test various ways to run the Python test suite. Use options close + to options used on the buildbot. + """ + + NTEST = 4 + + def setUp(self): + # Create NTEST tests doing nothing + self.tests = [] + for index in range(1, self.NTEST + 1): + name = self.create_test('noop%s' % index) + self.tests.append(name) + + self.python_args = ['-Wd', '-E', '-bb'] + self.regrtest_args = ['-uall', '-rwW', '--timeout', '3600', '-j4'] + if sys.platform == 'win32': + self.regrtest_args.append('-n') + + def run_tests(self, args): + res = script_helper.assert_python_ok(*args) + + stdout = os.fsdecode(res.out) + self.parse_random_seed(stdout) + self.check_executed_tests(stdout, self.tests) + + def test_script_regrtest(self): + # Lib/test/regrtest.py + script = os.path.join(ROOT_DIR, 'Lib', 'test', 'regrtest.py') + + args = [*self.python_args, script, *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_test(self): + # -m test + args = [*self.python_args, '-m', 'test', + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_regrtest(self): + # -m test.regrtest + args = [*self.python_args, '-m', 'test.regrtest', + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_autotest(self): + # -m test.autotest + args = [*self.python_args, '-m', 'test.autotest', + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_from_test_autotest(self): + # from test import autotest + code = 'from test import autotest' + args = [*self.python_args, '-c', code, + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_script_autotest(self): + # Lib/test/autotest.py + script = os.path.join(ROOT_DIR, 'Lib', 'test', 'autotest.py') + args = [*self.python_args, script, *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_tools_script_run_tests(self): + # Tools/scripts/run_tests.py + script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') + self.run_tests([script, *self.tests]) + + @unittest.skipUnless(sys.platform == 'win32', + 'test.bat is specific to Windows') + def test_tools_buildbot_test(self): + # Tools/buildbot/test.bat + script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat') + self.run_tests([script, *self.tests]) + + # FIXME: test PCbuild/rt.bat? + + +class ArgsTestCase(BaseTestCase): + """ + Test arguments of the Python test suite. + """ + + def run_tests(self, *args): + args = ['-m', 'test', *args] + res = script_helper.assert_python_ok(*args) + return os.fsdecode(res.out) + + def test_resources(self): + # test -u command line option + tests = {} + for resource in ('audio', 'network'): + code = 'from test import support\nsupport.requires(%r)' % resource + name = self.create_test('test_%s' % resource, module=code) + tests[resource] = name + test_names = sorted(tests.values()) + + # -u all: 2 resources enabled + stdout = self.run_tests('-u', 'all', *test_names) + self.check_executed_tests(stdout, test_names) + + # -u audio: 1 resource enabled + stdout = self.run_tests('-uaudio', *test_names) + self.check_executed_tests(stdout, test_names, + skipped=tests['network']) + + # no option: 0 resources enabled + stdout = self.run_tests(*test_names) + self.check_executed_tests(stdout, test_names, + skipped=test_names) + + def test_random(self): + # test -r and --randseed command line option + code = 'import random\nprint("TESTRANDOM: %s" % random.randint(1, 1000))' + test = self.create_test('random', module=code) + + # first run to get the output with the random seed + stdout = self.run_tests('-r', test) + randseed = self.parse_random_seed(stdout) + match = self.regex_search(r'TESTRANDOM: ([0-9]+)', stdout) + test_random = int(match.group(1)) + + # try to reproduce with the random seed + stdout = self.run_tests('-r', '--randseed=%s' % randseed, test) + randseed2 = self.parse_random_seed(stdout) + self.assertEqual(randseed2, randseed) + + match = self.regex_search(r'TESTRANDOM: ([0-9]+)', stdout) + test_random2 = int(match.group(1)) + self.assertEqual(test_random2, test_random) + + def test_fromfile(self): + # test --fromfile + ntest = 5 + tests = [self.create_test('test%s' % index) + for index in range(1, ntest + 1)] + + # Write the list of files using a format similar to regrtest output: + # [1/2] test_1 + # [2/2] test_2 + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + with open(filename, "w") as fp: + for index, name in enumerate(tests, 1): + print("[%s/%s] %s" % (index, ntest, name), file=fp) + + stdout = self.run_tests('--fromfile', filename) + self.check_executed_tests(stdout, tests) + + if __name__ == '__main__': unittest.main()