Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(1)

Side by Side Diff: Lib/test/test_regrtest.py

Issue 26295: Random failures when running test suite in parallel (-m test -j0) caused by test_regrtest
Patch Set: Created 3 years, 11 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « Lib/test/libregrtest/setup.py ('k') | Lib/test/test_zipfile.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 """ 1 """
2 Tests of regrtest.py. 2 Tests of regrtest.py.
3 3
4 Note: test_regrtest cannot be run twice in parallel. 4 Note: test_regrtest cannot be run twice in parallel.
5 """ 5 """
6 6
7 import argparse 7 import argparse
8 import contextlib 8 import contextlib
9 import faulthandler 9 import faulthandler
10 import getopt 10 import getopt
11 import io 11 import io
12 import os.path 12 import os.path
13 import platform 13 import platform
14 import re 14 import re
15 import subprocess 15 import subprocess
16 import sys 16 import sys
17 import sysconfig 17 import sysconfig
18 import tempfile
18 import textwrap 19 import textwrap
19 import unittest 20 import unittest
20 from test import libregrtest 21 from test import libregrtest
21 from test import support 22 from test import support
22 23
23 24
24 Py_DEBUG = hasattr(sys, 'getobjects') 25 Py_DEBUG = hasattr(sys, 'getobjects')
25 ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') 26 ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..')
26 ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) 27 ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
27 28
(...skipping 270 matching lines...) Expand 10 before | Expand all | Expand 10 after
298 ns = libregrtest._parse_args(['--quiet', 'foo']) 299 ns = libregrtest._parse_args(['--quiet', 'foo'])
299 self.assertTrue(ns.quiet) 300 self.assertTrue(ns.quiet)
300 self.assertEqual(ns.verbose, 0) 301 self.assertEqual(ns.verbose, 0)
301 self.assertEqual(ns.args, ['foo']) 302 self.assertEqual(ns.args, ['foo'])
302 303
303 304
304 class BaseTestCase(unittest.TestCase): 305 class BaseTestCase(unittest.TestCase):
305 TEST_UNIQUE_ID = 1 306 TEST_UNIQUE_ID = 1
306 TESTNAME_PREFIX = 'test_regrtest_' 307 TESTNAME_PREFIX = 'test_regrtest_'
307 TESTNAME_REGEX = r'test_[a-z0-9_]+' 308 TESTNAME_REGEX = r'test_[a-z0-9_]+'
309
310 def create_testdir(self):
311 self.tmpdir = tempfile.mkdtemp()
312 self.addCleanup(support.rmtree, self.tmpdir)
313
314 self.tmptestdir = os.path.join(self.tmpdir, 'test')
315 os.mkdir(self.tmptestdir)
308 316
309 def setUp(self): 317 def setUp(self):
310 self.testdir = os.path.realpath(os.path.dirname(__file__)) 318 self.testdir = os.path.realpath(os.path.dirname(__file__))
311 319
320 self.create_testdir()
321
312 # When test_regrtest is interrupted by CTRL+c, it can leave 322 # When test_regrtest is interrupted by CTRL+c, it can leave
313 # temporary test files 323 # temporary test files
314 remove = [entry.path 324 remove = [entry.path
315 for entry in os.scandir(self.testdir) 325 for entry in os.scandir(self.testdir)
316 if (entry.name.startswith(self.TESTNAME_PREFIX) 326 if (entry.name.startswith(self.TESTNAME_PREFIX)
317 and entry.name.endswith(".py"))] 327 and entry.name.endswith(".py"))]
318 for path in remove: 328 for path in remove:
319 print("WARNING: test_regrtest: remove %s" % path) 329 print("WARNING: test_regrtest: remove %s" % path)
320 support.unlink(path) 330 support.unlink(path)
321 331
322 def create_test(self, name=None, code=''): 332 def create_test(self, name=None, code=''):
323 if not name: 333 if not name:
324 name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID 334 name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID
325 BaseTestCase.TEST_UNIQUE_ID += 1 335 BaseTestCase.TEST_UNIQUE_ID += 1
326 336
327 # test_regrtest cannot be run twice in parallel because 337 # test_regrtest cannot be run twice in parallel because
328 # of setUp() and create_test() 338 # of setUp() and create_test()
329 name = self.TESTNAME_PREFIX + "%s_%s" % (os.getpid(), name) 339 name = self.TESTNAME_PREFIX + "%s_%s" % (os.getpid(), name)
330 path = os.path.join(self.testdir, name + '.py') 340 path = os.path.join(self.tmptestdir, name + '.py')
331 341
332 self.addCleanup(support.unlink, path) 342 self.addCleanup(support.unlink, path)
333 # Use 'x' mode to ensure that we do not override existing tests 343 # Use 'x' mode to ensure that we do not override existing tests
334 try: 344 try:
335 with open(path, 'x', encoding='utf-8') as fp: 345 with open(path, 'x', encoding='utf-8') as fp:
336 fp.write(code) 346 fp.write(code)
337 except PermissionError as exc: 347 except PermissionError as exc:
338 if not sysconfig.is_python_build(): 348 if not sysconfig.is_python_build():
339 self.skipTest("cannot write %s: %s" % (path, exc)) 349 self.skipTest("cannot write %s: %s" % (path, exc))
340 raise 350 raise
(...skipping 113 matching lines...) Expand 10 before | Expand all | Expand 10 after
454 464
455 NTEST = 4 465 NTEST = 4
456 466
457 def setUp(self): 467 def setUp(self):
458 super().setUp() 468 super().setUp()
459 469
460 # Create NTEST tests doing nothing 470 # Create NTEST tests doing nothing
461 self.tests = [self.create_test() for index in range(self.NTEST)] 471 self.tests = [self.create_test() for index in range(self.NTEST)]
462 472
463 self.python_args = ['-Wd', '-E', '-bb'] 473 self.python_args = ['-Wd', '-E', '-bb']
464 self.regrtest_args = ['-uall', '-rwW'] 474 self.regrtest_args = ['-uall', '-rwW', '--testdir=%s' % self.tmptestdir]
465 if hasattr(faulthandler, 'dump_traceback_later'): 475 if hasattr(faulthandler, 'dump_traceback_later'):
466 self.regrtest_args.extend(('--timeout', '3600', '-j4')) 476 self.regrtest_args.extend(('--timeout', '3600', '-j4'))
467 if sys.platform == 'win32': 477 if sys.platform == 'win32':
468 self.regrtest_args.append('-n') 478 self.regrtest_args.append('-n')
469 479
470 def check_output(self, output): 480 def check_output(self, output):
471 self.parse_random_seed(output) 481 self.parse_random_seed(output)
472 self.check_executed_tests(output, self.tests, randomize=True) 482 self.check_executed_tests(output, self.tests, randomize=True)
473 483
474 def run_tests(self, args): 484 def run_tests(self, args):
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after
511 # Lib/test/autotest.py 521 # Lib/test/autotest.py
512 script = os.path.join(self.testdir, 'autotest.py') 522 script = os.path.join(self.testdir, 'autotest.py')
513 args = [*self.python_args, script, *self.regrtest_args, *self.tests] 523 args = [*self.python_args, script, *self.regrtest_args, *self.tests]
514 self.run_tests(args) 524 self.run_tests(args)
515 525
516 @unittest.skipUnless(sysconfig.is_python_build(), 526 @unittest.skipUnless(sysconfig.is_python_build(),
517 'run_tests.py script is not installed') 527 'run_tests.py script is not installed')
518 def test_tools_script_run_tests(self): 528 def test_tools_script_run_tests(self):
519 # Tools/scripts/run_tests.py 529 # Tools/scripts/run_tests.py
520 script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') 530 script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py')
521 self.run_tests([script, *self.tests]) 531 args = [script, '--testdir=%s' % self.tmptestdir, *self.tests]
532 self.run_tests(args)
522 533
523 def run_batch(self, *args): 534 def run_batch(self, *args):
524 proc = self.run_command(args) 535 proc = self.run_command(args)
525 self.check_output(proc.stdout) 536 self.check_output(proc.stdout)
526 537
527 @unittest.skipUnless(sysconfig.is_python_build(), 538 @unittest.skipUnless(sysconfig.is_python_build(),
528 'test.bat script is not installed') 539 'test.bat script is not installed')
529 @unittest.skipUnless(sys.platform == 'win32', 'Windows only') 540 @unittest.skipUnless(sys.platform == 'win32', 'Windows only')
530 def test_tools_buildbot_test(self): 541 def test_tools_buildbot_test(self):
531 # Tools\buildbot\test.bat 542 # Tools\buildbot\test.bat
(...skipping 15 matching lines...) Expand all
547 if Py_DEBUG: 558 if Py_DEBUG:
548 rt_args.append('-d') # Debug build, use python_d.exe 559 rt_args.append('-d') # Debug build, use python_d.exe
549 self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests) 560 self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests)
550 561
551 562
552 class ArgsTestCase(BaseTestCase): 563 class ArgsTestCase(BaseTestCase):
553 """ 564 """
554 Test arguments of the Python test suite. 565 Test arguments of the Python test suite.
555 """ 566 """
556 567
557 def run_tests(self, *args, **kw): 568 def run_tests(self, *testargs, **kw):
558 return self.run_python(['-m', 'test', *args], **kw) 569 cmdargs = ['-m', 'test', '--testdir=%s' % self.tmptestdir, *testargs]
570 return self.run_python(cmdargs, **kw)
559 571
560 def test_failing_test(self): 572 def test_failing_test(self):
561 # test a failing test 573 # test a failing test
562 code = textwrap.dedent(""" 574 code = textwrap.dedent("""
563 import unittest 575 import unittest
564 576
565 class FailingTest(unittest.TestCase): 577 class FailingTest(unittest.TestCase):
566 def test_failing(self): 578 def test_failing(self):
567 self.fail("bug") 579 self.fail("bug")
568 """) 580 """)
(...skipping 175 matching lines...) Expand 10 before | Expand all | Expand 10 after
744 def test_list_tests(self): 756 def test_list_tests(self):
745 # test --list-tests 757 # test --list-tests
746 tests = [self.create_test() for i in range(5)] 758 tests = [self.create_test() for i in range(5)]
747 output = self.run_tests('--list-tests', *tests) 759 output = self.run_tests('--list-tests', *tests)
748 self.assertEqual(output.rstrip().splitlines(), 760 self.assertEqual(output.rstrip().splitlines(),
749 tests) 761 tests)
750 762
751 763
752 if __name__ == '__main__': 764 if __name__ == '__main__':
753 unittest.main() 765 unittest.main()
OLDNEW
« no previous file with comments | « Lib/test/libregrtest/setup.py ('k') | Lib/test/test_zipfile.py » ('j') | no next file with comments »

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+