diff -r 4752fafb579d Lib/test/regrtest.py --- a/Lib/test/regrtest.py Wed Jul 11 19:21:31 2012 +0200 +++ b/Lib/test/regrtest.py Thu Jul 12 20:07:02 2012 -0700 @@ -166,6 +166,8 @@ """ import builtins +import collections +import contextlib import faulthandler import getopt import io @@ -587,41 +589,28 @@ resource_denieds.append(test) if forever: - def test_forever(tests=list(selected)): - while True: - for test in tests: - yield test - if bad: - return - tests = test_forever() test_count = '' test_count_width = 3 else: - tests = iter(selected) test_count = '/{}'.format(len(selected)) test_count_width = len(test_count) - 1 if use_mp: - try: - from threading import Thread - except ImportError: + if threading is None: print("Multiprocess option requires thread support") sys.exit(2) + + def should_stop(): + return interrupted or (forever and bad) + + next_test = make_next_test(tests=selected, should_stop=should_stop, + use_mp=use_mp, forever=forever) + + if use_mp: from queue import Queue from subprocess import Popen, PIPE debug_output_pat = re.compile(r"\[\d+ refs\]$") output = Queue() - def tests_and_args(): - for test in tests: - args_tuple = ( - (test, verbose, quiet), - dict(huntrleaks=huntrleaks, use_resources=use_resources, - debug=debug, output_on_failure=verbose3, - timeout=timeout, failfast=failfast, - match_tests=match_tests) - ) - yield (test, args_tuple) - pending = tests_and_args() opt_args = support.args_from_interpreter_flags() base_cmd = [sys.executable] + opt_args + ['-m', 'test.regrtest'] def work(): @@ -629,10 +618,17 @@ try: while True: try: - test, args_tuple = next(pending) + test = next_test() except StopIteration: output.put((None, None, None, None)) return + args_tuple = ( + (test, verbose, quiet), + dict(huntrleaks=huntrleaks, use_resources=use_resources, + debug=debug, output_on_failure=verbose3, + timeout=timeout, failfast=failfast, + match_tests=match_tests) + ) # -E is needed by some tests, e.g. test_import # Running the child from the same working directory ensures # that TEMPDIR for the child is the same when @@ -660,7 +656,7 @@ except BaseException: output.put((None, None, None, None)) raise - workers = [Thread(target=work) for i in range(use_mp)] + workers = [threading.Thread(target=work) for i in range(use_mp)] for worker in workers: worker.start() finished = 0 @@ -691,11 +687,16 @@ test_index += 1 except KeyboardInterrupt: interrupted = True - pending.close() for worker in workers: worker.join() else: - for test_index, test in enumerate(tests, 1): + test_index = 0 + while True: + try: + test = next_test() + except StopIteration: + break + test_index += 1 if not quiet: fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" print(fmt.format( @@ -837,6 +838,47 @@ tests.append(mod) return stdtests + sorted(tests) +def make_next_test(tests, should_stop, use_mp, forever): + """Return a thread-safe next_test() function. + + The threading module must have already been imported if calling + this function with use_mp True. + + Arguments (selected): + + should_stop: a function that accepts no arguments and returns + whether to raise StopIteration on a call to next_test(). + + """ + pending = collections.deque(tests) + + if use_mp: + lock = threading.Lock() + deque_manager = lambda: lock + else: + @contextlib.contextmanager + def deque_manager(): + yield + + def next_test(): + while True: + if should_stop(): + break + try: + # We make no assumptions about the thread-safety of deque. + with deque_manager(): + return pending.popleft() + except IndexError: + # Don't loop forever if there are no tests. + if forever and tests: + with deque_manager(): + pending.extend(tests) + continue + break + raise StopIteration('no more tests') + + return next_test + def replace_stdout(): """Set stdout encoder error handler to backslashreplace (as stderr error handler) to avoid UnicodeEncodeError when printing a traceback""" diff -r 4752fafb579d Misc/NEWS --- a/Misc/NEWS Wed Jul 11 19:21:31 2012 +0200 +++ b/Misc/NEWS Thu Jul 12 20:07:02 2012 -0700 @@ -121,6 +121,9 @@ Tests ----- +- Issue #15320: Make iterating the list of tests thread-safe when running + tests in multiprocess mode. Patch by Chris Jerdonek. + - Issue #15300: Ensure the temporary test working directories are in the same parent folder when running tests in multiprocess mode from a Python build. Patch by Chris Jerdonek.