diff -r 976223ff4566 Lib/test/_test_multiprocessing.py --- a/Lib/test/_test_multiprocessing.py Mon May 11 14:55:54 2015 -0400 +++ b/Lib/test/_test_multiprocessing.py Sun Jun 14 11:38:28 2015 -0700 @@ -1,6 +1,7 @@ # # Unit tests for the multiprocessing package # +import subprocess import unittest import queue as pyqueue @@ -3435,9 +3436,9 @@ # start child process using unusual flags prog = ('from test._test_multiprocessing import TestFlags; ' + 'TestFlags.run_in_child()') - data = subprocess.check_output( - [sys.executable, '-E', '-S', '-O', '-c', prog]) - child_flags, grandchild_flags = json.loads(data.decode('ascii')) + result = test.support.script_helper.assert_python_ok( + '-E', '-S', '-O', '-c', prog) + child_flags, grandchild_flags = json.loads(result.out.decode('ascii')) self.assertEqual(child_flags, grandchild_flags) # @@ -3721,37 +3722,51 @@ "test semantics don't make sense on Windows") class TestSemaphoreTracker(unittest.TestCase): def test_semaphore_tracker(self): - import subprocess cmd = '''if 1: - import multiprocessing as mp, time, os + import multiprocessing as mp, time, os, sys mp.set_start_method("spawn") + + # Create two semaphores lock1 = mp.Lock() lock2 = mp.Lock() - os.write(%d, lock1._semlock.name.encode("ascii") + b"\\n") - os.write(%d, lock2._semlock.name.encode("ascii") + b"\\n") + + # Write semaphore names to the parent + print(lock1._semlock.name) + print(lock2._semlock.name) + sys.stdout.flush() + + # Give the parent process a chance to take actions before stopping + # this process time.sleep(10) ''' - r, w = os.pipe() - p = subprocess.Popen([sys.executable, - '-c', cmd % (w, w)], - pass_fds=[w], - stderr=subprocess.PIPE) - os.close(w) - with open(r, 'rb', closefd=True) as f: - name1 = f.readline().rstrip().decode('ascii') - name2 = f.readline().rstrip().decode('ascii') + p = test.support.script_helper.spawn_python('-c', cmd, stderr=subprocess.PIPE) + name1 = p.stdout.readline().rstrip().decode('ascii') + name2 = p.stdout.readline().rstrip().decode('ascii') + + # Unlink the semaphore in this process; semaphore_tracker will not be + # updated and still try to clean up this semaphore _multiprocessing.sem_unlink(name1) - p.terminate() - p.wait() + + run_result = test.support.script_helper.terminate_process(p) + # Wait to make sure the process finishes cleanup time.sleep(2.0) + + # Verify that name2 has already been removed (by semaphore_tracker when + # the process completed) and cannot now be unlinked because it doesn't + # exist with self.assertRaises(OSError) as ctx: _multiprocessing.sem_unlink(name2) # docs say it should be ENOENT, but OSX seems to give EINVAL self.assertIn(ctx.exception.errno, (errno.ENOENT, errno.EINVAL)) - err = p.stderr.read().decode('utf-8') - p.stderr.close() + + err = run_result.err.decode('utf-8') + # Verify that semaphore_tracker has detected that the two semaphores + # were not cleaned up expected = 'semaphore_tracker: There appear to be 2 leaked semaphores' - self.assertRegex(err, expected) + self.assertRegex(run_result.err.decode('utf-8'), expected) + + # semaphore_tracker should fail to clean up name1 because + # we unlinked it before the process completed self.assertRegex(err, 'semaphore_tracker: %r: \[Errno' % name1) # diff -r 976223ff4566 Lib/test/support/script_helper.py --- a/Lib/test/support/script_helper.py Mon May 11 14:55:54 2015 -0400 +++ b/Lib/test/support/script_helper.py Sun Jun 14 11:38:28 2015 -0700 @@ -6,11 +6,8 @@ import sys import os import os.path -import tempfile import subprocess import py_compile -import contextlib -import shutil import zipfile from importlib.util import source_from_cache @@ -54,22 +51,21 @@ _PythonRunResult = collections.namedtuple("_PythonRunResult", ("rc", "out", "err")) - -# Executing the interpreter in a subprocess -def run_python_until_end(*args, **env_vars): +def _get_python_process_args(*args, **env_vars): env_required = interpreter_requires_environment() if '__isolated' in env_vars: isolated = env_vars.pop('__isolated') else: isolated = not env_vars and not env_required - cmd_line = [sys.executable, '-X', 'faulthandler'] + command_line_args = [sys.executable, '-X', 'faulthandler'] if isolated: # isolated mode: ignore Python environment variables, ignore user # site-packages, and don't add the current directory to sys.path - cmd_line.append('-I') + command_line_args.append('-I') elif not env_vars and not env_required: # ignore Python environment variables - cmd_line.append('-E') + command_line_args.append('-E') + command_line_args.extend(args) # Need to preserve the original environment, for in-place testing of # shared library builds. env = os.environ.copy() @@ -78,22 +74,57 @@ if env_vars.pop('__cleanenv', None): env = {} env.update(env_vars) - cmd_line.extend(args) - p = subprocess.Popen(cmd_line, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=env) - try: - out, err = p.communicate() - finally: - subprocess._cleanup() - p.stdout.close() - p.stderr.close() - rc = p.returncode + return env, command_line_args + +def wait_for_process(p): + """ + Wait for the specified process to complete and return the result. + :param p: Instance of process returned by `Popen`. + :return: Instance of `_PythonRunResult` containing the result. + """ + out, err = p.communicate() + + subprocess._cleanup() + p.stdout.close() + p.stderr.close() + err = strip_python_stderr(err) - return _PythonRunResult(rc, out, err), cmd_line + return _PythonRunResult(p.returncode, out, err) + +def terminate_process(p): + """ + Terminate the specified process and return the result. + :param p: Instance of process returned by `Popen`. + :return: Instance of `_PythonRunResult` containing the result. + """ + p.terminate() + p.wait() + out = p.stdout.read() + err = p.stderr.read() + + subprocess._cleanup() + p.stdout.close() + p.stderr.close() + p.stdin.close() + + err = strip_python_stderr(err) + return _PythonRunResult(p.returncode, out, err) + +def run_python_until_end(*args, **env_vars): + """ + Run python interpreter process with the specified arguments, wait for + it to complete and return the results. + :param args: Arguments to the python process. + :param env_vars: Environment variables to apply to the process. + :return: `_PythonRunResult` containing the results. + """ + p = spawn_python(*args, env=env_vars, stderr=subprocess.PIPE) + run_result = wait_for_process(p) + return run_result def _assert_python(expected_success, *args, **env_vars): - res, cmd_line = run_python_until_end(*args, **env_vars) + _, cmd_line = _get_python_process_args(*args, **env_vars) + res = run_python_until_end(*args, **env_vars) if (res.rc and expected_success) or (not res.rc and not expected_success): # Limit to 80 lines to ASCII characters maxlen = 80 * 100 @@ -145,22 +176,25 @@ return _assert_python(False, *args, **env_vars) def spawn_python(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): - """Run a Python subprocess with the given arguments. + """Run a python subprocess with the given arguments. - kw is extra keyword args to pass to subprocess.Popen. Returns a Popen - object. + :param args: Command line args to pass to `Popen`. + :param stdout: File handle to use for stdout. + :param stderr: File handle to use for stderr. + :param kw: Extra keyword args to pass to `Popen`, including environment + variables, specified with the 'env' keyword. + :return: Instance of :class:`subprocess.Popen` """ - cmd_line = [sys.executable, '-E'] - cmd_line.extend(args) + kw.setdefault('env', dict(os.environ)) + kw['env'], command_line_args = _get_python_process_args(*args, **kw['env']) # Under Fedora (?), GNU readline can output junk on stderr when initialized, # depending on the TERM setting. Setting TERM=vt100 is supposed to disable # that. References: # - http://reinout.vanrees.org/weblog/2009/08/14/readline-invisible-character-hack.html # - http://stackoverflow.com/questions/15760712/python-readline-module-prints-escape-character-during-import # - http://lists.gnu.org/archive/html/bug-readline/2007-08/msg00004.html - env = kw.setdefault('env', dict(os.environ)) - env['TERM'] = 'vt100' - return subprocess.Popen(cmd_line, stdin=subprocess.PIPE, + kw['env']['TERM'] = 'vt100' + return subprocess.Popen(command_line_args, stdin=subprocess.PIPE, stdout=stdout, stderr=stderr, **kw) diff -r 976223ff4566 Lib/test/test_compile.py --- a/Lib/test/test_compile.py Mon May 11 14:55:54 2015 -0400 +++ b/Lib/test/test_compile.py Sun Jun 14 11:38:28 2015 -0700 @@ -502,7 +502,7 @@ fn = os.path.join(tmpd, "bad.py") with open(fn, "wb") as fp: fp.write(src) - res = script_helper.run_python_until_end(fn)[0] + res = script_helper.run_python_until_end(fn) self.assertIn(b"Non-UTF-8", res.err) @support.cpython_only diff -r 976223ff4566 Lib/test/test_io.py --- a/Lib/test/test_io.py Mon May 11 14:55:54 2015 -0400 +++ b/Lib/test/test_io.py Sun Jun 14 11:38:28 2015 -0700 @@ -3498,7 +3498,7 @@ file.write('!') file.flush() """.format_map(locals()) - res, _ = run_python_until_end("-c", code) + res = run_python_until_end("-c", code) err = res.err.decode() if res.rc != 0: # Failure: should be a fatal error diff -r 976223ff4566 Lib/test/test_script_helper.py --- a/Lib/test/test_script_helper.py Mon May 11 14:55:54 2015 -0400 +++ b/Lib/test/test_script_helper.py Sun Jun 14 11:38:28 2015 -0700 @@ -45,7 +45,7 @@ except RuntimeError as err: self.assertEqual('bail out of unittest', err.args[0]) self.assertEqual(1, mock_popen.call_count) - self.assertEqual(1, mock_ire_func.call_count) + self.assertTrue(mock_ire_func.call_count >= 1) popen_command = mock_popen.call_args[0][0] self.assertEqual(sys.executable, popen_command[0]) self.assertIn('None', popen_command)