diff -r 16f982f93a47 Doc/library/subprocess.rst --- a/Doc/library/subprocess.rst Wed Jan 28 12:06:39 2015 -0500 +++ b/Doc/library/subprocess.rst Wed Jan 28 12:13:39 2015 -0800 @@ -25,103 +25,28 @@ Using the :mod:`subprocess` Module ---------------------------------- -The recommended approach to invoking subprocesses is to use the following -convenience functions for all use cases they can handle. For more advanced +The recommended approach to invoking subprocesses is to use the :func:`run` +function for all use cases it can handle. For more advanced use cases, the underlying :class:`Popen` interface can be used directly. -.. function:: call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None) +.. function:: run(args, *, stdin=None, input=None, stdout=None, stderr=None, \ + shell=False, timeout=None, check_returncode=False) Run the command described by *args*. Wait for command to complete, then - return the :attr:`returncode` attribute. + return a :class:`CompletedProcess` instance. The arguments shown above are merely the most common ones, described below in :ref:`frequently-used-arguments` (hence the use of keyword-only notation in the abbreviated signature). The full function signature is largely the - same as that of the :class:`Popen` constructor - this function passes all - supplied arguments other than *timeout* directly through to that interface. + same as that of the :class:`Popen` constructor - apart from *timeout*, + *input* and *check_returncode*, all the arguments to this function + this function are passed through to that interface. - The *timeout* argument is passed to :meth:`Popen.wait`. If the timeout - expires, the child process will be killed and then waited for again. The - :exc:`TimeoutExpired` exception will be re-raised after the child process - has terminated. + This does not capture stdout or stderr by default. To do so, pass + :data:`PIPE` for the *stdout* and/or *stderr* arguments. - Examples:: - - >>> subprocess.call(["ls", "-l"]) - 0 - - >>> subprocess.call("exit 1", shell=True) - 1 - - .. note:: - - Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this - function. The child process will block if it generates enough - output to a pipe to fill up the OS pipe buffer as the pipes are - not being read from. - - .. versionchanged:: 3.3 - *timeout* was added. - - -.. function:: check_call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None) - - Run command with arguments. Wait for command to complete. If the return - code was zero then return, otherwise raise :exc:`CalledProcessError`. The - :exc:`CalledProcessError` object will have the return code in the - :attr:`~CalledProcessError.returncode` attribute. - - The arguments shown above are merely the most common ones, described below - in :ref:`frequently-used-arguments` (hence the use of keyword-only notation - in the abbreviated signature). The full function signature is largely the - same as that of the :class:`Popen` constructor - this function passes all - supplied arguments other than *timeout* directly through to that interface. - - The *timeout* argument is passed to :meth:`Popen.wait`. If the timeout - expires, the child process will be killed and then waited for again. The - :exc:`TimeoutExpired` exception will be re-raised after the child process - has terminated. - - Examples:: - - >>> subprocess.check_call(["ls", "-l"]) - 0 - - >>> subprocess.check_call("exit 1", shell=True) - Traceback (most recent call last): - ... - subprocess.CalledProcessError: Command 'exit 1' returned non-zero exit status 1 - - .. note:: - - Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this - function. The child process will block if it generates enough - output to a pipe to fill up the OS pipe buffer as the pipes are - not being read from. - - .. versionchanged:: 3.3 - *timeout* was added. - - -.. function:: check_output(args, *, input=None, stdin=None, stderr=None, shell=False, universal_newlines=False, timeout=None) - - Run command with arguments and return its output. - - If the return code was non-zero it raises a :exc:`CalledProcessError`. The - :exc:`CalledProcessError` object will have the return code in the - :attr:`~CalledProcessError.returncode` attribute and any output in the - :attr:`~CalledProcessError.output` attribute. - - The arguments shown above are merely the most common ones, described below - in :ref:`frequently-used-arguments` (hence the use of keyword-only notation - in the abbreviated signature). The full function signature is largely the - same as that of the :class:`Popen` constructor - this functions passes all - supplied arguments other than *input* and *timeout* directly through to - that interface. In addition, *stdout* is not permitted as an argument, as - it is used internally to collect the output from the subprocess. - - The *timeout* argument is passed to :meth:`Popen.wait`. If the timeout + The *timeout* argument is passed to :meth:`Popen.communicate`. If the timeout expires, the child process will be killed and then waited for again. The :exc:`TimeoutExpired` exception will be re-raised after the child process has terminated. @@ -132,53 +57,60 @@ is automatically created with ``stdin=PIPE``, and the *stdin* argument may not be used as well. + If *check_returncode* is True, and the process exits with a non-zero exit + code, a :exc:`CalledProcessError` exception will be raised. Attributes of + that exception hold the arguments, the exit code, and stdout and stderr if + they were captured. + Examples:: - >>> subprocess.check_output(["echo", "Hello World!"]) - b'Hello World!\n' + >>> subprocess.run(["ls", "-l"]) # doesn't capture output + CompletedProcess(args=['ls', '-l'], returncode=0) - >>> subprocess.check_output(["echo", "Hello World!"], universal_newlines=True) - 'Hello World!\n' - - >>> subprocess.check_output(["sed", "-e", "s/foo/bar/"], - ... input=b"when in the course of fooman events\n") - b'when in the course of barman events\n' - - >>> subprocess.check_output("exit 1", shell=True) + >>> subprocess.run("exit 1", shell=True, check_returncode=True) Traceback (most recent call last): - ... + ... subprocess.CalledProcessError: Command 'exit 1' returned non-zero exit status 1 - By default, this function will return the data as encoded bytes. The actual - encoding of the output data may depend on the command being invoked, so the - decoding to text will often need to be handled at the application level. + >>> subprocess.run(["ls", "-l", "/dev/null"], stdout=subprocess.PIPE) + CompletedProcess(args=['ls', '-l', '/dev/null'], returncode=0, + stdout=b'crw-rw-rw- 1 root root 1, 3 Jan 23 16:23 /dev/null\n') - This behaviour may be overridden by setting *universal_newlines* to - ``True`` as described below in :ref:`frequently-used-arguments`. + .. versionadded:: 3.5 - To also capture standard error in the result, use - ``stderr=subprocess.STDOUT``:: +.. class:: CompletedProcess - >>> subprocess.check_output( - ... "ls non_existent_file; exit 0", - ... stderr=subprocess.STDOUT, - ... shell=True) - 'ls: non_existent_file: No such file or directory\n' + The return value from :func:`run`, representing a process that has finished. - .. note:: + .. attribute:: args - Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this - function. The child process will block if it generates enough - output to a pipe to fill up the OS pipe buffer as the pipes are - not being read from. + The list of arguments used to launch the process. - .. versionadded:: 3.1 + .. attribute:: returncode - .. versionchanged:: 3.3 - *timeout* was added. + Exit status of the child process. Typically, an exit status of 0 indicates + that it ran successfully. - .. versionchanged:: 3.4 - *input* was added. + A negative value ``-N`` indicates that the child was terminated by signal + ``N`` (POSIX only). + + .. attribute:: stdout + + Captured stdout from the child process. A bytes sequence, or a string if + :func:`run` was called with ``universal_newlines=True``. None if stdout + was not captured. + + .. attribute:: stderr + + Captured stderr from the child process. A bytes sequence, or a string if + :func:`run` was called with ``universal_newlines=True``. None if stderr + was not captured. + + .. method:: check_returncode() + + If :attr:`returncode` is non-zero, raise a :exc:`CalledProcessError`. + + .. versionadded:: 3.5 .. data:: DEVNULL @@ -225,9 +157,14 @@ .. attribute:: output - Output of the child process if this exception is raised by + Output of the child process if it was captured by :func:`run` or :func:`check_output`. Otherwise, ``None``. + .. attribute:: stderr + + Stderr output of the child process if it was captured by :func:`run`. + Otherwise, ``None``. + .. versionadded:: 3.3 @@ -246,9 +183,13 @@ .. attribute:: output - Output of the child process if this exception is raised by + Output of the child process if it was captured by :func:`run` or :func:`check_output`. Otherwise, ``None``. + .. attribute:: stderr + + Stderr output of the child process if it was captured by :func:`run`. + Otherwise, ``None``. .. _frequently-used-arguments: @@ -852,6 +793,161 @@ This flag is ignored if :data:`CREATE_NEW_CONSOLE` is specified. +Older high-level API +-------------------- + +Prior to Python 3.5, these three functions comprised the high level API to +subprocess. You can now use :func:`run` in many cases, but lots of existing code +calls these functions. + +.. function:: call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None) + + Run the command described by *args*. Wait for command to complete, then + return the :attr:`returncode` attribute. + + The arguments shown above are merely the most common ones, described below + in :ref:`frequently-used-arguments` (hence the use of keyword-only notation + in the abbreviated signature). The full function signature is largely the + same as that of the :class:`Popen` constructor - this function passes all + supplied arguments other than *timeout* directly through to that interface. + + The *timeout* argument is passed to :meth:`Popen.wait`. If the timeout + expires, the child process will be killed and then waited for again. The + :exc:`TimeoutExpired` exception will be re-raised after the child process + has terminated. + + Examples:: + + >>> subprocess.call(["ls", "-l"]) + 0 + + >>> subprocess.call("exit 1", shell=True) + 1 + + .. note:: + + Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this + function. The child process will block if it generates enough + output to a pipe to fill up the OS pipe buffer as the pipes are + not being read from. + + .. versionchanged:: 3.3 + *timeout* was added. + +.. function:: check_call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None) + + Run command with arguments. Wait for command to complete. If the return + code was zero then return, otherwise raise :exc:`CalledProcessError`. The + :exc:`CalledProcessError` object will have the return code in the + :attr:`~CalledProcessError.returncode` attribute. + + The arguments shown above are merely the most common ones, described below + in :ref:`frequently-used-arguments` (hence the use of keyword-only notation + in the abbreviated signature). The full function signature is largely the + same as that of the :class:`Popen` constructor - this function passes all + supplied arguments other than *timeout* directly through to that interface. + + The *timeout* argument is passed to :meth:`Popen.wait`. If the timeout + expires, the child process will be killed and then waited for again. The + :exc:`TimeoutExpired` exception will be re-raised after the child process + has terminated. + + Examples:: + + >>> subprocess.check_call(["ls", "-l"]) + 0 + + >>> subprocess.check_call("exit 1", shell=True) + Traceback (most recent call last): + ... + subprocess.CalledProcessError: Command 'exit 1' returned non-zero exit status 1 + + .. note:: + + Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this + function. The child process will block if it generates enough + output to a pipe to fill up the OS pipe buffer as the pipes are + not being read from. + + .. versionchanged:: 3.3 + *timeout* was added. + + +.. function:: check_output(args, *, input=None, stdin=None, stderr=None, shell=False, universal_newlines=False, timeout=None) + + Run command with arguments and return its output. + + If the return code was non-zero it raises a :exc:`CalledProcessError`. The + :exc:`CalledProcessError` object will have the return code in the + :attr:`~CalledProcessError.returncode` attribute and any output in the + :attr:`~CalledProcessError.output` attribute. + + The arguments shown above are merely the most common ones, described below + in :ref:`frequently-used-arguments` (hence the use of keyword-only notation + in the abbreviated signature). The full function signature is largely the + same as that of the :class:`Popen` constructor - this functions passes all + supplied arguments other than *input* and *timeout* directly through to + that interface. In addition, *stdout* is not permitted as an argument, as + it is used internally to collect the output from the subprocess. + + The *timeout* argument is passed to :meth:`Popen.wait`. If the timeout + expires, the child process will be killed and then waited for again. The + :exc:`TimeoutExpired` exception will be re-raised after the child process + has terminated. + + The *input* argument is passed to :meth:`Popen.communicate` and thus to the + subprocess's stdin. If used it must be a byte sequence, or a string if + ``universal_newlines=True``. When used, the internal :class:`Popen` object + is automatically created with ``stdin=PIPE``, and the *stdin* argument may + not be used as well. + + Examples:: + + >>> subprocess.check_output(["echo", "Hello World!"]) + b'Hello World!\n' + + >>> subprocess.check_output(["echo", "Hello World!"], universal_newlines=True) + 'Hello World!\n' + + >>> subprocess.check_output(["sed", "-e", "s/foo/bar/"], + ... input=b"when in the course of fooman events\n") + b'when in the course of barman events\n' + + >>> subprocess.check_output("exit 1", shell=True) + Traceback (most recent call last): + ... + subprocess.CalledProcessError: Command 'exit 1' returned non-zero exit status 1 + + By default, this function will return the data as encoded bytes. The actual + encoding of the output data may depend on the command being invoked, so the + decoding to text will often need to be handled at the application level. + + This behaviour may be overridden by setting *universal_newlines* to + ``True`` as described below in :ref:`frequently-used-arguments`. + + To also capture standard error in the result, use + ``stderr=subprocess.STDOUT``:: + + >>> subprocess.check_output( + ... "ls non_existent_file; exit 0", + ... stderr=subprocess.STDOUT, + ... shell=True) + 'ls: non_existent_file: No such file or directory\n' + + .. note:: + + Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this + function. The child process will block if it generates enough + output to a pipe to fill up the OS pipe buffer as the pipes are + not being read from. + + .. versionadded:: 3.1 + + .. versionchanged:: 3.3 + *timeout* was added. + + .. versionchanged:: 3.4 + *input* was added. .. _subprocess-replacements: diff -r 16f982f93a47 Lib/subprocess.py --- a/Lib/subprocess.py Wed Jan 28 12:06:39 2015 -0500 +++ b/Lib/subprocess.py Wed Jan 28 12:13:39 2015 -0800 @@ -377,10 +377,11 @@ The exit status will be stored in the returncode attribute; check_output() will also store the output in the output attribute. """ - def __init__(self, returncode, cmd, output=None): + def __init__(self, returncode, cmd, output=None, stderr=None): self.returncode = returncode self.cmd = cmd self.output = output + self.stderr = stderr def __str__(self): return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode) @@ -389,10 +390,11 @@ """This exception is raised when the timeout expires while waiting for a child process. """ - def __init__(self, cmd, timeout, output=None): + def __init__(self, cmd, timeout, output=None, stderr=None): self.cmd = cmd self.timeout = timeout self.output = output + self.stderr = stderr def __str__(self): return ("Command '%s' timed out after %s seconds" % @@ -433,7 +435,7 @@ __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput", - "getoutput", "check_output", "CalledProcessError", "DEVNULL"] + "getoutput", "check_output", "run", "CalledProcessError", "DEVNULL"] if mswindows: from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, @@ -625,6 +627,101 @@ return output +class CompletedProcess(object): + """A process that has finished running. + + This is returned by run(). + """ + def __init__(self, args, returncode, stdout=None, stderr=None): + self.args = args + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + def __repr__(self): + args = ['args={!r}'.format(self.args), + 'returncode={!r}'.format(self.returncode)] + if self.stdout is not None: + args.append('stdout={!r}'.format(self.stdout)) + if self.stderr is not None: + args.append('stderr={!r}'.format(self.stderr)) + return "{}({})".format(type(self).__name__, ', '.join(args)) + + def check_returncode(self): + """Raise CalledProcessError if the exit code is non-zero.""" + if self.returncode: + raise CalledProcessError(self.returncode, self.args, self.stdout, + self.stderr) + + +def run(*popenargs, timeout=None, check_returncode=False, **kwargs): + r"""Run command with arguments and return a CompletedProcess instance. + + The returned instance will have attributes args, returncode, stdout and + stderr. By default, stdout and stderr are not captured, and those attributes + will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. + + If check_returncode is True and the exit code was non-zero, it raises a + CalledProcessError. The CalledProcessError object will have the return code + in the returncode attribute, and output & stderr attributes if those streams + were captured. + + If timeout is given, and the process takes too long, a TimeoutExpired + exception will be raised. + + The arguments are the same as for the Popen constructor. Example: + + >>> run(["ls", "-l", "/dev/null"], stdout=PIPE) + CompletedProcess(args=['ls', '-l', '/dev/null'], returncode=0, + stdout=b'crw-rw-rw- 1 root root 1, 3 Jan 23 16:23 /dev/null\n') + + >>> run(['/bin/sh', '-c', 'exit 1'], check_returncode=True) + Traceback (most recent call last): + ... + subprocess.CalledProcessError: Command '['/bin/sh', '-c', 'exit 1']' + returned non-zero exit status 1 + + There is an additional optional argument, "input", allowing you to + pass a string to the subprocess's stdin. If you use this argument + you may not also use the Popen constructor's "stdin" argument, as + it will be used internally. Example: + + >>> run(["sed", "-e", "s/foo/bar/"], stdout=PIPE, + ... input=b"when in the course of fooman events\n") + CompletedProcess(args=['sed', '-e', 's/foo/bar/'], returncode=0, + stdout=b'when in the course of barman events\n') + + If universal_newlines=True is passed, the "input" argument must be a + string and stdout/stderr in the returned object will be strings rather than + bytes. + """ + if 'input' in kwargs: + if 'stdin' in kwargs: + raise ValueError('stdin and input arguments may not both be used.') + inputdata = kwargs['input'] + del kwargs['input'] + kwargs['stdin'] = PIPE + else: + inputdata = None + with Popen(*popenargs, **kwargs) as process: + try: + stdout, stderr = process.communicate(inputdata, timeout=timeout) + except TimeoutExpired: + process.kill() + stdout, stderr = process.communicate() + raise TimeoutExpired(process.args, timeout, output=stdout, + stderr=stderr) + except: + process.kill() + process.wait() + raise + retcode = process.poll() + if check_returncode and retcode: + raise CalledProcessError(retcode, process.args, output=stdout, + stderr=stderr) + return CompletedProcess(process.args, retcode, stdout, stderr) + + def list2cmdline(seq): """ Translate a sequence of arguments into a command line diff -r 16f982f93a47 Lib/test/test_subprocess.py --- a/Lib/test/test_subprocess.py Wed Jan 28 12:06:39 2015 -0500 +++ b/Lib/test/test_subprocess.py Wed Jan 28 12:13:39 2015 -0800 @@ -1232,6 +1232,109 @@ fds_after_exception = os.listdir(fd_directory) self.assertEqual(fds_before_popen, fds_after_exception) + +class RunFuncTestCase(BaseTestCase): + def test_returncode(self): + # call() function with sequence argument + cp = subprocess.run([sys.executable, "-c", "import sys; sys.exit(47)"]) + self.assertEqual(cp.returncode, 47) + with self.assertRaises(subprocess.CalledProcessError): + cp.check_returncode() + + def test_check_returncode(self): + with self.assertRaises(subprocess.CalledProcessError) as c: + subprocess.run([sys.executable, "-c", "import sys; sys.exit(47)"], + check_returncode=True) + self.assertEqual(c.exception.returncode, 47) + + def test_check_returncode_zero(self): + # check_returncode shouldn't raise when returncode is zero + cp = subprocess.run([sys.executable, "-c", "import sys; sys.exit(0)"], + check_returncode=True) + self.assertEqual(cp.returncode, 0) + + def test_timeout(self): + # run() function with timeout argument; we want to test that the child + # process gets killed when the timeout expires. If the child isn't + # killed, this call will deadlock since subprocess.run waits for the + # child. + with self.assertRaises(subprocess.TimeoutExpired): + subprocess.run([sys.executable, "-c", "while True: pass"], + timeout=0.1) + + + def test_capture_stdout(self): + # capture stdout with zero return code + cp = subprocess.run([sys.executable, "-c", "print('BDFL')"], + stdout=subprocess.PIPE) + self.assertIn(b'BDFL', cp.stdout) + + def test_capture_stderr(self): + cp = subprocess.run( + [sys.executable, "-c", "import sys; sys.stderr.write('BDFL')"], + stderr=subprocess.PIPE) + self.assertIn(b'BDFL', cp.stderr) + + def test_check_output_stdin_arg(self): + # run() can be called with stdin set to a file + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + cp = subprocess.run( + [sys.executable, "-c", + "import sys; sys.stdout.write(sys.stdin.read().upper())"], + stdin=tf, stdout=subprocess.PIPE) + self.assertIn(b'PEAR', cp.stdout) + + def test_check_output_input_arg(self): + # check_output() can be called with input set to a string + cp = subprocess.run( + [sys.executable, "-c", + "import sys; sys.stdout.write(sys.stdin.read().upper())"], + input=b'pear', stdout=subprocess.PIPE) + self.assertIn(b'PEAR', cp.stdout) + + def test_check_output_stdin_with_input_arg(self): + # run() refuses to accept 'stdin' with 'input' + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + with self.assertRaises(ValueError) as c: + output = subprocess.run( + [sys.executable, "-c", "print('will not be run')"], + stdin=tf, input=b'hare') + self.fail("Expected ValueError when stdin and input args supplied.") + self.assertIn('stdin', c.exception.args[0]) + self.assertIn('input', c.exception.args[0]) + + def test_check_output_timeout(self): + # check_output() function with timeout arg + with self.assertRaises(subprocess.TimeoutExpired) as c: + cp = subprocess.run( + [sys.executable, "-c", + "import sys, time\n" + "sys.stdout.write('BDFL')\n" + "sys.stdout.flush()\n" + "time.sleep(3600)"], + # Some heavily loaded buildbots (sparc Debian 3.x) require + # this much time to start and print. + timeout=3, stdout=subprocess.PIPE) + self.fail("Expected TimeoutExpired.") + self.assertEqual(c.exception.output, b'BDFL') + + def test_run_kwargs(self): + # call() function with keyword args + newenv = os.environ.copy() + newenv["FRUIT"] = "banana" + cp = subprocess.run([sys.executable, "-c", + 'import sys, os;' + 'sys.exit(os.getenv("FRUIT")=="banana")'], + env=newenv) + self.assertEqual(cp.returncode, 1) + + @unittest.skipIf(mswindows, "POSIX specific tests") class POSIXProcessTestCase(BaseTestCase): @@ -2531,6 +2634,7 @@ HelperFunctionTests, CommandsWithSpaces, ContextManagerTests, + RunFuncTestCase, ) support.run_unittest(*unit_tests)