# HG changeset patch # User Steve Dower # Date 1473124876 25200 # Mon Sep 05 18:21:16 2016 -0700 # Node ID 152e731611abbf54cc94e628f143cf06877e8aef # Parent d52f10a0f10d157e74205922c2f0a90b7004022f Issue #6135: Adds encoding, errors and opener parameters to subprocess diff --git a/Lib/subprocess.py b/Lib/subprocess.py --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -30,7 +30,9 @@ preexec_fn=None, close_fds=True, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, - restore_signals=True, start_new_session=False, pass_fds=()): + restore_signals=True, start_new_session=False, pass_fds=(), + encoding=None, errors=None, + stdin_opener=None, stdout_opener=None, stderr_opener=None): Arguments are: @@ -104,8 +106,10 @@ If env is not None, it defines the environment variables for the new process. -If universal_newlines is False, the file objects stdin, stdout and stderr -are opened as binary files, and no line ending conversion is done. +If stdin_opener, stdout_opener or stderr_opener are provided, they are +called with the binary files opened for each stream respectively and the +returned object is used instead. In this case, the universal_newlines, +encoding and errors arguments have no effect for the stream. If universal_newlines is True, the file objects stdout and stderr are opened as a text file, but lines may be terminated by any of '\n', @@ -113,11 +117,13 @@ '\r\n', the Windows convention. All of these external representations are seen as '\n' by the Python program. Also, the newlines attribute of the file objects stdout, stdin and stderr are not updated by the -communicate() method. +communicate() method. The encoding and errors arguments are used when +opening as text. If not specified, the default encoding for +io.TextIOWrapper is used. -In either case, the process being communicated with should start up -expecting to receive bytes on its standard input and decode them with -the same encoding they are sent in. +If universal_newlines is False and no opener is specified, the file +objects stdin, stdout and stderr are opened as binary files, and no +line ending conversion is done. The startupinfo and creationflags, if given, will be passed to the underlying CreateProcess() function. They can specify things such as @@ -146,7 +152,7 @@ >>> subprocess.check_call(["ls", "-l"]) 0 -getstatusoutput(cmd): +getstatusoutput(cmd, **kwards): Return (status, output) of executing cmd in a shell. Execute the string 'cmd' in a shell with 'check_output' and @@ -164,7 +170,7 @@ >>> subprocess.getstatusoutput('/bin/junk') (256, 'sh: /bin/junk: not found') -getoutput(cmd): +getoutput(cmd, **kwards): Return output (stdout or stderr) of executing cmd in a shell. Like getstatusoutput(), except the exit status is ignored and the return @@ -804,13 +810,17 @@ # Various tools for executing commands and looking at their output and status. # -def getstatusoutput(cmd): +def getstatusoutput(cmd, universal_newlines=True, stderr=STDOUT, **kwargs): """ Return (status, output) of executing cmd in a shell. Execute the string 'cmd' in a shell with 'check_output' and return a 2-tuple (status, output). Universal newlines mode is used, meaning that the result with be decoded to a string. + The other arguments are the same as for the Popen constructor. + However, universal_newlines defaults to True and stderr defaults + to STDOUT. + A trailing newline is stripped from the output. The exit status for the command can be interpreted according to the rules for the function 'wait'. Example: @@ -824,7 +834,9 @@ (256, 'sh: /bin/junk: not found') """ try: - data = check_output(cmd, shell=True, universal_newlines=True, stderr=STDOUT) + kwargs['shell'] = True + data = check_output(cmd, universal_newlines=universal_newlines, + stderr=stderr, **kwargs) status = 0 except CalledProcessError as ex: data = ex.output @@ -833,9 +845,13 @@ data = data[:-1] return status, data -def getoutput(cmd): +def getoutput(cmd, **kwargs): """Return output (stdout or stderr) of executing cmd in a shell. + The other arguments are the same as for the Popen constructor. + However, universal_newlines defaults to True and stderr defaults + to STDOUT. + Like getstatusoutput(), except the exit status is ignored and the return value is a string containing the command's output. Example: @@ -843,7 +859,7 @@ >>> subprocess.getoutput('ls /bin/ls') '/bin/ls' """ - return getstatusoutput(cmd)[1] + return getstatusoutput(cmd, **kwargs)[1] _PLATFORM_DEFAULT_CLOSE_FDS = object() @@ -859,7 +875,8 @@ shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, restore_signals=True, start_new_session=False, - pass_fds=()): + pass_fds=(), encoding=None, errors=None, + stdin_opener=None, stdout_opener=None, stderr_opener=None): """Create new Popen instance.""" _cleanup() # Held while anything is calling waitpid before returncode has been @@ -946,17 +963,26 @@ if p2cwrite != -1: self.stdin = io.open(p2cwrite, 'wb', bufsize) - if universal_newlines: + if stdin_opener: + self.stdin = stdin_opener(self.stdin) + elif universal_newlines: self.stdin = io.TextIOWrapper(self.stdin, write_through=True, - line_buffering=(bufsize == 1)) + line_buffering=(bufsize == 1), + encoding=encoding, errors=errors) if c2pread != -1: self.stdout = io.open(c2pread, 'rb', bufsize) - if universal_newlines: - self.stdout = io.TextIOWrapper(self.stdout) + if stdout_opener: + self.stdout = stdout_opener(self.stdout) + elif universal_newlines: + self.stdout = io.TextIOWrapper(self.stdout, + encoding=encoding, errors=errors) if errread != -1: self.stderr = io.open(errread, 'rb', bufsize) - if universal_newlines: - self.stderr = io.TextIOWrapper(self.stderr) + if stderr_opener: + self.stderr = stderr_opener(self.stderr) + elif universal_newlines: + self.stderr = io.TextIOWrapper(self.stderr, + encoding=encoding, errors=errors) self._closed_child_pipe_fds = False try: diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -891,29 +891,19 @@ # # UTF-16 and UTF-32-BE are sufficient to check both with BOM and # without, and UTF-16 and UTF-32. - import _bootlocale for encoding in ['utf-16', 'utf-32-be']: - old_getpreferredencoding = _bootlocale.getpreferredencoding - # Indirectly via io.TextIOWrapper, Popen() defaults to - # locale.getpreferredencoding(False) and earlier in Python 3.2 to - # locale.getpreferredencoding(). - def getpreferredencoding(do_setlocale=True): - return encoding code = ("import sys; " r"sys.stdout.buffer.write('1\r\n2\r3\n4'.encode('%s'))" % encoding) args = [sys.executable, '-c', code] - try: - _bootlocale.getpreferredencoding = getpreferredencoding - # We set stdin to be non-None because, as of this writing, - # a different code path is used when the number of pipes is - # zero or one. - popen = subprocess.Popen(args, universal_newlines=True, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - stdout, stderr = popen.communicate(input='') - finally: - _bootlocale.getpreferredencoding = old_getpreferredencoding + # We set stdin to be non-None because, as of this writing, + # a different code path is used when the number of pipes is + # zero or one. + popen = subprocess.Popen(args, universal_newlines=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + encoding=encoding) + stdout, stderr = popen.communicate(input='') self.assertEqual(stdout, '1\n2\n3\n4') def test_no_leaking(self): @@ -1333,6 +1323,28 @@ env=newenv) self.assertEqual(cp.returncode, 33) + def test_openers(self): + # check stdin is passed to the opener + objs = [] + def check_opener(f): + objs.append(f) + return f + + p = subprocess.Popen([sys.executable, "-V"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin_opener=check_opener, + stdout_opener=check_opener, + stderr_opener=check_opener) + p.wait() + + self.assertEqual(3, len(objs)) + self.assertIsInstance(objs[0], io.BufferedWriter) + self.assertIsInstance(objs[1], io.BufferedReader) + self.assertIsInstance(objs[2], io.BufferedReader) + for f in objs: + f.close() @unittest.skipIf(mswindows, "POSIX specific tests") class POSIXProcessTestCase(BaseTestCase):