diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index afa1e86c05..ee79c34071 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -447,17 +447,20 @@ functions. common use of *preexec_fn* to call os.setsid() in the child. If *close_fds* is true, all file descriptors except :const:`0`, :const:`1` and - :const:`2` will be closed before the child process is executed. (POSIX only). - The default varies by platform: Always true on POSIX. On Windows it is - true when *stdin*/*stdout*/*stderr* are :const:`None`, false otherwise. + :const:`2` will be closed before the child process is executed. On Windows, if *close_fds* is true then no handles will be inherited by the - child process. Note that on Windows, you cannot set *close_fds* to true and - also redirect the standard handles by setting *stdin*, *stdout* or *stderr*. + child process unless explicitly passed in the ``handle_list`` element of + :attr:`STARTUPINFO.lpAttributeList`, or by standard handle redirection. .. versionchanged:: 3.2 The default for *close_fds* was changed from :const:`False` to what is described above. + .. versionchanged:: 3.7 + On Windows the default for *close_fds* was changed from :const:`False` to + :const:`True`. It's now possible to set *close_fds* to :const:`True` when + redirecting the standard handles. + *pass_fds* is an optional sequence of file descriptors to keep open between the parent and child. Providing any *pass_fds* forces *close_fds* to be :const:`True`. (POSIX only) @@ -797,6 +800,19 @@ on Windows. :data:`SW_HIDE` is provided for this attribute. It is used when :class:`Popen` is called with ``shell=True``. + .. attribute:: lpAttributeList + + A dictionary of additional attributes for process creation as given in + ``STARTUPINFOEX``, see + `UpdateProcThreadAttribute `__. + + Supported attributes: + + **handle_list** + When *close_fds* is :const:`True`, supplies a list of + handles that will be inherited. + + .. versionadded:: 3.7 Constants ^^^^^^^^^ diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 551aad342b..68ce6fc8c9 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -134,6 +134,7 @@ if _mswindows: self.hStdOutput = hStdOutput self.hStdError = hStdError self.wShowWindow = wShowWindow + self.lpAttributeList = {"handle_list": []} else: import _posixsubprocess import select @@ -608,25 +609,15 @@ class Popen(object): if not isinstance(bufsize, int): raise TypeError("bufsize must be an integer") + if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: + close_fds = True + if _mswindows: if preexec_fn is not None: raise ValueError("preexec_fn is not supported on Windows " "platforms") - any_stdio_set = (stdin is not None or stdout is not None or - stderr is not None) - if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: - if any_stdio_set: - close_fds = False - else: - close_fds = True - elif close_fds and any_stdio_set: - raise ValueError( - "close_fds is not supported on Windows platforms" - " if you redirect stdin/stdout/stderr") else: # POSIX - if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: - close_fds = True if pass_fds and not close_fds: warnings.warn("pass_fds overriding close_fds.", RuntimeWarning) close_fds = True @@ -951,6 +942,19 @@ class Popen(object): return Handle(h) + def _filter_handle_list(self, handle_list): + """Filter out console handles that can't be used + in lpAttributeList["handle_list"] and make sure the list + isn't empty""" + # An handle with it's lowest two bits set might be a special console + # handle that if passed in lpAttributeList["handle_list"], will + # cause it to fail. We use a set comprehension to remove duplicates. + return list({handle for handle in handle_list + if handle & 0x3 != 0x3 + or _winapi.GetFileType(handle) != + _winapi.FILE_TYPE_CHAR}) + + def _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, @@ -968,12 +972,40 @@ class Popen(object): # Process startup details if startupinfo is None: startupinfo = STARTUPINFO() - if -1 not in (p2cread, c2pwrite, errwrite): + + use_std_handles = -1 not in (p2cread, c2pwrite, errwrite) + if use_std_handles: startupinfo.dwFlags |= _winapi.STARTF_USESTDHANDLES startupinfo.hStdInput = p2cread startupinfo.hStdOutput = c2pwrite startupinfo.hStdError = errwrite + attribute_list = startupinfo.lpAttributeList + have_handle_list = (attribute_list and + "handle_list" in attribute_list and + attribute_list["handle_list"]) + + # If we were given an handle_list or need to create one + if have_handle_list or (use_std_handles and close_fds): + if attribute_list is None: + attribute_list = startupinfo.lpAttributeList = {} + handle_list = attribute_list.setdefault("handle_list", []) + + if use_std_handles: + handle_list += [p2cread, c2pwrite, errwrite] + + handle_list[:] = self._filter_handle_list(handle_list) + + if handle_list: + if have_handle_list and not close_fds: + warnings.warn("startupinfo.lpAttributeList['handle_list'] " + "overriding close_fds", RuntimeWarning) + + # When using the handle_list we always request to inherit + # handles but the only handles that will be inherited are + # the ones in the handle_list + close_fds = False + if shell: startupinfo.dwFlags |= _winapi.STARTF_USESHOWWINDOW startupinfo.wShowWindow = _winapi.SW_HIDE diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index f01bd1a695..0e8880e76a 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -2601,11 +2601,6 @@ class Win32ProcessTestCase(BaseTestCase): [sys.executable, "-c", "import sys; sys.exit(47)"], preexec_fn=lambda: 1) - self.assertRaises(ValueError, subprocess.call, - [sys.executable, "-c", - "import sys; sys.exit(47)"], - stdout=subprocess.PIPE, - close_fds=True) def test_close_fds(self): # close file descriptors @@ -2614,6 +2609,56 @@ class Win32ProcessTestCase(BaseTestCase): close_fds=True) self.assertEqual(rc, 47) + def test_close_fds_with_stdio(self): + import msvcrt + + fds = os.pipe() + self.addCleanup(os.close, fds[0]) + self.addCleanup(os.close, fds[1]) + + handles = [] + for fd in fds: + os.set_inheritable(fd, True) + handles.append(msvcrt.get_osfhandle(fd)) + + p = subprocess.Popen([sys.executable, "-c", + "import msvcrt; print(msvcrt.open_osfhandle({}, 0))".format(handles[0])], + stdout=subprocess.PIPE, close_fds=False) + stdout, stderr = p.communicate() + self.assertEqual(p.returncode, 0) + int(stdout.strip()) # Check that stdout is an integer + + p = subprocess.Popen([sys.executable, "-c", + "import msvcrt; print(msvcrt.open_osfhandle({}, 0))".format(handles[0])], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) + stdout, stderr = p.communicate() + self.assertEqual(p.returncode, 1) + self.assertIn(b"OSError", stderr) + + # The same as the previous call, but with an empty handle_list + handle_list = [] + startupinfo = subprocess.STARTUPINFO() + startupinfo.lpAttributeList = {"handle_list": handle_list} + p = subprocess.Popen([sys.executable, "-c", + "import msvcrt; print(msvcrt.open_osfhandle({}, 0))".format(handles[0])], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + startupinfo=startupinfo, close_fds=True) + stdout, stderr = p.communicate() + self.assertEqual(p.returncode, 1) + self.assertIn(b"OSError", stderr) + + def test_empty_attribute_list(self): + startupinfo = subprocess.STARTUPINFO() + startupinfo.lpAttributeList = {} + subprocess.call([sys.executable, "-c", "import sys; sys.exit(0)"], + startupinfo=startupinfo) + + def test_empty_handle_list(self): + startupinfo = subprocess.STARTUPINFO() + startupinfo.lpAttributeList = {"handle_list": []} + subprocess.call([sys.executable, "-c", "import sys; sys.exit(0)"], + startupinfo=startupinfo) + def test_shell_sequence(self): # Run command through the shell (sequence) newenv = os.environ.copy() diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 8d01b376f2..21658bc57f 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -795,6 +795,157 @@ getenvironment(PyObject* environment) return NULL; } +static LPHANDLE +gethandlelist(PyObject *mapping, const char *name, Py_ssize_t *size) +{ + LPHANDLE ret = NULL; + PyObject *value_fast = NULL; + PyObject *value; + Py_ssize_t i; + + value = PyMapping_GetItemString(mapping, name); + if (!value) { + PyErr_Clear(); + return NULL; + } + + if (value == Py_None) { + goto cleanup; + } + + value_fast = PySequence_Fast(value, "handle_list must be a sequence or None"); + if (value_fast == NULL) + goto cleanup; + + *size = PySequence_Fast_GET_SIZE(value_fast) * sizeof(HANDLE); + + /* Passing an empty array causes CreateProcess to fail so just don't set it */ + if (*size == 0) { + goto cleanup; + } + + ret = PyMem_Malloc(*size); + if (ret == NULL) + goto cleanup; + + for (i = 0; i < PySequence_Fast_GET_SIZE(value_fast); i++) { + ret[i] = PYNUM_TO_HANDLE(PySequence_Fast_GET_ITEM(value_fast, i)); + } + +cleanup: + Py_DECREF(value); + Py_XDECREF(value_fast); + return ret; +} + +typedef struct { + LPPROC_THREAD_ATTRIBUTE_LIST attribute_list; + LPHANDLE handle_list; +} AttributeList; + +static void +freeattributelist(AttributeList *attribute_list) +{ + if (attribute_list->attribute_list != NULL) { + DeleteProcThreadAttributeList(attribute_list->attribute_list); + PyMem_Free(attribute_list->attribute_list); + } + + PyMem_Free(attribute_list->handle_list); + + memset(attribute_list, 0, sizeof(*attribute_list)); +} + +static int +getattributelist(PyObject *obj, const char *name, AttributeList *attribute_list) +{ + int ret = 0; + BOOL result; + PyObject *value; + Py_ssize_t handle_list_size; + DWORD attribute_count = 0; + SIZE_T attribute_list_size; + + value = PyObject_GetAttrString(obj, name); + if (!value) { + PyErr_Clear(); /* FIXME: propagate error? */ + return 0; + } + + if (value == Py_None) { + ret = 0; + goto cleanup; + } + + if (!PyMapping_Check(value)) { + ret = -1; + PyErr_Format(PyExc_TypeError, "%s must be a mapping or None", name); + goto cleanup; + } + + attribute_list->handle_list = gethandlelist(value, "handle_list", &handle_list_size); + if (attribute_list->handle_list == NULL && PyErr_Occurred()) { + ret = -1; + goto cleanup; + } + + if (attribute_list->handle_list != NULL) + ++attribute_count; + + /* Get how many bytes we need for the attribute list */ + result = InitializeProcThreadAttributeList(NULL, attribute_count, 0, &attribute_list_size); + if (result || GetLastError() != ERROR_INSUFFICIENT_BUFFER) { + ret = -1; + PyErr_SetFromWindowsErr(GetLastError()); + goto cleanup; + } + + attribute_list->attribute_list = PyMem_Malloc(attribute_list_size); + if (attribute_list->attribute_list == NULL) { + ret = -1; + goto cleanup; + } + + result = InitializeProcThreadAttributeList( + attribute_list->attribute_list, + attribute_count, + 0, + &attribute_list_size); + if (!result) { + /* So that we won't call DeleteProcThreadAttributeList */ + PyMem_Free(attribute_list->attribute_list); + attribute_list->attribute_list = NULL; + + ret = -1; + PyErr_SetFromWindowsErr(GetLastError()); + goto cleanup; + } + + if (attribute_list->handle_list != NULL) { + result = UpdateProcThreadAttribute( + attribute_list->attribute_list, + 0, + PROC_THREAD_ATTRIBUTE_HANDLE_LIST, + attribute_list->handle_list, + handle_list_size, + NULL, + NULL); + if (!result) { + ret = -1; + PyErr_SetFromWindowsErr(GetLastError()); + goto cleanup; + } + } + +cleanup: + Py_DECREF(value); + + if (ret < 0) + freeattributelist(attribute_list); + + return ret; +} + /*[clinic input] _winapi.CreateProcess @@ -826,63 +977,74 @@ _winapi_CreateProcess_impl(PyObject *module, Py_UNICODE *application_name, PyObject *startup_info) /*[clinic end generated code: output=4652a33aff4b0ae1 input=4a43b05038d639bb]*/ { + PyObject *ret = NULL; BOOL result; PROCESS_INFORMATION pi; - STARTUPINFOW si; - PyObject* environment; + STARTUPINFOEXW si; + PyObject *environment = NULL; wchar_t *wenvironment; + AttributeList attribute_list = {0}; ZeroMemory(&si, sizeof(si)); - si.cb = sizeof(si); + si.StartupInfo.cb = sizeof(si); /* note: we only support a small subset of all SI attributes */ - si.dwFlags = getulong(startup_info, "dwFlags"); - si.wShowWindow = (WORD)getulong(startup_info, "wShowWindow"); - si.hStdInput = gethandle(startup_info, "hStdInput"); - si.hStdOutput = gethandle(startup_info, "hStdOutput"); - si.hStdError = gethandle(startup_info, "hStdError"); + si.StartupInfo.dwFlags = getulong(startup_info, "dwFlags"); + si.StartupInfo.wShowWindow = (WORD)getulong(startup_info, "wShowWindow"); + si.StartupInfo.hStdInput = gethandle(startup_info, "hStdInput"); + si.StartupInfo.hStdOutput = gethandle(startup_info, "hStdOutput"); + si.StartupInfo.hStdError = gethandle(startup_info, "hStdError"); if (PyErr_Occurred()) - return NULL; + goto cleanup; if (env_mapping != Py_None) { environment = getenvironment(env_mapping); if (! environment) - return NULL; + goto cleanup; wenvironment = PyUnicode_AsUnicode(environment); if (wenvironment == NULL) - { - Py_XDECREF(environment); - return NULL; - } + goto cleanup; } else { environment = NULL; wenvironment = NULL; } + if (getattributelist(startup_info, "lpAttributeList", &attribute_list) < 0) + goto cleanup; + + si.lpAttributeList = attribute_list.attribute_list; + Py_BEGIN_ALLOW_THREADS result = CreateProcessW(application_name, command_line, NULL, NULL, inherit_handles, - creation_flags | CREATE_UNICODE_ENVIRONMENT, + creation_flags | EXTENDED_STARTUPINFO_PRESENT | + CREATE_UNICODE_ENVIRONMENT, wenvironment, current_directory, - &si, + (LPSTARTUPINFOW)&si, &pi); Py_END_ALLOW_THREADS - Py_XDECREF(environment); + if (!result) { + PyErr_SetFromWindowsErr(GetLastError()); + goto cleanup; + } - if (! result) - return PyErr_SetFromWindowsErr(GetLastError()); + ret = Py_BuildValue("NNkk", + HANDLE_TO_PYNUM(pi.hProcess), + HANDLE_TO_PYNUM(pi.hThread), + pi.dwProcessId, + pi.dwThreadId); + +cleanup: + Py_XDECREF(environment); + freeattributelist(&attribute_list); - return Py_BuildValue("NNkk", - HANDLE_TO_PYNUM(pi.hProcess), - HANDLE_TO_PYNUM(pi.hThread), - pi.dwProcessId, - pi.dwThreadId); + return ret; } /*[clinic input] @@ -1472,6 +1634,30 @@ _winapi_WriteFile_impl(PyObject *module, HANDLE handle, PyObject *buffer, return Py_BuildValue("II", written, err); } +/*[clinic input] +_winapi.GetFileType -> DWORD + + handle: HANDLE +[clinic start generated code]*/ + +static DWORD +_winapi_GetFileType_impl(PyObject *module, HANDLE handle) +/*[clinic end generated code: output=92b8466ac76ecc17 input=0058366bc40bbfbf]*/ +{ + DWORD result; + + Py_BEGIN_ALLOW_THREADS + result = GetFileType(handle); + Py_END_ALLOW_THREADS + + if (result == FILE_TYPE_UNKNOWN && GetLastError() != NO_ERROR) { + PyErr_SetFromWindowsErr(0); + return -1; + } + + return result; +} + static PyMethodDef winapi_functions[] = { _WINAPI_CLOSEHANDLE_METHODDEF @@ -1498,6 +1684,7 @@ static PyMethodDef winapi_functions[] = { _WINAPI_WAITFORMULTIPLEOBJECTS_METHODDEF _WINAPI_WAITFORSINGLEOBJECT_METHODDEF _WINAPI_WRITEFILE_METHODDEF + _WINAPI_GETFILETYPE_METHODDEF {NULL, NULL} }; @@ -1578,6 +1765,11 @@ PyInit__winapi(void) WINAPI_CONSTANT(F_DWORD, WAIT_OBJECT_0); WINAPI_CONSTANT(F_DWORD, WAIT_ABANDONED_0); WINAPI_CONSTANT(F_DWORD, WAIT_TIMEOUT); + WINAPI_CONSTANT(F_DWORD, FILE_TYPE_UNKNOWN); + WINAPI_CONSTANT(F_DWORD, FILE_TYPE_DISK); + WINAPI_CONSTANT(F_DWORD, FILE_TYPE_CHAR); + WINAPI_CONSTANT(F_DWORD, FILE_TYPE_PIPE); + WINAPI_CONSTANT(F_DWORD, FILE_TYPE_REMOTE); WINAPI_CONSTANT("i", NULL); diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 431580293e..c625a4d73d 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -941,4 +941,38 @@ _winapi_WriteFile(PyObject *module, PyObject **args, Py_ssize_t nargs, PyObject exit: return return_value; } -/*[clinic end generated code: output=2beb984508fb040a input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_winapi_GetFileType__doc__, +"GetFileType($module, /, handle)\n" +"--\n" +"\n"); + +#define _WINAPI_GETFILETYPE_METHODDEF \ + {"GetFileType", (PyCFunction)_winapi_GetFileType, METH_FASTCALL, _winapi_GetFileType__doc__}, + +static DWORD +_winapi_GetFileType_impl(PyObject *module, HANDLE handle); + +static PyObject * +_winapi_GetFileType(PyObject *module, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"handle", NULL}; + static _PyArg_Parser _parser = {"" F_HANDLE ":GetFileType", _keywords, 0}; + HANDLE handle; + DWORD _return_value; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &handle)) { + goto exit; + } + _return_value = _winapi_GetFileType_impl(module, handle); + if ((_return_value == DWORD_MAX) && PyErr_Occurred()) { + goto exit; + } + return_value = Py_BuildValue("k", _return_value); + +exit: + return return_value; +} +/*[clinic end generated code: output=90c10be97410f00e input=a9049054013a1b77]*/