# HG changeset patch # User Steve Dower # Date 1472617356 25200 # Tue Aug 30 21:22:36 2016 -0700 # Node ID 65c4396853723badffd15c64b4f8825d313a051d # Parent 386aa9738c6bdf3d99e3c76bd00fec0efde09f97 Issue #1602: Windows console doesn't input or print Unicode (PEP 528) Closes #17602: Adds a readline implementation for the Windows console diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -72,6 +72,8 @@ Windows improvements: +* PEP 528: :ref:`Change Windows console encoding to UTF-8 ` + * The ``py.exe`` launcher, when used interactively, no longer prefers Python 2 over Python 3 when the user doesn't specify a version (via command line arguments or a config file). Handling of shebang lines diff --git a/Include/pydebug.h b/Include/pydebug.h --- a/Include/pydebug.h +++ b/Include/pydebug.h @@ -24,6 +24,10 @@ PyAPI_DATA(int) Py_HashRandomizationFlag; PyAPI_DATA(int) Py_IsolatedFlag; +#ifdef MS_WINDOWS +PyAPI_DATA(int) Py_LegacyWindowsStdioFlag; +#endif + /* this is a wrapper around getenv() that pays attention to Py_IgnoreEnvironmentFlag. It should be used for getting variables like PYTHONPATH and PYTHONHOME from the environment */ diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1480,7 +1480,7 @@ singles = ["fchdir", "dup", "fdopen", "fdatasync", "fstat", "fstatvfs", "fsync", "tcgetpgrp", "ttyname"] #singles.append("close") - #We omit close because it doesn'r raise an exception on some platforms + #We omit close because it doesn't raise an exception on some platforms def get_single(f): def helper(self): if hasattr(os, f): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -170,6 +170,8 @@ Windows ------- +- Issue #1602: Windows console doesn't input or print Unicode (PEP 528) + - Issue #27756: Adds new icons for Python files and processes on Windows. Designs by Cherry Wang. @@ -409,7 +411,6 @@ - Issue #10910: Avoid C++ compilation errors on FreeBSD and OS X. Also update FreedBSD version checks for the original ctype UTF-8 workaround. - What's New in Python 3.6.0 alpha 3 ================================== diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -20,6 +20,9 @@ #include #endif /* HAVE_SYS_STAT_H */ +#ifdef MS_WINDOWS +#include +#endif /* Various interned strings */ @@ -52,7 +55,6 @@ PyObject *_PyIO_empty_bytes; PyObject *_PyIO_zero; - PyDoc_STRVAR(module_doc, "The io module provides the Python interfaces to stream handling. The\n" "builtin open function is defined in this module.\n" @@ -362,8 +364,18 @@ } /* Create the Raw file stream */ - raw = PyObject_CallFunction((PyObject *)&PyFileIO_Type, - "OsiO", path_or_fd, rawmode, closefd, opener); + { + PyObject *RawIO_class = (PyObject *)&PyFileIO_Type; +#ifdef MS_WINDOWS + if (!Py_LegacyWindowsStdioFlag && _PyIO_get_console_type(path_or_fd) != '\0') { + RawIO_class = (PyObject *)&PyWindowsConsoleIO_Type; + encoding = "utf-8"; + } +#endif + raw = PyObject_CallFunction(RawIO_class, + "OsiO", path_or_fd, rawmode, closefd, opener); + } + if (raw == NULL) goto error; result = raw; @@ -708,6 +720,12 @@ PyStringIO_Type.tp_base = &PyTextIOBase_Type; ADD_TYPE(&PyStringIO_Type, "StringIO"); +#ifdef MS_WINDOWS + /* WindowsConsoleIO */ + PyWindowsConsoleIO_Type.tp_base = &PyRawIOBase_Type; + ADD_TYPE(&PyWindowsConsoleIO_Type, "WindowsConsoleIO"); +#endif + /* BufferedReader */ PyBufferedReader_Type.tp_base = &PyBufferedIOBase_Type; ADD_TYPE(&PyBufferedReader_Type, "BufferedReader"); diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -19,6 +19,12 @@ extern PyTypeObject PyTextIOWrapper_Type; extern PyTypeObject PyIncrementalNewlineDecoder_Type; +#ifndef Py_LIMITED_API +#ifdef MS_WINDOWS +extern PyTypeObject PyWindowsConsoleIO_Type; +#define PyWindowsConsoleIO_Check(op) (PyObject_TypeCheck((op), &PyWindowsConsoleIO_Type)) +#endif /* MS_WINDOWS */ +#endif /* Py_LIMITED_API */ extern int _PyIO_ConvertSsize_t(PyObject *, void *); @@ -145,6 +151,10 @@ extern _PyIO_State *_PyIO_get_module_state(void); extern PyObject *_PyIO_get_locale_module(_PyIO_State *); +#ifdef MS_WINDOWS +extern char _PyIO_get_console_type(PyObject *); +#endif + extern PyObject *_PyIO_str_close; extern PyObject *_PyIO_str_closed; extern PyObject *_PyIO_str_decode; diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h new file mode 100644 --- /dev/null +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -0,0 +1,330 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io_WindowsConsoleIO_close__doc__, +"close($self, /)\n" +"--\n" +"\n" +"Close the handle.\n" +"\n" +"A closed handle cannot be used for further I/O operations. close() may be\n" +"called more than once without error."); + +#define _IO_WINDOWSCONSOLEIO_CLOSE_METHODDEF \ + {"close", (PyCFunction)_io_WindowsConsoleIO_close, METH_NOARGS, _io_WindowsConsoleIO_close__doc__}, + +static PyObject * +_io_WindowsConsoleIO_close_impl(winconsoleio *self); + +static PyObject * +_io_WindowsConsoleIO_close(winconsoleio *self, PyObject *Py_UNUSED(ignored)) +{ + return _io_WindowsConsoleIO_close_impl(self); +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io_WindowsConsoleIO___init____doc__, +"WindowsConsoleIO(file, mode=\'r\', closefd=True, opener=None)\n" +"--\n" +"\n" +"Open a console buffer by file descriptor.\n" +"\n" +"The mode can be \'rb\' (default), or \'wb\' for reading or writing bytes. All\n" +"other mode characters will be ignored. Mode \'b\' will be assumed if it is\n" +"omitted. The *opener* parameter is always ignored."); + +static int +_io_WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, + const char *mode, int closefd, + PyObject *opener); + +static int +_io_WindowsConsoleIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + static char *_keywords[] = {"file", "mode", "closefd", "opener", NULL}; + PyObject *nameobj; + const char *mode = "r"; + int closefd = 1; + PyObject *opener = Py_None; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|siO:WindowsConsoleIO", _keywords, + &nameobj, &mode, &closefd, &opener)) { + goto exit; + } + return_value = _io_WindowsConsoleIO___init___impl((winconsoleio *)self, nameobj, mode, closefd, opener); + +exit: + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io_WindowsConsoleIO_fileno__doc__, +"fileno($self, /)\n" +"--\n" +"\n" +"Return the underlying file descriptor (an integer).\n" +"\n" +"fileno is only set when a file descriptor is used to open\n" +"one of the standard streams."); + +#define _IO_WINDOWSCONSOLEIO_FILENO_METHODDEF \ + {"fileno", (PyCFunction)_io_WindowsConsoleIO_fileno, METH_NOARGS, _io_WindowsConsoleIO_fileno__doc__}, + +static PyObject * +_io_WindowsConsoleIO_fileno_impl(winconsoleio *self); + +static PyObject * +_io_WindowsConsoleIO_fileno(winconsoleio *self, PyObject *Py_UNUSED(ignored)) +{ + return _io_WindowsConsoleIO_fileno_impl(self); +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io_WindowsConsoleIO_readable__doc__, +"readable($self, /)\n" +"--\n" +"\n" +"True if console is an input buffer."); + +#define _IO_WINDOWSCONSOLEIO_READABLE_METHODDEF \ + {"readable", (PyCFunction)_io_WindowsConsoleIO_readable, METH_NOARGS, _io_WindowsConsoleIO_readable__doc__}, + +static PyObject * +_io_WindowsConsoleIO_readable_impl(winconsoleio *self); + +static PyObject * +_io_WindowsConsoleIO_readable(winconsoleio *self, PyObject *Py_UNUSED(ignored)) +{ + return _io_WindowsConsoleIO_readable_impl(self); +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io_WindowsConsoleIO_writable__doc__, +"writable($self, /)\n" +"--\n" +"\n" +"True if console is an output buffer."); + +#define _IO_WINDOWSCONSOLEIO_WRITABLE_METHODDEF \ + {"writable", (PyCFunction)_io_WindowsConsoleIO_writable, METH_NOARGS, _io_WindowsConsoleIO_writable__doc__}, + +static PyObject * +_io_WindowsConsoleIO_writable_impl(winconsoleio *self); + +static PyObject * +_io_WindowsConsoleIO_writable(winconsoleio *self, PyObject *Py_UNUSED(ignored)) +{ + return _io_WindowsConsoleIO_writable_impl(self); +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io_WindowsConsoleIO_readinto__doc__, +"readinto($self, buffer, /)\n" +"--\n" +"\n" +"Same as RawIOBase.readinto()."); + +#define _IO_WINDOWSCONSOLEIO_READINTO_METHODDEF \ + {"readinto", (PyCFunction)_io_WindowsConsoleIO_readinto, METH_O, _io_WindowsConsoleIO_readinto__doc__}, + +static PyObject * +_io_WindowsConsoleIO_readinto_impl(winconsoleio *self, Py_buffer *buffer); + +static PyObject * +_io_WindowsConsoleIO_readinto(winconsoleio *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_buffer buffer = {NULL, NULL}; + + if (!PyArg_Parse(arg, "w*:readinto", &buffer)) { + goto exit; + } + return_value = _io_WindowsConsoleIO_readinto_impl(self, &buffer); + +exit: + /* Cleanup for buffer */ + if (buffer.obj) { + PyBuffer_Release(&buffer); + } + + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io_WindowsConsoleIO_readall__doc__, +"readall($self, /)\n" +"--\n" +"\n" +"Read all data from the console, returned as bytes.\n" +"\n" +"Return an empty bytes object at EOF."); + +#define _IO_WINDOWSCONSOLEIO_READALL_METHODDEF \ + {"readall", (PyCFunction)_io_WindowsConsoleIO_readall, METH_NOARGS, _io_WindowsConsoleIO_readall__doc__}, + +static PyObject * +_io_WindowsConsoleIO_readall_impl(winconsoleio *self); + +static PyObject * +_io_WindowsConsoleIO_readall(winconsoleio *self, PyObject *Py_UNUSED(ignored)) +{ + return _io_WindowsConsoleIO_readall_impl(self); +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io_WindowsConsoleIO_read__doc__, +"read($self, size=-1, /)\n" +"--\n" +"\n" +"Read at most size bytes, returned as bytes.\n" +"\n" +"Only makes one system call when size is a positive integer,\n" +"so less data may be returned than requested.\n" +"Return an empty bytes object at EOF."); + +#define _IO_WINDOWSCONSOLEIO_READ_METHODDEF \ + {"read", (PyCFunction)_io_WindowsConsoleIO_read, METH_VARARGS, _io_WindowsConsoleIO_read__doc__}, + +static PyObject * +_io_WindowsConsoleIO_read_impl(winconsoleio *self, Py_ssize_t size); + +static PyObject * +_io_WindowsConsoleIO_read(winconsoleio *self, PyObject *args) +{ + PyObject *return_value = NULL; + Py_ssize_t size = -1; + + if (!PyArg_ParseTuple(args, "|O&:read", + _PyIO_ConvertSsize_t, &size)) { + goto exit; + } + return_value = _io_WindowsConsoleIO_read_impl(self, size); + +exit: + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io_WindowsConsoleIO_write__doc__, +"write($self, b, /)\n" +"--\n" +"\n" +"Write buffer b to file, return number of bytes written.\n" +"\n" +"Only makes one system call, so not all of the data may be written.\n" +"The number of bytes actually written is returned."); + +#define _IO_WINDOWSCONSOLEIO_WRITE_METHODDEF \ + {"write", (PyCFunction)_io_WindowsConsoleIO_write, METH_O, _io_WindowsConsoleIO_write__doc__}, + +static PyObject * +_io_WindowsConsoleIO_write_impl(winconsoleio *self, Py_buffer *b); + +static PyObject * +_io_WindowsConsoleIO_write(winconsoleio *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_buffer b = {NULL, NULL}; + + if (!PyArg_Parse(arg, "y*:write", &b)) { + goto exit; + } + return_value = _io_WindowsConsoleIO_write_impl(self, &b); + +exit: + /* Cleanup for b */ + if (b.obj) { + PyBuffer_Release(&b); + } + + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_io_WindowsConsoleIO_isatty__doc__, +"isatty($self, /)\n" +"--\n" +"\n" +"Always True."); + +#define _IO_WINDOWSCONSOLEIO_ISATTY_METHODDEF \ + {"isatty", (PyCFunction)_io_WindowsConsoleIO_isatty, METH_NOARGS, _io_WindowsConsoleIO_isatty__doc__}, + +static PyObject * +_io_WindowsConsoleIO_isatty_impl(winconsoleio *self); + +static PyObject * +_io_WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) +{ + return _io_WindowsConsoleIO_isatty_impl(self); +} + +#endif /* defined(MS_WINDOWS) */ + +#ifndef _IO_WINDOWSCONSOLEIO_CLOSE_METHODDEF + #define _IO_WINDOWSCONSOLEIO_CLOSE_METHODDEF +#endif /* !defined(_IO_WINDOWSCONSOLEIO_CLOSE_METHODDEF) */ + +#ifndef _IO_WINDOWSCONSOLEIO_FILENO_METHODDEF + #define _IO_WINDOWSCONSOLEIO_FILENO_METHODDEF +#endif /* !defined(_IO_WINDOWSCONSOLEIO_FILENO_METHODDEF) */ + +#ifndef _IO_WINDOWSCONSOLEIO_READABLE_METHODDEF + #define _IO_WINDOWSCONSOLEIO_READABLE_METHODDEF +#endif /* !defined(_IO_WINDOWSCONSOLEIO_READABLE_METHODDEF) */ + +#ifndef _IO_WINDOWSCONSOLEIO_WRITABLE_METHODDEF + #define _IO_WINDOWSCONSOLEIO_WRITABLE_METHODDEF +#endif /* !defined(_IO_WINDOWSCONSOLEIO_WRITABLE_METHODDEF) */ + +#ifndef _IO_WINDOWSCONSOLEIO_READINTO_METHODDEF + #define _IO_WINDOWSCONSOLEIO_READINTO_METHODDEF +#endif /* !defined(_IO_WINDOWSCONSOLEIO_READINTO_METHODDEF) */ + +#ifndef _IO_WINDOWSCONSOLEIO_READALL_METHODDEF + #define _IO_WINDOWSCONSOLEIO_READALL_METHODDEF +#endif /* !defined(_IO_WINDOWSCONSOLEIO_READALL_METHODDEF) */ + +#ifndef _IO_WINDOWSCONSOLEIO_READ_METHODDEF + #define _IO_WINDOWSCONSOLEIO_READ_METHODDEF +#endif /* !defined(_IO_WINDOWSCONSOLEIO_READ_METHODDEF) */ + +#ifndef _IO_WINDOWSCONSOLEIO_WRITE_METHODDEF + #define _IO_WINDOWSCONSOLEIO_WRITE_METHODDEF +#endif /* !defined(_IO_WINDOWSCONSOLEIO_WRITE_METHODDEF) */ + +#ifndef _IO_WINDOWSCONSOLEIO_ISATTY_METHODDEF + #define _IO_WINDOWSCONSOLEIO_ISATTY_METHODDEF +#endif /* !defined(_IO_WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ +/*[clinic end generated code: output=37c16b7dc1e03513 input=a9049054013a1b77]*/ diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c new file mode 100644 --- /dev/null +++ b/Modules/_io/winconsoleio.c @@ -0,0 +1,970 @@ +/* + An implementation of Windows console I/O + + Classes defined here: WinConsoleReader, WinConsoleWriter + + Written by Steve Dower +*/ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" + +#ifdef MS_WINDOWS + +#include "structmember.h" +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif +#include /* For offsetof */ + +#define WIN32_LEAN_AND_MEAN +#include + +#include "_iomodule.h" + +/* BUFSIZ determines how many characters can be types at the console + before it starts blocking. */ +#if BUFSIZ < (16*1024) +#define SMALLCHUNK (16*1024) +#elif (BUFSIZ >= (2 << 25)) +#error "unreasonable BUFSIZ > 64MB defined" +#else +#define SMALLCHUNK BUFSIZ +#endif + +char _get_console_type(HANDLE handle) { + DWORD mode, peek_count; + + if (handle == INVALID_HANDLE_VALUE) + return '\0'; + + if (!GetConsoleMode(handle, &mode)) + return '\0'; + + /* Peek at the handle to see whether it is an input or output handle */ + if (GetNumberOfConsoleInputEvents(handle, &peek_count)) + return 'r'; + return 'w'; +} + +char _PyIO_get_console_type(PyObject *path_or_fd) { + int fd; + + fd = PyLong_AsLong(path_or_fd); + PyErr_Clear(); + if (fd >= 0) { + HANDLE handle; + _Py_BEGIN_SUPPRESS_IPH + handle = (HANDLE)_get_osfhandle(fd); + _Py_END_SUPPRESS_IPH + if (!handle) + return '\0'; + return _get_console_type(handle); + } + + /* TODO: Support opening console using name */ + return '\0'; +} + +/*[clinic input] +module _io +class _io.WindowsConsoleIO "winconsoleio *" "&PyWindowsConsoleIO_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=431722d0fd743878]*/ + +/*[python input] +class io_ssize_t_converter(CConverter): + type = 'Py_ssize_t' + converter = '_PyIO_ConvertSsize_t' +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=d0a811d3cbfd1b33]*/ + +typedef struct { + PyObject_HEAD + HANDLE handle; + unsigned int created : 1; + unsigned int readable : 1; + unsigned int writable : 1; + unsigned int closehandle : 1; + char finalizing; + unsigned int blksize; + unsigned int fd; + PyObject *weakreflist; + PyObject *dict; + char buf[4]; +} winconsoleio; + +PyTypeObject PyWindowsConsoleIO_Type; + +_Py_IDENTIFIER(name); + +int +_PyWindowsConsoleIO_closed(PyObject *self) +{ + return ((winconsoleio *)self)->handle == INVALID_HANDLE_VALUE; +} + + +/* Returns 0 on success, -1 with exception set on failure. */ +static int +internal_close(winconsoleio *self) +{ + if (self->handle != INVALID_HANDLE_VALUE) { + if (self->closehandle) { + CloseHandle(self->handle); + } + self->handle = INVALID_HANDLE_VALUE; + } + return 0; +} + +/*[clinic input] +_io.WindowsConsoleIO.close + +Close the handle. + +A closed handle cannot be used for further I/O operations. close() may be +called more than once without error. +[clinic start generated code]*/ + +static PyObject * +_io_WindowsConsoleIO_close_impl(winconsoleio *self) +/*[clinic end generated code: output=222098bea11857e5 input=5cdeb454f05e5b9c]*/ +{ + PyObject *res; + PyObject *exc, *val, *tb; + int rc; + _Py_IDENTIFIER(close); + res = _PyObject_CallMethodId((PyObject*)&PyRawIOBase_Type, + &PyId_close, "O", self); + if (!self->closehandle) { + self->handle = INVALID_HANDLE_VALUE; + return res; + } + if (res == NULL) + PyErr_Fetch(&exc, &val, &tb); + rc = internal_close(self); + if (res == NULL) + _PyErr_ChainExceptions(exc, val, tb); + if (rc < 0) + Py_CLEAR(res); + return res; +} + +static PyObject * +winconsoleio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + winconsoleio *self; + + assert(type != NULL && type->tp_alloc != NULL); + + self = (winconsoleio *) type->tp_alloc(type, 0); + if (self != NULL) { + self->handle = INVALID_HANDLE_VALUE; + self->created = 0; + self->readable = 0; + self->writable = 0; + self->closehandle = 0; + self->blksize = 0; + self->weakreflist = NULL; + } + + return (PyObject *) self; +} + +/*[clinic input] +_io.WindowsConsoleIO.__init__ + file as nameobj: object + mode: str = "r" + closefd: int(c_default="1") = True + opener: object = None + +Open a console buffer by file descriptor. + +The mode can be 'rb' (default), or 'wb' for reading or writing bytes. All +other mode characters will be ignored. Mode 'b' will be assumed if it is +omitted. The *opener* parameter is always ignored. +[clinic start generated code]*/ + +static int +_io_WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, + const char *mode, int closefd, + PyObject *opener) +/*[clinic end generated code: output=d75a8c3247929b4e input=83eae0cdf4b7e95f]*/ +{ + const char *name = NULL; + PyObject *stringobj = NULL; + const char *s; + Py_UNICODE *widename = NULL; + int ret = 0; + int rwa = 0; + int fd = -1; + int fd_is_own = 0; + + assert(PyWindowsConsoleIO_Check(self)); + if (self->handle >= 0) { + if (self->closehandle) { + /* Have to close the existing file first. */ + if (internal_close(self) < 0) + return -1; + } + else + self->handle = INVALID_HANDLE_VALUE; + } + + if (PyFloat_Check(nameobj)) { + PyErr_SetString(PyExc_TypeError, + "integer argument expected, got float"); + return -1; + } + + fd = _PyLong_AsInt(nameobj); + if (fd < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "negative file descriptor"); + return -1; + } + PyErr_Clear(); + } + self->fd = fd; + + if (PyUnicode_Check(nameobj)) { + Py_ssize_t length; + widename = PyUnicode_AsUnicodeAndSize(nameobj, &length); + if (widename == NULL) + return -1; + if (wcslen(widename) != length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + return -1; + } + } else if (fd < 0) { + if (!PyUnicode_FSConverter(nameobj, &stringobj)) { + return -1; + } + name = PyBytes_AS_STRING(stringobj); + } + + s = mode; + while (*s) { + switch (*s++) { + case '+': + case 'a': + case 'b': + case 'x': + break; + case 'r': + if (rwa) + goto bad_mode; + rwa = 1; + self->readable = 1; + break; + case 'w': + if (rwa) + goto bad_mode; + rwa = 1; + self->writable = 1; + break; + default: + PyErr_Format(PyExc_ValueError, + "invalid mode: %.200s", mode); + goto error; + } + } + + if (!rwa) + goto bad_mode; + + if (fd >= 0) { + _Py_BEGIN_SUPPRESS_IPH + self->handle = (HANDLE)_get_osfhandle(fd); + _Py_END_SUPPRESS_IPH + self->closehandle = 0; + } + else { + DWORD access = GENERIC_READ; + + self->closehandle = 1; + if (!closefd) { + PyErr_SetString(PyExc_ValueError, + "Cannot use closefd=False with file name"); + goto error; + } + + if (self->writable) + access |= GENERIC_WRITE; + + Py_BEGIN_ALLOW_THREADS + if (widename != NULL) + self->handle = CreateFileW(widename, access, + FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL); + else + self->handle = CreateFileA(name, access, + FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL); + Py_END_ALLOW_THREADS + + if (self->handle == INVALID_HANDLE_VALUE) { + PyErr_SetExcFromWindowsErrWithFilenameObject(PyExc_OSError, GetLastError(), nameobj); + goto error; + } + } + + if (self->writable && _get_console_type(self->handle) != 'w') { + PyErr_SetString(PyExc_ValueError, + "Cannot open console input buffer for writing"); + goto error; + } + if (self->readable && _get_console_type(self->handle) != 'r') { + PyErr_SetString(PyExc_ValueError, + "Cannot open console output buffer for reading"); + goto error; + } + + self->blksize = DEFAULT_BUFFER_SIZE; + memset(self->buf, 0, 4); + + if (_PyObject_SetAttrId((PyObject *)self, &PyId_name, nameobj) < 0) + goto error; + + goto done; + +bad_mode: + PyErr_SetString(PyExc_ValueError, + "Must have exactly one of read or write mode"); +error: + ret = -1; + internal_close(self); + +done: + Py_CLEAR(stringobj); + return ret; +} + +static int +winconsoleio_traverse(winconsoleio *self, visitproc visit, void *arg) +{ + Py_VISIT(self->dict); + return 0; +} + +static int +winconsoleio_clear(winconsoleio *self) +{ + Py_CLEAR(self->dict); + return 0; +} + +static void +winconsoleio_dealloc(winconsoleio *self) +{ + self->finalizing = 1; + if (_PyIOBase_finalize((PyObject *) self) < 0) + return; + _PyObject_GC_UNTRACK(self); + if (self->weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *) self); + Py_CLEAR(self->dict); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyObject * +err_closed(void) +{ + PyErr_SetString(PyExc_ValueError, "I/O operation on closed file"); + return NULL; +} + +static PyObject * +err_mode(const char *action) +{ + _PyIO_State *state = IO_STATE(); + if (state != NULL) + PyErr_Format(state->unsupported_operation, + "Console buffer does not support %s", action); + return NULL; +} + +/*[clinic input] +_io.WindowsConsoleIO.fileno + +Return the underlying file descriptor (an integer). + +fileno is only set when a file descriptor is used to open +one of the standard streams. + +[clinic start generated code]*/ + +static PyObject * +_io_WindowsConsoleIO_fileno_impl(winconsoleio *self) +/*[clinic end generated code: output=8ce2675b8b2e70aa input=5abcbca2c1a39670]*/ +{ + if (self->fd < 0) + return err_mode("fileno"); + return PyLong_FromLong(self->fd); +} + +/*[clinic input] +_io.WindowsConsoleIO.readable + +True if console is an input buffer. +[clinic start generated code]*/ + +static PyObject * +_io_WindowsConsoleIO_readable_impl(winconsoleio *self) +/*[clinic end generated code: output=8cc7d865bea4671b input=9cffdf2ab99ca138]*/ +{ + if (self->handle == INVALID_HANDLE_VALUE) + return err_closed(); + return PyBool_FromLong((long) self->readable); +} + +/*[clinic input] +_io.WindowsConsoleIO.writable + +True if console is an output buffer. +[clinic start generated code]*/ + +static PyObject * +_io_WindowsConsoleIO_writable_impl(winconsoleio *self) +/*[clinic end generated code: output=480ef155e305ed74 input=940422ab3d91c8d2]*/ +{ + if (self->handle == INVALID_HANDLE_VALUE) + return err_closed(); + return PyBool_FromLong((long) self->writable); +} + +static Py_ssize_t +_buflen(winconsoleio *self) +{ + for (int i = 0; i < 4; ++i) { + if (!self->buf[i]) + return i; + } + return 4; +} + +static Py_ssize_t +_copyfrombuf(winconsoleio *self, char *buf, Py_ssize_t len) +{ + Py_ssize_t n = 0; + + while (self->buf[0] && len--) { + n += 1; + buf[0] = self->buf[0]; + self->buf[0] = self->buf[1]; + self->buf[1] = self->buf[2]; + self->buf[2] = self->buf[3]; + self->buf[3] = 0; + } + + return n; +} + +static Py_ssize_t +readinto(winconsoleio *self, char *buf, Py_ssize_t len) +{ + BOOL res; + wchar_t *wbuf; + DWORD n, wlen, read_len = 0; + + if (self->handle == INVALID_HANDLE_VALUE) { + err_closed(); + return -1; + } + if (!self->readable) { + err_mode("reading"); + return -1; + } + if (len == 0) + return 0; + + /* Each character may take up to 4 bytes in the final buffer. + This is highly conservative, but necessary to avoid + failure for any given Unicode input (e.g. \U0010ffff). + If the caller requests fewer than 4 bytes, we buffer one + character. + */ + wlen = len / 4; + if (wlen == 0) { + wlen = 1; + } + + read_len = _copyfrombuf(self, buf, len); + if (read_len) { + buf = &buf[read_len]; + len -= read_len; + wlen -= 1; + } + if (len == read_len || wlen == 0) + return read_len; + + wbuf = (wchar_t*)PyMem_Malloc(wlen * sizeof(wchar_t)); + + Py_BEGIN_ALLOW_THREADS + res = ReadConsoleW(self->handle, wbuf, wlen, &n, NULL); + + if (res) { + if (len < 4) { + if (WideCharToMultiByte(CP_UTF8, 0, wbuf, n, + self->buf, sizeof(self->buf) / sizeof(self->buf[0]), + NULL, NULL)) + read_len = _copyfrombuf(self, buf, len); + } else { + read_len = WideCharToMultiByte(CP_UTF8, 0, wbuf, n, + buf, len, NULL, NULL); + } + if (!read_len) { + if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + /* Calculate the needed buffer for a more useful error, as this + means our "/ 4" logic above is insufficient for some input. + */ + read_len = WideCharToMultiByte(CP_UTF8, 0, wbuf, n, + NULL, 0, NULL, NULL); + } + res = 0; + } + } + Py_END_ALLOW_THREADS + + PyMem_Free(wbuf); + + if (!res) { + if (read_len) + PyErr_Format(PyExc_SystemError, + "Buffer had room for %d bytes but %d bytes required", + len, read_len); + else + PyErr_SetFromWindowsErr(0); + return -1; + } + + return read_len; +} + +/*[clinic input] +_io.WindowsConsoleIO.readinto + buffer: Py_buffer(accept={rwbuffer}) + / + +Same as RawIOBase.readinto(). +[clinic start generated code]*/ + +static PyObject * +_io_WindowsConsoleIO_readinto_impl(winconsoleio *self, Py_buffer *buffer) +/*[clinic end generated code: output=ccf1ee4b17df834d input=8c5b9abb7c2dba64]*/ +{ + Py_ssize_t len = readinto(self, buffer->buf, buffer->len); + if (len < 0) + return NULL; + + return PyLong_FromSsize_t(len); +} + +static DWORD +new_buffersize(winconsoleio *self, DWORD currentsize) +{ + DWORD addend; + + /* Expand the buffer by an amount proportional to the current size, + giving us amortized linear-time behavior. For bigger sizes, use a + less-than-double growth factor to avoid excessive allocation. */ + if (currentsize > 65536) + addend = currentsize >> 3; + else + addend = 256 + currentsize; + if (addend < SMALLCHUNK) + /* Avoid tiny read() calls. */ + addend = SMALLCHUNK; + return addend + currentsize; +} + +/*[clinic input] +_io.WindowsConsoleIO.readall + +Read all data from the console, returned as bytes. + +Return an empty bytes object at EOF. +[clinic start generated code]*/ + +static PyObject * +_io_WindowsConsoleIO_readall_impl(winconsoleio *self) +/*[clinic end generated code: output=524a2ffc0b46dc4e input=f02b3ad8e47b0417]*/ +{ + BOOL res; + wchar_t *buf; + DWORD bufsize, n, len = 0; + PyObject *bytes; + Py_ssize_t bytes_size, rn; + + if (self->handle == INVALID_HANDLE_VALUE) + return err_closed(); + + bufsize = SMALLCHUNK; + + buf = (wchar_t*)PyMem_Malloc(bufsize * sizeof(wchar_t)); + if (buf == NULL) + return NULL; + + while (1) { + if (len >= (Py_ssize_t)bufsize) { + DWORD newsize = new_buffersize(self, len); + if (newsize < bufsize) { + PyErr_SetString(PyExc_OverflowError, + "unbounded read returned more bytes " + "than a Python bytes object can hold"); + PyMem_Free(buf); + return NULL; + } + bufsize = newsize; + + buf = PyMem_Realloc(buf, bufsize * sizeof(wchar_t)); + if (!buf) { + PyMem_Free(buf); + return NULL; + } + } + + Py_BEGIN_ALLOW_THREADS + res = ReadConsoleW(self->handle, buf + len, bufsize - len, &n, NULL); + Py_END_ALLOW_THREADS + + /* Operation may be aborted without failing the call */ + if (!res || n == 0) { + int err = GetLastError(); + if (err == ERROR_OPERATION_ABORTED) { + HANDLE hInterruptEvent = _PyOS_SigintEvent(); + PyMem_Free(buf); + if (WaitForSingleObjectEx(hInterruptEvent, 10, FALSE) == WAIT_OBJECT_0) { + ResetEvent(hInterruptEvent); + PyErr_SetNone(PyExc_KeyboardInterrupt); + } else { + PyErr_SetFromWindowsErr(err); + } + return NULL; + } else if (!res) { + PyMem_Free(buf); + return PyErr_SetFromWindowsErr(err); + } + } + + if (n == 0 || buf[len] == '\x1a') + break; + + len += n; + } + + if (len > 0 && buf[0] == '\x1a') { + PyMem_Free(buf); + return PyBytes_FromStringAndSize(NULL, 0); + } + + Py_BEGIN_ALLOW_THREADS + bytes_size = WideCharToMultiByte(CP_UTF8, 0, buf, len, + NULL, 0, NULL, NULL); + Py_END_ALLOW_THREADS + + if (!bytes_size) { + DWORD err = GetLastError(); + PyMem_Free(buf); + return PyErr_SetFromWindowsErr(err); + } + + bytes_size += _buflen(self); + bytes = PyBytes_FromStringAndSize(NULL, bytes_size); + rn = _copyfrombuf(self, bytes, bytes_size); + + Py_BEGIN_ALLOW_THREADS + bytes_size = WideCharToMultiByte(CP_UTF8, 0, buf, len, + &PyBytes_AS_STRING(bytes)[rn], bytes_size - rn, NULL, NULL); + Py_END_ALLOW_THREADS + + if (!bytes_size) { + DWORD err = GetLastError(); + PyMem_Free(buf); + Py_CLEAR(bytes); + return PyErr_SetFromWindowsErr(err); + } + + PyMem_Free(buf); + if (bytes_size < PyBytes_GET_SIZE(bytes)) { + if (_PyBytes_Resize(&bytes, n * sizeof(wchar_t)) < 0) { + Py_CLEAR(bytes); + return NULL; + } + } + return bytes; +} + +/*[clinic input] +_io.WindowsConsoleIO.read + size: io_ssize_t = -1 + / + +Read at most size bytes, returned as bytes. + +Only makes one system call when size is a positive integer, +so less data may be returned than requested. +Return an empty bytes object at EOF. +[clinic start generated code]*/ + +static PyObject * +_io_WindowsConsoleIO_read_impl(winconsoleio *self, Py_ssize_t size) +/*[clinic end generated code: output=8300f301cef9e00c input=a5c2d9aa1d03f708]*/ +{ + PyObject *bytes; + Py_ssize_t bytes_size; + + if (self->handle == INVALID_HANDLE_VALUE) + return err_closed(); + if (!self->readable) + return err_mode("reading"); + + if (size < 0) + return _io_WindowsConsoleIO_readall_impl(self); + if (size > ULONG_MAX) + size = ULONG_MAX; + + bytes = PyBytes_FromStringAndSize(NULL, size); + if (bytes == NULL) + return NULL; + + bytes_size = readinto(self, PyBytes_AS_STRING(bytes), PyBytes_GET_SIZE(bytes)); + if (bytes_size < 0) { + Py_CLEAR(bytes); + return NULL; + } + + if (bytes_size < PyBytes_GET_SIZE(bytes)) { + if (_PyBytes_Resize(&bytes, bytes_size) < 0) { + Py_CLEAR(bytes); + return NULL; + } + } + + return bytes; +} + +/*[clinic input] +_io.WindowsConsoleIO.write + b: Py_buffer + / + +Write buffer b to file, return number of bytes written. + +Only makes one system call, so not all of the data may be written. +The number of bytes actually written is returned. +[clinic start generated code]*/ + +static PyObject * +_io_WindowsConsoleIO_write_impl(winconsoleio *self, Py_buffer *b) +/*[clinic end generated code: output=931b5512f7615d6f input=fe61e17f6cd6781b]*/ +{ + BOOL res = TRUE; + wchar_t *wbuf; + DWORD len, wlen, n = 0; + + if (self->handle == INVALID_HANDLE_VALUE) + return err_closed(); + if (!self->writable) + return err_mode("writing"); + + len = b->len; + + Py_BEGIN_ALLOW_THREADS + wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, NULL, 0); + + /* issue11395 there is an unspecified upper bound on how many bytes + can be written at once. We cap at 32k - the caller will have to + handle partial writes. + Since we don't know how many input bytes are being ignored, we + have to reduce and recalculate. */ + while (wlen > 32766 / sizeof(wchar_t)) { + len /= 2; + wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, NULL, 0); + } + Py_END_ALLOW_THREADS + + if (!wlen) + return PyErr_SetFromWindowsErr(0); + + wbuf = (wchar_t*)PyMem_Malloc(wlen * sizeof(wchar_t)); + + Py_BEGIN_ALLOW_THREADS + wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, wbuf, wlen); + if (wlen) { + res = WriteConsoleW(self->handle, wbuf, wlen, &n, NULL); + if (n < wlen) { + /* Wrote fewer characters than expected, which means our + * len value may be wrong. So recalculate it from the + * characters that were written. As this could potentially + * result in a different value, we also validate that value. + */ + len = WideCharToMultiByte(CP_UTF8, 0, wbuf, n, + NULL, 0, NULL, NULL); + if (len) { + wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, + NULL, 0); + assert(wlen == len); + } + } + } else + res = 0; + Py_END_ALLOW_THREADS + + if (!res) { + DWORD err = GetLastError(); + PyMem_Free(wbuf); + return PyErr_SetFromWindowsErr(err); + } + + PyMem_Free(wbuf); + return PyLong_FromSsize_t(len); +} + +static PyObject * +winconsoleio_repr(winconsoleio *self) +{ + if (self->handle == INVALID_HANDLE_VALUE) + return PyUnicode_FromFormat("<_io.WindowsConsoleIO [closed]>"); + + if (self->readable) + return PyUnicode_FromFormat("<_io.WindowsConsoleIO mode='rb' closefd=%s>", + self->closehandle ? "True" : "False"); + if (self->writable) + return PyUnicode_FromFormat("<_io.WindowsConsoleIO mode='wb' closefd=%s>", + self->closehandle ? "True" : "False"); + + PyErr_SetString(PyExc_SystemError, "WindowsConsoleIO has invalid mode"); + return NULL; +} + +/*[clinic input] +_io.WindowsConsoleIO.isatty + +Always True. +[clinic start generated code]*/ + +static PyObject * +_io_WindowsConsoleIO_isatty_impl(winconsoleio *self) +/*[clinic end generated code: output=ee9b776418c2b0dd input=97521b5fd13237ca]*/ +{ + if (self->handle == INVALID_HANDLE_VALUE) + return err_closed(); + + Py_RETURN_TRUE; +} + +static PyObject * +winconsoleio_getstate(winconsoleio *self) +{ + PyErr_Format(PyExc_TypeError, + "cannot serialize '%s' object", Py_TYPE(self)->tp_name); + return NULL; +} + +#include "clinic/winconsoleio.c.h" + +static PyMethodDef winconsoleio_methods[] = { + _IO_WINDOWSCONSOLEIO_READ_METHODDEF + _IO_WINDOWSCONSOLEIO_READALL_METHODDEF + _IO_WINDOWSCONSOLEIO_READINTO_METHODDEF + _IO_WINDOWSCONSOLEIO_WRITE_METHODDEF + _IO_WINDOWSCONSOLEIO_CLOSE_METHODDEF + _IO_WINDOWSCONSOLEIO_READABLE_METHODDEF + _IO_WINDOWSCONSOLEIO_WRITABLE_METHODDEF + _IO_WINDOWSCONSOLEIO_FILENO_METHODDEF + _IO_WINDOWSCONSOLEIO_ISATTY_METHODDEF + {"__getstate__", (PyCFunction)winconsoleio_getstate, METH_NOARGS, NULL}, + {NULL, NULL} /* sentinel */ +}; + +/* 'closed' and 'mode' are attributes for compatibility with FileIO. */ + +static PyObject * +get_closed(winconsoleio *self, void *closure) +{ + return PyBool_FromLong((long)(self->handle == INVALID_HANDLE_VALUE)); +} + +static PyObject * +get_closefd(winconsoleio *self, void *closure) +{ + return PyBool_FromLong((long)(self->closehandle)); +} + +static PyObject * +get_mode(winconsoleio *self, void *closure) +{ + return PyUnicode_FromString(self->readable ? "rb" : "wb"); +} + +static PyGetSetDef winconsoleio_getsetlist[] = { + {"closed", (getter)get_closed, NULL, "True if the file is closed"}, + {"closefd", (getter)get_closefd, NULL, + "True if the file descriptor will be closed by close()."}, + {"mode", (getter)get_mode, NULL, "String giving the file mode"}, + {NULL}, +}; + +static PyMemberDef winconsoleio_members[] = { + {"_blksize", T_UINT, offsetof(winconsoleio, blksize), 0}, + {"_finalizing", T_BOOL, offsetof(winconsoleio, finalizing), 0}, + {NULL} +}; + +PyTypeObject PyWindowsConsoleIO_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_io.WindowsConsoleIO", + sizeof(winconsoleio), + 0, + (destructor)winconsoleio_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + (reprfunc)winconsoleio_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */ + _io_WindowsConsoleIO___init____doc__, /* tp_doc */ + (traverseproc)winconsoleio_traverse, /* tp_traverse */ + (inquiry)winconsoleio_clear, /* tp_clear */ + 0, /* tp_richcompare */ + offsetof(winconsoleio, weakreflist), /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + winconsoleio_methods, /* tp_methods */ + winconsoleio_members, /* tp_members */ + winconsoleio_getsetlist, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + offsetof(winconsoleio, dict), /* tp_dictoffset */ + _io_WindowsConsoleIO___init__, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + winconsoleio_new, /* tp_new */ + PyObject_GC_Del, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + 0, /* tp_finalize */ +}; + +#endif /* MS_WINDOWS */ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -266,6 +266,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -593,6 +593,9 @@ Modules\_io + + Modules\_io + Modules\_io diff --git a/Parser/myreadline.c b/Parser/myreadline.c --- a/Parser/myreadline.c +++ b/Parser/myreadline.c @@ -10,6 +10,7 @@ */ #include "Python.h" + #ifdef MS_WINDOWS #define WIN32_LEAN_AND_MEAN #include "windows.h" @@ -101,6 +102,100 @@ /* NOTREACHED */ } +#ifdef MS_WINDOWS +/* Readline implementation using ReadConsoleW */ + +extern char _get_console_type(HANDLE handle); + +char * +_PyOS_WindowsConsoleReadline(HANDLE hStdIn) +{ + static wchar_t wbuf_local[1024 * 32]; + const DWORD chunk_size = 1024; + + DWORD n_read, total_read, wbuflen, u8len; + wchar_t *wbuf; + char *buf = NULL; + int err = 0; + + n_read = 0; + total_read = 0; + wbuf = wbuf_local; + wbuflen = sizeof(wbuf_local) / sizeof(wbuf_local[0]) - 1; + while (1) { + if (!ReadConsoleW(hStdIn, &wbuf[total_read], wbuflen - total_read, &n_read, NULL)) { + err = GetLastError(); + goto exit; + } + if (n_read == 0) { + int s; + err = GetLastError(); + if (err != ERROR_OPERATION_ABORTED) + goto exit; + err = 0; + HANDLE hInterruptEvent = _PyOS_SigintEvent(); + if (WaitForSingleObjectEx(hInterruptEvent, 100, FALSE) + == WAIT_OBJECT_0) { + ResetEvent(hInterruptEvent); +#ifdef WITH_THREAD + PyEval_RestoreThread(_PyOS_ReadlineTState); +#endif + s = PyErr_CheckSignals(); +#ifdef WITH_THREAD + PyEval_SaveThread(); +#endif + if (s < 0) + goto exit; + } + break; + } + + total_read += n_read; + if (total_read == 0 || wbuf[total_read - 1] == L'\n') { + break; + } + wbuflen += chunk_size; + if (wbuf == wbuf_local) { + wbuf[total_read] = '\0'; + wbuf = (wchar_t*)PyMem_RawMalloc(wbuflen * sizeof(wchar_t)); + if (wbuf) + wcscpy_s(wbuf, wbuflen, wbuf_local); + } + else + wbuf = (wchar_t*)PyMem_RawRealloc(wbuf, wbuflen * sizeof(wchar_t)); + } + + if (wbuf[0] == '\x1a') { + buf = PyMem_RawMalloc(1); + if (buf) + buf[0] = '\0'; + goto exit; + } + + u8len = WideCharToMultiByte(CP_UTF8, 0, wbuf, total_read, NULL, 0, NULL, NULL); + buf = PyMem_RawMalloc(u8len + 1); + u8len = WideCharToMultiByte(CP_UTF8, 0, wbuf, total_read, buf, u8len, NULL, NULL); + buf[u8len] = '\0'; + +exit: + if (wbuf != wbuf_local) + PyMem_RawFree(wbuf); + + if (err) { +#ifdef WITH_THREAD + PyEval_RestoreThread(_PyOS_ReadlineTState); +#endif + PyErr_SetFromWindowsErr(err); +#ifdef WITH_THREAD + PyEval_SaveThread(); +#endif + } + + return buf; +} + +#endif + /* Readline implementation using fgets() */ @@ -110,6 +205,25 @@ size_t n; char *p, *pr; +#ifdef MS_WINDOWS + if (!Py_LegacyWindowsStdioFlag && sys_stdin == stdin) { + HANDLE hStdIn; + + _Py_BEGIN_SUPPRESS_IPH + hStdIn = (HANDLE)_get_osfhandle(fileno(sys_stdin)); + _Py_END_SUPPRESS_IPH + + if (_get_console_type(hStdIn) == 'r') { + fflush(sys_stdout); + if (prompt) + fprintf(stderr, "%s", prompt); + fflush(stderr); + clearerr(sys_stdin); + return _PyOS_WindowsConsoleReadline(hStdIn); + } + } +#endif + n = 100; p = (char *)PyMem_RawMalloc(n); if (p == NULL) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -31,6 +31,9 @@ #ifdef MS_WINDOWS #undef BYTE #include "windows.h" + +extern PyTypeObject PyWindowsConsoleIO_Type; +#define PyWindowsConsoleIO_Check(op) (PyObject_TypeCheck((op), &PyWindowsConsoleIO_Type)) #endif _Py_IDENTIFIER(flush); @@ -90,6 +93,9 @@ int Py_UnbufferedStdioFlag = 0; /* Unbuffered binary std{in,out,err} */ int Py_HashRandomizationFlag = 0; /* for -R and PYTHONHASHSEED */ int Py_IsolatedFlag = 0; /* for -I, isolate from user's env */ +#ifdef MS_WINDOWS +int Py_LegacyWindowsStdioFlag = 0; /* Uses FileIO instead of WindowsConsoleIO */ +#endif PyThreadState *_Py_Finalizing = NULL; @@ -151,6 +157,12 @@ return -3; } } +#ifdef MS_WINDOWS + if (_Py_StandardStreamEncoding) { + /* Overriding the stream encoding implies legacy streams */ + Py_LegacyWindowsStdioFlag = 1; + } +#endif return 0; } @@ -321,6 +333,10 @@ check its value further. */ if ((p = Py_GETENV("PYTHONHASHSEED")) && *p != '\0') Py_HashRandomizationFlag = add_flag(Py_HashRandomizationFlag, p); +#ifdef MS_WINDOWS + if ((p = Py_GETENV("PYTHONLEGACYWINDOWSSTDIO")) && *p != '\0') + Py_LegacyWindowsStdioFlag = add_flag(Py_LegacyWindowsStdioFlag, p); +#endif _PyRandom_Init(); @@ -1069,6 +1085,12 @@ Py_INCREF(raw); } +#ifdef MS_WINDOWS + /* Windows console IO is always UTF-8 encoded */ + if (PyWindowsConsoleIO_Check(raw)) + encoding = "utf-8"; +#endif + text = PyUnicode_FromString(name); if (text == NULL || _PyObject_SetAttrId(raw, &PyId_name, text) < 0) goto error;