# HG changeset patch # User Steve Dower # Date 1475255623 14400 # Fri Sep 30 13:13:43 2016 -0400 # Branch 3.6 # Node ID e4eea8d0a034feec9a703ca155cf04194e62605a # Parent 08119dc47cffa2af6ff37a8db467e63552504fb5 Issue #28217: Adds _testconsole module to test console input. Fixes some issues found by the tests. diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_winconsoleio.py --- a/Lib/test/test_winconsoleio.py +++ b/Lib/test/test_winconsoleio.py @@ -1,12 +1,4 @@ '''Tests for WindowsConsoleIO - -Unfortunately, most testing requires interactive use, since we have no -API to read back from a real console, and this class is only for use -with real consoles. - -Instead, we validate that basic functionality such as opening, closing -and in particular fileno() work, but are forced to leave real testing -to real people with real keyborads. ''' import io @@ -16,6 +8,8 @@ if sys.platform != 'win32': raise unittest.SkipTest("test only relevant on win32") +from _testconsole import write_input + ConIO = io._WindowsConsoleIO class WindowsConsoleIOTests(unittest.TestCase): @@ -83,5 +77,65 @@ f.close() f.close() + def assertStdinRoundTrip(self, text): + stdin = open('CONIN$', 'r') + old_stdin = sys.stdin + try: + sys.stdin = stdin + write_input( + stdin.buffer.raw, + (text + '\r\n').encode('utf-16-le', 'surrogatepass') + ) + actual = input() + finally: + sys.stdin = old_stdin + self.assertEqual(actual, text) + + def test_input(self): + # ASCII + self.assertStdinRoundTrip('abc123') + # Non-ASCII + self.assertStdinRoundTrip('ϼўТλФЙ') + # Combining characters + self.assertStdinRoundTrip('A͏B ﬖ̳AA̝') + # Non-BMP + self.assertStdinRoundTrip('\U00100000\U0010ffff\U0010fffd') + + def test_partial_reads(self): + # Test that reading less than 1 full character works when stdin + # contains multibyte UTF-8 sequences + source = 'ϼўТλФЙ\r\n'.encode('utf-16-le') + expected = 'ϼўТλФЙ\r\n'.encode('utf-8') + for read_count in range(1, 16): + stdin = open('CONIN$', 'rb', buffering=0) + write_input(stdin, source) + + actual = b'' + while not actual.endswith(b'\n'): + b = stdin.read(read_count) + actual += b + + self.assertEqual(actual, expected, 'stdin.read({})'.format(read_count)) + stdin.close() + + def test_partial_surrogate_reads(self): + # Test that reading less than 1 full character works when stdin + # contains surrogate pairs that cannot be decoded to UTF-8 without + # reading an extra character. + source = '\U00101FFF\U00101001\r\n'.encode('utf-16-le') + expected = '\U00101FFF\U00101001\r\n'.encode('utf-8') + for read_count in range(1, 16): + stdin = open('CONIN$', 'rb', buffering=0) + write_input(stdin, source) + + actual = b'' + while not actual.endswith(b'\n'): + b = stdin.read(read_count) + actual += b + + self.assertEqual(actual, expected, 'stdin.read({})'.format(read_count)) + stdin.close() + + if __name__ == "__main__": unittest.main() diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -22,7 +22,8 @@ #ifndef Py_LIMITED_API #ifdef MS_WINDOWS extern PyTypeObject PyWindowsConsoleIO_Type; -#define PyWindowsConsoleIO_Check(op) (PyObject_TypeCheck((op), &PyWindowsConsoleIO_Type)) +PyAPI_DATA(PyObject *) _PyWindowsConsoleIO_Type; +#define PyWindowsConsoleIO_Check(op) (PyObject_TypeCheck((op), (PyTypeObject*)_PyWindowsConsoleIO_Type)) #endif /* MS_WINDOWS */ #endif /* Py_LIMITED_API */ diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -39,6 +39,11 @@ /* BUFMAX determines how many bytes can be read in one go. */ #define BUFMAX (32*1024*1024) +/* SMALLBUF determines how many utf-8 characters will be + buffered within the stream, in order to support reads + of less than one character */ +#define SMALLBUF 4 + char _get_console_type(HANDLE handle) { DWORD mode, peek_count; @@ -125,7 +130,8 @@ unsigned int blksize; PyObject *weakreflist; PyObject *dict; - char buf[4]; + char buf[SMALLBUF]; + wchar_t wbuf; } winconsoleio; PyTypeObject PyWindowsConsoleIO_Type; @@ -500,11 +506,11 @@ static DWORD _buflen(winconsoleio *self) { - for (DWORD i = 0; i < 4; ++i) { + for (DWORD i = 0; i < SMALLBUF; ++i) { if (!self->buf[i]) return i; } - return 4; + return SMALLBUF; } static DWORD @@ -513,12 +519,10 @@ DWORD n = 0; while (self->buf[0] && len--) { - n += 1; - buf[0] = self->buf[0]; - self->buf[0] = self->buf[1]; - self->buf[1] = self->buf[2]; - self->buf[2] = self->buf[3]; - self->buf[3] = 0; + buf[n++] = self->buf[0]; + for (int i = 1; i < SMALLBUF; ++i) + self->buf[i - 1] = self->buf[i]; + self->buf[SMALLBUF - 1] = 0; } return n; @@ -531,10 +535,13 @@ wchar_t *buf = (wchar_t*)PyMem_Malloc(maxlen * sizeof(wchar_t)); if (!buf) goto error; + *readlen = 0; + //DebugBreak(); Py_BEGIN_ALLOW_THREADS - for (DWORD off = 0; off < maxlen; off += BUFSIZ) { + DWORD off = 0; + while (off < maxlen) { DWORD n, len = min(maxlen - off, BUFSIZ); SetLastError(0); BOOL res = ReadConsoleW(handle, &buf[off], len, &n, NULL); @@ -550,7 +557,7 @@ err = 0; HANDLE hInterruptEvent = _PyOS_SigintEvent(); if (WaitForSingleObjectEx(hInterruptEvent, 100, FALSE) - == WAIT_OBJECT_0) { + == WAIT_OBJECT_0) { ResetEvent(hInterruptEvent); Py_BLOCK_THREADS sig = PyErr_CheckSignals(); @@ -568,7 +575,30 @@ /* If the buffer ended with a newline, break out */ if (buf[*readlen - 1] == '\n') break; + /* If the buffer ends with a high surrogate, expand the + buffer and read an extra character. */ + WORD char_type; + if (off + BUFSIZ >= maxlen && + GetStringTypeW(CT_CTYPE3, &buf[*readlen - 1], 1, &char_type) && + char_type == C3_HIGHSURROGATE) { + wchar_t *newbuf; + maxlen += 1; + Py_BLOCK_THREADS + newbuf = (wchar_t*)PyMem_Realloc(buf, maxlen * sizeof(wchar_t)); + Py_UNBLOCK_THREADS + if (!newbuf) { + sig = -1; + break; + } + buf = newbuf; + /* Only advance by n and not BUFSIZ in this case */ + off += n; + continue; + } + + off += BUFSIZ; } + Py_END_ALLOW_THREADS if (sig) @@ -1110,4 +1140,6 @@ 0, /* tp_finalize */ }; +PyAPI_DATA(PyObject *) _PyWindowsConsoleIO_Type = (PyObject*)&PyWindowsConsoleIO_Type; + #endif /* MS_WINDOWS */ diff --git a/PC/_testconsole.c b/PC/_testconsole.c new file mode 100644 --- /dev/null +++ b/PC/_testconsole.c @@ -0,0 +1,131 @@ + +/* Testing module for multi-phase initialization of extension modules (PEP 489) + */ + +#include "Python.h" + +#ifdef MS_WINDOWS + +#include "..\modules\_io\_iomodule.h" + +#define WIN32_LEAN_AND_MEAN +#include +#include + + /* The full definition is in iomodule. We reproduce + enough here to get the handle, which is all we want. */ +typedef struct { + PyObject_HEAD + HANDLE handle; +} winconsoleio; + + +static int execfunc(PyObject *m) +{ + return 0; +} + +PyModuleDef_Slot testconsole_slots[] = { + {Py_mod_exec, execfunc}, + {0, NULL}, +}; + +/*[clinic input] +module _testconsole + +_testconsole.write_input + file: object + s: PyBytesObject + +Writes UTF-16-LE encoded bytes to the console as if typed by a user. +[clinic start generated code]*/ + +static PyObject * +_testconsole_write_input_impl(PyObject *module, PyObject *file, + PyBytesObject *s) +/*[clinic end generated code: output=48f9563db34aedb3 input=4c774f2d05770bc6]*/ +{ + INPUT_RECORD *rec = NULL; + + if (!PyWindowsConsoleIO_Check(file)) { + PyErr_SetString(PyExc_TypeError, "expected raw console object"); + return NULL; + } + + const wchar_t *p = (const wchar_t *)PyBytes_AS_STRING(s); + DWORD size = (DWORD)PyBytes_GET_SIZE(s) / sizeof(wchar_t); + + rec = (INPUT_RECORD*)PyMem_Malloc(sizeof(INPUT_RECORD) * size); + if (!rec) + goto error; + memset(rec, 0, sizeof(INPUT_RECORD) * size); + + INPUT_RECORD *prec = rec; + for (DWORD i = 0; i < size; ++i, ++p, ++prec) { + prec->EventType = KEY_EVENT; + prec->Event.KeyEvent.bKeyDown = TRUE; + prec->Event.KeyEvent.wRepeatCount = 10; + prec->Event.KeyEvent.uChar.UnicodeChar = *p; + } + + HANDLE hInput = ((winconsoleio*)file)->handle; + DWORD total = 0; + while (total < size) { + DWORD wrote; + if (!WriteConsoleInputW(hInput, &rec[total], (size - total), &wrote)) { + PyErr_SetFromWindowsErr(0); + goto error; + } + total += wrote; + } + + PyMem_Free((void*)rec); + + Py_RETURN_NONE; +error: + if (rec) + PyMem_Free((void*)rec); + return NULL; +} + +/*[clinic input] +_testconsole.read_output + file: object + +Reads a str from the console as written to stdout. +[clinic start generated code]*/ + +static PyObject * +_testconsole_read_output_impl(PyObject *module, PyObject *file) +/*[clinic end generated code: output=876310d81a73e6d2 input=b3521f64b1b558e3]*/ +{ + Py_RETURN_NONE; +} + +#include "clinic\_testconsole.c.h" + +PyMethodDef testconsole_methods[] = { + _TESTCONSOLE_WRITE_INPUT_METHODDEF + _TESTCONSOLE_READ_OUTPUT_METHODDEF + {NULL, NULL} +}; + +static PyModuleDef testconsole_def = { + PyModuleDef_HEAD_INIT, /* m_base */ + "_testconsole", /* m_name */ + PyDoc_STR("Test module for the Windows console"), /* m_doc */ + 0, /* m_size */ + testconsole_methods, /* m_methods */ + testconsole_slots, /* m_slots */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL, /* m_free */ +}; + +PyMODINIT_FUNC +PyInit__testconsole(PyObject *spec) +{ + return PyModuleDef_Init(&testconsole_def); +} + +#endif /* MS_WINDOWS */ diff --git a/PC/clinic/_testconsole.c.h b/PC/clinic/_testconsole.c.h new file mode 100644 --- /dev/null +++ b/PC/clinic/_testconsole.c.h @@ -0,0 +1,82 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_testconsole_write_input__doc__, +"write_input($module, /, file, s)\n" +"--\n" +"\n" +"Writes UTF-16-LE encoded bytes to the console as if typed by a user."); + +#define _TESTCONSOLE_WRITE_INPUT_METHODDEF \ + {"write_input", (PyCFunction)_testconsole_write_input, METH_FASTCALL, _testconsole_write_input__doc__}, + +static PyObject * +_testconsole_write_input_impl(PyObject *module, PyObject *file, + PyBytesObject *s); + +static PyObject * +_testconsole_write_input(PyObject *module, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"file", "s", NULL}; + static _PyArg_Parser _parser = {"OS:write_input", _keywords, 0}; + PyObject *file; + PyBytesObject *s; + + if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, + &file, &s)) { + goto exit; + } + return_value = _testconsole_write_input_impl(module, file, s); + +exit: + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(_testconsole_read_output__doc__, +"read_output($module, /, file)\n" +"--\n" +"\n" +"Reads a str from the console as written to stdout."); + +#define _TESTCONSOLE_READ_OUTPUT_METHODDEF \ + {"read_output", (PyCFunction)_testconsole_read_output, METH_FASTCALL, _testconsole_read_output__doc__}, + +static PyObject * +_testconsole_read_output_impl(PyObject *module, PyObject *file); + +static PyObject * +_testconsole_read_output(PyObject *module, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"file", NULL}; + static _PyArg_Parser _parser = {"O:read_output", _keywords, 0}; + PyObject *file; + + if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, + &file)) { + goto exit; + } + return_value = _testconsole_read_output_impl(module, file); + +exit: + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#ifndef _TESTCONSOLE_WRITE_INPUT_METHODDEF + #define _TESTCONSOLE_WRITE_INPUT_METHODDEF +#endif /* !defined(_TESTCONSOLE_WRITE_INPUT_METHODDEF) */ + +#ifndef _TESTCONSOLE_READ_OUTPUT_METHODDEF + #define _TESTCONSOLE_READ_OUTPUT_METHODDEF +#endif /* !defined(_TESTCONSOLE_READ_OUTPUT_METHODDEF) */ +/*[clinic end generated code: output=3a8dc0c421807c41 input=a9049054013a1b77]*/ diff --git a/PCbuild/_testconsole.vcxproj b/PCbuild/_testconsole.vcxproj new file mode 100644 --- /dev/null +++ b/PCbuild/_testconsole.vcxproj @@ -0,0 +1,83 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {B244E787-C445-441C-BDF4-5A4F1A3A1E51} + Win32Proj + _testconsole + false + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + + _CONSOLE;%(PreprocessorDefinitions) + + + Console + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + + \ No newline at end of file diff --git a/PCbuild/_testconsole.vcxproj.filters b/PCbuild/_testconsole.vcxproj.filters new file mode 100644 --- /dev/null +++ b/PCbuild/_testconsole.vcxproj.filters @@ -0,0 +1,22 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + \ No newline at end of file diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -61,7 +61,7 @@ - + diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -92,6 +92,8 @@ EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pyshellext", "pyshellext.vcxproj", "{0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testconsole", "_testconsole.vcxproj", "{B244E787-C445-441C-BDF4-5A4F1A3A1E51}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -708,6 +710,22 @@ {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|Win32.Build.0 = Release|Win32 {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|x64.ActiveCfg = Release|x64 {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|x64.Build.0 = Release|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|Win32.ActiveCfg = Debug|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|Win32.Build.0 = Debug|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|x64.ActiveCfg = Debug|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|x64.Build.0 = Debug|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|Win32.Build.0 = Release|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|x64.ActiveCfg = Release|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|x64.Build.0 = Release|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|Win32.Build.0 = Release|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|x64.ActiveCfg = Release|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|x64.Build.0 = Release|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|Win32.ActiveCfg = Release|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|Win32.Build.0 = Release|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|x64.ActiveCfg = Release|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Tools/msi/test/test_files.wxs b/Tools/msi/test/test_files.wxs --- a/Tools/msi/test/test_files.wxs +++ b/Tools/msi/test/test_files.wxs @@ -17,6 +17,9 @@ + + + @@ -37,6 +40,9 @@ + + + @@ -57,6 +63,9 @@ + + + @@ -72,6 +81,9 @@ + + +