# HG changeset patch # Parent 2917a3ce988ec4277efc048930131256acb06140 diff -r 2917a3ce988e Lib/test/test_readline.py --- a/Lib/test/test_readline.py Sat May 21 21:36:29 2016 +0300 +++ b/Lib/test/test_readline.py Tue May 31 13:47:34 2016 +0000 @@ -116,14 +116,63 @@ output = run_pty(self.auto_history_script.format(False)) self.assertIn(b"History length: 0\r\n", output) + def test_nonascii_completion(self): + script = """\ +import readline -def run_pty(script, input=b"dummy input\r"): +# TODO: adapt for Editline +readline.parse_and_bind("Control-b: backward-char") +readline.parse_and_bind("TAB: complete") + +def pre_input_hook(): + readline.insert_text("[\\xEFnserted]") + readline.redisplay() +readline.set_pre_input_hook(pre_input_hook) + +readline.set_completer_delims("|") + +def completer(text, state): + if state > 0: + return None + print("text", ascii(text)) + print("line", ascii(readline.get_line_buffer())) + print("indexes", readline.get_begidx(), readline.get_endidx()) + return "[c\\xF6mpletion]" +readline.set_completer(completer) + +print("result", ascii(input())) +""" + + # Input line, move cursor back to before "[after]", press Tab for + # completion, and then Return + input = "|[t\xEBxt][after]" + "\x02" * 7 + "\t\r" + try: + output = run_pty(script, input) + except UnicodeEncodeError as err: + self.skipTest("Locale cannot encode test data: {}".format(err)) + self.assertIn(b"text '[t\\xebxt]'\r\n", output) + self.assertIn(b"line '[\\xefnserted]|[t\\xebxt][after]'\r\n", + output) + self.assertIn(b"indexes 11 17\r\n", output) + self.assertIn(b"result '[\\xefnserted]|[c\\xf6mpletion][after]'\r\n", + output) + +def run_pty(script, input="dummy input\r"): pty = import_module('pty') output = bytearray() [master, slave] = pty.openpty() - args = (sys.executable, '-c', script) - proc = subprocess.Popen(args, stdin=slave, stdout=slave, stderr=slave) - os.close(slave) + try: + input = input.encode(os.device_encoding(slave)) + + args = (sys.executable, '-c', script) + proc = subprocess.Popen(args, + stdin=slave, stdout=slave, stderr=slave) + except: + os.close(master) + raise + finally: + os.close(slave) + with ExitStack() as cleanup: cleanup.enter_context(proc) def terminate(proc): @@ -148,7 +197,7 @@ try: chunk = os.read(master, 0x10000) except OSError as err: - # Linux raises EIO when the slave is closed + # Linux raises EIO when slave is closed (Issue 5380) if err.errno != EIO: raise chunk = b"" @@ -156,7 +205,13 @@ return output output.extend(chunk) if events & selectors.EVENT_WRITE: - input = input[os.write(master, input):] + try: + input = input[os.write(master, input):] + except OSError as err: + # Apparently EIO means the slave was closed + if err.errno != EIO: + raise + input = b"" # Stop writing if not input: sel.modify(master, selectors.EVENT_READ) diff -r 2917a3ce988e Modules/readline.c --- a/Modules/readline.c Sat May 21 21:36:29 2016 +0300 +++ b/Modules/readline.c Tue May 31 13:47:34 2016 +0000 @@ -128,20 +128,40 @@ #define readlinestate_global ((readlinestate *)PyModule_GetState(PyState_FindModule(&readlinemodule))) +/* Convert to/from multibyte C strings */ + +static PyObject * +encode(PyObject *b) +{ + return PyUnicode_EncodeLocale(b, "surrogateescape"); +} + +static PyObject * +decode(const char *s) +{ + return PyUnicode_DecodeLocale(s, "surrogateescape"); +} + + /* Exported function to send one line to readline's init file parser */ static PyObject * -parse_and_bind(PyObject *self, PyObject *args) +parse_and_bind(PyObject *self, PyObject *string) { - char *s, *copy; - if (!PyArg_ParseTuple(args, "s:parse_and_bind", &s)) + char *copy; + PyObject *encoded = encode(string); + if (encoded == NULL) { return NULL; + } /* Make a copy -- rl_parse_and_bind() modifies its argument */ /* Bernard Herzog */ - copy = PyMem_Malloc(1 + strlen(s)); - if (copy == NULL) + copy = PyMem_Malloc(1 + PyBytes_Size(encoded)); + if (copy == NULL) { + Py_DECREF(encoded); return PyErr_NoMemory(); - strcpy(copy, s); + } + strcpy(copy, PyBytes_AsString(encoded)); + Py_DECREF(encoded); rl_parse_and_bind(copy); PyMem_Free(copy); /* Free the copy */ Py_RETURN_NONE; @@ -439,17 +459,18 @@ /* Set the tab-completion word-delimiters that readline uses */ static PyObject * -set_completer_delims(PyObject *self, PyObject *args) +set_completer_delims(PyObject *self, PyObject *string) { char *break_chars; - - if (!PyArg_ParseTuple(args, "s:set_completer_delims", &break_chars)) { + PyObject *encoded = encode(string); + if (encoded == NULL) { return NULL; } /* Keep a reference to the allocated memory in the module state in case some other module modifies rl_completer_word_break_characters (see issue #17289). */ - break_chars = strdup(break_chars); + break_chars = strdup(PyBytes_AsString(encoded)); + Py_DECREF(encoded); if (break_chars) { free(completer_word_break_characters); completer_word_break_characters = break_chars; @@ -529,10 +550,11 @@ py_replace_history(PyObject *self, PyObject *args) { int entry_number; - char *line; + PyObject *line; + PyObject *encoded; HIST_ENTRY *old_entry; - if (!PyArg_ParseTuple(args, "is:replace_history_item", &entry_number, + if (!PyArg_ParseTuple(args, "iU:replace_history_item", &entry_number, &line)) { return NULL; } @@ -541,7 +563,12 @@ "History index cannot be negative"); return NULL; } - old_entry = replace_history_entry(entry_number, line, (void *)NULL); + encoded = encode(line); + if (encoded == NULL) { + return NULL; + } + old_entry = replace_history_entry(entry_number, PyBytes_AsString(encoded), (void *)NULL); + Py_DECREF(encoded); if (!old_entry) { PyErr_Format(PyExc_ValueError, "No history item at position %d", @@ -560,14 +587,14 @@ /* Add a line to the history buffer */ static PyObject * -py_add_history(PyObject *self, PyObject *args) +py_add_history(PyObject *self, PyObject *string) { - char *line; - - if(!PyArg_ParseTuple(args, "s:add_history", &line)) { + PyObject *encoded = encode(string); + if (encoded == NULL) { return NULL; } - add_history(line); + add_history(PyBytes_AsString(encoded)); + Py_DECREF(encoded); Py_RETURN_NONE; } @@ -599,7 +626,7 @@ static PyObject * get_completer_delims(PyObject *self, PyObject *noarg) { - return PyUnicode_FromString(rl_completer_word_break_characters); + return decode(rl_completer_word_break_characters); } PyDoc_STRVAR(doc_get_completer_delims, @@ -689,7 +716,7 @@ } #endif /* __APPLE__ */ if ((hist_ent = history_get(idx))) - return PyUnicode_FromString(hist_ent->line); + return decode(hist_ent->line); else { Py_RETURN_NONE; } @@ -718,7 +745,7 @@ static PyObject * get_line_buffer(PyObject *self, PyObject *noarg) { - return PyUnicode_FromString(rl_line_buffer); + return decode(rl_line_buffer); } PyDoc_STRVAR(doc_get_line_buffer, @@ -746,12 +773,14 @@ /* Exported function to insert text into the line buffer */ static PyObject * -insert_text(PyObject *self, PyObject *args) +insert_text(PyObject *self, PyObject *string) { - char *s; - if (!PyArg_ParseTuple(args, "s:insert_text", &s)) + PyObject *encoded = encode(string); + if (encoded == NULL) { return NULL; - rl_insert_text(s); + } + rl_insert_text(PyBytes_AsString(encoded)); + Py_DECREF(encoded); Py_RETURN_NONE; } @@ -779,9 +808,9 @@ static struct PyMethodDef readline_methods[] = { - {"parse_and_bind", parse_and_bind, METH_VARARGS, doc_parse_and_bind}, + {"parse_and_bind", parse_and_bind, METH_O, doc_parse_and_bind}, {"get_line_buffer", get_line_buffer, METH_NOARGS, doc_get_line_buffer}, - {"insert_text", insert_text, METH_VARARGS, doc_insert_text}, + {"insert_text", insert_text, METH_O, doc_insert_text}, {"redisplay", redisplay, METH_NOARGS, doc_redisplay}, {"read_init_file", read_init_file, METH_VARARGS, doc_read_init_file}, {"read_history_file", read_history_file, @@ -808,9 +837,9 @@ {"get_endidx", get_endidx, METH_NOARGS, doc_get_endidx}, {"set_completer_delims", set_completer_delims, - METH_VARARGS, doc_set_completer_delims}, + METH_O, doc_set_completer_delims}, {"set_auto_history", py_set_auto_history, METH_VARARGS, doc_set_auto_history}, - {"add_history", py_add_history, METH_VARARGS, doc_add_history}, + {"add_history", py_add_history, METH_O, doc_add_history}, {"remove_history_item", py_remove_history, METH_VARARGS, doc_remove_history}, {"replace_history_item", py_replace_history, METH_VARARGS, doc_replace_history}, {"get_completer_delims", get_completer_delims, @@ -907,7 +936,7 @@ int num_matches, int max_length) { int i; - PyObject *m=NULL, *s=NULL, *r=NULL; + PyObject *sub, *m=NULL, *s=NULL, *r=NULL; #ifdef WITH_THREAD PyGILState_STATE gilstate = PyGILState_Ensure(); #endif @@ -915,16 +944,19 @@ if (m == NULL) goto error; for (i = 0; i < num_matches; i++) { - s = PyUnicode_FromString(matches[i+1]); + s = decode(matches[i+1]); if (s == NULL) goto error; if (PyList_SetItem(m, i, s) == -1) goto error; } + sub = decode(matches[0]); + if (sub == NULL) + goto error; r = PyObject_CallFunction(readlinestate_global->completion_display_matches_hook, - "sOi", matches[0], m, max_length); + "NNi", sub, m, max_length); - Py_DECREF(m); m=NULL; + m = NULL; if (r == NULL || (r != Py_None && PyLong_AsLong(r) == -1 && PyErr_Occurred())) { @@ -972,22 +1004,26 @@ { char *result = NULL; if (readlinestate_global->completer != NULL) { - PyObject *r; + PyObject *r = NULL, *t; #ifdef WITH_THREAD PyGILState_STATE gilstate = PyGILState_Ensure(); #endif rl_attempted_completion_over = 1; - r = PyObject_CallFunction(readlinestate_global->completer, "si", text, state); + t = decode(text); + if (t == NULL) + goto error; + r = PyObject_CallFunction(readlinestate_global->completer, "Ni", t, state); if (r == NULL) goto error; if (r == Py_None) { result = NULL; } else { - char *s = _PyUnicode_AsString(r); - if (s == NULL) + PyObject *encoded = encode(r); + if (encoded == NULL) goto error; - result = strdup(s); + result = strdup(PyBytes_AsString(encoded)); + Py_DECREF(encoded); } Py_DECREF(r); goto done; @@ -1011,6 +1047,8 @@ flex_complete(const char *text, int start, int end) { char **result; + char saved; + size_t start_size, end_size; #ifdef WITH_THREAD PyGILState_STATE gilstate = PyGILState_Ensure(); #endif @@ -1020,6 +1058,20 @@ #ifdef HAVE_RL_COMPLETION_SUPPRESS_APPEND rl_completion_suppress_append = 0; #endif + + saved = rl_line_buffer[start]; + rl_line_buffer[start] = 0; + PyMem_RawFree(Py_DecodeLocale(rl_line_buffer, &start_size)); + rl_line_buffer[start] = saved; + saved = rl_line_buffer[end]; + rl_line_buffer[end] = 0; + PyMem_RawFree(Py_DecodeLocale(rl_line_buffer + start, &end_size)); + rl_line_buffer[end] = saved; + if (start_size >= 0 and end_size >= 0) { + start = (int)start_size; + end = start + (int)end_size; + } + Py_XDECREF(readlinestate_global->begidx); Py_XDECREF(readlinestate_global->endidx); readlinestate_global->begidx = PyLong_FromLong((long) start);