diff -r 2917a3ce988e Modules/readline.c --- a/Modules/readline.c Sat May 21 21:36:29 2016 +0300 +++ b/Modules/readline.c Sun May 22 18:08:07 2016 +0300 @@ -128,20 +128,40 @@ static PyModuleDef readlinemodule; #define readlinestate_global ((readlinestate *)PyModule_GetState(PyState_FindModule(&readlinemodule))) +/* Convert to/from multibyte C strings */ + +static PyObject * +encode(PyObject *b) +{ + return PyUnicode_EncodeLocale(b, "surrogateescape"); +} + +static PyObject * +decode(const char *s) +{ + return PyUnicode_DecodeLocale(s, "surrogateescape"); +} + + /* Exported function to send one line to readline's init file parser */ static PyObject * -parse_and_bind(PyObject *self, PyObject *args) +parse_and_bind(PyObject *self, PyObject *string) { - char *s, *copy; - if (!PyArg_ParseTuple(args, "s:parse_and_bind", &s)) + char *copy; + PyObject *encoded = encode(string); + if (encoded == NULL) { return NULL; + } /* Make a copy -- rl_parse_and_bind() modifies its argument */ /* Bernard Herzog */ - copy = PyMem_Malloc(1 + strlen(s)); - if (copy == NULL) + copy = PyMem_Malloc(1 + PyBytes_Size(encoded)); + if (copy == NULL) { + Py_DECREF(encoded); return PyErr_NoMemory(); - strcpy(copy, s); + } + strcpy(copy, PyBytes_AsString(encoded)); + Py_DECREF(encoded); rl_parse_and_bind(copy); PyMem_Free(copy); /* Free the copy */ Py_RETURN_NONE; @@ -439,17 +459,18 @@ get the ending index of the completion s /* Set the tab-completion word-delimiters that readline uses */ static PyObject * -set_completer_delims(PyObject *self, PyObject *args) +set_completer_delims(PyObject *self, PyObject *string) { char *break_chars; - - if (!PyArg_ParseTuple(args, "s:set_completer_delims", &break_chars)) { + PyObject *encoded = encode(string); + if (encoded == NULL) { return NULL; } /* Keep a reference to the allocated memory in the module state in case some other module modifies rl_completer_word_break_characters (see issue #17289). */ - break_chars = strdup(break_chars); + break_chars = strdup(PyBytes_AsString(encoded)); + Py_DECREF(encoded); if (break_chars) { free(completer_word_break_characters); completer_word_break_characters = break_chars; @@ -529,10 +550,11 @@ static PyObject * py_replace_history(PyObject *self, PyObject *args) { int entry_number; - char *line; + PyObject *line; + PyObject *encoded; HIST_ENTRY *old_entry; - if (!PyArg_ParseTuple(args, "is:replace_history_item", &entry_number, + if (!PyArg_ParseTuple(args, "iU:replace_history_item", &entry_number, &line)) { return NULL; } @@ -541,7 +563,12 @@ py_replace_history(PyObject *self, PyObj "History index cannot be negative"); return NULL; } - old_entry = replace_history_entry(entry_number, line, (void *)NULL); + encoded = encode(line); + if (encoded == NULL) { + return NULL; + } + old_entry = replace_history_entry(entry_number, PyBytes_AsString(encoded), (void *)NULL); + Py_DECREF(encoded); if (!old_entry) { PyErr_Format(PyExc_ValueError, "No history item at position %d", @@ -560,14 +587,14 @@ replaces history item given by its posit /* Add a line to the history buffer */ static PyObject * -py_add_history(PyObject *self, PyObject *args) +py_add_history(PyObject *self, PyObject *string) { - char *line; - - if(!PyArg_ParseTuple(args, "s:add_history", &line)) { + PyObject *encoded = encode(string); + if (encoded == NULL) { return NULL; } - add_history(line); + add_history(PyBytes_AsString(encoded)); + Py_DECREF(encoded); Py_RETURN_NONE; } @@ -599,7 +626,7 @@ Enables or disables automatic history.") static PyObject * get_completer_delims(PyObject *self, PyObject *noarg) { - return PyUnicode_FromString(rl_completer_word_break_characters); + return decode(rl_completer_word_break_characters); } PyDoc_STRVAR(doc_get_completer_delims, @@ -689,7 +716,7 @@ get_history_item(PyObject *self, PyObjec } #endif /* __APPLE__ */ if ((hist_ent = history_get(idx))) - return PyUnicode_FromString(hist_ent->line); + return decode(hist_ent->line); else { Py_RETURN_NONE; } @@ -718,7 +745,7 @@ return the current (not the maximum) len static PyObject * get_line_buffer(PyObject *self, PyObject *noarg) { - return PyUnicode_FromString(rl_line_buffer); + return decode(rl_line_buffer); } PyDoc_STRVAR(doc_get_line_buffer, @@ -746,12 +773,14 @@ Clear the current readline history."); /* Exported function to insert text into the line buffer */ static PyObject * -insert_text(PyObject *self, PyObject *args) +insert_text(PyObject *self, PyObject *string) { - char *s; - if (!PyArg_ParseTuple(args, "s:insert_text", &s)) + PyObject *encoded = encode(string); + if (encoded == NULL) { return NULL; - rl_insert_text(s); + } + rl_insert_text(PyBytes_AsString(encoded)); + Py_DECREF(encoded); Py_RETURN_NONE; } @@ -779,9 +808,9 @@ contents of the line buffer."); static struct PyMethodDef readline_methods[] = { - {"parse_and_bind", parse_and_bind, METH_VARARGS, doc_parse_and_bind}, + {"parse_and_bind", parse_and_bind, METH_O, doc_parse_and_bind}, {"get_line_buffer", get_line_buffer, METH_NOARGS, doc_get_line_buffer}, - {"insert_text", insert_text, METH_VARARGS, doc_insert_text}, + {"insert_text", insert_text, METH_O, doc_insert_text}, {"redisplay", redisplay, METH_NOARGS, doc_redisplay}, {"read_init_file", read_init_file, METH_VARARGS, doc_read_init_file}, {"read_history_file", read_history_file, @@ -808,9 +837,9 @@ static struct PyMethodDef readline_metho {"get_endidx", get_endidx, METH_NOARGS, doc_get_endidx}, {"set_completer_delims", set_completer_delims, - METH_VARARGS, doc_set_completer_delims}, + METH_O, doc_set_completer_delims}, {"set_auto_history", py_set_auto_history, METH_VARARGS, doc_set_auto_history}, - {"add_history", py_add_history, METH_VARARGS, doc_add_history}, + {"add_history", py_add_history, METH_O, doc_add_history}, {"remove_history_item", py_remove_history, METH_VARARGS, doc_remove_history}, {"replace_history_item", py_replace_history, METH_VARARGS, doc_replace_history}, {"get_completer_delims", get_completer_delims, @@ -907,7 +936,7 @@ on_completion_display_matches_hook(char int num_matches, int max_length) { int i; - PyObject *m=NULL, *s=NULL, *r=NULL; + PyObject *sub, *m=NULL, *s=NULL, *r=NULL; #ifdef WITH_THREAD PyGILState_STATE gilstate = PyGILState_Ensure(); #endif @@ -915,16 +944,19 @@ on_completion_display_matches_hook(char if (m == NULL) goto error; for (i = 0; i < num_matches; i++) { - s = PyUnicode_FromString(matches[i+1]); + s = decode(matches[i+1]); if (s == NULL) goto error; if (PyList_SetItem(m, i, s) == -1) goto error; } + sub = decode(matches[0]); + if (sub == NULL) + goto error; r = PyObject_CallFunction(readlinestate_global->completion_display_matches_hook, - "sOi", matches[0], m, max_length); + "NNi", sub, m, max_length); - Py_DECREF(m); m=NULL; + m = NULL; if (r == NULL || (r != Py_None && PyLong_AsLong(r) == -1 && PyErr_Occurred())) { @@ -972,22 +1004,26 @@ on_completion(const char *text, int stat { char *result = NULL; if (readlinestate_global->completer != NULL) { - PyObject *r; + PyObject *r = NULL, *t; #ifdef WITH_THREAD PyGILState_STATE gilstate = PyGILState_Ensure(); #endif rl_attempted_completion_over = 1; - r = PyObject_CallFunction(readlinestate_global->completer, "si", text, state); + t = decode(text); + if (t == NULL) + goto error; + r = PyObject_CallFunction(readlinestate_global->completer, "Ni", t, state); if (r == NULL) goto error; if (r == Py_None) { result = NULL; } else { - char *s = _PyUnicode_AsString(r); - if (s == NULL) + PyObject *encoded = encode(r); + if (encoded == NULL) goto error; - result = strdup(s); + result = strdup(PyBytes_AsString(encoded)); + Py_DECREF(encoded); } Py_DECREF(r); goto done; @@ -1011,6 +1047,7 @@ static char ** flex_complete(const char *text, int start, int end) { char **result; + PyObject *s1, *s2; #ifdef WITH_THREAD PyGILState_STATE gilstate = PyGILState_Ensure(); #endif @@ -1020,6 +1057,22 @@ flex_complete(const char *text, int star #ifdef HAVE_RL_COMPLETION_SUPPRESS_APPEND rl_completion_suppress_append = 0; #endif + s1 = PyUnicode_DecodeLocaleAndSize(rl_line_buffer, start, + "surrogateescape"); + if (s1 == NULL) { + PyErr_Clear(); + } + s2 = PyUnicode_DecodeLocaleAndSize(rl_line_buffer + start, end - start, + "surrogateescape"); + if (s2 == NULL) { + PyErr_Clear(); + } + if (s1 != NULL && s2 != NULL) { + start = PyUnicode_GetLength(s1); + end = start + PyUnicode_GetLength(s2); + } + Py_XDECREF(s1); + Py_XDECREF(s2); Py_XDECREF(readlinestate_global->begidx); Py_XDECREF(readlinestate_global->endidx); readlinestate_global->begidx = PyLong_FromLong((long) start);