diff -r 47b4dbd451f5 Objects/stringlib/join.h --- a/Objects/stringlib/join.h Wed Sep 07 09:31:52 2016 -0700 +++ b/Objects/stringlib/join.h Wed Sep 07 11:14:10 2016 -0700 @@ -7,17 +7,18 @@ Py_LOCAL_INLINE(PyObject *) STRINGLIB(bytes_join)(PyObject *sep, PyObject *iterable) { + PyObject *seq; + Py_ssize_t seqlen; char *sepstr = STRINGLIB_STR(sep); const Py_ssize_t seplen = STRINGLIB_LEN(sep); PyObject *res = NULL; char *p; - Py_ssize_t seqlen = 0; Py_ssize_t sz = 0; Py_ssize_t i, nbufs; - PyObject *seq, *item; + PyObject *item; Py_buffer *buffers = NULL; -#define NB_STATIC_BUFFERS 10 - Py_buffer static_buffers[NB_STATIC_BUFFERS]; + Py_buffer static_buffers[10]; + int exact; seq = PySequence_Fast(iterable, "can only join an iterable"); if (seq == NULL) { @@ -29,112 +30,175 @@ STRINGLIB(bytes_join)(PyObject *sep, PyO Py_DECREF(seq); return STRINGLIB_NEW(NULL, 0); } + #ifndef STRINGLIB_MUTABLE if (seqlen == 1) { item = PySequence_Fast_GET_ITEM(seq, 0); if (STRINGLIB_CHECK_EXACT(item)) { + Py_DECREF(seq); + Py_INCREF(item); - Py_DECREF(seq); return item; } } #endif - if (seqlen > NB_STATIC_BUFFERS) { - buffers = PyMem_NEW(Py_buffer, seqlen); - if (buffers == NULL) { - Py_DECREF(seq); - PyErr_NoMemory(); - return NULL; - } - } - else { - buffers = static_buffers; - } - /* Here is the general case. Do a pre-pass to figure out the total - * amount of space we'll need (sz), and see whether all arguments are - * bytes-like. - */ - for (i = 0, nbufs = 0; i < seqlen; i++) { + /* If all items are bytes or bytearray: avoid Py_buffer API and avoid + allocation a temporary array of Py_buffer */ + sz = 0; + exact = 1; + for (i = 0; i < seqlen; i++) { Py_ssize_t itemlen; + item = PySequence_Fast_GET_ITEM(seq, i); if (PyBytes_CheckExact(item)) { - /* Fast path. */ - Py_INCREF(item); - buffers[i].obj = item; - buffers[i].buf = PyBytes_AS_STRING(item); - buffers[i].len = PyBytes_GET_SIZE(item); + itemlen = PyBytes_GET_SIZE(item); } - else if (PyObject_GetBuffer(item, &buffers[i], PyBUF_SIMPLE) != 0) { - PyErr_Format(PyExc_TypeError, - "sequence item %zd: expected a bytes-like object, " - "%.80s found", - i, Py_TYPE(item)->tp_name); - goto error; + else if (PyByteArray_CheckExact(item)) { + itemlen = PyByteArray_GET_SIZE(item); } - nbufs = i + 1; /* for error cleanup */ - itemlen = buffers[i].len; + else { + exact = 0; + break; + } + if (itemlen > PY_SSIZE_T_MAX - sz) { PyErr_SetString(PyExc_OverflowError, "join() result is too long"); - goto error; + return NULL; } sz += itemlen; - if (i != 0) { - if (seplen > PY_SSIZE_T_MAX - sz) { + + /* if all items are bytes or bytearray, the sequence length cannot + change, but check just in case */ + assert(seqlen == PySequence_Fast_GET_SIZE(seq)); + } + + if (!exact) { + if ((size_t)seqlen > Py_ARRAY_LENGTH(static_buffers)) { + buffers = PyMem_NEW(Py_buffer, seqlen); + if (buffers == NULL) { + PyErr_NoMemory(); + return NULL; + } + } + else { + buffers = static_buffers; + } + + /* Here is the general case. Do a pre-pass to figure out the total + * amount of space we'll need (sz), and see whether all arguments are + * bytes-like. + */ + sz = 0; + for (i = 0, nbufs = 0; i < seqlen; i++) { + Py_ssize_t itemlen; + item = PySequence_Fast_GET_ITEM(seq, i); + if (PyBytes_CheckExact(item)) { + /* Fast path. */ + Py_INCREF(item); + buffers[i].obj = item; + buffers[i].buf = PyBytes_AS_STRING(item); + buffers[i].len = PyBytes_GET_SIZE(item); + } + else if (PyObject_GetBuffer(item, &buffers[i], PyBUF_SIMPLE) != 0) { + PyErr_Format(PyExc_TypeError, + "sequence item %zd: expected a bytes-like object, " + "%.80s found", + i, Py_TYPE(item)->tp_name); + goto error; + } + nbufs = i + 1; /* for error cleanup */ + itemlen = buffers[i].len; + if (itemlen > PY_SSIZE_T_MAX - sz) { PyErr_SetString(PyExc_OverflowError, "join() result is too long"); goto error; } - sz += seplen; + sz += itemlen; + + /* PyObject_GetBuffer() can indirectly modify iterable (list) */ + if (seqlen != PySequence_Fast_GET_SIZE(seq)) { + PyErr_SetString(PyExc_RuntimeError, + "sequence changed size during iteration"); + goto error; + } } - if (seqlen != PySequence_Fast_GET_SIZE(seq)) { - PyErr_SetString(PyExc_RuntimeError, - "sequence changed size during iteration"); - goto error; + } + + if (seqlen > 1 && seplen) { + if (seplen > (PY_SSIZE_T_MAX - sz) / (seqlen - 1)) { + PyErr_SetString(PyExc_OverflowError, + "join() result is too long"); + return NULL; } + sz += seplen * (seqlen - 1); } /* Allocate result space. */ res = STRINGLIB_NEW(NULL, sz); - if (res == NULL) + if (res == NULL) { goto error; + } /* Catenate everything. */ p = STRINGLIB_STR(res); - if (!seplen) { - /* fast path */ - for (i = 0; i < nbufs; i++) { - Py_ssize_t n = buffers[i].len; - char *q = buffers[i].buf; + if (exact) { + for (i = 0; i < seqlen; i++) { + Py_ssize_t n; + char *q; + + if (seplen && i) { + Py_MEMCPY(p, sepstr, seplen); + p += seplen; + } + + item = PySequence_Fast_GET_ITEM(seq, i); + if (PyBytes_CheckExact(item)) { + q = PyBytes_AS_STRING(item); + n = PyBytes_GET_SIZE(item); + } + else { + assert(PyByteArray_CheckExact(item)); + q = PyByteArray_AS_STRING(item); + n = PyByteArray_GET_SIZE(item); + } + Py_MEMCPY(p, q, n); p += n; } - goto done; } - for (i = 0; i < nbufs; i++) { - Py_ssize_t n; - char *q; - if (i) { - Py_MEMCPY(p, sepstr, seplen); - p += seplen; + else { + for (i = 0; i < nbufs; i++) { + Py_ssize_t n; + char *q; + if (seplen && i) { + Py_MEMCPY(p, sepstr, seplen); + p += seplen; + } + n = buffers[i].len; + q = buffers[i].buf; + Py_MEMCPY(p, q, n); + p += n; } - n = buffers[i].len; - q = buffers[i].buf; - Py_MEMCPY(p, q, n); - p += n; } + assert(p == STRINGLIB_STR(res) + sz); goto done; error: res = NULL; done: Py_DECREF(seq); - for (i = 0; i < nbufs; i++) - PyBuffer_Release(&buffers[i]); - if (buffers != static_buffers) - PyMem_FREE(buffers); + + if (buffers) { + for (i = 0; i < nbufs; i++) { + PyBuffer_Release(&buffers[i]); + } + if (buffers != static_buffers) { + PyMem_Free(buffers); + } + } + + assert(!res || seqlen == PySequence_Fast_GET_SIZE(seq)); return res; } - -#undef NB_STATIC_BUFFERS