diff -r 601045ceff94 Lib/test/test_memoryio.py --- a/Lib/test/test_memoryio.py Thu Aug 14 22:26:38 2014 +0300 +++ b/Lib/test/test_memoryio.py Thu Aug 14 23:02:29 2014 +0300 @@ -712,12 +712,11 @@ @support.cpython_only def test_sizeof(self): - basesize = support.calcobjsize('P2nN2PnP') + basesize = support.calcobjsize('P2n2Pn') check = self.check_sizeof self.assertEqual(object.__sizeof__(io.BytesIO()), basesize) check(io.BytesIO(), basesize ) - check(io.BytesIO(b'a'), basesize + 1 ) - check(io.BytesIO(b'a' * 1000), basesize + 1000) + check(io.BytesIO(b'a' * 1000), basesize + sys.getsizeof(b'a' * 1000)) # Various tests of copy-on-write behaviour for BytesIO. diff -r 601045ceff94 Modules/_io/bytesio.c --- a/Modules/_io/bytesio.c Thu Aug 14 22:26:38 2014 +0300 +++ b/Modules/_io/bytesio.c Thu Aug 14 23:02:29 2014 +0300 @@ -4,17 +4,12 @@ typedef struct { PyObject_HEAD - char *buf; + PyObject *buf; Py_ssize_t pos; Py_ssize_t string_size; - size_t buf_size; PyObject *dict; PyObject *weakreflist; Py_ssize_t exports; - /** If `initvalue' != NULL, `buf' is a read-only pointer into the PyBytes - * referenced by `initvalue'. It must be copied prior to mutation, and - * released during finalization */ - PyObject *initvalue; } bytesio; typedef struct { @@ -22,12 +17,18 @@ bytesio *source; } bytesiobuf; +/* The bytesio object can be in three states: + * Py_REFCNT(buf) == 1, exports == 0. + * Py_REFCNT(buf) > 1. exports == 0, string_size == PyBytes_GET_SIZE(buf), + first modification or export causes the internal buffer copying. + * exports > 0. Py_REFCNT(buf) == 1, any modifications are forbidden. +*/ -#define CHECK_CLOSED(self, ret) \ +#define CHECK_CLOSED(self) \ if ((self)->buf == NULL) { \ PyErr_SetString(PyExc_ValueError, \ "I/O operation on closed file."); \ - return ret; \ + return NULL; \ } #define CHECK_EXPORTS(self) \ @@ -37,45 +38,8 @@ return NULL; \ } -/* Ensure we have a buffer suitable for writing, in the case that an initvalue - * object was provided, and we're currently borrowing its buffer. `size' - * indicates the new buffer size allocated as part of unsharing, to avoid a - * redundant reallocation caused by any subsequent mutation. `truncate' - * indicates whether truncation should occur if `size` < self->string_size. - * - * Do nothing if the buffer wasn't shared. Returns 0 on success, or sets an - * exception and returns -1 on failure. Existing state is preserved on failure. - */ -static int -unshare(bytesio *self, size_t preferred_size, int truncate) -{ - if (self->initvalue) { - Py_ssize_t copy_size; - char *new_buf; +#define SHARED_BUF(self) (Py_REFCNT((self)->buf) > 1) - if((! truncate) && preferred_size < self->string_size) { - preferred_size = self->string_size; - } - - new_buf = (char *)PyMem_Malloc(preferred_size); - if (new_buf == NULL) { - PyErr_NoMemory(); - return -1; - } - - copy_size = self->string_size; - if (copy_size > preferred_size) { - copy_size = preferred_size; - } - - memcpy(new_buf, self->buf, copy_size); - Py_CLEAR(self->initvalue); - self->buf = new_buf; - self->buf_size = preferred_size; - self->string_size = (Py_ssize_t) copy_size; - } - return 0; -} /* Internal routine to get a line from the buffer of a BytesIO object. Returns the length between the current position to the @@ -89,7 +53,7 @@ assert(self->buf != NULL); /* Move to the end of the line, up to the end of the string, s. */ - start = self->buf + self->pos; + start = PyBytes_AS_STRING(self->buf) + self->pos; maxlen = self->string_size - self->pos; if (len < 0 || len > maxlen) len = maxlen; @@ -107,6 +71,27 @@ return len; } +/* Internal routine for detaching the shared buffer of BytesIO objects. + The caller should ensure that the 'size' argument is non-negative and + not lesser than self->string_size. Returns 0 on success, -1 otherwise. */ +static int +unshare_buffer(bytesio *self, size_t size) +{ + PyObject *new_buf, *old_buf; + assert(SHARED_BUF(self)); + assert(self->exports == 0); + assert(size >= self->string_size); + new_buf = PyBytes_FromStringAndSize(NULL, size); + if (new_buf == NULL) + return -1; + memcpy(PyBytes_AS_STRING(new_buf), PyBytes_AS_STRING(self->buf), + self->string_size); + old_buf = self->buf; + self->buf = new_buf; + Py_DECREF(old_buf); + return 0; +} + /* Internal routine for changing the size of the buffer of BytesIO objects. The caller should ensure that the 'size' argument is non-negative. Returns 0 on success, -1 otherwise. */ @@ -115,8 +100,7 @@ { /* Here, unsigned types are used to avoid dealing with signed integer overflow, which is undefined in C. */ - size_t alloc = self->buf_size; - char *new_buf = NULL; + size_t alloc = PyBytes_GET_SIZE(self->buf); assert(self->buf != NULL); @@ -127,30 +111,32 @@ if (size < alloc / 2) { /* Major downsize; resize down to exact size. */ - alloc = size + 1; + alloc = size; } else if (size < alloc) { /* Within allocated size; quick exit */ return 0; } - else if (size <= alloc * 1.125) { + else if (size <= alloc + (alloc / 2)) { /* Moderate upsize; overallocate similar to list_resize() */ - alloc = size + (size >> 3) + (size < 9 ? 3 : 6); + alloc = size + (size / 2) + (size < 9 ? 3 : 6); } else { /* Major upsize; resize up to exact size */ - alloc = size + 1; + alloc = size; } if (alloc > ((size_t)-1) / sizeof(char)) goto overflow; - new_buf = (char *)PyMem_Realloc(self->buf, alloc * sizeof(char)); - if (new_buf == NULL) { - PyErr_NoMemory(); - return -1; + + if (SHARED_BUF(self)) { + if (unshare_buffer(self, alloc) < 0) + return -1; } - self->buf_size = alloc; - self->buf = new_buf; + else { + if (_PyBytes_Resize(&self->buf, alloc) < 0) + return -1; + } return 0; @@ -165,19 +151,16 @@ static Py_ssize_t write_bytes(bytesio *self, const char *bytes, Py_ssize_t len) { - size_t desired; - assert(self->buf != NULL); assert(self->pos >= 0); assert(len >= 0); - desired = (size_t)self->pos + len; - if (unshare(self, desired, 0) < 0) { - return -1; + if ((size_t)self->pos + len > PyBytes_GET_SIZE(self->buf)) { + if (resize_buffer(self, (size_t)self->pos + len) < 0) + return -1; } - - if (desired > self->buf_size) { - if (resize_buffer(self, (size_t)self->pos + len) < 0) + else if (SHARED_BUF(self)) { + if (unshare_buffer(self, self->string_size) < 0) return -1; } @@ -190,13 +173,13 @@ | | <--to pad-->|<---to write---> | 0 buf position */ - memset(self->buf + self->string_size, '\0', + memset(PyBytes_AS_STRING(self->buf) + self->string_size, '\0', (self->pos - self->string_size) * sizeof(char)); } /* Copy the data to the internal buffer, overwriting some of the existing data if self->pos < self->string_size. */ - memcpy(self->buf + self->pos, bytes, len); + memcpy(PyBytes_AS_STRING(self->buf) + self->pos, bytes, len); self->pos += len; /* Set the new length of the internal string if it has changed. */ @@ -207,74 +190,6 @@ return len; } -/* Release or free any existing buffer, and place the BytesIO in the closed - * state. */ -static void -reset(bytesio *self) -{ - if (self->initvalue) { - Py_CLEAR(self->initvalue); - } else if (self->buf) { - PyMem_Free(self->buf); - } - self->buf = NULL; - self->string_size = 0; - self->pos = 0; -} - -/* Reinitialize with a new heap-allocated buffer of size `size`. Returns 0 on - * success, or sets an exception and returns -1 on failure. Existing state is - * preserved on failure. */ -static int -reinit_private(bytesio *self, Py_ssize_t size) -{ - char *tmp = (char *)PyMem_Malloc(size); - if (tmp == NULL) { - PyErr_NoMemory(); - return -1; - } - reset(self); - self->buf = tmp; - self->buf_size = size; - return 0; -} - -/* Internal version of BytesIO.__init__; resets the object to its initial - * (closed) state before repopulating it, optionally by sharing a PyBytes - * buffer provided by `initvalue'. Returns 0 on success, or sets an exception - * and returns -1 on failure. */ -static int -reinit(bytesio *self, PyObject *initvalue) -{ - CHECK_CLOSED(self, -1); - - if (initvalue == NULL || initvalue == Py_None) { - if (reinit_private(self, 0) < 0) { - return -1; - } - } else if (PyBytes_CheckExact(initvalue)) { - reset(self); - Py_INCREF(initvalue); - self->initvalue = initvalue; - self->buf = PyBytes_AS_STRING(initvalue); - self->buf_size = PyBytes_GET_SIZE(initvalue); - self->string_size = PyBytes_GET_SIZE(initvalue); - } else { - Py_buffer buf; - if (PyObject_GetBuffer(initvalue, &buf, PyBUF_CONTIG_RO) < 0) { - return -1; - } - if (reinit_private(self, buf.len) < 0) { - PyBuffer_Release(&buf); - return -1; - } - memcpy(self->buf, buf.buf, buf.len); - self->string_size = buf.len; - PyBuffer_Release(&buf); - } - return 0; -} - static PyObject * bytesio_get_closed(bytesio *self) { @@ -299,7 +214,7 @@ static PyObject * return_not_closed(bytesio *self) { - CHECK_CLOSED(self, NULL); + CHECK_CLOSED(self); Py_RETURN_TRUE; } @@ -309,7 +224,7 @@ static PyObject * bytesio_flush(bytesio *self) { - CHECK_CLOSED(self, NULL); + CHECK_CLOSED(self); Py_RETURN_NONE; } @@ -325,7 +240,7 @@ bytesiobuf *buf; PyObject *view; - CHECK_CLOSED(self, NULL); + CHECK_CLOSED(self); buf = (bytesiobuf *) type->tp_alloc(type, 0); if (buf == NULL) @@ -345,8 +260,23 @@ static PyObject * bytesio_getvalue(bytesio *self) { - CHECK_CLOSED(self, NULL); - return PyBytes_FromStringAndSize(self->buf, self->string_size); + CHECK_CLOSED(self); + if (self->string_size <= 1 || self->exports > 0) + return PyBytes_FromStringAndSize(PyBytes_AS_STRING(self->buf), + self->string_size); + + if (self->string_size != PyBytes_GET_SIZE(self->buf)) { + if (SHARED_BUF(self)) { + if (unshare_buffer(self, self->string_size) < 0) + return NULL; + } + else { + if (_PyBytes_Resize(&self->buf, self->string_size) < 0) + return NULL; + } + } + Py_INCREF(self->buf); + return self->buf; } PyDoc_STRVAR(isatty_doc, @@ -358,7 +288,7 @@ static PyObject * bytesio_isatty(bytesio *self) { - CHECK_CLOSED(self, NULL); + CHECK_CLOSED(self); Py_RETURN_FALSE; } @@ -368,7 +298,7 @@ static PyObject * bytesio_tell(bytesio *self) { - CHECK_CLOSED(self, NULL); + CHECK_CLOSED(self); return PyLong_FromSsize_t(self->pos); } @@ -385,7 +315,7 @@ char *output; PyObject *arg = Py_None; - CHECK_CLOSED(self, NULL); + CHECK_CLOSED(self); if (!PyArg_ParseTuple(args, "|O:read", &arg)) return NULL; @@ -414,7 +344,15 @@ } assert(self->buf != NULL); - output = self->buf + self->pos; + if (size > 1 && + self->pos == 0 && size == PyBytes_GET_SIZE(self->buf) && + self->exports == 0) { + self->pos += size; + Py_INCREF(self->buf); + return self->buf; + } + + output = PyBytes_AS_STRING(self->buf) + self->pos; self->pos += size; return PyBytes_FromStringAndSize(output, size); @@ -454,7 +392,7 @@ char *output; PyObject *arg = Py_None; - CHECK_CLOSED(self, NULL); + CHECK_CLOSED(self); if (!PyArg_ParseTuple(args, "|O:readline", &arg)) return NULL; @@ -476,7 +414,15 @@ n = scan_eol(self, size); - output = self->buf + self->pos; + if (n > 1 && + self->pos == n && n == PyBytes_GET_SIZE(self->buf) && + self->exports == 0) { + self->pos += n; + Py_INCREF(self->buf); + return self->buf; + } + + output = PyBytes_AS_STRING(self->buf) + self->pos; self->pos += n; return PyBytes_FromStringAndSize(output, n); } @@ -496,7 +442,7 @@ char *output; PyObject *arg = Py_None; - CHECK_CLOSED(self, NULL); + CHECK_CLOSED(self); if (!PyArg_ParseTuple(args, "|O:readlines", &arg)) return NULL; @@ -521,7 +467,7 @@ if (!result) return NULL; - output = self->buf + self->pos; + output = PyBytes_AS_STRING(self->buf) + self->pos; while ((n = scan_eol(self, -1)) != 0) { self->pos += n; line = PyBytes_FromStringAndSize(output, n); @@ -556,7 +502,7 @@ void *raw_buffer; Py_ssize_t len, n; - CHECK_CLOSED(self, NULL); + CHECK_CLOSED(self); if (PyObject_AsWriteBuffer(buffer, &raw_buffer, &len) == -1) return NULL; @@ -569,7 +515,7 @@ len = 0; } - memcpy(raw_buffer, self->buf + self->pos, len); + memcpy(raw_buffer, PyBytes_AS_STRING(self->buf) + self->pos, len); assert(self->pos + len < PY_SSIZE_T_MAX); assert(len >= 0); self->pos += len; @@ -589,7 +535,7 @@ Py_ssize_t size; PyObject *arg = Py_None; - CHECK_CLOSED(self, NULL); + CHECK_CLOSED(self); CHECK_EXPORTS(self); if (!PyArg_ParseTuple(args, "|O:truncate", &arg)) @@ -616,10 +562,6 @@ return NULL; } - if (unshare(self, size, 1) < 0) { - return NULL; - } - if (size < self->string_size) { self->string_size = size; if (resize_buffer(self, size) < 0) @@ -635,14 +577,22 @@ const char *next; Py_ssize_t n; - CHECK_CLOSED(self, NULL); + CHECK_CLOSED(self); n = scan_eol(self, -1); if (n == 0) return NULL; - next = self->buf + self->pos; + if (n > 1 && + self->pos == 0 && n == PyBytes_GET_SIZE(self->buf) && + self->exports == 0) { + self->pos += n; + Py_INCREF(self->buf); + return self->buf; + } + + next = PyBytes_AS_STRING(self->buf) + self->pos; self->pos += n; return PyBytes_FromStringAndSize(next, n); } @@ -662,7 +612,7 @@ Py_ssize_t pos; int mode = 0; - CHECK_CLOSED(self, NULL); + CHECK_CLOSED(self); if (!PyArg_ParseTuple(args, "n|i:seek", &pos, &mode)) return NULL; @@ -717,7 +667,7 @@ Py_buffer buf; PyObject *result = NULL; - CHECK_CLOSED(self, NULL); + CHECK_CLOSED(self); CHECK_EXPORTS(self); if (PyObject_GetBuffer(obj, &buf, PyBUF_CONTIG_RO) < 0) @@ -745,7 +695,7 @@ PyObject *it, *item; PyObject *ret; - CHECK_CLOSED(self, NULL); + CHECK_CLOSED(self); it = PyObject_GetIter(v); if (it == NULL) @@ -775,7 +725,7 @@ static PyObject * bytesio_close(bytesio *self) { - reset(self); + Py_CLEAR(self->buf); Py_RETURN_NONE; } @@ -823,11 +773,11 @@ static PyObject * bytesio_setstate(bytesio *self, PyObject *state) { + PyObject *result; PyObject *position_obj; PyObject *dict; Py_ssize_t pos; - CHECK_EXPORTS(self); assert(state != NULL); /* We allow the state tuple to be longer than 3, because we may need @@ -839,13 +789,18 @@ Py_TYPE(self)->tp_name, Py_TYPE(state)->tp_name); return NULL; } + CHECK_EXPORTS(self); + /* Reset the object to its default state. This is only needed to handle + the case of repeated calls to __setstate__. */ + self->string_size = 0; + self->pos = 0; - /* Reset the object to its default state and set the value of the internal - * buffer. If state[0] does not support the buffer protocol, reinit() will - * raise the appropriate TypeError. */ - if (reinit(self, PyTuple_GET_ITEM(state, 0)) < 0) { + /* Set the value of the internal buffer. If state[0] does not support the + buffer protocol, bytesio_write will raise the appropriate TypeError. */ + result = bytesio_write(self, PyTuple_GET_ITEM(state, 0)); + if (result == NULL) return NULL; - } + Py_DECREF(result); /* Set carefully the position value. Alternatively, we could use the seek method instead of modifying self->pos directly to better protect the @@ -900,9 +855,7 @@ "deallocated BytesIO object has exported buffers"); PyErr_Print(); } - - reset(self); - + Py_CLEAR(self->buf); Py_CLEAR(self->dict); if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); @@ -922,7 +875,7 @@ /* tp_alloc initializes all the fields to zero. So we don't have to initialize them here. */ - self->buf = (char *)PyMem_Malloc(0); + self->buf = PyBytes_FromStringAndSize(NULL, 0); if (self->buf == NULL) { Py_DECREF(self); return PyErr_NoMemory(); @@ -941,7 +894,33 @@ &initvalue)) return -1; - return reinit(self, initvalue); + /* In case, __init__ is called multiple times. */ + self->string_size = 0; + self->pos = 0; + + if (self->exports > 0) { + PyErr_SetString(PyExc_BufferError, + "Existing exports of data: object cannot be re-sized"); + return -1; + } + if (initvalue && initvalue != Py_None) { + if (PyBytes_CheckExact(initvalue)) { + Py_INCREF(initvalue); + Py_XDECREF(self->buf); + self->buf = initvalue; + self->string_size = PyBytes_GET_SIZE(initvalue); + } + else { + PyObject *res; + res = bytesio_write(self, initvalue); + if (res == NULL) + return -1; + Py_DECREF(res); + self->pos = 0; + } + } + + return 0; } static PyObject * @@ -950,8 +929,8 @@ Py_ssize_t res; res = sizeof(bytesio); - if (self->buf) - res += self->buf_size; + if (self->buf && !SHARED_BUF(self)) + res += _PySys_GetSizeOf(self->buf); return PyLong_FromSsize_t(res); } @@ -1061,11 +1040,16 @@ { int ret; bytesio *b = (bytesio *) obj->source; + if (SHARED_BUF(b)) { + if (unshare_buffer(b, b->string_size) < 0) + return -1; + } if (view == NULL) { b->exports++; return 0; } - ret = PyBuffer_FillInfo(view, (PyObject*)obj, b->buf, b->string_size, + ret = PyBuffer_FillInfo(view, (PyObject*)obj, + PyBytes_AS_STRING(b->buf), b->string_size, 0, flags); if (ret >= 0) { b->exports++;