Index: Lib/test/test_array.py =================================================================== --- Lib/test/test_array.py (revision 75063) +++ Lib/test/test_array.py (working copy) @@ -8,6 +8,8 @@ from weakref import proxy import array, cStringIO from cPickle import loads, dumps, HIGHEST_PROTOCOL +import hashlib +import operator class ArraySubclass(array.array): pass @@ -751,6 +753,38 @@ a = array.array(self.typecode, self.example) b = buffer(a) self.assertEqual(b[0], a.tostring()[0]) + m = memoryview(a) + expected = m.tobytes() + self.assertEqual(a.tostring(), expected) + self.assertEqual(a.tostring()[0], expected[0]) + # Resizing is forbidden when there are buffer exports. + # For issue 4509, we also check after each error that + # the array was not modified. + self.assertRaises(BufferError, a.append, a[0]) + self.assertEqual(m.tobytes(), expected) + self.assertRaises(BufferError, a.extend, a[0:1]) + self.assertEqual(m.tobytes(), expected) + self.assertRaises(BufferError, a.remove, a[0]) + self.assertEqual(m.tobytes(), expected) + self.assertRaises(BufferError, a.pop, 0) + self.assertEqual(m.tobytes(), expected) + self.assertRaises(BufferError, a.fromlist, a.tolist()) + self.assertEqual(m.tobytes(), expected) + self.assertRaises(BufferError, a.fromstring, a.tostring()) + self.assertEqual(m.tobytes(), expected) + if self.typecode == 'u': + self.assertRaises(BufferError, a.fromunicode, a.tounicode()) + self.assertEqual(m.tobytes(), expected) + self.assertRaises(BufferError, operator.imul, a, 2) + self.assertEqual(m.tobytes(), expected) + self.assertRaises(BufferError, operator.imul, a, 0) + self.assertEqual(m.tobytes(), expected) + self.assertRaises(BufferError, operator.setitem, a, slice(0, 0), a) + self.assertEqual(m.tobytes(), expected) + self.assertRaises(BufferError, operator.delitem, a, 0) + self.assertEqual(m.tobytes(), expected) + self.assertRaises(BufferError, operator.delitem, a, slice(0, 1)) + self.assertEqual(m.tobytes(), expected) def test_weakref(self): s = array.array(self.typecode, self.example) @@ -773,6 +807,8 @@ # SF bug #1486663 -- this used to erroneously raise a TypeError ArraySubclassWithKwargs('b', newarg=1) + def test_new_buffer_api(self): + sha1digest = hashlib.sha1(array.array(self.typecode, self.example)).hexdigest() class StringTest(BaseTest): Index: Modules/arraymodule.c =================================================================== --- Modules/arraymodule.c (revision 75063) +++ Modules/arraymodule.c (working copy) @@ -26,6 +26,7 @@ int itemsize; PyObject * (*getitem)(struct arrayobject *, Py_ssize_t); int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *); + char *formats; }; typedef struct arrayobject { @@ -34,6 +35,7 @@ Py_ssize_t allocated; struct arraydescr *ob_descr; PyObject *weakreflist; /* List of weak references */ + int ob_exports; /* Number of exported buffers */ } arrayobject; static PyTypeObject Arraytype; @@ -47,6 +49,12 @@ char *items; size_t _new_size; + if (self->ob_exports > 0 && newsize != Py_SIZE(self)) { + PyErr_SetString(PyExc_BufferError, + "cannot resize an array that is exporting buffers"); + return -1; + } + /* Bypass realloc() when a previous overallocation is large enough to accommodate the newsize. If the newsize is 16 smaller than the current size, then proceed with the realloc() to shrink the list. @@ -59,6 +67,14 @@ return 0; } + if (newsize == 0) { + PyMem_FREE(self->ob_item); + self->ob_item = NULL; + Py_SIZE(self) = 0; + self->allocated = 0; + return 0; + } + /* This over-allocates proportional to the array size, making room * for additional growth. The over-allocation is mild, but is * enough to give linear-time amortized behavior over a long @@ -391,20 +407,20 @@ /* Description of types */ static struct arraydescr descriptors[] = { - {'c', sizeof(char), c_getitem, c_setitem}, - {'b', sizeof(char), b_getitem, b_setitem}, - {'B', sizeof(char), BB_getitem, BB_setitem}, + {'c', sizeof(char), c_getitem, c_setitem, "c"}, + {'b', sizeof(char), b_getitem, b_setitem, "b"}, + {'B', sizeof(char), BB_getitem, BB_setitem, "B"}, #ifdef Py_USING_UNICODE - {'u', sizeof(Py_UNICODE), u_getitem, u_setitem}, + {'u', sizeof(Py_UNICODE), u_getitem, u_setitem, "u"}, #endif - {'h', sizeof(short), h_getitem, h_setitem}, - {'H', sizeof(short), HH_getitem, HH_setitem}, - {'i', sizeof(int), i_getitem, i_setitem}, - {'I', sizeof(int), II_getitem, II_setitem}, - {'l', sizeof(long), l_getitem, l_setitem}, - {'L', sizeof(long), LL_getitem, LL_setitem}, - {'f', sizeof(float), f_getitem, f_setitem}, - {'d', sizeof(double), d_getitem, d_setitem}, + {'h', sizeof(short), h_getitem, h_setitem, "h"}, + {'H', sizeof(short), HH_getitem, HH_setitem, "H"}, + {'i', sizeof(int), i_getitem, i_setitem, "i"}, + {'I', sizeof(int), II_getitem, II_setitem, "I"}, + {'l', sizeof(long), l_getitem, l_setitem, "l"}, + {'L', sizeof(long), LL_getitem, LL_setitem, "L"}, + {'f', sizeof(float), f_getitem, f_setitem, "f"}, + {'d', sizeof(double), d_getitem, d_setitem, "d"}, {'\0', 0, 0, 0} /* Sentinel */ }; @@ -446,6 +462,7 @@ return PyErr_NoMemory(); } } + op->ob_exports = 0; return (PyObject *) op; } @@ -737,6 +754,14 @@ ihigh = Py_SIZE(a); item = a->ob_item; d = n - (ihigh-ilow); + /* Issue #4509: If the array has exported buffers and the slice + assignment would change the size of the array, fail early to make + sure we don't modify it. */ + if (d != 0 && a->ob_exports > 0) { + PyErr_SetString(PyExc_BufferError, + "cannot resize an array that is exporting buffers"); + return -1; + } if (d < 0) { /* Delete -d items */ memmove(item + (ihigh+d)*a->ob_descr->itemsize, item + ihigh*a->ob_descr->itemsize, @@ -814,8 +839,7 @@ static int array_do_extend(arrayobject *self, PyObject *bb) { - Py_ssize_t size; - char *old_item; + Py_ssize_t size, oldsize, bbsize; if (!array_Check(bb)) return array_iter_extend(self, bb); @@ -830,18 +854,14 @@ PyErr_NoMemory(); return -1; } - size = Py_SIZE(self) + Py_SIZE(b); - old_item = self->ob_item; - PyMem_RESIZE(self->ob_item, char, size*self->ob_descr->itemsize); - if (self->ob_item == NULL) { - self->ob_item = old_item; - PyErr_NoMemory(); + oldsize = Py_SIZE(self); + /* Get the size of bb before resizing the array since bb could be self. */ + bbsize = Py_SIZE(bb); + size = oldsize + Py_SIZE(b); + if (array_resize(self, size) == -1) return -1; - } - memcpy(self->ob_item + Py_SIZE(self)*self->ob_descr->itemsize, - b->ob_item, Py_SIZE(b)*b->ob_descr->itemsize); - Py_SIZE(self) = size; - self->allocated = size; + memcpy(self->ob_item + oldsize * self->ob_descr->itemsize, + b->ob_item, bbsize * b->ob_descr->itemsize); return 0; #undef b @@ -877,27 +897,15 @@ return PyErr_NoMemory(); } size = Py_SIZE(self) * self->ob_descr->itemsize; - if (n == 0) { - PyMem_FREE(items); - self->ob_item = NULL; - Py_SIZE(self) = 0; - self->allocated = 0; + if (n > 0 && size > PY_SSIZE_T_MAX / n) { + return PyErr_NoMemory(); } - else { - if (size > PY_SSIZE_T_MAX / n) { - return PyErr_NoMemory(); - } - PyMem_RESIZE(items, char, n * size); - if (items == NULL) - return PyErr_NoMemory(); - p = items; - for (i = 1; i < n; i++) { - p += size; - memcpy(p, items, size); - } - self->ob_item = items; - Py_SIZE(self) *= n; - self->allocated = Py_SIZE(self); + if (array_resize(self, n * Py_SIZE(self)) == -1) + return NULL; + items = p = self->ob_item; + for (i = 1; i < n; i++) { + p += size; + memcpy(p, items, size); } } Py_INCREF(self); @@ -1297,7 +1305,6 @@ array_fromlist(arrayobject *self, PyObject *list) { Py_ssize_t n; - Py_ssize_t itemsize = self->ob_descr->itemsize; if (!PyList_Check(list)) { PyErr_SetString(PyExc_TypeError, "arg must be list"); @@ -1305,28 +1312,15 @@ } n = PyList_Size(list); if (n > 0) { - char *item = self->ob_item; - Py_ssize_t i; - PyMem_RESIZE(item, char, (Py_SIZE(self) + n) * itemsize); - if (item == NULL) { - PyErr_NoMemory(); + Py_ssize_t i, old_size; + old_size = Py_SIZE(self); + if (array_resize(self, old_size + n) == -1) return NULL; - } - self->ob_item = item; - Py_SIZE(self) += n; - self->allocated = Py_SIZE(self); for (i = 0; i < n; i++) { PyObject *v = PyList_GetItem(list, i); if ((*self->ob_descr->setitem)(self, Py_SIZE(self) - n + i, v) != 0) { - Py_SIZE(self) -= n; - if (itemsize && (self->ob_size > PY_SSIZE_T_MAX / itemsize)) { - return PyErr_NoMemory(); - } - PyMem_RESIZE(item, char, - Py_SIZE(self) * itemsize); - self->ob_item = item; - self->allocated = Py_SIZE(self); + array_resize(self, old_size); return NULL; } } @@ -1381,21 +1375,15 @@ } n = n / itemsize; if (n > 0) { - char *item = self->ob_item; - if ((n > PY_SSIZE_T_MAX - Py_SIZE(self)) || - ((Py_SIZE(self) + n) > PY_SSIZE_T_MAX / itemsize)) { + Py_ssize_t old_size = Py_SIZE(self); + if ((n > PY_SSIZE_T_MAX - old_size) || + ((old_size + n) > PY_SSIZE_T_MAX / itemsize)) { return PyErr_NoMemory(); } - PyMem_RESIZE(item, char, (Py_SIZE(self) + n) * itemsize); - if (item == NULL) { - PyErr_NoMemory(); + if (array_resize(self, old_size + n) == -1) return NULL; - } - self->ob_item = item; - Py_SIZE(self) += n; - self->allocated = Py_SIZE(self); - memcpy(item + (Py_SIZE(self) - n) * itemsize, - str, itemsize*n); + memcpy(self->ob_item + old_size * itemsize, + str, n * itemsize); } Py_INCREF(Py_None); return Py_None; @@ -1443,19 +1431,10 @@ return NULL; } if (n > 0) { - Py_UNICODE *item = (Py_UNICODE *) self->ob_item; - if (Py_SIZE(self) > PY_SSIZE_T_MAX - n) { - return PyErr_NoMemory(); - } - PyMem_RESIZE(item, Py_UNICODE, Py_SIZE(self) + n); - if (item == NULL) { - PyErr_NoMemory(); + Py_ssize_t old_size = Py_SIZE(self); + if (array_resize(self, old_size + n) == -1) return NULL; - } - self->ob_item = (char *) item; - Py_SIZE(self) += n; - self->allocated = Py_SIZE(self); - memcpy(item + Py_SIZE(self) - n, + memcpy(self->ob_item + old_size * sizeof(Py_UNICODE), ustr, n * sizeof(Py_UNICODE)); } @@ -1768,6 +1747,16 @@ if ((step > 0 && stop < start) || (step < 0 && stop > start)) stop = start; + + /* Issue #4509: If the array has exported buffers and the slice + assignment would change the size of the array, fail early to make + sure we don't modify it. */ + if ((needed == 0 || slicelength != needed) && self->ob_exports > 0) { + PyErr_SetString(PyExc_BufferError, + "cannot resize an array that is exporting buffers"); + return -1; + } + if (step == 1) { if (slicelength > needed) { memmove(self->ob_item + (start + needed) * itemsize, @@ -1883,6 +1872,50 @@ return 1; } +static int +array_buffer_getbuf(arrayobject *self, Py_buffer *view, int flags) +{ + if (view==NULL) goto finish; + + view->buf = (void *)self->ob_item; + view->obj = (PyObject*)self; + Py_INCREF(self); + if (view->buf == NULL) + view->buf = (void *)emptybuf; + view->len = (Py_SIZE(self)) * self->ob_descr->itemsize; + view->readonly = 0; + view->ndim = 1; + view->itemsize = self->ob_descr->itemsize; + view->suboffsets = NULL; + view->shape = NULL; + if ((flags & PyBUF_ND)==PyBUF_ND) { + view->shape = &((Py_SIZE(self))); + } + view->strides = NULL; + if ((flags & PyBUF_STRIDES)==PyBUF_STRIDES) + view->strides = &(view->itemsize); + view->format = NULL; + view->internal = NULL; + if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) { + view->format = self->ob_descr->formats; +#ifdef Py_UNICODE_WIDE + if (self->ob_descr->typecode == 'u') { + view->format = "w"; + } +#endif + } + + finish: + self->ob_exports++; + return 0; +} + +static void +array_buffer_relbuf(arrayobject *self, Py_buffer *view) +{ + self->ob_exports--; +} + static PySequenceMethods array_as_sequence = { (lenfunc)array_length, /*sq_length*/ (binaryfunc)array_concat, /*sq_concat*/ @@ -1901,6 +1934,8 @@ (writebufferproc)array_buffer_getwritebuf, (segcountproc)array_buffer_getsegcount, NULL, + (getbufferproc)array_buffer_getbuf, + (releasebufferproc)array_buffer_relbuf }; static PyObject * @@ -2097,7 +2132,8 @@ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ &array_as_buffer, /* tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_WEAKREFS, /* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_WEAKREFS | Py_TPFLAGS_HAVE_NEWBUFFER, /* tp_flags */ arraytype_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */