diff -r 1dd11321f1de Include/object.h --- a/Include/object.h Mon Dec 08 12:23:22 2008 +0100 +++ b/Include/object.h Tue Dec 09 16:02:25 2008 +0100 @@ -142,18 +142,20 @@ typedef int(*objobjargproc)(PyObject *, /* buffer interface */ typedef struct bufferinfo { - void *buf; - PyObject *obj; /* owned reference */ - Py_ssize_t len; - Py_ssize_t itemsize; /* This is Py_ssize_t so it can be - pointed to by strides in simple case.*/ - int readonly; - int ndim; - char *format; - Py_ssize_t *shape; - Py_ssize_t *strides; - Py_ssize_t *suboffsets; - void *internal; + void *buf; + PyObject *obj; /* owned reference */ + Py_ssize_t len; + Py_ssize_t itemsize; /* This is Py_ssize_t so it can be + pointed to by strides in simple case.*/ + int readonly; + int ndim; + char *format; + Py_ssize_t *shape; + Py_ssize_t *strides; + Py_ssize_t *suboffsets; + Py_ssize_t smalltable[2]; /* static store for shape and strides of + mono-dimensional buffers. */ + void *internal; } Py_buffer; typedef int (*getbufferproc)(PyObject *, Py_buffer *, int); diff -r 1dd11321f1de Lib/test/test_memoryview.py --- a/Lib/test/test_memoryview.py Mon Dec 08 12:23:22 2008 +0100 +++ b/Lib/test/test_memoryview.py Tue Dec 09 16:02:25 2008 +0100 @@ -8,24 +8,30 @@ import test.support import sys import gc import weakref +import array -class CommonMemoryTests: - # - # Tests common to direct memoryviews and sliced memoryviews - # +class AbstractMemoryTests: + source_bytes = b"abcdef" - base_object = b"abcdef" + @property + def _source(self): + return self.source_bytes + + @property + def _types(self): + return filter(None, [self.ro_type, self.rw_type]) def check_getitem_with_type(self, tp): - b = tp(self.base_object) + item = self.getitem_type + b = tp(self._source) oldrefcount = sys.getrefcount(b) m = self._view(b) - self.assertEquals(m[0], b"a") + self.assertEquals(m[0], item(b"a")) self.assert_(isinstance(m[0], bytes), type(m[0])) - self.assertEquals(m[5], b"f") - self.assertEquals(m[-1], b"f") - self.assertEquals(m[-6], b"a") + self.assertEquals(m[5], item(b"f")) + self.assertEquals(m[-1], item(b"f")) + self.assertEquals(m[-6], item(b"a")) # Bounds checking self.assertRaises(IndexError, lambda: m[6]) self.assertRaises(IndexError, lambda: m[-7]) @@ -38,14 +44,14 @@ class CommonMemoryTests: m = None self.assertEquals(sys.getrefcount(b), oldrefcount) - def test_getitem_readonly(self): - self.check_getitem_with_type(bytes) - - def test_getitem_writable(self): - self.check_getitem_with_type(bytearray) + def test_getitem(self): + for tp in self._types: + self.check_getitem_with_type(tp) def test_setitem_readonly(self): - b = self.base_object + if not self.ro_type: + return + b = self.ro_type(self._source) oldrefcount = sys.getrefcount(b) m = self._view(b) def setitem(value): @@ -57,27 +63,30 @@ class CommonMemoryTests: self.assertEquals(sys.getrefcount(b), oldrefcount) def test_setitem_writable(self): - b = bytearray(self.base_object) + if not self.rw_type: + return + tp = self.rw_type + b = self.rw_type(self._source) oldrefcount = sys.getrefcount(b) m = self._view(b) - m[0] = b"0" - self._check_contents(b, b"0bcdef") - m[1:3] = b"12" - self._check_contents(b, b"012def") - m[1:1] = b"" - self._check_contents(b, b"012def") - m[:] = b"abcdef" - self._check_contents(b, b"abcdef") + m[0] = tp(b"0") + self._check_contents(tp, b, b"0bcdef") + m[1:3] = tp(b"12") + self._check_contents(tp, b, b"012def") + m[1:1] = tp(b"") + self._check_contents(tp, b, b"012def") + m[:] = tp(b"abcdef") + self._check_contents(tp, b, b"abcdef") # Overlapping copies of a view into itself m[0:3] = m[2:5] - self._check_contents(b, b"cdedef") - m[:] = b"abcdef" + self._check_contents(tp, b, b"cdedef") + m[:] = tp(b"abcdef") m[2:5] = m[0:3] - self._check_contents(b, b"ababcf") + self._check_contents(tp, b, b"ababcf") def setitem(key, value): - m[key] = value + m[key] = tp(value) # Bounds checking self.assertRaises(IndexError, setitem, 6, b"a") self.assertRaises(IndexError, setitem, -7, b"a") @@ -96,159 +105,224 @@ class CommonMemoryTests: m = None self.assertEquals(sys.getrefcount(b), oldrefcount) - def test_len(self): - self.assertEquals(len(self._view(self.base_object)), 6) - def test_tobytes(self): - m = self._view(self.base_object) - b = m.tobytes() - self.assertEquals(b, b"abcdef") - self.assert_(isinstance(b, bytes), type(b)) + for tp in self._types: + m = self._view(tp(self._source)) + b = m.tobytes() + # This calls self.getitem_type() on each separate byte of b"abcdef" + expected = b"".join( + self.getitem_type(bytes([c])) for c in b"abcdef") + self.assertEquals(b, expected) + self.assert_(isinstance(b, bytes), type(b)) def test_tolist(self): - m = self._view(self.base_object) - l = m.tolist() - self.assertEquals(l, list(b"abcdef")) + for tp in self._types: + m = self._view(tp(self._source)) + l = m.tolist() + self.assertEquals(l, list(b"abcdef")) def test_compare(self): # memoryviews can compare for equality with other objects # having the buffer interface. - m = self._view(self.base_object) - for tp in (bytes, bytearray): - self.assertTrue(m == tp(b"abcdef")) - self.assertFalse(m != tp(b"abcdef")) - self.assertFalse(m == tp(b"abcde")) - self.assertTrue(m != tp(b"abcde")) - self.assertFalse(m == tp(b"abcde1")) - self.assertTrue(m != tp(b"abcde1")) - self.assertTrue(m == m) - self.assertTrue(m == m[:]) - self.assertTrue(m[0:6] == m[:]) - self.assertFalse(m[0:5] == m) - - # Comparison with objects which don't support the buffer API - self.assertFalse(m == "abc") - self.assertTrue(m != "abc") - self.assertFalse("abc" == m) - self.assertTrue("abc" != m) - - # Unordered comparisons - for c in (m, b"abcdef"): - self.assertRaises(TypeError, lambda: m < c) - self.assertRaises(TypeError, lambda: c <= m) - self.assertRaises(TypeError, lambda: m >= c) - self.assertRaises(TypeError, lambda: c > m) + for tp in self._types: + m = self._view(tp(self._source)) + for tp_comp in self._types: + self.assertTrue(m == tp_comp(b"abcdef")) + self.assertFalse(m != tp_comp(b"abcdef")) + self.assertFalse(m == tp_comp(b"abcde")) + self.assertTrue(m != tp_comp(b"abcde")) + self.assertFalse(m == tp_comp(b"abcde1")) + self.assertTrue(m != tp_comp(b"abcde1")) + self.assertTrue(m == m) + self.assertTrue(m == m[:]) + self.assertTrue(m[0:6] == m[:]) + self.assertFalse(m[0:5] == m) + + # Comparison with objects which don't support the buffer API + self.assertFalse(m == "abcdef") + self.assertTrue(m != "abcdef") + self.assertFalse("abcdef" == m) + self.assertTrue("abcdef" != m) + + # Unordered comparisons + for c in (m, b"abcdef"): + self.assertRaises(TypeError, lambda: m < c) + self.assertRaises(TypeError, lambda: c <= m) + self.assertRaises(TypeError, lambda: m >= c) + self.assertRaises(TypeError, lambda: c > m) def check_attributes_with_type(self, tp): - b = tp(self.base_object) - m = self._view(b) - self.assertEquals(m.format, 'B') - self.assertEquals(m.itemsize, 1) + m = self._view(tp(self._source)) + self.assertEquals(m.format, self.format) + self.assertEquals(m.itemsize, self.itemsize) self.assertEquals(m.ndim, 1) self.assertEquals(m.shape, (6,)) - self.assertEquals(len(m), 6) - self.assertEquals(m.strides, (1,)) + self.assertEquals(len(m), 6 * self.itemsize) + self.assertEquals(m.strides, (self.itemsize,)) self.assertEquals(m.suboffsets, None) return m def test_attributes_readonly(self): - m = self.check_attributes_with_type(bytes) + if not self.ro_type: + return + m = self.check_attributes_with_type(self.ro_type) self.assertEquals(m.readonly, True) def test_attributes_writable(self): - m = self.check_attributes_with_type(bytearray) + if not self.rw_type: + return + m = self.check_attributes_with_type(self.rw_type) self.assertEquals(m.readonly, False) def test_getbuffer(self): # Test PyObject_GetBuffer() on a memoryview object. - b = self.base_object - oldrefcount = sys.getrefcount(b) - m = self._view(b) - oldviewrefcount = sys.getrefcount(m) - s = str(m, "utf-8") - self._check_contents(b, s.encode("utf-8")) - self.assertEquals(sys.getrefcount(m), oldviewrefcount) - m = None - self.assertEquals(sys.getrefcount(b), oldrefcount) + for tp in self._types: + b = tp(self._source) + oldrefcount = sys.getrefcount(b) + m = self._view(b) + oldviewrefcount = sys.getrefcount(m) + s = str(m, "utf-8") + self._check_contents(tp, b, s.encode("utf-8")) + self.assertEquals(sys.getrefcount(m), oldviewrefcount) + m = None + self.assertEquals(sys.getrefcount(b), oldrefcount) def test_gc(self): - class MyBytes(bytes): - pass - class MyObject: - pass + for tp in self._types: + if not isinstance(tp, type): + # If tp is a factory rather than a plain type, skip + continue - # Create a reference cycle through a memoryview object - b = MyBytes(b'abc') - m = self._view(b) - o = MyObject() - b.m = m - b.o = o - wr = weakref.ref(o) - b = m = o = None - # The cycle must be broken - gc.collect() - self.assert_(wr() is None, wr()) + class MySource(tp): + pass + class MyObject: + pass + + # Create a reference cycle through a memoryview object + b = MySource(tp(b'abc')) + m = self._view(b) + o = MyObject() + b.m = m + b.o = o + wr = weakref.ref(o) + b = m = o = None + # The cycle must be broken + gc.collect() + self.assert_(wr() is None, wr()) -class MemoryviewTest(unittest.TestCase, CommonMemoryTests): +# Variations on source objects for the buffer: bytes-like objects, then arrays +# with itemsize > 1. +# NOTE: support for multi-dimensional objects is unimplemented. +class BaseBytesMemoryTests(AbstractMemoryTests): + ro_type = bytes + rw_type = bytearray + getitem_type = bytes + itemsize = 1 + format = 'B' + +class BaseArrayMemoryTests(AbstractMemoryTests): + ro_type = None + rw_type = lambda self, b: array.array('i', list(b)) + getitem_type = lambda self, b: array.array('i', list(b)).tostring() + itemsize = array.array('i').itemsize + format = 'i' + + def test_getbuffer(self): + # XXX Test should be adapted for non-byte buffers + pass + + def test_tolist(self): + # XXX NotImplementedError: tolist() only supports byte views + pass + + +# Variations on indirection levels: memoryview, slice of memoryview, +# slice of slice of memoryview. +# This is important to test allocation subtleties. + +class BaseMemoryviewTests: def _view(self, obj): return memoryview(obj) - def _check_contents(self, obj, contents): - self.assertEquals(obj, contents) + def _check_contents(self, tp, obj, contents): + self.assertEquals(obj, tp(contents)) - def test_constructor(self): - ob = b'test' - self.assert_(memoryview(ob)) - self.assert_(memoryview(object=ob)) - self.assertRaises(TypeError, memoryview) - self.assertRaises(TypeError, memoryview, ob, ob) - self.assertRaises(TypeError, memoryview, argument=ob) - self.assertRaises(TypeError, memoryview, ob, argument=True) - - def test_array_assign(self): - # Issue #4569: segfault when mutating a memoryview with itemsize != 1 - from array import array - a = array('i', range(10)) - m = memoryview(a) - new_a = array('i', range(9, -1, -1)) - m[:] = new_a - self.assertEquals(a, new_a) - - -class MemorySliceTest(unittest.TestCase, CommonMemoryTests): - base_object = b"XabcdefY" +class BaseMemorySliceTests: + source_bytes = b"XabcdefY" def _view(self, obj): m = memoryview(obj) return m[1:7] - def _check_contents(self, obj, contents): - self.assertEquals(obj[1:7], contents) + def _check_contents(self, tp, obj, contents): + self.assertEquals(obj[1:7], tp(contents)) def test_refs(self): - m = memoryview(b"ab") - oldrefcount = sys.getrefcount(m) - m[1:2] - self.assertEquals(sys.getrefcount(m), oldrefcount) + for tp in self._types: + m = memoryview(tp(self._source)) + oldrefcount = sys.getrefcount(m) + m[1:2] + self.assertEquals(sys.getrefcount(m), oldrefcount) - -class MemorySliceSliceTest(unittest.TestCase, CommonMemoryTests): - base_object = b"XabcdefY" +class BaseMemorySliceSliceTests: + source_bytes = b"XabcdefY" def _view(self, obj): m = memoryview(obj) return m[:7][1:] - def _check_contents(self, obj, contents): - self.assertEquals(obj[1:7], contents) + def _check_contents(self, tp, obj, contents): + self.assertEquals(obj[1:7], tp(contents)) + + +# Concrete test classes + +class BytesMemoryviewTest(unittest.TestCase, + BaseMemoryviewTests, BaseBytesMemoryTests): + + def test_constructor(self): + for tp in self._types: + ob = tp(self._source) + self.assert_(memoryview(ob)) + self.assert_(memoryview(object=ob)) + self.assertRaises(TypeError, memoryview) + self.assertRaises(TypeError, memoryview, ob, ob) + self.assertRaises(TypeError, memoryview, argument=ob) + self.assertRaises(TypeError, memoryview, ob, argument=True) + +class ArrayMemoryviewTest(unittest.TestCase, + BaseMemoryviewTests, BaseArrayMemoryTests): + + def test_array_assign(self): + # Issue #4569: segfault when mutating a memoryview with itemsize != 1 + a = array.array('i', range(10)) + m = memoryview(a) + new_a = array.array('i', range(9, -1, -1)) + m[:] = new_a + self.assertEquals(a, new_a) + + +class BytesMemorySliceTest(unittest.TestCase, + BaseMemorySliceTests, BaseBytesMemoryTests): + pass + +class ArrayMemorySliceTest(unittest.TestCase, + BaseMemorySliceTests, BaseArrayMemoryTests): + pass + +class BytesMemorySliceSliceTest(unittest.TestCase, + BaseMemorySliceSliceTests, BaseBytesMemoryTests): + pass + +class ArrayMemorySliceSliceTest(unittest.TestCase, + BaseMemorySliceSliceTests, BaseArrayMemoryTests): + pass def test_main(): - test.support.run_unittest( - MemoryviewTest, MemorySliceTest, MemorySliceSliceTest) - + test.support.run_unittest(__name__) if __name__ == "__main__": test_main() diff -r 1dd11321f1de Lib/test/test_sys.py --- a/Lib/test/test_sys.py Mon Dec 08 12:23:22 2008 +0100 +++ b/Lib/test/test_sys.py Tue Dec 09 16:02:25 2008 +0100 @@ -559,7 +559,7 @@ class SizeofTest(unittest.TestCase): check(32768*32768-1, size(vh) + 2*self.H) check(32768*32768, size(vh) + 3*self.H) # memory - check(memoryview(b''), size(h + 'P PP2P2i5P')) + check(memoryview(b''), size(h + 'P PP2P2i7P')) # module check(unittest, size(h + '3P')) # None diff -r 1dd11321f1de Objects/memoryobject.c --- a/Objects/memoryobject.c Mon Dec 08 12:23:22 2008 +0100 +++ b/Objects/memoryobject.c Tue Dec 09 16:02:25 2008 +0100 @@ -3,19 +3,7 @@ #include "Python.h" -static void -dup_buffer(Py_buffer *dest, Py_buffer *src) -{ - *dest = *src; - if (src->shape == &(src->len)) - dest->shape = &(dest->len); - if (src->strides == &(src->itemsize)) - dest->strides = &(dest->itemsize); -} - -/* XXX The buffer API should mandate that the shape array be non-NULL, but - it would complicate some code since the (de)allocation semantics of shape - are not specified. */ +/* XXX The buffer API should mandate that the shape array be non-NULL */ static Py_ssize_t get_shape0(Py_buffer *buf) { @@ -25,6 +13,20 @@ get_shape0(Py_buffer *buf) return buf->len / buf->itemsize; } +static void +dup_buffer(Py_buffer *dest, Py_buffer *src) +{ + *dest = *src; + if (src->ndim == 1) { + dest->shape = &(dest->smalltable[0]); + dest->shape[0] = get_shape0(src); + } + if (src->ndim == 1 && src->strides != NULL) { + dest->strides = &(dest->smalltable[1]); + dest->strides[0] = src->strides[0]; + } +} + static int memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) { @@ -449,8 +451,6 @@ memory_tolist(PyMemoryViewObject *mem, P return res; } - - static PyMethodDef memory_methods[] = { {"tobytes", (PyCFunction)memory_tobytes, METH_NOARGS, NULL}, {"tolist", (PyCFunction)memory_tolist, METH_NOARGS, NULL}, @@ -474,19 +474,19 @@ memory_dealloc(PyMemoryViewObject *self) PyObject_CopyData(PyTuple_GET_ITEM(self->base,0), PyTuple_GET_ITEM(self->base,1)); - /* The view member should have readonly == -1 in - this instance indicating that the memory can - be "locked" and was locked and will be unlocked - again after this call. - */ - PyBuffer_Release(&(self->view)); - } - else { - PyBuffer_Release(&(self->view)); - } - Py_CLEAR(self->base); + /* The view member should have readonly == -1 in + this instance indicating that the memory can + be "locked" and was locked and will be unlocked + again after this call. + */ + PyBuffer_Release(&(self->view)); } - PyObject_GC_Del(self); + else { + PyBuffer_Release(&(self->view)); + } + Py_CLEAR(self->base); + } + PyObject_GC_Del(self); } static PyObject * @@ -512,16 +512,10 @@ memory_str(PyMemoryViewObject *self) } /* Sequence methods */ - static Py_ssize_t memory_length(PyMemoryViewObject *self) { - Py_buffer view; - - if (PyObject_GetBuffer((PyObject *)self, &view, PyBUF_FULL) < 0) - return -1; - PyBuffer_Release(&view); - return view.len; + return self->view.len; } /* @@ -603,26 +597,25 @@ memory_subscript(PyMemoryViewObject *sel int newflags = view->readonly ? PyBUF_CONTIG_RO : PyBUF_CONTIG; - /* XXX There should be an API to create a subbuffer */ - if (view->obj != NULL) { - if (PyObject_GetBuffer(view->obj, - &newview, newflags) == -1) - return NULL; - } - else { - newview = *view; - } - newview.buf = newbuf; - newview.len = slicelength; - newview.format = view->format; - if (view->shape == &(view->len)) - newview.shape = &(newview.len); - if (view->strides == &(view->itemsize)) - newview.strides = &(newview.itemsize); - return PyMemoryView_FromBuffer(&newview); - } - PyErr_SetNone(PyExc_NotImplementedError); - return NULL; + /* XXX There should be an API to create a subbuffer */ + if (view->obj != NULL) { + if (PyObject_GetBuffer(view->obj, + &newview, newflags) == -1) + return NULL; + } + else { + newview = *view; + } + newview.buf = newbuf; + newview.len = slicelength * newview.itemsize; + newview.format = view->format; + newview.shape = &(newview.smalltable[0]); + newview.shape[0] = slicelength; + newview.strides = &(newview.itemsize); + return PyMemoryView_FromBuffer(&newview); + } + PyErr_SetNone(PyExc_NotImplementedError); + return NULL; } PyErr_Format(PyExc_TypeError, "cannot index memory using \"%.200s\"", @@ -747,7 +740,7 @@ memory_richcompare(PyObject *v, PyObject if (vv.itemsize != ww.itemsize || vv.len != ww.len) goto _end; - equal = !memcmp(vv.buf, ww.buf, vv.len * vv.itemsize); + equal = !memcmp(vv.buf, ww.buf, vv.len); _end: PyBuffer_Release(&vv);