diff -r ce52310f61a0 -r 718801740bde Include/memoryobject.h --- a/Include/memoryobject.h Sun Jul 03 13:17:06 2011 +0200 +++ b/Include/memoryobject.h Tue Jul 05 12:34:54 2011 +0200 @@ -6,8 +6,10 @@ extern "C" { #endif +PyAPI_DATA(PyTypeObject) PyManagedBuffer_Type; PyAPI_DATA(PyTypeObject) PyMemoryView_Type; +#define PyManagedBuffer_Check(op) (Py_TYPE(op) == &PyManagedBuffer_Type) #define PyMemoryView_Check(op) (Py_TYPE(op) == &PyMemoryView_Type) #ifndef Py_LIMITED_API @@ -68,6 +70,21 @@ #ifndef Py_LIMITED_API typedef struct { PyObject_HEAD + /* All subsequent memoryviews are based on the master buffer. */ + int released; + Py_buffer master; +} PyManagedBufferObject; + +/* To provide a sliced view, shape, strides and suboffsets may need to be + altered. If ndim <= Py_MEMORYVIEW_MAXSTATIC, the view uses the static + arrays. Otherwise, dynamic memory will be allocated. */ +#define Py_MEMORYVIEW_MAXSTATIC 3 +typedef struct { + PyObject_HEAD + PyManagedBufferObject *mbuf; + Py_ssize_t shape[Py_MEMORYVIEW_MAXSTATIC]; + Py_ssize_t strides[Py_MEMORYVIEW_MAXSTATIC]; + Py_ssize_t suboffsets[Py_MEMORYVIEW_MAXSTATIC]; Py_buffer view; } PyMemoryViewObject; #endif diff -r ce52310f61a0 -r 718801740bde Include/object.h --- a/Include/object.h Sun Jul 03 13:17:06 2011 +0200 +++ b/Include/object.h Tue Jul 05 12:34:54 2011 +0200 @@ -157,8 +157,6 @@ Py_ssize_t *shape; Py_ssize_t *strides; Py_ssize_t *suboffsets; - Py_ssize_t smalltable[2]; /* static store for shape and strides of - mono-dimensional buffers. */ void *internal; } Py_buffer; diff -r ce52310f61a0 -r 718801740bde Lib/test/test_sys.py --- a/Lib/test/test_sys.py Sun Jul 03 13:17:06 2011 +0200 +++ b/Lib/test/test_sys.py Tue Jul 05 12:34:54 2011 +0200 @@ -770,7 +770,7 @@ check(int(PyLong_BASE**2-1), size(vh) + 2*self.longdigit) check(int(PyLong_BASE**2), size(vh) + 3*self.longdigit) # memory - check(memoryview(b''), size(h + 'PP2P2i7P')) + check(memoryview(b''), size(h + 'P3P3P3P2P2P2i5P')) # module check(unittest, size(h + '3P')) # None diff -r ce52310f61a0 -r 718801740bde Objects/memoryobject.c --- a/Objects/memoryobject.c Sun Jul 03 13:17:06 2011 +0200 +++ b/Objects/memoryobject.c Tue Jul 05 12:34:54 2011 +0200 @@ -1,10 +1,150 @@ - /* Memoryview object implementation */ #include "Python.h" +/* + ManagedBuffer Object: + --------------------- + + The purpose of this object is to facilitate the handling of chained + memoryviews that have the same underlying exporting object. PEP-3118 + allows the underlying object to change while a view is exported. This + could lead to unexpected results when constructing a new memoryview + from an existing memoryview. + + Rather than repeatedly redirecting buffer requests to the original base + object, all chained memoryviews use a single buffer snapshot. This + snapshot is generated by the constructor PyManagedBuffer_FromObject(). + + Ownership rules: + ---------------- + + The master buffer inside a PyManagedBuffer is filled in by the original + base object and is read-only for all consumers. + + A new MemoryView makes a private copy of the master buffer. All arrays + pointed to in the copy are shared and thus read-only. If they need to + be modified, either the static arrays in the struct or newly allocated + memory must be used. + + Reference count assumptions: + ---------------------------- + + The 'obj' member of a Py_buffer must either be NULL or refer to the + exporting base object. In the Python codebase, all getbufferprocs + return a new reference to view.obj (example: bytes_buffer_getbuffer). + + XXX view.obj should be documented and getbufferproc should be REQUIRED + XXX to return a new reference to view.obj. + + PyBuffer_Release() decrements view.obj (if non-NULL), so the + releasebufferprocs should NOT decrement view.obj. +*/ + +PyObject * +PyManagedBuffer_FromObject(PyObject *base) +{ + PyManagedBufferObject *mbuf; + + mbuf = (PyManagedBufferObject *) + PyObject_GC_New(PyManagedBufferObject, &PyManagedBuffer_Type); + if (mbuf == NULL) + return NULL; + mbuf->master.obj = NULL; + mbuf->released = 0; + + /* If 'base' supports writable buffer exports, the memoryview object + will be readable and writable, otherwise it will be read-only. */ + if (PyObject_GetBuffer(base, &mbuf->master, PyBUF_FULL) < 0) { + PyErr_Clear(); + if (PyObject_GetBuffer(base, &mbuf->master, PyBUF_FULL_RO) < 0) { + _PyObject_GC_TRACK(mbuf); + Py_DECREF(mbuf); + return NULL; + } + } + + /* Assume that master.obj is a new reference to base. */ + assert(mbuf->master.obj == base); + + _PyObject_GC_TRACK(mbuf); + return (PyObject *)mbuf; +} + +/* XXX Do we need mbuf_new()? */ + +static int +mbuf_release(PyManagedBufferObject *self) +{ + if (self->released) + return 0; + + if (Py_REFCNT(self) > 1) { + PyErr_SetString(PyExc_BufferError, "several memoryviews are active"); + return -1; + } + + /* PyBuffer_Release decrements master.obj. */ + PyBuffer_Release(&(self->master)); + self->released = 1; + return 0; +} + +static void +mbuf_dealloc(PyManagedBufferObject *self) +{ + _PyObject_GC_UNTRACK(self); + (void)mbuf_release(self); + PyObject_GC_Del(self); +} + +static int +mbuf_traverse(PyManagedBufferObject *self, visitproc visit, void *arg) +{ + if (self->master.obj != NULL) + Py_VISIT(self->master.obj); + return 0; +} + +static int +mbuf_clear(PyManagedBufferObject *self) +{ + PyBuffer_Release(&self->master); + self->released = 1; + return 0; +} + +PyTypeObject PyManagedBuffer_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "managedbuffer", + sizeof(PyManagedBufferObject), + 0, + (destructor)mbuf_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)mbuf_traverse, /* tp_traverse */ + (inquiry)mbuf_clear /* tp_clear */ +}; + + +/* Actual MemoryView Object */ + #define IS_RELEASED(memobj) \ - (((PyMemoryViewObject *) memobj)->view.buf == NULL) + (((PyMemoryViewObject *) memobj)->mbuf->released) #define CHECK_RELEASED(memobj) \ if (IS_RELEASED(memobj)) { \ @@ -20,6 +160,20 @@ return -1; \ } +#define CHECK_SHAPE(buf) \ + if (buf->shape == NULL) { \ + PyErr_SetString(PyExc_TypeError, \ + "shape information missing"); \ + return NULL; \ + } + +#define CHECK_SHAPE_INT(buf) \ + if (buf->shape == NULL) { \ + PyErr_SetString(PyExc_TypeError, \ + "shape information missing"); \ + return -1; \ + } + static Py_ssize_t get_shape0(Py_buffer *buf) { @@ -33,38 +187,6 @@ return -1; } -static void -dup_buffer(Py_buffer *dest, Py_buffer *src) -{ - *dest = *src; - if (src->ndim == 1 && src->shape != NULL) { - dest->shape = &(dest->smalltable[0]); - dest->shape[0] = get_shape0(src); - } - if (src->ndim == 1 && src->strides != NULL) { - dest->strides = &(dest->smalltable[1]); - dest->strides[0] = src->strides[0]; - } -} - -static int -memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) -{ - int res = 0; - CHECK_RELEASED_INT(self); - if (self->view.obj != NULL) - res = PyObject_GetBuffer(self->view.obj, view, flags); - if (view) - dup_buffer(view, &self->view); - return res; -} - -static void -memory_releasebuf(PyMemoryViewObject *self, Py_buffer *view) -{ - PyBuffer_Release(view); -} - PyDoc_STRVAR(memory_doc, "memoryview(object)\n\ \n\ @@ -73,46 +195,68 @@ PyObject * PyMemoryView_FromBuffer(Py_buffer *info) { - PyMemoryViewObject *mview; + PyManagedBufferObject *mbuf; + PyObject *mview; if (info->buf == NULL) { PyErr_SetString(PyExc_ValueError, "cannot make memory view from a buffer with a NULL data pointer"); return NULL; } - mview = (PyMemoryViewObject *) - PyObject_GC_New(PyMemoryViewObject, &PyMemoryView_Type); - if (mview == NULL) + + mbuf = (PyManagedBufferObject *) + PyObject_GC_New(PyManagedBufferObject, &PyManagedBuffer_Type); + if (mbuf == NULL) return NULL; - dup_buffer(&mview->view, info); - /* NOTE: mview->view.obj should already have been incref'ed as - part of PyBuffer_FillInfo(). */ - _PyObject_GC_TRACK(mview); - return (PyObject *)mview; + + /* info->obj is either NULL or we own the reference. */ + mbuf->released = 0; + mbuf->master = *info; + + _PyObject_GC_TRACK(mbuf); + mview = PyMemoryView_FromObject((PyObject *)mbuf); + Py_DECREF(mbuf); + return mview; } PyObject * -PyMemoryView_FromObject(PyObject *base) +PyMemoryView_FromObject(PyObject *v) { PyMemoryViewObject *mview; - Py_buffer view; + PyManagedBufferObject *mbuf; - if (!PyObject_CheckBuffer(base)) { + if (PyMemoryView_Check(v)) { + CHECK_RELEASED(v); + mbuf = ((PyMemoryViewObject *)v)->mbuf; + Py_INCREF(mbuf); + } + else if (PyObject_CheckBuffer(v)) { + mbuf = (PyManagedBufferObject *)PyManagedBuffer_FromObject(v); + if (mbuf == NULL) + return NULL; + } + else if (PyManagedBuffer_Check(v)) { + mbuf = (PyManagedBufferObject *)v; + Py_INCREF(mbuf); + } + else { PyErr_SetString(PyExc_TypeError, "cannot make memory view because object does " "not have the buffer interface"); return NULL; } - if (PyObject_GetBuffer(base, &view, PyBUF_FULL_RO) < 0) - return NULL; - - mview = (PyMemoryViewObject *)PyMemoryView_FromBuffer(&view); + mview = (PyMemoryViewObject *) + PyObject_GC_New(PyMemoryViewObject, &PyMemoryView_Type); if (mview == NULL) { - PyBuffer_Release(&view); + Py_DECREF(mbuf); return NULL; } + mview->mbuf = mbuf; + mview->view = mbuf->master; + + _PyObject_GC_TRACK(mview); return (PyObject *)mview; } @@ -130,6 +274,77 @@ return PyMemoryView_FromObject(obj); } +/* Buffer methods */ +static int +memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) +{ + CHECK_RELEASED_INT(self); + + /* XXX All flags are ignored, except for this basic check. */ + if ((flags&PyBUF_WRITABLE) && self->view.readonly) + return -1; + + *view = self->view; + view->obj = (PyObject *)self; + Py_INCREF(self); + Py_INCREF(self->mbuf); + return 0; +} + +static void +memory_releasebuf(PyMemoryViewObject *self, Py_buffer *view) +{ + /* self->view.obj is decremented in PyBuffer_Release(). */ + Py_DECREF(self->mbuf); + return; +} + +static PyObject * +memory_release(PyMemoryViewObject *self) +{ + if (!IS_RELEASED(self)) { + if (mbuf_release(self->mbuf) < 0) + return NULL; + self->view.obj = NULL; + self->view.buf = NULL; + } + Py_RETURN_NONE; +} + +static void +memory_dealloc(PyMemoryViewObject *self) +{ + _PyObject_GC_UNTRACK(self); + Py_DECREF(self->mbuf); + PyObject_GC_Del(self); +} + +static int +memory_traverse(PyMemoryViewObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->mbuf); + return 0; +} + +static int +memory_clear(PyMemoryViewObject *self) +{ + return 0; +} + +static PyObject * +memory_enter(PyObject *self, PyObject *args) +{ + CHECK_RELEASED(self); + Py_INCREF(self); + return self; +} + +static PyObject * +memory_exit(PyObject *self, PyObject *args) +{ + return memory_release((PyMemoryViewObject *)self); +} static void _strided_copy_nd(char *dest, char *src, int nd, Py_ssize_t *shape, @@ -262,7 +477,6 @@ PyMemoryViewObject *mem; PyObject *bytes; Py_buffer *view; - int flags; char *dest; if (!PyObject_CheckBuffer(obj)) { @@ -271,26 +485,23 @@ return NULL; } - mem = PyObject_GC_New(PyMemoryViewObject, &PyMemoryView_Type); + mem = (PyMemoryViewObject *)PyMemoryView_FromObject(obj); if (mem == NULL) return NULL; view = &mem->view; - flags = PyBUF_FULL_RO; - switch(buffertype) { - case PyBUF_WRITE: - flags = PyBUF_FULL; - break; - } - - if (PyObject_GetBuffer(obj, view, flags) != 0) { + if ((buffertype == PyBUF_WRITE && view->readonly) || + (buffertype == PyBUF_READ && !view->readonly)) { + /* XXX Should PyMemoryView_FromObject take a flags parameter? */ + PyErr_SetString(PyExc_BufferError, + "mismatch between requested buffertype and " + "underlying PyManagedBuffer"); Py_DECREF(mem); return NULL; } if (PyBuffer_IsContiguous(view, fort)) { /* no copy needed */ - _PyObject_GC_TRACK(mem); return (PyObject *)mem; } /* otherwise a copy is needed */ @@ -298,9 +509,18 @@ Py_DECREF(mem); PyErr_SetString(PyExc_BufferError, "writable contiguous buffer requested " - "for a non-contiguousobject."); + "for a non-contiguous object."); return NULL; } + + PyErr_SetString(PyExc_NotImplementedError, + "feature is not implemented"); + Py_DECREF(mem); + return NULL; + /* XXX NOT REACHED: The following lines need a test case. Currently + 'dest' is filled in but not used: 'mem' is returned unchanged. + */ + bytes = PyBytes_FromStringAndSize(NULL, view->len); if (bytes == NULL) { Py_DECREF(mem); @@ -323,7 +543,6 @@ } PyBuffer_Release(view); /* XXX ? */ } - _PyObject_GC_TRACK(mem); return (PyObject *)mem; } @@ -460,49 +679,16 @@ return res; } -static void -do_release(PyMemoryViewObject *self) -{ - if (self->view.obj != NULL) { - PyBuffer_Release(&(self->view)); - } - self->view.obj = NULL; - self->view.buf = NULL; -} - -static PyObject * -memory_enter(PyObject *self, PyObject *args) -{ - CHECK_RELEASED(self); - Py_INCREF(self); - return self; -} - -static PyObject * -memory_exit(PyObject *self, PyObject *args) -{ - do_release((PyMemoryViewObject *) self); - Py_RETURN_NONE; -} - static PyMethodDef memory_methods[] = { - {"release", memory_exit, METH_NOARGS}, - {"tobytes", (PyCFunction)memory_tobytes, METH_NOARGS, NULL}, - {"tolist", (PyCFunction)memory_tolist, METH_NOARGS, NULL}, + {"release", (PyCFunction)memory_release, METH_NOARGS}, + {"tobytes", (PyCFunction)memory_tobytes, METH_NOARGS, NULL}, + {"tolist", (PyCFunction)memory_tolist, METH_NOARGS, NULL}, {"__enter__", memory_enter, METH_NOARGS}, - {"__exit__", memory_exit, METH_VARARGS}, - {NULL, NULL} /* sentinel */ + {"__exit__", memory_exit, METH_VARARGS}, + {NULL, NULL} }; -static void -memory_dealloc(PyMemoryViewObject *self) -{ - _PyObject_GC_UNTRACK(self); - do_release(self); - PyObject_GC_Del(self); -} - static PyObject * memory_repr(PyMemoryViewObject *self) { @@ -512,11 +698,13 @@ return PyUnicode_FromFormat("", self); } + /* Sequence methods */ static Py_ssize_t memory_length(PyMemoryViewObject *self) { CHECK_RELEASED_INT(self); + /* XXX handle ndim > 1 */ return get_shape0(&self->view); } @@ -536,12 +724,14 @@ } if (view->ndim == 1) { /* Return a bytes object */ - char *ptr; - ptr = (char *)view->buf; + char *ptr = (char *)view->buf; + + CHECK_SHAPE(view); + if (result < 0) { - result += get_shape0(view); + result += view->shape[0]; } - if ((result < 0) || (result >= get_shape0(view))) { + if ((result < 0) || (result >= view->shape[0])) { PyErr_SetString(PyExc_IndexError, "index out of bounds"); return NULL; @@ -556,14 +746,26 @@ } return PyBytes_FromStringAndSize(ptr, view->itemsize); } else { + PyErr_SetString(PyExc_NotImplementedError, + "multi-dimensional sub-views are not implemented"); + return NULL; +#if 0 /* Return a new memory-view object */ Py_buffer newview; memset(&newview, 0, sizeof(newview)); /* XXX: This needs to be fixed so it actually returns a sub-view */ return PyMemoryView_FromBuffer(&newview); +#endif } } +static void +init_arrays(PyMemoryViewObject *m) +{ + m->view.shape = m->shape; + m->view.strides = m->strides; +} + /* mem[obj] returns a bytes object holding the data for one element if obj fully indexes the memory view or another memory-view object @@ -601,35 +803,40 @@ else if (PySlice_Check(key)) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(key, get_shape0(view), + CHECK_SHAPE(view); + + if (PySlice_GetIndicesEx(key, view->shape[0], &start, &stop, &step, &slicelength) < 0) { return NULL; } if (step == 1 && view->ndim == 1) { - Py_buffer newview; - void *newbuf = (char *) view->buf - + start * view->itemsize; - int newflags = view->readonly - ? PyBUF_CONTIG_RO : PyBUF_CONTIG; - - /* XXX There should be an API to create a subbuffer */ - if (view->obj != NULL) { - if (PyObject_GetBuffer(view->obj, &newview, newflags) == -1) - return NULL; + + PyMemoryViewObject *sliced; + void *newbuf = (char *) view->buf + start * view->itemsize; + + if (!PyBuffer_IsContiguous(&self->view, 'A')) { + PyErr_SetString(PyExc_BufferError, + "sliced view requires a contiguous base"); + return NULL; } - else { - newview = *view; - } - newview.buf = newbuf; - newview.len = slicelength * newview.itemsize; - newview.format = view->format; - newview.shape = &(newview.smalltable[0]); - newview.shape[0] = slicelength; - newview.strides = &(newview.itemsize); - return PyMemoryView_FromBuffer(&newview); + + sliced = (PyMemoryViewObject *) + PyMemoryView_FromObject((PyObject *)self); + if (sliced == NULL) + return NULL; + + init_arrays(sliced); + + sliced->view.buf = newbuf; + sliced->view.len = slicelength * view->itemsize; + sliced->view.format = view->format; + sliced->view.shape[0] = slicelength; + sliced->view.strides[0] = view->itemsize; + return (PyObject *)sliced; } - PyErr_SetNone(PyExc_NotImplementedError); + PyErr_SetString(PyExc_NotImplementedError, + "multi-dimensional slices are not implemented"); return NULL; } PyErr_Format(PyExc_TypeError, @@ -663,14 +870,17 @@ PyErr_SetNone(PyExc_NotImplementedError); return -1; } + + CHECK_SHAPE_INT(view); + if (PyIndex_Check(key)) { start = PyNumber_AsSsize_t(key, NULL); if (start == -1 && PyErr_Occurred()) return -1; if (start < 0) { - start += get_shape0(view); + start += view->shape[0]; } - if ((start < 0) || (start >= get_shape0(view))) { + if ((start < 0) || (start >= view->shape[0])) { PyErr_SetString(PyExc_IndexError, "index out of bounds"); return -1; @@ -680,7 +890,7 @@ else if (PySlice_Check(key)) { Py_ssize_t stop, step; - if (PySlice_GetIndicesEx(key, get_shape0(view), + if (PySlice_GetIndicesEx(key, view->shape[0], &start, &stop, &step, &len) < 0) { return -1; } @@ -778,22 +988,6 @@ } -static int -memory_traverse(PyMemoryViewObject *self, visitproc visit, void *arg) -{ - if (self->view.obj != NULL) - Py_VISIT(self->view.obj); - return 0; -} - -static int -memory_clear(PyMemoryViewObject *self) -{ - PyBuffer_Release(&self->view); - return 0; -} - - /* As mapping */ static PyMappingMethods memory_as_mapping = { (lenfunc)memory_length, /* mp_length */ diff -r ce52310f61a0 -r 718801740bde Objects/object.c --- a/Objects/object.c Sun Jul 03 13:17:06 2011 +0200 +++ b/Objects/object.c Tue Jul 05 12:34:54 2011 +0200 @@ -1409,6 +1409,9 @@ if (PyType_Ready(&PyProperty_Type) < 0) Py_FatalError("Can't initialize property type"); + if (PyType_Ready(&PyManagedBuffer_Type) < 0) + Py_FatalError("Can't initialize managed buffer type"); + if (PyType_Ready(&PyMemoryView_Type) < 0) Py_FatalError("Can't initialize memoryview type");