# HG changeset patch # User Antoine Pitrou # Date 1236984626 -3600 diff -r d4af54cc46df -r 474bd6421c3f Doc/c-api/buffer.rst --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -2,10 +2,11 @@ .. _bufferobjects: -Buffer Objects --------------- +Buffers and Memoryview Objects +------------------------------ .. sectionauthor:: Greg Stein +.. sectionauthor:: Benjamin Peterson .. index:: @@ -28,9 +29,296 @@ interface can be written to a file. Ther :cfunc:`PyArg_ParseTuple` that operate against an object's buffer interface, returning data from the target object. +Starting from version 1.6, Python has been providing Python-level buffer +objects and a C-level buffer API so that any builtin or used-defined type +can expose its characteristics. Both, however, have been deprecated because +of various shortcomings, and have been officially removed in Python 3.0 in +favour of a new C-level buffer API and a new Python-level object named +:class:`memoryview`. + +The new buffer API has been backported to Python 2.6, and the +:class:`memoryview` object has been backported to Python 2.7. It is strongly +advised to use them rather than the old APIs, unless you are blocked from +doing so for compatibility reasons. + + +The new-style Py_buffer struct +============================== + + +.. ctype:: Py_buffer + + .. cmember:: void *buf + + A pointer to the start of the memory for the object. + + .. cmember:: Py_ssize_t len + :noindex: + + The total length of the memory in bytes. + + .. cmember:: int readonly + + An indicator of whether the buffer is read only. + + .. cmember:: const char *format + :noindex: + + A *NULL* terminated string in :mod:`struct` module style syntax giving the + contents of the elements available through the buffer. If this is *NULL*, + ``"B"`` (unsigned bytes) is assumed. + + .. cmember:: int ndim + + The number of dimensions the memory represents as a multi-dimensional + array. If it is 0, :cdata:`strides` and :cdata:`suboffsets` must be + *NULL*. + + .. cmember:: Py_ssize_t *shape + + An array of :ctype:`Py_ssize_t`\s the length of :cdata:`ndim` giving the + shape of the memory as a multi-dimensional array. Note that + ``((*shape)[0] * ... * (*shape)[ndims-1])*itemsize`` should be equal to + :cdata:`len`. + + .. cmember:: Py_ssize_t *strides + + An array of :ctype:`Py_ssize_t`\s the length of :cdata:`ndim` giving the + number of bytes to skip to get to a new element in each dimension. + + .. cmember:: Py_ssize_t *suboffsets + + An array of :ctype:`Py_ssize_t`\s the length of :cdata:`ndim`. If these + suboffset numbers are greater than or equal to 0, then the value stored + along the indicated dimension is a pointer and the suboffset value + dictates how many bytes to add to the pointer after de-referencing. A + suboffset value that it negative indicates that no de-referencing should + occur (striding in a contiguous memory block). + + Here is a function that returns a pointer to the element in an N-D array + pointed to by an N-dimesional index when there are both non-NULL strides + and suboffsets:: + + void *get_item_pointer(int ndim, void *buf, Py_ssize_t *strides, + Py_ssize_t *suboffsets, Py_ssize_t *indices) { + char *pointer = (char*)buf; + int i; + for (i = 0; i < ndim; i++) { + pointer += strides[i] * indices[i]; + if (suboffsets[i] >=0 ) { + pointer = *((char**)pointer) + suboffsets[i]; + } + } + return (void*)pointer; + } + + + .. cmember:: Py_ssize_t itemsize + + This is a storage for the itemsize (in bytes) of each element of the + shared memory. It is technically un-necessary as it can be obtained using + :cfunc:`PyBuffer_SizeFromFormat`, however an exporter may know this + information without parsing the format string and it is necessary to know + the itemsize for proper interpretation of striding. Therefore, storing it + is more convenient and faster. + + .. cmember:: void *internal + + This is for use internally by the exporting object. For example, this + might be re-cast as an integer by the exporter and used to store flags + about whether or not the shape, strides, and suboffsets arrays must be + freed when the buffer is released. The consumer should never alter this + value. + + +Buffer related functions +======================== + + +.. cfunction:: int PyObject_CheckBuffer(PyObject *obj) + + Return 1 if *obj* supports the buffer interface otherwise 0. + + +.. cfunction:: int PyObject_GetBuffer(PyObject *obj, PyObject *view, int flags) + + Export *obj* into a :ctype:`Py_buffer`, *view*. These arguments must + never be *NULL*. The *flags* argument is a bit field indicating what kind + of buffer the caller is prepared to deal with and therefore what kind of + buffer the exporter is allowed to return. The buffer interface allows for + complicated memory sharing possibilities, but some caller may not be able + to handle all the complexibity but may want to see if the exporter will + let them take a simpler view to its memory. + + Some exporters may not be able to share memory in every possible way and + may need to raise errors to signal to some consumers that something is + just not possible. These errors should be a :exc:`BufferError` unless + there is another error that is actually causing the problem. The exporter + can use flags information to simplify how much of the :cdata:`Py_buffer` + structure is filled in with non-default values and/or raise an error if + the object can't support a simpler view of its memory. + + 0 is returned on success and -1 on error. + + The following table gives possible values to the *flags* arguments. + + +------------------------------+---------------------------------------------------+ + | Flag | Description | + +==============================+===================================================+ + | :cmacro:`PyBUF_SIMPLE` | This is the default flag state. The returned | + | | buffer may or may not have writable memory. The | + | | format of the data will be assumed to be unsigned | + | | bytes. This is a "stand-alone" flag constant. It | + | | never needs to be '|'d to the others. The exporter| + | | will raise an error if it cannot provide such a | + | | contiguous buffer of bytes. | + | | | + +------------------------------+---------------------------------------------------+ + | :cmacro:`PyBUF_WRITABLE` | The returned buffer must be writable. If it is | + | | not writable, then raise an error. | + +------------------------------+---------------------------------------------------+ + | :cmacro:`PyBUF_STRIDES` | This implies :cmacro:`PyBUF_ND`. The returned | + | | buffer must provide strides information (i.e. the | + | | strides cannot be NULL). This would be used when | + | | the consumer can handle strided, discontiguous | + | | arrays. Handling strides automatically assumes | + | | you can handle shape. The exporter can raise an | + | | error if a strided representation of the data is | + | | not possible (i.e. without the suboffsets). | + | | | + +------------------------------+---------------------------------------------------+ + | :cmacro:`PyBUF_ND` | The returned buffer must provide shape | + | | information. The memory will be assumed C-style | + | | contiguous (last dimension varies the | + | | fastest). The exporter may raise an error if it | + | | cannot provide this kind of contiguous buffer. If | + | | this is not given then shape will be *NULL*. | + | | | + | | | + | | | + +------------------------------+---------------------------------------------------+ + |:cmacro:`PyBUF_C_CONTIGUOUS` | These flags indicate that the contiguity returned | + |:cmacro:`PyBUF_F_CONTIGUOUS` | buffer must be respectively, C-contiguous (last | + |:cmacro:`PyBUF_ANY_CONTIGUOUS`| dimension varies the fastest), Fortran contiguous | + | | (first dimension varies the fastest) or either | + | | one. All of these flags imply | + | | :cmacro:`PyBUF_STRIDES` and guarantee that the | + | | strides buffer info structure will be filled in | + | | correctly. | + | | | + +------------------------------+---------------------------------------------------+ + | :cmacro:`PyBUF_INDIRECT` | This flag indicates the returned buffer must have | + | | suboffsets information (which can be NULL if no | + | | suboffsets are needed). This can be used when | + | | the consumer can handle indirect array | + | | referencing implied by these suboffsets. This | + | | implies :cmacro:`PyBUF_STRIDES`. | + | | | + | | | + | | | + +------------------------------+---------------------------------------------------+ + | :cmacro:`PyBUF_FORMAT` | The returned buffer must have true format | + | | information if this flag is provided. This would | + | | be used when the consumer is going to be checking | + | | for what 'kind' of data is actually stored. An | + | | exporter should always be able to provide this | + | | information if requested. If format is not | + | | explicitly requested then the format must be | + | | returned as *NULL* (which means ``'B'``, or | + | | unsigned bytes) | + +------------------------------+---------------------------------------------------+ + | :cmacro:`PyBUF_STRIDED` | This is equivalent to ``(PyBUF_STRIDES | | + | | PyBUF_WRITABLE)``. | + +------------------------------+---------------------------------------------------+ + | :cmacro:`PyBUF_STRIDED_RO` | This is equivalent to ``(PyBUF_STRIDES)``. | + | | | + +------------------------------+---------------------------------------------------+ + | :cmacro:`PyBUF_RECORDS` | This is equivalent to ``(PyBUF_STRIDES | | + | | PyBUF_FORMAT | PyBUF_WRITABLE)``. | + +------------------------------+---------------------------------------------------+ + | :cmacro:`PyBUF_RECORDS_RO` | This is equivalent to ``(PyBUF_STRIDES | | + | | PyBUF_FORMAT)``. | + +------------------------------+---------------------------------------------------+ + | :cmacro:`PyBUF_FULL` | This is equivalent to ``(PyBUF_INDIRECT | | + | | PyBUF_FORMAT | PyBUF_WRITABLE)``. | + +------------------------------+---------------------------------------------------+ + | :cmacro:`PyBUF_FULL_RO`` | This is equivalent to ``(PyBUF_INDIRECT | | + | | PyBUF_FORMAT)``. | + +------------------------------+---------------------------------------------------+ + | :cmacro:`PyBUF_CONTIG` | This is equivalent to ``(PyBUF_ND | | + | | PyBUF_WRITABLE)``. | + +------------------------------+---------------------------------------------------+ + | :cmacro:`PyBUF_CONTIG_RO` | This is equivalent to ``(PyBUF_ND)``. | + | | | + +------------------------------+---------------------------------------------------+ + + +.. cfunction:: void PyBuffer_Release(PyObject *obj, Py_buffer *view) + + Release the buffer *view* over *obj*. This shouldd be called when the buffer + is no longer being used as it may free memory from it. + + +.. cfunction:: Py_ssize_t PyBuffer_SizeFromFormat(const char *) + + Return the implied :cdata:`~Py_buffer.itemsize` from the struct-stype + :cdata:`~Py_buffer.format`. + + +.. cfunction:: int PyObject_CopyToObject(PyObject *obj, void *buf, Py_ssize_t len, char fortran) + + Copy *len* bytes of data pointed to by the contiguous chunk of memory pointed + to by *buf* into the buffer exported by obj. The buffer must of course be + writable. Return 0 on success and return -1 and raise an error on failure. + If the object does not have a writable buffer, then an error is raised. If + *fortran* is ``'F'``, then if the object is multi-dimensional, then the data + will be copied into the array in Fortran-style (first dimension varies the + fastest). If *fortran* is ``'C'``, then the data will be copied into the + array in C-style (last dimension varies the fastest). If *fortran* is + ``'A'``, then it does not matter and the copy will be made in whatever way is + more efficient. + + +.. cfunction:: int PyBuffer_IsContiguous(Py_buffer *view, char fortran) + + Return 1 if the memory defined by the *view* is C-style (*fortran* is + ``'C'``) or Fortran-style (*fortran* is ``'F'``) contiguous or either one + (*fortran* is ``'A'``). Return 0 otherwise. + + +.. cfunction:: void PyBuffer_FillContiguousStrides(int ndim, Py_ssize_t *shape, Py_ssize_t *strides, Py_ssize_t itemsize, char fortran) + + Fill the *strides* array with byte-strides of a contiguous (C-style if + *fortran* is ``'C'`` or Fortran-style if *fortran* is ``'F'`` array of the + given shape with the given number of bytes per element. + + +.. cfunction:: int PyBuffer_FillInfo(Py_buffer *view, void *buf, Py_ssize_t len, int readonly, int infoflags) + + Fill in a buffer-info structure, *view*, correctly for an exporter that can + only share a contiguous chunk of memory of "unsigned bytes" of the given + length. Return 0 on success and -1 (with raising an error) on error. + + +MemoryView objects +================== + +A memoryview object is an extended buffer object that could replace the buffer +object (but doesn't have to as that could be kept as a simple 1-d memoryview +object). It, unlike :ctype:`Py_buffer`, is a Python object (exposed as +:class:`memoryview` in :mod:`builtins`), so it can be used with Python code. + +.. cfunction:: PyObject* PyMemoryView_FromObject(PyObject *obj) + + Return a memoryview object from an object that defines the buffer interface. + + +Old-style buffer objects +======================== + .. index:: single: PyBufferProcs -More information on the buffer interface is provided in the section +More information on the old buffer interface is provided in the section :ref:`buffer-structs`, under the description for :ctype:`PyBufferProcs`. A "buffer object" is defined in the :file:`bufferobject.h` header (included by diff -r d4af54cc46df -r 474bd6421c3f Doc/c-api/objbuffer.rst --- a/Doc/c-api/objbuffer.rst +++ b/Doc/c-api/objbuffer.rst @@ -2,8 +2,15 @@ .. _abstract-buffer: -Buffer Protocol -=============== + +Old Buffer Protocol +=================== + +This section describes the legacy buffer protocol, which has been introduced +in Python 1.6. It is still supported but deprecated in the Python 2.x series. +Python 3.0 introduces a new buffer protocol which fixes weaknesses and +shortcomings of the protocol, and has been backported to Python 2.6. +See :ref:`bufferobjects` for more information. .. cfunction:: int PyObject_AsCharBuffer(PyObject *obj, const char **buffer, Py_ssize_t *buffer_len) diff -r d4af54cc46df -r 474bd6421c3f Doc/library/functions.rst --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -672,6 +672,13 @@ available. They are listed here in alph Added support for the optional *key* argument. +.. function:: memoryview(obj) + :noindex: + + Return a "memory view" object created from the given argument. See + :ref:`typememoryview` for more information. + + .. function:: min(iterable[, args...][key]) With a single argument *iterable*, return the smallest item of a non-empty diff -r d4af54cc46df -r 474bd6421c3f Doc/library/stdtypes.rst --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2354,6 +2354,104 @@ the particular object. state. +.. _typememoryview: + +memoryview Types +================ + +:class:`memoryview`\s allow Python code to access the internal data of an object +that supports the buffer protocol without copying. Memory can be interpreted as +simple bytes or complex data structures. + +.. class:: memoryview(obj) + + Create a :class:`memoryview` that references *obj*. *obj* must support the + buffer protocol. Builtin objects that support the buffer protocol include + :class:`str` and :class:`bytearray` (but not :class:`unicode`). + + ``len(view)`` returns the total number of bytes in the memoryview, *view*. + + A :class:`memoryview` supports slicing to expose its data. Taking a single + index will return a single byte. Full slicing will result in a subview:: + + >>> v = memoryview('abcefg') + >>> v[1] + 'b' + >>> v[-1] + 'g' + >>> v[1:4] + + >>> str(v[1:4]) + 'bce' + >>> v[3:-1] + + >>> str(v[4:-1]) + 'f' + + If the object the memory view is over supports changing its data, the + memoryview supports slice assignment:: + + >>> data = bytearray('abcefg') + >>> v = memoryview(data) + >>> v.readonly + False + >>> v[0] = 'z' + >>> data + bytearray(b'zbcefg') + >>> v[1:4] = '123' + >>> data + bytearray(b'z123fg') + >>> v[2] = 'spam' + Traceback (most recent call last): + File "", line 1, in + ValueError: cannot modify size of memoryview object + + Notice how the size of the memoryview object can not be changed. + + + :class:`memoryview` has two methods: + + .. method:: tobytes() + + Return the data in the buffer as a bytestring (an object of class + :class:`str`). + + .. method:: tolist() + + Return the data in the buffer as a list of integers. :: + + >>> memoryview(b'abc').tolist() + [97, 98, 99] + + There are also several readonly attributes available: + + .. attribute:: format + + A string containing the format (in :mod:`struct` module style) for each + element in the view. This defaults to ``'B'``, a simple bytestring. + + .. attribute:: itemsize + + The size in bytes of each element of the memoryview. + + .. attribute:: shape + + A tuple of integers the length of :attr:`ndim` giving the shape of the + memory as a N-dimensional array. + + .. attribute:: ndim + + An integer indicating how many dimensions of a multi-dimensional array the + memory represents. + + .. attribute:: strides + + A tuple of integers the length of :attr:`ndim` giving the size in bytes to + access each element for each dimension of the array. + + .. memoryview.suboffsets isn't documented because it only seems useful for C + + .. _typecontextmanager: Context Manager Types diff -r d4af54cc46df -r 474bd6421c3f Doc/tutorial/modules.rst --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -328,8 +328,8 @@ want a list of those, they are defined i 'enumerate', 'eval', 'execfile', 'exit', 'file', 'filter', 'float', 'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'help', 'hex', 'id', 'input', 'int', 'intern', 'isinstance', 'issubclass', 'iter', - 'len', 'license', 'list', 'locals', 'long', 'map', 'max', 'min', - 'object', 'oct', 'open', 'ord', 'pow', 'property', 'quit', 'range', + 'len', 'license', 'list', 'locals', 'long', 'map', 'max', 'memoryview', + 'min', 'object', 'oct', 'open', 'ord', 'pow', 'property', 'quit', 'range', 'raw_input', 'reduce', 'reload', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type', 'unichr', 'unicode', 'vars', 'xrange', 'zip'] diff -r d4af54cc46df -r 474bd6421c3f Include/Python.h --- a/Include/Python.h +++ b/Include/Python.h @@ -92,7 +92,7 @@ #endif #include "rangeobject.h" #include "stringobject.h" -/* #include "memoryobject.h" */ +#include "memoryobject.h" #include "bufferobject.h" #include "bytesobject.h" #include "bytearrayobject.h" diff -r d4af54cc46df -r 474bd6421c3f Include/memoryobject.h --- /dev/null +++ b/Include/memoryobject.h @@ -0,0 +1,74 @@ +/* Memory view object. In Python this is available as "memoryview". */ + +#ifndef Py_MEMORYOBJECT_H +#define Py_MEMORYOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +PyAPI_DATA(PyTypeObject) PyMemoryView_Type; + +#define PyMemoryView_Check(op) (Py_TYPE(op) == &PyMemoryView_Type) + +/* Get a pointer to the underlying Py_buffer of a memoryview object. */ +#define PyMemoryView_GET_BUFFER(op) (&((PyMemoryViewObject *)(op))->view) +/* Get a pointer to the PyObject from which originates a memoryview object. */ +#define PyMemoryView_GET_BASE(op) (((PyMemoryViewObject *)(op))->view.obj) + + +PyAPI_FUNC(PyObject *) PyMemoryView_GetContiguous(PyObject *base, + int buffertype, + char fort); + + /* Return a contiguous chunk of memory representing the buffer + from an object in a memory view object. If a copy is made then the + base object for the memory view will be a *new* bytes object. + + Otherwise, the base-object will be the object itself and no + data-copying will be done. + + The buffertype argument can be PyBUF_READ, PyBUF_WRITE, + PyBUF_SHADOW to determine whether the returned buffer + should be READONLY, WRITABLE, or set to update the + original buffer if a copy must be made. If buffertype is + PyBUF_WRITE and the buffer is not contiguous an error will + be raised. In this circumstance, the user can use + PyBUF_SHADOW to ensure that a a writable temporary + contiguous buffer is returned. The contents of this + contiguous buffer will be copied back into the original + object after the memoryview object is deleted as long as + the original object is writable and allows setting an + exclusive write lock. If this is not allowed by the + original object, then a BufferError is raised. + + If the object is multi-dimensional and if fortran is 'F', + the first dimension of the underlying array will vary the + fastest in the buffer. If fortran is 'C', then the last + dimension will vary the fastest (C-style contiguous). If + fortran is 'A', then it does not matter and you will get + whatever the object decides is more efficient. + + A new reference is returned that must be DECREF'd when finished. + */ + +PyAPI_FUNC(PyObject *) PyMemoryView_FromObject(PyObject *base); + +PyAPI_FUNC(PyObject *) PyMemoryView_FromBuffer(Py_buffer *info); + /* create new if bufptr is NULL + will be a new bytesobject in base */ + + +/* The struct is declared here so that macros can work, but it shouldn't + be considered public. Don't access those fields directly, use the macros + and functions instead! */ +typedef struct { + PyObject_HEAD + PyObject *base; + Py_buffer view; +} PyMemoryViewObject; + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_MEMORYOBJECT_H */ diff -r d4af54cc46df -r 474bd6421c3f Include/object.h --- a/Include/object.h +++ b/Include/object.h @@ -159,21 +159,23 @@ typedef Py_ssize_t (*writebufferproc)(Py typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *); typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **); + /* Py3k buffer interface */ - typedef struct bufferinfo { - void *buf; - PyObject *obj; /* borrowed reference */ - Py_ssize_t len; - Py_ssize_t itemsize; /* This is Py_ssize_t so it can be - pointed to by strides in simple case.*/ - int readonly; - int ndim; - char *format; - Py_ssize_t *shape; - Py_ssize_t *strides; - Py_ssize_t *suboffsets; - void *internal; + void *buf; + PyObject *obj; /* owned reference */ + Py_ssize_t len; + Py_ssize_t itemsize; /* This is Py_ssize_t so it can be + pointed to by strides in simple case.*/ + int readonly; + int ndim; + char *format; + Py_ssize_t *shape; + Py_ssize_t *strides; + Py_ssize_t *suboffsets; + Py_ssize_t smalltable[2]; /* static store for shape and strides of + mono-dimensional buffers. */ + void *internal; } Py_buffer; typedef int (*getbufferproc)(PyObject *, Py_buffer *, int); diff -r d4af54cc46df -r 474bd6421c3f Lib/test/test_memoryview.py --- /dev/null +++ b/Lib/test/test_memoryview.py @@ -0,0 +1,328 @@ +"""Unit tests for the memoryview + +XXX We need more tests! Some tests are in test_bytes +""" + +import unittest +import sys +import gc +import weakref +import array +from test import test_support + + +class AbstractMemoryTests: + source_bytes = b"abcdef" + + @property + def _source(self): + return self.source_bytes + + @property + def _types(self): + return filter(None, [self.ro_type, self.rw_type]) + + def check_getitem_with_type(self, tp): + item = self.getitem_type + b = tp(self._source) + oldrefcount = sys.getrefcount(b) + m = self._view(b) + self.assertEquals(m[0], item(b"a")) + self.assert_(isinstance(m[0], bytes), type(m[0])) + self.assertEquals(m[5], item(b"f")) + self.assertEquals(m[-1], item(b"f")) + self.assertEquals(m[-6], item(b"a")) + # Bounds checking + self.assertRaises(IndexError, lambda: m[6]) + self.assertRaises(IndexError, lambda: m[-7]) + self.assertRaises(IndexError, lambda: m[sys.maxsize]) + self.assertRaises(IndexError, lambda: m[-sys.maxsize]) + # Type checking + self.assertRaises(TypeError, lambda: m[None]) + self.assertRaises(TypeError, lambda: m[0.0]) + self.assertRaises(TypeError, lambda: m["a"]) + m = None + self.assertEquals(sys.getrefcount(b), oldrefcount) + + def test_getitem(self): + for tp in self._types: + self.check_getitem_with_type(tp) + + def test_setitem_readonly(self): + if not self.ro_type: + return + b = self.ro_type(self._source) + oldrefcount = sys.getrefcount(b) + m = self._view(b) + def setitem(value): + m[0] = value + self.assertRaises(TypeError, setitem, b"a") + self.assertRaises(TypeError, setitem, 65) + self.assertRaises(TypeError, setitem, memoryview(b"a")) + m = None + self.assertEquals(sys.getrefcount(b), oldrefcount) + + def test_setitem_writable(self): + if not self.rw_type: + return + tp = self.rw_type + b = self.rw_type(self._source) + oldrefcount = sys.getrefcount(b) + m = self._view(b) + m[0] = tp(b"0") + self._check_contents(tp, b, b"0bcdef") + m[1:3] = tp(b"12") + self._check_contents(tp, b, b"012def") + m[1:1] = tp(b"") + self._check_contents(tp, b, b"012def") + m[:] = tp(b"abcdef") + self._check_contents(tp, b, b"abcdef") + + # Overlapping copies of a view into itself + m[0:3] = m[2:5] + self._check_contents(tp, b, b"cdedef") + m[:] = tp(b"abcdef") + m[2:5] = m[0:3] + self._check_contents(tp, b, b"ababcf") + + def setitem(key, value): + m[key] = tp(value) + # Bounds checking + self.assertRaises(IndexError, setitem, 6, b"a") + self.assertRaises(IndexError, setitem, -7, b"a") + self.assertRaises(IndexError, setitem, sys.maxsize, b"a") + self.assertRaises(IndexError, setitem, -sys.maxsize, b"a") + # Wrong index/slice types + self.assertRaises(TypeError, setitem, 0.0, b"a") + self.assertRaises(TypeError, setitem, (0,), b"a") + self.assertRaises(TypeError, setitem, "a", b"a") + # Trying to resize the memory object + self.assertRaises(ValueError, setitem, 0, b"") + self.assertRaises(ValueError, setitem, 0, b"ab") + self.assertRaises(ValueError, setitem, slice(1,1), b"a") + self.assertRaises(ValueError, setitem, slice(0,2), b"a") + + m = None + self.assertEquals(sys.getrefcount(b), oldrefcount) + + def test_tobytes(self): + for tp in self._types: + m = self._view(tp(self._source)) + b = m.tobytes() + # This calls self.getitem_type() on each separate byte of b"abcdef" + expected = b"".join( + self.getitem_type(c) for c in b"abcdef") + self.assertEquals(b, expected) + self.assert_(isinstance(b, bytes), type(b)) + + def test_tolist(self): + for tp in self._types: + m = self._view(tp(self._source)) + l = m.tolist() + self.assertEquals(l, map(ord, b"abcdef")) + + def test_compare(self): + # memoryviews can compare for equality with other objects + # having the buffer interface. + for tp in self._types: + m = self._view(tp(self._source)) + for tp_comp in self._types: + self.assertTrue(m == tp_comp(b"abcdef")) + self.assertFalse(m != tp_comp(b"abcdef")) + self.assertFalse(m == tp_comp(b"abcde")) + self.assertTrue(m != tp_comp(b"abcde")) + self.assertFalse(m == tp_comp(b"abcde1")) + self.assertTrue(m != tp_comp(b"abcde1")) + self.assertTrue(m == m) + self.assertTrue(m == m[:]) + self.assertTrue(m[0:6] == m[:]) + self.assertFalse(m[0:5] == m) + + # Comparison with objects which don't support the buffer API + self.assertFalse(m == u"abcdef") + self.assertTrue(m != u"abcdef") + self.assertFalse(u"abcdef" == m) + self.assertTrue(u"abcdef" != m) + + # Unordered comparisons are unimplemented, and therefore give + # arbitrary results (they raise a TypeError in py3k) + + def check_attributes_with_type(self, tp): + m = self._view(tp(self._source)) + self.assertEquals(m.format, self.format) + self.assertEquals(m.itemsize, self.itemsize) + self.assertEquals(m.ndim, 1) + self.assertEquals(m.shape, (6,)) + self.assertEquals(len(m), 6) + self.assertEquals(m.strides, (self.itemsize,)) + self.assertEquals(m.suboffsets, None) + return m + + def test_attributes_readonly(self): + if not self.ro_type: + return + m = self.check_attributes_with_type(self.ro_type) + self.assertEquals(m.readonly, True) + + def test_attributes_writable(self): + if not self.rw_type: + return + m = self.check_attributes_with_type(self.rw_type) + self.assertEquals(m.readonly, False) + + # Disabled: unicode uses the old buffer API in 2.x + + #def test_getbuffer(self): + ## Test PyObject_GetBuffer() on a memoryview object. + #for tp in self._types: + #b = tp(self._source) + #oldrefcount = sys.getrefcount(b) + #m = self._view(b) + #oldviewrefcount = sys.getrefcount(m) + #s = unicode(m, "utf-8") + #self._check_contents(tp, b, s.encode("utf-8")) + #self.assertEquals(sys.getrefcount(m), oldviewrefcount) + #m = None + #self.assertEquals(sys.getrefcount(b), oldrefcount) + + def test_gc(self): + for tp in self._types: + if not isinstance(tp, type): + # If tp is a factory rather than a plain type, skip + continue + + class MySource(tp): + pass + class MyObject: + pass + + # Create a reference cycle through a memoryview object + b = MySource(tp(b'abc')) + m = self._view(b) + o = MyObject() + b.m = m + b.o = o + wr = weakref.ref(o) + b = m = o = None + # The cycle must be broken + gc.collect() + self.assert_(wr() is None, wr()) + + +# Variations on source objects for the buffer: bytes-like objects, then arrays +# with itemsize > 1. +# NOTE: support for multi-dimensional objects is unimplemented. + +class BaseBytesMemoryTests(AbstractMemoryTests): + ro_type = bytes + rw_type = bytearray + getitem_type = bytes + itemsize = 1 + format = 'B' + +# Disabled: array.array() does not support the new buffer API in 2.x + +#class BaseArrayMemoryTests(AbstractMemoryTests): + #ro_type = None + #rw_type = lambda self, b: array.array('i', map(ord, b)) + #getitem_type = lambda self, b: array.array('i', map(ord, b)).tostring() + #itemsize = array.array('i').itemsize + #format = 'i' + + #def test_getbuffer(self): + ## XXX Test should be adapted for non-byte buffers + #pass + + #def test_tolist(self): + ## XXX NotImplementedError: tolist() only supports byte views + #pass + + +# Variations on indirection levels: memoryview, slice of memoryview, +# slice of slice of memoryview. +# This is important to test allocation subtleties. + +class BaseMemoryviewTests: + def _view(self, obj): + return memoryview(obj) + + def _check_contents(self, tp, obj, contents): + self.assertEquals(obj, tp(contents)) + +class BaseMemorySliceTests: + source_bytes = b"XabcdefY" + + def _view(self, obj): + m = memoryview(obj) + return m[1:7] + + def _check_contents(self, tp, obj, contents): + self.assertEquals(obj[1:7], tp(contents)) + + def test_refs(self): + for tp in self._types: + m = memoryview(tp(self._source)) + oldrefcount = sys.getrefcount(m) + m[1:2] + self.assertEquals(sys.getrefcount(m), oldrefcount) + +class BaseMemorySliceSliceTests: + source_bytes = b"XabcdefY" + + def _view(self, obj): + m = memoryview(obj) + return m[:7][1:] + + def _check_contents(self, tp, obj, contents): + self.assertEquals(obj[1:7], tp(contents)) + + +# Concrete test classes + +class BytesMemoryviewTest(unittest.TestCase, + BaseMemoryviewTests, BaseBytesMemoryTests): + + def test_constructor(self): + for tp in self._types: + ob = tp(self._source) + self.assert_(memoryview(ob)) + self.assert_(memoryview(object=ob)) + self.assertRaises(TypeError, memoryview) + self.assertRaises(TypeError, memoryview, ob, ob) + self.assertRaises(TypeError, memoryview, argument=ob) + self.assertRaises(TypeError, memoryview, ob, argument=True) + +#class ArrayMemoryviewTest(unittest.TestCase, + #BaseMemoryviewTests, BaseArrayMemoryTests): + + #def test_array_assign(self): + ## Issue #4569: segfault when mutating a memoryview with itemsize != 1 + #a = array.array('i', range(10)) + #m = memoryview(a) + #new_a = array.array('i', range(9, -1, -1)) + #m[:] = new_a + #self.assertEquals(a, new_a) + + +class BytesMemorySliceTest(unittest.TestCase, + BaseMemorySliceTests, BaseBytesMemoryTests): + pass + +#class ArrayMemorySliceTest(unittest.TestCase, + #BaseMemorySliceTests, BaseArrayMemoryTests): + #pass + +class BytesMemorySliceSliceTest(unittest.TestCase, + BaseMemorySliceSliceTests, BaseBytesMemoryTests): + pass + +#class ArrayMemorySliceSliceTest(unittest.TestCase, + #BaseMemorySliceSliceTests, BaseArrayMemoryTests): + #pass + + +def test_main(): + test_support.run_unittest(__name__) + +if __name__ == "__main__": + test_main() diff -r d4af54cc46df -r 474bd6421c3f Makefile.pre.in --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -320,6 +320,7 @@ OBJECT_OBJS= \ Objects/listobject.o \ Objects/longobject.o \ Objects/dictobject.o \ + Objects/memoryobject.o \ Objects/methodobject.o \ Objects/moduleobject.o \ Objects/object.o \ @@ -617,6 +618,7 @@ PYTHON_HEADERS= \ Include/longintrepr.h \ Include/longobject.h \ Include/marshal.h \ + Include/memoryobject.h \ Include/metagrammar.h \ Include/methodobject.h \ Include/modsupport.h \ diff -r d4af54cc46df -r 474bd6421c3f Objects/abstract.c --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -439,7 +439,7 @@ PyBuffer_GetPointer(Py_buffer *view, Py_ } -static void +void _add_one_to_index_F(int nd, Py_ssize_t *index, Py_ssize_t *shape) { int k; @@ -455,7 +455,7 @@ _add_one_to_index_F(int nd, Py_ssize_t * } } -static void +void _add_one_to_index_C(int nd, Py_ssize_t *index, Py_ssize_t *shape) { int k; diff -r d4af54cc46df -r 474bd6421c3f Objects/memoryobject.c --- /dev/null +++ b/Objects/memoryobject.c @@ -0,0 +1,834 @@ + +/* Memoryview object implementation */ + +#include "Python.h" + +static Py_ssize_t +get_shape0(Py_buffer *buf) +{ + if (buf->shape != NULL) + return buf->shape[0]; + if (buf->ndim == 0) + return 1; + PyErr_SetString(PyExc_TypeError, + "exported buffer does not have any shape information associated " + "to it"); + return -1; +} + +static void +dup_buffer(Py_buffer *dest, Py_buffer *src) +{ + *dest = *src; + if (src->ndim == 1 && src->shape != NULL) { + dest->shape = &(dest->smalltable[0]); + dest->shape[0] = get_shape0(src); + } + if (src->ndim == 1 && src->strides != NULL) { + dest->strides = &(dest->smalltable[1]); + dest->strides[0] = src->strides[0]; + } +} + +static int +memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) +{ + int res = 0; + /* XXX for whatever reason fixing the flags seems necessary */ + if (self->view.readonly) + flags &= ~PyBUF_WRITABLE; + if (self->view.obj != NULL) + res = PyObject_GetBuffer(self->view.obj, view, flags); + if (view) + dup_buffer(view, &self->view); + return res; +} + +static void +memory_releasebuf(PyMemoryViewObject *self, Py_buffer *view) +{ + PyBuffer_Release(view); +} + +PyDoc_STRVAR(memory_doc, +"memoryview(object)\n\ +\n\ +Create a new memoryview object which references the given object."); + +PyObject * +PyMemoryView_FromBuffer(Py_buffer *info) +{ + PyMemoryViewObject *mview; + + mview = (PyMemoryViewObject *) + PyObject_GC_New(PyMemoryViewObject, &PyMemoryView_Type); + if (mview == NULL) + return NULL; + mview->base = NULL; + dup_buffer(&mview->view, info); + /* NOTE: mview->view.obj should already have been incref'ed as + part of PyBuffer_FillInfo(). */ + _PyObject_GC_TRACK(mview); + return (PyObject *)mview; +} + +PyObject * +PyMemoryView_FromObject(PyObject *base) +{ + PyMemoryViewObject *mview; + + if (!PyObject_CheckBuffer(base)) { + PyErr_SetString(PyExc_TypeError, + "cannot make memory view because object does " + "not have the buffer interface"); + return NULL; + } + + mview = (PyMemoryViewObject *) + PyObject_GC_New(PyMemoryViewObject, &PyMemoryView_Type); + if (mview == NULL) + return NULL; + + mview->base = NULL; + if (PyObject_GetBuffer(base, &(mview->view), PyBUF_FULL_RO) < 0) { + Py_DECREF(mview); + return NULL; + } + + mview->base = base; + Py_INCREF(base); + _PyObject_GC_TRACK(mview); + return (PyObject *)mview; +} + +static PyObject * +memory_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) +{ + PyObject *obj; + static char *kwlist[] = {"object", 0}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:memoryview", kwlist, + &obj)) { + return NULL; + } + + return PyMemoryView_FromObject(obj); +} + + +static void +_strided_copy_nd(char *dest, char *src, int nd, Py_ssize_t *shape, + Py_ssize_t *strides, Py_ssize_t itemsize, char fort) +{ + int k; + Py_ssize_t outstride; + + if (nd==0) { + memcpy(dest, src, itemsize); + } + else if (nd == 1) { + for (k = 0; kndim > PY_SSIZE_T_MAX / sizeof(Py_ssize_t)) { + PyErr_NoMemory(); + return -1; + } + + indices = (Py_ssize_t *)PyMem_Malloc(sizeof(Py_ssize_t)*view->ndim); + if (indices == NULL) { + PyErr_NoMemory(); + return -1; + } + for (k=0; kndim;k++) { + indices[k] = 0; + } + + elements = 1; + for (k=0; kndim; k++) { + elements *= view->shape[k]; + } + if (fort == 'F') { + func = _add_one_to_index_F; + } + else { + func = _add_one_to_index_C; + } + while (elements--) { + func(view->ndim, indices, view->shape); + ptr = PyBuffer_GetPointer(view, indices); + memcpy(dest, ptr, view->itemsize); + dest += view->itemsize; + } + + PyMem_Free(indices); + return 0; +} + +/* + Get a the data from an object as a contiguous chunk of memory (in + either 'C' or 'F'ortran order) even if it means copying it into a + separate memory area. + + Returns a new reference to a Memory view object. If no copy is needed, + the memory view object points to the original memory and holds a + lock on the original. If a copy is needed, then the memory view object + points to a brand-new Bytes object (and holds a memory lock on it). + + buffertype + + PyBUF_READ buffer only needs to be read-only + PyBUF_WRITE buffer needs to be writable (give error if not contiguous) + PyBUF_SHADOW buffer needs to be writable so shadow it with + a contiguous buffer if it is not. The view will point to + the shadow buffer which can be written to and then + will be copied back into the other buffer when the memory + view is de-allocated. While the shadow buffer is + being used, it will have an exclusive write lock on + the original buffer. + */ + +PyObject * +PyMemoryView_GetContiguous(PyObject *obj, int buffertype, char fort) +{ + PyMemoryViewObject *mem; + PyObject *bytes; + Py_buffer *view; + int flags; + char *dest; + + if (!PyObject_CheckBuffer(obj)) { + PyErr_SetString(PyExc_TypeError, + "object does not have the buffer interface"); + return NULL; + } + + mem = PyObject_GC_New(PyMemoryViewObject, &PyMemoryView_Type); + if (mem == NULL) + return NULL; + + view = &mem->view; + flags = PyBUF_FULL_RO; + switch(buffertype) { + case PyBUF_WRITE: + flags = PyBUF_FULL; + break; + } + + if (PyObject_GetBuffer(obj, view, flags) != 0) { + Py_DECREF(mem); + return NULL; + } + + if (PyBuffer_IsContiguous(view, fort)) { + /* no copy needed */ + Py_INCREF(obj); + mem->base = obj; + _PyObject_GC_TRACK(mem); + return (PyObject *)mem; + } + /* otherwise a copy is needed */ + if (buffertype == PyBUF_WRITE) { + Py_DECREF(mem); + PyErr_SetString(PyExc_BufferError, + "writable contiguous buffer requested " + "for a non-contiguousobject."); + return NULL; + } + bytes = PyBytes_FromStringAndSize(NULL, view->len); + if (bytes == NULL) { + Py_DECREF(mem); + return NULL; + } + dest = PyBytes_AS_STRING(bytes); + /* different copying strategy depending on whether + or not any pointer de-referencing is needed + */ + /* strided or in-direct copy */ + if (view->suboffsets==NULL) { + _strided_copy_nd(dest, view->buf, view->ndim, view->shape, + view->strides, view->itemsize, fort); + } + else { + if (_indirect_copy_nd(dest, view, fort) < 0) { + Py_DECREF(bytes); + Py_DECREF(mem); + return NULL; + } + } + if (buffertype == PyBUF_SHADOW) { + /* return a shadowed memory-view object */ + view->buf = dest; + mem->base = PyTuple_Pack(2, obj, bytes); + Py_DECREF(bytes); + if (mem->base == NULL) { + Py_DECREF(mem); + return NULL; + } + } + else { + PyBuffer_Release(view); /* XXX ? */ + /* steal the reference */ + mem->base = bytes; + } + _PyObject_GC_TRACK(mem); + return (PyObject *)mem; +} + + +static PyObject * +memory_format_get(PyMemoryViewObject *self) +{ + return PyUnicode_FromString(self->view.format); +} + +static PyObject * +memory_itemsize_get(PyMemoryViewObject *self) +{ + return PyLong_FromSsize_t(self->view.itemsize); +} + +static PyObject * +_IntTupleFromSsizet(int len, Py_ssize_t *vals) +{ + int i; + PyObject *o; + PyObject *intTuple; + + if (vals == NULL) { + Py_INCREF(Py_None); + return Py_None; + } + intTuple = PyTuple_New(len); + if (!intTuple) return NULL; + for(i=0; iview.ndim, self->view.shape); +} + +static PyObject * +memory_strides_get(PyMemoryViewObject *self) +{ + return _IntTupleFromSsizet(self->view.ndim, self->view.strides); +} + +static PyObject * +memory_suboffsets_get(PyMemoryViewObject *self) +{ + return _IntTupleFromSsizet(self->view.ndim, self->view.suboffsets); +} + +static PyObject * +memory_readonly_get(PyMemoryViewObject *self) +{ + return PyBool_FromLong(self->view.readonly); +} + +static PyObject * +memory_ndim_get(PyMemoryViewObject *self) +{ + return PyLong_FromLong(self->view.ndim); +} + +static PyGetSetDef memory_getsetlist[] ={ + {"format", (getter)memory_format_get, NULL, NULL}, + {"itemsize", (getter)memory_itemsize_get, NULL, NULL}, + {"shape", (getter)memory_shape_get, NULL, NULL}, + {"strides", (getter)memory_strides_get, NULL, NULL}, + {"suboffsets", (getter)memory_suboffsets_get, NULL, NULL}, + {"readonly", (getter)memory_readonly_get, NULL, NULL}, + {"ndim", (getter)memory_ndim_get, NULL, NULL}, + {NULL, NULL, NULL, NULL}, +}; + + +static PyObject * +memory_tobytes(PyMemoryViewObject *self, PyObject *noargs) +{ + Py_buffer view; + PyObject *res; + + if (PyObject_GetBuffer((PyObject *)self, &view, PyBUF_FULL) < 0) + return NULL; + + res = PyBytes_FromStringAndSize(NULL, view.len); + PyBuffer_ToContiguous(PyBytes_AS_STRING(res), &view, view.len, 'C'); + PyBuffer_Release(&view); + return res; +} + +/* TODO: rewrite this function using the struct module to unpack + each buffer item */ + +static PyObject * +memory_tolist(PyMemoryViewObject *mem, PyObject *noargs) +{ + Py_buffer *view = &(mem->view); + Py_ssize_t i; + PyObject *res, *item; + char *buf; + + if (strcmp(view->format, "B") || view->itemsize != 1) { + PyErr_SetString(PyExc_NotImplementedError, + "tolist() only supports byte views"); + return NULL; + } + if (view->ndim != 1) { + PyErr_SetString(PyExc_NotImplementedError, + "tolist() only supports one-dimensional objects"); + return NULL; + } + res = PyList_New(view->len); + if (res == NULL) + return NULL; + buf = view->buf; + for (i = 0; i < view->len; i++) { + item = PyInt_FromLong((unsigned char) *buf); + if (item == NULL) { + Py_DECREF(res); + return NULL; + } + PyList_SET_ITEM(res, i, item); + buf++; + } + return res; +} + +static PyMethodDef memory_methods[] = { + {"tobytes", (PyCFunction)memory_tobytes, METH_NOARGS, NULL}, + {"tolist", (PyCFunction)memory_tolist, METH_NOARGS, NULL}, + {NULL, NULL} /* sentinel */ +}; + + +static void +memory_dealloc(PyMemoryViewObject *self) +{ + _PyObject_GC_UNTRACK(self); + if (self->view.obj != NULL) { + if (self->base && PyTuple_Check(self->base)) { + /* Special case when first element is generic object + with buffer interface and the second element is a + contiguous "shadow" that must be copied back into + the data areay of the first tuple element before + releasing the buffer on the first element. + */ + + PyObject_CopyData(PyTuple_GET_ITEM(self->base,0), + PyTuple_GET_ITEM(self->base,1)); + + /* The view member should have readonly == -1 in + this instance indicating that the memory can + be "locked" and was locked and will be unlocked + again after this call. + */ + PyBuffer_Release(&(self->view)); + } + else { + PyBuffer_Release(&(self->view)); + } + Py_CLEAR(self->base); + } + PyObject_GC_Del(self); +} + +static PyObject * +memory_repr(PyMemoryViewObject *self) +{ + return PyUnicode_FromFormat("", self); +} + +/* Sequence methods */ +static Py_ssize_t +memory_length(PyMemoryViewObject *self) +{ + return get_shape0(&self->view); +} + +/* + mem[obj] returns a bytes object holding the data for one element if + obj fully indexes the memory view or another memory-view object + if it does not. + + 0-d memory-view objects can be referenced using ... or () but + not with anything else. + */ +static PyObject * +memory_subscript(PyMemoryViewObject *self, PyObject *key) +{ + Py_buffer *view; + view = &(self->view); + + if (view->ndim == 0) { + if (key == Py_Ellipsis || + (PyTuple_Check(key) && PyTuple_GET_SIZE(key)==0)) { + Py_INCREF(self); + return (PyObject *)self; + } + else { + PyErr_SetString(PyExc_IndexError, + "invalid indexing of 0-dim memory"); + return NULL; + } + } + if (PyIndex_Check(key)) { + Py_ssize_t result; + result = PyNumber_AsSsize_t(key, NULL); + if (result == -1 && PyErr_Occurred()) + return NULL; + if (view->ndim == 1) { + /* Return a bytes object */ + char *ptr; + ptr = (char *)view->buf; + if (result < 0) { + result += get_shape0(view); + } + if ((result < 0) || (result >= get_shape0(view))) { + PyErr_SetString(PyExc_IndexError, + "index out of bounds"); + return NULL; + } + if (view->strides == NULL) + ptr += view->itemsize * result; + else + ptr += view->strides[0] * result; + if (view->suboffsets != NULL && + view->suboffsets[0] >= 0) { + ptr = *((char **)ptr) + view->suboffsets[0]; + } + return PyBytes_FromStringAndSize(ptr, view->itemsize); + } + else { + /* Return a new memory-view object */ + Py_buffer newview; + memset(&newview, 0, sizeof(newview)); + /* XXX: This needs to be fixed so it + actually returns a sub-view + */ + return PyMemoryView_FromBuffer(&newview); + } + } + else if (PySlice_Check(key)) { + Py_ssize_t start, stop, step, slicelength; + + if (PySlice_GetIndicesEx((PySliceObject*)key, get_shape0(view), + &start, &stop, &step, &slicelength) < 0) { + return NULL; + } + + if (step == 1 && view->ndim == 1) { + Py_buffer newview; + void *newbuf = (char *) view->buf + + start * view->itemsize; + int newflags = view->readonly + ? PyBUF_CONTIG_RO : PyBUF_CONTIG; + + /* XXX There should be an API to create a subbuffer */ + if (view->obj != NULL) { + if (PyObject_GetBuffer(view->obj, &newview, newflags) == -1) + return NULL; + } + else { + newview = *view; + } + newview.buf = newbuf; + newview.len = slicelength * newview.itemsize; + newview.format = view->format; + newview.shape = &(newview.smalltable[0]); + newview.shape[0] = slicelength; + newview.strides = &(newview.itemsize); + return PyMemoryView_FromBuffer(&newview); + } + PyErr_SetNone(PyExc_NotImplementedError); + return NULL; + } + PyErr_Format(PyExc_TypeError, + "cannot index memory using \"%.200s\"", + key->ob_type->tp_name); + return NULL; +} + + +/* Need to support assigning memory if we can */ +static int +memory_ass_sub(PyMemoryViewObject *self, PyObject *key, PyObject *value) +{ + Py_ssize_t start, len, bytelen, i; + Py_buffer srcview; + Py_buffer *view = &(self->view); + char *srcbuf, *destbuf; + + if (view->readonly) { + PyErr_SetString(PyExc_TypeError, + "cannot modify read-only memory"); + return -1; + } + if (view->ndim != 1) { + PyErr_SetNone(PyExc_NotImplementedError); + return -1; + } + if (PyIndex_Check(key)) { + start = PyNumber_AsSsize_t(key, NULL); + if (start == -1 && PyErr_Occurred()) + return -1; + if (start < 0) { + start += get_shape0(view); + } + if ((start < 0) || (start >= get_shape0(view))) { + PyErr_SetString(PyExc_IndexError, + "index out of bounds"); + return -1; + } + len = 1; + } + else if (PySlice_Check(key)) { + Py_ssize_t stop, step; + + if (PySlice_GetIndicesEx((PySliceObject*)key, get_shape0(view), + &start, &stop, &step, &len) < 0) { + return -1; + } + if (step != 1) { + PyErr_SetNone(PyExc_NotImplementedError); + return -1; + } + } + else { + PyErr_Format(PyExc_TypeError, + "cannot index memory using \"%.200s\"", + key->ob_type->tp_name); + return -1; + } + if (PyObject_GetBuffer(value, &srcview, PyBUF_CONTIG_RO) == -1) { + return -1; + } + /* XXX should we allow assignment of different item sizes + as long as the byte length is the same? + (e.g. assign 2 shorts to a 4-byte slice) */ + if (srcview.itemsize != view->itemsize) { + PyErr_Format(PyExc_TypeError, + "mismatching item sizes for \"%.200s\" and \"%.200s\"", + view->obj->ob_type->tp_name, srcview.obj->ob_type->tp_name); + goto _error; + } + bytelen = len * view->itemsize; + if (bytelen != srcview.len) { + PyErr_SetString(PyExc_ValueError, + "cannot modify size of memoryview object"); + goto _error; + } + /* Do the actual copy */ + destbuf = (char *) view->buf + start * view->itemsize; + srcbuf = (char *) srcview.buf; + if (destbuf + bytelen < srcbuf || srcbuf + bytelen < destbuf) + /* No overlapping */ + memcpy(destbuf, srcbuf, bytelen); + else if (destbuf < srcbuf) { + /* Copy in ascending order */ + for (i = 0; i < bytelen; i++) + destbuf[i] = srcbuf[i]; + } + else { + /* Copy in descencing order */ + for (i = bytelen - 1; i >= 0; i--) + destbuf[i] = srcbuf[i]; + } + + PyBuffer_Release(&srcview); + return 0; + +_error: + PyBuffer_Release(&srcview); + return -1; +} + +static PyObject * +memory_richcompare(PyObject *v, PyObject *w, int op) +{ + Py_buffer vv, ww; + int equal = 0; + PyObject *res; + + vv.obj = NULL; + ww.obj = NULL; + if (op != Py_EQ && op != Py_NE) + goto _notimpl; + if (PyObject_GetBuffer(v, &vv, PyBUF_CONTIG_RO) == -1) { + PyErr_Clear(); + goto _notimpl; + } + if (PyObject_GetBuffer(w, &ww, PyBUF_CONTIG_RO) == -1) { + PyErr_Clear(); + goto _notimpl; + } + + if (vv.itemsize != ww.itemsize || vv.len != ww.len) + goto _end; + + equal = !memcmp(vv.buf, ww.buf, vv.len); + +_end: + PyBuffer_Release(&vv); + PyBuffer_Release(&ww); + if ((equal && op == Py_EQ) || (!equal && op == Py_NE)) + res = Py_True; + else + res = Py_False; + Py_INCREF(res); + return res; + +_notimpl: + PyBuffer_Release(&vv); + PyBuffer_Release(&ww); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + + +static int +memory_traverse(PyMemoryViewObject *self, visitproc visit, void *arg) +{ + if (self->base != NULL) + Py_VISIT(self->base); + if (self->view.obj != NULL) + Py_VISIT(self->view.obj); + return 0; +} + +static int +memory_clear(PyMemoryViewObject *self) +{ + Py_CLEAR(self->base); + PyBuffer_Release(&self->view); + return 0; +} + + +/* As mapping */ +static PyMappingMethods memory_as_mapping = { + (lenfunc)memory_length, /* mp_length */ + (binaryfunc)memory_subscript, /* mp_subscript */ + (objobjargproc)memory_ass_sub, /* mp_ass_subscript */ +}; + + +/* Buffer methods */ +static PyBufferProcs memory_as_buffer = { + 0, /* bf_getreadbuffer */ + 0, /* bf_getwritebuffer */ + 0, /* bf_getsegcount */ + 0, /* bf_getcharbuffer */ + (getbufferproc)memory_getbuf, /* bf_getbuffer */ + (releasebufferproc)memory_releasebuf, /* bf_releasebuffer */ +}; + + +PyTypeObject PyMemoryView_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "memoryview", + sizeof(PyMemoryViewObject), + 0, + (destructor)memory_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + (reprfunc)memory_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + &memory_as_mapping, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + &memory_as_buffer, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_HAVE_NEWBUFFER, /* tp_flags */ + memory_doc, /* tp_doc */ + (traverseproc)memory_traverse, /* tp_traverse */ + (inquiry)memory_clear, /* tp_clear */ + memory_richcompare, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + memory_methods, /* tp_methods */ + 0, /* tp_members */ + memory_getsetlist, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + memory_new, /* tp_new */ +}; diff -r d4af54cc46df -r 474bd6421c3f Objects/typeobject.c --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2304,6 +2304,8 @@ type_new(PyTypeObject *metatype, PyObjec Py_TPFLAGS_BASETYPE; if (base->tp_flags & Py_TPFLAGS_HAVE_GC) type->tp_flags |= Py_TPFLAGS_HAVE_GC; + if (base->tp_flags & Py_TPFLAGS_HAVE_NEWBUFFER) + type->tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER; /* It's a new-style number unless it specifically inherits any old-style numeric behavior */ @@ -3596,6 +3598,8 @@ add_getset(PyTypeObject *type, PyGetSetD return 0; } +#define BUFFER_FLAGS (Py_TPFLAGS_HAVE_GETCHARBUFFER | Py_TPFLAGS_HAVE_NEWBUFFER) + static void inherit_special(PyTypeObject *type, PyTypeObject *base) { @@ -3603,9 +3607,9 @@ inherit_special(PyTypeObject *type, PyTy /* Special flag magic */ if (!type->tp_as_buffer && base->tp_as_buffer) { - type->tp_flags &= ~Py_TPFLAGS_HAVE_GETCHARBUFFER; + type->tp_flags &= ~BUFFER_FLAGS; type->tp_flags |= - base->tp_flags & Py_TPFLAGS_HAVE_GETCHARBUFFER; + base->tp_flags & BUFFER_FLAGS; } if (!type->tp_as_sequence && base->tp_as_sequence) { type->tp_flags &= ~Py_TPFLAGS_HAVE_SEQUENCE_IN; diff -r d4af54cc46df -r 474bd6421c3f Python/bltinmodule.c --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2573,7 +2573,7 @@ _PyBuiltin_Init(void) SETBUILTIN("True", Py_True); SETBUILTIN("basestring", &PyBaseString_Type); SETBUILTIN("bool", &PyBool_Type); - /* SETBUILTIN("memoryview", &PyMemoryView_Type); */ + SETBUILTIN("memoryview", &PyMemoryView_Type); SETBUILTIN("bytearray", &PyByteArray_Type); SETBUILTIN("bytes", &PyString_Type); SETBUILTIN("buffer", &PyBuffer_Type);