diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2700,25 +2700,7 @@ A string containing the format (in :mod:`struct` module style) for each element in the view. A memoryview can be created from exporters with arbitrary format strings, but some methods (e.g. :meth:`tolist`) are - restricted to native single element formats. Special care must be taken - when comparing memoryviews. Since comparisons are required to return a - value for ``==`` and ``!=``, two memoryviews referencing the same - exporter can compare as not-equal if the exporter's format is not - understood:: - - >>> from ctypes import BigEndianStructure, c_long - >>> class BEPoint(BigEndianStructure): - ... _fields_ = [("x", c_long), ("y", c_long)] - ... - >>> point = BEPoint(100, 200) - >>> a = memoryview(point) - >>> b = memoryview(point) - >>> a == b - False - >>> a.tolist() - Traceback (most recent call last): - File "", line 1, in - NotImplementedError: memoryview: unsupported format T{>l:x:>l:y:} + restricted to native single element formats. .. attribute:: itemsize diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -32,6 +32,11 @@ struct = None try: + import ctypes +except ImportError: + ctypes = None + +try: with warnings.catch_warnings(): from numpy import ndarray as numpy_array except ImportError: @@ -835,8 +840,6 @@ # test tobytes() self.assertEqual(result.tobytes(), b) - if not buf_err and is_memoryview_format(fmt): - # lst := expected multi-dimensional logical representation # flatten(lst) := elements in C-order ff = fmt if fmt else 'B' @@ -877,8 +880,10 @@ # To 'C' contig = py_buffer_to_contiguous(result, 'C', PyBUF_FULL_RO) self.assertEqual(len(contig), nmemb * itemsize) - initlst = [struct.unpack_from(fmt, contig, n*itemsize)[0] + initlst = [struct.unpack_from(fmt, contig, n*itemsize) for n in range(nmemb)] + if len(initlst[0]) == 1: + initlst = [v[0] for v in initlst] y = ndarray(initlst, shape=shape, flags=ro, format=fmt) self.assertEqual(memoryview(y), memoryview(result)) @@ -886,8 +891,10 @@ # To 'F' contig = py_buffer_to_contiguous(result, 'F', PyBUF_FULL_RO) self.assertEqual(len(contig), nmemb * itemsize) - initlst = [struct.unpack_from(fmt, contig, n*itemsize)[0] + initlst = [struct.unpack_from(fmt, contig, n*itemsize) for n in range(nmemb)] + if len(initlst[0]) == 1: + initlst = [v[0] for v in initlst] y = ndarray(initlst, shape=shape, flags=ro|ND_FORTRAN, format=fmt) @@ -896,8 +903,10 @@ # To 'A' contig = py_buffer_to_contiguous(result, 'A', PyBUF_FULL_RO) self.assertEqual(len(contig), nmemb * itemsize) - initlst = [struct.unpack_from(fmt, contig, n*itemsize)[0] + initlst = [struct.unpack_from(fmt, contig, n*itemsize) for n in range(nmemb)] + if len(initlst[0]) == 1: + initlst = [v[0] for v in initlst] f = ND_FORTRAN if is_contiguous(result, 'F') else 0 y = ndarray(initlst, shape=shape, flags=f|ro, format=fmt) @@ -3064,24 +3073,6 @@ self.assertNotEqual(v, c) self.assertNotEqual(c, v) - # Not implemented formats. Ugly, but inevitable. This is the same as - # issue #2531: equality is also used for membership testing and must - # return a result. - a = ndarray([(1, 1.5), (2, 2.7)], shape=[2], format='ld') - v = memoryview(a) - self.assertNotEqual(v, a) - self.assertNotEqual(a, v) - - a = ndarray([b'12345'], shape=[1], format="s") - v = memoryview(a) - self.assertNotEqual(v, a) - self.assertNotEqual(a, v) - - nd = ndarray([(1,1,1), (2,2,2), (3,3,3)], shape=[3], format='iii') - v = memoryview(nd) - self.assertNotEqual(v, nd) - self.assertNotEqual(nd, v) - # '@' prefix can be dropped: nd1 = ndarray([1,2,3], shape=[3], format='@i') nd2 = ndarray([1,2,3], shape=[3], format='i') @@ -3413,6 +3404,123 @@ self.assertEqual(w, nd2) self.assertEqual(v, w) + # Identical formats allowed in comparisons and tobytes(). + a = ndarray([(1, 1.5), (2, 2.7)], shape=[2], format='ld') + v = memoryview(a) + self.assertEqual(v, a) + self.assertEqual(a, v) + + a = ndarray([b'12345'], shape=[1], format="s") + v = memoryview(a) + self.assertEqual(v, a) + self.assertEqual(a, v) + + nd = ndarray([(1,1,1), (2,2,2), (3,3,3)], shape=[3], format='iii') + v = memoryview(nd) + self.assertEqual(v, nd) + self.assertEqual(nd, v) + + a = array.array('u', 'xyz') + v = memoryview(a) + self.assertEqual(a, v) + self.assertEqual(v, a) + + # Logically identical formats allowed in comparisons and tobytes(). + fmt1 = '=L' + fmt2 = 'L' + x = ndarray([1,2,3], shape=[3], format=fmt1) + y = ndarray([1,2,3], shape=[3], format=fmt2) + a = memoryview(x) + b = memoryview(y) + self.assertEqual(a, b) + self.assertEqual(b, a) + self.assertEqual(a, y) + self.assertEqual(b, x) + + t = (-529, 576, -625, 676, -729) + fmt1 = '=hQiLl' + fmt2 = 'hQiLl' + + x = ndarray([t for _ in range(120)], shape=[5,2,2,3,2], format=fmt1, + flags=ND_PIL) + y = ndarray([t for _ in range(120)], shape=[5,2,2,3,2], format=fmt2, + flags=ND_FORTRAN) + a = memoryview(x) + b = memoryview(y) + self.assertEqual(a, b) + self.assertEqual(b, a) + self.assertEqual(a, y) + self.assertEqual(b, x) + + # Formats that are not considered equal. + fmt1 = '=I' + fmt2 = '=L' + x = ndarray([1,2,3], shape=[3], format=fmt1) + y = ndarray([1,2,3], shape=[3], format=fmt2) + a = memoryview(x) + b = memoryview(y) + self.assertNotEqual(a, b) + self.assertNotEqual(b, a) + self.assertNotEqual(a, y) + self.assertNotEqual(b, x) + self.assertEqual(a, x) + self.assertEqual(b, y) + + t = (-529, 576, -625, 676, -729) + fmt1 = 'hQiLl' + fmt2 = 'hQiLl' + + x = ndarray([t for _ in range(120)], shape=[5,2,2,3,2], format=fmt1, + flags=ND_PIL) + y = ndarray([t for _ in range(120)], shape=[5,2,2,3,2], format=fmt2, + flags=ND_FORTRAN) + a = memoryview(x) + b = memoryview(y) + self.assertNotEqual(a, b) + self.assertNotEqual(b, a) + self.assertNotEqual(a, y) + self.assertNotEqual(b, x) + self.assertEqual(a, x) + self.assertEqual(b, y) + + # Items not equal. + for byteorder in ['=', '<', '>', '!']: + x = ndarray([2**63]*120, shape=[3,5,2,2,2], format=byteorder+'Q') + y = ndarray([2**63]*120, shape=[3,5,2,2,2], format=byteorder+'Q', + flags=ND_WRITABLE|ND_FORTRAN) + y[2][3][1][1][1] = 1 + a = memoryview(x) + b = memoryview(y) + self.assertEqual(a, x) + self.assertEqual(b, y) + self.assertNotEqual(a, b) + self.assertNotEqual(a, y) + self.assertNotEqual(b, x) + + x = ndarray([(2**63, 2**31, 2**15)]*120, shape=[3,5,2,2,2], + format=byteorder+'QLH') + y = ndarray([(2**63, 2**31, 2**15)]*120, shape=[3,5,2,2,2], + format=byteorder+'QLH', flags=ND_WRITABLE|ND_FORTRAN) + y[2][3][1][1][1] = (1, 1, 1) + a = memoryview(x) + b = memoryview(y) + self.assertEqual(a, x) + self.assertEqual(b, y) + self.assertNotEqual(a, b) + self.assertNotEqual(a, y) + self.assertNotEqual(b, x) + + # ctypes format strings + if ctypes: + # format: "T{>l:x:>l:y:}" + class BEPoint(ctypes.BigEndianStructure): + _fields_ = [("x", ctypes.c_long), ("y", ctypes.c_long)] + point = BEPoint(100, 200) + a = memoryview(point) + b = memoryview(point) + self.assertEqual(a, b) + self.assertRaises(NotImplementedError, a.tolist) + def test_memoryview_check_released(self): a = array.array('d', [1.1, 2.2, 3.3]) @@ -3456,9 +3564,27 @@ def test_memoryview_tobytes(self): # Many implicit tests are already in self.verify(). - nd = ndarray([-529, 576, -625, 676, -729], shape=[5], format='@h') - + t = (-529, 576, -625, 676, -729) + + nd = ndarray(t, shape=[5], format='@h') m = memoryview(nd) + self.assertEqual(m, nd) + self.assertEqual(m.tobytes(), nd.tobytes()) + + nd = ndarray([t], shape=[1], format='>hQiLl') + m = memoryview(nd) + self.assertEqual(m, nd) + self.assertEqual(m.tobytes(), nd.tobytes()) + + nd = ndarray([t for _ in range(12)], shape=[2,2,3], format='=hQiLl') + m = memoryview(nd) + self.assertEqual(m, nd) + self.assertEqual(m.tobytes(), nd.tobytes()) + + nd = ndarray([t for _ in range(120)], shape=[5,2,2,3,2], + format='strides[src->ndim-1] == src->itemsize); } +/* struct module properties of a single item */ +#define MV_LITTLE_ENDIAN 0x001 /* byte order */ +#define MV_NATIVE_SIZE 0x002 /* native or standard size */ +#define MV_NATIVE_ALIGNMENT 0x004 /* native or no alignment */ +Py_LOCAL_INLINE(int) +format_type(const char **fmt) +{ + int native_le = 1; + char *p = (char *) &native_le; + /* assume big endian and standard size */ + int type = 0; + + native_le = (*p == 1); + + switch (**fmt) { + case '=': + type |= native_le; + (*fmt)++; + break; + case '<': + type |= MV_LITTLE_ENDIAN; + (*fmt)++; + break; + case '>': case '!': + (*fmt)++; + break; + case '@': + (*fmt)++; /* fall through */ + default: + type |= (native_le|MV_NATIVE_SIZE|MV_NATIVE_ALIGNMENT); + break; + } + + return type; +} + +/* Compare two formats in struct module syntax. If byte order, size and + alignment as determined by the (optional) byte order specifier are + equal and the remainder of the two format strings is equal, return 0. + Return nonzero otherwise. */ +static int +fmtcmp(const char *p, const char *q) +{ + if (format_type(&p) != format_type(&q)) { + return -1; + } + + return strcmp(p, q); +} + /* Check that the logical structure of the destination and source buffers is identical. */ static int cmp_structure(Py_buffer *dest, Py_buffer *src) { - const char *dfmt, *sfmt; int i; assert(dest->format && src->format); - dfmt = dest->format[0] == '@' ? dest->format+1 : dest->format; - sfmt = src->format[0] == '@' ? src->format+1 : src->format; - - if (strcmp(dfmt, sfmt) != 0 || + + if (fmtcmp(dest->format, src->format) != 0 || dest->itemsize != src->itemsize || dest->ndim != src->ndim) { goto value_error; @@ -2271,11 +2318,11 @@ } while (0) Py_LOCAL_INLINE(int) -unpack_cmp(const char *p, const char *q, const char *fmt) +unpack_cmp(const char *p, const char *q, char fmt, Py_ssize_t itemsize) { int equal; - switch (fmt[0]) { + switch (fmt) { /* signed integers and fast path for 'B' */ case 'B': return *((unsigned char *)p) == *((unsigned char *)q); @@ -2317,8 +2364,8 @@ /* pointer */ case 'P': CMP_SINGLE(p, q, void *); return equal; - /* Py_NotImplemented */ - default: return -1; + /* not a native single character format: use itemsize */ + default: return !memcmp(p, q, itemsize); } } @@ -2327,7 +2374,7 @@ cmp_base(const char *p, const char *q, const Py_ssize_t *shape, const Py_ssize_t *pstrides, const Py_ssize_t *psuboffsets, const Py_ssize_t *qstrides, const Py_ssize_t *qsuboffsets, - const char *fmt) + char fmt, Py_ssize_t itemsize) { Py_ssize_t i; int equal; @@ -2335,7 +2382,7 @@ for (i = 0; i < shape[0]; p+=pstrides[0], q+=qstrides[0], i++) { const char *xp = ADJUST_PTR(p, psuboffsets); const char *xq = ADJUST_PTR(q, qsuboffsets); - equal = unpack_cmp(xp, xq, fmt); + equal = unpack_cmp(xp, xq, fmt, itemsize); if (equal <= 0) return equal; } @@ -2350,7 +2397,7 @@ Py_ssize_t ndim, const Py_ssize_t *shape, const Py_ssize_t *pstrides, const Py_ssize_t *psuboffsets, const Py_ssize_t *qstrides, const Py_ssize_t *qsuboffsets, - const char *fmt) + char fmt, Py_ssize_t itemsize) { Py_ssize_t i; int equal; @@ -2364,7 +2411,7 @@ return cmp_base(p, q, shape, pstrides, psuboffsets, qstrides, qsuboffsets, - fmt); + fmt, itemsize); } for (i = 0; i < shape[0]; p+=pstrides[0], q+=qstrides[0], i++) { @@ -2373,7 +2420,7 @@ equal = cmp_rec(xp, xq, ndim-1, shape+1, pstrides+1, psuboffsets ? psuboffsets+1 : NULL, qstrides+1, qsuboffsets ? qsuboffsets+1 : NULL, - fmt); + fmt, itemsize); if (equal <= 0) return equal; } @@ -2386,7 +2433,7 @@ { PyObject *res; Py_buffer wbuf, *vv, *ww = NULL; - const char *vfmt, *wfmt; + char fmt; int equal = -1; /* Py_NotImplemented */ if (op != Py_EQ && op != Py_NE) @@ -2414,33 +2461,30 @@ ww = &wbuf; } - vfmt = adjust_fmt(vv); - wfmt = adjust_fmt(ww); - if (vfmt == NULL || wfmt == NULL) { - PyErr_Clear(); - goto result; /* Py_NotImplemented */ - } - if (cmp_structure(vv, ww) < 0) { PyErr_Clear(); equal = 0; goto result; } + if (get_native_fmtchar(&fmt, vv->format) < 0) { + fmt = '_'; /* not a native single character format */ + } + if (vv->ndim == 0) { - equal = unpack_cmp(vv->buf, ww->buf, vfmt); + equal = unpack_cmp(vv->buf, ww->buf, fmt, vv->itemsize); } else if (vv->ndim == 1) { equal = cmp_base(vv->buf, ww->buf, vv->shape, vv->strides, vv->suboffsets, ww->strides, ww->suboffsets, - vfmt); + fmt, vv->itemsize); } else { equal = cmp_rec(vv->buf, ww->buf, vv->ndim, vv->shape, vv->strides, vv->suboffsets, ww->strides, ww->suboffsets, - vfmt); + fmt, vv->itemsize); } result: