Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(29607)

Side by Side Diff: Modules/_struct.c

Issue 17804: streaming struct unpacking
Patch Set: Created 6 years, 5 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « Lib/test/test_struct.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 /* struct module -- pack values into and (out of) bytes objects */ 1 /* struct module -- pack values into and (out of) bytes objects */
2 2
3 /* New version supporting byte order, alignment and size options, 3 /* New version supporting byte order, alignment and size options,
4 character strings, and unsigned numbers */ 4 character strings, and unsigned numbers */
5 5
6 #define PY_SSIZE_T_CLEAN 6 #define PY_SSIZE_T_CLEAN
7 7
8 #include "Python.h" 8 #include "Python.h"
9 #include "structmember.h" 9 #include "structmember.h"
10 #include <ctype.h> 10 #include <ctype.h>
(...skipping 1229 matching lines...) Expand 10 before | Expand all | Expand 10 after
1240 if (e->alignment && size > 0) { 1240 if (e->alignment && size > 0) {
1241 extra = (e->alignment - 1) - (size - 1) % (e->alignment); 1241 extra = (e->alignment - 1) - (size - 1) % (e->alignment);
1242 if (extra > PY_SSIZE_T_MAX - size) 1242 if (extra > PY_SSIZE_T_MAX - size)
1243 return -1; 1243 return -1;
1244 size += extra; 1244 size += extra;
1245 } 1245 }
1246 } 1246 }
1247 return size; 1247 return size;
1248 } 1248 }
1249 1249
1250 /*
1251 * Struct object implementation.
1252 */
1250 1253
1251 /* calculate the size of a format string */ 1254 /* calculate the size of a format string */
1252 1255
1253 static int 1256 static int
1254 prepare_s(PyStructObject *self) 1257 prepare_s(PyStructObject *self)
1255 { 1258 {
1256 const formatdef *f; 1259 const formatdef *f;
1257 const formatdef *e; 1260 const formatdef *e;
1258 formatcode *codes; 1261 formatcode *codes;
1259 1262
(...skipping 286 matching lines...) Expand 10 before | Expand all | Expand 10 after
1546 if (offset < 0 || vbuf.len - offset < soself->s_size) { 1549 if (offset < 0 || vbuf.len - offset < soself->s_size) {
1547 PyErr_Format(StructError, 1550 PyErr_Format(StructError,
1548 "unpack_from requires a buffer of at least %zd bytes", 1551 "unpack_from requires a buffer of at least %zd bytes",
1549 soself->s_size); 1552 soself->s_size);
1550 PyBuffer_Release(&vbuf); 1553 PyBuffer_Release(&vbuf);
1551 return NULL; 1554 return NULL;
1552 } 1555 }
1553 result = s_unpack_internal(soself, (char*)vbuf.buf + offset); 1556 result = s_unpack_internal(soself, (char*)vbuf.buf + offset);
1554 PyBuffer_Release(&vbuf); 1557 PyBuffer_Release(&vbuf);
1555 return result; 1558 return result;
1559 }
1560
1561
1562 /* Unpack iterator type */
1563
1564 typedef struct {
1565 PyObject_HEAD
1566 PyStructObject *so;
1567 Py_buffer buf;
1568 Py_ssize_t index;
1569 } unpackiterobject;
1570
1571 static void
1572 unpackiter_dealloc(unpackiterobject *self)
1573 {
1574 Py_XDECREF(self->so);
1575 PyBuffer_Release(&self->buf);
1576 PyObject_GC_Del(self);
1577 }
1578
1579 static int
1580 unpackiter_traverse(unpackiterobject *self, visitproc visit, void *arg)
1581 {
1582 Py_VISIT(self->so);
1583 Py_VISIT(self->buf.obj);
1584 return 0;
1585 }
1586
1587 static PyObject *
1588 unpackiter_len(unpackiterobject *self)
1589 {
1590 Py_ssize_t len;
1591 if (self->so == NULL)
1592 len = 0;
1593 else
1594 len = (self->buf.len - self->index) / self->so->s_size;
1595 return PyLong_FromSsize_t(len);
1596 }
1597
1598 static PyMethodDef unpackiter_methods[] = {
1599 {"__length_hint__", (PyCFunction) unpackiter_len, METH_NOARGS, NULL},
1600 {NULL, NULL} /* sentinel */
1601 };
1602
1603 static PyObject *
1604 unpackiter_iternext(unpackiterobject *self)
1605 {
1606 PyObject *result;
1607 if (self->so == NULL)
1608 return NULL;
1609 if (self->index >= self->buf.len) {
1610 /* Iterator exhausted */
1611 Py_CLEAR(self->so);
1612 PyBuffer_Release(&self->buf);
1613 return NULL;
1614 }
1615 assert(self->index + self->so->s_size <= self->buf.len);
1616 result = s_unpack_internal(self->so,
1617 (char*) self->buf.buf + self->index);
1618 self->index += self->so->s_size;
1619 return result;
1620 }
1621
1622 PyTypeObject unpackiter_type = {
1623 PyVarObject_HEAD_INIT(&PyType_Type, 0)
1624 "unpack_iterator", /* tp_name */
1625 sizeof(unpackiterobject), /* tp_basicsize */
1626 0, /* tp_itemsize */
1627 (destructor)unpackiter_dealloc, /* tp_dealloc */
1628 0, /* tp_print */
1629 0, /* tp_getattr */
1630 0, /* tp_setattr */
1631 0, /* tp_reserved */
1632 0, /* tp_repr */
1633 0, /* tp_as_number */
1634 0, /* tp_as_sequence */
1635 0, /* tp_as_mapping */
1636 0, /* tp_hash */
1637 0, /* tp_call */
1638 0, /* tp_str */
1639 PyObject_GenericGetAttr, /* tp_getattro */
1640 0, /* tp_setattro */
1641 0, /* tp_as_buffer */
1642 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
1643 0, /* tp_doc */
1644 (traverseproc)unpackiter_traverse, /* tp_traverse */
1645 0, /* tp_clear */
1646 0, /* tp_richcompare */
1647 0, /* tp_weaklistoffset */
1648 PyObject_SelfIter, /* tp_iter */
1649 (iternextfunc)unpackiter_iternext, /* tp_iternext */
1650 unpackiter_methods /* tp_methods */
1651 };
1652
1653 PyDoc_STRVAR(s_iter_unpack__doc__,
1654 "S.iter_unpack(buffer) -> iterator(v1, v2, ...)\n\
1655 \n\
1656 Return an iterator yielding tuples unpacked from the given bytes\n\
1657 source, like a repeated invocation of unpack_from(). Requires\n\
1658 that the bytes length be a multiple of the struct size.");
1659
1660 static PyObject *
1661 s_iter_unpack(PyObject *_so, PyObject *input)
1662 {
1663 PyStructObject *so = (PyStructObject *) _so;
1664 unpackiterobject *self;
1665
1666 assert(PyStruct_Check(_so));
1667 assert(so->s_codes != NULL);
1668
1669 if (so->s_size == 0) {
1670 PyErr_Format(StructError,
1671 "cannot iteratively unpack with a struct of length 0");
1672 return NULL;
1673 }
1674
1675 self = (unpackiterobject *) PyType_GenericAlloc(&unpackiter_type, 0);
1676
1677 if (PyObject_GetBuffer(input, &self->buf, PyBUF_SIMPLE) < 0) {
1678 Py_DECREF(self);
1679 return NULL;
1680 }
1681 if (self->buf.len % so->s_size != 0) {
1682 PyErr_Format(StructError,
1683 "iterative unpacking requires a bytes length "
1684 "multiple of %zd",
1685 so->s_size);
1686 Py_DECREF(self);
1687 return NULL;
1688 }
1689 Py_INCREF(so);
1690 self->so = so;
1691 self->index = 0;
1692 return (PyObject *) self;
1556 } 1693 }
1557 1694
1558 1695
1559 /* 1696 /*
1560 * Guts of the pack function. 1697 * Guts of the pack function.
1561 * 1698 *
1562 * Takes a struct object, a tuple of arguments, and offset in that tuple of 1699 * Takes a struct object, a tuple of arguments, and offset in that tuple of
1563 * argument for where to start processing the arguments for packing, and a 1700 * argument for where to start processing the arguments for packing, and a
1564 * character buffer for writing the packed string. The caller must insure 1701 * character buffer for writing the packed string. The caller must insure
1565 * that the buffer may contain the required length for packing the arguments. 1702 * that the buffer may contain the required length for packing the arguments.
(...skipping 203 matching lines...) Expand 10 before | Expand all | Expand 10 after
1769 { 1906 {
1770 Py_ssize_t size; 1907 Py_ssize_t size;
1771 1908
1772 size = sizeof(PyStructObject) + sizeof(formatcode) * (self->s_len + 1); 1909 size = sizeof(PyStructObject) + sizeof(formatcode) * (self->s_len + 1);
1773 return PyLong_FromSsize_t(size); 1910 return PyLong_FromSsize_t(size);
1774 } 1911 }
1775 1912
1776 /* List of functions */ 1913 /* List of functions */
1777 1914
1778 static struct PyMethodDef s_methods[] = { 1915 static struct PyMethodDef s_methods[] = {
1916 {"iter_unpack", s_iter_unpack, METH_O, s_iter_unpack__doc__},
1779 {"pack", s_pack, METH_VARARGS, s_pack__doc__}, 1917 {"pack", s_pack, METH_VARARGS, s_pack__doc__},
1780 {"pack_into", s_pack_into, METH_VARARGS, s_pack_into__doc__}, 1918 {"pack_into", s_pack_into, METH_VARARGS, s_pack_into__doc__},
1781 {"unpack", s_unpack, METH_O, s_unpack__doc__}, 1919 {"unpack", s_unpack, METH_O, s_unpack__doc__},
1782 {"unpack_from", (PyCFunction)s_unpack_from, METH_VARARGS|METH_KEYWORDS, 1920 {"unpack_from", (PyCFunction)s_unpack_from, METH_VARARGS|METH_KEYWORDS,
1783 s_unpack_from__doc__}, 1921 s_unpack_from__doc__},
1784 {"__sizeof__", (PyCFunction)s_sizeof, METH_NOARGS, s_sizeof__doc__}, 1922 {"__sizeof__", (PyCFunction)s_sizeof, METH_NOARGS, s_sizeof__doc__},
1785 {NULL, NULL} /* sentinel */ 1923 {NULL, NULL} /* sentinel */
1786 }; 1924 };
1787 1925
1788 PyDoc_STRVAR(s__doc__, 1926 PyDoc_STRVAR(s__doc__,
(...skipping 226 matching lines...) Expand 10 before | Expand all | Expand 10 after
2015 return NULL; 2153 return NULL;
2016 2154
2017 s_object = cache_struct(fmt); 2155 s_object = cache_struct(fmt);
2018 if (s_object == NULL) { 2156 if (s_object == NULL) {
2019 Py_DECREF(newargs); 2157 Py_DECREF(newargs);
2020 return NULL; 2158 return NULL;
2021 } 2159 }
2022 result = s_unpack_from(s_object, newargs, kwds); 2160 result = s_unpack_from(s_object, newargs, kwds);
2023 Py_DECREF(newargs); 2161 Py_DECREF(newargs);
2024 Py_DECREF(s_object); 2162 Py_DECREF(s_object);
2163 return result;
2164 }
2165
2166 PyDoc_STRVAR(iter_unpack_doc,
2167 "iter_unpack(fmt, buffer) -> iterator(v1, v2, ...)\n\
2168 \n\
2169 Return an iterator yielding tuples unpacked from the given bytes\n\
2170 source according to the format string, like a repeated invocation of\n\
2171 unpack_from(). Requires that the bytes length be a multiple of the\n\
2172 format struct size.");
2173
2174 static PyObject *
2175 iter_unpack(PyObject *self, PyObject *args)
2176 {
2177 PyObject *s_object, *fmt, *input, *result;
2178
2179 if (!PyArg_ParseTuple(args, "OO:iter_unpack", &fmt, &input))
2180 return NULL;
2181
2182 s_object = cache_struct(fmt);
2183 if (s_object == NULL)
2184 return NULL;
2185 result = s_iter_unpack(s_object, input);
2186 Py_DECREF(s_object);
2025 return result; 2187 return result;
2026 } 2188 }
2027 2189
2028 static struct PyMethodDef module_functions[] = { 2190 static struct PyMethodDef module_functions[] = {
2029 {"_clearcache", (PyCFunction)clearcache, METH_NOARGS, clearcac he_doc}, 2191 {"_clearcache", (PyCFunction)clearcache, METH_NOARGS, clearcac he_doc},
2030 {"calcsize", calcsize, METH_O, calcsize_doc}, 2192 {"calcsize", calcsize, METH_O, calcsize_doc},
2193 {"iter_unpack", iter_unpack, METH_VARARGS, iter_unpack_doc},
2031 {"pack", pack, METH_VARARGS, pack_doc}, 2194 {"pack", pack, METH_VARARGS, pack_doc},
2032 {"pack_into", pack_into, METH_VARARGS, pack_into_doc}, 2195 {"pack_into", pack_into, METH_VARARGS, pack_into_doc},
2033 {"unpack", unpack, METH_VARARGS, unpack_doc}, 2196 {"unpack", unpack, METH_VARARGS, unpack_doc},
2034 {"unpack_from", (PyCFunction)unpack_from, 2197 {"unpack_from", (PyCFunction)unpack_from,
2035 METH_VARARGS|METH_KEYWORDS, unpack_from_doc}, 2198 METH_VARARGS|METH_KEYWORDS, unpack_from_doc},
2036 {NULL, NULL} /* sentinel */ 2199 {NULL, NULL} /* sentinel */
2037 }; 2200 };
2038 2201
2039 2202
2040 /* Module initialization */ 2203 /* Module initialization */
(...skipping 101 matching lines...) Expand 10 before | Expand all | Expand 10 after
2142 } 2305 }
2143 2306
2144 Py_INCREF(StructError); 2307 Py_INCREF(StructError);
2145 PyModule_AddObject(m, "error", StructError); 2308 PyModule_AddObject(m, "error", StructError);
2146 2309
2147 Py_INCREF((PyObject*)&PyStructType); 2310 Py_INCREF((PyObject*)&PyStructType);
2148 PyModule_AddObject(m, "Struct", (PyObject*)&PyStructType); 2311 PyModule_AddObject(m, "Struct", (PyObject*)&PyStructType);
2149 2312
2150 return m; 2313 return m;
2151 } 2314 }
OLDNEW
« no previous file with comments | « Lib/test/test_struct.py ('k') | no next file » | no next file with comments »

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+