Index: Include/bytesobject.h =================================================================== --- Include/bytesobject.h (revision 58109) +++ Include/bytesobject.h (working copy) @@ -18,15 +18,16 @@ * to contain a char pointer, not an unsigned char pointer. */ /* Object layout */ typedef struct { PyObject_VAR_HEAD /* XXX(nnorwitz): should ob_exports be Py_ssize_t? */ - int ob_exports; /* how many buffer exports */ + int ob_exports; /* How many buffer exports */ + int ob_readonly_exports; /* How many buffer exports as readonly */ Py_ssize_t ob_alloc; /* How many bytes allocated */ char *ob_bytes; } PyBytesObject; /* Type object */ PyAPI_DATA(PyTypeObject) PyBytes_Type; Index: Objects/bytesobject.c =================================================================== --- Objects/bytesobject.c (revision 58109) +++ Objects/bytesobject.c (working copy) @@ -2,14 +2,114 @@ /* XXX TO DO: optimizations */ #define PY_SSIZE_T_CLEAN #include "Python.h" #include "structmember.h" +/* + * Constants for use with the PyBytesObject.ob_readonly_exports. + * XXX(gps) rename these and move them to the header file. + */ +#define MAX_READONLY_EXPORTS (INT_MAX) +#define MAX_EXPORTS (INT_MAX) + +/* + * Should we bounds check PyBytesObject.ob_exports and + * ob_readonly_exports when we increment them? + */ +#if MAX_READONLY_EXPORTS <= USHRT_MAX +#define BOUNDS_CHECK_EXPORTS 1 +#else +#undef BOUNDS_CHECK_EXPORTS +#endif + +/* + * TODO(gps) Do we want to provide an exported interface for any of + * these inlines for use by C code that uses Bytes objects directly + * rather than the buffer API? I suggest C code should prefer to use + * the buffer API (though it is heavier weight). Exporting is_readonly() + * might be useful? + */ + +/* + * Is this bytes object currently read only? 0: no, 1: yes + */ +Py_LOCAL_INLINE(int) is_readonly(PyBytesObject *obj) +{ + assert(obj->ob_readonly_exports <= obj->ob_exports); + return (obj->ob_readonly_exports > 0 && obj->ob_exports == 0); +} + +/* + * Increment the export count. For use by getbuffer. + * + * Returns: 0 on success, -1 on failure with an exception set. + * (-1 matches the required buffer API getbuffer return value) + */ +Py_LOCAL_INLINE(int) inc_exports(PyBytesObject *obj) +{ +#ifdef BOUNDS_CHECK_EXPORTS + if (MAX_EXPORTS <= obj->ob_exports) { + /* XXX(gps): include object id in the error? */ + PyErr_SetString(PyExc_RuntimeError, + "ob_exports overflow"); + return -1; + } +#endif + obj->ob_exports++; + return 0; +} + +/* + * Decrement the export count. For use by releasebuffer. + */ +Py_LOCAL_INLINE(void) dec_exports(PyBytesObject *obj) +{ + obj->ob_exports--; +} + + +/* + * Increment the readonly export count if the object is mutable. + * Must be called with the GIL held. + * + * For use by the buffer API to implement PyBUF_LOCKDATA requests. + * + * Returns: 0 on success, -1 on failure with an exception set. + * (-1 matches the required buffer API getbuffer return value) + */ +Py_LOCAL_INLINE(int) inc_readonly_exports(PyBytesObject *obj) +{ +#ifdef BOUNDS_CHECK_EXPORTS + if (MAX_READONLY_EXPORTS <= obj->ob_readonly_exports) { + /* XXX(gps): include object id in the error? */ + PyErr_SetString(PyExc_RuntimeError, + "ob_readonly_exports overflow"); + return 1; + } +#endif + obj->ob_readonly_exports++; + return 0; +} + + +/* + * Decrement the readonly export count. + * Must be called with the GIL held. + * + * For use by the buffer API to implement PyBUF_LOCKDATA requests. + */ +Py_LOCAL_INLINE(void) dec_readonly_exports(PyBytesObject *obj) +{ + assert(obj->ob_readonly_exports <= obj->ob_exports); + obj->ob_readonly_exports--; +} + + /* The nullbytes are used by the stringlib during partition. * If partition is removed from bytes, nullbytes and its helper * Init/Fini should also be removed. */ static PyBytesObject *nullbytes = NULL; void @@ -23,14 +123,15 @@ { nullbytes = PyObject_New(PyBytesObject, &PyBytes_Type); if (nullbytes == NULL) return 0; nullbytes->ob_bytes = NULL; Py_Size(nullbytes) = nullbytes->ob_alloc = 0; nullbytes->ob_exports = 0; + nullbytes->ob_readonly_exports = 0; return 1; } /* end nullbytes support */ /* Helpers */ @@ -50,35 +151,48 @@ } static int bytes_getbuffer(PyBytesObject *obj, PyBuffer *view, int flags) { int ret; void *ptr; + int readonly = 0; if (view == NULL) { - obj->ob_exports++; - return 0; + return inc_exports(obj); } if (obj->ob_bytes == NULL) ptr = ""; else ptr = obj->ob_bytes; - ret = PyBuffer_FillInfo(view, ptr, Py_Size(obj), 0, flags); + if (((flags & PyBUF_LOCKDATA) == PyBUF_LOCKDATA) && + obj->ob_exports == 0) { + if (inc_readonly_exports(obj)) + return -1; + readonly = -1; + } else { + readonly = is_readonly(obj); + } + ret = PyBuffer_FillInfo(view, ptr, Py_Size(obj), readonly, flags); if (ret >= 0) { - obj->ob_exports++; + return inc_exports(obj); } return ret; } static void bytes_releasebuffer(PyBytesObject *obj, PyBuffer *view) { - obj->ob_exports--; + dec_exports(obj); + if (view && view->readonly == -1) + dec_readonly_exports(obj); } +/* Internal function: + * Get a simple buffer view for read-only use with the GIL held. + */ static Py_ssize_t _getbuffer(PyObject *obj, PyBuffer *view) { PyBufferProcs *buffer = Py_Type(obj)->tp_as_buffer; if (buffer == NULL || PyUnicode_Check(obj) || @@ -130,14 +244,15 @@ if (bytes != NULL) memcpy(new->ob_bytes, bytes, size); new->ob_bytes[size] = '\0'; /* Trailing null byte */ } Py_Size(new) = size; new->ob_alloc = alloc; new->ob_exports = 0; + new->ob_readonly_exports = 0; return (PyObject *)new; } Py_ssize_t PyBytes_Size(PyObject *self) { @@ -152,24 +267,34 @@ { assert(self != NULL); assert(PyBytes_Check(self)); return PyBytes_AS_STRING(self); } +#define SET_RO_ERROR(bo) do { \ + PyErr_SetString(PyExc_BufferError, \ + "Readonly export exists: object cannot be modified"); \ + } while (0); + int PyBytes_Resize(PyObject *self, Py_ssize_t size) { void *sval; Py_ssize_t alloc = ((PyBytesObject *)self)->ob_alloc; assert(self != NULL); assert(PyBytes_Check(self)); assert(size >= 0); + if (is_readonly((PyBytesObject *)self)) { + SET_RO_ERROR(self); + return -1; + } + if (size < alloc / 2) { /* Major downsize; resize down to exact size */ alloc = size + 1; } else if (size < alloc) { /* Within allocated size; quick exit */ Py_Size(self) = size; @@ -276,15 +401,21 @@ mysize = Py_Size(self); size = mysize + vo.len; if (size < 0) { PyObject_ReleaseBuffer(other, &vo); return PyErr_NoMemory(); } + if (size < self->ob_alloc) { + if (is_readonly((PyBytesObject *)self)) { + SET_RO_ERROR(self); + PyObject_ReleaseBuffer(other, &vo); + return NULL; + } Py_Size(self) = size; self->ob_bytes[Py_Size(self)] = '\0'; /* Trailing null byte */ } else if (PyBytes_Resize((PyObject *)self, size) < 0) { PyObject_ReleaseBuffer(other, &vo); return NULL; } @@ -328,15 +459,20 @@ if (count < 0) count = 0; mysize = Py_Size(self); size = mysize * count; if (count != 0 && size / count != mysize) return PyErr_NoMemory(); + if (size < self->ob_alloc) { + if (is_readonly((PyBytesObject *)self)) { + SET_RO_ERROR(self); + return NULL; + } Py_Size(self) = size; self->ob_bytes[Py_Size(self)] = '\0'; /* Trailing null byte */ } else if (PyBytes_Resize((PyObject *)self, size) < 0) return NULL; if (mysize == 1) @@ -488,14 +624,20 @@ Py_Type(values)->tp_name); return -1; } needed = vbytes.len; bytes = vbytes.buf; } + if (is_readonly((PyBytesObject *)self)) { + SET_RO_ERROR(self); + res = -1; + goto finish; + } + if (lo < 0) lo = 0; if (hi < lo) hi = lo; if (hi > Py_Size(self)) hi = Py_Size(self); @@ -531,15 +673,14 @@ Py_Size(self) - lo - needed); } } if (needed > 0) memcpy(self->ob_bytes + lo, bytes, needed); - finish: if (vbytes.len != -1) PyObject_ReleaseBuffer(values, &vbytes); return res; } static int @@ -554,14 +695,19 @@ PyErr_SetString(PyExc_IndexError, "bytes index out of range"); return -1; } if (value == NULL) return bytes_setslice(self, i, i+1, NULL); + if (is_readonly((PyBytesObject *)self)) { + SET_RO_ERROR(self); + return -1; + } + ival = PyNumber_AsSsize_t(value, PyExc_ValueError); if (ival == -1 && PyErr_Occurred()) return -1; if (ival < 0 || ival >= 256) { PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); return -1; @@ -573,14 +719,19 @@ static int bytes_ass_subscript(PyBytesObject *self, PyObject *item, PyObject *values) { Py_ssize_t start, stop, step, slicelen, needed; char *bytes; + if (is_readonly((PyBytesObject *)self)) { + SET_RO_ERROR(self); + return -1; + } + if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) return -1; if (i < 0) @@ -1329,14 +1480,16 @@ "B.translate(table [,deletechars]) -> bytes\n\ \n\ Return a copy of the bytes B, where all characters occurring\n\ in the optional argument deletechars are removed, and the\n\ remaining characters have been mapped through the given\n\ translation table, which must be a bytes of length 256."); +/* XXX(gps): bytes could also use an in place bytes_itranslate method? */ + static PyObject * bytes_translate(PyBytesObject *self, PyObject *args) { register char *input, *output; register const char *table; register Py_ssize_t i, c, changed = 0; PyObject *input_obj = (PyObject*)self; @@ -2024,14 +2177,16 @@ PyDoc_STRVAR(replace__doc__, "B.replace (old, new[, count]) -> bytes\n\ \n\ Return a copy of bytes B with all occurrences of subsection\n\ old replaced by new. If the optional argument count is\n\ given, only the first count occurrences are replaced."); +/* XXX(gps): bytes could also use an in place bytes_ireplace method? */ + static PyObject * bytes_replace(PyBytesObject *self, PyObject *args) { Py_ssize_t count = -1; PyObject *from, *to, *res; PyBuffer vfrom, vto; @@ -2471,14 +2626,19 @@ Reverse the order of the values in bytes in place."); static PyObject * bytes_reverse(PyBytesObject *self, PyObject *unused) { char swap, *head, *tail; Py_ssize_t i, j, n = Py_Size(self); + if (is_readonly((PyBytesObject *)self)) { + SET_RO_ERROR(self); + return NULL; + } + j = n / 2; head = self->ob_bytes; tail = head + n - 1; for (i = 0; i < j; i++) { swap = *head; *head++ = *tail; *tail-- = swap; @@ -2496,14 +2656,19 @@ { int value; Py_ssize_t where, n = Py_Size(self); if (!PyArg_ParseTuple(args, "ni:insert", &where, &value)) return NULL; + if (is_readonly((PyBytesObject *)self)) { + SET_RO_ERROR(self); + return NULL; + } + if (n == PY_SSIZE_T_MAX) { PyErr_SetString(PyExc_OverflowError, "cannot add more objects to bytes"); return NULL; } if (value < 0 || value >= 256) { PyErr_SetString(PyExc_ValueError, @@ -2561,14 +2726,19 @@ { int value; Py_ssize_t where = -1, n = Py_Size(self); if (!PyArg_ParseTuple(args, "|n:pop", &where)) return NULL; + if (is_readonly((PyBytesObject *)self)) { + SET_RO_ERROR(self); + return NULL; + } + if (n == 0) { PyErr_SetString(PyExc_OverflowError, "cannot pop an empty bytes"); return NULL; } if (where < 0) where += Py_Size(self); @@ -2594,14 +2764,19 @@ { int value; Py_ssize_t where, n = Py_Size(self); if (! _getbytevalue(arg, &value)) return NULL; + if (is_readonly((PyBytesObject *)self)) { + SET_RO_ERROR(self); + return NULL; + } + for (where = 0; where < n; where++) { if (self->ob_bytes[where] == value) break; } if (where == n) { PyErr_SetString(PyExc_ValueError, "value not found in bytes"); return NULL;