diff -r 4d65893a956f Doc/c-api/arg.rst --- a/Doc/c-api/arg.rst Sat Aug 02 23:58:05 2008 +0200 +++ b/Doc/c-api/arg.rst Sun Aug 03 00:44:56 2008 +0200 @@ -40,12 +40,24 @@ variable(s) whose address should be pass other read-buffer compatible objects pass back a reference to the raw internal data representation. +``s*`` (string, Unicode, or any buffer compatible object) [Py_buffer \*] + Similar to ``s#``, this code fills a Py_buffer structure provided by the caller. + The buffer gets locked, so that the caller can subsequently use the buffer even + inside a ``Py_BEGIN_ALLOW_THREADS`` block; the caller is responsible for calling + ``PyBuffer_Release`` with the structure after it has processed the data. + + .. versionadded:: 2.6 + ``z`` (string or ``None``) [const char \*] Like ``s``, but the Python object may also be ``None``, in which case the C pointer is set to *NULL*. ``z#`` (string or ``None`` or any read buffer compatible object) [const char \*, int] This is to ``s#`` as ``z`` is to ``s``. + +``z*`` (string or ``None`` or any buffer compatible object) [Py_buffer*] + This is to ``s*`` as ``z`` is to ``s``. + .. versionadded:: 2.6 ``u`` (Unicode object) [Py_UNICODE \*] Convert a Python Unicode object to a C pointer to a NUL-terminated buffer of @@ -240,6 +252,10 @@ variable(s) whose address should be pass single-segment buffer objects are accepted; :exc:`TypeError` is raised for all others. +``w*`` (read-write byte-oriented buffer) [Py_buffer \*] + This is to ``w`` what ``s*`` is to ``s``. + .. versionadded:: 2.6 + ``(items)`` (tuple) [*matching-items*] The object must be a Python sequence whose length is the number of format units in *items*. The C arguments must correspond to the individual format units in diff -r 4d65893a956f Include/abstract.h --- a/Include/abstract.h Sat Aug 02 23:58:05 2008 +0200 +++ b/Include/abstract.h Sun Aug 03 00:44:56 2008 +0200 @@ -623,7 +623,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx per element. */ - PyAPI_FUNC(int) PyBuffer_FillInfo(Py_buffer *view, void *buf, + PyAPI_FUNC(int) PyBuffer_FillInfo(Py_buffer *view, PyObject *o, void *buf, Py_ssize_t len, int readonly, int flags); @@ -632,6 +632,11 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx "unsigned bytes" of the given length. Returns 0 on success and -1 (with raising an error) on error. */ + + PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view); + + /* Releases a Py_buffer obtained from getbuffer ParseTuple's s*. + */ PyAPI_FUNC(PyObject *) PyObject_Format(PyObject* obj, PyObject *format_spec); diff -r 4d65893a956f Include/object.h --- a/Include/object.h Sat Aug 02 23:58:05 2008 +0200 +++ b/Include/object.h Sun Aug 03 00:44:56 2008 +0200 @@ -162,7 +162,8 @@ typedef Py_ssize_t (*charbufferproc)(PyO /* Py3k buffer interface */ typedef struct bufferinfo { - void *buf; + void *buf; + PyObject *obj; /* borrowed reference */ Py_ssize_t len; Py_ssize_t itemsize; /* This is Py_ssize_t so it can be pointed to by strides in simple case.*/ diff -r 4d65893a956f Modules/_codecsmodule.c --- a/Modules/_codecsmodule.c Sat Aug 02 23:58:05 2008 +0200 +++ b/Modules/_codecsmodule.c Sun Aug 03 00:44:56 2008 +0200 @@ -232,20 +232,20 @@ utf_7_decode(PyObject *self, utf_7_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int final = 0; Py_ssize_t consumed; PyObject *decoded = NULL; - if (!PyArg_ParseTuple(args, "t#|zi:utf_7_decode", - &data, &size, &errors, &final)) - return NULL; - consumed = size; + if (!PyArg_ParseTuple(args, "s*|zi:utf_7_decode", + &pbuf, &errors, &final)) + return NULL; + consumed = pbuf.len; - decoded = PyUnicode_DecodeUTF7Stateful(data, size, errors, - final ? NULL : &consumed); + decoded = PyUnicode_DecodeUTF7Stateful(pbuf.buf, pbuf.len, errors, + final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (decoded == NULL) return NULL; return codec_tuple(decoded, consumed); @@ -255,24 +255,20 @@ utf_8_decode(PyObject *self, utf_8_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int final = 0; Py_ssize_t consumed; PyObject *decoded = NULL; - if (!PyArg_ParseTuple(args, "t#|zi:utf_8_decode", - &data, &size, &errors, &final)) + if (!PyArg_ParseTuple(args, "s*|zi:utf_8_decode", + &pbuf, &errors, &final)) return NULL; - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative argument"); - return 0; - } - consumed = size; - - decoded = PyUnicode_DecodeUTF8Stateful(data, size, errors, + consumed = pbuf.len; + + decoded = PyUnicode_DecodeUTF8Stateful(pbuf.buf, pbuf.len, errors, final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (decoded == NULL) return NULL; return codec_tuple(decoded, consumed); @@ -282,24 +278,20 @@ utf_16_decode(PyObject *self, utf_16_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int byteorder = 0; int final = 0; Py_ssize_t consumed; PyObject *decoded; - if (!PyArg_ParseTuple(args, "t#|zi:utf_16_decode", - &data, &size, &errors, &final)) + if (!PyArg_ParseTuple(args, "s*|zi:utf_16_decode", + &pbuf, &errors, &final)) return NULL; - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative argument"); - return 0; - } - consumed = size; /* This is overwritten unless final is true. */ - decoded = PyUnicode_DecodeUTF16Stateful(data, size, errors, &byteorder, - final ? NULL : &consumed); + consumed = pbuf.len; /* This is overwritten unless final is true. */ + decoded = PyUnicode_DecodeUTF16Stateful(pbuf.buf, pbuf.len, errors, + &byteorder, final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (decoded == NULL) return NULL; return codec_tuple(decoded, consumed); @@ -309,53 +301,45 @@ utf_16_le_decode(PyObject *self, utf_16_le_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int byteorder = -1; int final = 0; Py_ssize_t consumed; PyObject *decoded = NULL; - if (!PyArg_ParseTuple(args, "t#|zi:utf_16_le_decode", - &data, &size, &errors, &final)) + if (!PyArg_ParseTuple(args, "s*|zi:utf_16_le_decode", + &pbuf, &errors, &final)) return NULL; - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative argument"); - return 0; - } - consumed = size; /* This is overwritten unless final is true. */ - decoded = PyUnicode_DecodeUTF16Stateful(data, size, errors, + consumed = pbuf.len; /* This is overwritten unless final is true. */ + decoded = PyUnicode_DecodeUTF16Stateful(pbuf.buf, pbuf.len, errors, &byteorder, final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (decoded == NULL) return NULL; return codec_tuple(decoded, consumed); - } static PyObject * utf_16_be_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int byteorder = 1; int final = 0; Py_ssize_t consumed; PyObject *decoded = NULL; - if (!PyArg_ParseTuple(args, "t#|zi:utf_16_be_decode", - &data, &size, &errors, &final)) + if (!PyArg_ParseTuple(args, "s*|zi:utf_16_be_decode", + &pbuf, &errors, &final)) return NULL; - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative argument"); - return 0; - } - consumed = size; /* This is overwritten unless final is true. */ - decoded = PyUnicode_DecodeUTF16Stateful(data, size, errors, + + consumed = pbuf.len; /* This is overwritten unless final is true. */ + decoded = PyUnicode_DecodeUTF16Stateful(pbuf.buf, pbuf.len, errors, &byteorder, final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (decoded == NULL) return NULL; return codec_tuple(decoded, consumed); @@ -373,24 +357,20 @@ utf_16_ex_decode(PyObject *self, utf_16_ex_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int byteorder = 0; PyObject *unicode, *tuple; int final = 0; Py_ssize_t consumed; - if (!PyArg_ParseTuple(args, "t#|zii:utf_16_ex_decode", - &data, &size, &errors, &byteorder, &final)) + if (!PyArg_ParseTuple(args, "s*|zii:utf_16_ex_decode", + &pbuf, &errors, &byteorder, &final)) return NULL; - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative argument"); - return 0; - } - consumed = size; /* This is overwritten unless final is true. */ - unicode = PyUnicode_DecodeUTF16Stateful(data, size, errors, &byteorder, - final ? NULL : &consumed); + consumed = pbuf.len; /* This is overwritten unless final is true. */ + unicode = PyUnicode_DecodeUTF16Stateful(pbuf.buf, pbuf.len, errors, + &byteorder, final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (unicode == NULL) return NULL; tuple = Py_BuildValue("Oni", unicode, consumed, byteorder); @@ -402,24 +382,20 @@ utf_32_decode(PyObject *self, utf_32_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int byteorder = 0; int final = 0; Py_ssize_t consumed; PyObject *decoded; - if (!PyArg_ParseTuple(args, "t#|zi:utf_32_decode", - &data, &size, &errors, &final)) + if (!PyArg_ParseTuple(args, "s*|zi:utf_32_decode", + &pbuf, &errors, &final)) return NULL; - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative argument"); - return 0; - } - consumed = size; /* This is overwritten unless final is true. */ - decoded = PyUnicode_DecodeUTF32Stateful(data, size, errors, &byteorder, - final ? NULL : &consumed); + consumed = pbuf.len; /* This is overwritten unless final is true. */ + decoded = PyUnicode_DecodeUTF32Stateful(pbuf.buf, pbuf.len, errors, + &byteorder, final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (decoded == NULL) return NULL; return codec_tuple(decoded, consumed); @@ -429,53 +405,43 @@ utf_32_le_decode(PyObject *self, utf_32_le_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int byteorder = -1; int final = 0; Py_ssize_t consumed; - PyObject *decoded = NULL; + PyObject *decoded; - if (!PyArg_ParseTuple(args, "t#|zi:utf_32_le_decode", - &data, &size, &errors, &final)) + if (!PyArg_ParseTuple(args, "s*|zi:utf_32_le_decode", + &pbuf, &errors, &final)) return NULL; - - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative argument"); - return 0; - } - consumed = size; /* This is overwritten unless final is true. */ - decoded = PyUnicode_DecodeUTF32Stateful(data, size, errors, - &byteorder, final ? NULL : &consumed); + consumed = pbuf.len; /* This is overwritten unless final is true. */ + decoded = PyUnicode_DecodeUTF32Stateful(pbuf.buf, pbuf.len, errors, + &byteorder, final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (decoded == NULL) return NULL; return codec_tuple(decoded, consumed); - } static PyObject * utf_32_be_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int byteorder = 1; int final = 0; Py_ssize_t consumed; - PyObject *decoded = NULL; + PyObject *decoded; - if (!PyArg_ParseTuple(args, "t#|zi:utf_32_be_decode", - &data, &size, &errors, &final)) + if (!PyArg_ParseTuple(args, "s*|zi:utf_32_be_decode", + &pbuf, &errors, &final)) return NULL; - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative argument"); - return 0; - } - consumed = size; /* This is overwritten unless final is true. */ - decoded = PyUnicode_DecodeUTF32Stateful(data, size, errors, - &byteorder, final ? NULL : &consumed); + consumed = pbuf.len; /* This is overwritten unless final is true. */ + decoded = PyUnicode_DecodeUTF32Stateful(pbuf.buf, pbuf.len, errors, + &byteorder, final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (decoded == NULL) return NULL; return codec_tuple(decoded, consumed); @@ -493,24 +459,20 @@ utf_32_ex_decode(PyObject *self, utf_32_ex_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int byteorder = 0; PyObject *unicode, *tuple; int final = 0; Py_ssize_t consumed; - if (!PyArg_ParseTuple(args, "t#|zii:utf_32_ex_decode", - &data, &size, &errors, &byteorder, &final)) + if (!PyArg_ParseTuple(args, "s*|zii:utf_32_ex_decode", + &pbuf, &errors, &byteorder, &final)) return NULL; - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative argument"); - return 0; - } - consumed = size; /* This is overwritten unless final is true. */ - unicode = PyUnicode_DecodeUTF32Stateful(data, size, errors, &byteorder, - final ? NULL : &consumed); + consumed = pbuf.len; /* This is overwritten unless final is true. */ + unicode = PyUnicode_DecodeUTF32Stateful(pbuf.buf, pbuf.len, errors, + &byteorder, final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (unicode == NULL) return NULL; tuple = Py_BuildValue("Oni", unicode, consumed, byteorder); @@ -522,83 +484,88 @@ unicode_escape_decode(PyObject *self, unicode_escape_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; + PyObject *unicode; - if (!PyArg_ParseTuple(args, "t#|z:unicode_escape_decode", - &data, &size, &errors)) + if (!PyArg_ParseTuple(args, "s*|z:unicode_escape_decode", + &pbuf, &errors)) return NULL; - return codec_tuple(PyUnicode_DecodeUnicodeEscape(data, size, errors), - size); + unicode = PyUnicode_DecodeUnicodeEscape(pbuf.buf, pbuf.len, errors); + PyBuffer_Release(&pbuf); + return codec_tuple(unicode, pbuf.len); } static PyObject * raw_unicode_escape_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; + PyObject *unicode; - if (!PyArg_ParseTuple(args, "t#|z:raw_unicode_escape_decode", - &data, &size, &errors)) + if (!PyArg_ParseTuple(args, "s*|z:raw_unicode_escape_decode", + &pbuf, &errors)) return NULL; - return codec_tuple(PyUnicode_DecodeRawUnicodeEscape(data, size, errors), - size); + unicode = PyUnicode_DecodeRawUnicodeEscape(pbuf.buf, pbuf.len, errors); + PyBuffer_Release(&pbuf); + return codec_tuple(unicode, pbuf.len); } static PyObject * latin_1_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; + PyObject *unicode; const char *errors = NULL; - if (!PyArg_ParseTuple(args, "t#|z:latin_1_decode", - &data, &size, &errors)) + if (!PyArg_ParseTuple(args, "s*|z:latin_1_decode", + &pbuf, &errors)) return NULL; - return codec_tuple(PyUnicode_DecodeLatin1(data, size, errors), - size); + unicode = PyUnicode_DecodeLatin1(pbuf.buf, pbuf.len, errors); + PyBuffer_Release(&pbuf); + return codec_tuple(unicode, pbuf.len); } static PyObject * ascii_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; + PyObject *unicode; const char *errors = NULL; - if (!PyArg_ParseTuple(args, "t#|z:ascii_decode", - &data, &size, &errors)) + if (!PyArg_ParseTuple(args, "s*|z:ascii_decode", + &pbuf, &errors)) return NULL; - return codec_tuple(PyUnicode_DecodeASCII(data, size, errors), - size); + unicode = PyUnicode_DecodeASCII(pbuf.buf, pbuf.len, errors); + PyBuffer_Release(&pbuf); + return codec_tuple(unicode, pbuf.len); } static PyObject * charmap_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; + PyObject *unicode; const char *errors = NULL; PyObject *mapping = NULL; - if (!PyArg_ParseTuple(args, "t#|zO:charmap_decode", - &data, &size, &errors, &mapping)) + if (!PyArg_ParseTuple(args, "s*|zO:charmap_decode", + &pbuf, &errors, &mapping)) return NULL; if (mapping == Py_None) mapping = NULL; - return codec_tuple(PyUnicode_DecodeCharmap(data, size, mapping, errors), - size); + unicode = PyUnicode_DecodeCharmap(pbuf.buf, pbuf.len, mapping, errors); + PyBuffer_Release(&pbuf); + return codec_tuple(unicode, pbuf.len); } #if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) @@ -607,21 +574,23 @@ mbcs_decode(PyObject *self, mbcs_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size, consumed; + Py_buffer pbuf; const char *errors = NULL; int final = 0; - PyObject *decoded; + Py_ssize_t consumed; + PyObject *decoded = NULL; - if (!PyArg_ParseTuple(args, "t#|zi:mbcs_decode", - &data, &size, &errors, &final)) + if (!PyArg_ParseTuple(args, "s*|zi:mbcs_decode", + &pbuf, &errors, &final)) return NULL; + consumed = pbuf.len; - decoded = PyUnicode_DecodeMBCSStateful( - data, size, errors, final ? NULL : &consumed); - if (!decoded) + decoded = PyUnicode_DecodeMBCSStateful(pbuf.buf, pbuf.len, errors, + final ? NULL : &consumed); + PyBuffer_Release(&pbuf); + if (decoded == NULL) return NULL; - return codec_tuple(decoded, final ? size : consumed); + return codec_tuple(decoded, consumed); } #endif /* MS_WINDOWS */ diff -r 4d65893a956f Modules/_fileio.c --- a/Modules/_fileio.c Sat Aug 02 23:58:05 2008 +0200 +++ b/Modules/_fileio.c Sun Aug 03 00:44:56 2008 +0200 @@ -357,7 +357,7 @@ static PyObject * static PyObject * fileio_readinto(PyFileIOObject *self, PyObject *args) { - char *ptr; + Py_buffer pbuf; Py_ssize_t n; if (self->fd < 0) @@ -365,13 +365,14 @@ fileio_readinto(PyFileIOObject *self, Py if (!self->readable) return err_mode("reading"); - if (!PyArg_ParseTuple(args, "w#", &ptr, &n)) + if (!PyArg_ParseTuple(args, "w*", &pbuf)) return NULL; Py_BEGIN_ALLOW_THREADS errno = 0; - n = read(self->fd, ptr, n); + n = read(self->fd, pbuf.buf, pbuf.len); Py_END_ALLOW_THREADS + PyBuffer_Release(&pbuf); if (n < 0) { if (errno == EAGAIN) Py_RETURN_NONE; @@ -489,21 +490,23 @@ static PyObject * static PyObject * fileio_write(PyFileIOObject *self, PyObject *args) { + Py_buffer pbuf; Py_ssize_t n; - char *ptr; if (self->fd < 0) return err_closed(); if (!self->writable) return err_mode("writing"); - if (!PyArg_ParseTuple(args, "s#", &ptr, &n)) + if (!PyArg_ParseTuple(args, "s*", &pbuf)) return NULL; Py_BEGIN_ALLOW_THREADS errno = 0; - n = write(self->fd, ptr, n); + n = write(self->fd, pbuf.buf, pbuf.len); Py_END_ALLOW_THREADS + + PyBuffer_Release(&pbuf); if (n < 0) { if (errno == EAGAIN) diff -r 4d65893a956f Modules/_multiprocessing/connection.h --- a/Modules/_multiprocessing/connection.h Sat Aug 02 23:58:05 2008 +0200 +++ b/Modules/_multiprocessing/connection.h Sun Aug 03 00:44:56 2008 +0200 @@ -187,21 +187,25 @@ connection_recvbytes_into(ConnectionObje char *freeme = NULL, *buffer = NULL; Py_ssize_t res, length, offset = 0; PyObject *result = NULL; + Py_buffer pbuf; - if (!PyArg_ParseTuple(args, "w#|" F_PY_SSIZE_T, - &buffer, &length, &offset)) + CHECK_READABLE(self); + + if (!PyArg_ParseTuple(args, "w*|" F_PY_SSIZE_T, + &pbuf, &offset)) return NULL; - CHECK_READABLE(self); + buffer = pbuf.buf; + length = pbuf.len; if (offset < 0) { PyErr_SetString(PyExc_ValueError, "negative offset"); - return NULL; + goto _error; } if (offset > length) { PyErr_SetString(PyExc_ValueError, "offset too large"); - return NULL; + goto _error; } res = conn_recv_string(self, buffer+offset, length-offset, @@ -231,11 +235,17 @@ connection_recvbytes_into(ConnectionObje PyErr_SetObject(BufferTooShort, result); Py_DECREF(result); } - return NULL; + goto _error; } } +_cleanup: + PyBuffer_Release(&pbuf); return result; + +_error: + result = NULL; + goto _cleanup; } /* diff -r 4d65893a956f Modules/posixmodule.c --- a/Modules/posixmodule.c Sat Aug 02 23:58:05 2008 +0200 +++ b/Modules/posixmodule.c Sun Aug 03 00:44:56 2008 +0200 @@ -6334,15 +6334,16 @@ static PyObject * static PyObject * posix_write(PyObject *self, PyObject *args) { + Py_buffer pbuf; int fd; Py_ssize_t size; - char *buffer; - - if (!PyArg_ParseTuple(args, "is#:write", &fd, &buffer, &size)) - return NULL; - Py_BEGIN_ALLOW_THREADS - size = write(fd, buffer, (size_t)size); - Py_END_ALLOW_THREADS + + if (!PyArg_ParseTuple(args, "is*:write", &fd, &pbuf)) + return NULL; + Py_BEGIN_ALLOW_THREADS + size = write(fd, pbuf.buf, (size_t)pbuf.len); + Py_END_ALLOW_THREADS + PyBuffer_Release(&pbuf); if (size < 0) return posix_error(); return PyInt_FromSsize_t(size); diff -r 4d65893a956f Modules/socketmodule.c --- a/Modules/socketmodule.c Sat Aug 02 23:58:05 2008 +0200 +++ b/Modules/socketmodule.c Sun Aug 03 00:44:56 2008 +0200 @@ -2647,12 +2647,17 @@ sock_send(PySocketSockObject *s, PyObjec { char *buf; int len, n = -1, flags = 0, timeout; - - if (!PyArg_ParseTuple(args, "s#|i:send", &buf, &len, &flags)) - return NULL; - - if (!IS_SELECTABLE(s)) + Py_buffer pbuf; + + if (!PyArg_ParseTuple(args, "s*|i:send", &pbuf, &flags)) + return NULL; + + if (!IS_SELECTABLE(s)) { + PyBuffer_Release(&pbuf); return select_error(); + } + buf = pbuf.buf; + len = pbuf.len; Py_BEGIN_ALLOW_THREADS timeout = internal_select(s, 1); @@ -2663,6 +2668,8 @@ sock_send(PySocketSockObject *s, PyObjec n = send(s->sock_fd, buf, len, flags); #endif Py_END_ALLOW_THREADS + + PyBuffer_Release(&pbuf); if (timeout == 1) { PyErr_SetString(socket_timeout, "timed out"); @@ -2688,12 +2695,17 @@ sock_sendall(PySocketSockObject *s, PyOb { char *buf; int len, n = -1, flags = 0, timeout; - - if (!PyArg_ParseTuple(args, "s#|i:sendall", &buf, &len, &flags)) - return NULL; - - if (!IS_SELECTABLE(s)) + Py_buffer pbuf; + + if (!PyArg_ParseTuple(args, "s*|i:sendall", &pbuf, &flags)) + return NULL; + buf = pbuf.buf; + len = pbuf.len; + + if (!IS_SELECTABLE(s)) { + PyBuffer_Release(&pbuf); return select_error(); + } Py_BEGIN_ALLOW_THREADS do { @@ -2712,6 +2724,7 @@ sock_sendall(PySocketSockObject *s, PyOb len -= n; } while (len > 0); Py_END_ALLOW_THREADS + PyBuffer_Release(&pbuf); if (timeout == 1) { PyErr_SetString(socket_timeout, "timed out"); @@ -2738,24 +2751,32 @@ static PyObject * static PyObject * sock_sendto(PySocketSockObject *s, PyObject *args) { + Py_buffer pbuf; PyObject *addro; char *buf; - sock_addr_t addrbuf; - int addrlen, len, n = -1, flags, timeout; + Py_ssize_t len; + sock_addr_t addrbuf; + int addrlen, n = -1, flags, timeout; flags = 0; - if (!PyArg_ParseTuple(args, "s#O:sendto", &buf, &len, &addro)) { + if (!PyArg_ParseTuple(args, "s*O:sendto", &pbuf, &addro)) { PyErr_Clear(); - if (!PyArg_ParseTuple(args, "s#iO:sendto", - &buf, &len, &flags, &addro)) - return NULL; - } - - if (!IS_SELECTABLE(s)) + if (!PyArg_ParseTuple(args, "s*iO:sendto", + &pbuf, &flags, &addro)) + return NULL; + } + buf = pbuf.buf; + len = pbuf.len; + + if (!IS_SELECTABLE(s)) { + PyBuffer_Release(&pbuf); return select_error(); - - if (!getsockaddrarg(s, addro, SAS2SA(&addrbuf), &addrlen)) - return NULL; + } + + if (!getsockaddrarg(s, addro, SAS2SA(&addrbuf), &addrlen)) { + PyBuffer_Release(&pbuf); + return NULL; + } Py_BEGIN_ALLOW_THREADS timeout = internal_select(s, 1); @@ -2763,6 +2784,7 @@ sock_sendto(PySocketSockObject *s, PyObj n = sendto(s->sock_fd, buf, len, flags, SAS2SA(&addrbuf), addrlen); Py_END_ALLOW_THREADS + PyBuffer_Release(&pbuf); if (timeout == 1) { PyErr_SetString(socket_timeout, "timed out"); return NULL; diff -r 4d65893a956f Objects/abstract.c --- a/Objects/abstract.c Sat Aug 02 23:58:05 2008 +0200 +++ b/Objects/abstract.c Sun Aug 03 00:44:56 2008 +0200 @@ -681,7 +681,7 @@ PyBuffer_FillContiguousStrides(int nd, P } int -PyBuffer_FillInfo(Py_buffer *view, void *buf, Py_ssize_t len, +PyBuffer_FillInfo(Py_buffer *view, PyObject *obj, void *buf, Py_ssize_t len, int readonly, int flags) { if (view == NULL) return 0; @@ -692,6 +692,7 @@ PyBuffer_FillInfo(Py_buffer *view, void return -1; } + view->obj = obj; view->buf = buf; view->len = len; view->readonly = readonly; @@ -709,6 +710,17 @@ PyBuffer_FillInfo(Py_buffer *view, void view->suboffsets = NULL; view->internal = NULL; return 0; +} + +void +PyBuffer_Release(Py_buffer *view) +{ + PyObject *obj = view->obj; + if (!obj || !Py_TYPE(obj)->tp_as_buffer || !Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer) + /* Unmanaged buffer */ + return; + Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer(obj, view); + } PyObject * diff -r 4d65893a956f Objects/bytearrayobject.c --- a/Objects/bytearrayobject.c Sat Aug 02 23:58:05 2008 +0200 +++ b/Objects/bytearrayobject.c Sun Aug 03 00:44:56 2008 +0200 @@ -123,7 +123,7 @@ bytes_getbuffer(PyByteArrayObject *obj, ptr = ""; else ptr = obj->ob_bytes; - ret = PyBuffer_FillInfo(view, ptr, Py_SIZE(obj), 0, flags); + ret = PyBuffer_FillInfo(view, (PyObject*)obj, ptr, Py_SIZE(obj), 0, flags); if (ret >= 0) { obj->ob_exports++; } @@ -1075,6 +1075,11 @@ static void static void bytes_dealloc(PyByteArrayObject *self) { + if (self->ob_exports > 0) { + PyErr_SetString(PyExc_SystemError, + "deallocated bytearray object has exported buffers"); + PyErr_Print(); + } if (self->ob_bytes != 0) { PyMem_Free(self->ob_bytes); } diff -r 4d65893a956f Objects/fileobject.c --- a/Objects/fileobject.c Sat Aug 02 23:58:05 2008 +0200 +++ b/Objects/fileobject.c Sun Aug 03 00:44:56 2008 +0200 @@ -1007,6 +1007,7 @@ file_readinto(PyFileObject *f, PyObject char *ptr; Py_ssize_t ntodo; Py_ssize_t ndone, nnow; + Py_buffer pbuf; if (f->f_fp == NULL) return err_closed(); @@ -1015,8 +1016,10 @@ file_readinto(PyFileObject *f, PyObject (f->f_bufend - f->f_bufptr) > 0 && f->f_buf[0] != '\0') return err_iterbuffered(); - if (!PyArg_ParseTuple(args, "w#", &ptr, &ntodo)) + if (!PyArg_ParseTuple(args, "w*", &pbuf)) return NULL; + ptr = pbuf.buf; + ntodo = pbuf.len; ndone = 0; while (ntodo > 0) { FILE_BEGIN_ALLOW_THREADS(f) @@ -1029,11 +1032,13 @@ file_readinto(PyFileObject *f, PyObject break; PyErr_SetFromErrno(PyExc_IOError); clearerr(f->f_fp); + PyBuffer_Release(&pbuf); return NULL; } ndone += nnow; ntodo -= nnow; } + PyBuffer_Release(&pbuf); return PyInt_FromSsize_t(ndone); } @@ -1611,17 +1616,26 @@ static PyObject * static PyObject * file_write(PyFileObject *f, PyObject *args) { + Py_buffer pbuf; char *s; Py_ssize_t n, n2; if (f->f_fp == NULL) return err_closed(); - if (!PyArg_ParseTuple(args, f->f_binary ? "s#" : "t#", &s, &n)) + if (f->f_binary) { + if (!PyArg_ParseTuple(args, "s*", &pbuf)) + return NULL; + s = pbuf.buf; + n = pbuf.len; + } else + if (!PyArg_ParseTuple(args, "t#", &s, &n)) return NULL; f->f_softspace = 0; FILE_BEGIN_ALLOW_THREADS(f) errno = 0; n2 = fwrite(s, 1, n, f->f_fp); FILE_END_ALLOW_THREADS(f) + if (f->f_binary) + PyBuffer_Release(&pbuf); if (n2 != n) { PyErr_SetFromErrno(PyExc_IOError); clearerr(f->f_fp); diff -r 4d65893a956f Objects/stringobject.c --- a/Objects/stringobject.c Sat Aug 02 23:58:05 2008 +0200 +++ b/Objects/stringobject.c Sun Aug 03 00:44:56 2008 +0200 @@ -1328,7 +1328,8 @@ static int static int string_buffer_getbuffer(PyStringObject *self, Py_buffer *view, int flags) { - return PyBuffer_FillInfo(view, (void *)self->ob_sval, Py_SIZE(self), + return PyBuffer_FillInfo(view, (PyObject*)self, + (void *)self->ob_sval, Py_SIZE(self), 1, flags); } @@ -1359,7 +1360,7 @@ static PyBufferProcs string_as_buffer = }; - + #define LEFTSTRIP 0 #define RIGHTSTRIP 1 #define BOTHSTRIP 2 @@ -3996,7 +3997,7 @@ PyDoc_STRVAR(p_format__doc__, \n\ "); - + static PyMethodDef string_methods[] = { /* Counterparts of the obsolete stropmodule functions; except diff -r 4d65893a956f Python/getargs.c --- a/Python/getargs.c Sat Aug 02 23:58:05 2008 +0200 +++ b/Python/getargs.c Sun Aug 03 00:44:56 2008 +0200 @@ -44,6 +44,7 @@ static char *convertsimple(PyObject *, c static char *convertsimple(PyObject *, const char **, va_list *, int, char *, size_t, PyObject **); static Py_ssize_t convertbuffer(PyObject *, void **p, char **); +static int getbuffer(PyObject *, Py_buffer *, char**); static int vgetargskeywords(PyObject *, PyObject *, const char *, char **, va_list *, int); @@ -773,7 +774,32 @@ convertsimple(PyObject *arg, const char } case 's': {/* string */ - if (*format == '#') { + if (*format == '*') { + Py_buffer *p = (Py_buffer *)va_arg(*p_va, Py_buffer *); + + if (PyString_Check(arg)) { + PyBuffer_FillInfo(p, arg, + PyString_AS_STRING(arg), PyString_GET_SIZE(arg), + 1, 0); + } +#ifdef Py_USING_UNICODE + else if (PyUnicode_Check(arg)) { + uarg = UNICODE_DEFAULT_ENCODING(arg); + if (uarg == NULL) + return converterr(CONV_UNICODE, + arg, msgbuf, bufsize); + PyBuffer_FillInfo(p, arg, + PyString_AS_STRING(uarg), PyString_GET_SIZE(uarg), + 1, 0); + } +#endif + else { /* any buffer-like object */ + char *buf; + if (getbuffer(arg, p, &buf) < 0) + return converterr(buf, arg, msgbuf, bufsize); + } + format++; + } else if (*format == '#') { void **p = (void **)va_arg(*p_va, char **); FETCH_SIZE; @@ -823,7 +849,34 @@ convertsimple(PyObject *arg, const char } case 'z': {/* string, may be NULL (None) */ - if (*format == '#') { /* any buffer-like object */ + if (*format == '*') { + Py_buffer *p = (Py_buffer *)va_arg(*p_va, Py_buffer *); + + if (arg == Py_None) + PyBuffer_FillInfo(p, NULL, NULL, 0, 1, 0); + else if (PyString_Check(arg)) { + PyBuffer_FillInfo(p, arg, + PyString_AS_STRING(arg), PyString_GET_SIZE(arg), + 1, 0); + } +#ifdef Py_USING_UNICODE + else if (PyUnicode_Check(arg)) { + uarg = UNICODE_DEFAULT_ENCODING(arg); + if (uarg == NULL) + return converterr(CONV_UNICODE, + arg, msgbuf, bufsize); + PyBuffer_FillInfo(p, arg, + PyString_AS_STRING(uarg), PyString_GET_SIZE(uarg), + 1, 0); + } +#endif + else { /* any buffer-like object */ + char *buf; + if (getbuffer(arg, p, &buf) < 0) + return converterr(buf, arg, msgbuf, bufsize); + } + format++; + } else if (*format == '#') { /* any buffer-like object */ void **p = (void **)va_arg(*p_va, char **); FETCH_SIZE; @@ -1144,22 +1197,48 @@ convertsimple(PyObject *arg, const char case 'w': { /* memory buffer, read-write access */ void **p = va_arg(*p_va, void **); + void *res; PyBufferProcs *pb = arg->ob_type->tp_as_buffer; Py_ssize_t count; - - if (pb == NULL || + + if (pb && pb->bf_releasebuffer && *format != '*') + /* Buffer must be released, yet caller does not use + the Py_buffer protocol. */ + return converterr("pinned buffer", arg, msgbuf, bufsize); + + if (pb && pb->bf_getbuffer && *format == '*') { + /* Caller is interested in Py_buffer, and the object + supports it directly. */ + format++; + if (pb->bf_getbuffer(arg, (Py_buffer*)p, PyBUF_WRITABLE) < 0) { + PyErr_Clear(); + return converterr("read-write buffer", arg, msgbuf, bufsize); + } + if (!PyBuffer_IsContiguous((Py_buffer*)p, 'C')) + return converterr("contiguous buffer", arg, msgbuf, bufsize); + break; + } + + if (pb == NULL || pb->bf_getwritebuffer == NULL || pb->bf_getsegcount == NULL) return converterr("read-write buffer", arg, msgbuf, bufsize); if ((*pb->bf_getsegcount)(arg, NULL) != 1) return converterr("single-segment read-write buffer", arg, msgbuf, bufsize); - if ((count = pb->bf_getwritebuffer(arg, 0, p)) < 0) + if ((count = pb->bf_getwritebuffer(arg, 0, &res)) < 0) return converterr("(unspecified)", arg, msgbuf, bufsize); - if (*format == '#') { - FETCH_SIZE; - STORE_SIZE(count); + if (*format == '*') { + PyBuffer_FillInfo((Py_buffer*)p, arg, res, count, 1, 0); format++; + } + else { + *p = res; + if (*format == '#') { + FETCH_SIZE; + STORE_SIZE(count); + format++; + } } break; } @@ -1186,6 +1265,11 @@ convertsimple(PyObject *arg, const char "string or single-segment read-only buffer", arg, msgbuf, bufsize); + if (pb->bf_releasebuffer) + return converterr( + "string or pinned buffer", + arg, msgbuf, bufsize); + count = pb->bf_getcharbuffer(arg, 0, p); if (count < 0) return converterr("(unspecified)", arg, msgbuf, bufsize); @@ -1212,7 +1296,8 @@ convertbuffer(PyObject *arg, void **p, c Py_ssize_t count; if (pb == NULL || pb->bf_getreadbuffer == NULL || - pb->bf_getsegcount == NULL) { + pb->bf_getsegcount == NULL || + pb->bf_releasebuffer != NULL) { *errmsg = "string or read-only buffer"; return -1; } @@ -1226,6 +1311,33 @@ convertbuffer(PyObject *arg, void **p, c return count; } +static int +getbuffer(PyObject *arg, Py_buffer *view, char**errmsg) +{ + void *buf; + Py_ssize_t count; + PyBufferProcs *pb = arg->ob_type->tp_as_buffer; + if (pb == NULL) { + *errmsg = "string or buffer"; + return -1; + } + if (pb->bf_getbuffer) { + if (pb->bf_getbuffer(arg, view, 0) < 0) + return -1; + if (!PyBuffer_IsContiguous(view, 'C')) { + *errmsg = "contiguous buffer"; + return -1; + } + return 0; + } + + count = convertbuffer(arg, &buf, errmsg); + if (count < 0) + return count; + PyBuffer_FillInfo(view, NULL, buf, count, 1, 0); + return 0; +} + /* Support for keyword arguments donated by Geoff Philbrick */ @@ -1566,6 +1678,8 @@ skipitem(const char **p_format, va_list else (void) va_arg(*p_va, int *); format++; + } else if ((c == 's' || c == 'z') && *format == '*') { + format++; } break; }