diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 3deafdfbe01..5ceab58cd2b 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -40,6 +40,14 @@ variable(s) whose address should be passed. other read-buffer compatible objects pass back a reference to the raw internal data representation. +``s*`` (string, Unicode, or any buffer compatible object) [Py_buffer \*] + Similar to ``s#``, this code fills a Py_buffer structure provided by the caller. + The buffer gets locked, so that the caller can subsequently use the buffer even + inside a ``Py_BEGIN_ALLOW_THREADS`` block; the caller is responsible for calling + ``PyBuffer_Release`` with the structure after it has processed the data. + + .. versionadded:: 2.6 + ``z`` (string or ``None``) [const char \*] Like ``s``, but the Python object may also be ``None``, in which case the C pointer is set to *NULL*. @@ -47,6 +55,10 @@ variable(s) whose address should be passed. ``z#`` (string or ``None`` or any read buffer compatible object) [const char \*, int] This is to ``s#`` as ``z`` is to ``s``. +``z*`` (string or ``None`` or any buffer compatible object) [Py_buffer*] + This is to ``s*`` as ``z`` is to ``s``. + .. versionadded:: 2.6 + ``u`` (Unicode object) [Py_UNICODE \*] Convert a Python Unicode object to a C pointer to a NUL-terminated buffer of 16-bit Unicode (UTF-16) data. As with ``s``, there is no need to provide @@ -240,6 +252,10 @@ variable(s) whose address should be passed. single-segment buffer objects are accepted; :exc:`TypeError` is raised for all others. +``w*`` (read-write byte-oriented buffer) [Py_buffer \*] + This is to ``w`` what ``s*`` is to ``s``. + .. versionadded:: 2.6 + ``(items)`` (tuple) [*matching-items*] The object must be a Python sequence whose length is the number of format units in *items*. The C arguments must correspond to the individual format units in diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index c0ecf012929..0f8b88325c8 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -1064,7 +1064,7 @@ about the object's memory representation. Objects can use this operation to lock memory in place while an external caller could be modifying the contents, so there's a corresponding -``PyObject_ReleaseBuffer(PyObject *obj, Py_buffer *view)`` to +``PyBuffer_Release(Py_buffer *view)`` to indicate that the external caller is done. The **flags** argument to :cfunc:`PyObject_GetBuffer` specifies @@ -2841,7 +2841,7 @@ Changes to Python's build process and to the C API include: * The new buffer interface, previously described in `the PEP 3118 section <#pep-3118-revised-buffer-protocol>`__, - adds :cfunc:`PyObject_GetBuffer` and :cfunc:`PyObject_ReleaseBuffer`, + adds :cfunc:`PyObject_GetBuffer` and :cfunc:`PyBuffer_Release`, as well as a few other functions. * Python's use of the C stdio library is now thread-safe, or at least diff --git a/Include/abstract.h b/Include/abstract.h index 3e5cf129f8d..04c007a47dc 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -549,24 +549,6 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/ */ - PyAPI_FUNC(void) PyObject_ReleaseBuffer(PyObject *obj, Py_buffer *view); - - - /* C-API version of the releasebuffer function call. It - checks to make sure the object has the required function - pointer and issues the call. The obj must have the buffer - interface or this function will cause a segfault (i.e. it - is assumed to be called only after a corresponding - getbuffer which already verified the existence of the - tp_as_buffer pointer). - - Returns 0 on success and -1 (with an error raised) on - failure. This function always succeeds (as a NO-OP) if - there is no releasebuffer function for the object so that - it can always be called when the consumer is done with the - buffer - */ - PyAPI_FUNC(void *) PyBuffer_GetPointer(Py_buffer *view, Py_ssize_t *indices); /* Get the memory area pointed to by the indices for the buffer given. @@ -623,7 +605,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/ per element. */ - PyAPI_FUNC(int) PyBuffer_FillInfo(Py_buffer *view, void *buf, + PyAPI_FUNC(int) PyBuffer_FillInfo(Py_buffer *view, PyObject *o, void *buf, Py_ssize_t len, int readonly, int flags); @@ -633,6 +615,11 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/ and -1 (with raising an error) on error. */ + PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view); + + /* Releases a Py_buffer obtained from getbuffer ParseTuple's s*. + */ + PyAPI_FUNC(PyObject *) PyObject_Format(PyObject* obj, PyObject *format_spec); /* diff --git a/Include/object.h b/Include/object.h index c00df74e6e1..6c040340394 100644 --- a/Include/object.h +++ b/Include/object.h @@ -162,7 +162,8 @@ typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **); /* Py3k buffer interface */ typedef struct bufferinfo { - void *buf; + void *buf; + PyObject *obj; /* borrowed reference */ Py_ssize_t len; Py_ssize_t itemsize; /* This is Py_ssize_t so it can be pointed to by strides in simple case.*/ diff --git a/Misc/NEWS b/Misc/NEWS index 398740224f0..0a66c78f3d6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,10 @@ What's New in Python 2.6 beta 3? Core and Builtins ----------------- +- Issue #3139: Make buffer-interface thread-safe wrt. PyArg_ParseTuple, + by denying s# to parse objects that have a releasebuffer procedure, + and introducing s*. + - Issue #3537: Fix an assertion failure when an empty but presized dict object was stored in the freelist. diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index d4eb0d5b9c0..9250e3edd32 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -232,20 +232,20 @@ static PyObject * utf_7_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int final = 0; Py_ssize_t consumed; PyObject *decoded = NULL; - if (!PyArg_ParseTuple(args, "t#|zi:utf_7_decode", - &data, &size, &errors, &final)) - return NULL; - consumed = size; + if (!PyArg_ParseTuple(args, "s*|zi:utf_7_decode", + &pbuf, &errors, &final)) + return NULL; + consumed = pbuf.len; - decoded = PyUnicode_DecodeUTF7Stateful(data, size, errors, - final ? NULL : &consumed); + decoded = PyUnicode_DecodeUTF7Stateful(pbuf.buf, pbuf.len, errors, + final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (decoded == NULL) return NULL; return codec_tuple(decoded, consumed); @@ -255,24 +255,20 @@ static PyObject * utf_8_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int final = 0; Py_ssize_t consumed; PyObject *decoded = NULL; - if (!PyArg_ParseTuple(args, "t#|zi:utf_8_decode", - &data, &size, &errors, &final)) + if (!PyArg_ParseTuple(args, "s*|zi:utf_8_decode", + &pbuf, &errors, &final)) return NULL; - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative argument"); - return 0; - } - consumed = size; - - decoded = PyUnicode_DecodeUTF8Stateful(data, size, errors, + consumed = pbuf.len; + + decoded = PyUnicode_DecodeUTF8Stateful(pbuf.buf, pbuf.len, errors, final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (decoded == NULL) return NULL; return codec_tuple(decoded, consumed); @@ -282,24 +278,20 @@ static PyObject * utf_16_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int byteorder = 0; int final = 0; Py_ssize_t consumed; PyObject *decoded; - if (!PyArg_ParseTuple(args, "t#|zi:utf_16_decode", - &data, &size, &errors, &final)) + if (!PyArg_ParseTuple(args, "s*|zi:utf_16_decode", + &pbuf, &errors, &final)) return NULL; - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative argument"); - return 0; - } - consumed = size; /* This is overwritten unless final is true. */ - decoded = PyUnicode_DecodeUTF16Stateful(data, size, errors, &byteorder, - final ? NULL : &consumed); + consumed = pbuf.len; /* This is overwritten unless final is true. */ + decoded = PyUnicode_DecodeUTF16Stateful(pbuf.buf, pbuf.len, errors, + &byteorder, final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (decoded == NULL) return NULL; return codec_tuple(decoded, consumed); @@ -309,53 +301,45 @@ static PyObject * utf_16_le_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int byteorder = -1; int final = 0; Py_ssize_t consumed; PyObject *decoded = NULL; - if (!PyArg_ParseTuple(args, "t#|zi:utf_16_le_decode", - &data, &size, &errors, &final)) + if (!PyArg_ParseTuple(args, "s*|zi:utf_16_le_decode", + &pbuf, &errors, &final)) return NULL; - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative argument"); - return 0; - } - consumed = size; /* This is overwritten unless final is true. */ - decoded = PyUnicode_DecodeUTF16Stateful(data, size, errors, + consumed = pbuf.len; /* This is overwritten unless final is true. */ + decoded = PyUnicode_DecodeUTF16Stateful(pbuf.buf, pbuf.len, errors, &byteorder, final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (decoded == NULL) return NULL; return codec_tuple(decoded, consumed); - } static PyObject * utf_16_be_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int byteorder = 1; int final = 0; Py_ssize_t consumed; PyObject *decoded = NULL; - if (!PyArg_ParseTuple(args, "t#|zi:utf_16_be_decode", - &data, &size, &errors, &final)) + if (!PyArg_ParseTuple(args, "s*|zi:utf_16_be_decode", + &pbuf, &errors, &final)) return NULL; - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative argument"); - return 0; - } - consumed = size; /* This is overwritten unless final is true. */ - decoded = PyUnicode_DecodeUTF16Stateful(data, size, errors, + + consumed = pbuf.len; /* This is overwritten unless final is true. */ + decoded = PyUnicode_DecodeUTF16Stateful(pbuf.buf, pbuf.len, errors, &byteorder, final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (decoded == NULL) return NULL; return codec_tuple(decoded, consumed); @@ -373,24 +357,20 @@ static PyObject * utf_16_ex_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int byteorder = 0; PyObject *unicode, *tuple; int final = 0; Py_ssize_t consumed; - if (!PyArg_ParseTuple(args, "t#|zii:utf_16_ex_decode", - &data, &size, &errors, &byteorder, &final)) + if (!PyArg_ParseTuple(args, "s*|zii:utf_16_ex_decode", + &pbuf, &errors, &byteorder, &final)) return NULL; - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative argument"); - return 0; - } - consumed = size; /* This is overwritten unless final is true. */ - unicode = PyUnicode_DecodeUTF16Stateful(data, size, errors, &byteorder, - final ? NULL : &consumed); + consumed = pbuf.len; /* This is overwritten unless final is true. */ + unicode = PyUnicode_DecodeUTF16Stateful(pbuf.buf, pbuf.len, errors, + &byteorder, final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (unicode == NULL) return NULL; tuple = Py_BuildValue("Oni", unicode, consumed, byteorder); @@ -402,24 +382,20 @@ static PyObject * utf_32_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int byteorder = 0; int final = 0; Py_ssize_t consumed; PyObject *decoded; - if (!PyArg_ParseTuple(args, "t#|zi:utf_32_decode", - &data, &size, &errors, &final)) + if (!PyArg_ParseTuple(args, "s*|zi:utf_32_decode", + &pbuf, &errors, &final)) return NULL; - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative argument"); - return 0; - } - consumed = size; /* This is overwritten unless final is true. */ - decoded = PyUnicode_DecodeUTF32Stateful(data, size, errors, &byteorder, - final ? NULL : &consumed); + consumed = pbuf.len; /* This is overwritten unless final is true. */ + decoded = PyUnicode_DecodeUTF32Stateful(pbuf.buf, pbuf.len, errors, + &byteorder, final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (decoded == NULL) return NULL; return codec_tuple(decoded, consumed); @@ -429,53 +405,43 @@ static PyObject * utf_32_le_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int byteorder = -1; int final = 0; Py_ssize_t consumed; - PyObject *decoded = NULL; + PyObject *decoded; - if (!PyArg_ParseTuple(args, "t#|zi:utf_32_le_decode", - &data, &size, &errors, &final)) + if (!PyArg_ParseTuple(args, "s*|zi:utf_32_le_decode", + &pbuf, &errors, &final)) return NULL; - - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative argument"); - return 0; - } - consumed = size; /* This is overwritten unless final is true. */ - decoded = PyUnicode_DecodeUTF32Stateful(data, size, errors, - &byteorder, final ? NULL : &consumed); + consumed = pbuf.len; /* This is overwritten unless final is true. */ + decoded = PyUnicode_DecodeUTF32Stateful(pbuf.buf, pbuf.len, errors, + &byteorder, final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (decoded == NULL) return NULL; return codec_tuple(decoded, consumed); - } static PyObject * utf_32_be_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int byteorder = 1; int final = 0; Py_ssize_t consumed; - PyObject *decoded = NULL; + PyObject *decoded; - if (!PyArg_ParseTuple(args, "t#|zi:utf_32_be_decode", - &data, &size, &errors, &final)) + if (!PyArg_ParseTuple(args, "s*|zi:utf_32_be_decode", + &pbuf, &errors, &final)) return NULL; - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative argument"); - return 0; - } - consumed = size; /* This is overwritten unless final is true. */ - decoded = PyUnicode_DecodeUTF32Stateful(data, size, errors, - &byteorder, final ? NULL : &consumed); + consumed = pbuf.len; /* This is overwritten unless final is true. */ + decoded = PyUnicode_DecodeUTF32Stateful(pbuf.buf, pbuf.len, errors, + &byteorder, final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (decoded == NULL) return NULL; return codec_tuple(decoded, consumed); @@ -493,24 +459,20 @@ static PyObject * utf_32_ex_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; int byteorder = 0; PyObject *unicode, *tuple; int final = 0; Py_ssize_t consumed; - if (!PyArg_ParseTuple(args, "t#|zii:utf_32_ex_decode", - &data, &size, &errors, &byteorder, &final)) + if (!PyArg_ParseTuple(args, "s*|zii:utf_32_ex_decode", + &pbuf, &errors, &byteorder, &final)) return NULL; - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative argument"); - return 0; - } - consumed = size; /* This is overwritten unless final is true. */ - unicode = PyUnicode_DecodeUTF32Stateful(data, size, errors, &byteorder, - final ? NULL : &consumed); + consumed = pbuf.len; /* This is overwritten unless final is true. */ + unicode = PyUnicode_DecodeUTF32Stateful(pbuf.buf, pbuf.len, errors, + &byteorder, final ? NULL : &consumed); + PyBuffer_Release(&pbuf); if (unicode == NULL) return NULL; tuple = Py_BuildValue("Oni", unicode, consumed, byteorder); @@ -522,83 +484,88 @@ static PyObject * unicode_escape_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; + PyObject *unicode; - if (!PyArg_ParseTuple(args, "t#|z:unicode_escape_decode", - &data, &size, &errors)) + if (!PyArg_ParseTuple(args, "s*|z:unicode_escape_decode", + &pbuf, &errors)) return NULL; - return codec_tuple(PyUnicode_DecodeUnicodeEscape(data, size, errors), - size); + unicode = PyUnicode_DecodeUnicodeEscape(pbuf.buf, pbuf.len, errors); + PyBuffer_Release(&pbuf); + return codec_tuple(unicode, pbuf.len); } static PyObject * raw_unicode_escape_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; const char *errors = NULL; + PyObject *unicode; - if (!PyArg_ParseTuple(args, "t#|z:raw_unicode_escape_decode", - &data, &size, &errors)) + if (!PyArg_ParseTuple(args, "s*|z:raw_unicode_escape_decode", + &pbuf, &errors)) return NULL; - return codec_tuple(PyUnicode_DecodeRawUnicodeEscape(data, size, errors), - size); + unicode = PyUnicode_DecodeRawUnicodeEscape(pbuf.buf, pbuf.len, errors); + PyBuffer_Release(&pbuf); + return codec_tuple(unicode, pbuf.len); } static PyObject * latin_1_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; + PyObject *unicode; const char *errors = NULL; - if (!PyArg_ParseTuple(args, "t#|z:latin_1_decode", - &data, &size, &errors)) + if (!PyArg_ParseTuple(args, "s*|z:latin_1_decode", + &pbuf, &errors)) return NULL; - return codec_tuple(PyUnicode_DecodeLatin1(data, size, errors), - size); + unicode = PyUnicode_DecodeLatin1(pbuf.buf, pbuf.len, errors); + PyBuffer_Release(&pbuf); + return codec_tuple(unicode, pbuf.len); } static PyObject * ascii_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; + PyObject *unicode; const char *errors = NULL; - if (!PyArg_ParseTuple(args, "t#|z:ascii_decode", - &data, &size, &errors)) + if (!PyArg_ParseTuple(args, "s*|z:ascii_decode", + &pbuf, &errors)) return NULL; - return codec_tuple(PyUnicode_DecodeASCII(data, size, errors), - size); + unicode = PyUnicode_DecodeASCII(pbuf.buf, pbuf.len, errors); + PyBuffer_Release(&pbuf); + return codec_tuple(unicode, pbuf.len); } static PyObject * charmap_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size; + Py_buffer pbuf; + PyObject *unicode; const char *errors = NULL; PyObject *mapping = NULL; - if (!PyArg_ParseTuple(args, "t#|zO:charmap_decode", - &data, &size, &errors, &mapping)) + if (!PyArg_ParseTuple(args, "s*|zO:charmap_decode", + &pbuf, &errors, &mapping)) return NULL; if (mapping == Py_None) mapping = NULL; - return codec_tuple(PyUnicode_DecodeCharmap(data, size, mapping, errors), - size); + unicode = PyUnicode_DecodeCharmap(pbuf.buf, pbuf.len, mapping, errors); + PyBuffer_Release(&pbuf); + return codec_tuple(unicode, pbuf.len); } #if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) @@ -607,21 +574,23 @@ static PyObject * mbcs_decode(PyObject *self, PyObject *args) { - const char *data; - Py_ssize_t size, consumed; + Py_buffer pbuf; const char *errors = NULL; int final = 0; - PyObject *decoded; + Py_ssize_t consumed; + PyObject *decoded = NULL; - if (!PyArg_ParseTuple(args, "t#|zi:mbcs_decode", - &data, &size, &errors, &final)) + if (!PyArg_ParseTuple(args, "s*|zi:mbcs_decode", + &pbuf, &errors, &final)) return NULL; + consumed = pbuf.len; - decoded = PyUnicode_DecodeMBCSStateful( - data, size, errors, final ? NULL : &consumed); - if (!decoded) + decoded = PyUnicode_DecodeMBCSStateful(pbuf.buf, pbuf.len, errors, + final ? NULL : &consumed); + PyBuffer_Release(&pbuf); + if (decoded == NULL) return NULL; - return codec_tuple(decoded, final ? size : consumed); + return codec_tuple(decoded, consumed); } #endif /* MS_WINDOWS */ diff --git a/Modules/_fileio.c b/Modules/_fileio.c index 21608f1ef1a..58462dd74b5 100644 --- a/Modules/_fileio.c +++ b/Modules/_fileio.c @@ -357,7 +357,7 @@ fileio_seekable(PyFileIOObject *self) static PyObject * fileio_readinto(PyFileIOObject *self, PyObject *args) { - char *ptr; + Py_buffer pbuf; Py_ssize_t n; if (self->fd < 0) @@ -365,13 +365,14 @@ fileio_readinto(PyFileIOObject *self, PyObject *args) if (!self->readable) return err_mode("reading"); - if (!PyArg_ParseTuple(args, "w#", &ptr, &n)) + if (!PyArg_ParseTuple(args, "w*", &pbuf)) return NULL; Py_BEGIN_ALLOW_THREADS errno = 0; - n = read(self->fd, ptr, n); + n = read(self->fd, pbuf.buf, pbuf.len); Py_END_ALLOW_THREADS + PyBuffer_Release(&pbuf); if (n < 0) { if (errno == EAGAIN) Py_RETURN_NONE; @@ -489,22 +490,24 @@ fileio_read(PyFileIOObject *self, PyObject *args) static PyObject * fileio_write(PyFileIOObject *self, PyObject *args) { + Py_buffer pbuf; Py_ssize_t n; - char *ptr; if (self->fd < 0) return err_closed(); if (!self->writable) return err_mode("writing"); - if (!PyArg_ParseTuple(args, "s#", &ptr, &n)) + if (!PyArg_ParseTuple(args, "s*", &pbuf)) return NULL; Py_BEGIN_ALLOW_THREADS errno = 0; - n = write(self->fd, ptr, n); + n = write(self->fd, pbuf.buf, pbuf.len); Py_END_ALLOW_THREADS + PyBuffer_Release(&pbuf); + if (n < 0) { if (errno == EAGAIN) Py_RETURN_NONE; diff --git a/Modules/_multiprocessing/connection.h b/Modules/_multiprocessing/connection.h index 4b475c6bf72..66a3e8ad86a 100644 --- a/Modules/_multiprocessing/connection.h +++ b/Modules/_multiprocessing/connection.h @@ -187,21 +187,25 @@ connection_recvbytes_into(ConnectionObject *self, PyObject *args) char *freeme = NULL, *buffer = NULL; Py_ssize_t res, length, offset = 0; PyObject *result = NULL; - - if (!PyArg_ParseTuple(args, "w#|" F_PY_SSIZE_T, - &buffer, &length, &offset)) - return NULL; + Py_buffer pbuf; CHECK_READABLE(self); + + if (!PyArg_ParseTuple(args, "w*|" F_PY_SSIZE_T, + &pbuf, &offset)) + return NULL; + + buffer = pbuf.buf; + length = pbuf.len; if (offset < 0) { PyErr_SetString(PyExc_ValueError, "negative offset"); - return NULL; + goto _error; } if (offset > length) { PyErr_SetString(PyExc_ValueError, "offset too large"); - return NULL; + goto _error; } res = conn_recv_string(self, buffer+offset, length-offset, @@ -231,11 +235,17 @@ connection_recvbytes_into(ConnectionObject *self, PyObject *args) PyErr_SetObject(BufferTooShort, result); Py_DECREF(result); } - return NULL; + goto _error; } } +_cleanup: + PyBuffer_Release(&pbuf); return result; + +_error: + result = NULL; + goto _cleanup; } /* diff --git a/Modules/bz2module.c b/Modules/bz2module.c index 39292197d8e..b5542bf968d 100644 --- a/Modules/bz2module.c +++ b/Modules/bz2module.c @@ -792,12 +792,15 @@ static PyObject * BZ2File_write(BZ2FileObject *self, PyObject *args) { PyObject *ret = NULL; + Py_buffer pbuf; char *buf; int len; int bzerror; - if (!PyArg_ParseTuple(args, "s#:write", &buf, &len)) + if (!PyArg_ParseTuple(args, "s*:write", &pbuf)) return NULL; + buf = pbuf.buf; + len = pbuf.len; ACQUIRE_LOCK(self); switch (self->mode) { @@ -831,6 +834,7 @@ BZ2File_write(BZ2FileObject *self, PyObject *args) ret = Py_None; cleanup: + PyBuffer_Release(&pbuf); RELEASE_LOCK(self); return ret; } @@ -1549,6 +1553,7 @@ and return what is left in the internal buffers.\n\ static PyObject * BZ2Comp_compress(BZ2CompObject *self, PyObject *args) { + Py_buffer pdata; char *data; int datasize; int bufsize = SMALLCHUNK; @@ -1557,11 +1562,15 @@ BZ2Comp_compress(BZ2CompObject *self, PyObject *args) bz_stream *bzs = &self->bzs; int bzerror; - if (!PyArg_ParseTuple(args, "s#:compress", &data, &datasize)) + if (!PyArg_ParseTuple(args, "s*:compress", &pdata)) return NULL; + data = pdata.buf; + datasize = pdata.len; - if (datasize == 0) + if (datasize == 0) { + PyBuffer_Release(&pdata); return PyString_FromString(""); + } ACQUIRE_LOCK(self); if (!self->running) { @@ -1606,10 +1615,12 @@ BZ2Comp_compress(BZ2CompObject *self, PyObject *args) _PyString_Resize(&ret, (Py_ssize_t)(BZS_TOTAL_OUT(bzs) - totalout)); RELEASE_LOCK(self); + PyBuffer_Release(&pdata); return ret; error: RELEASE_LOCK(self); + PyBuffer_Release(&pdata); Py_XDECREF(ret); return NULL; } @@ -1833,6 +1844,7 @@ unused_data attribute.\n\ static PyObject * BZ2Decomp_decompress(BZ2DecompObject *self, PyObject *args) { + Py_buffer pdata; char *data; int datasize; int bufsize = SMALLCHUNK; @@ -1841,8 +1853,10 @@ BZ2Decomp_decompress(BZ2DecompObject *self, PyObject *args) bz_stream *bzs = &self->bzs; int bzerror; - if (!PyArg_ParseTuple(args, "s#:decompress", &data, &datasize)) + if (!PyArg_ParseTuple(args, "s*:decompress", &pdata)) return NULL; + data = pdata.buf; + datasize = pdata.len; ACQUIRE_LOCK(self); if (!self->running) { @@ -1899,10 +1913,12 @@ BZ2Decomp_decompress(BZ2DecompObject *self, PyObject *args) _PyString_Resize(&ret, (Py_ssize_t)(BZS_TOTAL_OUT(bzs) - totalout)); RELEASE_LOCK(self); + PyBuffer_Release(&pdata); return ret; error: RELEASE_LOCK(self); + PyBuffer_Release(&pdata); Py_XDECREF(ret); return NULL; } @@ -2041,6 +2057,7 @@ static PyObject * bz2_compress(PyObject *self, PyObject *args, PyObject *kwargs) { int compresslevel=9; + Py_buffer pdata; char *data; int datasize; int bufsize; @@ -2050,14 +2067,17 @@ bz2_compress(PyObject *self, PyObject *args, PyObject *kwargs) int bzerror; static char *kwlist[] = {"data", "compresslevel", 0}; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|i", - kwlist, &data, &datasize, + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|i", + kwlist, &pdata, &compresslevel)) return NULL; + data = pdata.buf; + datasize = pdata.len; if (compresslevel < 1 || compresslevel > 9) { PyErr_SetString(PyExc_ValueError, "compresslevel must be between 1 and 9"); + PyBuffer_Release(&pdata); return NULL; } @@ -2066,8 +2086,10 @@ bz2_compress(PyObject *self, PyObject *args, PyObject *kwargs) bufsize = datasize + (datasize/100+1) + 600; ret = PyString_FromStringAndSize(NULL, bufsize); - if (!ret) + if (!ret) { + PyBuffer_Release(&pdata); return NULL; + } memset(bzs, 0, sizeof(bz_stream)); @@ -2079,6 +2101,7 @@ bz2_compress(PyObject *self, PyObject *args, PyObject *kwargs) bzerror = BZ2_bzCompressInit(bzs, compresslevel, 0, 0); if (bzerror != BZ_OK) { Util_CatchBZ2Error(bzerror); + PyBuffer_Release(&pdata); Py_DECREF(ret); return NULL; } @@ -2092,6 +2115,7 @@ bz2_compress(PyObject *self, PyObject *args, PyObject *kwargs) } else if (bzerror != BZ_FINISH_OK) { BZ2_bzCompressEnd(bzs); Util_CatchBZ2Error(bzerror); + PyBuffer_Release(&pdata); Py_DECREF(ret); return NULL; } @@ -2099,6 +2123,7 @@ bz2_compress(PyObject *self, PyObject *args, PyObject *kwargs) bufsize = Util_NewBufferSize(bufsize); if (_PyString_Resize(&ret, bufsize) < 0) { BZ2_bzCompressEnd(bzs); + PyBuffer_Release(&pdata); Py_DECREF(ret); return NULL; } @@ -2111,6 +2136,7 @@ bz2_compress(PyObject *self, PyObject *args, PyObject *kwargs) _PyString_Resize(&ret, (Py_ssize_t)BZS_TOTAL_OUT(bzs)); BZ2_bzCompressEnd(bzs); + PyBuffer_Release(&pdata); return ret; } @@ -2124,6 +2150,7 @@ use an instance of BZ2Decompressor instead.\n\ static PyObject * bz2_decompress(PyObject *self, PyObject *args) { + Py_buffer pdata; char *data; int datasize; int bufsize = SMALLCHUNK; @@ -2132,15 +2159,21 @@ bz2_decompress(PyObject *self, PyObject *args) bz_stream *bzs = &_bzs; int bzerror; - if (!PyArg_ParseTuple(args, "s#:decompress", &data, &datasize)) + if (!PyArg_ParseTuple(args, "s*:decompress", &pdata)) return NULL; + data = pdata.buf; + datasize = pdata.len; - if (datasize == 0) + if (datasize == 0) { + PyBuffer_Release(&pdata); return PyString_FromString(""); + } ret = PyString_FromStringAndSize(NULL, bufsize); - if (!ret) + if (!ret) { + PyBuffer_Release(&pdata); return NULL; + } memset(bzs, 0, sizeof(bz_stream)); @@ -2153,6 +2186,7 @@ bz2_decompress(PyObject *self, PyObject *args) if (bzerror != BZ_OK) { Util_CatchBZ2Error(bzerror); Py_DECREF(ret); + PyBuffer_Release(&pdata); return NULL; } @@ -2165,6 +2199,7 @@ bz2_decompress(PyObject *self, PyObject *args) } else if (bzerror != BZ_OK) { BZ2_bzDecompressEnd(bzs); Util_CatchBZ2Error(bzerror); + PyBuffer_Release(&pdata); Py_DECREF(ret); return NULL; } @@ -2172,6 +2207,7 @@ bz2_decompress(PyObject *self, PyObject *args) BZ2_bzDecompressEnd(bzs); PyErr_SetString(PyExc_ValueError, "couldn't find end of stream"); + PyBuffer_Release(&pdata); Py_DECREF(ret); return NULL; } @@ -2179,6 +2215,7 @@ bz2_decompress(PyObject *self, PyObject *args) bufsize = Util_NewBufferSize(bufsize); if (_PyString_Resize(&ret, bufsize) < 0) { BZ2_bzDecompressEnd(bzs); + PyBuffer_Release(&pdata); Py_DECREF(ret); return NULL; } @@ -2190,6 +2227,7 @@ bz2_decompress(PyObject *self, PyObject *args) if (bzs->avail_out != 0) _PyString_Resize(&ret, (Py_ssize_t)BZS_TOTAL_OUT(bzs)); BZ2_bzDecompressEnd(bzs); + PyBuffer_Release(&pdata); return ret; } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f18e1546eb0..8104ccd6455 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -6334,15 +6334,16 @@ Write a string to a file descriptor."); static PyObject * posix_write(PyObject *self, PyObject *args) { + Py_buffer pbuf; int fd; Py_ssize_t size; - char *buffer; - if (!PyArg_ParseTuple(args, "is#:write", &fd, &buffer, &size)) + if (!PyArg_ParseTuple(args, "is*:write", &fd, &pbuf)) return NULL; Py_BEGIN_ALLOW_THREADS - size = write(fd, buffer, (size_t)size); + size = write(fd, pbuf.buf, (size_t)pbuf.len); Py_END_ALLOW_THREADS + PyBuffer_Release(&pbuf); if (size < 0) return posix_error(); return PyInt_FromSsize_t(size); diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index e4c8ac8a3b1..deff28a1be9 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -2647,12 +2647,17 @@ sock_send(PySocketSockObject *s, PyObject *args) { char *buf; int len, n = -1, flags = 0, timeout; + Py_buffer pbuf; - if (!PyArg_ParseTuple(args, "s#|i:send", &buf, &len, &flags)) + if (!PyArg_ParseTuple(args, "s*|i:send", &pbuf, &flags)) return NULL; - if (!IS_SELECTABLE(s)) + if (!IS_SELECTABLE(s)) { + PyBuffer_Release(&pbuf); return select_error(); + } + buf = pbuf.buf; + len = pbuf.len; Py_BEGIN_ALLOW_THREADS timeout = internal_select(s, 1); @@ -2664,6 +2669,8 @@ sock_send(PySocketSockObject *s, PyObject *args) #endif Py_END_ALLOW_THREADS + PyBuffer_Release(&pbuf); + if (timeout == 1) { PyErr_SetString(socket_timeout, "timed out"); return NULL; @@ -2688,12 +2695,17 @@ sock_sendall(PySocketSockObject *s, PyObject *args) { char *buf; int len, n = -1, flags = 0, timeout; + Py_buffer pbuf; - if (!PyArg_ParseTuple(args, "s#|i:sendall", &buf, &len, &flags)) + if (!PyArg_ParseTuple(args, "s*|i:sendall", &pbuf, &flags)) return NULL; + buf = pbuf.buf; + len = pbuf.len; - if (!IS_SELECTABLE(s)) + if (!IS_SELECTABLE(s)) { + PyBuffer_Release(&pbuf); return select_error(); + } Py_BEGIN_ALLOW_THREADS do { @@ -2712,6 +2724,7 @@ sock_sendall(PySocketSockObject *s, PyObject *args) len -= n; } while (len > 0); Py_END_ALLOW_THREADS + PyBuffer_Release(&pbuf); if (timeout == 1) { PyErr_SetString(socket_timeout, "timed out"); @@ -2738,24 +2751,32 @@ to tell how much data has been sent."); static PyObject * sock_sendto(PySocketSockObject *s, PyObject *args) { + Py_buffer pbuf; PyObject *addro; char *buf; + Py_ssize_t len; sock_addr_t addrbuf; - int addrlen, len, n = -1, flags, timeout; + int addrlen, n = -1, flags, timeout; flags = 0; - if (!PyArg_ParseTuple(args, "s#O:sendto", &buf, &len, &addro)) { + if (!PyArg_ParseTuple(args, "s*O:sendto", &pbuf, &addro)) { PyErr_Clear(); - if (!PyArg_ParseTuple(args, "s#iO:sendto", - &buf, &len, &flags, &addro)) + if (!PyArg_ParseTuple(args, "s*iO:sendto", + &pbuf, &flags, &addro)) return NULL; } + buf = pbuf.buf; + len = pbuf.len; - if (!IS_SELECTABLE(s)) + if (!IS_SELECTABLE(s)) { + PyBuffer_Release(&pbuf); return select_error(); + } - if (!getsockaddrarg(s, addro, SAS2SA(&addrbuf), &addrlen)) + if (!getsockaddrarg(s, addro, SAS2SA(&addrbuf), &addrlen)) { + PyBuffer_Release(&pbuf); return NULL; + } Py_BEGIN_ALLOW_THREADS timeout = internal_select(s, 1); @@ -2763,6 +2784,7 @@ sock_sendto(PySocketSockObject *s, PyObject *args) n = sendto(s->sock_fd, buf, len, flags, SAS2SA(&addrbuf), addrlen); Py_END_ALLOW_THREADS + PyBuffer_Release(&pbuf); if (timeout == 1) { PyErr_SetString(socket_timeout, "timed out"); return NULL; diff --git a/Objects/abstract.c b/Objects/abstract.c index 5cabe589809..5fb89f36728 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -362,16 +362,6 @@ PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) return (*(obj->ob_type->tp_as_buffer->bf_getbuffer))(obj, view, flags); } -void -PyObject_ReleaseBuffer(PyObject *obj, Py_buffer *view) -{ - if (obj->ob_type->tp_as_buffer != NULL && - obj->ob_type->tp_as_buffer->bf_releasebuffer != NULL) { - (*(obj->ob_type->tp_as_buffer->bf_releasebuffer))(obj, view); - } -} - - static int _IsFortranContiguous(Py_buffer *view) { @@ -603,15 +593,15 @@ int PyObject_CopyData(PyObject *dest, PyObject *src) if (PyObject_GetBuffer(dest, &view_dest, PyBUF_FULL) != 0) return -1; if (PyObject_GetBuffer(src, &view_src, PyBUF_FULL_RO) != 0) { - PyObject_ReleaseBuffer(dest, &view_dest); + PyBuffer_Release(&view_dest); return -1; } if (view_dest.len < view_src.len) { PyErr_SetString(PyExc_BufferError, "destination is too small to receive data from source"); - PyObject_ReleaseBuffer(dest, &view_dest); - PyObject_ReleaseBuffer(src, &view_src); + PyBuffer_Release(&view_dest); + PyBuffer_Release(&view_src); return -1; } @@ -621,8 +611,8 @@ int PyObject_CopyData(PyObject *dest, PyObject *src) PyBuffer_IsContiguous(&view_src, 'F'))) { /* simplest copy is all that is needed */ memcpy(view_dest.buf, view_src.buf, view_src.len); - PyObject_ReleaseBuffer(dest, &view_dest); - PyObject_ReleaseBuffer(src, &view_src); + PyBuffer_Release(&view_dest); + PyBuffer_Release(&view_src); return 0; } @@ -632,8 +622,8 @@ int PyObject_CopyData(PyObject *dest, PyObject *src) indices = (Py_ssize_t *)PyMem_Malloc(sizeof(Py_ssize_t)*view_src.ndim); if (indices == NULL) { PyErr_NoMemory(); - PyObject_ReleaseBuffer(dest, &view_dest); - PyObject_ReleaseBuffer(src, &view_src); + PyBuffer_Release(&view_dest); + PyBuffer_Release(&view_src); return -1; } for (k=0; kobj = obj; view->buf = buf; view->len = len; view->readonly = readonly; @@ -711,6 +702,17 @@ PyBuffer_FillInfo(Py_buffer *view, void *buf, Py_ssize_t len, return 0; } +void +PyBuffer_Release(Py_buffer *view) +{ + PyObject *obj = view->obj; + if (!obj || !Py_TYPE(obj)->tp_as_buffer || !Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer) + /* Unmanaged buffer */ + return; + Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer(obj, view); + +} + PyObject * PyObject_Format(PyObject* obj, PyObject *format_spec) { diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index bc02106e786..9ff44586f73 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -123,7 +123,7 @@ bytes_getbuffer(PyByteArrayObject *obj, Py_buffer *view, int flags) ptr = ""; else ptr = obj->ob_bytes; - ret = PyBuffer_FillInfo(view, ptr, Py_SIZE(obj), 0, flags); + ret = PyBuffer_FillInfo(view, (PyObject*)obj, ptr, Py_SIZE(obj), 0, flags); if (ret >= 0) { obj->ob_exports++; } @@ -302,9 +302,9 @@ PyByteArray_Concat(PyObject *a, PyObject *b) done: if (va.len != -1) - PyObject_ReleaseBuffer(a, &va); + PyBuffer_Release(&va); if (vb.len != -1) - PyObject_ReleaseBuffer(b, &vb); + PyBuffer_Release(&vb); return (PyObject *)result; } @@ -332,7 +332,7 @@ bytes_iconcat(PyByteArrayObject *self, PyObject *other) mysize = Py_SIZE(self); size = mysize + vo.len; if (size < 0) { - PyObject_ReleaseBuffer(other, &vo); + PyBuffer_Release(&vo); return PyErr_NoMemory(); } if (size < self->ob_alloc) { @@ -340,11 +340,11 @@ bytes_iconcat(PyByteArrayObject *self, PyObject *other) self->ob_bytes[Py_SIZE(self)] = '\0'; /* Trailing null byte */ } else if (PyByteArray_Resize((PyObject *)self, size) < 0) { - PyObject_ReleaseBuffer(other, &vo); + PyBuffer_Release(&vo); return NULL; } memcpy(self->ob_bytes + mysize, vo.buf, vo.len); - PyObject_ReleaseBuffer(other, &vo); + PyBuffer_Release(&vo); Py_INCREF(self); return (PyObject *)self; } @@ -555,7 +555,7 @@ bytes_setslice(PyByteArrayObject *self, Py_ssize_t lo, Py_ssize_t hi, finish: if (vbytes.len != -1) - PyObject_ReleaseBuffer(values, &vbytes); + PyBuffer_Release(&vbytes); return res; } @@ -841,10 +841,10 @@ bytes_init(PyByteArrayObject *self, PyObject *args, PyObject *kwds) if (PyByteArray_Resize((PyObject *)self, size) < 0) goto fail; if (PyBuffer_ToContiguous(self->ob_bytes, &view, size, 'C') < 0) goto fail; - PyObject_ReleaseBuffer(arg, &view); + PyBuffer_Release(&view); return 0; fail: - PyObject_ReleaseBuffer(arg, &view); + PyBuffer_Release(&view); return -1; } @@ -1031,7 +1031,7 @@ bytes_richcompare(PyObject *self, PyObject *other, int op) other_size = _getbuffer(other, &other_bytes); if (other_size < 0) { PyErr_Clear(); - PyObject_ReleaseBuffer(self, &self_bytes); + PyBuffer_Release(&self_bytes); Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } @@ -1066,8 +1066,8 @@ bytes_richcompare(PyObject *self, PyObject *other, int op) } res = cmp ? Py_True : Py_False; - PyObject_ReleaseBuffer(self, &self_bytes); - PyObject_ReleaseBuffer(other, &other_bytes); + PyBuffer_Release(&self_bytes); + PyBuffer_Release(&other_bytes); Py_INCREF(res); return res; } @@ -1075,6 +1075,11 @@ bytes_richcompare(PyObject *self, PyObject *other, int op) static void bytes_dealloc(PyByteArrayObject *self) { + if (self->ob_exports > 0) { + PyErr_SetString(PyExc_SystemError, + "deallocated bytearray object has exported buffers"); + PyErr_Print(); + } if (self->ob_bytes != 0) { PyMem_Free(self->ob_bytes); } @@ -1142,7 +1147,7 @@ bytes_find_internal(PyByteArrayObject *self, PyObject *args, int dir) res = stringlib_rfind_slice( PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), subbuf.buf, subbuf.len, start, end); - PyObject_ReleaseBuffer(subobj, &subbuf); + PyBuffer_Release(&subbuf); return res; } @@ -1192,7 +1197,7 @@ bytes_count(PyByteArrayObject *self, PyObject *args) count_obj = PyInt_FromSsize_t( stringlib_count(str + start, end - start, vsub.buf, vsub.len) ); - PyObject_ReleaseBuffer(sub_obj, &vsub); + PyBuffer_Release(&vsub); return count_obj; } @@ -1268,7 +1273,7 @@ bytes_contains(PyObject *self, PyObject *arg) return -1; pos = stringlib_find(PyByteArray_AS_STRING(self), Py_SIZE(self), varg.buf, varg.len, 0); - PyObject_ReleaseBuffer(arg, &varg); + PyBuffer_Release(&varg); return pos >= 0; } if (ival < 0 || ival >= 256) { @@ -1318,7 +1323,7 @@ _bytes_tailmatch(PyByteArrayObject *self, PyObject *substr, Py_ssize_t start, rv = ! memcmp(str+start, vsubstr.buf, vsubstr.len); done: - PyObject_ReleaseBuffer(substr, &vsubstr); + PyBuffer_Release(&vsubstr); return rv; } @@ -1498,9 +1503,9 @@ bytes_translate(PyByteArrayObject *self, PyObject *args) PyByteArray_Resize(result, output - output_start); done: - PyObject_ReleaseBuffer(tableobj, &vtable); + PyBuffer_Release(&vtable); if (delobj != NULL) - PyObject_ReleaseBuffer(delobj, &vdel); + PyBuffer_Release(&vdel); return result; } @@ -2122,7 +2127,7 @@ bytes_replace(PyByteArrayObject *self, PyObject *args) if (_getbuffer(from, &vfrom) < 0) return NULL; if (_getbuffer(to, &vto) < 0) { - PyObject_ReleaseBuffer(from, &vfrom); + PyBuffer_Release(&vfrom); return NULL; } @@ -2130,8 +2135,8 @@ bytes_replace(PyByteArrayObject *self, PyObject *args) vfrom.buf, vfrom.len, vto.buf, vto.len, count); - PyObject_ReleaseBuffer(from, &vfrom); - PyObject_ReleaseBuffer(to, &vto); + PyBuffer_Release(&vfrom); + PyBuffer_Release(&vto); return res; } @@ -2287,7 +2292,7 @@ bytes_split(PyByteArrayObject *self, PyObject *args) if (n == 0) { PyErr_SetString(PyExc_ValueError, "empty separator"); - PyObject_ReleaseBuffer(subobj, &vsub); + PyBuffer_Release(&vsub); return NULL; } if (n == 1) @@ -2295,7 +2300,7 @@ bytes_split(PyByteArrayObject *self, PyObject *args) list = PyList_New(PREALLOC_SIZE(maxsplit)); if (list == NULL) { - PyObject_ReleaseBuffer(subobj, &vsub); + PyBuffer_Release(&vsub); return NULL; } @@ -2323,12 +2328,12 @@ bytes_split(PyByteArrayObject *self, PyObject *args) #endif SPLIT_ADD(s, i, len); FIX_PREALLOC_SIZE(list); - PyObject_ReleaseBuffer(subobj, &vsub); + PyBuffer_Release(&vsub); return list; onError: Py_DECREF(list); - PyObject_ReleaseBuffer(subobj, &vsub); + PyBuffer_Release(&vsub); return NULL; } @@ -2519,7 +2524,7 @@ bytes_rsplit(PyByteArrayObject *self, PyObject *args) if (n == 0) { PyErr_SetString(PyExc_ValueError, "empty separator"); - PyObject_ReleaseBuffer(subobj, &vsub); + PyBuffer_Release(&vsub); return NULL; } else if (n == 1) @@ -2527,7 +2532,7 @@ bytes_rsplit(PyByteArrayObject *self, PyObject *args) list = PyList_New(PREALLOC_SIZE(maxsplit)); if (list == NULL) { - PyObject_ReleaseBuffer(subobj, &vsub); + PyBuffer_Release(&vsub); return NULL; } @@ -2548,12 +2553,12 @@ bytes_rsplit(PyByteArrayObject *self, PyObject *args) FIX_PREALLOC_SIZE(list); if (PyList_Reverse(list) < 0) goto onError; - PyObject_ReleaseBuffer(subobj, &vsub); + PyBuffer_Release(&vsub); return list; onError: Py_DECREF(list); - PyObject_ReleaseBuffer(subobj, &vsub); + PyBuffer_Release(&vsub); return NULL; } @@ -2828,7 +2833,7 @@ bytes_strip(PyByteArrayObject *self, PyObject *args) else right = rstrip_helper(myptr, mysize, argptr, argsize); if (arg != Py_None) - PyObject_ReleaseBuffer(arg, &varg); + PyBuffer_Release(&varg); return PyByteArray_FromStringAndSize(self->ob_bytes + left, right - left); } @@ -2861,7 +2866,7 @@ bytes_lstrip(PyByteArrayObject *self, PyObject *args) left = lstrip_helper(myptr, mysize, argptr, argsize); right = mysize; if (arg != Py_None) - PyObject_ReleaseBuffer(arg, &varg); + PyBuffer_Release(&varg); return PyByteArray_FromStringAndSize(self->ob_bytes + left, right - left); } @@ -2894,7 +2899,7 @@ bytes_rstrip(PyByteArrayObject *self, PyObject *args) left = 0; right = rstrip_helper(myptr, mysize, argptr, argsize); if (arg != Py_None) - PyObject_ReleaseBuffer(arg, &varg); + PyBuffer_Release(&varg); return PyByteArray_FromStringAndSize(self->ob_bytes + left, right - left); } diff --git a/Objects/fileobject.c b/Objects/fileobject.c index 7c49afff069..a8e95a26975 100644 --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -1007,6 +1007,7 @@ file_readinto(PyFileObject *f, PyObject *args) char *ptr; Py_ssize_t ntodo; Py_ssize_t ndone, nnow; + Py_buffer pbuf; if (f->f_fp == NULL) return err_closed(); @@ -1015,8 +1016,10 @@ file_readinto(PyFileObject *f, PyObject *args) (f->f_bufend - f->f_bufptr) > 0 && f->f_buf[0] != '\0') return err_iterbuffered(); - if (!PyArg_ParseTuple(args, "w#", &ptr, &ntodo)) + if (!PyArg_ParseTuple(args, "w*", &pbuf)) return NULL; + ptr = pbuf.buf; + ntodo = pbuf.len; ndone = 0; while (ntodo > 0) { FILE_BEGIN_ALLOW_THREADS(f) @@ -1029,11 +1032,13 @@ file_readinto(PyFileObject *f, PyObject *args) break; PyErr_SetFromErrno(PyExc_IOError); clearerr(f->f_fp); + PyBuffer_Release(&pbuf); return NULL; } ndone += nnow; ntodo -= nnow; } + PyBuffer_Release(&pbuf); return PyInt_FromSsize_t(ndone); } @@ -1611,17 +1616,26 @@ error: static PyObject * file_write(PyFileObject *f, PyObject *args) { + Py_buffer pbuf; char *s; Py_ssize_t n, n2; if (f->f_fp == NULL) return err_closed(); - if (!PyArg_ParseTuple(args, f->f_binary ? "s#" : "t#", &s, &n)) + if (f->f_binary) { + if (!PyArg_ParseTuple(args, "s*", &pbuf)) + return NULL; + s = pbuf.buf; + n = pbuf.len; + } else + if (!PyArg_ParseTuple(args, "t#", &s, &n)) return NULL; f->f_softspace = 0; FILE_BEGIN_ALLOW_THREADS(f) errno = 0; n2 = fwrite(s, 1, n, f->f_fp); FILE_END_ALLOW_THREADS(f) + if (f->f_binary) + PyBuffer_Release(&pbuf); if (n2 != n) { PyErr_SetFromErrno(PyExc_IOError); clearerr(f->f_fp); diff --git a/Objects/stringobject.c b/Objects/stringobject.c index 6d53a099f87..5bf4add229c 100644 --- a/Objects/stringobject.c +++ b/Objects/stringobject.c @@ -1328,7 +1328,8 @@ string_buffer_getcharbuf(PyStringObject *self, Py_ssize_t index, const char **pt static int string_buffer_getbuffer(PyStringObject *self, Py_buffer *view, int flags) { - return PyBuffer_FillInfo(view, (void *)self->ob_sval, Py_SIZE(self), + return PyBuffer_FillInfo(view, (PyObject*)self, + (void *)self->ob_sval, Py_SIZE(self), 1, flags); } @@ -1359,7 +1360,7 @@ static PyBufferProcs string_as_buffer = { }; - + #define LEFTSTRIP 0 #define RIGHTSTRIP 1 #define BOTHSTRIP 2 @@ -3996,7 +3997,7 @@ PyDoc_STRVAR(p_format__doc__, \n\ "); - + static PyMethodDef string_methods[] = { /* Counterparts of the obsolete stropmodule functions; except diff --git a/Python/getargs.c b/Python/getargs.c index 162cf6fa98a..7c9774fb82e 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -44,6 +44,7 @@ static char *converttuple(PyObject *, const char **, va_list *, int, static char *convertsimple(PyObject *, const char **, va_list *, int, char *, size_t, PyObject **); static Py_ssize_t convertbuffer(PyObject *, void **p, char **); +static int getbuffer(PyObject *, Py_buffer *, char**); static int vgetargskeywords(PyObject *, PyObject *, const char *, char **, va_list *, int); @@ -773,7 +774,32 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } case 's': {/* string */ - if (*format == '#') { + if (*format == '*') { + Py_buffer *p = (Py_buffer *)va_arg(*p_va, Py_buffer *); + + if (PyString_Check(arg)) { + PyBuffer_FillInfo(p, arg, + PyString_AS_STRING(arg), PyString_GET_SIZE(arg), + 1, 0); + } +#ifdef Py_USING_UNICODE + else if (PyUnicode_Check(arg)) { + uarg = UNICODE_DEFAULT_ENCODING(arg); + if (uarg == NULL) + return converterr(CONV_UNICODE, + arg, msgbuf, bufsize); + PyBuffer_FillInfo(p, arg, + PyString_AS_STRING(uarg), PyString_GET_SIZE(uarg), + 1, 0); + } +#endif + else { /* any buffer-like object */ + char *buf; + if (getbuffer(arg, p, &buf) < 0) + return converterr(buf, arg, msgbuf, bufsize); + } + format++; + } else if (*format == '#') { void **p = (void **)va_arg(*p_va, char **); FETCH_SIZE; @@ -823,7 +849,34 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } case 'z': {/* string, may be NULL (None) */ - if (*format == '#') { /* any buffer-like object */ + if (*format == '*') { + Py_buffer *p = (Py_buffer *)va_arg(*p_va, Py_buffer *); + + if (arg == Py_None) + PyBuffer_FillInfo(p, NULL, NULL, 0, 1, 0); + else if (PyString_Check(arg)) { + PyBuffer_FillInfo(p, arg, + PyString_AS_STRING(arg), PyString_GET_SIZE(arg), + 1, 0); + } +#ifdef Py_USING_UNICODE + else if (PyUnicode_Check(arg)) { + uarg = UNICODE_DEFAULT_ENCODING(arg); + if (uarg == NULL) + return converterr(CONV_UNICODE, + arg, msgbuf, bufsize); + PyBuffer_FillInfo(p, arg, + PyString_AS_STRING(uarg), PyString_GET_SIZE(uarg), + 1, 0); + } +#endif + else { /* any buffer-like object */ + char *buf; + if (getbuffer(arg, p, &buf) < 0) + return converterr(buf, arg, msgbuf, bufsize); + } + format++; + } else if (*format == '#') { /* any buffer-like object */ void **p = (void **)va_arg(*p_va, char **); FETCH_SIZE; @@ -1144,23 +1197,49 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'w': { /* memory buffer, read-write access */ void **p = va_arg(*p_va, void **); + void *res; PyBufferProcs *pb = arg->ob_type->tp_as_buffer; Py_ssize_t count; - - if (pb == NULL || + + if (pb && pb->bf_releasebuffer && *format != '*') + /* Buffer must be released, yet caller does not use + the Py_buffer protocol. */ + return converterr("pinned buffer", arg, msgbuf, bufsize); + + if (pb && pb->bf_getbuffer && *format == '*') { + /* Caller is interested in Py_buffer, and the object + supports it directly. */ + format++; + if (pb->bf_getbuffer(arg, (Py_buffer*)p, PyBUF_WRITABLE) < 0) { + PyErr_Clear(); + return converterr("read-write buffer", arg, msgbuf, bufsize); + } + if (!PyBuffer_IsContiguous((Py_buffer*)p, 'C')) + return converterr("contiguous buffer", arg, msgbuf, bufsize); + break; + } + + if (pb == NULL || pb->bf_getwritebuffer == NULL || pb->bf_getsegcount == NULL) return converterr("read-write buffer", arg, msgbuf, bufsize); if ((*pb->bf_getsegcount)(arg, NULL) != 1) return converterr("single-segment read-write buffer", arg, msgbuf, bufsize); - if ((count = pb->bf_getwritebuffer(arg, 0, p)) < 0) + if ((count = pb->bf_getwritebuffer(arg, 0, &res)) < 0) return converterr("(unspecified)", arg, msgbuf, bufsize); - if (*format == '#') { - FETCH_SIZE; - STORE_SIZE(count); + if (*format == '*') { + PyBuffer_FillInfo((Py_buffer*)p, arg, res, count, 1, 0); format++; } + else { + *p = res; + if (*format == '#') { + FETCH_SIZE; + STORE_SIZE(count); + format++; + } + } break; } @@ -1186,6 +1265,11 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, "string or single-segment read-only buffer", arg, msgbuf, bufsize); + if (pb->bf_releasebuffer) + return converterr( + "string or pinned buffer", + arg, msgbuf, bufsize); + count = pb->bf_getcharbuffer(arg, 0, p); if (count < 0) return converterr("(unspecified)", arg, msgbuf, bufsize); @@ -1212,7 +1296,8 @@ convertbuffer(PyObject *arg, void **p, char **errmsg) Py_ssize_t count; if (pb == NULL || pb->bf_getreadbuffer == NULL || - pb->bf_getsegcount == NULL) { + pb->bf_getsegcount == NULL || + pb->bf_releasebuffer != NULL) { *errmsg = "string or read-only buffer"; return -1; } @@ -1226,6 +1311,33 @@ convertbuffer(PyObject *arg, void **p, char **errmsg) return count; } +static int +getbuffer(PyObject *arg, Py_buffer *view, char**errmsg) +{ + void *buf; + Py_ssize_t count; + PyBufferProcs *pb = arg->ob_type->tp_as_buffer; + if (pb == NULL) { + *errmsg = "string or buffer"; + return -1; + } + if (pb->bf_getbuffer) { + if (pb->bf_getbuffer(arg, view, 0) < 0) + return -1; + if (!PyBuffer_IsContiguous(view, 'C')) { + *errmsg = "contiguous buffer"; + return -1; + } + return 0; + } + + count = convertbuffer(arg, &buf, errmsg); + if (count < 0) + return count; + PyBuffer_FillInfo(view, NULL, buf, count, 1, 0); + return 0; +} + /* Support for keyword arguments donated by Geoff Philbrick */ @@ -1566,6 +1678,8 @@ skipitem(const char **p_format, va_list *p_va, int flags) else (void) va_arg(*p_va, int *); format++; + } else if ((c == 's' || c == 'z') && *format == '*') { + format++; } break; }