From 9795ca44fbf6d94789ee7053b81ff18f008c41ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristj=C3=A1n=20Valur=20J=C3=B3nsson?= Date: Tue, 19 Mar 2013 17:17:47 -0700 Subject: [PATCH] Issue #10212: Support new buffer interface for struct.unpack and cStringIO --- Lib/test/test_StringIO.py | 5 +++- Lib/test/test_struct.py | 11 +++++++++ Misc/NEWS | 4 ++++ Modules/_struct.c | 29 +++++++++++++++-------- Modules/cStringIO.c | 49 ++++++++++++++++++++------------------- 5 files changed, 63 insertions(+), 35 deletions(-) diff --git a/Lib/test/test_StringIO.py b/Lib/test/test_StringIO.py index 37a825f14e3..42f307ad578 100644 --- a/Lib/test/test_StringIO.py +++ b/Lib/test/test_StringIO.py @@ -20,7 +20,6 @@ class TestGenericStringIO(unittest.TestCase): constructor = str def setUp(self): - self._line = self.constructor(self._line) self._lines = self.constructor((self._line + '\n') * 5) self._fp = self.MODULE.StringIO(self._lines) @@ -210,12 +209,16 @@ class TestBufferStringIO(TestStringIO): class TestBuffercStringIO(TestcStringIO): constructor = buffer +class TestMemoryviewcStringIO(TestcStringIO): + constructor = memoryview + def test_main(): test_support.run_unittest(TestStringIO, TestcStringIO) with test_support.check_py3k_warnings(("buffer.. not supported", DeprecationWarning)): test_support.run_unittest(TestBufferStringIO, TestBuffercStringIO) + test_support.run_unittest(TestMemoryviewcStringIO) if __name__ == '__main__': test_main() diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index c8dc6f17042..f1b5d9a1cde 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -496,6 +496,17 @@ class StructTest(unittest.TestCase): self.test_unpack_from(cls=buffer) + def test_unpack_with_memoryview(self): + with check_py3k_warnings(("buffer.. not supported in 3.x", + DeprecationWarning)): + # SF bug 1563759: struct.unpack doesn't support buffer protocol objects + data1 = memoryview('\x12\x34\x56\x78') + for data in [data1,]: + value, = struct.unpack('>I', data) + self.assertEqual(value, 0x12345678) + + self.test_unpack_from(cls=memoryview) + def test_bool(self): class ExplodingBool(object): def __nonzero__(self): diff --git a/Misc/NEWS b/Misc/NEWS index f309e6f3d23..1f3601cd1fc 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -9,6 +9,8 @@ What's New in Python 2.7.4 Core and Builtins ----------------- +- Issue #10211: Buffer objects expose the new buffer interface internally + - Issue #16445: Fixed potential segmentation fault when deleting an exception message. @@ -214,6 +216,8 @@ Core and Builtins Library ------- +- Issue #10212: cStringIO and struct.unpack support new buffer objects. + - Issue #12098: multiprocessing on Windows now starts child processes using the same sys.flags as the current process. Initial patch by Sergey Mezentsev. diff --git a/Modules/_struct.c b/Modules/_struct.c index d8c932427d6..80569561f98 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -1439,6 +1439,7 @@ strings."); static PyObject * s_unpack(PyObject *self, PyObject *inputstr) { + Py_buffer buf; char *start; Py_ssize_t len; PyObject *args=NULL, *result; @@ -1454,12 +1455,17 @@ s_unpack(PyObject *self, PyObject *inputstr) args = PyTuple_Pack(1, inputstr); if (args == NULL) return NULL; - if (!PyArg_ParseTuple(args, "s#:unpack", &start, &len)) + if (!PyArg_ParseTuple(args, "s*:unpack", &buf)) goto fail; - if (soself->s_size != len) + start = buf.buf; + len = buf.len; + if (soself->s_size != len) { + PyBuffer_Release(&buf); goto fail; + } result = s_unpack_internal(soself, start); Py_DECREF(args); + PyBuffer_Release(&buf); return result; fail: @@ -1482,24 +1488,24 @@ static PyObject * s_unpack_from(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"buffer", "offset", 0}; -#if (PY_VERSION_HEX < 0x02050000) - static char *fmt = "z#|i:unpack_from"; -#else - static char *fmt = "z#|n:unpack_from"; -#endif + static char *fmt = "z*|n:unpack_from"; + Py_buffer buf; Py_ssize_t buffer_len = 0, offset = 0; char *buffer = NULL; PyStructObject *soself = (PyStructObject *)self; + PyObject *result; assert(PyStruct_Check(self)); assert(soself->s_codes != NULL); if (!PyArg_ParseTupleAndKeywords(args, kwds, fmt, kwlist, - &buffer, &buffer_len, &offset)) + &buf, &offset)) return NULL; - + buffer = buf.buf; + buffer_len = buf.len; if (buffer == NULL) { PyErr_Format(StructError, "unpack_from requires a buffer argument"); + PyBuffer_Release(&buf); return NULL; } @@ -1510,9 +1516,12 @@ s_unpack_from(PyObject *self, PyObject *args, PyObject *kwds) PyErr_Format(StructError, "unpack_from requires a buffer of at least %zd bytes", soself->s_size); + PyBuffer_Release(&buf); return NULL; } - return s_unpack_internal(soself, buffer + offset); + result = s_unpack_internal(soself, buffer + offset); + PyBuffer_Release(&buf); + return result; } diff --git a/Modules/cStringIO.c b/Modules/cStringIO.c index 566851f66d8..5b78789d5af 100644 --- a/Modules/cStringIO.c +++ b/Modules/cStringIO.c @@ -66,9 +66,7 @@ typedef struct { /* Subtype of IOobject */ PyObject_HEAD char *buf; Py_ssize_t pos, string_size; - /* We store a reference to the object here in order to keep - the buffer alive during the lifetime of the Iobject. */ - PyObject *pbuf; + Py_buffer pbuf; } Iobject; /* IOobject (common) methods */ @@ -448,12 +446,14 @@ O_cwrite(PyObject *self, const char *c, Py_ssize_t len) { static PyObject * O_write(Oobject *self, PyObject *args) { - char *c; - int l; + Py_buffer buf; + int result; - if (!PyArg_ParseTuple(args, "t#:write", &c, &l)) return NULL; + if (!PyArg_ParseTuple(args, "s*:write", &buf)) return NULL; - if (O_cwrite((PyObject*)self,c,l) < 0) return NULL; + result = O_cwrite((PyObject*)self, buf.buf, buf.len); + PyBuffer_Release(&buf); + if (result < 0) return NULL; Py_INCREF(Py_None); return Py_None; @@ -606,7 +606,7 @@ newOobject(int size) { static PyObject * I_close(Iobject *self, PyObject *unused) { - Py_CLEAR(self->pbuf); + PyBuffer_Release(&self->pbuf); self->buf = NULL; self->pos = self->string_size = 0; @@ -635,7 +635,7 @@ static struct PyMethodDef I_methods[] = { static void I_dealloc(Iobject *self) { - Py_XDECREF(self->pbuf); + PyBuffer_Release(&self->pbuf); PyObject_Del(self); } @@ -680,25 +680,26 @@ static PyTypeObject Itype = { static PyObject * newIobject(PyObject *s) { Iobject *self; - char *buf; - Py_ssize_t size; + Py_buffer buf; + PyObject *args; + int result; - if (PyUnicode_Check(s)) { - if (PyObject_AsCharBuffer(s, (const char **)&buf, &size) != 0) + args = Py_BuildValue("(O)", s); + if (args == NULL) + return NULL; + result = PyArg_ParseTuple(args, "s*:StringIO", &buf); + Py_DECREF(args); + if (!result) return NULL; - } - else if (PyObject_AsReadBuffer(s, (const void **)&buf, &size)) { - PyErr_Format(PyExc_TypeError, "expected read buffer, %.200s found", - s->ob_type->tp_name); - return NULL; - } self = PyObject_New(Iobject, &Itype); - if (!self) return NULL; - Py_INCREF(s); - self->buf=buf; - self->string_size=size; - self->pbuf=s; + if (!self) { + PyBuffer_Release(&buf); + return NULL; + } + self->buf=buf.buf; + self->string_size=buf.len; + self->pbuf=buf; self->pos=0; return (PyObject*)self;