From 616d28566b6b61ec92059b0151b79bdcf487bc5a Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 19 Aug 2008 22:09:34 +0000 Subject: [PATCH] Issue #2394: implement more of the memoryview API. --- Lib/test/test_codecs.py | 2 +- Lib/test/test_memoryview.py | 192 +++++++++++++++++++++++++++- Misc/NEWS | 7 ++ Objects/memoryobject.c | 242 ++++++++++++++++++++++++++++++++---- 4 files changed, 418 insertions(+), 25 deletions(-) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index a7da809ccb0..e485fdd2fe8 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -621,7 +621,7 @@ class UTF8SigTest(ReadTest): def test_bug1601501(self): # SF bug #1601501: check that the codec works with a buffer - str(b"\xef\xbb\xbf", "utf-8-sig") + self.assertEquals(str(b"\xef\xbb\xbf", "utf-8-sig"), "") def test_bom(self): d = codecs.getincrementaldecoder("utf-8-sig")() diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index b3d478df17f..dca48b1b5b2 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -5,8 +5,166 @@ XXX We need more tests! Some tests are in test_bytes import unittest import test.support +import sys -class MemoryviewTest(unittest.TestCase): + +class CommonMemoryTests: + # + # Tests common to direct memoryviews and sliced memoryviews + # + + base_object = b"abcdef" + + def check_getitem_with_type(self, tp): + b = tp(self.base_object) + oldrefcount = sys.getrefcount(b) + m = self._view(b) + self.assertEquals(m[0], b"a") + self.assert_(isinstance(m[0], bytes), type(m[0])) + self.assertEquals(m[5], b"f") + self.assertEquals(m[-1], b"f") + self.assertEquals(m[-6], b"a") + # Bounds checking + self.assertRaises(IndexError, lambda: m[6]) + self.assertRaises(IndexError, lambda: m[-7]) + self.assertRaises(IndexError, lambda: m[sys.maxsize]) + self.assertRaises(IndexError, lambda: m[-sys.maxsize]) + # Type checking + self.assertRaises(TypeError, lambda: m[None]) + self.assertRaises(TypeError, lambda: m[0.0]) + self.assertRaises(TypeError, lambda: m["a"]) + m = None + self.assertEquals(sys.getrefcount(b), oldrefcount) + + def test_getitem_readonly(self): + self.check_getitem_with_type(bytes) + + def test_getitem_writable(self): + self.check_getitem_with_type(bytearray) + + def test_setitem_readonly(self): + b = self.base_object + oldrefcount = sys.getrefcount(b) + m = self._view(b) + def setitem(value): + m[0] = value + self.assertRaises(TypeError, setitem, b"a") + self.assertRaises(TypeError, setitem, 65) + self.assertRaises(TypeError, setitem, memoryview(b"a")) + m = None + self.assertEquals(sys.getrefcount(b), oldrefcount) + + def test_setitem_writable(self): + b = bytearray(self.base_object) + oldrefcount = sys.getrefcount(b) + m = self._view(b) + m[0] = b"0" + self._check_contents(b, b"0bcdef") + m[1:3] = b"12" + self._check_contents(b, b"012def") + m[1:1] = b"" + self._check_contents(b, b"012def") + m[:] = b"abcdef" + self._check_contents(b, b"abcdef") + + # Overlapping copies of a view into itself + m[0:3] = m[2:5] + self._check_contents(b, b"cdedef") + m[:] = b"abcdef" + m[2:5] = m[0:3] + self._check_contents(b, b"ababcf") + + def setitem(key, value): + m[key] = value + # Bounds checking + self.assertRaises(IndexError, setitem, 6, b"a") + self.assertRaises(IndexError, setitem, -7, b"a") + self.assertRaises(IndexError, setitem, sys.maxsize, b"a") + self.assertRaises(IndexError, setitem, -sys.maxsize, b"a") + # Wrong index/slice types + self.assertRaises(TypeError, setitem, 0.0, b"a") + self.assertRaises(TypeError, setitem, (0,), b"a") + self.assertRaises(TypeError, setitem, "a", b"a") + # Trying to resize the memory object + self.assertRaises(ValueError, setitem, 0, b"") + self.assertRaises(ValueError, setitem, 0, b"ab") + self.assertRaises(ValueError, setitem, slice(1,1), b"a") + self.assertRaises(ValueError, setitem, slice(0,2), b"a") + + m = None + self.assertEquals(sys.getrefcount(b), oldrefcount) + + def test_len(self): + self.assertEquals(len(self._view(self.base_object)), 6) + + def test_tobytes(self): + m = self._view(self.base_object) + b = m.tobytes() + self.assertEquals(b, b"abcdef") + self.assert_(isinstance(b, bytes), type(b)) + + def test_tolist(self): + m = self._view(self.base_object) + l = m.tolist() + self.assertEquals(l, list(b"abcdef")) + + def test_compare(self): + # memoryviews can compare for equality with other objects + # having the buffer interface. + m = self._view(self.base_object) + for tp in (bytes, bytearray): + self.assertTrue(m == tp(b"abcdef")) + self.assertFalse(m != tp(b"abcdef")) + self.assertFalse(m == tp(b"abcde")) + self.assertTrue(m != tp(b"abcde")) + self.assertFalse(m == tp(b"abcde1")) + self.assertTrue(m != tp(b"abcde1")) + self.assertTrue(m == m) + self.assertTrue(m == m[:]) + self.assertTrue(m[0:6] == m[:]) + self.assertFalse(m[0:5] == m) + + # Comparison with objects which don't support the buffer API + self.assertFalse(m == "abc") + self.assertTrue(m != "abc") + self.assertFalse("abc" == m) + self.assertTrue("abc" != m) + + # Unordered comparisons + for c in (m, b"abcdef"): + self.assertRaises(TypeError, lambda: m < c) + self.assertRaises(TypeError, lambda: c <= m) + self.assertRaises(TypeError, lambda: m >= c) + self.assertRaises(TypeError, lambda: c > m) + + def check_attributes_with_type(self, tp): + b = tp(self.base_object) + m = self._view(b) + self.assertEquals(m.format, 'B') + self.assertEquals(m.itemsize, 1) + self.assertEquals(m.ndim, 1) + self.assertEquals(m.shape, (6,)) + self.assertEquals(m.size, 6) + self.assertEquals(m.strides, (1,)) + self.assertEquals(m.suboffsets, None) + return m + + def test_attributes_readonly(self): + m = self.check_attributes_with_type(bytes) + self.assertEquals(m.readonly, True) + + def test_attributes_writable(self): + m = self.check_attributes_with_type(bytearray) + self.assertEquals(m.readonly, False) + + +class MemoryviewTest(unittest.TestCase, CommonMemoryTests): + + def _view(self, obj): + return memoryview(obj) + + def _check_contents(self, obj, contents): + self.assertEquals(obj, contents) def test_constructor(self): ob = b'test' @@ -17,8 +175,38 @@ class MemoryviewTest(unittest.TestCase): self.assertRaises(TypeError, memoryview, argument=ob) self.assertRaises(TypeError, memoryview, ob, argument=True) + +class MemorySliceTest(unittest.TestCase, CommonMemoryTests): + base_object = b"XabcdefY" + + def _view(self, obj): + m = memoryview(obj) + return m[1:7] + + def _check_contents(self, obj, contents): + self.assertEquals(obj[1:7], contents) + + def test_refs(self): + m = memoryview(b"ab") + oldrefcount = sys.getrefcount(m) + m[1:2] + self.assertEquals(sys.getrefcount(m), oldrefcount) + + +class MemorySliceSliceTest(unittest.TestCase, CommonMemoryTests): + base_object = b"XabcdefY" + + def _view(self, obj): + m = memoryview(obj) + return m[:7][1:] + + def _check_contents(self, obj, contents): + self.assertEquals(obj[1:7], contents) + + def test_main(): - test.support.run_unittest(MemoryviewTest) + test.support.run_unittest( + MemoryviewTest, MemorySliceTest, MemorySliceSliceTest) if __name__ == "__main__": diff --git a/Misc/NEWS b/Misc/NEWS index c3cf8bee982..bd7fdbade25 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,13 @@ What's new in Python 3.0b3? Core and Builtins ----------------- +- Issue #2394: implement more of the memoryview API, with the caveat that + only one-dimensional contiguous buffers are supported and exercised right + now. Slicing, slice assignment and comparison (equality and inequality) + have been added. Also, the tolist() method has been implemented, but only + for byte buffers. Endly, the API has been updated to return bytes objects + wherever it used to return bytearrays. + - Issue #3560: clean up the new C PyMemoryView API so that naming is internally consistent; add macros PyMemoryView_GET_BASE() and PyMemoryView_GET_BUFFER() to access useful properties of a memory views diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index c108363bf83..bd5820f0f08 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -13,8 +13,8 @@ memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) } if (self->view.obj == NULL) return 0; - return self->view.obj->ob_type->tp_as_buffer->bf_getbuffer(self->base, NULL, - PyBUF_FULL); + return self->view.obj->ob_type->tp_as_buffer->bf_getbuffer( + self->view.obj, NULL, PyBUF_FULL); } static void @@ -37,9 +37,14 @@ PyMemoryView_FromBuffer(Py_buffer *info) &PyMemoryView_Type); if (mview == NULL) return NULL; mview->base = NULL; + /* XXX there should be an API to duplicate a buffer object */ mview->view = *info; - if (info->obj) - Py_INCREF(mview->view.obj); + if (info->shape == &(info->len)) + mview->view.shape = &(mview->view.len); + if (info->strides == &(info->itemsize)) + mview->view.strides = &(mview->view.itemsize); + /* NOTE: mview->view.obj should already have been incref'ed as + part of PyBuffer_FillInfo(). */ return (PyObject *)mview; } @@ -258,12 +263,12 @@ PyMemoryView_GetContiguous(PyObject *obj, int buffertype, char fort) "for a non-contiguousobject."); return NULL; } - bytes = PyByteArray_FromStringAndSize(NULL, view->len); + bytes = PyBytes_FromStringAndSize(NULL, view->len); if (bytes == NULL) { PyBuffer_Release(view); return NULL; } - dest = PyByteArray_AS_STRING(bytes); + dest = PyBytes_AS_STRING(bytes); /* different copying strategy depending on whether or not any pointer de-referencing is needed */ @@ -386,17 +391,45 @@ static PyGetSetDef memory_getsetlist[] ={ static PyObject * memory_tobytes(PyMemoryViewObject *mem, PyObject *noargs) { - return PyByteArray_FromObject((PyObject *)mem); + return PyObject_CallFunctionObjArgs( + (PyObject *) &PyBytes_Type, mem, NULL); } +/* TODO: rewrite this function using the struct module to unpack + each buffer item */ + static PyObject * memory_tolist(PyMemoryViewObject *mem, PyObject *noargs) { - /* This should construct a (nested) list of unpacked objects - possibly using the struct module. - */ - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_buffer *view = &(mem->view); + Py_ssize_t i; + PyObject *res, *item; + char *buf; + + if (strcmp(view->format, "B") || view->itemsize != 1) { + PyErr_SetString(PyExc_NotImplementedError, + "tolist() only supports byte views"); + return NULL; + } + if (view->ndim != 1) { + PyErr_SetString(PyExc_NotImplementedError, + "tolist() only supports one-dimensional objects"); + return NULL; + } + res = PyList_New(view->len); + if (res == NULL) + return NULL; + buf = view->buf; + for (i = 0; i < view->len; i++) { + item = PyLong_FromUnsignedLong((unsigned char) *buf); + if (item == NULL) { + Py_DECREF(res); + return NULL; + } + PyList_SET_ITEM(res, i, item); + buf++; + } + return res; } @@ -412,7 +445,7 @@ static void memory_dealloc(PyMemoryViewObject *self) { if (self->view.obj != NULL) { - if (PyTuple_Check(self->base)) { + if (self->base && PyTuple_Check(self->base)) { /* Special case when first element is generic object with buffer interface and the second element is a contiguous "shadow" that must be copied back into @@ -454,8 +487,8 @@ memory_str(PyMemoryViewObject *self) if (PyObject_GetBuffer((PyObject *)self, &view, PyBUF_FULL) < 0) return NULL; - res = PyByteArray_FromStringAndSize(NULL, view.len); - PyBuffer_ToContiguous(PyByteArray_AS_STRING(res), &view, view.len, 'C'); + res = PyBytes_FromStringAndSize(NULL, view.len); + PyBuffer_ToContiguous(PyBytes_AS_STRING(res), &view, view.len, 'C'); PyBuffer_Release(&view); return res; } @@ -511,7 +544,7 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key) if (result < 0) { result += view->shape[0]; } - if ((result < 0) || (result > view->shape[0])) { + if ((result < 0) || (result >= view->shape[0])) { PyErr_SetString(PyExc_IndexError, "index out of bounds"); return NULL; @@ -525,7 +558,7 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key) { ptr = *((char **)ptr) + view->suboffsets[0]; } - return PyByteArray_FromStringAndSize(ptr, view->itemsize); + return PyBytes_FromStringAndSize(ptr, view->itemsize); } else { /* Return a new memory-view object */ @@ -537,10 +570,46 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key) return PyMemoryView_FromBuffer(&newview); } } + else if (PySlice_Check(key)) { + Py_ssize_t start, stop, step, slicelength; - /* Need to support getting a sliced view */ - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + if (PySlice_GetIndicesEx((PySliceObject*)key, view->len, + &start, &stop, &step, &slicelength) < 0) { + return NULL; + } + + if (step == 1 && view->ndim == 1) { + Py_buffer newview; + void *newbuf = (char *) view->buf + + start * view->itemsize; + int newflags = view->readonly + ? PyBUF_CONTIG_RO : PyBUF_CONTIG; + + /* XXX There should be an API to create a subbuffer */ + if (view->obj != NULL) { + if (PyObject_GetBuffer(view->obj, + &newview, newflags) == -1) + return NULL; + } + else { + newview = *view; + } + newview.buf = newbuf; + newview.len = slicelength; + newview.format = view->format; + if (view->shape == &(view->len)) + newview.shape = &(newview.len); + if (view->strides == &(view->itemsize)) + newview.strides = &(newview.itemsize); + return PyMemoryView_FromBuffer(&newview); + } + PyErr_SetNone(PyExc_NotImplementedError); + return NULL; + } + PyErr_Format(PyExc_TypeError, + "cannot index memory using \"%.200s\"", + key->ob_type->tp_name); + return NULL; } @@ -548,9 +617,138 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key) static int memory_ass_sub(PyMemoryViewObject *self, PyObject *key, PyObject *value) { - return 0; + Py_ssize_t start, len, bytelen, i; + Py_buffer srcview; + Py_buffer *view = &(self->view); + char *srcbuf, *destbuf; + + if (view->readonly) { + PyErr_SetString(PyExc_TypeError, + "cannot modify read-only memory"); + return -1; + } + if (view->ndim != 1) { + PyErr_SetNone(PyExc_NotImplementedError); + return -1; + } + if (PyIndex_Check(key)) { + start = PyNumber_AsSsize_t(key, NULL); + if (start == -1 && PyErr_Occurred()) + return -1; + if (start < 0) { + start += view->shape[0]; + } + if ((start < 0) || (start >= view->shape[0])) { + PyErr_SetString(PyExc_IndexError, + "index out of bounds"); + return -1; + } + len = 1; + } + else if (PySlice_Check(key)) { + Py_ssize_t stop, step; + + if (PySlice_GetIndicesEx((PySliceObject*)key, view->len, + &start, &stop, &step, &len) < 0) { + return -1; + } + if (step != 1) { + PyErr_SetNone(PyExc_NotImplementedError); + return -1; + } + } + else { + PyErr_Format(PyExc_TypeError, + "cannot index memory using \"%.200s\"", + key->ob_type->tp_name); + return -1; + } + if (PyObject_GetBuffer(value, &srcview, PyBUF_CONTIG_RO) == -1) { + return -1; + } + /* XXX should we allow assignment of different item sizes + as long as the byte length is the same? + (e.g. assign 2 shorts to a 4-byte slice) */ + if (srcview.itemsize != view->itemsize) { + PyErr_Format(PyExc_TypeError, + "mismatching item sizes for \"%.200s\" and \"%.200s\"", + view->obj->ob_type->tp_name, srcview.obj->ob_type->tp_name); + goto _error; + } + if (srcview.len != len) { + PyErr_SetString(PyExc_ValueError, + "cannot modify size of memoryview object"); + goto _error; + } + /* Do the actual copy */ + destbuf = (char *) view->buf + start * view->itemsize; + srcbuf = (char *) srcview.buf; + bytelen = len * view->itemsize; + if (destbuf + bytelen < srcbuf || srcbuf + bytelen < destbuf) + /* No overlapping */ + memcpy(destbuf, srcbuf, bytelen); + else if (destbuf < srcbuf) { + /* Copy in ascending order */ + for (i = 0; i < bytelen; i++) + destbuf[i] = srcbuf[i]; + } + else { + /* Copy in descencing order */ + for (i = bytelen - 1; i >= 0; i--) + destbuf[i] = srcbuf[i]; + } + + PyBuffer_Release(&srcview); + return 0; + +_error: + PyBuffer_Release(&srcview); + return -1; } +static PyObject * +memory_richcompare(PyObject *v, PyObject *w, int op) +{ + Py_buffer vv, ww; + int equal = 0; + PyObject *res; + + vv.obj = NULL; + ww.obj = NULL; + if (op != Py_EQ && op != Py_NE) + goto _notimpl; + if (PyObject_GetBuffer(v, &vv, PyBUF_CONTIG_RO) == -1) { + PyErr_Clear(); + goto _notimpl; + } + if (PyObject_GetBuffer(w, &ww, PyBUF_CONTIG_RO) == -1) { + PyErr_Clear(); + goto _notimpl; + } + + if (vv.itemsize != ww.itemsize || vv.len != ww.len) + goto _end; + + equal = !memcmp(vv.buf, ww.buf, vv.len * vv.itemsize); + +_end: + PyBuffer_Release(&vv); + PyBuffer_Release(&ww); + if ((equal && op == Py_EQ) || (!equal && op == Py_NE)) + res = Py_True; + else + res = Py_False; + Py_INCREF(res); + return res; + +_notimpl: + PyBuffer_Release(&vv); + PyBuffer_Release(&ww); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + + /* As mapping */ static PyMappingMethods memory_as_mapping = { (lenfunc)memory_length, /*mp_length*/ @@ -591,7 +789,7 @@ PyTypeObject PyMemoryView_Type = { memory_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ - 0, /* tp_richcompare */ + memory_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */