Issue #2394: implement more of the memoryview API.

Lib/test/test_codecs.py

@@ -621,7 +621,7 @@ class UTF8SigTest(ReadTest):
 
     def test_bug1601501(self):
         # SF bug #1601501: check that the codec works with a buffer
-        str(b"\xef\xbb\xbf", "utf-8-sig")
+        self.assertEquals(str(b"\xef\xbb\xbf", "utf-8-sig"), "")
 
     def test_bom(self):
         d = codecs.getincrementaldecoder("utf-8-sig")()

Lib/test/test_memoryview.py

@@ -5,8 +5,166 @@ XXX We need more tests! Some tests are in test_bytes
 
 import unittest
 import test.support
+import sys
 
-class MemoryviewTest(unittest.TestCase):
+
+class CommonMemoryTests:
+    #
+    # Tests common to direct memoryviews and sliced memoryviews
+    #
+
+    base_object = b"abcdef"
+
+    def check_getitem_with_type(self, tp):
+        b = tp(self.base_object)
+        oldrefcount = sys.getrefcount(b)
+        m = self._view(b)
+        self.assertEquals(m[0], b"a")
+        self.assert_(isinstance(m[0], bytes), type(m[0]))
+        self.assertEquals(m[5], b"f")
+        self.assertEquals(m[-1], b"f")
+        self.assertEquals(m[-6], b"a")
+        # Bounds checking
+        self.assertRaises(IndexError, lambda: m[6])
+        self.assertRaises(IndexError, lambda: m[-7])
+        self.assertRaises(IndexError, lambda: m[sys.maxsize])
+        self.assertRaises(IndexError, lambda: m[-sys.maxsize])
+        # Type checking
+        self.assertRaises(TypeError, lambda: m[None])
+        self.assertRaises(TypeError, lambda: m[0.0])
+        self.assertRaises(TypeError, lambda: m["a"])
+        m = None
+        self.assertEquals(sys.getrefcount(b), oldrefcount)
+
+    def test_getitem_readonly(self):
+        self.check_getitem_with_type(bytes)
+
+    def test_getitem_writable(self):
+        self.check_getitem_with_type(bytearray)
+
+    def test_setitem_readonly(self):
+        b = self.base_object
+        oldrefcount = sys.getrefcount(b)
+        m = self._view(b)
+        def setitem(value):
+            m[0] = value
+        self.assertRaises(TypeError, setitem, b"a")
+        self.assertRaises(TypeError, setitem, 65)
+        self.assertRaises(TypeError, setitem, memoryview(b"a"))
+        m = None
+        self.assertEquals(sys.getrefcount(b), oldrefcount)
+
+    def test_setitem_writable(self):
+        b = bytearray(self.base_object)
+        oldrefcount = sys.getrefcount(b)
+        m = self._view(b)
+        m[0] = b"0"
+        self._check_contents(b, b"0bcdef")
+        m[1:3] = b"12"
+        self._check_contents(b, b"012def")
+        m[1:1] = b""
+        self._check_contents(b, b"012def")
+        m[:] = b"abcdef"
+        self._check_contents(b, b"abcdef")
+
+        # Overlapping copies of a view into itself
+        m[0:3] = m[2:5]
+        self._check_contents(b, b"cdedef")
+        m[:] = b"abcdef"
+        m[2:5] = m[0:3]
+        self._check_contents(b, b"ababcf")
+
+        def setitem(key, value):
+            m[key] = value
+        # Bounds checking
+        self.assertRaises(IndexError, setitem, 6, b"a")
+        self.assertRaises(IndexError, setitem, -7, b"a")
+        self.assertRaises(IndexError, setitem, sys.maxsize, b"a")
+        self.assertRaises(IndexError, setitem, -sys.maxsize, b"a")
+        # Wrong index/slice types
+        self.assertRaises(TypeError, setitem, 0.0, b"a")
+        self.assertRaises(TypeError, setitem, (0,), b"a")
+        self.assertRaises(TypeError, setitem, "a", b"a")
+        # Trying to resize the memory object
+        self.assertRaises(ValueError, setitem, 0, b"")
+        self.assertRaises(ValueError, setitem, 0, b"ab")
+        self.assertRaises(ValueError, setitem, slice(1,1), b"a")
+        self.assertRaises(ValueError, setitem, slice(0,2), b"a")
+
+        m = None
+        self.assertEquals(sys.getrefcount(b), oldrefcount)
+
+    def test_len(self):
+        self.assertEquals(len(self._view(self.base_object)), 6)
+
+    def test_tobytes(self):
+        m = self._view(self.base_object)
+        b = m.tobytes()
+        self.assertEquals(b, b"abcdef")
+        self.assert_(isinstance(b, bytes), type(b))
+
+    def test_tolist(self):
+        m = self._view(self.base_object)
+        l = m.tolist()
+        self.assertEquals(l, list(b"abcdef"))
+
+    def test_compare(self):
+        # memoryviews can compare for equality with other objects
+        # having the buffer interface.
+        m = self._view(self.base_object)
+        for tp in (bytes, bytearray):
+            self.assertTrue(m == tp(b"abcdef"))
+            self.assertFalse(m != tp(b"abcdef"))
+            self.assertFalse(m == tp(b"abcde"))
+            self.assertTrue(m != tp(b"abcde"))
+            self.assertFalse(m == tp(b"abcde1"))
+            self.assertTrue(m != tp(b"abcde1"))
+        self.assertTrue(m == m)
+        self.assertTrue(m == m[:])
+        self.assertTrue(m[0:6] == m[:])
+        self.assertFalse(m[0:5] == m)
+
+        # Comparison with objects which don't support the buffer API
+        self.assertFalse(m == "abc")
+        self.assertTrue(m != "abc")
+        self.assertFalse("abc" == m)
+        self.assertTrue("abc" != m)
+
+        # Unordered comparisons
+        for c in (m, b"abcdef"):
+            self.assertRaises(TypeError, lambda: m < c)
+            self.assertRaises(TypeError, lambda: c <= m)
+            self.assertRaises(TypeError, lambda: m >= c)
+            self.assertRaises(TypeError, lambda: c > m)
+
+    def check_attributes_with_type(self, tp):
+        b = tp(self.base_object)
+        m = self._view(b)
+        self.assertEquals(m.format, 'B')
+        self.assertEquals(m.itemsize, 1)
+        self.assertEquals(m.ndim, 1)
+        self.assertEquals(m.shape, (6,))
+        self.assertEquals(m.size, 6)
+        self.assertEquals(m.strides, (1,))
+        self.assertEquals(m.suboffsets, None)
+        return m
+
+    def test_attributes_readonly(self):
+        m = self.check_attributes_with_type(bytes)
+        self.assertEquals(m.readonly, True)
+
+    def test_attributes_writable(self):
+        m = self.check_attributes_with_type(bytearray)
+        self.assertEquals(m.readonly, False)
+
+
+class MemoryviewTest(unittest.TestCase, CommonMemoryTests):
+
+    def _view(self, obj):
+        return memoryview(obj)
+
+    def _check_contents(self, obj, contents):
+        self.assertEquals(obj, contents)
 
     def test_constructor(self):
         ob = b'test'
@@ -17,8 +175,38 @@ class MemoryviewTest(unittest.TestCase):
         self.assertRaises(TypeError, memoryview, argument=ob)
         self.assertRaises(TypeError, memoryview, ob, argument=True)
 
+
+class MemorySliceTest(unittest.TestCase, CommonMemoryTests):
+    base_object = b"XabcdefY"
+
+    def _view(self, obj):
+        m = memoryview(obj)
+        return m[1:7]
+
+    def _check_contents(self, obj, contents):
+        self.assertEquals(obj[1:7], contents)
+
+    def test_refs(self):
+        m = memoryview(b"ab")
+        oldrefcount = sys.getrefcount(m)
+        m[1:2]
+        self.assertEquals(sys.getrefcount(m), oldrefcount)
+
+
+class MemorySliceSliceTest(unittest.TestCase, CommonMemoryTests):
+    base_object = b"XabcdefY"
+
+    def _view(self, obj):
+        m = memoryview(obj)
+        return m[:7][1:]
+
+    def _check_contents(self, obj, contents):
+        self.assertEquals(obj[1:7], contents)
+
+
 def test_main():
-    test.support.run_unittest(MemoryviewTest)
+    test.support.run_unittest(
+        MemoryviewTest, MemorySliceTest, MemorySliceSliceTest)
 
 
 if __name__ == "__main__":

Misc/NEWS

@@ -12,6 +12,13 @@ What's new in Python 3.0b3?
 Core and Builtins
 -----------------
 
+- Issue #2394: implement more of the memoryview API, with the caveat that
+  only one-dimensional contiguous buffers are supported and exercised right
+  now. Slicing, slice assignment and comparison (equality and inequality)
+  have been added. Also, the tolist() method has been implemented, but only
+  for byte buffers. Endly, the API has been updated to return bytes objects
+  wherever it used to return bytearrays.
+
 - Issue #3560: clean up the new C PyMemoryView API so that naming is
   internally consistent; add macros PyMemoryView_GET_BASE() and
   PyMemoryView_GET_BUFFER() to access useful properties of a memory views

Objects/memoryobject.c

@@ -13,8 +13,8 @@ memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags)
 	}
 	if (self->view.obj == NULL)
 		return 0;
-        return self->view.obj->ob_type->tp_as_buffer->bf_getbuffer(self->base, NULL,
+        return self->view.obj->ob_type->tp_as_buffer->bf_getbuffer(
-                                                               PyBUF_FULL);
+            self->view.obj, NULL, PyBUF_FULL);
 }
 
 static void
@@ -37,9 +37,14 @@ PyMemoryView_FromBuffer(Py_buffer *info)
 						   &PyMemoryView_Type);
 	if (mview == NULL) return NULL;
 	mview->base = NULL;
+        /* XXX there should be an API to duplicate a buffer object */
 	mview->view = *info;
-	if (info->obj)
+        if (info->shape == &(info->len))
-		Py_INCREF(mview->view.obj);
+            mview->view.shape = &(mview->view.len);
+        if (info->strides == &(info->itemsize))
+            mview->view.strides = &(mview->view.itemsize);
+        /* NOTE: mview->view.obj should already have been incref'ed as
+           part of PyBuffer_FillInfo(). */
 	return (PyObject *)mview;
 }
 
@@ -258,12 +263,12 @@ PyMemoryView_GetContiguous(PyObject *obj, int buffertype, char fort)
                                 "for a non-contiguousobject.");
                 return NULL;
         }
-        bytes = PyByteArray_FromStringAndSize(NULL, view->len);
+        bytes = PyBytes_FromStringAndSize(NULL, view->len);
         if (bytes == NULL) {
                 PyBuffer_Release(view);
                 return NULL;
         }
-        dest = PyByteArray_AS_STRING(bytes);
+        dest = PyBytes_AS_STRING(bytes);
         /* different copying strategy depending on whether
            or not any pointer de-referencing is needed
         */
@@ -386,17 +391,45 @@ static PyGetSetDef memory_getsetlist[] ={
 static PyObject *
 memory_tobytes(PyMemoryViewObject *mem, PyObject *noargs)
 {
-        return PyByteArray_FromObject((PyObject *)mem);
+	return PyObject_CallFunctionObjArgs(
+		(PyObject *) &PyBytes_Type, mem, NULL);
 }
 
+/* TODO: rewrite this function using the struct module to unpack
+   each buffer item */
+
 static PyObject *
 memory_tolist(PyMemoryViewObject *mem, PyObject *noargs)
 {
-	/* This should construct a (nested) list of unpacked objects
+	Py_buffer *view = &(mem->view);
-	   possibly using the struct module.
+	Py_ssize_t i;
-	 */
+	PyObject *res, *item;
-        Py_INCREF(Py_NotImplemented);
+	char *buf;
-        return Py_NotImplemented;
+
+	if (strcmp(view->format, "B") || view->itemsize != 1) {
+		PyErr_SetString(PyExc_NotImplementedError, 
+			"tolist() only supports byte views");
+		return NULL;
+	}
+	if (view->ndim != 1) {
+		PyErr_SetString(PyExc_NotImplementedError, 
+			"tolist() only supports one-dimensional objects");
+		return NULL;
+	}
+	res = PyList_New(view->len);
+	if (res == NULL)
+		return NULL;
+	buf = view->buf;
+	for (i = 0; i < view->len; i++) {
+		item = PyLong_FromUnsignedLong((unsigned char) *buf);
+		if (item == NULL) {
+			Py_DECREF(res);
+			return NULL;
+		}
+		PyList_SET_ITEM(res, i, item);
+		buf++;
+	}
+	return res;
 }
 
 
@@ -412,7 +445,7 @@ static void
 memory_dealloc(PyMemoryViewObject *self)
 {
         if (self->view.obj != NULL) {
-            if (PyTuple_Check(self->base)) {
+            if (self->base && PyTuple_Check(self->base)) {
                 /* Special case when first element is generic object
                    with buffer interface and the second element is a
                    contiguous "shadow" that must be copied back into
@@ -454,8 +487,8 @@ memory_str(PyMemoryViewObject *self)
         if (PyObject_GetBuffer((PyObject *)self, &view, PyBUF_FULL) < 0)
                 return NULL;
 
-	res = PyByteArray_FromStringAndSize(NULL, view.len);
+	res = PyBytes_FromStringAndSize(NULL, view.len);
-        PyBuffer_ToContiguous(PyByteArray_AS_STRING(res), &view, view.len, 'C');
+        PyBuffer_ToContiguous(PyBytes_AS_STRING(res), &view, view.len, 'C');
         PyBuffer_Release(&view);
         return res;
 }
@@ -511,7 +544,7 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key)
 			if (result < 0) {
 				result += view->shape[0];
 			}
-			if ((result < 0) || (result > view->shape[0])) {
+			if ((result < 0) || (result >= view->shape[0])) {
 				PyErr_SetString(PyExc_IndexError,
 						"index out of bounds");
 				return NULL;
@@ -525,7 +558,7 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key)
                         {
 				ptr = *((char **)ptr) + view->suboffsets[0];
 			}
-			return PyByteArray_FromStringAndSize(ptr, view->itemsize);
+			return PyBytes_FromStringAndSize(ptr, view->itemsize);
 		}
 		else {
 			/* Return a new memory-view object */
@@ -537,10 +570,46 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key)
 			return PyMemoryView_FromBuffer(&newview);
 		}
 	}
+	else if (PySlice_Check(key)) {
+		Py_ssize_t start, stop, step, slicelength;
 
-	/* Need to support getting a sliced view */
+		if (PySlice_GetIndicesEx((PySliceObject*)key, view->len,
-        Py_INCREF(Py_NotImplemented);
+				 &start, &stop, &step, &slicelength) < 0) {
-        return Py_NotImplemented;
+			return NULL;
+		}
+
+		if (step == 1 && view->ndim == 1) {
+			Py_buffer newview;
+			void *newbuf = (char *) view->buf
+						+ start * view->itemsize;
+			int newflags = view->readonly
+				? PyBUF_CONTIG_RO : PyBUF_CONTIG;
+
+			/* XXX There should be an API to create a subbuffer */
+			if (view->obj != NULL) {
+				if (PyObject_GetBuffer(view->obj,
+						&newview, newflags) == -1)
+					return NULL;
+			}
+			else {
+				newview = *view;
+			}
+			newview.buf = newbuf;
+			newview.len = slicelength;
+			newview.format = view->format;
+			if (view->shape == &(view->len))
+				newview.shape = &(newview.len);
+			if (view->strides == &(view->itemsize))
+				newview.strides = &(newview.itemsize);
+			return PyMemoryView_FromBuffer(&newview);
+		}
+		PyErr_SetNone(PyExc_NotImplementedError);
+		return NULL;
+	}
+	PyErr_Format(PyExc_TypeError,
+		"cannot index memory using \"%.200s\"", 
+		key->ob_type->tp_name);
+	return NULL;
 }
 
 
@@ -548,9 +617,138 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key)
 static int
 memory_ass_sub(PyMemoryViewObject *self, PyObject *key, PyObject *value)
 {
-        return 0;
+    Py_ssize_t start, len, bytelen, i;
+    Py_buffer srcview;
+    Py_buffer *view = &(self->view);
+    char *srcbuf, *destbuf;
+
+    if (view->readonly) {
+        PyErr_SetString(PyExc_TypeError,
+            "cannot modify read-only memory");
+        return -1;
+    }
+    if (view->ndim != 1) {
+        PyErr_SetNone(PyExc_NotImplementedError);
+        return -1;
+    }
+    if (PyIndex_Check(key)) {
+        start = PyNumber_AsSsize_t(key, NULL);
+        if (start == -1 && PyErr_Occurred())
+            return -1;
+        if (start < 0) {
+            start += view->shape[0];
+        }
+        if ((start < 0) || (start >= view->shape[0])) {
+            PyErr_SetString(PyExc_IndexError,
+                            "index out of bounds");
+            return -1;
+        }
+        len = 1;
+    }
+    else if (PySlice_Check(key)) {
+        Py_ssize_t stop, step;
+
+        if (PySlice_GetIndicesEx((PySliceObject*)key, view->len,
+                         &start, &stop, &step, &len) < 0) {
+            return -1;
+        }
+        if (step != 1) {
+            PyErr_SetNone(PyExc_NotImplementedError);
+            return -1;
+        }
+    }
+    else {
+        PyErr_Format(PyExc_TypeError,
+            "cannot index memory using \"%.200s\"", 
+            key->ob_type->tp_name);
+        return -1;
+    }
+    if (PyObject_GetBuffer(value, &srcview, PyBUF_CONTIG_RO) == -1) {
+        return -1;
+    }
+    /* XXX should we allow assignment of different item sizes
+       as long as the byte length is the same?
+       (e.g. assign 2 shorts to a 4-byte slice) */
+    if (srcview.itemsize != view->itemsize) {
+        PyErr_Format(PyExc_TypeError,
+            "mismatching item sizes for \"%.200s\" and \"%.200s\"", 
+            view->obj->ob_type->tp_name, srcview.obj->ob_type->tp_name);
+        goto _error;
+    }
+    if (srcview.len != len) {
+        PyErr_SetString(PyExc_ValueError,
+            "cannot modify size of memoryview object");
+        goto _error;
+    }
+    /* Do the actual copy */
+    destbuf = (char *) view->buf + start * view->itemsize;
+    srcbuf = (char *) srcview.buf;
+    bytelen = len * view->itemsize;
+    if (destbuf + bytelen < srcbuf || srcbuf + bytelen < destbuf)
+        /* No overlapping */
+        memcpy(destbuf, srcbuf, bytelen);
+    else if (destbuf < srcbuf) {
+        /* Copy in ascending order */
+        for (i = 0; i < bytelen; i++)
+            destbuf[i] = srcbuf[i];
+    }
+    else {
+        /* Copy in descencing order */
+        for (i = bytelen - 1; i >= 0; i--)
+            destbuf[i] = srcbuf[i];
+    }
+
+    PyBuffer_Release(&srcview);
+    return 0;
+
+_error:
+    PyBuffer_Release(&srcview);
+    return -1;
 }
 
+static PyObject *
+memory_richcompare(PyObject *v, PyObject *w, int op)
+{
+	Py_buffer vv, ww;
+	int equal = 0;
+	PyObject *res;
+
+	vv.obj = NULL;
+	ww.obj = NULL;
+	if (op != Py_EQ && op != Py_NE)
+		goto _notimpl;
+	if (PyObject_GetBuffer(v, &vv, PyBUF_CONTIG_RO) == -1) {
+		PyErr_Clear();
+		goto _notimpl;
+	}
+	if (PyObject_GetBuffer(w, &ww, PyBUF_CONTIG_RO) == -1) {
+		PyErr_Clear();
+		goto _notimpl;
+	}
+
+	if (vv.itemsize != ww.itemsize || vv.len != ww.len)
+		goto _end;
+
+	equal = !memcmp(vv.buf, ww.buf, vv.len * vv.itemsize);
+
+_end:
+	PyBuffer_Release(&vv);
+	PyBuffer_Release(&ww);
+	if ((equal && op == Py_EQ) || (!equal && op == Py_NE))
+		res = Py_True;
+	else
+		res = Py_False;
+	Py_INCREF(res);
+	return res;
+
+_notimpl:
+	PyBuffer_Release(&vv);
+	PyBuffer_Release(&ww);
+	Py_INCREF(Py_NotImplemented);
+	return Py_NotImplemented;
+}
+
+
 /* As mapping */
 static PyMappingMethods memory_as_mapping = {
 	(lenfunc)memory_length, /*mp_length*/
@@ -591,7 +789,7 @@ PyTypeObject PyMemoryView_Type = {
 	memory_doc,				/* tp_doc */
 	0,					/* tp_traverse */
 	0,					/* tp_clear */
-	0,		                	/* tp_richcompare */
+	memory_richcompare,                     /* tp_richcompare */
 	0,					/* tp_weaklistoffset */
 	0,					/* tp_iter */
 	0,					/* tp_iternext */