Issue #2394: implement more of the memoryview API.

This commit is contained in:
Antoine Pitrou 2008-08-19 22:09:34 +00:00
parent 4aeec04624
commit 616d28566b
4 changed files with 418 additions and 25 deletions

View File

@ -621,7 +621,7 @@ class UTF8SigTest(ReadTest):
def test_bug1601501(self):
# SF bug #1601501: check that the codec works with a buffer
str(b"\xef\xbb\xbf", "utf-8-sig")
self.assertEquals(str(b"\xef\xbb\xbf", "utf-8-sig"), "")
def test_bom(self):
d = codecs.getincrementaldecoder("utf-8-sig")()

View File

@ -5,8 +5,166 @@ XXX We need more tests! Some tests are in test_bytes
import unittest
import test.support
import sys
class MemoryviewTest(unittest.TestCase):
class CommonMemoryTests:
#
# Tests common to direct memoryviews and sliced memoryviews
#
base_object = b"abcdef"
def check_getitem_with_type(self, tp):
b = tp(self.base_object)
oldrefcount = sys.getrefcount(b)
m = self._view(b)
self.assertEquals(m[0], b"a")
self.assert_(isinstance(m[0], bytes), type(m[0]))
self.assertEquals(m[5], b"f")
self.assertEquals(m[-1], b"f")
self.assertEquals(m[-6], b"a")
# Bounds checking
self.assertRaises(IndexError, lambda: m[6])
self.assertRaises(IndexError, lambda: m[-7])
self.assertRaises(IndexError, lambda: m[sys.maxsize])
self.assertRaises(IndexError, lambda: m[-sys.maxsize])
# Type checking
self.assertRaises(TypeError, lambda: m[None])
self.assertRaises(TypeError, lambda: m[0.0])
self.assertRaises(TypeError, lambda: m["a"])
m = None
self.assertEquals(sys.getrefcount(b), oldrefcount)
def test_getitem_readonly(self):
self.check_getitem_with_type(bytes)
def test_getitem_writable(self):
self.check_getitem_with_type(bytearray)
def test_setitem_readonly(self):
b = self.base_object
oldrefcount = sys.getrefcount(b)
m = self._view(b)
def setitem(value):
m[0] = value
self.assertRaises(TypeError, setitem, b"a")
self.assertRaises(TypeError, setitem, 65)
self.assertRaises(TypeError, setitem, memoryview(b"a"))
m = None
self.assertEquals(sys.getrefcount(b), oldrefcount)
def test_setitem_writable(self):
b = bytearray(self.base_object)
oldrefcount = sys.getrefcount(b)
m = self._view(b)
m[0] = b"0"
self._check_contents(b, b"0bcdef")
m[1:3] = b"12"
self._check_contents(b, b"012def")
m[1:1] = b""
self._check_contents(b, b"012def")
m[:] = b"abcdef"
self._check_contents(b, b"abcdef")
# Overlapping copies of a view into itself
m[0:3] = m[2:5]
self._check_contents(b, b"cdedef")
m[:] = b"abcdef"
m[2:5] = m[0:3]
self._check_contents(b, b"ababcf")
def setitem(key, value):
m[key] = value
# Bounds checking
self.assertRaises(IndexError, setitem, 6, b"a")
self.assertRaises(IndexError, setitem, -7, b"a")
self.assertRaises(IndexError, setitem, sys.maxsize, b"a")
self.assertRaises(IndexError, setitem, -sys.maxsize, b"a")
# Wrong index/slice types
self.assertRaises(TypeError, setitem, 0.0, b"a")
self.assertRaises(TypeError, setitem, (0,), b"a")
self.assertRaises(TypeError, setitem, "a", b"a")
# Trying to resize the memory object
self.assertRaises(ValueError, setitem, 0, b"")
self.assertRaises(ValueError, setitem, 0, b"ab")
self.assertRaises(ValueError, setitem, slice(1,1), b"a")
self.assertRaises(ValueError, setitem, slice(0,2), b"a")
m = None
self.assertEquals(sys.getrefcount(b), oldrefcount)
def test_len(self):
self.assertEquals(len(self._view(self.base_object)), 6)
def test_tobytes(self):
m = self._view(self.base_object)
b = m.tobytes()
self.assertEquals(b, b"abcdef")
self.assert_(isinstance(b, bytes), type(b))
def test_tolist(self):
m = self._view(self.base_object)
l = m.tolist()
self.assertEquals(l, list(b"abcdef"))
def test_compare(self):
# memoryviews can compare for equality with other objects
# having the buffer interface.
m = self._view(self.base_object)
for tp in (bytes, bytearray):
self.assertTrue(m == tp(b"abcdef"))
self.assertFalse(m != tp(b"abcdef"))
self.assertFalse(m == tp(b"abcde"))
self.assertTrue(m != tp(b"abcde"))
self.assertFalse(m == tp(b"abcde1"))
self.assertTrue(m != tp(b"abcde1"))
self.assertTrue(m == m)
self.assertTrue(m == m[:])
self.assertTrue(m[0:6] == m[:])
self.assertFalse(m[0:5] == m)
# Comparison with objects which don't support the buffer API
self.assertFalse(m == "abc")
self.assertTrue(m != "abc")
self.assertFalse("abc" == m)
self.assertTrue("abc" != m)
# Unordered comparisons
for c in (m, b"abcdef"):
self.assertRaises(TypeError, lambda: m < c)
self.assertRaises(TypeError, lambda: c <= m)
self.assertRaises(TypeError, lambda: m >= c)
self.assertRaises(TypeError, lambda: c > m)
def check_attributes_with_type(self, tp):
b = tp(self.base_object)
m = self._view(b)
self.assertEquals(m.format, 'B')
self.assertEquals(m.itemsize, 1)
self.assertEquals(m.ndim, 1)
self.assertEquals(m.shape, (6,))
self.assertEquals(m.size, 6)
self.assertEquals(m.strides, (1,))
self.assertEquals(m.suboffsets, None)
return m
def test_attributes_readonly(self):
m = self.check_attributes_with_type(bytes)
self.assertEquals(m.readonly, True)
def test_attributes_writable(self):
m = self.check_attributes_with_type(bytearray)
self.assertEquals(m.readonly, False)
class MemoryviewTest(unittest.TestCase, CommonMemoryTests):
def _view(self, obj):
return memoryview(obj)
def _check_contents(self, obj, contents):
self.assertEquals(obj, contents)
def test_constructor(self):
ob = b'test'
@ -17,8 +175,38 @@ class MemoryviewTest(unittest.TestCase):
self.assertRaises(TypeError, memoryview, argument=ob)
self.assertRaises(TypeError, memoryview, ob, argument=True)
class MemorySliceTest(unittest.TestCase, CommonMemoryTests):
base_object = b"XabcdefY"
def _view(self, obj):
m = memoryview(obj)
return m[1:7]
def _check_contents(self, obj, contents):
self.assertEquals(obj[1:7], contents)
def test_refs(self):
m = memoryview(b"ab")
oldrefcount = sys.getrefcount(m)
m[1:2]
self.assertEquals(sys.getrefcount(m), oldrefcount)
class MemorySliceSliceTest(unittest.TestCase, CommonMemoryTests):
base_object = b"XabcdefY"
def _view(self, obj):
m = memoryview(obj)
return m[:7][1:]
def _check_contents(self, obj, contents):
self.assertEquals(obj[1:7], contents)
def test_main():
test.support.run_unittest(MemoryviewTest)
test.support.run_unittest(
MemoryviewTest, MemorySliceTest, MemorySliceSliceTest)
if __name__ == "__main__":

View File

@ -12,6 +12,13 @@ What's new in Python 3.0b3?
Core and Builtins
-----------------
- Issue #2394: implement more of the memoryview API, with the caveat that
only one-dimensional contiguous buffers are supported and exercised right
now. Slicing, slice assignment and comparison (equality and inequality)
have been added. Also, the tolist() method has been implemented, but only
for byte buffers. Endly, the API has been updated to return bytes objects
wherever it used to return bytearrays.
- Issue #3560: clean up the new C PyMemoryView API so that naming is
internally consistent; add macros PyMemoryView_GET_BASE() and
PyMemoryView_GET_BUFFER() to access useful properties of a memory views

View File

@ -13,8 +13,8 @@ memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags)
}
if (self->view.obj == NULL)
return 0;
return self->view.obj->ob_type->tp_as_buffer->bf_getbuffer(self->base, NULL,
PyBUF_FULL);
return self->view.obj->ob_type->tp_as_buffer->bf_getbuffer(
self->view.obj, NULL, PyBUF_FULL);
}
static void
@ -37,9 +37,14 @@ PyMemoryView_FromBuffer(Py_buffer *info)
&PyMemoryView_Type);
if (mview == NULL) return NULL;
mview->base = NULL;
/* XXX there should be an API to duplicate a buffer object */
mview->view = *info;
if (info->obj)
Py_INCREF(mview->view.obj);
if (info->shape == &(info->len))
mview->view.shape = &(mview->view.len);
if (info->strides == &(info->itemsize))
mview->view.strides = &(mview->view.itemsize);
/* NOTE: mview->view.obj should already have been incref'ed as
part of PyBuffer_FillInfo(). */
return (PyObject *)mview;
}
@ -258,12 +263,12 @@ PyMemoryView_GetContiguous(PyObject *obj, int buffertype, char fort)
"for a non-contiguousobject.");
return NULL;
}
bytes = PyByteArray_FromStringAndSize(NULL, view->len);
bytes = PyBytes_FromStringAndSize(NULL, view->len);
if (bytes == NULL) {
PyBuffer_Release(view);
return NULL;
}
dest = PyByteArray_AS_STRING(bytes);
dest = PyBytes_AS_STRING(bytes);
/* different copying strategy depending on whether
or not any pointer de-referencing is needed
*/
@ -386,17 +391,45 @@ static PyGetSetDef memory_getsetlist[] ={
static PyObject *
memory_tobytes(PyMemoryViewObject *mem, PyObject *noargs)
{
return PyByteArray_FromObject((PyObject *)mem);
return PyObject_CallFunctionObjArgs(
(PyObject *) &PyBytes_Type, mem, NULL);
}
/* TODO: rewrite this function using the struct module to unpack
each buffer item */
static PyObject *
memory_tolist(PyMemoryViewObject *mem, PyObject *noargs)
{
/* This should construct a (nested) list of unpacked objects
possibly using the struct module.
*/
Py_INCREF(Py_NotImplemented);
return Py_NotImplemented;
Py_buffer *view = &(mem->view);
Py_ssize_t i;
PyObject *res, *item;
char *buf;
if (strcmp(view->format, "B") || view->itemsize != 1) {
PyErr_SetString(PyExc_NotImplementedError,
"tolist() only supports byte views");
return NULL;
}
if (view->ndim != 1) {
PyErr_SetString(PyExc_NotImplementedError,
"tolist() only supports one-dimensional objects");
return NULL;
}
res = PyList_New(view->len);
if (res == NULL)
return NULL;
buf = view->buf;
for (i = 0; i < view->len; i++) {
item = PyLong_FromUnsignedLong((unsigned char) *buf);
if (item == NULL) {
Py_DECREF(res);
return NULL;
}
PyList_SET_ITEM(res, i, item);
buf++;
}
return res;
}
@ -412,7 +445,7 @@ static void
memory_dealloc(PyMemoryViewObject *self)
{
if (self->view.obj != NULL) {
if (PyTuple_Check(self->base)) {
if (self->base && PyTuple_Check(self->base)) {
/* Special case when first element is generic object
with buffer interface and the second element is a
contiguous "shadow" that must be copied back into
@ -454,8 +487,8 @@ memory_str(PyMemoryViewObject *self)
if (PyObject_GetBuffer((PyObject *)self, &view, PyBUF_FULL) < 0)
return NULL;
res = PyByteArray_FromStringAndSize(NULL, view.len);
PyBuffer_ToContiguous(PyByteArray_AS_STRING(res), &view, view.len, 'C');
res = PyBytes_FromStringAndSize(NULL, view.len);
PyBuffer_ToContiguous(PyBytes_AS_STRING(res), &view, view.len, 'C');
PyBuffer_Release(&view);
return res;
}
@ -511,7 +544,7 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key)
if (result < 0) {
result += view->shape[0];
}
if ((result < 0) || (result > view->shape[0])) {
if ((result < 0) || (result >= view->shape[0])) {
PyErr_SetString(PyExc_IndexError,
"index out of bounds");
return NULL;
@ -525,7 +558,7 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key)
{
ptr = *((char **)ptr) + view->suboffsets[0];
}
return PyByteArray_FromStringAndSize(ptr, view->itemsize);
return PyBytes_FromStringAndSize(ptr, view->itemsize);
}
else {
/* Return a new memory-view object */
@ -537,10 +570,46 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key)
return PyMemoryView_FromBuffer(&newview);
}
}
else if (PySlice_Check(key)) {
Py_ssize_t start, stop, step, slicelength;
/* Need to support getting a sliced view */
Py_INCREF(Py_NotImplemented);
return Py_NotImplemented;
if (PySlice_GetIndicesEx((PySliceObject*)key, view->len,
&start, &stop, &step, &slicelength) < 0) {
return NULL;
}
if (step == 1 && view->ndim == 1) {
Py_buffer newview;
void *newbuf = (char *) view->buf
+ start * view->itemsize;
int newflags = view->readonly
? PyBUF_CONTIG_RO : PyBUF_CONTIG;
/* XXX There should be an API to create a subbuffer */
if (view->obj != NULL) {
if (PyObject_GetBuffer(view->obj,
&newview, newflags) == -1)
return NULL;
}
else {
newview = *view;
}
newview.buf = newbuf;
newview.len = slicelength;
newview.format = view->format;
if (view->shape == &(view->len))
newview.shape = &(newview.len);
if (view->strides == &(view->itemsize))
newview.strides = &(newview.itemsize);
return PyMemoryView_FromBuffer(&newview);
}
PyErr_SetNone(PyExc_NotImplementedError);
return NULL;
}
PyErr_Format(PyExc_TypeError,
"cannot index memory using \"%.200s\"",
key->ob_type->tp_name);
return NULL;
}
@ -548,8 +617,137 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key)
static int
memory_ass_sub(PyMemoryViewObject *self, PyObject *key, PyObject *value)
{
return 0;
Py_ssize_t start, len, bytelen, i;
Py_buffer srcview;
Py_buffer *view = &(self->view);
char *srcbuf, *destbuf;
if (view->readonly) {
PyErr_SetString(PyExc_TypeError,
"cannot modify read-only memory");
return -1;
}
if (view->ndim != 1) {
PyErr_SetNone(PyExc_NotImplementedError);
return -1;
}
if (PyIndex_Check(key)) {
start = PyNumber_AsSsize_t(key, NULL);
if (start == -1 && PyErr_Occurred())
return -1;
if (start < 0) {
start += view->shape[0];
}
if ((start < 0) || (start >= view->shape[0])) {
PyErr_SetString(PyExc_IndexError,
"index out of bounds");
return -1;
}
len = 1;
}
else if (PySlice_Check(key)) {
Py_ssize_t stop, step;
if (PySlice_GetIndicesEx((PySliceObject*)key, view->len,
&start, &stop, &step, &len) < 0) {
return -1;
}
if (step != 1) {
PyErr_SetNone(PyExc_NotImplementedError);
return -1;
}
}
else {
PyErr_Format(PyExc_TypeError,
"cannot index memory using \"%.200s\"",
key->ob_type->tp_name);
return -1;
}
if (PyObject_GetBuffer(value, &srcview, PyBUF_CONTIG_RO) == -1) {
return -1;
}
/* XXX should we allow assignment of different item sizes
as long as the byte length is the same?
(e.g. assign 2 shorts to a 4-byte slice) */
if (srcview.itemsize != view->itemsize) {
PyErr_Format(PyExc_TypeError,
"mismatching item sizes for \"%.200s\" and \"%.200s\"",
view->obj->ob_type->tp_name, srcview.obj->ob_type->tp_name);
goto _error;
}
if (srcview.len != len) {
PyErr_SetString(PyExc_ValueError,
"cannot modify size of memoryview object");
goto _error;
}
/* Do the actual copy */
destbuf = (char *) view->buf + start * view->itemsize;
srcbuf = (char *) srcview.buf;
bytelen = len * view->itemsize;
if (destbuf + bytelen < srcbuf || srcbuf + bytelen < destbuf)
/* No overlapping */
memcpy(destbuf, srcbuf, bytelen);
else if (destbuf < srcbuf) {
/* Copy in ascending order */
for (i = 0; i < bytelen; i++)
destbuf[i] = srcbuf[i];
}
else {
/* Copy in descencing order */
for (i = bytelen - 1; i >= 0; i--)
destbuf[i] = srcbuf[i];
}
PyBuffer_Release(&srcview);
return 0;
_error:
PyBuffer_Release(&srcview);
return -1;
}
static PyObject *
memory_richcompare(PyObject *v, PyObject *w, int op)
{
Py_buffer vv, ww;
int equal = 0;
PyObject *res;
vv.obj = NULL;
ww.obj = NULL;
if (op != Py_EQ && op != Py_NE)
goto _notimpl;
if (PyObject_GetBuffer(v, &vv, PyBUF_CONTIG_RO) == -1) {
PyErr_Clear();
goto _notimpl;
}
if (PyObject_GetBuffer(w, &ww, PyBUF_CONTIG_RO) == -1) {
PyErr_Clear();
goto _notimpl;
}
if (vv.itemsize != ww.itemsize || vv.len != ww.len)
goto _end;
equal = !memcmp(vv.buf, ww.buf, vv.len * vv.itemsize);
_end:
PyBuffer_Release(&vv);
PyBuffer_Release(&ww);
if ((equal && op == Py_EQ) || (!equal && op == Py_NE))
res = Py_True;
else
res = Py_False;
Py_INCREF(res);
return res;
_notimpl:
PyBuffer_Release(&vv);
PyBuffer_Release(&ww);
Py_INCREF(Py_NotImplemented);
return Py_NotImplemented;
}
/* As mapping */
static PyMappingMethods memory_as_mapping = {
@ -591,7 +789,7 @@ PyTypeObject PyMemoryView_Type = {
memory_doc, /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
memory_richcompare, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */