traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Issue #4580: slicing of memoryviews when itemsize != 1 is wrong. 

Also fix len() to return number of items rather than length in bytes.

I'm sorry it was not possible for me to work on this without reindenting
a bit some stuff around. The indentation in memoryobject.c is a mess,
I'll open a separate bug for it.
This commit is contained in:
Antoine Pitrou 2009-01-03 16:59:18 +00:00
parent 8bcddcabd7
commit c3b39245a7
8 changed files with 297 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
Include/object.h
View File

@ -144,16 +144,18 @@ typedef int(*objobjargproc)(PyObject *, PyObject *, PyObject *);
typedef struct bufferinfo {
void *buf;
PyObject *obj; /* owned reference */
Py_ssize_t len;
Py_ssize_t itemsize; /* This is Py_ssize_t so it can be
pointed to by strides in simple case.*/
int readonly;
int ndim;
char *format;
Py_ssize_t *shape;
Py_ssize_t *strides;
Py_ssize_t *suboffsets;
void *internal;
Py_ssize_t len;
Py_ssize_t itemsize; /* This is Py_ssize_t so it can be
pointed to by strides in simple case.*/
int readonly;
int ndim;
char *format;
Py_ssize_t *shape;
Py_ssize_t *strides;
Py_ssize_t *suboffsets;
Py_ssize_t smalltable[2]; /* static store for shape and strides of
mono-dimensional buffers. */
void *internal;
} Py_buffer;
typedef int (*getbufferproc)(PyObject *, Py_buffer *, int);

14
Lib/ctypes/test/test_pep3118.py
View File

@ -25,7 +25,10 @@ class Test(unittest.TestCase):
v = memoryview(ob)
try:
self.failUnlessEqual(normalize(v.format), normalize(fmt))
self.failUnlessEqual(len(v), sizeof(ob))
if shape is not None:
self.failUnlessEqual(len(v), shape[0])
else:
self.failUnlessEqual(len(v) * sizeof(itemtp), sizeof(ob))
self.failUnlessEqual(v.itemsize, sizeof(itemtp))
self.failUnlessEqual(v.shape, shape)
# ctypes object always have a non-strided memory block
@ -37,7 +40,7 @@ class Test(unittest.TestCase):
n = 1
for dim in v.shape:
n = n * dim
self.failUnlessEqual(v.itemsize * n, len(v))
self.failUnlessEqual(n * v.itemsize, len(v.tobytes()))
except:
# so that we can see the failing type
print(tp)
@ -49,7 +52,10 @@ class Test(unittest.TestCase):
v = memoryview(ob)
try:
self.failUnlessEqual(v.format, fmt)
self.failUnlessEqual(len(v), sizeof(ob))
if shape is not None:
self.failUnlessEqual(len(v), shape[0])
else:
self.failUnlessEqual(len(v) * sizeof(itemtp), sizeof(ob))
self.failUnlessEqual(v.itemsize, sizeof(itemtp))
self.failUnlessEqual(v.shape, shape)
# ctypes object always have a non-strided memory block
@ -61,7 +67,7 @@ class Test(unittest.TestCase):
n = 1
for dim in v.shape:
n = n * dim
self.failUnlessEqual(v.itemsize * n, len(v))
self.failUnlessEqual(n, len(v))
except:
# so that we can see the failing type
print(tp)

2
Lib/test/test_io.py
View File

@ -260,7 +260,7 @@ class IOTest(unittest.TestCase):
def test_array_writes(self):
a = array.array('i', range(10))
n = len(memoryview(a))
n = len(a.tostring())
f = io.open(support.TESTFN, "wb", 0)
self.assertEqual(f.write(a), n)
f.close()

336
Lib/test/test_memoryview.py
View File

@ -8,24 +8,30 @@ import test.support
import sys
import gc
import weakref
import array
class CommonMemoryTests:
#
# Tests common to direct memoryviews and sliced memoryviews
#
class AbstractMemoryTests:
source_bytes = b"abcdef"
base_object = b"abcdef"
@property
def _source(self):
return self.source_bytes
@property
def _types(self):
return filter(None, [self.ro_type, self.rw_type])
def check_getitem_with_type(self, tp):
b = tp(self.base_object)
item = self.getitem_type
b = tp(self._source)
oldrefcount = sys.getrefcount(b)
m = self._view(b)
self.assertEquals(m[0], b"a")
self.assertEquals(m[0], item(b"a"))
self.assert_(isinstance(m[0], bytes), type(m[0]))
self.assertEquals(m[5], b"f")
self.assertEquals(m[-1], b"f")
self.assertEquals(m[-6], b"a")
self.assertEquals(m[5], item(b"f"))
self.assertEquals(m[-1], item(b"f"))
self.assertEquals(m[-6], item(b"a"))
# Bounds checking
self.assertRaises(IndexError, lambda: m[6])
self.assertRaises(IndexError, lambda: m[-7])
@ -38,14 +44,14 @@ class CommonMemoryTests:
m = None
self.assertEquals(sys.getrefcount(b), oldrefcount)
def test_getitem_readonly(self):
self.check_getitem_with_type(bytes)
def test_getitem_writable(self):
self.check_getitem_with_type(bytearray)
def test_getitem(self):
for tp in self._types:
self.check_getitem_with_type(tp)
def test_setitem_readonly(self):
b = self.base_object
if not self.ro_type:
return
b = self.ro_type(self._source)
oldrefcount = sys.getrefcount(b)
m = self._view(b)
def setitem(value):
@ -57,27 +63,30 @@ class CommonMemoryTests:
self.assertEquals(sys.getrefcount(b), oldrefcount)
def test_setitem_writable(self):
b = bytearray(self.base_object)
if not self.rw_type:
return
tp = self.rw_type
b = self.rw_type(self._source)
oldrefcount = sys.getrefcount(b)
m = self._view(b)
m[0] = b"0"
self._check_contents(b, b"0bcdef")
m[1:3] = b"12"
self._check_contents(b, b"012def")
m[1:1] = b""
self._check_contents(b, b"012def")
m[:] = b"abcdef"
self._check_contents(b, b"abcdef")
m[0] = tp(b"0")
self._check_contents(tp, b, b"0bcdef")
m[1:3] = tp(b"12")
self._check_contents(tp, b, b"012def")
m[1:1] = tp(b"")
self._check_contents(tp, b, b"012def")
m[:] = tp(b"abcdef")
self._check_contents(tp, b, b"abcdef")
# Overlapping copies of a view into itself
m[0:3] = m[2:5]
self._check_contents(b, b"cdedef")
m[:] = b"abcdef"
self._check_contents(tp, b, b"cdedef")
m[:] = tp(b"abcdef")
m[2:5] = m[0:3]
self._check_contents(b, b"ababcf")
self._check_contents(tp, b, b"ababcf")
def setitem(key, value):
m[key] = value
m[key] = tp(value)
# Bounds checking
self.assertRaises(IndexError, setitem, 6, b"a")
self.assertRaises(IndexError, setitem, -7, b"a")
@ -96,159 +105,224 @@ class CommonMemoryTests:
m = None
self.assertEquals(sys.getrefcount(b), oldrefcount)
def test_len(self):
self.assertEquals(len(self._view(self.base_object)), 6)
def test_tobytes(self):
m = self._view(self.base_object)
b = m.tobytes()
self.assertEquals(b, b"abcdef")
self.assert_(isinstance(b, bytes), type(b))
for tp in self._types:
m = self._view(tp(self._source))
b = m.tobytes()
# This calls self.getitem_type() on each separate byte of b"abcdef"
expected = b"".join(
self.getitem_type(bytes([c])) for c in b"abcdef")
self.assertEquals(b, expected)
self.assert_(isinstance(b, bytes), type(b))
def test_tolist(self):
m = self._view(self.base_object)
l = m.tolist()
self.assertEquals(l, list(b"abcdef"))
for tp in self._types:
m = self._view(tp(self._source))
l = m.tolist()
self.assertEquals(l, list(b"abcdef"))
def test_compare(self):
# memoryviews can compare for equality with other objects
# having the buffer interface.
m = self._view(self.base_object)
for tp in (bytes, bytearray):
self.assertTrue(m == tp(b"abcdef"))
self.assertFalse(m != tp(b"abcdef"))
self.assertFalse(m == tp(b"abcde"))
self.assertTrue(m != tp(b"abcde"))
self.assertFalse(m == tp(b"abcde1"))
self.assertTrue(m != tp(b"abcde1"))
self.assertTrue(m == m)
self.assertTrue(m == m[:])
self.assertTrue(m[0:6] == m[:])
self.assertFalse(m[0:5] == m)
for tp in self._types:
m = self._view(tp(self._source))
for tp_comp in self._types:
self.assertTrue(m == tp_comp(b"abcdef"))
self.assertFalse(m != tp_comp(b"abcdef"))
self.assertFalse(m == tp_comp(b"abcde"))
self.assertTrue(m != tp_comp(b"abcde"))
self.assertFalse(m == tp_comp(b"abcde1"))
self.assertTrue(m != tp_comp(b"abcde1"))
self.assertTrue(m == m)
self.assertTrue(m == m[:])
self.assertTrue(m[0:6] == m[:])
self.assertFalse(m[0:5] == m)
# Comparison with objects which don't support the buffer API
self.assertFalse(m == "abc")
self.assertTrue(m != "abc")
self.assertFalse("abc" == m)
self.assertTrue("abc" != m)
# Comparison with objects which don't support the buffer API
self.assertFalse(m == "abcdef")
self.assertTrue(m != "abcdef")
self.assertFalse("abcdef" == m)
self.assertTrue("abcdef" != m)
# Unordered comparisons
for c in (m, b"abcdef"):
self.assertRaises(TypeError, lambda: m < c)
self.assertRaises(TypeError, lambda: c <= m)
self.assertRaises(TypeError, lambda: m >= c)
self.assertRaises(TypeError, lambda: c > m)
# Unordered comparisons
for c in (m, b"abcdef"):
self.assertRaises(TypeError, lambda: m < c)
self.assertRaises(TypeError, lambda: c <= m)
self.assertRaises(TypeError, lambda: m >= c)
self.assertRaises(TypeError, lambda: c > m)
def check_attributes_with_type(self, tp):
b = tp(self.base_object)
m = self._view(b)
self.assertEquals(m.format, 'B')
self.assertEquals(m.itemsize, 1)
m = self._view(tp(self._source))
self.assertEquals(m.format, self.format)
self.assertEquals(m.itemsize, self.itemsize)
self.assertEquals(m.ndim, 1)
self.assertEquals(m.shape, (6,))
self.assertEquals(len(m), 6)
self.assertEquals(m.strides, (1,))
self.assertEquals(m.strides, (self.itemsize,))
self.assertEquals(m.suboffsets, None)
return m
def test_attributes_readonly(self):
m = self.check_attributes_with_type(bytes)
if not self.ro_type:
return
m = self.check_attributes_with_type(self.ro_type)
self.assertEquals(m.readonly, True)
def test_attributes_writable(self):
m = self.check_attributes_with_type(bytearray)
if not self.rw_type:
return
m = self.check_attributes_with_type(self.rw_type)
self.assertEquals(m.readonly, False)
def test_getbuffer(self):
# Test PyObject_GetBuffer() on a memoryview object.
b = self.base_object
oldrefcount = sys.getrefcount(b)
m = self._view(b)
oldviewrefcount = sys.getrefcount(m)
s = str(m, "utf-8")
self._check_contents(b, s.encode("utf-8"))
self.assertEquals(sys.getrefcount(m), oldviewrefcount)
m = None
self.assertEquals(sys.getrefcount(b), oldrefcount)
for tp in self._types:
b = tp(self._source)
oldrefcount = sys.getrefcount(b)
m = self._view(b)
oldviewrefcount = sys.getrefcount(m)
s = str(m, "utf-8")
self._check_contents(tp, b, s.encode("utf-8"))
self.assertEquals(sys.getrefcount(m), oldviewrefcount)
m = None
self.assertEquals(sys.getrefcount(b), oldrefcount)
def test_gc(self):
class MyBytes(bytes):
pass
class MyObject:
pass
for tp in self._types:
if not isinstance(tp, type):
# If tp is a factory rather than a plain type, skip
continue
# Create a reference cycle through a memoryview object
b = MyBytes(b'abc')
m = self._view(b)
o = MyObject()
b.m = m
b.o = o
wr = weakref.ref(o)
b = m = o = None
# The cycle must be broken
gc.collect()
self.assert_(wr() is None, wr())
class MySource(tp):
pass
class MyObject:
pass
# Create a reference cycle through a memoryview object
b = MySource(tp(b'abc'))
m = self._view(b)
o = MyObject()
b.m = m
b.o = o
wr = weakref.ref(o)
b = m = o = None
# The cycle must be broken
gc.collect()
self.assert_(wr() is None, wr())
class MemoryviewTest(unittest.TestCase, CommonMemoryTests):
# Variations on source objects for the buffer: bytes-like objects, then arrays
# with itemsize > 1.
# NOTE: support for multi-dimensional objects is unimplemented.
class BaseBytesMemoryTests(AbstractMemoryTests):
ro_type = bytes
rw_type = bytearray
getitem_type = bytes
itemsize = 1
format = 'B'
class BaseArrayMemoryTests(AbstractMemoryTests):
ro_type = None
rw_type = lambda self, b: array.array('i', list(b))
getitem_type = lambda self, b: array.array('i', list(b)).tostring()
itemsize = array.array('i').itemsize
format = 'i'
def test_getbuffer(self):
# XXX Test should be adapted for non-byte buffers
pass
def test_tolist(self):
# XXX NotImplementedError: tolist() only supports byte views
pass
# Variations on indirection levels: memoryview, slice of memoryview,
# slice of slice of memoryview.
# This is important to test allocation subtleties.
class BaseMemoryviewTests:
def _view(self, obj):
return memoryview(obj)
def _check_contents(self, obj, contents):
self.assertEquals(obj, contents)
def _check_contents(self, tp, obj, contents):
self.assertEquals(obj, tp(contents))
def test_constructor(self):
ob = b'test'
self.assert_(memoryview(ob))
self.assert_(memoryview(object=ob))
self.assertRaises(TypeError, memoryview)
self.assertRaises(TypeError, memoryview, ob, ob)
self.assertRaises(TypeError, memoryview, argument=ob)
self.assertRaises(TypeError, memoryview, ob, argument=True)
def test_array_assign(self):
# Issue #4569: segfault when mutating a memoryview with itemsize != 1
from array import array
a = array('i', range(10))
m = memoryview(a)
new_a = array('i', range(9, -1, -1))
m[:] = new_a
self.assertEquals(a, new_a)
class MemorySliceTest(unittest.TestCase, CommonMemoryTests):
base_object = b"XabcdefY"
class BaseMemorySliceTests:
source_bytes = b"XabcdefY"
def _view(self, obj):
m = memoryview(obj)
return m[1:7]
def _check_contents(self, obj, contents):
self.assertEquals(obj[1:7], contents)
def _check_contents(self, tp, obj, contents):
self.assertEquals(obj[1:7], tp(contents))
def test_refs(self):
m = memoryview(b"ab")
oldrefcount = sys.getrefcount(m)
m[1:2]
self.assertEquals(sys.getrefcount(m), oldrefcount)
for tp in self._types:
m = memoryview(tp(self._source))
oldrefcount = sys.getrefcount(m)
m[1:2]
self.assertEquals(sys.getrefcount(m), oldrefcount)
class MemorySliceSliceTest(unittest.TestCase, CommonMemoryTests):
base_object = b"XabcdefY"
class BaseMemorySliceSliceTests:
source_bytes = b"XabcdefY"
def _view(self, obj):
m = memoryview(obj)
return m[:7][1:]
def _check_contents(self, obj, contents):
self.assertEquals(obj[1:7], contents)
def _check_contents(self, tp, obj, contents):
self.assertEquals(obj[1:7], tp(contents))
# Concrete test classes
class BytesMemoryviewTest(unittest.TestCase,
BaseMemoryviewTests, BaseBytesMemoryTests):
def test_constructor(self):
for tp in self._types:
ob = tp(self._source)
self.assert_(memoryview(ob))
self.assert_(memoryview(object=ob))
self.assertRaises(TypeError, memoryview)
self.assertRaises(TypeError, memoryview, ob, ob)
self.assertRaises(TypeError, memoryview, argument=ob)
self.assertRaises(TypeError, memoryview, ob, argument=True)
class ArrayMemoryviewTest(unittest.TestCase,
BaseMemoryviewTests, BaseArrayMemoryTests):
def test_array_assign(self):
# Issue #4569: segfault when mutating a memoryview with itemsize != 1
a = array.array('i', range(10))
m = memoryview(a)
new_a = array.array('i', range(9, -1, -1))
m[:] = new_a
self.assertEquals(a, new_a)
class BytesMemorySliceTest(unittest.TestCase,
BaseMemorySliceTests, BaseBytesMemoryTests):
pass
class ArrayMemorySliceTest(unittest.TestCase,
BaseMemorySliceTests, BaseArrayMemoryTests):
pass
class BytesMemorySliceSliceTest(unittest.TestCase,
BaseMemorySliceSliceTests, BaseBytesMemoryTests):
pass
class ArrayMemorySliceSliceTest(unittest.TestCase,
BaseMemorySliceSliceTests, BaseArrayMemoryTests):
pass
def test_main():
test.support.run_unittest(
MemoryviewTest, MemorySliceTest, MemorySliceSliceTest)
test.support.run_unittest(__name__)
if __name__ == "__main__":
test_main()

2
Lib/test/test_sys.py
View File

@ -559,7 +559,7 @@ class SizeofTest(unittest.TestCase):
check(32768*32768-1, size(vh) + 2*self.H)
check(32768*32768, size(vh) + 3*self.H)
# memory
check(memoryview(b''), size(h + 'P PP2P2i5P'))
check(memoryview(b''), size(h + 'P PP2P2i7P'))
# module
check(unittest, size(h + '3P'))
# None

4
Misc/NEWS
View File

@ -12,6 +12,10 @@ What's New in Python 3.1 alpha 0
Core and Builtins
-----------------
- Issue #4580: Fix slicing of memoryviews when the item size is greater than
one byte. Also fixes the meaning of len() so that it returns the number of
items, rather than the size in bytes.
- Issue #4075: Use OutputDebugStringW in Py_FatalError.
- Issue #4747: When the terminal does not use utf-8, executing a script with

131
Objects/memoryobject.c
View File

@ -3,26 +3,31 @@
#include "Python.h"
static void
dup_buffer(Py_buffer *dest, Py_buffer *src)
{
*dest = *src;
if (src->shape == &(src->len))
dest->shape = &(dest->len);
if (src->strides == &(src->itemsize))
dest->strides = &(dest->itemsize);
}
/* XXX The buffer API should mandate that the shape array be non-NULL, but
it would complicate some code since the (de)allocation semantics of shape
are not specified. */
static Py_ssize_t
get_shape0(Py_buffer *buf)
{
if (buf->shape != NULL)
return buf->shape[0];
assert(buf->ndim == 1 && buf->itemsize > 0);
return buf->len / buf->itemsize;
if (buf->ndim == 0)
return 1;
PyErr_SetString(PyExc_TypeError,
"exported buffer does not have any shape information associated "
"to it");
return -1;
}
static void
dup_buffer(Py_buffer *dest, Py_buffer *src)
{
*dest = *src;
if (src->ndim == 1 && src->shape != NULL) {
dest->shape = &(dest->smalltable[0]);
dest->shape[0] = get_shape0(src);
}
if (src->ndim == 1 && src->strides != NULL) {
dest->strides = &(dest->smalltable[1]);
dest->strides[0] = src->strides[0];
}
}
static int
@ -449,8 +454,6 @@ memory_tolist(PyMemoryViewObject *mem, PyObject *noargs)
return res;
}
static PyMethodDef memory_methods[] = {
{"tobytes", (PyCFunction)memory_tobytes, METH_NOARGS, NULL},
{"tolist", (PyCFunction)memory_tolist, METH_NOARGS, NULL},
@ -474,19 +477,19 @@ memory_dealloc(PyMemoryViewObject *self)
PyObject_CopyData(PyTuple_GET_ITEM(self->base,0),
PyTuple_GET_ITEM(self->base,1));
/* The view member should have readonly == -1 in
this instance indicating that the memory can
be "locked" and was locked and will be unlocked
again after this call.
*/
PyBuffer_Release(&(self->view));
}
else {
PyBuffer_Release(&(self->view));
}
Py_CLEAR(self->base);
/* The view member should have readonly == -1 in
this instance indicating that the memory can
be "locked" and was locked and will be unlocked
again after this call.
*/
PyBuffer_Release(&(self->view));
}
PyObject_GC_Del(self);
else {
PyBuffer_Release(&(self->view));
}
Py_CLEAR(self->base);
}
PyObject_GC_Del(self);
}
static PyObject *
@ -512,16 +515,10 @@ memory_str(PyMemoryViewObject *self)
}
/* Sequence methods */
static Py_ssize_t
memory_length(PyMemoryViewObject *self)
{
Py_buffer view;
if (PyObject_GetBuffer((PyObject *)self, &view, PyBUF_FULL) < 0)
return -1;
PyBuffer_Release(&view);
return view.len;
return get_shape0(&self->view);
}
/*
@ -589,40 +586,38 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key)
}
}
else if (PySlice_Check(key)) {
Py_ssize_t start, stop, step, slicelength;
Py_ssize_t start, stop, step, slicelength;
if (PySlice_GetIndicesEx((PySliceObject*)key, get_shape0(view),
&start, &stop, &step, &slicelength) < 0) {
return NULL;
}
&start, &stop, &step, &slicelength) < 0) {
return NULL;
}
if (step == 1 && view->ndim == 1) {
Py_buffer newview;
void *newbuf = (char *) view->buf
+ start * view->itemsize;
int newflags = view->readonly
? PyBUF_CONTIG_RO : PyBUF_CONTIG;
if (step == 1 && view->ndim == 1) {
Py_buffer newview;
void *newbuf = (char *) view->buf
+ start * view->itemsize;
int newflags = view->readonly
? PyBUF_CONTIG_RO : PyBUF_CONTIG;
/* XXX There should be an API to create a subbuffer */
if (view->obj != NULL) {
if (PyObject_GetBuffer(view->obj,
&newview, newflags) == -1)
return NULL;
}
else {
newview = *view;
}
newview.buf = newbuf;
newview.len = slicelength;
newview.format = view->format;
if (view->shape == &(view->len))
newview.shape = &(newview.len);
if (view->strides == &(view->itemsize))
newview.strides = &(newview.itemsize);
return PyMemoryView_FromBuffer(&newview);
}
PyErr_SetNone(PyExc_NotImplementedError);
return NULL;
/* XXX There should be an API to create a subbuffer */
if (view->obj != NULL) {
if (PyObject_GetBuffer(view->obj, &newview, newflags) == -1)
return NULL;
}
else {
newview = *view;
}
newview.buf = newbuf;
newview.len = slicelength * newview.itemsize;
newview.format = view->format;
newview.shape = &(newview.smalltable[0]);
newview.shape[0] = slicelength;
newview.strides = &(newview.itemsize);
return PyMemoryView_FromBuffer(&newview);
}
PyErr_SetNone(PyExc_NotImplementedError);
return NULL;
}
PyErr_Format(PyExc_TypeError,
"cannot index memory using \"%.200s\"",
@ -747,7 +742,7 @@ memory_richcompare(PyObject *v, PyObject *w, int op)
if (vv.itemsize != ww.itemsize || vv.len != ww.len)
goto _end;
equal = !memcmp(vv.buf, ww.buf, vv.len * vv.itemsize);
equal = !memcmp(vv.buf, ww.buf, vv.len);
_end:
PyBuffer_Release(&vv);

2
Objects/unicodeobject.c
View File

@ -1205,7 +1205,7 @@ PyObject *PyUnicode_Decode(const char *s,
/* Decode via the codec registry */
buffer = NULL;
if (PyBuffer_FillInfo(&info, NULL, (void *)s, size, 1, PyBUF_SIMPLE) < 0)
if (PyBuffer_FillInfo(&info, NULL, (void *)s, size, 1, PyBUF_FULL_RO) < 0)
goto onError;
buffer = PyMemoryView_FromBuffer(&info);
if (buffer == NULL)