traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

merge 3.5

This commit is contained in:
Benjamin Peterson 2016-10-05 22:54:19 -07:00
parent a88614233e 4c8b2cd126
commit 87845bcb4d
3 changed files with 97 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Lib/test/test_mmap.py
View File

@ -720,6 +720,20 @@ class MmapTests(unittest.TestCase):
self.assertEqual(mm.write(b"yz"), 2)
self.assertEqual(mm.write(b"python"), 6)
@unittest.skipIf(os.name == 'nt', 'cannot resize anonymous mmaps on Windows')
def test_resize_past_pos(self):
m = mmap.mmap(-1, 8192)
self.addCleanup(m.close)
m.read(5000)
try:
m.resize(4096)
except SystemError:
self.skipTest("resizing not supported")
self.assertEqual(m.read(14), b'')
self.assertRaises(ValueError, m.read_byte)
self.assertRaises(ValueError, m.write_byte, 42)
self.assertRaises(ValueError, m.write, b'abc')
class LargeMmapTests(unittest.TestCase):

3
Misc/NEWS
View File

@ -404,6 +404,9 @@ Library
pickling and text representation purposes. Patch by Emanuel Barry and
Serhiy Storchaka.
- Fix possible integer overflows and crashes in the mmap module with unusual
usage patterns.
- Issue #1703178: Fix the ability to pass the --link-objects option to the
distutils build_ext command.

193
Modules/mmapmodule.c
View File

@ -90,8 +90,8 @@ typedef enum
typedef struct {
PyObject_HEAD
char * data;
size_t size;
size_t pos; /* relative to offset */
Py_ssize_t size;
Py_ssize_t pos; /* relative to offset */
#ifdef MS_WINDOWS
long long offset;
#else
@ -210,33 +210,32 @@ mmap_read_byte_method(mmap_object *self,
PyObject *unused)
{
CHECK_VALID(NULL);
if (self->pos < self->size) {
char value = self->data[self->pos];
self->pos += 1;
return Py_BuildValue("B", (unsigned char)value);
} else {
if (self->pos >= self->size) {
PyErr_SetString(PyExc_ValueError, "read byte out of range");
return NULL;
}
return PyLong_FromLong((unsigned char)self->data[self->pos++]);
}
static PyObject *
mmap_read_line_method(mmap_object *self,
PyObject *unused)
{
char *start = self->data+self->pos;
char *eof = self->data+self->size;
char *eol;
Py_ssize_t remaining;
char *start, *eol;
PyObject *result;
CHECK_VALID(NULL);
eol = memchr(start, '\n', self->size - self->pos);
remaining = (self->pos < self->size) ? self->size - self->pos : 0;
if (!remaining)
return PyBytes_FromString("");
start = self->data + self->pos;
eol = memchr(start, '\n', remaining);
if (!eol)
eol = eof;
eol = self->data + self->size;
else
++eol; /* we're interested in the position after the
newline. */
++eol; /* advance past newline */
result = PyBytes_FromStringAndSize(start, (eol - start));
self->pos += (eol - start);
return result;
@ -268,7 +267,7 @@ static PyObject *
mmap_read_method(mmap_object *self,
PyObject *args)
{
Py_ssize_t num_bytes = -1, n;
Py_ssize_t num_bytes, remaining;
PyObject *result;
CHECK_VALID(NULL);
@ -276,20 +275,10 @@ mmap_read_method(mmap_object *self,
return(NULL);
/* silently 'adjust' out-of-range requests */
assert(self->size >= self->pos);
n = self->size - self->pos;
/* The difference can overflow, only if self->size is greater than
* PY_SSIZE_T_MAX. But then the operation cannot possibly succeed,
* because the mapped area and the returned string each need more
* than half of the addressable memory. So we clip the size, and let
* the code below raise MemoryError.
*/
if (n < 0)
n = PY_SSIZE_T_MAX;
if (num_bytes < 0 || num_bytes > n) {
num_bytes = n;
}
result = PyBytes_FromStringAndSize(self->data+self->pos, num_bytes);
remaining = (self->pos < self->size) ? self->size - self->pos : 0;
if (num_bytes < 0 || num_bytes > remaining)
num_bytes = remaining;
result = PyBytes_FromStringAndSize(&self->data[self->pos], num_bytes);
self->pos += num_bytes;
return result;
}
@ -317,14 +306,14 @@ mmap_gfind(mmap_object *self,
start += self->size;
if (start < 0)
start = 0;
else if ((size_t)start > self->size)
else if (start > self->size)
start = self->size;
if (end < 0)
end += self->size;
if (end < 0)
end = 0;
else if ((size_t)end > self->size)
else if (end > self->size)
end = self->size;
start_p = self->data + start;
@ -389,27 +378,24 @@ mmap_write_method(mmap_object *self,
PyObject *args)
{
Py_buffer data;
PyObject *result;
CHECK_VALID(NULL);
if (!PyArg_ParseTuple(args, "y*:write", &data))
return(NULL);
if (!is_writable(self)) {
if (!is_writable(self))
return NULL;
if (self->pos > self->size || self->size - self->pos < data.len) {
PyBuffer_Release(&data);
PyErr_SetString(PyExc_ValueError, "data out of range");
return NULL;
}
if ((self->pos + data.len) > self->size) {
PyErr_SetString(PyExc_ValueError, "data out of range");
PyBuffer_Release(&data);
return NULL;
}
memcpy(self->data + self->pos, data.buf, data.len);
self->pos = self->pos + data.len;
result = PyLong_FromSsize_t(data.len);
memcpy(&self->data[self->pos], data.buf, data.len);
self->pos += data.len;
PyBuffer_Release(&data);
return result;
return PyLong_FromSsize_t(data.len);
}
static PyObject *
@ -426,8 +412,7 @@ mmap_write_byte_method(mmap_object *self,
return NULL;
if (self->pos < self->size) {
*(self->data+self->pos) = value;
self->pos += 1;
self->data[self->pos++] = value;
Py_INCREF(Py_None);
return Py_None;
}
@ -496,8 +481,14 @@ mmap_resize_method(mmap_object *self,
if (!PyArg_ParseTuple(args, "n:resize", &new_size) ||
!is_resizeable(self)) {
return NULL;
}
if (new_size < 0 || PY_SSIZE_T_MAX - new_size < self->offset) {
PyErr_SetString(PyExc_ValueError, "new size out of range");
return NULL;
}
{
#ifdef MS_WINDOWS
} else {
DWORD dwErrCode = 0;
DWORD off_hi, off_lo, newSizeLow, newSizeHigh;
/* First, unmap the file view */
@ -547,15 +538,13 @@ mmap_resize_method(mmap_object *self,
#ifdef UNIX
#ifndef HAVE_MREMAP
} else {
PyErr_SetString(PyExc_SystemError,
"mmap: resizing not available--no mremap()");
return NULL;
#else
} else {
void *newmap;
if (ftruncate(self->fd, self->offset + new_size) == -1) {
if (self->fd != -1 && ftruncate(self->fd, self->offset + new_size) == -1) {
PyErr_SetFromErrno(PyExc_OSError);
return NULL;
}
@ -563,11 +552,11 @@ mmap_resize_method(mmap_object *self,
#ifdef MREMAP_MAYMOVE
newmap = mremap(self->data, self->size, new_size, MREMAP_MAYMOVE);
#else
#if defined(__NetBSD__)
newmap = mremap(self->data, self->size, self->data, new_size, 0);
#else
newmap = mremap(self->data, self->size, new_size, 0);
#endif /* __NetBSD__ */
#if defined(__NetBSD__)
newmap = mremap(self->data, self->size, self->data, new_size, 0);
#else
newmap = mremap(self->data, self->size, new_size, 0);
#endif /* __NetBSD__ */
#endif
if (newmap == (void *)-1)
{
@ -598,7 +587,7 @@ mmap_flush_method(mmap_object *self, PyObject *args)
CHECK_VALID(NULL);
if (!PyArg_ParseTuple(args, "|nn:flush", &offset, &size))
return NULL;
if ((size_t)(offset + size) > self->size) {
if (size < 0 || offset < 0 || self->size - offset < size) {
PyErr_SetString(PyExc_ValueError, "flush values out of range");
return NULL;
}
@ -631,20 +620,18 @@ mmap_seek_method(mmap_object *self, PyObject *args)
if (!PyArg_ParseTuple(args, "n|i:seek", &dist, &how))
return NULL;
else {
size_t where;
Py_ssize_t where;
switch (how) {
case 0: /* relative to start */
if (dist < 0)
goto onoutofrange;
where = dist;
break;
case 1: /* relative to current position */
if ((Py_ssize_t)self->pos + dist < 0)
if (PY_SSIZE_T_MAX - self->pos < dist)
goto onoutofrange;
where = self->pos + dist;
break;
case 2: /* relative to end */
if ((Py_ssize_t)self->size + dist < 0)
if (PY_SSIZE_T_MAX - self->size < dist)
goto onoutofrange;
where = self->size + dist;
break;
@ -652,7 +639,7 @@ mmap_seek_method(mmap_object *self, PyObject *args)
PyErr_SetString(PyExc_ValueError, "unknown seek type");
return NULL;
}
if (where > self->size)
if (where > self->size || where < 0)
goto onoutofrange;
self->pos = where;
Py_INCREF(Py_None);
@ -667,23 +654,27 @@ mmap_seek_method(mmap_object *self, PyObject *args)
static PyObject *
mmap_move_method(mmap_object *self, PyObject *args)
{
unsigned long dest, src, cnt;
Py_ssize_t dest, src, cnt;
CHECK_VALID(NULL);
if (!PyArg_ParseTuple(args, "kkk:move", &dest, &src, &cnt) ||
if (!PyArg_ParseTuple(args, "nnn:move", &dest, &src, &cnt) ||
!is_writable(self)) {
return NULL;
} else {
/* bounds check the values */
if ((cnt + dest) < cnt || (cnt + src) < cnt ||
src > self->size || (src + cnt) > self->size ||
dest > self->size || (dest + cnt) > self->size) {
PyErr_SetString(PyExc_ValueError,
"source, destination, or count out of range");
return NULL;
}
memmove(self->data+dest, self->data+src, cnt);
if (dest < 0 || src < 0 || cnt < 0)
goto bounds;
if (self->size - dest < cnt || self->size - src < cnt)
goto bounds;
memmove(&self->data[dest], &self->data[src], cnt);
Py_INCREF(Py_None);
return Py_None;
bounds:
PyErr_SetString(PyExc_ValueError,
"source, destination, or count out of range");
return NULL;
}
}
@ -786,7 +777,7 @@ static PyObject *
mmap_item(mmap_object *self, Py_ssize_t i)
{
CHECK_VALID(NULL);
if (i < 0 || (size_t)i >= self->size) {
if (i < 0 || i >= self->size) {
PyErr_SetString(PyExc_IndexError, "mmap index out of range");
return NULL;
}
@ -803,7 +794,7 @@ mmap_subscript(mmap_object *self, PyObject *item)
return NULL;
if (i < 0)
i += self->size;
if (i < 0 || (size_t)i >= self->size) {
if (i < 0 || i >= self->size) {
PyErr_SetString(PyExc_IndexError,
"mmap index out of range");
return NULL;
@ -871,7 +862,7 @@ mmap_ass_item(mmap_object *self, Py_ssize_t i, PyObject *v)
const char *buf;
CHECK_VALID(-1);
if (i < 0 || (size_t)i >= self->size) {
if (i < 0 || i >= self->size) {
PyErr_SetString(PyExc_IndexError, "mmap index out of range");
return -1;
}
@ -908,7 +899,7 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value)
return -1;
if (i < 0)
i += self->size;
if (i < 0 || (size_t)i >= self->size) {
if (i < 0 || i >= self->size) {
PyErr_SetString(PyExc_IndexError,
"mmap index out of range");
return -1;
@ -1075,32 +1066,6 @@ static PyTypeObject mmap_object_type = {
};
/* extract the map size from the given PyObject
Returns -1 on error, with an appropriate Python exception raised. On
success, the map size is returned. */
static Py_ssize_t
_GetMapSize(PyObject *o, const char* param)
{
if (o == NULL)
return 0;
if (PyIndex_Check(o)) {
Py_ssize_t i = PyNumber_AsSsize_t(o, PyExc_OverflowError);
if (i==-1 && PyErr_Occurred())
return -1;
if (i < 0) {
PyErr_Format(PyExc_OverflowError,
"memory mapped %s must be positive",
param);
return -1;
}
return i;
}
PyErr_SetString(PyExc_TypeError, "map size must be an integral value");
return -1;
}
#ifdef UNIX
#ifdef HAVE_LARGEFILE_SUPPORT
#define _Py_PARSE_OFF_T "L"
@ -1113,7 +1078,6 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
{
struct _Py_stat_struct status;
mmap_object *m_obj;
PyObject *map_size_obj = NULL;
Py_ssize_t map_size;
off_t offset = 0;
int fd, flags = MAP_SHARED, prot = PROT_WRITE | PROT_READ;
@ -1123,13 +1087,15 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
"flags", "prot",
"access", "offset", NULL};
if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|iii" _Py_PARSE_OFF_T, keywords,
&fd, &map_size_obj, &flags, &prot,
if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|iii" _Py_PARSE_OFF_T, keywords,
&fd, &map_size, &flags, &prot,
&access, &offset))
return NULL;
map_size = _GetMapSize(map_size_obj, "size");
if (map_size < 0)
if (map_size < 0) {
PyErr_SetString(PyExc_OverflowError,
"memory mapped length must be postiive");
return NULL;
}
if (offset < 0) {
PyErr_SetString(PyExc_OverflowError,
"memory mapped offset must be positive");
@ -1195,7 +1161,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
return NULL;
}
map_size = (Py_ssize_t) (status.st_size - offset);
} else if (offset + map_size > status.st_size) {
} else if (offset > status.st_size || status.st_size - offset < map_size) {
PyErr_SetString(PyExc_ValueError,
"mmap length is greater than file size");
return NULL;
@ -1204,8 +1170,8 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
m_obj = (mmap_object *)type->tp_alloc(type, 0);
if (m_obj == NULL) {return NULL;}
m_obj->data = NULL;
m_obj->size = (size_t) map_size;
m_obj->pos = (size_t) 0;
m_obj->size = map_size;
m_obj->pos = 0;
m_obj->weakreflist = NULL;
m_obj->exports = 0;
m_obj->offset = offset;
@ -1265,7 +1231,6 @@ static PyObject *
new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
{
mmap_object *m_obj;
PyObject *map_size_obj = NULL;
Py_ssize_t map_size;
long long offset = 0, size;
DWORD off_hi; /* upper 32 bits of offset */
@ -1282,8 +1247,8 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
"tagname",
"access", "offset", NULL };
if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|ziL", keywords,
&fileno, &map_size_obj,
if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|ziL", keywords,
&fileno, &map_size,
&tagname, &access, &offset)) {
return NULL;
}
@ -1306,9 +1271,11 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
"mmap invalid access parameter.");
}
map_size = _GetMapSize(map_size_obj, "size");
if (map_size < 0)
if (map_size < 0) {
PyErr_SetString(PyExc_OverflowError,
"memory mapped length must be postiive");
return NULL;
}
if (offset < 0) {
PyErr_SetString(PyExc_OverflowError,
"memory mapped offset must be positive");