traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

merge 3.5

This commit is contained in:
Benjamin Peterson 2016-10-05 22:54:19 -07:00
parent a88614233e 4c8b2cd126
commit 87845bcb4d
3 changed files with 97 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Lib/test/test_mmap.py
View File

@ -720,6 +720,20 @@ class MmapTests(unittest.TestCase):
self.assertEqual(mm.write(b"yz"), 2) self.assertEqual(mm.write(b"yz"), 2)
self.assertEqual(mm.write(b"python"), 6) self.assertEqual(mm.write(b"python"), 6)
@unittest.skipIf(os.name == 'nt', 'cannot resize anonymous mmaps on Windows')
def test_resize_past_pos(self):
m = mmap.mmap(-1, 8192)
self.addCleanup(m.close)
m.read(5000)
try:
m.resize(4096)
except SystemError:
self.skipTest("resizing not supported")
self.assertEqual(m.read(14), b'')
self.assertRaises(ValueError, m.read_byte)
self.assertRaises(ValueError, m.write_byte, 42)
self.assertRaises(ValueError, m.write, b'abc')
class LargeMmapTests(unittest.TestCase): class LargeMmapTests(unittest.TestCase):

3
Misc/NEWS
View File

@ -404,6 +404,9 @@ Library
pickling and text representation purposes. Patch by Emanuel Barry and pickling and text representation purposes. Patch by Emanuel Barry and
Serhiy Storchaka. Serhiy Storchaka.
- Fix possible integer overflows and crashes in the mmap module with unusual
usage patterns.
- Issue #1703178: Fix the ability to pass the --link-objects option to the - Issue #1703178: Fix the ability to pass the --link-objects option to the
distutils build_ext command. distutils build_ext command.

193
Modules/mmapmodule.c
View File

@ -90,8 +90,8 @@ typedef enum
typedef struct { typedef struct {
PyObject_HEAD PyObject_HEAD
char * data; char * data;
size_t size; Py_ssize_t size;
size_t pos; /* relative to offset */ Py_ssize_t pos; /* relative to offset */
#ifdef MS_WINDOWS #ifdef MS_WINDOWS
long long offset; long long offset;
#else #else
@ -210,33 +210,32 @@ mmap_read_byte_method(mmap_object *self,
PyObject *unused) PyObject *unused)
{ {
CHECK_VALID(NULL); CHECK_VALID(NULL);
if (self->pos < self->size) { if (self->pos >= self->size) {
char value = self->data[self->pos];
self->pos += 1;
return Py_BuildValue("B", (unsigned char)value);
} else {
PyErr_SetString(PyExc_ValueError, "read byte out of range"); PyErr_SetString(PyExc_ValueError, "read byte out of range");
return NULL; return NULL;
} }
return PyLong_FromLong((unsigned char)self->data[self->pos++]);
} }
static PyObject * static PyObject *
mmap_read_line_method(mmap_object *self, mmap_read_line_method(mmap_object *self,
PyObject *unused) PyObject *unused)
{ {
char *start = self->data+self->pos; Py_ssize_t remaining;
char *eof = self->data+self->size; char *start, *eol;
char *eol;
PyObject *result; PyObject *result;
CHECK_VALID(NULL); CHECK_VALID(NULL);
eol = memchr(start, '\n', self->size - self->pos); remaining = (self->pos < self->size) ? self->size - self->pos : 0;
if (!remaining)
return PyBytes_FromString("");
start = self->data + self->pos;
eol = memchr(start, '\n', remaining);
if (!eol) if (!eol)
eol = eof; eol = self->data + self->size;
else else
++eol; /* we're interested in the position after the ++eol; /* advance past newline */
newline. */
result = PyBytes_FromStringAndSize(start, (eol - start)); result = PyBytes_FromStringAndSize(start, (eol - start));
self->pos += (eol - start); self->pos += (eol - start);
return result; return result;
@ -268,7 +267,7 @@ static PyObject *
mmap_read_method(mmap_object *self, mmap_read_method(mmap_object *self,
PyObject *args) PyObject *args)
{ {
Py_ssize_t num_bytes = -1, n; Py_ssize_t num_bytes, remaining;
PyObject *result; PyObject *result;
CHECK_VALID(NULL); CHECK_VALID(NULL);
@ -276,20 +275,10 @@ mmap_read_method(mmap_object *self,
return(NULL); return(NULL);
/* silently 'adjust' out-of-range requests */ /* silently 'adjust' out-of-range requests */
assert(self->size >= self->pos); remaining = (self->pos < self->size) ? self->size - self->pos : 0;
n = self->size - self->pos; if (num_bytes < 0 || num_bytes > remaining)
/* The difference can overflow, only if self->size is greater than num_bytes = remaining;
* PY_SSIZE_T_MAX. But then the operation cannot possibly succeed, result = PyBytes_FromStringAndSize(&self->data[self->pos], num_bytes);
* because the mapped area and the returned string each need more
* than half of the addressable memory. So we clip the size, and let
* the code below raise MemoryError.
*/
if (n < 0)
n = PY_SSIZE_T_MAX;
if (num_bytes < 0 || num_bytes > n) {
num_bytes = n;
}
result = PyBytes_FromStringAndSize(self->data+self->pos, num_bytes);
self->pos += num_bytes; self->pos += num_bytes;
return result; return result;
} }
@ -317,14 +306,14 @@ mmap_gfind(mmap_object *self,
start += self->size; start += self->size;
if (start < 0) if (start < 0)
start = 0; start = 0;
else if ((size_t)start > self->size) else if (start > self->size)
start = self->size; start = self->size;
if (end < 0) if (end < 0)
end += self->size; end += self->size;
if (end < 0) if (end < 0)
end = 0; end = 0;
else if ((size_t)end > self->size) else if (end > self->size)
end = self->size; end = self->size;
start_p = self->data + start; start_p = self->data + start;
@ -389,27 +378,24 @@ mmap_write_method(mmap_object *self,
PyObject *args) PyObject *args)
{ {
Py_buffer data; Py_buffer data;
PyObject *result;
CHECK_VALID(NULL); CHECK_VALID(NULL);
if (!PyArg_ParseTuple(args, "y*:write", &data)) if (!PyArg_ParseTuple(args, "y*:write", &data))
return(NULL); return(NULL);
if (!is_writable(self)) { if (!is_writable(self))
return NULL;
if (self->pos > self->size || self->size - self->pos < data.len) {
PyBuffer_Release(&data); PyBuffer_Release(&data);
PyErr_SetString(PyExc_ValueError, "data out of range");
return NULL; return NULL;
} }
if ((self->pos + data.len) > self->size) { memcpy(&self->data[self->pos], data.buf, data.len);
PyErr_SetString(PyExc_ValueError, "data out of range"); self->pos += data.len;
PyBuffer_Release(&data);
return NULL;
}
memcpy(self->data + self->pos, data.buf, data.len);
self->pos = self->pos + data.len;
result = PyLong_FromSsize_t(data.len);
PyBuffer_Release(&data); PyBuffer_Release(&data);
return result; return PyLong_FromSsize_t(data.len);
} }
static PyObject * static PyObject *
@ -426,8 +412,7 @@ mmap_write_byte_method(mmap_object *self,
return NULL; return NULL;
if (self->pos < self->size) { if (self->pos < self->size) {
*(self->data+self->pos) = value; self->data[self->pos++] = value;
self->pos += 1;
Py_INCREF(Py_None); Py_INCREF(Py_None);
return Py_None; return Py_None;
} }
@ -496,8 +481,14 @@ mmap_resize_method(mmap_object *self,
if (!PyArg_ParseTuple(args, "n:resize", &new_size) || if (!PyArg_ParseTuple(args, "n:resize", &new_size) ||
!is_resizeable(self)) { !is_resizeable(self)) {
return NULL; return NULL;
}
if (new_size < 0 || PY_SSIZE_T_MAX - new_size < self->offset) {
PyErr_SetString(PyExc_ValueError, "new size out of range");
return NULL;
}
{
#ifdef MS_WINDOWS #ifdef MS_WINDOWS
} else {
DWORD dwErrCode = 0; DWORD dwErrCode = 0;
DWORD off_hi, off_lo, newSizeLow, newSizeHigh; DWORD off_hi, off_lo, newSizeLow, newSizeHigh;
/* First, unmap the file view */ /* First, unmap the file view */
@ -547,15 +538,13 @@ mmap_resize_method(mmap_object *self,
#ifdef UNIX #ifdef UNIX
#ifndef HAVE_MREMAP #ifndef HAVE_MREMAP
} else {
PyErr_SetString(PyExc_SystemError, PyErr_SetString(PyExc_SystemError,
"mmap: resizing not available--no mremap()"); "mmap: resizing not available--no mremap()");
return NULL; return NULL;
#else #else
} else {
void *newmap; void *newmap;
if (ftruncate(self->fd, self->offset + new_size) == -1) { if (self->fd != -1 && ftruncate(self->fd, self->offset + new_size) == -1) {
PyErr_SetFromErrno(PyExc_OSError); PyErr_SetFromErrno(PyExc_OSError);
return NULL; return NULL;
} }
@ -563,11 +552,11 @@ mmap_resize_method(mmap_object *self,
#ifdef MREMAP_MAYMOVE #ifdef MREMAP_MAYMOVE
newmap = mremap(self->data, self->size, new_size, MREMAP_MAYMOVE); newmap = mremap(self->data, self->size, new_size, MREMAP_MAYMOVE);
#else #else
#if defined(__NetBSD__) #if defined(__NetBSD__)
newmap = mremap(self->data, self->size, self->data, new_size, 0); newmap = mremap(self->data, self->size, self->data, new_size, 0);
#else #else
newmap = mremap(self->data, self->size, new_size, 0); newmap = mremap(self->data, self->size, new_size, 0);
#endif /* __NetBSD__ */ #endif /* __NetBSD__ */
#endif #endif
if (newmap == (void *)-1) if (newmap == (void *)-1)
{ {
@ -598,7 +587,7 @@ mmap_flush_method(mmap_object *self, PyObject *args)
CHECK_VALID(NULL); CHECK_VALID(NULL);
if (!PyArg_ParseTuple(args, "|nn:flush", &offset, &size)) if (!PyArg_ParseTuple(args, "|nn:flush", &offset, &size))
return NULL; return NULL;
if ((size_t)(offset + size) > self->size) { if (size < 0 || offset < 0 || self->size - offset < size) {
PyErr_SetString(PyExc_ValueError, "flush values out of range"); PyErr_SetString(PyExc_ValueError, "flush values out of range");
return NULL; return NULL;
} }
@ -631,20 +620,18 @@ mmap_seek_method(mmap_object *self, PyObject *args)
if (!PyArg_ParseTuple(args, "n|i:seek", &dist, &how)) if (!PyArg_ParseTuple(args, "n|i:seek", &dist, &how))
return NULL; return NULL;
else { else {
size_t where; Py_ssize_t where;
switch (how) { switch (how) {
case 0: /* relative to start */ case 0: /* relative to start */
if (dist < 0)
goto onoutofrange;
where = dist; where = dist;
break; break;
case 1: /* relative to current position */ case 1: /* relative to current position */
if ((Py_ssize_t)self->pos + dist < 0) if (PY_SSIZE_T_MAX - self->pos < dist)
goto onoutofrange; goto onoutofrange;
where = self->pos + dist; where = self->pos + dist;
break; break;
case 2: /* relative to end */ case 2: /* relative to end */
if ((Py_ssize_t)self->size + dist < 0) if (PY_SSIZE_T_MAX - self->size < dist)
goto onoutofrange; goto onoutofrange;
where = self->size + dist; where = self->size + dist;
break; break;
@ -652,7 +639,7 @@ mmap_seek_method(mmap_object *self, PyObject *args)
PyErr_SetString(PyExc_ValueError, "unknown seek type"); PyErr_SetString(PyExc_ValueError, "unknown seek type");
return NULL; return NULL;
} }
if (where > self->size) if (where > self->size || where < 0)
goto onoutofrange; goto onoutofrange;
self->pos = where; self->pos = where;
Py_INCREF(Py_None); Py_INCREF(Py_None);
@ -667,23 +654,27 @@ mmap_seek_method(mmap_object *self, PyObject *args)
static PyObject * static PyObject *
mmap_move_method(mmap_object *self, PyObject *args) mmap_move_method(mmap_object *self, PyObject *args)
{ {
unsigned long dest, src, cnt; Py_ssize_t dest, src, cnt;
CHECK_VALID(NULL); CHECK_VALID(NULL);
if (!PyArg_ParseTuple(args, "kkk:move", &dest, &src, &cnt) || if (!PyArg_ParseTuple(args, "nnn:move", &dest, &src, &cnt) ||
!is_writable(self)) { !is_writable(self)) {
return NULL; return NULL;
} else { } else {
/* bounds check the values */ /* bounds check the values */
if ((cnt + dest) < cnt || (cnt + src) < cnt || if (dest < 0 || src < 0 || cnt < 0)
src > self->size || (src + cnt) > self->size || goto bounds;
dest > self->size || (dest + cnt) > self->size) { if (self->size - dest < cnt || self->size - src < cnt)
PyErr_SetString(PyExc_ValueError, goto bounds;
"source, destination, or count out of range");
return NULL; memmove(&self->data[dest], &self->data[src], cnt);
}
memmove(self->data+dest, self->data+src, cnt);
Py_INCREF(Py_None); Py_INCREF(Py_None);
return Py_None; return Py_None;
bounds:
PyErr_SetString(PyExc_ValueError,
"source, destination, or count out of range");
return NULL;
} }
} }
@ -786,7 +777,7 @@ static PyObject *
mmap_item(mmap_object *self, Py_ssize_t i) mmap_item(mmap_object *self, Py_ssize_t i)
{ {
CHECK_VALID(NULL); CHECK_VALID(NULL);
if (i < 0 || (size_t)i >= self->size) { if (i < 0 || i >= self->size) {
PyErr_SetString(PyExc_IndexError, "mmap index out of range"); PyErr_SetString(PyExc_IndexError, "mmap index out of range");
return NULL; return NULL;
} }
@ -803,7 +794,7 @@ mmap_subscript(mmap_object *self, PyObject *item)
return NULL; return NULL;
if (i < 0) if (i < 0)
i += self->size; i += self->size;
if (i < 0 || (size_t)i >= self->size) { if (i < 0 || i >= self->size) {
PyErr_SetString(PyExc_IndexError, PyErr_SetString(PyExc_IndexError,
"mmap index out of range"); "mmap index out of range");
return NULL; return NULL;
@ -871,7 +862,7 @@ mmap_ass_item(mmap_object *self, Py_ssize_t i, PyObject *v)
const char *buf; const char *buf;
CHECK_VALID(-1); CHECK_VALID(-1);
if (i < 0 || (size_t)i >= self->size) { if (i < 0 || i >= self->size) {
PyErr_SetString(PyExc_IndexError, "mmap index out of range"); PyErr_SetString(PyExc_IndexError, "mmap index out of range");
return -1; return -1;
} }
@ -908,7 +899,7 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value)
return -1; return -1;
if (i < 0) if (i < 0)
i += self->size; i += self->size;
if (i < 0 || (size_t)i >= self->size) { if (i < 0 || i >= self->size) {
PyErr_SetString(PyExc_IndexError, PyErr_SetString(PyExc_IndexError,
"mmap index out of range"); "mmap index out of range");
return -1; return -1;
@ -1075,32 +1066,6 @@ static PyTypeObject mmap_object_type = {
}; };
/* extract the map size from the given PyObject
Returns -1 on error, with an appropriate Python exception raised. On
success, the map size is returned. */
static Py_ssize_t
_GetMapSize(PyObject *o, const char* param)
{
if (o == NULL)
return 0;
if (PyIndex_Check(o)) {
Py_ssize_t i = PyNumber_AsSsize_t(o, PyExc_OverflowError);
if (i==-1 && PyErr_Occurred())
return -1;
if (i < 0) {
PyErr_Format(PyExc_OverflowError,
"memory mapped %s must be positive",
param);
return -1;
}
return i;
}
PyErr_SetString(PyExc_TypeError, "map size must be an integral value");
return -1;
}
#ifdef UNIX #ifdef UNIX
#ifdef HAVE_LARGEFILE_SUPPORT #ifdef HAVE_LARGEFILE_SUPPORT
#define _Py_PARSE_OFF_T "L" #define _Py_PARSE_OFF_T "L"
@ -1113,7 +1078,6 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
{ {
struct _Py_stat_struct status; struct _Py_stat_struct status;
mmap_object *m_obj; mmap_object *m_obj;
PyObject *map_size_obj = NULL;
Py_ssize_t map_size; Py_ssize_t map_size;
off_t offset = 0; off_t offset = 0;
int fd, flags = MAP_SHARED, prot = PROT_WRITE | PROT_READ; int fd, flags = MAP_SHARED, prot = PROT_WRITE | PROT_READ;
@ -1123,13 +1087,15 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
"flags", "prot", "flags", "prot",
"access", "offset", NULL}; "access", "offset", NULL};
if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|iii" _Py_PARSE_OFF_T, keywords, if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|iii" _Py_PARSE_OFF_T, keywords,
&fd, &map_size_obj, &flags, &prot, &fd, &map_size, &flags, &prot,
&access, &offset)) &access, &offset))
return NULL; return NULL;
map_size = _GetMapSize(map_size_obj, "size"); if (map_size < 0) {
if (map_size < 0) PyErr_SetString(PyExc_OverflowError,
"memory mapped length must be postiive");
return NULL; return NULL;
}
if (offset < 0) { if (offset < 0) {
PyErr_SetString(PyExc_OverflowError, PyErr_SetString(PyExc_OverflowError,
"memory mapped offset must be positive"); "memory mapped offset must be positive");
@ -1195,7 +1161,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
return NULL; return NULL;
} }
map_size = (Py_ssize_t) (status.st_size - offset); map_size = (Py_ssize_t) (status.st_size - offset);
} else if (offset + map_size > status.st_size) { } else if (offset > status.st_size || status.st_size - offset < map_size) {
PyErr_SetString(PyExc_ValueError, PyErr_SetString(PyExc_ValueError,
"mmap length is greater than file size"); "mmap length is greater than file size");
return NULL; return NULL;
@ -1204,8 +1170,8 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
m_obj = (mmap_object *)type->tp_alloc(type, 0); m_obj = (mmap_object *)type->tp_alloc(type, 0);
if (m_obj == NULL) {return NULL;} if (m_obj == NULL) {return NULL;}
m_obj->data = NULL; m_obj->data = NULL;
m_obj->size = (size_t) map_size; m_obj->size = map_size;
m_obj->pos = (size_t) 0; m_obj->pos = 0;
m_obj->weakreflist = NULL; m_obj->weakreflist = NULL;
m_obj->exports = 0; m_obj->exports = 0;
m_obj->offset = offset; m_obj->offset = offset;
@ -1265,7 +1231,6 @@ static PyObject *
new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
{ {
mmap_object *m_obj; mmap_object *m_obj;
PyObject *map_size_obj = NULL;
Py_ssize_t map_size; Py_ssize_t map_size;
long long offset = 0, size; long long offset = 0, size;
DWORD off_hi; /* upper 32 bits of offset */ DWORD off_hi; /* upper 32 bits of offset */
@ -1282,8 +1247,8 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
"tagname", "tagname",
"access", "offset", NULL }; "access", "offset", NULL };
if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|ziL", keywords, if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|ziL", keywords,
&fileno, &map_size_obj, &fileno, &map_size,
&tagname, &access, &offset)) { &tagname, &access, &offset)) {
return NULL; return NULL;
} }
@ -1306,9 +1271,11 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
"mmap invalid access parameter."); "mmap invalid access parameter.");
} }
map_size = _GetMapSize(map_size_obj, "size"); if (map_size < 0) {
if (map_size < 0) PyErr_SetString(PyExc_OverflowError,
"memory mapped length must be postiive");
return NULL; return NULL;
}
if (offset < 0) { if (offset < 0) {
PyErr_SetString(PyExc_OverflowError, PyErr_SetString(PyExc_OverflowError,
"memory mapped offset must be positive"); "memory mapped offset must be positive");