traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add phuang patch from Issue 708374 which adds offset parameter to mmap module.

This commit is contained in:
Travis E. Oliphant 2007-10-23 02:40:56 +00:00
parent 5e81270b22
commit 8feafab346
3 changed files with 130 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Doc/library/mmap.rst
View File

@ -40,7 +40,7 @@ not update the underlying file.
length.
.. function:: mmap(fileno, length[, tagname[, access]])
.. function:: mmap(fileno, length[, tagname[, access[, offset]]])
**(Windows version)** Maps *length* bytes from the file specified by the file
handle *fileno*, and returns a mmap object. If *length* is larger than the
@ -56,8 +56,12 @@ not update the underlying file.
the mapping is created without a name. Avoiding the use of the tag parameter
will assist in keeping your code portable between Unix and Windows.
*offset* may be specified as a non-negative integer offset. mmap references will
be relative to the offset from the beginning of the file. *offset* defaults to 0.
*offset* must be a multiple of the ALLOCATIONGRANULARITY.
.. function:: mmap(fileno, length[, flags[, prot[, access]]])
.. function:: mmap(fileno, length[, flags[, prot[, access[, offset]]]])
:noindex:
**(Unix version)** Maps *length* bytes from the file specified by the file
@ -79,6 +83,10 @@ not update the underlying file.
parameter. It is an error to specify both *flags*, *prot* and *access*. See
the description of *access* above for information on how to use this parameter.
*offset* may be specified as a non-negative integer offset. mmap references will
be relative to the offset from the beginning of the file. *offset* defaults to 0.
*offset* must be a multiple of the PAGESIZE or ALLOCATIONGRANULARITY.
Memory-mapped file objects support the following methods:
@ -171,3 +179,4 @@ Memory-mapped file objects support the following methods:
created with :const:`ACCESS_READ`, then writing to it will throw a
:exc:`TypeError` exception.

44
Lib/test/test_mmap.py
View File

@ -340,6 +340,50 @@ class MmapTests(unittest.TestCase):
m[start:stop:step] = data
self.assertEquals(m[:], "".join(L))
def make_mmap_file (self, f, halfsize):
# Write 2 pages worth of data to the file
f.write ('\0' * halfsize)
f.write ('foo')
f.write ('\0' * (halfsize - 3))
f.flush ()
return mmap.mmap (f.fileno(), 0)
def test_offset (self):
f = open (TESTFN, 'w+b')
try: # unlink TESTFN no matter what
halfsize = mmap.ALLOCATIONGRANULARITY
m = self.make_mmap_file (f, halfsize)
m.close ()
f.close ()
mapsize = halfsize * 2
# Try invalid offset
f = open(TESTFN, "r+b")
for offset in [-2, -1, None]:
try:
m = mmap.mmap(f.fileno(), mapsize, offset=offset)
self.assertEqual(0, 1)
except (ValueError, TypeError, OverflowError):
pass
else:
self.assertEqual(0, 0)
f.close()
# Try valid offset, hopefully 8192 works on all OSes
f = open(TESTFN, "r+b")
m = mmap.mmap(f.fileno(), mapsize - halfsize, offset=halfsize)
self.assertEqual(m[0:3], 'foo')
f.close()
m.close()
finally:
f.close()
try:
os.unlink(TESTFN)
except OSError:
pass
def test_main():
run_unittest(MmapTests)

109
Modules/mmapmodule.c
View File

@ -3,6 +3,9 @@
/ Hacked for Unix by AMK
/ $Id$
/ Modified to support mmap with offset - to map a 'window' of a file
/ Author: Yotam Medini yotamm@mellanox.co.il
/
/ mmapmodule.cpp -- map a view of a file into memory
/
/ todo: need permission flags, perhaps a 'chsize' analog
@ -31,6 +34,16 @@ my_getpagesize(void)
GetSystemInfo(&si);
return si.dwPageSize;
}
static int
my_getallocationgranularity (void)
{
SYSTEM_INFO si;
GetSystemInfo(&si);
return si.dwAllocationGranularity;
}
#endif
#ifdef UNIX
@ -43,6 +56,8 @@ my_getpagesize(void)
{
return sysconf(_SC_PAGESIZE);
}
#define my_getallocationgranularity my_getpagesize
#else
#define my_getpagesize getpagesize
#endif
@ -74,7 +89,8 @@ typedef struct {
PyObject_HEAD
char * data;
size_t size;
size_t pos;
size_t pos; /* relative to offset */
size_t offset;
#ifdef MS_WINDOWS
HANDLE map_handle;
@ -387,18 +403,22 @@ mmap_resize_method(mmap_object *self,
#ifdef MS_WINDOWS
} else {
DWORD dwErrCode = 0;
DWORD newSizeLow, newSizeHigh;
DWORD off_hi, off_lo, newSizeLow, newSizeHigh;
/* First, unmap the file view */
UnmapViewOfFile(self->data);
/* Close the mapping object */
CloseHandle(self->map_handle);
/* Move to the desired EOF position */
#if SIZEOF_SIZE_T > 4
newSizeHigh = (DWORD)(new_size >> 32);
newSizeLow = (DWORD)(new_size & 0xFFFFFFFF);
newSizeHigh = (DWORD)((self->offset + new_size) >> 32);
newSizeLow = (DWORD)((self->offset + new_size) & 0xFFFFFFFF);
off_hi = (DWORD)(self->offset >> 32);
off_lo = (DWORD)(self->offset & 0xFFFFFFFF);
#else
newSizeHigh = 0;
newSizeLow = (DWORD)new_size;
off_hi = 0;
off_lo = (DWORD)self->offset;
#endif
SetFilePointer(self->file_handle,
newSizeLow, &newSizeHigh, FILE_BEGIN);
@ -409,15 +429,15 @@ mmap_resize_method(mmap_object *self,
self->file_handle,
NULL,
PAGE_READWRITE,
newSizeHigh,
newSizeLow,
0,
0,
self->tagname);
if (self->map_handle != NULL) {
self->data = (char *) MapViewOfFile(self->map_handle,
FILE_MAP_WRITE,
0,
0,
0);
off_hi,
off_lo,
new_size);
if (self->data != NULL) {
self->size = new_size;
Py_INCREF(Py_None);
@ -962,15 +982,18 @@ static PyTypeObject mmap_object_type = {
Returns -1 on error, with an appropriate Python exception raised. On
success, the map size is returned. */
static Py_ssize_t
_GetMapSize(PyObject *o)
_GetMapSize(PyObject *o, const char* param)
{
if (o == NULL)
return 0;
if (PyIndex_Check(o)) {
Py_ssize_t i = PyNumber_AsSsize_t(o, PyExc_OverflowError);
if (i==-1 && PyErr_Occurred())
return -1;
if (i < 0) {
PyErr_SetString(PyExc_OverflowError,
"memory mapped size must be positive");
PyErr_Format(PyExc_OverflowError,
"memory mapped %s must be positive",
param);
return -1;
}
return i;
@ -988,22 +1011,25 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
struct stat st;
#endif
mmap_object *m_obj;
PyObject *map_size_obj = NULL;
Py_ssize_t map_size;
PyObject *map_size_obj = NULL, *offset_obj = NULL;
Py_ssize_t map_size, offset;
int fd, flags = MAP_SHARED, prot = PROT_WRITE | PROT_READ;
int devzero = -1;
int access = (int)ACCESS_DEFAULT;
static char *keywords[] = {"fileno", "length",
"flags", "prot",
"access", NULL};
"access", "offset", NULL};
if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|iii", keywords,
if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|iiiO", keywords,
&fd, &map_size_obj, &flags, &prot,
&access))
&access, &offset_obj))
return NULL;
map_size = _GetMapSize(map_size_obj);
map_size = _GetMapSize(map_size_obj, "size");
if (map_size < 0)
return NULL;
offset = _GetMapSize(offset_obj, "offset");
if (offset < 0)
return NULL;
if ((access != (int)ACCESS_DEFAULT) &&
((flags != MAP_SHARED) || (prot != (PROT_WRITE | PROT_READ))))
@ -1038,7 +1064,7 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
if (fstat(fd, &st) == 0 && S_ISREG(st.st_mode)) {
if (map_size == 0) {
map_size = st.st_size;
} else if ((size_t)map_size > st.st_size) {
} else if ((size_t)offset + (size_t)map_size > st.st_size) {
PyErr_SetString(PyExc_ValueError,
"mmap length is greater than file size");
return NULL;
@ -1050,6 +1076,7 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
m_obj->data = NULL;
m_obj->size = (size_t) map_size;
m_obj->pos = (size_t) 0;
m_obj->offset = offset;
if (fd == -1) {
m_obj->fd = -1;
/* Assume the caller wants to map anonymous memory.
@ -1076,10 +1103,10 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
return NULL;
}
}
m_obj->data = mmap(NULL, map_size,
prot, flags,
fd, 0);
fd, offset);
if (devzero != -1) {
close(devzero);
@ -1101,10 +1128,12 @@ static PyObject *
new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
{
mmap_object *m_obj;
PyObject *map_size_obj = NULL;
Py_ssize_t map_size;
DWORD size_hi; /* upper 32 bits of m_obj->size */
DWORD size_lo; /* lower 32 bits of m_obj->size */
PyObject *map_size_obj = NULL, *offset_obj = NULL;
Py_ssize_t map_size, offset;
DWORD off_hi; /* upper 32 bits of offset */
DWORD off_lo; /* lower 32 bits of offset */
DWORD size_hi; /* upper 32 bits of size */
DWORD size_lo; /* lower 32 bits of size */
char *tagname = "";
DWORD dwErr = 0;
int fileno;
@ -1113,11 +1142,11 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
DWORD flProtect, dwDesiredAccess;
static char *keywords[] = { "fileno", "length",
"tagname",
"access", NULL };
"access", "offset", NULL };
if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|zi", keywords,
if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|ziO", keywords,
&fileno, &map_size_obj,
&tagname, &access)) {
&tagname, &access, &offset_obj)) {
return NULL;
}
@ -1139,9 +1168,12 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
"mmap invalid access parameter.");
}
map_size = _GetMapSize(map_size_obj);
map_size = _GetMapSize(map_size_obj, "size");
if (map_size < 0)
return NULL;
offset = _GetMapSize(offset_obj, "offset");
if (offset < 0)
return NULL;
/* assume -1 and 0 both mean invalid filedescriptor
to 'anonymously' map memory.
@ -1170,6 +1202,7 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
m_obj->file_handle = INVALID_HANDLE_VALUE;
m_obj->map_handle = INVALID_HANDLE_VALUE;
m_obj->tagname = NULL;
m_obj->offset = offset;
if (fh) {
/* It is necessary to duplicate the handle, so the
@ -1238,12 +1271,18 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
* right by 32, so we need different code.
*/
#if SIZEOF_SIZE_T > 4
size_hi = (DWORD)(m_obj->size >> 32);
size_lo = (DWORD)(m_obj->size & 0xFFFFFFFF);
size_hi = (DWORD)((offset + m_obj->size) >> 32);
size_lo = (DWORD)((offset + m_obj->size) & 0xFFFFFFFF);
off_hi = (DWORD)(offset >> 32);
off_lo = (DWORD)(offset & 0xFFFFFFFF);
#else
size_hi = 0;
size_lo = (DWORD)m_obj->size;
size_lo = (DWORD)(offset + m_obj->size);
off_hi = 0;
off_lo = (DWORD)offset;
#endif
/* For files, it would be sufficient to pass 0 as size.
For anonymous maps, we have to pass the size explicitly. */
m_obj->map_handle = CreateFileMapping(m_obj->file_handle,
NULL,
flProtect,
@ -1253,8 +1292,8 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
if (m_obj->map_handle != NULL) {
m_obj->data = (char *) MapViewOfFile(m_obj->map_handle,
dwDesiredAccess,
0,
0,
off_hi,
off_lo,
0);
if (m_obj->data != NULL)
return (PyObject *)m_obj;
@ -1329,6 +1368,8 @@ PyMODINIT_FUNC
setint(dict, "PAGESIZE", (long)my_getpagesize());
setint(dict, "ALLOCATIONGRANULARITY", (long)my_getallocationgranularity());
setint(dict, "ACCESS_READ", ACCESS_READ);
setint(dict, "ACCESS_WRITE", ACCESS_WRITE);
setint(dict, "ACCESS_COPY", ACCESS_COPY);