[Python-checkins] r58598 - in python/trunk: Doc/library/mmap.rst Lib/test/test_mmap.py Modules/mmapmodule.c
travis.oliphant
python-checkins at python.org
Tue Oct 23 04:40:57 CEST 2007
Author: travis.oliphant
Date: Tue Oct 23 04:40:56 2007
New Revision: 58598
Modified:
python/trunk/Doc/library/mmap.rst
python/trunk/Lib/test/test_mmap.py
python/trunk/Modules/mmapmodule.c
Log:
Add phuang patch from Issue 708374 which adds offset parameter to mmap module.
Modified: python/trunk/Doc/library/mmap.rst
==============================================================================
--- python/trunk/Doc/library/mmap.rst (original)
+++ python/trunk/Doc/library/mmap.rst Tue Oct 23 04:40:56 2007
@@ -40,7 +40,7 @@
length.
-.. function:: mmap(fileno, length[, tagname[, access]])
+.. function:: mmap(fileno, length[, tagname[, access[, offset]]])
**(Windows version)** Maps *length* bytes from the file specified by the file
handle *fileno*, and returns a mmap object. If *length* is larger than the
@@ -56,8 +56,12 @@
the mapping is created without a name. Avoiding the use of the tag parameter
will assist in keeping your code portable between Unix and Windows.
+ *offset* may be specified as a non-negative integer offset. mmap references will
+ be relative to the offset from the beginning of the file. *offset* defaults to 0.
+ *offset* must be a multiple of the ALLOCATIONGRANULARITY.
-.. function:: mmap(fileno, length[, flags[, prot[, access]]])
+
+.. function:: mmap(fileno, length[, flags[, prot[, access[, offset]]]])
:noindex:
**(Unix version)** Maps *length* bytes from the file specified by the file
@@ -79,6 +83,10 @@
parameter. It is an error to specify both *flags*, *prot* and *access*. See
the description of *access* above for information on how to use this parameter.
+ *offset* may be specified as a non-negative integer offset. mmap references will
+ be relative to the offset from the beginning of the file. *offset* defaults to 0.
+ *offset* must be a multiple of the PAGESIZE or ALLOCATIONGRANULARITY.
+
Memory-mapped file objects support the following methods:
@@ -171,3 +179,4 @@
created with :const:`ACCESS_READ`, then writing to it will throw a
:exc:`TypeError` exception.
+
Modified: python/trunk/Lib/test/test_mmap.py
==============================================================================
--- python/trunk/Lib/test/test_mmap.py (original)
+++ python/trunk/Lib/test/test_mmap.py Tue Oct 23 04:40:56 2007
@@ -340,6 +340,50 @@
m[start:stop:step] = data
self.assertEquals(m[:], "".join(L))
+ def make_mmap_file (self, f, halfsize):
+ # Write 2 pages worth of data to the file
+ f.write ('\0' * halfsize)
+ f.write ('foo')
+ f.write ('\0' * (halfsize - 3))
+ f.flush ()
+ return mmap.mmap (f.fileno(), 0)
+
+ def test_offset (self):
+ f = open (TESTFN, 'w+b')
+
+ try: # unlink TESTFN no matter what
+ halfsize = mmap.ALLOCATIONGRANULARITY
+ m = self.make_mmap_file (f, halfsize)
+ m.close ()
+ f.close ()
+
+ mapsize = halfsize * 2
+ # Try invalid offset
+ f = open(TESTFN, "r+b")
+ for offset in [-2, -1, None]:
+ try:
+ m = mmap.mmap(f.fileno(), mapsize, offset=offset)
+ self.assertEqual(0, 1)
+ except (ValueError, TypeError, OverflowError):
+ pass
+ else:
+ self.assertEqual(0, 0)
+ f.close()
+
+ # Try valid offset, hopefully 8192 works on all OSes
+ f = open(TESTFN, "r+b")
+ m = mmap.mmap(f.fileno(), mapsize - halfsize, offset=halfsize)
+ self.assertEqual(m[0:3], 'foo')
+ f.close()
+ m.close()
+
+ finally:
+ f.close()
+ try:
+ os.unlink(TESTFN)
+ except OSError:
+ pass
+
def test_main():
run_unittest(MmapTests)
Modified: python/trunk/Modules/mmapmodule.c
==============================================================================
--- python/trunk/Modules/mmapmodule.c (original)
+++ python/trunk/Modules/mmapmodule.c Tue Oct 23 04:40:56 2007
@@ -3,6 +3,9 @@
/ Hacked for Unix by AMK
/ $Id$
+ / Modified to support mmap with offset - to map a 'window' of a file
+ / Author: Yotam Medini yotamm at mellanox.co.il
+ /
/ mmapmodule.cpp -- map a view of a file into memory
/
/ todo: need permission flags, perhaps a 'chsize' analog
@@ -31,6 +34,16 @@
GetSystemInfo(&si);
return si.dwPageSize;
}
+
+static int
+my_getallocationgranularity (void)
+{
+
+ SYSTEM_INFO si;
+ GetSystemInfo(&si);
+ return si.dwAllocationGranularity;
+}
+
#endif
#ifdef UNIX
@@ -43,6 +56,8 @@
{
return sysconf(_SC_PAGESIZE);
}
+
+#define my_getallocationgranularity my_getpagesize
#else
#define my_getpagesize getpagesize
#endif
@@ -74,7 +89,8 @@
PyObject_HEAD
char * data;
size_t size;
- size_t pos;
+ size_t pos; /* relative to offset */
+ size_t offset;
#ifdef MS_WINDOWS
HANDLE map_handle;
@@ -387,18 +403,22 @@
#ifdef MS_WINDOWS
} else {
DWORD dwErrCode = 0;
- DWORD newSizeLow, newSizeHigh;
+ DWORD off_hi, off_lo, newSizeLow, newSizeHigh;
/* First, unmap the file view */
UnmapViewOfFile(self->data);
/* Close the mapping object */
CloseHandle(self->map_handle);
/* Move to the desired EOF position */
#if SIZEOF_SIZE_T > 4
- newSizeHigh = (DWORD)(new_size >> 32);
- newSizeLow = (DWORD)(new_size & 0xFFFFFFFF);
+ newSizeHigh = (DWORD)((self->offset + new_size) >> 32);
+ newSizeLow = (DWORD)((self->offset + new_size) & 0xFFFFFFFF);
+ off_hi = (DWORD)(self->offset >> 32);
+ off_lo = (DWORD)(self->offset & 0xFFFFFFFF);
#else
newSizeHigh = 0;
newSizeLow = (DWORD)new_size;
+ off_hi = 0;
+ off_lo = (DWORD)self->offset;
#endif
SetFilePointer(self->file_handle,
newSizeLow, &newSizeHigh, FILE_BEGIN);
@@ -409,15 +429,15 @@
self->file_handle,
NULL,
PAGE_READWRITE,
- newSizeHigh,
- newSizeLow,
+ 0,
+ 0,
self->tagname);
if (self->map_handle != NULL) {
self->data = (char *) MapViewOfFile(self->map_handle,
FILE_MAP_WRITE,
- 0,
- 0,
- 0);
+ off_hi,
+ off_lo,
+ new_size);
if (self->data != NULL) {
self->size = new_size;
Py_INCREF(Py_None);
@@ -962,15 +982,18 @@
Returns -1 on error, with an appropriate Python exception raised. On
success, the map size is returned. */
static Py_ssize_t
-_GetMapSize(PyObject *o)
+_GetMapSize(PyObject *o, const char* param)
{
+ if (o == NULL)
+ return 0;
if (PyIndex_Check(o)) {
Py_ssize_t i = PyNumber_AsSsize_t(o, PyExc_OverflowError);
if (i==-1 && PyErr_Occurred())
return -1;
if (i < 0) {
- PyErr_SetString(PyExc_OverflowError,
- "memory mapped size must be positive");
+ PyErr_Format(PyExc_OverflowError,
+ "memory mapped %s must be positive",
+ param);
return -1;
}
return i;
@@ -988,22 +1011,25 @@
struct stat st;
#endif
mmap_object *m_obj;
- PyObject *map_size_obj = NULL;
- Py_ssize_t map_size;
+ PyObject *map_size_obj = NULL, *offset_obj = NULL;
+ Py_ssize_t map_size, offset;
int fd, flags = MAP_SHARED, prot = PROT_WRITE | PROT_READ;
int devzero = -1;
int access = (int)ACCESS_DEFAULT;
static char *keywords[] = {"fileno", "length",
"flags", "prot",
- "access", NULL};
+ "access", "offset", NULL};
- if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|iii", keywords,
+ if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|iiiO", keywords,
&fd, &map_size_obj, &flags, &prot,
- &access))
+ &access, &offset_obj))
return NULL;
- map_size = _GetMapSize(map_size_obj);
+ map_size = _GetMapSize(map_size_obj, "size");
if (map_size < 0)
return NULL;
+ offset = _GetMapSize(offset_obj, "offset");
+ if (offset < 0)
+ return NULL;
if ((access != (int)ACCESS_DEFAULT) &&
((flags != MAP_SHARED) || (prot != (PROT_WRITE | PROT_READ))))
@@ -1038,7 +1064,7 @@
if (fstat(fd, &st) == 0 && S_ISREG(st.st_mode)) {
if (map_size == 0) {
map_size = st.st_size;
- } else if ((size_t)map_size > st.st_size) {
+ } else if ((size_t)offset + (size_t)map_size > st.st_size) {
PyErr_SetString(PyExc_ValueError,
"mmap length is greater than file size");
return NULL;
@@ -1050,6 +1076,7 @@
m_obj->data = NULL;
m_obj->size = (size_t) map_size;
m_obj->pos = (size_t) 0;
+ m_obj->offset = offset;
if (fd == -1) {
m_obj->fd = -1;
/* Assume the caller wants to map anonymous memory.
@@ -1076,10 +1103,10 @@
return NULL;
}
}
-
+
m_obj->data = mmap(NULL, map_size,
prot, flags,
- fd, 0);
+ fd, offset);
if (devzero != -1) {
close(devzero);
@@ -1101,10 +1128,12 @@
new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
{
mmap_object *m_obj;
- PyObject *map_size_obj = NULL;
- Py_ssize_t map_size;
- DWORD size_hi; /* upper 32 bits of m_obj->size */
- DWORD size_lo; /* lower 32 bits of m_obj->size */
+ PyObject *map_size_obj = NULL, *offset_obj = NULL;
+ Py_ssize_t map_size, offset;
+ DWORD off_hi; /* upper 32 bits of offset */
+ DWORD off_lo; /* lower 32 bits of offset */
+ DWORD size_hi; /* upper 32 bits of size */
+ DWORD size_lo; /* lower 32 bits of size */
char *tagname = "";
DWORD dwErr = 0;
int fileno;
@@ -1113,11 +1142,11 @@
DWORD flProtect, dwDesiredAccess;
static char *keywords[] = { "fileno", "length",
"tagname",
- "access", NULL };
+ "access", "offset", NULL };
- if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|zi", keywords,
+ if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|ziO", keywords,
&fileno, &map_size_obj,
- &tagname, &access)) {
+ &tagname, &access, &offset_obj)) {
return NULL;
}
@@ -1139,9 +1168,12 @@
"mmap invalid access parameter.");
}
- map_size = _GetMapSize(map_size_obj);
+ map_size = _GetMapSize(map_size_obj, "size");
if (map_size < 0)
return NULL;
+ offset = _GetMapSize(offset_obj, "offset");
+ if (offset < 0)
+ return NULL;
/* assume -1 and 0 both mean invalid filedescriptor
to 'anonymously' map memory.
@@ -1170,6 +1202,7 @@
m_obj->file_handle = INVALID_HANDLE_VALUE;
m_obj->map_handle = INVALID_HANDLE_VALUE;
m_obj->tagname = NULL;
+ m_obj->offset = offset;
if (fh) {
/* It is necessary to duplicate the handle, so the
@@ -1238,12 +1271,18 @@
* right by 32, so we need different code.
*/
#if SIZEOF_SIZE_T > 4
- size_hi = (DWORD)(m_obj->size >> 32);
- size_lo = (DWORD)(m_obj->size & 0xFFFFFFFF);
+ size_hi = (DWORD)((offset + m_obj->size) >> 32);
+ size_lo = (DWORD)((offset + m_obj->size) & 0xFFFFFFFF);
+ off_hi = (DWORD)(offset >> 32);
+ off_lo = (DWORD)(offset & 0xFFFFFFFF);
#else
size_hi = 0;
- size_lo = (DWORD)m_obj->size;
+ size_lo = (DWORD)(offset + m_obj->size);
+ off_hi = 0;
+ off_lo = (DWORD)offset;
#endif
+ /* For files, it would be sufficient to pass 0 as size.
+ For anonymous maps, we have to pass the size explicitly. */
m_obj->map_handle = CreateFileMapping(m_obj->file_handle,
NULL,
flProtect,
@@ -1253,8 +1292,8 @@
if (m_obj->map_handle != NULL) {
m_obj->data = (char *) MapViewOfFile(m_obj->map_handle,
dwDesiredAccess,
- 0,
- 0,
+ off_hi,
+ off_lo,
0);
if (m_obj->data != NULL)
return (PyObject *)m_obj;
@@ -1329,6 +1368,8 @@
setint(dict, "PAGESIZE", (long)my_getpagesize());
+ setint(dict, "ALLOCATIONGRANULARITY", (long)my_getallocationgranularity());
+
setint(dict, "ACCESS_READ", ACCESS_READ);
setint(dict, "ACCESS_WRITE", ACCESS_WRITE);
setint(dict, "ACCESS_COPY", ACCESS_COPY);
More information about the Python-checkins
mailing list