Index: Doc/lib/libmmap.tex =================================================================== --- Doc/lib/libmmap.tex (Revision 52498) +++ Doc/lib/libmmap.tex (Arbeitskopie) @@ -40,7 +40,8 @@ the underlying file. \versionchanged[To map anonymous memory, -1 should be passed as the fileno along with the length]{2.5} -\begin{funcdesc}{mmap}{fileno, length\optional{, tagname\optional{, access}}} +\begin{funcdesc}{mmap}{fileno, length\optional{, tagname\optional{, + access\optional{, offset}}}} \strong{(Windows version)} Maps \var{length} bytes from the file specified by the file handle \var{fileno}, and returns a mmap object. If \var{length} is larger than the current size of the file, @@ -57,10 +58,15 @@ mapping is created without a name. Avoiding the use of the tag parameter will assist in keeping your code portable between \UNIX{} and Windows. + + \var{offset} may be specified as a non-negative integer offset. + mmap references will be relative to the offset from the beginning + of the file. +\versionchanged[The offset parameter was added]{2.3} \end{funcdesc} \begin{funcdescni}{mmap}{fileno, length\optional{, flags\optional{, - prot\optional{, access}}}} + prot\optional{, access\optional{, offset}}}}} \strong{(\UNIX{} version)} Maps \var{length} bytes from the file specified by the file descriptor \var{fileno}, and returns a mmap object. If \var{length} is \code{0}, the maximum length of the map @@ -83,6 +89,11 @@ as an optional keyword parameter. It is an error to specify both \var{flags}, \var{prot} and \var{access}. See the description of \var{access} above for information on how to use this parameter. + + \var{offset} may be specified as a non-negative integer offset. + mmap references will be relative to the offset from the beginning + of the file. +\versionchanged[The offset parameter was added]{2.3} \end{funcdescni} Index: Lib/test/output/test_mmap =================================================================== --- Lib/test/output/test_mmap (Revision 52498) +++ Lib/test/output/test_mmap (Arbeitskopie) @@ -1,38 +1,2 @@ test_mmap - - Position of foo: 1.0 pages - Length of file: 2.0 pages - Contents of byte 0: '\x00' - Contents of first 3 bytes: '\x00\x00\x00' - - Modifying file's content... - Contents of byte 0: '3' - Contents of first 3 bytes: '3\x00\x00' - Contents of second page: '\x00foobar\x00' - Regex match on mmap (page start, length of match): 1.0 6 - Seek to zeroth byte - Seek to 42nd byte - Seek to last byte - Try to seek to negative position... - Try to seek beyond end of mmap... - Try to seek to negative position... - Attempting resize() - Creating 10 byte test data file. - Opening mmap with access=ACCESS_READ - Ensuring that readonly mmap can't be slice assigned. - Ensuring that readonly mmap can't be item assigned. - Ensuring that readonly mmap can't be write() to. - Ensuring that readonly mmap can't be write_byte() to. - Ensuring that readonly mmap can't be resized. - Opening mmap with size too big - Opening mmap with access=ACCESS_WRITE - Modifying write-through memory map. - Opening mmap with access=ACCESS_COPY - Modifying copy-on-write memory map. - Ensuring copy-on-write maps cannot be resized. - Ensuring invalid access parameter raises exception. - Try opening a bad file descriptor... - Ensuring that passing 0 as map length sets map size to current file size. - Ensuring that passing 0 as map length sets map size to current file size. - anonymous mmap.mmap(-1, PAGESIZE)... Test passed Index: Lib/test/test_mmap.py =================================================================== --- Lib/test/test_mmap.py (Revision 52498) +++ Lib/test/test_mmap.py (Arbeitskopie) @@ -1,9 +1,55 @@ from test.test_support import verify, vereq, TESTFN import mmap -import os, re +import os, re, sys PAGESIZE = mmap.PAGESIZE +def make_mmap_file(f, halfsize): + # Write 2 pages worth of data to the file + f.write('\0'* halfsize) + f.write('foo') + f.write('\0'* (halfsize-3) ) + f.flush() + return mmap.mmap(f.fileno(), 2 * PAGESIZE) + +def test_offset(): + f = open(TESTFN, 'w+b') + + try: # unlink TESTFN no matter what + halfsize = PAGESIZE + if sys.platform.startswith('win'): + # halfsize must be a multiple of the allocation granularity (64k) + halfsize = 64 * 1024 * 1024 + m = make_mmap_file(f, halfsize) + m.close() + f.close() + + mapsize = halfsize * 2 + # Try invalid offset + f = open(TESTFN, "r+b") + for offset in [-2, -1, None]: + try: + m = mmap.mmap(f.fileno(), mapsize, offset=offset) + except (ValueError, TypeError, OverflowError): + pass + else: + verify(0, "Invalid offset should raise ValueError.") + f.close() + + # Try valid offset, hopefully 8192 works on all OSes + f = open(TESTFN, "r+b") + m = mmap.mmap(f.fileno(), mapsize - halfsize, offset=halfsize) + verify(m[0:3] == 'foo', "Invalid data found using offset.") + f.close() + m.close() + + finally: + f.close() + try: + os.unlink(TESTFN) + except OSError: + pass + def test_both(): "Test mmap module on Unix systems and Windows" @@ -13,12 +59,7 @@ f = open(TESTFN, 'w+') try: # unlink TESTFN no matter what - # Write 2 pages worth of data to the file - f.write('\0'* PAGESIZE) - f.write('foo') - f.write('\0'* (PAGESIZE-3) ) - f.flush() - m = mmap.mmap(f.fileno(), 2 * PAGESIZE) + m = make_mmap_file(f, PAGESIZE) f.close() # Simple sanity checks @@ -390,6 +431,7 @@ m[x] = ch = chr(x & 255) vereq(m[x], ch) -test_both() -test_anon() +#test_both() +#test_anon() +test_offset() print ' Test passed' Index: Modules/mmapmodule.c =================================================================== --- Modules/mmapmodule.c (Revision 52498) +++ Modules/mmapmodule.c (Arbeitskopie) @@ -3,6 +3,9 @@ / Hacked for Unix by AMK / $Id$ + / Modified to support mmap with offset - to map a 'window' of a file + / Author: Yotam Medini yotamm@mellanox.co.il + / / mmapmodule.cpp -- map a view of a file into memory / / todo: need permission flags, perhaps a 'chsize' analog @@ -74,7 +77,8 @@ PyObject_HEAD char * data; size_t size; - size_t pos; + size_t pos; /* relative to offset */ + size_t offset; /* always 0 for MS_WINDOWS */ #ifdef MS_WINDOWS HANDLE map_handle; @@ -258,7 +262,7 @@ for (i = 0; i < len && needle[i] == p[i]; ++i) /* nothing */; if (i == len) { - return PyInt_FromSsize_t(p - self->data); + return PyInt_FromSsize_t(self->offset + p - self->data); } } return PyInt_FromLong(-1); @@ -387,18 +391,22 @@ #ifdef MS_WINDOWS } else { DWORD dwErrCode = 0; - DWORD newSizeLow, newSizeHigh; + DWORD off_hi, off_lo, newSizeLow, newSizeHigh; /* First, unmap the file view */ UnmapViewOfFile(self->data); /* Close the mapping object */ CloseHandle(self->map_handle); /* Move to the desired EOF position */ #if SIZEOF_SIZE_T > 4 - newSizeHigh = (DWORD)(new_size >> 32); - newSizeLow = (DWORD)(new_size & 0xFFFFFFFF); + newSizeHigh = (DWORD)((self->offset + new_size) >> 32); + newSizeLow = (DWORD)((self->offset + new_size) & 0xFFFFFFFF); + off_hi = (DWORD)(self->offset >> 32); + off_lo = (DWORD)(self->offset & 0xFFFFFFFF); #else newSizeHigh = 0; newSizeLow = (DWORD)new_size; + off_hi = 0; + off_lo = (DWORD)self->offset; #endif SetFilePointer(self->file_handle, newSizeLow, &newSizeHigh, FILE_BEGIN); @@ -409,15 +417,15 @@ self->file_handle, NULL, PAGE_READWRITE, - newSizeHigh, - newSizeLow, + 0, + 0, self->tagname); if (self->map_handle != NULL) { self->data = (char *) MapViewOfFile(self->map_handle, FILE_MAP_WRITE, - 0, - 0, - 0); + off_hi, + off_lo, + new_size); if (self->data != NULL) { self->size = new_size; Py_INCREF(Py_None); @@ -470,7 +478,7 @@ mmap_tell_method(mmap_object *self, PyObject *unused) { CHECK_VALID(NULL); - return PyInt_FromSize_t(self->pos); + return PyInt_FromSize_t(self->offset + self->pos); } static PyObject * @@ -515,6 +523,7 @@ size_t where; switch (how) { case 0: /* relative to start */ + dist -= self->offset; if (dist < 0) goto onoutofrange; where = dist; @@ -554,6 +563,8 @@ !is_writeable(self)) { return NULL; } else { + dest -= self->offset; + src -= self->offset; /* bounds check the values */ if (/* end of source after end of data?? */ ((src+count) > self->size) @@ -813,15 +824,18 @@ Returns -1 on error, with an appropriate Python exception raised. On success, the map size is returned. */ static Py_ssize_t -_GetMapSize(PyObject *o) +_GetMapSize(PyObject *o, const char* param) { + if (o == NULL) + return 0; if (PyIndex_Check(o)) { Py_ssize_t i = PyNumber_AsSsize_t(o, PyExc_OverflowError); if (i==-1 && PyErr_Occurred()) return -1; if (i < 0) { - PyErr_SetString(PyExc_OverflowError, - "memory mapped size must be positive"); + PyErr_Format(PyExc_OverflowError, + "memory mapped %s must be positive", + param); return -1; } return i; @@ -839,22 +853,25 @@ struct stat st; #endif mmap_object *m_obj; - PyObject *map_size_obj = NULL; - Py_ssize_t map_size; + PyObject *map_size_obj = NULL, *offset_obj = NULL; + Py_ssize_t map_size, offset; int fd, flags = MAP_SHARED, prot = PROT_WRITE | PROT_READ; int devzero = -1; int access = (int)ACCESS_DEFAULT; static char *keywords[] = {"fileno", "length", "flags", "prot", - "access", NULL}; + "access", "offset", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|iii", keywords, + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|iiiO", keywords, &fd, &map_size_obj, &flags, &prot, - &access)) + &access, &offset_obj)) return NULL; - map_size = _GetMapSize(map_size_obj); + map_size = _GetMapSize(map_size_obj, "size"); if (map_size < 0) return NULL; + offset = _GetMapSize(offset_obj, "offset"); + if (offset < 0) + return NULL; if ((access != (int)ACCESS_DEFAULT) && ((flags != MAP_SHARED) || (prot != (PROT_WRITE | PROT_READ)))) @@ -889,7 +906,7 @@ if (fstat(fd, &st) == 0 && S_ISREG(st.st_mode)) { if (map_size == 0) { map_size = st.st_size; - } else if ((size_t)map_size > st.st_size) { + } else if ((size_t)offset + (size_t)map_size > st.st_size) { PyErr_SetString(PyExc_ValueError, "mmap length is greater than file size"); return NULL; @@ -901,6 +918,7 @@ m_obj->data = NULL; m_obj->size = (size_t) map_size; m_obj->pos = (size_t) 0; + m_obj->offset = offset; if (fd == -1) { m_obj->fd = -1; /* Assume the caller wants to map anonymous memory. @@ -930,7 +948,7 @@ m_obj->data = mmap(NULL, map_size, prot, flags, - fd, 0); + fd, offset); if (devzero != -1) { close(devzero); @@ -952,10 +970,12 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict) { mmap_object *m_obj; - PyObject *map_size_obj = NULL; - Py_ssize_t map_size; - DWORD size_hi; /* upper 32 bits of m_obj->size */ - DWORD size_lo; /* lower 32 bits of m_obj->size */ + PyObject *map_size_obj = NULL, *offset_obj = NULL; + Py_ssize_t map_size, offset; + DWORD off_hi; /* upper 32 bits of offset */ + DWORD off_lo; /* lower 32 bits of offset */ + DWORD size_hi; /* upper 32 bits of size */ + DWORD size_lo; /* lower 32 bits of size */ char *tagname = ""; DWORD dwErr = 0; int fileno; @@ -964,11 +984,11 @@ DWORD flProtect, dwDesiredAccess; static char *keywords[] = { "fileno", "length", "tagname", - "access", NULL }; + "access", "offset", NULL }; - if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|zi", keywords, + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|ziO", keywords, &fileno, &map_size_obj, - &tagname, &access)) { + &tagname, &access, &offset_obj)) { return NULL; } @@ -990,9 +1010,12 @@ "mmap invalid access parameter."); } - map_size = _GetMapSize(map_size_obj); + map_size = _GetMapSize(map_size_obj, "size"); if (map_size < 0) return NULL; + offset = _GetMapSize(offset_obj, "offset"); + if (offset < 0) + return NULL; /* assume -1 and 0 both mean invalid filedescriptor to 'anonymously' map memory. @@ -1021,6 +1044,7 @@ m_obj->file_handle = INVALID_HANDLE_VALUE; m_obj->map_handle = INVALID_HANDLE_VALUE; m_obj->tagname = NULL; + m_obj->offset = offset; if (fh) { /* It is necessary to duplicate the handle, so the @@ -1089,12 +1113,18 @@ * right by 32, so we need different code. */ #if SIZEOF_SIZE_T > 4 - size_hi = (DWORD)(m_obj->size >> 32); - size_lo = (DWORD)(m_obj->size & 0xFFFFFFFF); + size_hi = (DWORD)((offset + m_obj->size) >> 32); + size_lo = (DWORD)((offset + m_obj->size) & 0xFFFFFFFF); + off_hi = (DWORD)(offset >> 32); + off_lo = (DWORD)(offset & 0xFFFFFFFF); #else size_hi = 0; - size_lo = (DWORD)m_obj->size; + size_lo = (DWORD)(offset + m_obj->size); + off_hi = 0; + off_lo = (DWORD)offset; #endif + /* For files, it would be sufficient to pass 0 as size. + For anonymous maps, we have to pass the size explicitly. */ m_obj->map_handle = CreateFileMapping(m_obj->file_handle, NULL, flProtect, @@ -1104,8 +1134,8 @@ if (m_obj->map_handle != NULL) { m_obj->data = (char *) MapViewOfFile(m_obj->map_handle, dwDesiredAccess, - 0, - 0, + off_hi, + off_lo, 0); if (m_obj->data != NULL) return (PyObject *)m_obj;