[pypy-commit] pypy merge-2.7.2: Fix for mmap when trying to specify an offset that's past the end of a file.
alex_gaynor
noreply at buildbot.pypy.org
Sun Jan 22 22:26:52 CET 2012
Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: merge-2.7.2
Changeset: r51667:b0b180ee94ea
Date: 2012-01-22 15:26 -0600
http://bitbucket.org/pypy/pypy/changeset/b0b180ee94ea/
Log: Fix for mmap when trying to specify an offset that's past the end of
a file.
diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -8,7 +8,7 @@
space = gettestobjspace(usemodules=('mmap',))
cls.space = space
cls.w_tmpname = space.wrap(str(udir.join('mmap-')))
-
+
def test_page_size(self):
import mmap
assert mmap.PAGESIZE > 0
@@ -43,7 +43,7 @@
raises(TypeError, mmap, "foo")
raises(TypeError, mmap, 0, "foo")
-
+
if os.name == "posix":
raises(ValueError, mmap, 0, 1, 2, 3, 4)
raises(TypeError, mmap, 0, 1, 2, 3, "foo", 5)
@@ -72,7 +72,7 @@
from mmap import mmap
f = open(self.tmpname + "a", "w+")
-
+
f.write("c")
f.flush()
raises(ValueError, mmap, f.fileno(), 123)
@@ -81,18 +81,18 @@
def test_create(self):
from mmap import mmap
f = open(self.tmpname + "b", "w+")
-
+
f.write("c")
f.flush()
m = mmap(f.fileno(), 1)
assert m.read(99) == "c"
-
+
f.close()
def test_close(self):
from mmap import mmap
f = open(self.tmpname + "c", "w+")
-
+
f.write("c")
f.flush()
m = mmap(f.fileno(), 1)
@@ -131,7 +131,7 @@
def test_read(self):
from mmap import mmap
f = open(self.tmpname + "f", "w+")
-
+
f.write("foobar")
f.flush()
m = mmap(f.fileno(), 6)
@@ -217,7 +217,7 @@
def test_is_modifiable(self):
import mmap
f = open(self.tmpname + "h", "w+")
-
+
f.write("foobar")
f.flush()
m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ)
@@ -229,7 +229,7 @@
def test_seek(self):
from mmap import mmap
f = open(self.tmpname + "i", "w+")
-
+
f.write("foobar")
f.flush()
m = mmap(f.fileno(), 6)
@@ -270,7 +270,7 @@
def test_write_byte(self):
import mmap
f = open(self.tmpname + "k", "w+")
-
+
f.write("foobar")
f.flush()
m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ)
@@ -286,7 +286,7 @@
def test_size(self):
from mmap import mmap
f = open(self.tmpname + "l", "w+")
-
+
f.write("foobar")
f.flush()
m = mmap(f.fileno(), 5)
@@ -297,7 +297,7 @@
def test_tell(self):
from mmap import mmap
f = open(self.tmpname + "m", "w+")
-
+
f.write("c")
f.flush()
m = mmap(f.fileno(), 1)
@@ -308,7 +308,7 @@
def test_flush(self):
from mmap import mmap
f = open(self.tmpname + "n", "w+")
-
+
f.write("foobar")
f.flush()
m = mmap(f.fileno(), 6)
@@ -319,10 +319,18 @@
m.close()
f.close()
+ def test_length_0_large_offset(self):
+ import mmap
+
+ with open(self.tmpname, "wb") as f:
+ f.write(115699 * 'm')
+ with open(self.tmpname, "w+b") as f:
+ raises(ValueError, mmap.mmap, f.fileno(), 0, offset=2147418112)
+
def test_move(self):
import mmap
f = open(self.tmpname + "o", "w+")
-
+
f.write("foobar")
f.flush()
m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ)
@@ -339,15 +347,15 @@
assert a == "frarar"
m.close()
f.close()
-
+
def test_resize(self):
import sys
if ("darwin" in sys.platform) or ("freebsd" in sys.platform):
skip("resize does not work under OSX or FreeBSD")
-
+
import mmap
import os
-
+
f = open(self.tmpname + "p", "w+")
f.write("foobar")
f.flush()
@@ -368,10 +376,10 @@
import sys
if ("darwin" not in sys.platform) and ("freebsd" not in sys.platform):
skip("resize works under not OSX or FreeBSD")
-
+
import mmap
import os
-
+
f = open(self.tmpname + "p", "w+")
f.write("foobar")
f.flush()
@@ -388,7 +396,7 @@
def test_len(self):
from mmap import mmap
-
+
f = open(self.tmpname + "q", "w+")
f.write("foobar")
f.flush()
@@ -397,14 +405,14 @@
assert len(m) == 6
m.close()
f.close()
-
+
def test_get_item(self):
from mmap import mmap
-
+
f = open(self.tmpname + "r", "w+")
f.write("foobar")
f.flush()
-
+
m = mmap(f.fileno(), 6)
fn = lambda: m["foo"]
raises(TypeError, fn)
@@ -416,10 +424,10 @@
assert m[4:1:-2] == 'ao'
m.close()
f.close()
-
+
def test_set_item(self):
import mmap
-
+
f = open(self.tmpname + "s", "w+")
f.write("foobar")
f.flush()
@@ -455,14 +463,14 @@
assert data == "yxxBaR"
m.close()
f.close()
-
+
def test_del_item(self):
from mmap import mmap
-
+
f = open(self.tmpname + "t", "w+")
f.write("foobar")
f.flush()
-
+
m = mmap(f.fileno(), 6)
def fn(): del m["foo"]
raises(TypeError, fn)
@@ -475,11 +483,11 @@
def test_concatenation(self):
from mmap import mmap
-
+
f = open(self.tmpname + "u", "w+")
f.write("foobar")
f.flush()
-
+
m = mmap(f.fileno(), 6)
def fn(): m + 1
raises((SystemError, TypeError), fn) # SystemError is in CPython,
@@ -492,11 +500,11 @@
def test_repeatition(self):
from mmap import mmap
-
+
f = open(self.tmpname + "v", "w+")
f.write("foobar")
f.flush()
-
+
m = mmap(f.fileno(), 6)
def fn(): m * 1
raises((SystemError, TypeError), fn) # SystemError is in CPython,
@@ -506,14 +514,14 @@
raises((SystemError, TypeError), fn)
m.close()
f.close()
-
+
def test_slicing(self):
from mmap import mmap
f = open(self.tmpname + "v", "w+")
f.write("foobar")
f.flush()
-
+
f.seek(0)
m = mmap(f.fileno(), 6)
assert m[-3:7] == "bar"
@@ -589,11 +597,11 @@
from mmap import PAGESIZE
import sys
import os
-
+
filename = self.tmpname + "w"
-
+
f = open(filename, "w+")
-
+
# write 2 pages worth of data to the file
f.write('\0' * PAGESIZE)
f.write('foo')
@@ -601,24 +609,24 @@
f.flush()
m = mmap.mmap(f.fileno(), 2 * PAGESIZE)
f.close()
-
+
# sanity checks
assert m.find("foo") == PAGESIZE
assert len(m) == 2 * PAGESIZE
assert m[0] == '\0'
assert m[0:3] == '\0\0\0'
-
+
# modify the file's content
m[0] = '3'
m[PAGESIZE+3:PAGESIZE+3+3] = 'bar'
-
+
# check that the modification worked
assert m[0] == '3'
assert m[0:3] == '3\0\0'
assert m[PAGESIZE-1:PAGESIZE+7] == '\0foobar\0'
m.flush()
-
+
# test seeking around
m.seek(0,0)
assert m.tell() == 0
@@ -626,28 +634,28 @@
assert m.tell() == 42
m.seek(0, 2)
assert m.tell() == len(m)
-
+
raises(ValueError, m.seek, -1)
raises(ValueError, m.seek, 1, 2)
raises(ValueError, m.seek, -len(m) - 1, 2)
-
+
# try resizing map
if not (("darwin" in sys.platform) or ("freebsd" in sys.platform)):
m.resize(512)
-
+
assert len(m) == 512
raises(ValueError, m.seek, 513, 0)
-
+
# check that the underlying file is truncated too
f = open(filename)
f.seek(0, 2)
assert f.tell() == 512
f.close()
assert m.size() == 512
-
+
m.close()
f.close()
-
+
# test access=ACCESS_READ
mapsize = 10
f = open(filename, "wb")
@@ -667,21 +675,21 @@
if not (("darwin" in sys.platform) or ("freebsd" in sys.platform)):
raises(TypeError, m.resize, 2 * mapsize)
assert open(filename, "rb").read() == 'a' * mapsize
-
+
# opening with size too big
f = open(filename, "r+b")
if not os.name == "nt":
# this should work under windows
raises(ValueError, mmap.mmap, f.fileno(), mapsize + 1)
f.close()
-
+
# if _MS_WINDOWS:
# # repair damage from the resizing test.
# f = open(filename, 'r+b')
# f.truncate(mapsize)
# f.close()
m.close()
-
+
# test access=ACCESS_WRITE"
f = open(filename, "r+b")
m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_WRITE)
@@ -696,7 +704,7 @@
stuff = f.read()
f.close()
assert stuff == 'c' * mapsize
-
+
# test access=ACCESS_COPY
f = open(filename, "r+b")
m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_COPY)
@@ -710,12 +718,12 @@
raises(TypeError, m.resize, 2 * mapsize)
m.close()
f.close()
-
+
# test invalid access
f = open(filename, "r+b")
raises(ValueError, mmap.mmap, f.fileno(), mapsize, access=4)
f.close()
-
+
# test incompatible parameters
if os.name == "posix":
f = open(filename, "r+b")
@@ -723,10 +731,10 @@
prot=mmap.PROT_READ, access=mmap.ACCESS_WRITE)
f.close()
-
+
# bad file descriptor
raises(EnvironmentError, mmap.mmap, -2, 4096)
-
+
# do a tougher .find() test. SF bug 515943 pointed out that, in 2.2,
# searching for data with embedded \0 bytes didn't work.
f = open(filename, 'w+')
@@ -736,14 +744,14 @@
f.flush()
m = mmap.mmap(f.fileno(), n)
f.close()
-
+
for start in range(n + 1):
for finish in range(start, n + 1):
sl = data[start:finish]
assert m.find(sl) == data.find(sl)
assert m.find(sl + 'x') == -1
m.close()
-
+
# test mapping of entire file by passing 0 for map length
f = open(filename, "w+")
f.write(2**16 * 'm')
@@ -754,7 +762,7 @@
assert m.read(2**16) == 2**16 * "m"
m.close()
f.close()
-
+
# make move works everywhere (64-bit format problem earlier)
f = open(filename, 'w+')
f.write("ABCDEabcde")
diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -43,7 +43,7 @@
constants = {}
if _POSIX:
# constants, look in sys/mman.h and platform docs for the meaning
- # some constants are linux only so they will be correctly exposed outside
+ # some constants are linux only so they will be correctly exposed outside
# depending on the OS
constant_names = ['MAP_SHARED', 'MAP_PRIVATE',
'PROT_READ', 'PROT_WRITE',
@@ -136,7 +136,7 @@
)
SYSINFO_UNION = rffi.CStruct(
- 'union SYSINFO_UNION',
+ 'union SYSINFO_UNION',
("dwOemId", DWORD),
("_struct_", SYSINFO_STRUCT),
)
@@ -250,7 +250,7 @@
elif _POSIX:
self.fd = -1
self.closed = False
-
+
def check_valid(self):
if _MS_WINDOWS:
to_close = self.map_handle == INVALID_HANDLE
@@ -259,11 +259,11 @@
if to_close:
raise RValueError("map closed or invalid")
-
+
def check_writeable(self):
if not (self.access != ACCESS_READ):
raise RTypeError("mmap can't modify a readonly memory map.")
-
+
def check_resizeable(self):
if not (self.access == ACCESS_WRITE or self.access == _ACCESS_DEFAULT):
raise RTypeError("mmap can't resize a readonly or copy-on-write memory map.")
@@ -273,7 +273,7 @@
assert size >= 0
self.data = data
self.size = size
-
+
def close(self):
if _MS_WINDOWS:
if self.size > 0:
@@ -302,7 +302,7 @@
def unmapview(self):
UnmapViewOfFile(self.getptr(0))
-
+
def read_byte(self):
self.check_valid()
@@ -312,7 +312,7 @@
return value
else:
raise RValueError("read byte out of range")
-
+
def readline(self):
self.check_valid()
@@ -327,7 +327,7 @@
res = "".join([data[i] for i in range(self.pos, eol)])
self.pos += len(res)
return res
-
+
def read(self, num=-1):
self.check_valid()
@@ -389,10 +389,10 @@
def seek(self, pos, whence=0):
self.check_valid()
-
+
dist = pos
how = whence
-
+
if how == 0: # relative to start
where = dist
elif how == 1: # relative to current position
@@ -404,16 +404,16 @@
if not (0 <= where <= self.size):
raise RValueError("seek out of range")
-
+
self.pos = where
-
+
def tell(self):
self.check_valid()
return self.pos
-
+
def file_size(self):
self.check_valid()
-
+
size = self.size
if _MS_WINDOWS:
if self.file_handle != INVALID_HANDLE:
@@ -433,11 +433,11 @@
else:
size = int(size)
return size
-
+
def write(self, data):
- self.check_valid()
+ self.check_valid()
self.check_writeable()
-
+
data_len = len(data)
if self.pos + data_len > self.size:
raise RValueError("data out of range")
@@ -447,10 +447,10 @@
for i in range(data_len):
internaldata[start+i] = data[i]
self.pos = start + data_len
-
+
def write_byte(self, byte):
self.check_valid()
-
+
if len(byte) != 1:
raise RTypeError("write_byte() argument must be char")
@@ -491,14 +491,14 @@
if res == -1:
errno = rposix.get_errno()
raise OSError(errno, os.strerror(errno))
-
+
return 0
-
+
def move(self, dest, src, count):
self.check_valid()
-
+
self.check_writeable()
-
+
# check boundings
if (src < 0 or dest < 0 or count < 0 or
src + count > self.size or dest + count > self.size):
@@ -507,19 +507,19 @@
datasrc = self.getptr(src)
datadest = self.getptr(dest)
c_memmove(datadest, datasrc, count)
-
+
def resize(self, newsize):
self.check_valid()
-
+
self.check_resizeable()
-
+
if _POSIX:
if not has_mremap:
raise RValueError("mmap: resizing not available--no mremap()")
-
+
# resize the underlying file first
os.ftruncate(self.fd, self.offset + newsize)
-
+
# now resize the mmap
newdata = c_mremap(self.getptr(0), self.size, newsize,
MREMAP_MAYMOVE or 0)
@@ -573,9 +573,9 @@
def len(self):
self.check_valid()
-
+
return self.size
-
+
def getitem(self, index):
self.check_valid()
# simplified version, for rpython
@@ -650,6 +650,8 @@
size = int(size)
if stat.S_ISREG(mode):
if map_size == 0:
+ if offset > st[stat.ST_SIZE]:
+ raise RValueError("mmap offset is greater than file size")
map_size = size
elif map_size > size:
raise RValueError("mmap length is greater than file size")
@@ -673,7 +675,7 @@
if res == rffi.cast(PTR, -1):
errno = rposix.get_errno()
raise OSError(errno, os.strerror(errno))
-
+
m.setdata(res, map_size)
return m
@@ -704,7 +706,7 @@
alloc._annenforceargs_ = (int,)
free = c_munmap_safe
-
+
elif _MS_WINDOWS:
def mmap(fileno, length, tagname="", access=_ACCESS_DEFAULT, offset=0):
# check size boundaries
@@ -712,11 +714,11 @@
map_size = length
if offset < 0:
raise RValueError("negative offset")
-
+
flProtect = 0
dwDesiredAccess = 0
fh = NULL_HANDLE
-
+
if access == ACCESS_READ:
flProtect = PAGE_READONLY
dwDesiredAccess = FILE_MAP_READ
@@ -728,7 +730,7 @@
dwDesiredAccess = FILE_MAP_COPY
else:
raise RValueError("mmap invalid access parameter.")
-
+
# assume -1 and 0 both mean invalid file descriptor
# to 'anonymously' map memory.
if fileno != -1 and fileno != 0:
@@ -739,7 +741,7 @@
# Win9x appears to need us seeked to zero
# SEEK_SET = 0
# libc._lseek(fileno, 0, SEEK_SET)
-
+
m = MMap(access, offset)
m.file_handle = INVALID_HANDLE
m.map_handle = INVALID_HANDLE
@@ -752,7 +754,7 @@
res = DuplicateHandle(GetCurrentProcess(), # source process handle
fh, # handle to be duplicated
GetCurrentProcess(), # target process handle
- handle_ref, # result
+ handle_ref, # result
0, # access - ignored due to options value
False, # inherited by child procs?
DUPLICATE_SAME_ACCESS) # options
@@ -761,7 +763,7 @@
m.file_handle = handle_ref[0]
finally:
lltype.free(handle_ref, flavor='raw')
-
+
if not map_size:
low, high = _get_file_size(fh)
if _64BIT:
@@ -775,7 +777,7 @@
if tagname:
m.tagname = tagname
-
+
# DWORD is a 4-byte int. If int > 4-byte it must be divided
if _64BIT:
size_hi = (map_size + offset) >> 32
@@ -807,7 +809,7 @@
def alloc(map_size):
"""Allocate memory. This is intended to be used by the JIT,
- so the memory has the executable bit set.
+ so the memory has the executable bit set.
XXX implement me: it should get allocated internally in
case of a sandboxed process
"""
@@ -825,5 +827,5 @@
def free(ptr, map_size):
VirtualFree(ptr, 0, MEM_RELEASE)
-
+
# register_external here?
More information about the pypy-commit
mailing list