[Python-checkins] bpo-32941: Add madvise() for mmap objects (GH-6172)
Antoine Pitrou
webhook-mailer at python.org
Mon May 27 12:48:20 EDT 2019
https://github.com/python/cpython/commit/02db696732c031d9a0265dc9bbf4f5b1fad042b3
commit: 02db696732c031d9a0265dc9bbf4f5b1fad042b3
branch: master
author: Zackery Spytz <zspytz at gmail.com>
committer: Antoine Pitrou <antoine at python.org>
date: 2019-05-27T18:48:16+02:00
summary:
bpo-32941: Add madvise() for mmap objects (GH-6172)
Allow mmap objects to access the madvise() system call.
files:
A Misc/NEWS.d/next/Library/2018-03-20-20-57-00.bpo-32941.9FU0gL.rst
M Doc/library/mmap.rst
M Doc/whatsnew/3.8.rst
M Lib/test/test_mmap.py
M Modules/mmapmodule.c
M configure
M configure.ac
M pyconfig.h.in
diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst
index a82caf86e801..c7a13abad888 100644
--- a/Doc/library/mmap.rst
+++ b/Doc/library/mmap.rst
@@ -203,6 +203,20 @@ To map anonymous memory, -1 should be passed as the fileno along with the length
exception was raised on error under Unix.
+ .. method:: madvise(option[, start[, length]])
+
+ Send advice *option* to the kernel about the memory region beginning at
+ *start* and extending *length* bytes. *option* must be one of the
+ :ref:`MADV_* constants <madvise-constants>` available on the system. If
+ *start* and *length* are omitted, the entire mapping is spanned. On
+ some systems (including Linux), *start* must be a multiple of the
+ :const:`PAGESIZE`.
+
+ Availability: Systems with the ``madvise()`` system call.
+
+ .. versionadded:: 3.8
+
+
.. method:: move(dest, src, count)
Copy the *count* bytes starting at offset *src* to the destination index
@@ -292,3 +306,38 @@ To map anonymous memory, -1 should be passed as the fileno along with the length
position of the file pointer; the file position is advanced by ``1``. If
the mmap was created with :const:`ACCESS_READ`, then writing to it will
raise a :exc:`TypeError` exception.
+
+.. _madvise-constants:
+
+MADV_* Constants
+++++++++++++++++
+
+.. data:: MADV_NORMAL
+ MADV_RANDOM
+ MADV_SEQUENTIAL
+ MADV_WILLNEED
+ MADV_DONTNEED
+ MADV_REMOVE
+ MADV_DONTFORK
+ MADV_DOFORK
+ MADV_HWPOISON
+ MADV_MERGEABLE
+ MADV_UNMERGEABLE
+ MADV_SOFT_OFFLINE
+ MADV_HUGEPAGE
+ MADV_NOHUGEPAGE
+ MADV_DONTDUMP
+ MADV_DODUMP
+ MADV_FREE
+ MADV_NOSYNC
+ MADV_AUTOSYNC
+ MADV_NOCORE
+ MADV_CORE
+ MADV_PROTECT
+
+ These options can be passed to :meth:`mmap.madvise`. Not every option will
+ be present on every system.
+
+ Availability: Systems with the madvise() system call.
+
+ .. versionadded:: 3.8
diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 6102d8c357ca..fd5e6496ba2e 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -460,6 +460,15 @@ numbers. (Contributed by Pablo Galindo in :issue:`35606`)
Added new function :func:`math.isqrt` for computing integer square roots.
(Contributed by Mark Dickinson in :issue:`36887`.)
+
+mmap
+----
+
+The :class:`mmap.mmap` class now has an :meth:`~mmap.mmap.madvise` method to
+access the ``madvise()`` system call.
+(Contributed by Zackery Spytz in :issue:`32941`.)
+
+
os
--
diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py
index 495d24ad8077..7b2b100dce01 100644
--- a/Lib/test/test_mmap.py
+++ b/Lib/test/test_mmap.py
@@ -739,6 +739,25 @@ def test_flush_return_value(self):
# See bpo-34754 for details.
self.assertRaises(OSError, mm.flush, 1, len(b'python'))
+ @unittest.skipUnless(hasattr(mmap.mmap, 'madvise'), 'needs madvise')
+ def test_madvise(self):
+ size = 8192
+ m = mmap.mmap(-1, size)
+
+ with self.assertRaisesRegex(ValueError, "madvise start out of bounds"):
+ m.madvise(mmap.MADV_NORMAL, size)
+ with self.assertRaisesRegex(ValueError, "madvise start out of bounds"):
+ m.madvise(mmap.MADV_NORMAL, -1)
+ with self.assertRaisesRegex(ValueError, "madvise length invalid"):
+ m.madvise(mmap.MADV_NORMAL, 0, -1)
+ with self.assertRaisesRegex(OverflowError, "madvise length too large"):
+ m.madvise(mmap.MADV_NORMAL, PAGESIZE, sys.maxsize)
+ self.assertEqual(m.madvise(mmap.MADV_NORMAL), None)
+ self.assertEqual(m.madvise(mmap.MADV_NORMAL, PAGESIZE), None)
+ self.assertEqual(m.madvise(mmap.MADV_NORMAL, PAGESIZE, size), None)
+ self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, 2), None)
+ self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, size), None)
+
class LargeMmapTests(unittest.TestCase):
diff --git a/Misc/NEWS.d/next/Library/2018-03-20-20-57-00.bpo-32941.9FU0gL.rst b/Misc/NEWS.d/next/Library/2018-03-20-20-57-00.bpo-32941.9FU0gL.rst
new file mode 100644
index 000000000000..f7668aecda6b
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-03-20-20-57-00.bpo-32941.9FU0gL.rst
@@ -0,0 +1,2 @@
+Allow :class:`mmap.mmap` objects to access the madvise() system call
+(through :meth:`mmap.mmap.madvise`).
diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c
index fdd60bbb6eef..36cbaf9fb8b2 100644
--- a/Modules/mmapmodule.c
+++ b/Modules/mmapmodule.c
@@ -708,11 +708,54 @@ mmap__sizeof__method(mmap_object *self, void *unused)
}
#endif
+#ifdef HAVE_MADVISE
+static PyObject *
+mmap_madvise_method(mmap_object *self, PyObject *args)
+{
+ int option;
+ Py_ssize_t start = 0, length;
+
+ CHECK_VALID(NULL);
+ length = self->size;
+
+ if (!PyArg_ParseTuple(args, "i|nn:madvise", &option, &start, &length)) {
+ return NULL;
+ }
+
+ if (start < 0 || start >= self->size) {
+ PyErr_SetString(PyExc_ValueError, "madvise start out of bounds");
+ return NULL;
+ }
+ if (length < 0) {
+ PyErr_SetString(PyExc_ValueError, "madvise length invalid");
+ return NULL;
+ }
+ if (PY_SSIZE_T_MAX - start < length) {
+ PyErr_SetString(PyExc_OverflowError, "madvise length too large");
+ return NULL;
+ }
+
+ if (start + length > self->size) {
+ length = self->size - start;
+ }
+
+ if (madvise(self->data + start, length, option) != 0) {
+ PyErr_SetFromErrno(PyExc_OSError);
+ return NULL;
+ }
+
+ Py_RETURN_NONE;
+}
+#endif // HAVE_MADVISE
+
static struct PyMethodDef mmap_object_methods[] = {
{"close", (PyCFunction) mmap_close_method, METH_NOARGS},
{"find", (PyCFunction) mmap_find_method, METH_VARARGS},
{"rfind", (PyCFunction) mmap_rfind_method, METH_VARARGS},
{"flush", (PyCFunction) mmap_flush_method, METH_VARARGS},
+#ifdef HAVE_MADVISE
+ {"madvise", (PyCFunction) mmap_madvise_method, METH_VARARGS},
+#endif
{"move", (PyCFunction) mmap_move_method, METH_VARARGS},
{"read", (PyCFunction) mmap_read_method, METH_VARARGS},
{"read_byte", (PyCFunction) mmap_read_byte_method, METH_NOARGS},
@@ -1494,5 +1537,80 @@ PyInit_mmap(void)
setint(dict, "ACCESS_READ", ACCESS_READ);
setint(dict, "ACCESS_WRITE", ACCESS_WRITE);
setint(dict, "ACCESS_COPY", ACCESS_COPY);
+
+#ifdef HAVE_MADVISE
+ // Conventional advice values
+#ifdef MADV_NORMAL
+ setint(dict, "MADV_NORMAL", MADV_NORMAL);
+#endif
+#ifdef MADV_RANDOM
+ setint(dict, "MADV_RANDOM", MADV_RANDOM);
+#endif
+#ifdef MADV_SEQUENTIAL
+ setint(dict, "MADV_SEQUENTIAL", MADV_SEQUENTIAL);
+#endif
+#ifdef MADV_WILLNEED
+ setint(dict, "MADV_WILLNEED", MADV_WILLNEED);
+#endif
+#ifdef MADV_DONTNEED
+ setint(dict, "MADV_DONTNEED", MADV_DONTNEED);
+#endif
+
+ // Linux-specific advice values
+#ifdef MADV_REMOVE
+ setint(dict, "MADV_REMOVE", MADV_REMOVE);
+#endif
+#ifdef MADV_DONTFORK
+ setint(dict, "MADV_DONTFORK", MADV_DONTFORK);
+#endif
+#ifdef MADV_DOFORK
+ setint(dict, "MADV_DOFORK", MADV_DOFORK);
+#endif
+#ifdef MADV_HWPOISON
+ setint(dict, "MADV_HWPOISON", MADV_HWPOISON);
+#endif
+#ifdef MADV_MERGEABLE
+ setint(dict, "MADV_MERGEABLE", MADV_MERGEABLE);
+#endif
+#ifdef MADV_UNMERGEABLE
+ setint(dict, "MADV_UNMERGEABLE", MADV_UNMERGEABLE);
+#endif
+#ifdef MADV_SOFT_OFFLINE
+ setint(dict, "MADV_SOFT_OFFLINE", MADV_SOFT_OFFLINE);
+#endif
+#ifdef MADV_HUGEPAGE
+ setint(dict, "MADV_HUGEPAGE", MADV_HUGEPAGE);
+#endif
+#ifdef MADV_NOHUGEPAGE
+ setint(dict, "MADV_NOHUGEPAGE", MADV_NOHUGEPAGE);
+#endif
+#ifdef MADV_DONTDUMP
+ setint(dict, "MADV_DONTDUMP", MADV_DONTDUMP);
+#endif
+#ifdef MADV_DODUMP
+ setint(dict, "MADV_DODUMP", MADV_DODUMP);
+#endif
+#ifdef MADV_FREE // (Also present on FreeBSD and macOS.)
+ setint(dict, "MADV_FREE", MADV_FREE);
+#endif
+
+ // FreeBSD-specific
+#ifdef MADV_NOSYNC
+ setint(dict, "MADV_NOSYNC", MADV_NOSYNC);
+#endif
+#ifdef MADV_AUTOSYNC
+ setint(dict, "MADV_AUTOSYNC", MADV_AUTOSYNC);
+#endif
+#ifdef MADV_NOCORE
+ setint(dict, "MADV_NOCORE", MADV_NOCORE);
+#endif
+#ifdef MADV_CORE
+ setint(dict, "MADV_CORE", MADV_CORE);
+#endif
+#ifdef MADV_PROTECT
+ setint(dict, "MADV_PROTECT", MADV_PROTECT);
+#endif
+#endif // HAVE_MADVISE
+
return module;
}
diff --git a/configure b/configure
index bc276ac58362..e8cbfeb154a5 100755
--- a/configure
+++ b/configure
@@ -11471,7 +11471,7 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \
getpriority getresuid getresgid getpwent getpwnam_r getpwuid_r getspnam getspent getsid getwd \
if_nameindex \
- initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \
+ initgroups kill killpg lchmod lchown lockf linkat lstat lutimes madvise mmap \
memrchr mbrtowc mkdirat mkfifo \
mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \
posix_fallocate posix_fadvise posix_spawn posix_spawnp pread preadv preadv2 \
diff --git a/configure.ac b/configure.ac
index 5e565191f27d..c743edfdeb18 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3527,7 +3527,7 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \
getpriority getresuid getresgid getpwent getpwnam_r getpwuid_r getspnam getspent getsid getwd \
if_nameindex \
- mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \
+ madvise mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \
posix_fallocate posix_fadvise posix_spawn posix_spawnp pread preadv preadv2 \
pthread_condattr_setclock pthread_init pthread_kill putenv pwrite pwritev pwritev2 \
readlink readlinkat readv realpath renameat \
diff --git a/pyconfig.h.in b/pyconfig.h.in
index dd5f2e393be0..cc6435541646 100644
--- a/pyconfig.h.in
+++ b/pyconfig.h.in
@@ -658,6 +658,9 @@
/* Define to 1 if you have the `lutimes' function. */
#undef HAVE_LUTIMES
+/* Define to 1 if you have the `madvise' function. */
+#undef HAVE_MADVISE
+
/* Define this if you have the makedev macro. */
#undef HAVE_MAKEDEV
More information about the Python-checkins
mailing list