[Python-checkins] bpo-26826: Expose copy_file_range in the os module (GH-7255)
Pablo Galindo
webhook-mailer at python.org
Fri May 31 14:39:53 EDT 2019
https://github.com/python/cpython/commit/aac4d0342c3e692731c189d003dbd73a8c681a34
commit: aac4d0342c3e692731c189d003dbd73a8c681a34
branch: master
author: Pablo Galindo <Pablogsal at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-05-31T19:39:47+01:00
summary:
bpo-26826: Expose copy_file_range in the os module (GH-7255)
files:
A Misc/NEWS.d/next/Core and Builtins/2018-05-30-23-43-03.bpo-26826.NkRzjb.rst
M Doc/library/os.rst
M Lib/test/test_os.py
M Modules/clinic/posixmodule.c.h
M Modules/posixmodule.c
M aclocal.m4
M configure
M configure.ac
M pyconfig.h.in
diff --git a/Doc/library/os.rst b/Doc/library/os.rst
index b53fd71e65b3..107764ba4d53 100644
--- a/Doc/library/os.rst
+++ b/Doc/library/os.rst
@@ -707,6 +707,28 @@ as internal buffering of data.
pass
+.. function:: copy_file_range(src, dst, count, offset_src=None, offset_dst=None)
+
+ Copy *count* bytes from file descriptor *src*, starting from offset
+ *offset_src*, to file descriptor *dst*, starting from offset *offset_dst*.
+ If *offset_src* is None, then *src* is read from the current position;
+ respectively for *offset_dst*. The files pointed by *src* and *dst*
+ must reside in the same filesystem, otherwise an :exc:`OSError` is
+ raised with :attr:`~OSError.errno` set to :data:`errno.EXDEV`.
+
+ This copy is done without the additional cost of transferring data
+ from the kernel to user space and then back into the kernel. Additionally,
+ some filesystems could implement extra optimizations. The copy is done as if
+ both files are opened as binary.
+
+ The return value is the amount of bytes copied. This could be less than the
+ amount requested.
+
+ .. availability:: Linux kernel >= 4.5 or glibc >= 2.27.
+
+ .. versionadded:: 3.8
+
+
.. function:: device_encoding(fd)
Return a string describing the encoding of the device associated with *fd*
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index f17a19a7585d..a8eae6162057 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -231,6 +231,89 @@ def test_symlink_keywords(self):
except (NotImplementedError, OSError):
pass # No OS support or unprivileged user
+ @unittest.skipUnless(hasattr(os, 'copy_file_range'), 'test needs os.copy_file_range()')
+ def test_copy_file_range_invalid_values(self):
+ with self.assertRaises(ValueError):
+ os.copy_file_range(0, 1, -10)
+
+ @unittest.skipUnless(hasattr(os, 'copy_file_range'), 'test needs os.copy_file_range()')
+ def test_copy_file_range(self):
+ TESTFN2 = support.TESTFN + ".3"
+ data = b'0123456789'
+
+ create_file(support.TESTFN, data)
+ self.addCleanup(support.unlink, support.TESTFN)
+
+ in_file = open(support.TESTFN, 'rb')
+ self.addCleanup(in_file.close)
+ in_fd = in_file.fileno()
+
+ out_file = open(TESTFN2, 'w+b')
+ self.addCleanup(support.unlink, TESTFN2)
+ self.addCleanup(out_file.close)
+ out_fd = out_file.fileno()
+
+ try:
+ i = os.copy_file_range(in_fd, out_fd, 5)
+ except OSError as e:
+ # Handle the case in which Python was compiled
+ # in a system with the syscall but without support
+ # in the kernel.
+ if e.errno != errno.ENOSYS:
+ raise
+ self.skipTest(e)
+ else:
+ # The number of copied bytes can be less than
+ # the number of bytes originally requested.
+ self.assertIn(i, range(0, 6));
+
+ with open(TESTFN2, 'rb') as in_file:
+ self.assertEqual(in_file.read(), data[:i])
+
+ @unittest.skipUnless(hasattr(os, 'copy_file_range'), 'test needs os.copy_file_range()')
+ def test_copy_file_range_offset(self):
+ TESTFN4 = support.TESTFN + ".4"
+ data = b'0123456789'
+ bytes_to_copy = 6
+ in_skip = 3
+ out_seek = 5
+
+ create_file(support.TESTFN, data)
+ self.addCleanup(support.unlink, support.TESTFN)
+
+ in_file = open(support.TESTFN, 'rb')
+ self.addCleanup(in_file.close)
+ in_fd = in_file.fileno()
+
+ out_file = open(TESTFN4, 'w+b')
+ self.addCleanup(support.unlink, TESTFN4)
+ self.addCleanup(out_file.close)
+ out_fd = out_file.fileno()
+
+ try:
+ i = os.copy_file_range(in_fd, out_fd, bytes_to_copy,
+ offset_src=in_skip,
+ offset_dst=out_seek)
+ except OSError as e:
+ # Handle the case in which Python was compiled
+ # in a system with the syscall but without support
+ # in the kernel.
+ if e.errno != errno.ENOSYS:
+ raise
+ self.skipTest(e)
+ else:
+ # The number of copied bytes can be less than
+ # the number of bytes originally requested.
+ self.assertIn(i, range(0, bytes_to_copy+1));
+
+ with open(TESTFN4, 'rb') as in_file:
+ read = in_file.read()
+ # seeked bytes (5) are zero'ed
+ self.assertEqual(read[:out_seek], b'\x00'*out_seek)
+ # 012 are skipped (in_skip)
+ # 345678 are copied in the file (in_skip + bytes_to_copy)
+ self.assertEqual(read[out_seek:],
+ data[in_skip:in_skip+i])
# Test attributes on return values from os.*stat* family.
class StatAttributeTests(unittest.TestCase):
diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-05-30-23-43-03.bpo-26826.NkRzjb.rst b/Misc/NEWS.d/next/Core and Builtins/2018-05-30-23-43-03.bpo-26826.NkRzjb.rst
new file mode 100644
index 000000000000..27d7f82a672d
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2018-05-30-23-43-03.bpo-26826.NkRzjb.rst
@@ -0,0 +1 @@
+Expose :func:`copy_file_range` as a low level API in the :mod:`os` module.
diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h
index 13f25460b4f6..22cb94761de5 100644
--- a/Modules/clinic/posixmodule.c.h
+++ b/Modules/clinic/posixmodule.c.h
@@ -5395,6 +5395,108 @@ os_pwritev(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
#endif /* (defined(HAVE_PWRITEV) || defined (HAVE_PWRITEV2)) */
+#if defined(HAVE_COPY_FILE_RANGE)
+
+PyDoc_STRVAR(os_copy_file_range__doc__,
+"copy_file_range($module, /, src, dst, count, offset_src=None,\n"
+" offset_dst=None)\n"
+"--\n"
+"\n"
+"Copy count bytes from one file descriptor to another.\n"
+"\n"
+" src\n"
+" Source file descriptor.\n"
+" dst\n"
+" Destination file descriptor.\n"
+" count\n"
+" Number of bytes to copy.\n"
+" offset_src\n"
+" Starting offset in src.\n"
+" offset_dst\n"
+" Starting offset in dst.\n"
+"\n"
+"If offset_src is None, then src is read from the current position;\n"
+"respectively for offset_dst.");
+
+#define OS_COPY_FILE_RANGE_METHODDEF \
+ {"copy_file_range", (PyCFunction)(void(*)(void))os_copy_file_range, METH_FASTCALL|METH_KEYWORDS, os_copy_file_range__doc__},
+
+static PyObject *
+os_copy_file_range_impl(PyObject *module, int src, int dst, Py_ssize_t count,
+ PyObject *offset_src, PyObject *offset_dst);
+
+static PyObject *
+os_copy_file_range(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+ PyObject *return_value = NULL;
+ static const char * const _keywords[] = {"src", "dst", "count", "offset_src", "offset_dst", NULL};
+ static _PyArg_Parser _parser = {NULL, _keywords, "copy_file_range", 0};
+ PyObject *argsbuf[5];
+ Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3;
+ int src;
+ int dst;
+ Py_ssize_t count;
+ PyObject *offset_src = Py_None;
+ PyObject *offset_dst = Py_None;
+
+ args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 5, 0, argsbuf);
+ if (!args) {
+ goto exit;
+ }
+ if (PyFloat_Check(args[0])) {
+ PyErr_SetString(PyExc_TypeError,
+ "integer argument expected, got float" );
+ goto exit;
+ }
+ src = _PyLong_AsInt(args[0]);
+ if (src == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ if (PyFloat_Check(args[1])) {
+ PyErr_SetString(PyExc_TypeError,
+ "integer argument expected, got float" );
+ goto exit;
+ }
+ dst = _PyLong_AsInt(args[1]);
+ if (dst == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ if (PyFloat_Check(args[2])) {
+ PyErr_SetString(PyExc_TypeError,
+ "integer argument expected, got float" );
+ goto exit;
+ }
+ {
+ Py_ssize_t ival = -1;
+ PyObject *iobj = PyNumber_Index(args[2]);
+ if (iobj != NULL) {
+ ival = PyLong_AsSsize_t(iobj);
+ Py_DECREF(iobj);
+ }
+ if (ival == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ count = ival;
+ }
+ if (!noptargs) {
+ goto skip_optional_pos;
+ }
+ if (args[3]) {
+ offset_src = args[3];
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+ offset_dst = args[4];
+skip_optional_pos:
+ return_value = os_copy_file_range_impl(module, src, dst, count, offset_src, offset_dst);
+
+exit:
+ return return_value;
+}
+
+#endif /* defined(HAVE_COPY_FILE_RANGE) */
+
#if defined(HAVE_MKFIFO)
PyDoc_STRVAR(os_mkfifo__doc__,
@@ -8460,6 +8562,10 @@ os__remove_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nar
#define OS_PWRITEV_METHODDEF
#endif /* !defined(OS_PWRITEV_METHODDEF) */
+#ifndef OS_COPY_FILE_RANGE_METHODDEF
+ #define OS_COPY_FILE_RANGE_METHODDEF
+#endif /* !defined(OS_COPY_FILE_RANGE_METHODDEF) */
+
#ifndef OS_MKFIFO_METHODDEF
#define OS_MKFIFO_METHODDEF
#endif /* !defined(OS_MKFIFO_METHODDEF) */
@@ -8635,4 +8741,4 @@ os__remove_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nar
#ifndef OS__REMOVE_DLL_DIRECTORY_METHODDEF
#define OS__REMOVE_DLL_DIRECTORY_METHODDEF
#endif /* !defined(OS__REMOVE_DLL_DIRECTORY_METHODDEF) */
-/*[clinic end generated code: output=855b81aafd05beed input=a9049054013a1b77]*/
+/*[clinic end generated code: output=b3ae8afd275ea5cd input=a9049054013a1b77]*/
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index 77a3700ab22e..8f6cffffcdfb 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -117,6 +117,10 @@ corresponding Unix manual entries for more information on calls.");
#include <sched.h>
#endif
+#ifdef HAVE_COPY_FILE_RANGE
+#include <unistd.h>
+#endif
+
#if !defined(CPU_ALLOC) && defined(HAVE_SCHED_SETAFFINITY)
#undef HAVE_SCHED_SETAFFINITY
#endif
@@ -9455,8 +9459,74 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset,
}
#endif /* HAVE_PWRITEV */
+#ifdef HAVE_COPY_FILE_RANGE
+/*[clinic input]
+
+os.copy_file_range
+ src: int
+ Source file descriptor.
+ dst: int
+ Destination file descriptor.
+ count: Py_ssize_t
+ Number of bytes to copy.
+ offset_src: object = None
+ Starting offset in src.
+ offset_dst: object = None
+ Starting offset in dst.
+
+Copy count bytes from one file descriptor to another.
+
+If offset_src is None, then src is read from the current position;
+respectively for offset_dst.
+[clinic start generated code]*/
+
+static PyObject *
+os_copy_file_range_impl(PyObject *module, int src, int dst, Py_ssize_t count,
+ PyObject *offset_src, PyObject *offset_dst)
+/*[clinic end generated code: output=1a91713a1d99fc7a input=42fdce72681b25a9]*/
+{
+ off_t offset_src_val, offset_dst_val;
+ off_t *p_offset_src = NULL;
+ off_t *p_offset_dst = NULL;
+ Py_ssize_t ret;
+ int async_err = 0;
+ /* The flags argument is provided to allow
+ * for future extensions and currently must be to 0. */
+ int flags = 0;
+
+
+ if (count < 0) {
+ PyErr_SetString(PyExc_ValueError, "negative value for 'count' not allowed");
+ return NULL;
+ }
+
+ if (offset_src != Py_None) {
+ if (!Py_off_t_converter(offset_src, &offset_src_val)) {
+ return NULL;
+ }
+ p_offset_src = &offset_src_val;
+ }
+ if (offset_dst != Py_None) {
+ if (!Py_off_t_converter(offset_dst, &offset_dst_val)) {
+ return NULL;
+ }
+ p_offset_dst = &offset_dst_val;
+ }
+ do {
+ Py_BEGIN_ALLOW_THREADS
+ ret = copy_file_range(src, p_offset_src, dst, p_offset_dst, count, flags);
+ Py_END_ALLOW_THREADS
+ } while (ret < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals()));
+
+ if (ret < 0) {
+ return (!async_err) ? posix_error() : NULL;
+ }
+
+ return PyLong_FromSsize_t(ret);
+}
+#endif /* HAVE_COPY_FILE_RANGE*/
#ifdef HAVE_MKFIFO
/*[clinic input]
@@ -13432,6 +13502,7 @@ static PyMethodDef posix_methods[] = {
OS_POSIX_SPAWN_METHODDEF
OS_POSIX_SPAWNP_METHODDEF
OS_READLINK_METHODDEF
+ OS_COPY_FILE_RANGE_METHODDEF
OS_RENAME_METHODDEF
OS_REPLACE_METHODDEF
OS_RMDIR_METHODDEF
diff --git a/aclocal.m4 b/aclocal.m4
index 85f00dd5fac7..3d6b1a375fdc 100644
--- a/aclocal.m4
+++ b/aclocal.m4
@@ -12,9 +12,9 @@
# PARTICULAR PURPOSE.
m4_ifndef([AC_CONFIG_MACRO_DIRS], [m4_defun([_AM_CONFIG_MACRO_DIRS], [])m4_defun([AC_CONFIG_MACRO_DIRS], [_AM_CONFIG_MACRO_DIRS($@)])])
-dnl pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*-
-dnl serial 11 (pkg-config-0.29.1)
-dnl
+# pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*-
+# serial 11 (pkg-config-0.29.1)
+
dnl Copyright © 2004 Scott James Remnant <scott at netsplit.com>.
dnl Copyright © 2012-2015 Dan Nicholson <dbn.lists at gmail.com>
dnl
@@ -288,5 +288,73 @@ AS_VAR_COPY([$1], [pkg_cv_][$1])
AS_VAR_IF([$1], [""], [$5], [$4])dnl
])dnl PKG_CHECK_VAR
+dnl PKG_WITH_MODULES(VARIABLE-PREFIX, MODULES,
+dnl [ACTION-IF-FOUND],[ACTION-IF-NOT-FOUND],
+dnl [DESCRIPTION], [DEFAULT])
+dnl ------------------------------------------
+dnl
+dnl Prepare a "--with-" configure option using the lowercase
+dnl [VARIABLE-PREFIX] name, merging the behaviour of AC_ARG_WITH and
+dnl PKG_CHECK_MODULES in a single macro.
+AC_DEFUN([PKG_WITH_MODULES],
+[
+m4_pushdef([with_arg], m4_tolower([$1]))
+
+m4_pushdef([description],
+ [m4_default([$5], [build with ]with_arg[ support])])
+
+m4_pushdef([def_arg], [m4_default([$6], [auto])])
+m4_pushdef([def_action_if_found], [AS_TR_SH([with_]with_arg)=yes])
+m4_pushdef([def_action_if_not_found], [AS_TR_SH([with_]with_arg)=no])
+
+m4_case(def_arg,
+ [yes],[m4_pushdef([with_without], [--without-]with_arg)],
+ [m4_pushdef([with_without],[--with-]with_arg)])
+
+AC_ARG_WITH(with_arg,
+ AS_HELP_STRING(with_without, description[ @<:@default=]def_arg[@:>@]),,
+ [AS_TR_SH([with_]with_arg)=def_arg])
+
+AS_CASE([$AS_TR_SH([with_]with_arg)],
+ [yes],[PKG_CHECK_MODULES([$1],[$2],$3,$4)],
+ [auto],[PKG_CHECK_MODULES([$1],[$2],
+ [m4_n([def_action_if_found]) $3],
+ [m4_n([def_action_if_not_found]) $4])])
+
+m4_popdef([with_arg])
+m4_popdef([description])
+m4_popdef([def_arg])
+
+])dnl PKG_WITH_MODULES
+
+dnl PKG_HAVE_WITH_MODULES(VARIABLE-PREFIX, MODULES,
+dnl [DESCRIPTION], [DEFAULT])
+dnl -----------------------------------------------
+dnl
+dnl Convenience macro to trigger AM_CONDITIONAL after PKG_WITH_MODULES
+dnl check._[VARIABLE-PREFIX] is exported as make variable.
+AC_DEFUN([PKG_HAVE_WITH_MODULES],
+[
+PKG_WITH_MODULES([$1],[$2],,,[$3],[$4])
+
+AM_CONDITIONAL([HAVE_][$1],
+ [test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"])
+])dnl PKG_HAVE_WITH_MODULES
+
+dnl PKG_HAVE_DEFINE_WITH_MODULES(VARIABLE-PREFIX, MODULES,
+dnl [DESCRIPTION], [DEFAULT])
+dnl ------------------------------------------------------
+dnl
+dnl Convenience macro to run AM_CONDITIONAL and AC_DEFINE after
+dnl PKG_WITH_MODULES check. HAVE_[VARIABLE-PREFIX] is exported as make
+dnl and preprocessor variable.
+AC_DEFUN([PKG_HAVE_DEFINE_WITH_MODULES],
+[
+PKG_HAVE_WITH_MODULES([$1],[$2],[$3],[$4])
+
+AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"],
+ [AC_DEFINE([HAVE_][$1], 1, [Enable ]m4_tolower([$1])[ support])])
+])dnl PKG_HAVE_DEFINE_WITH_MODULES
+
m4_include([m4/ax_c_float_words_bigendian.m4])
m4_include([m4/ax_check_openssl.m4])
diff --git a/configure b/configure
index cacf9fc41894..b606fc808c17 100755
--- a/configure
+++ b/configure
@@ -785,7 +785,6 @@ infodir
docdir
oldincludedir
includedir
-runstatedir
localstatedir
sharedstatedir
sysconfdir
@@ -898,7 +897,6 @@ datadir='${datarootdir}'
sysconfdir='${prefix}/etc'
sharedstatedir='${prefix}/com'
localstatedir='${prefix}/var'
-runstatedir='${localstatedir}/run'
includedir='${prefix}/include'
oldincludedir='/usr/include'
docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
@@ -1151,15 +1149,6 @@ do
| -silent | --silent | --silen | --sile | --sil)
silent=yes ;;
- -runstatedir | --runstatedir | --runstatedi | --runstated \
- | --runstate | --runstat | --runsta | --runst | --runs \
- | --run | --ru | --r)
- ac_prev=runstatedir ;;
- -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
- | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
- | --run=* | --ru=* | --r=*)
- runstatedir=$ac_optarg ;;
-
-sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
ac_prev=sbindir ;;
-sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
@@ -1297,7 +1286,7 @@ fi
for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \
datadir sysconfdir sharedstatedir localstatedir includedir \
oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
- libdir localedir mandir runstatedir
+ libdir localedir mandir
do
eval ac_val=\$$ac_var
# Remove trailing slashes.
@@ -1450,7 +1439,6 @@ Fine tuning of the installation directories:
--sysconfdir=DIR read-only single-machine data [PREFIX/etc]
--sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com]
--localstatedir=DIR modifiable single-machine data [PREFIX/var]
- --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run]
--libdir=DIR object code libraries [EPREFIX/lib]
--includedir=DIR C header files [PREFIX/include]
--oldincludedir=DIR C header files for non-gcc [/usr/include]
@@ -11476,7 +11464,8 @@ fi
# checks for library functions
for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
- clock confstr ctermid dup3 execv explicit_bzero explicit_memset faccessat fchmod fchmodat fchown fchownat \
+ clock confstr copy_file_range ctermid dup3 execv explicit_bzero explicit_memset \
+ faccessat fchmod fchmodat fchown fchownat \
fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \
futimens futimes gai_strerror getentropy \
getgrgid_r getgrnam_r \
diff --git a/configure.ac b/configure.ac
index 1190b37e9f9d..3d589ac25891 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3520,7 +3520,8 @@ fi
# checks for library functions
AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
- clock confstr ctermid dup3 execv explicit_bzero explicit_memset faccessat fchmod fchmodat fchown fchownat \
+ clock confstr copy_file_range ctermid dup3 execv explicit_bzero explicit_memset \
+ faccessat fchmod fchmodat fchown fchownat \
fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \
futimens futimes gai_strerror getentropy \
getgrgid_r getgrnam_r \
diff --git a/pyconfig.h.in b/pyconfig.h.in
index b9bb3ffa6f69..20cc901e473b 100644
--- a/pyconfig.h.in
+++ b/pyconfig.h.in
@@ -148,6 +148,9 @@
/* Define to 1 if you have the `copysign' function. */
#undef HAVE_COPYSIGN
+/* Define to 1 if you have the `copy_file_range' function. */
+#undef HAVE_COPY_FILE_RANGE
+
/* Define to 1 if you have the <crypt.h> header file. */
#undef HAVE_CRYPT_H
More information about the Python-checkins
mailing list