[Python-checkins] bpo-35537: subprocess uses os.posix_spawn in some cases (GH-11452)
Victor Stinner
webhook-mailer at python.org
Tue Jan 15 18:02:39 EST 2019
https://github.com/python/cpython/commit/9daecf37a571e98aaf43a387bcc9e41a7132f477
commit: 9daecf37a571e98aaf43a387bcc9e41a7132f477
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-01-16T00:02:35+01:00
summary:
bpo-35537: subprocess uses os.posix_spawn in some cases (GH-11452)
The subprocess module can now use the os.posix_spawn() function
in some cases for better performance. Currently, it is only used on macOS
and Linux (using glibc 2.24 or newer) if all these conditions are met:
* executable path contains a directory
* close_fds=False
* preexec_fn, pass_fds, cwd, stdin, stdout, stderr
and start_new_session parameters are not set
Co-authored-by: Joannah Nanjekye <nanjekyejoannah at gmail.com>
files:
A Misc/NEWS.d/next/Library/2018-12-20-16-24-51.bpo-35537.z4E7aA.rst
M Doc/whatsnew/3.8.rst
M Lib/subprocess.py
M Lib/test/pythoninfo.py
diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 370ef4604834..053fe902c481 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -275,6 +275,15 @@ xml
Optimizations
=============
+* The :mod:`subprocess` module can now use the :func:`os.posix_spawn` function
+ in some cases for better performance. Currently, it is only used on macOS
+ and Linux (using glibc 2.24 or newer) if all these conditions are met:
+
+ * *close_fds* is false;
+ * *preexec_fn*, *pass_fds*, *cwd*, *stdin*, *stdout*, *stderr* and
+ *start_new_session* parameters are not set;
+ * the *executable* path contains a directory.
+
* :func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2`,
:func:`shutil.copytree` and :func:`shutil.move` use platform-specific
"fast-copy" syscalls on Linux, macOS and Solaris in order to copy the file
diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index 696617697047..b94575b8401e 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -606,6 +606,57 @@ def getoutput(cmd):
return getstatusoutput(cmd)[1]
+def _use_posix_spawn():
+ """Check is posix_spawn() can be used for subprocess.
+
+ subprocess requires a posix_spawn() implementation that reports properly
+ errors to the parent process, set errno on the following failures:
+
+ * process attribute actions failed
+ * file actions failed
+ * exec() failed
+
+ Prefer an implementation which can use vfork in some cases for best
+ performances.
+ """
+ if _mswindows or not hasattr(os, 'posix_spawn'):
+ # os.posix_spawn() is not available
+ return False
+
+ if sys.platform == 'darwin':
+ # posix_spawn() is a syscall on macOS and properly reports errors
+ return True
+
+ # Check libc name and runtime libc version
+ try:
+ ver = os.confstr('CS_GNU_LIBC_VERSION')
+ # parse 'glibc 2.28' as ('glibc', (2, 28))
+ parts = ver.split(maxsplit=1)
+ if len(parts) != 2:
+ # reject unknown format
+ raise ValueError
+ libc = parts[0]
+ version = tuple(map(int, parts[1].split('.')))
+
+ if sys.platform == 'linux' and libc == 'glibc' and version >= (2, 24):
+ # glibc 2.24 has a new Linux posix_spawn implementation using vfork
+ # which properly reports errors to the parent process.
+ return True
+ # Note: Don't use the POSIX implementation of glibc because it doesn't
+ # use vfork (even if glibc 2.26 added a pipe to properly report errors
+ # to the parent process).
+ except (AttributeError, ValueError, OSError):
+ # os.confstr() or CS_GNU_LIBC_VERSION value not available
+ pass
+
+ # By default, consider that the implementation does not properly report
+ # errors.
+ return False
+
+
+_USE_POSIX_SPAWN = _use_posix_spawn()
+
+
class Popen(object):
""" Execute a child program in a new process.
@@ -1390,6 +1441,23 @@ def _get_handles(self, stdin, stdout, stderr):
errread, errwrite)
+ def _posix_spawn(self, args, executable, env, restore_signals):
+ """Execute program using os.posix_spawn()."""
+ if env is None:
+ env = os.environ
+
+ kwargs = {}
+ if restore_signals:
+ # See _Py_RestoreSignals() in Python/pylifecycle.c
+ sigset = []
+ for signame in ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ'):
+ signum = getattr(signal, signame, None)
+ if signum is not None:
+ sigset.append(signum)
+ kwargs['setsigdef'] = sigset
+
+ self.pid = os.posix_spawn(executable, args, env, **kwargs)
+
def _execute_child(self, args, executable, preexec_fn, close_fds,
pass_fds, cwd, env,
startupinfo, creationflags, shell,
@@ -1414,6 +1482,20 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
if executable is None:
executable = args[0]
+
+ if (_USE_POSIX_SPAWN
+ and os.path.dirname(executable)
+ and preexec_fn is None
+ and not close_fds
+ and not pass_fds
+ and cwd is None
+ and p2cread == p2cwrite == -1
+ and c2pread == c2pwrite == -1
+ and errread == errwrite == -1
+ and not start_new_session):
+ self._posix_spawn(args, executable, env, restore_signals)
+ return
+
orig_executable = executable
# For transferring possible exec failure from child to parent.
diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py
index 7ce6bf7b1ab9..7e94a31cecea 100644
--- a/Lib/test/pythoninfo.py
+++ b/Lib/test/pythoninfo.py
@@ -610,6 +610,11 @@ def collect_get_config(info_add):
info_add('%s[%s]' % (prefix, key), repr(config[key]))
+def collect_subprocess(info_add):
+ import subprocess
+ copy_attributes(info_add, subprocess, 'subprocess.%s', ('_USE_POSIX_SPAWN',))
+
+
def collect_info(info):
error = False
info_add = info.add
@@ -639,6 +644,7 @@ def collect_info(info):
collect_cc,
collect_gdbm,
collect_get_config,
+ collect_subprocess,
# Collecting from tests should be last as they have side effects.
collect_test_socket,
diff --git a/Misc/NEWS.d/next/Library/2018-12-20-16-24-51.bpo-35537.z4E7aA.rst b/Misc/NEWS.d/next/Library/2018-12-20-16-24-51.bpo-35537.z4E7aA.rst
new file mode 100644
index 000000000000..b14d7493bc60
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-12-20-16-24-51.bpo-35537.z4E7aA.rst
@@ -0,0 +1,2 @@
+The :mod:`subprocess` module can now use the :func:`os.posix_spawn` function in
+some cases for better performance.
More information about the Python-checkins
mailing list