[Python-checkins] bpo-39763: distutils.spawn now uses subprocess (GH-18743)
Victor Stinner
webhook-mailer at python.org
Wed Mar 4 08:50:24 EST 2020
https://github.com/python/cpython/commit/1ec63b62035e73111e204a0e03b83503e1c58f2e
commit: 1ec63b62035e73111e204a0e03b83503e1c58f2e
branch: master
author: Victor Stinner <vstinner at python.org>
committer: GitHub <noreply at github.com>
date: 2020-03-04T14:50:19+01:00
summary:
bpo-39763: distutils.spawn now uses subprocess (GH-18743)
Reimplement distutils.spawn.spawn() function with the subprocess
module.
setup.py now uses a basic implementation of the subprocess module if
the subprocess module is not available: before required C extension
modules are built.
files:
A Misc/NEWS.d/next/Build/2020-03-02-14-44-09.bpo-39763.GGEwhH.rst
A Misc/NEWS.d/next/Library/2020-03-02-14-43-19.bpo-39763.5a822c.rst
M Lib/distutils/spawn.py
M Lib/distutils/tests/test_spawn.py
M setup.py
diff --git a/Lib/distutils/spawn.py b/Lib/distutils/spawn.py
index ceb94945dc8be..aad277b0ca776 100644
--- a/Lib/distutils/spawn.py
+++ b/Lib/distutils/spawn.py
@@ -8,11 +8,18 @@
import sys
import os
+import subprocess
from distutils.errors import DistutilsPlatformError, DistutilsExecError
from distutils.debug import DEBUG
from distutils import log
+
+if sys.platform == 'darwin':
+ _cfg_target = None
+ _cfg_target_split = None
+
+
def spawn(cmd, search_path=1, verbose=0, dry_run=0):
"""Run another program, specified as a command list 'cmd', in a new process.
@@ -32,64 +39,16 @@ def spawn(cmd, search_path=1, verbose=0, dry_run=0):
# cmd is documented as a list, but just in case some code passes a tuple
# in, protect our %-formatting code against horrible death
cmd = list(cmd)
- if os.name == 'posix':
- _spawn_posix(cmd, search_path, dry_run=dry_run)
- elif os.name == 'nt':
- _spawn_nt(cmd, search_path, dry_run=dry_run)
- else:
- raise DistutilsPlatformError(
- "don't know how to spawn programs on platform '%s'" % os.name)
-
-def _nt_quote_args(args):
- """Quote command-line arguments for DOS/Windows conventions.
-
- Just wraps every argument which contains blanks in double quotes, and
- returns a new argument list.
- """
- # XXX this doesn't seem very robust to me -- but if the Windows guys
- # say it'll work, I guess I'll have to accept it. (What if an arg
- # contains quotes? What other magic characters, other than spaces,
- # have to be escaped? Is there an escaping mechanism other than
- # quoting?)
- for i, arg in enumerate(args):
- if ' ' in arg:
- args[i] = '"%s"' % arg
- return args
-
-def _spawn_nt(cmd, search_path=1, verbose=0, dry_run=0):
- executable = cmd[0]
- cmd = _nt_quote_args(cmd)
- if search_path:
- # either we find one or it stays the same
- executable = find_executable(executable) or executable
- log.info(' '.join([executable] + cmd[1:]))
- if not dry_run:
- # spawn for NT requires a full path to the .exe
- try:
- rc = os.spawnv(os.P_WAIT, executable, cmd)
- except OSError as exc:
- # this seems to happen when the command isn't found
- if not DEBUG:
- cmd = executable
- raise DistutilsExecError(
- "command %r failed: %s" % (cmd, exc.args[-1]))
- if rc != 0:
- # and this reflects the command running but failing
- if not DEBUG:
- cmd = executable
- raise DistutilsExecError(
- "command %r failed with exit status %d" % (cmd, rc))
-
-if sys.platform == 'darwin':
- _cfg_target = None
- _cfg_target_split = None
-def _spawn_posix(cmd, search_path=1, verbose=0, dry_run=0):
log.info(' '.join(cmd))
if dry_run:
return
- executable = cmd[0]
- exec_fn = search_path and os.execvp or os.execv
+
+ if search_path:
+ executable = find_executable(cmd[0])
+ if executable is not None:
+ cmd[0] = executable
+
env = None
if sys.platform == 'darwin':
global _cfg_target, _cfg_target_split
@@ -111,60 +70,17 @@ def _spawn_posix(cmd, search_path=1, verbose=0, dry_run=0):
raise DistutilsPlatformError(my_msg)
env = dict(os.environ,
MACOSX_DEPLOYMENT_TARGET=cur_target)
- exec_fn = search_path and os.execvpe or os.execve
- pid = os.fork()
- if pid == 0: # in the child
- try:
- if env is None:
- exec_fn(executable, cmd)
- else:
- exec_fn(executable, cmd, env)
- except OSError as e:
- if not DEBUG:
- cmd = executable
- sys.stderr.write("unable to execute %r: %s\n"
- % (cmd, e.strerror))
- os._exit(1)
+ proc = subprocess.Popen(cmd, env=env)
+ proc.wait()
+ exitcode = proc.returncode
+
+ if exitcode:
if not DEBUG:
- cmd = executable
- sys.stderr.write("unable to execute %r for unknown reasons" % cmd)
- os._exit(1)
- else: # in the parent
- # Loop until the child either exits or is terminated by a signal
- # (ie. keep waiting if it's merely stopped)
- while True:
- try:
- pid, status = os.waitpid(pid, 0)
- except OSError as exc:
- if not DEBUG:
- cmd = executable
- raise DistutilsExecError(
- "command %r failed: %s" % (cmd, exc.args[-1]))
- if os.WIFSIGNALED(status):
- if not DEBUG:
- cmd = executable
- raise DistutilsExecError(
- "command %r terminated by signal %d"
- % (cmd, os.WTERMSIG(status)))
- elif os.WIFEXITED(status):
- exit_status = os.WEXITSTATUS(status)
- if exit_status == 0:
- return # hey, it succeeded!
- else:
- if not DEBUG:
- cmd = executable
- raise DistutilsExecError(
- "command %r failed with exit status %d"
- % (cmd, exit_status))
- elif os.WIFSTOPPED(status):
- continue
- else:
- if not DEBUG:
- cmd = executable
- raise DistutilsExecError(
- "unknown error executing %r: termination status %d"
- % (cmd, status))
+ cmd = cmd[0]
+ raise DistutilsExecError(
+ "command %r failed with exit code %s" % (cmd, exitcode))
+
def find_executable(executable, path=None):
"""Tries to find 'executable' in the directories listed in 'path'.
diff --git a/Lib/distutils/tests/test_spawn.py b/Lib/distutils/tests/test_spawn.py
index f9ae69ef86b3d..73b0f5cb7324c 100644
--- a/Lib/distutils/tests/test_spawn.py
+++ b/Lib/distutils/tests/test_spawn.py
@@ -8,7 +8,6 @@
from test import support as test_support
from distutils.spawn import find_executable
-from distutils.spawn import _nt_quote_args
from distutils.spawn import spawn
from distutils.errors import DistutilsExecError
from distutils.tests import support
@@ -17,16 +16,6 @@ class SpawnTestCase(support.TempdirManager,
support.LoggingSilencer,
unittest.TestCase):
- def test_nt_quote_args(self):
-
- for (args, wanted) in ((['with space', 'nospace'],
- ['"with space"', 'nospace']),
- (['nochange', 'nospace'],
- ['nochange', 'nospace'])):
- res = _nt_quote_args(args)
- self.assertEqual(res, wanted)
-
-
@unittest.skipUnless(os.name in ('nt', 'posix'),
'Runs only under posix or nt')
def test_spawn(self):
diff --git a/Misc/NEWS.d/next/Build/2020-03-02-14-44-09.bpo-39763.GGEwhH.rst b/Misc/NEWS.d/next/Build/2020-03-02-14-44-09.bpo-39763.GGEwhH.rst
new file mode 100644
index 0000000000000..e983b4f338a94
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2020-03-02-14-44-09.bpo-39763.GGEwhH.rst
@@ -0,0 +1,3 @@
+setup.py now uses a basic implementation of the :mod:`subprocess` module if
+the :mod:`subprocess` module is not available: before required C extension
+modules are built.
diff --git a/Misc/NEWS.d/next/Library/2020-03-02-14-43-19.bpo-39763.5a822c.rst b/Misc/NEWS.d/next/Library/2020-03-02-14-43-19.bpo-39763.5a822c.rst
new file mode 100644
index 0000000000000..73ea8f9ab69a6
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2020-03-02-14-43-19.bpo-39763.5a822c.rst
@@ -0,0 +1,2 @@
+Reimplement :func:`distutils.spawn.spawn` function with the
+:mod:`subprocess` module.
diff --git a/setup.py b/setup.py
index 51e67fe4a558b..c9f3c97238c22 100644
--- a/setup.py
+++ b/setup.py
@@ -10,6 +10,61 @@
import sysconfig
from glob import glob
+
+try:
+ import subprocess
+ del subprocess
+ SUBPROCESS_BOOTSTRAP = False
+except ImportError:
+ SUBPROCESS_BOOTSTRAP = True
+
+ # Bootstrap Python: distutils.spawn uses subprocess to build C extensions,
+ # subprocess requires C extensions built by setup.py like _posixsubprocess.
+ #
+ # Basic subprocess implementation for POSIX (setup.py is not used on
+ # Windows) which only uses os functions. Only implement features required
+ # by distutils.spawn.
+ #
+ # It is dropped from sys.modules as soon as all C extension modules
+ # are built.
+ class Popen:
+ def __init__(self, cmd, env=None):
+ self._cmd = cmd
+ self._env = env
+ self.returncode = None
+
+ def wait(self):
+ pid = os.fork()
+ if pid == 0:
+ # Child process
+ try:
+ if self._env is not None:
+ os.execve(self._cmd[0], self._cmd, self._env)
+ else:
+ os.execv(self._cmd[0], self._cmd)
+ finally:
+ os._exit(1)
+ else:
+ # Parent process
+ pid, status = os.waitpid(pid, 0)
+ if os.WIFSIGNALED(status):
+ self.returncode = -os.WTERMSIG(status)
+ elif os.WIFEXITED(status):
+ self.returncode = os.WEXITSTATUS(status)
+ elif os.WIFSTOPPED(status):
+ self.returncode = -os.WSTOPSIG(sts)
+ else:
+ # Should never happen
+ raise Exception("Unknown child exit status!")
+
+ return self.returncode
+
+ mod = type(sys)('subprocess')
+ mod.Popen = Popen
+ sys.modules['subprocess'] = mod
+ del mod
+
+
from distutils import log
from distutils.command.build_ext import build_ext
from distutils.command.build_scripts import build_scripts
@@ -391,6 +446,11 @@ def build_extensions(self):
build_ext.build_extensions(self)
+ if SUBPROCESS_BOOTSTRAP:
+ # Drop our custom subprocess module:
+ # use the newly built subprocess module
+ del sys.modules['subprocess']
+
for ext in self.extensions:
self.check_extension_import(ext)
More information about the Python-checkins
mailing list