[Python-checkins] gh-98360: multiprocessing now spawns children on Windows with correct argv[0] in virtual environments (GH-98462)
miss-islington
webhook-mailer at python.org
Thu Oct 20 10:33:24 EDT 2022
https://github.com/python/cpython/commit/ace6611de602906414d612b5649c13b9d8115a1e
commit: ace6611de602906414d612b5649c13b9d8115a1e
branch: 3.11
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2022-10-20T07:33:12-07:00
summary:
gh-98360: multiprocessing now spawns children on Windows with correct argv[0] in virtual environments (GH-98462)
(cherry picked from commit e48f9b2b7e73f4a89a9b9c287f3b93dc13a60460)
Co-authored-by: Steve Dower <steve.dower at python.org>
files:
A Lib/test/_test_venv_multiprocessing.py
A Misc/NEWS.d/next/Windows/2022-10-19-20-00-28.gh-issue-98360.O2m6YG.rst
M Lib/multiprocessing/popen_spawn_win32.py
M Lib/test/test_venv.py
diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py
index 9c4098d0fa4f..4d60ffc030be 100644
--- a/Lib/multiprocessing/popen_spawn_win32.py
+++ b/Lib/multiprocessing/popen_spawn_win32.py
@@ -54,19 +54,20 @@ def __init__(self, process_obj):
wfd = msvcrt.open_osfhandle(whandle, 0)
cmd = spawn.get_command_line(parent_pid=os.getpid(),
pipe_handle=rhandle)
- cmd = ' '.join('"%s"' % x for x in cmd)
python_exe = spawn.get_executable()
# bpo-35797: When running in a venv, we bypass the redirect
# executor and launch our base Python.
if WINENV and _path_eq(python_exe, sys.executable):
- python_exe = sys._base_executable
+ cmd[0] = python_exe = sys._base_executable
env = os.environ.copy()
env["__PYVENV_LAUNCHER__"] = sys.executable
else:
env = None
+ cmd = ' '.join('"%s"' % x for x in cmd)
+
with open(wfd, 'wb', closefd=True) as to_child:
# start process
try:
diff --git a/Lib/test/_test_venv_multiprocessing.py b/Lib/test/_test_venv_multiprocessing.py
new file mode 100644
index 000000000000..af72e915ba52
--- /dev/null
+++ b/Lib/test/_test_venv_multiprocessing.py
@@ -0,0 +1,40 @@
+import multiprocessing
+import random
+import sys
+import time
+
+def fill_queue(queue, code):
+ queue.put(code)
+
+
+def drain_queue(queue, code):
+ if code != queue.get():
+ sys.exit(1)
+
+
+def test_func():
+ code = random.randrange(0, 1000)
+ queue = multiprocessing.Queue()
+ fill_pool = multiprocessing.Process(
+ target=fill_queue,
+ args=(queue, code)
+ )
+ drain_pool = multiprocessing.Process(
+ target=drain_queue,
+ args=(queue, code)
+ )
+ drain_pool.start()
+ fill_pool.start()
+ fill_pool.join()
+ drain_pool.join()
+
+
+def main():
+ test_pool = multiprocessing.Process(target=test_func)
+ test_pool.start()
+ test_pool.join()
+ sys.exit(test_pool.exitcode)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py
index b0a86ee4abc5..c88df1795fda 100644
--- a/Lib/test/test_venv.py
+++ b/Lib/test/test_venv.py
@@ -20,7 +20,7 @@
from test.support import (captured_stdout, captured_stderr, requires_zlib,
skip_if_broken_multiprocessing_synchronize, verbose,
requires_subprocess, is_emscripten, is_wasi,
- requires_venv_with_pip)
+ requires_venv_with_pip, TEST_HOME_DIR)
from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree)
import unittest
import venv
@@ -482,6 +482,20 @@ def test_multiprocessing(self):
'pool.terminate()'])
self.assertEqual(out.strip(), "python".encode())
+ @requireVenvCreate
+ def test_multiprocessing_recursion(self):
+ """
+ Test that the multiprocessing is able to spawn itself
+ """
+ skip_if_broken_multiprocessing_synchronize()
+
+ rmtree(self.env_dir)
+ self.run_with_capture(venv.create, self.env_dir)
+ envpy = os.path.join(os.path.realpath(self.env_dir),
+ self.bindir, self.exe)
+ script = os.path.join(TEST_HOME_DIR, '_test_venv_multiprocessing.py')
+ subprocess.check_call([envpy, script])
+
@unittest.skipIf(os.name == 'nt', 'not relevant on Windows')
def test_deactivate_with_strict_bash_opts(self):
bash = shutil.which("bash")
diff --git a/Misc/NEWS.d/next/Windows/2022-10-19-20-00-28.gh-issue-98360.O2m6YG.rst b/Misc/NEWS.d/next/Windows/2022-10-19-20-00-28.gh-issue-98360.O2m6YG.rst
new file mode 100644
index 000000000000..61c1e5e837fe
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2022-10-19-20-00-28.gh-issue-98360.O2m6YG.rst
@@ -0,0 +1,4 @@
+Fixes :mod:`multiprocessing` spawning child processes on Windows from a
+virtual environment to ensure that child processes that also use
+:mod:`multiprocessing` to spawn more children will recognize that they are
+in a virtual environment.
More information about the Python-checkins
mailing list