[Python-checkins] gh-90876: Restore the ability to import multiprocessing when `sys.executable` is `None` (#106464)
gpshead
webhook-mailer at python.org
Thu Jul 6 18:46:53 EDT 2023
https://github.com/python/cpython/commit/c60df361ce2d734148d503f4a711e67c110fe223
commit: c60df361ce2d734148d503f4a711e67c110fe223
branch: main
author: Gregory P. Smith <greg at krypto.org>
committer: gpshead <greg at krypto.org>
date: 2023-07-06T22:46:50Z
summary:
gh-90876: Restore the ability to import multiprocessing when `sys.executable` is `None` (#106464)
Prevent `multiprocessing.spawn` from failing to *import* in environments
where `sys.executable` is `None`. This regressed in 3.11 with the addition
of support for path-like objects in multiprocessing.
Adds a test decorator to have tests only run when part of test_multiprocessing_spawn to `_test_multiprocessing.py` so we can start to avoid re-running the same not-global-state specific test in all 3 modes when there is no need.
files:
A Misc/NEWS.d/next/Library/2023-07-05-13-08-23.gh-issue-90876.Qvlkfl.rst
M Lib/multiprocessing/spawn.py
M Lib/test/_test_multiprocessing.py
diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py
index 09f8a229d7ccc..f1af770910471 100644
--- a/Lib/multiprocessing/spawn.py
+++ b/Lib/multiprocessing/spawn.py
@@ -31,11 +31,13 @@
WINSERVICE = False
else:
WINEXE = getattr(sys, 'frozen', False)
- WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
+ WINSERVICE = sys.executable and sys.executable.lower().endswith("pythonservice.exe")
def set_executable(exe):
global _python_exe
- if sys.platform == 'win32':
+ if exe is None:
+ _python_exe = exe
+ elif sys.platform == 'win32':
_python_exe = os.fsdecode(exe)
else:
_python_exe = os.fsencode(exe)
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index c101fe980ceed..c1f9487ae8051 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -13,6 +13,7 @@
import os
import gc
import errno
+import functools
import signal
import array
import socket
@@ -31,6 +32,7 @@
from test.support import hashlib_helper
from test.support import import_helper
from test.support import os_helper
+from test.support import script_helper
from test.support import socket_helper
from test.support import threading_helper
from test.support import warnings_helper
@@ -171,6 +173,59 @@ def check_enough_semaphores():
"to run the test (required: %d)." % nsems_min)
+def only_run_in_spawn_testsuite(reason):
+ """Returns a decorator: raises SkipTest when SM != spawn at test time.
+
+ This can be useful to save overall Python test suite execution time.
+ "spawn" is the universal mode available on all platforms so this limits the
+ decorated test to only execute within test_multiprocessing_spawn.
+
+ This would not be necessary if we refactored our test suite to split things
+ into other test files when they are not start method specific to be rerun
+ under all start methods.
+ """
+
+ def decorator(test_item):
+
+ @functools.wraps(test_item)
+ def spawn_check_wrapper(*args, **kwargs):
+ if (start_method := multiprocessing.get_start_method()) != "spawn":
+ raise unittest.SkipTest(f"{start_method=}, not 'spawn'; {reason}")
+ return test_item(*args, **kwargs)
+
+ return spawn_check_wrapper
+
+ return decorator
+
+
+class TestInternalDecorators(unittest.TestCase):
+ """Logic within a test suite that could errantly skip tests? Test it!"""
+
+ @unittest.skipIf(sys.platform == "win32", "test requires that fork exists.")
+ def test_only_run_in_spawn_testsuite(self):
+ if multiprocessing.get_start_method() != "spawn":
+ raise unittest.SkipTest("only run in test_multiprocessing_spawn.")
+
+ try:
+ @only_run_in_spawn_testsuite("testing this decorator")
+ def return_four_if_spawn():
+ return 4
+ except Exception as err:
+ self.fail(f"expected decorated `def` not to raise; caught {err}")
+
+ orig_start_method = multiprocessing.get_start_method(allow_none=True)
+ try:
+ multiprocessing.set_start_method("spawn", force=True)
+ self.assertEqual(return_four_if_spawn(), 4)
+ multiprocessing.set_start_method("fork", force=True)
+ with self.assertRaises(unittest.SkipTest) as ctx:
+ return_four_if_spawn()
+ self.assertIn("testing this decorator", str(ctx.exception))
+ self.assertIn("start_method=", str(ctx.exception))
+ finally:
+ multiprocessing.set_start_method(orig_start_method, force=True)
+
+
#
# Creates a wrapper for a function which records the time it takes to finish
#
@@ -5815,6 +5870,7 @@ def test_namespace(self):
class TestNamedResource(unittest.TestCase):
+ @only_run_in_spawn_testsuite("spawn specific test.")
def test_global_named_resource_spawn(self):
#
# gh-90549: Check that global named resources in main module
@@ -5825,22 +5881,18 @@ def test_global_named_resource_spawn(self):
with open(testfn, 'w', encoding='utf-8') as f:
f.write(textwrap.dedent('''\
import multiprocessing as mp
-
ctx = mp.get_context('spawn')
-
global_resource = ctx.Semaphore()
-
def submain(): pass
-
if __name__ == '__main__':
p = ctx.Process(target=submain)
p.start()
p.join()
'''))
- rc, out, err = test.support.script_helper.assert_python_ok(testfn)
+ rc, out, err = script_helper.assert_python_ok(testfn)
# on error, err = 'UserWarning: resource_tracker: There appear to
# be 1 leaked semaphore objects to clean up at shutdown'
- self.assertEqual(err, b'')
+ self.assertFalse(err, msg=err.decode('utf-8'))
class MiscTestCase(unittest.TestCase):
@@ -5849,6 +5901,24 @@ def test__all__(self):
support.check__all__(self, multiprocessing, extra=multiprocessing.__all__,
not_exported=['SUBDEBUG', 'SUBWARNING'])
+ @only_run_in_spawn_testsuite("avoids redundant testing.")
+ def test_spawn_sys_executable_none_allows_import(self):
+ # Regression test for a bug introduced in
+ # https://github.com/python/cpython/issues/90876 that caused an
+ # ImportError in multiprocessing when sys.executable was None.
+ # This can be true in embedded environments.
+ rc, out, err = script_helper.assert_python_ok(
+ "-c",
+ """if 1:
+ import sys
+ sys.executable = None
+ assert "multiprocessing" not in sys.modules, "already imported!"
+ import multiprocessing
+ import multiprocessing.spawn # This should not fail\n""",
+ )
+ self.assertEqual(rc, 0)
+ self.assertFalse(err, msg=err.decode('utf-8'))
+
#
# Mixins
diff --git a/Misc/NEWS.d/next/Library/2023-07-05-13-08-23.gh-issue-90876.Qvlkfl.rst b/Misc/NEWS.d/next/Library/2023-07-05-13-08-23.gh-issue-90876.Qvlkfl.rst
new file mode 100644
index 0000000000000..3e062b5add6d8
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2023-07-05-13-08-23.gh-issue-90876.Qvlkfl.rst
@@ -0,0 +1,3 @@
+Prevent :mod:`multiprocessing.spawn` from failing to *import* in environments
+where ``sys.executable`` is ``None``. This regressed in 3.11 with the addition
+of support for path-like objects in multiprocessing.
More information about the Python-checkins
mailing list