[Python-checkins] bpo-26732: fix too many fds in processes started with the "forkserver" method (#2813)
Antoine Pitrou
webhook-mailer at python.org
Sat Jul 22 07:22:57 EDT 2017
https://github.com/python/cpython/commit/896145d9d266ee2758cfcd7691238cbc1f9e1ab8
commit: 896145d9d266ee2758cfcd7691238cbc1f9e1ab8
branch: master
author: Antoine Pitrou <pitrou at free.fr>
committer: GitHub <noreply at github.com>
date: 2017-07-22T13:22:54+02:00
summary:
bpo-26732: fix too many fds in processes started with the "forkserver" method (#2813)
* bpo-26732: fix too many fds in processes started with the "forkserver" method
A child process would inherit as many fds as the number of still-running children.
* Add blurb and test comment
files:
A Misc/NEWS.d/next/Library/2017-07-22-12-12-42.bpo-26732.lYLWBH.rst
M Lib/multiprocessing/forkserver.py
M Lib/test/_test_multiprocessing.py
M Lib/test/libregrtest/refleak.py
M Lib/test/support/__init__.py
diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py
index b9f9b9dd8b5..69b842aa939 100644
--- a/Lib/multiprocessing/forkserver.py
+++ b/Lib/multiprocessing/forkserver.py
@@ -236,8 +236,11 @@ def sigchld_handler(*_unused):
code = 1
try:
listener.close()
+ selector.close()
+ unused_fds = [alive_r, child_w, sig_r, sig_w]
+ unused_fds.extend(pid_to_fd.values())
code = _serve_one(child_r, fds,
- (alive_r, child_w, sig_r, sig_w),
+ unused_fds,
old_handlers)
except Exception:
sys.excepthook(*sys.exc_info())
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index 329a6d29aca..a14fa7422e7 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -494,6 +494,40 @@ def test_lose_target_ref(self):
self.assertIs(wr(), None)
self.assertEqual(q.get(), 5)
+ @classmethod
+ def _test_child_fd_inflation(self, evt, q):
+ q.put(test.support.fd_count())
+ evt.wait()
+
+ def test_child_fd_inflation(self):
+ # Number of fds in child processes should not grow with the
+ # number of running children.
+ if self.TYPE == 'threads':
+ self.skipTest('test not appropriate for {}'.format(self.TYPE))
+
+ sm = multiprocessing.get_start_method()
+ if sm == 'fork':
+ # The fork method by design inherits all fds from the parent,
+ # trying to go against it is a lost battle
+ self.skipTest('test not appropriate for {}'.format(sm))
+
+ N = 5
+ evt = self.Event()
+ q = self.Queue()
+
+ procs = [self.Process(target=self._test_child_fd_inflation, args=(evt, q))
+ for i in range(N)]
+ for p in procs:
+ p.start()
+
+ try:
+ fd_counts = [q.get() for i in range(N)]
+ self.assertEqual(len(set(fd_counts)), 1, fd_counts)
+
+ finally:
+ evt.set()
+ for p in procs:
+ p.join()
#
#
diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py
index 8e93816d965..efe52107cb4 100644
--- a/Lib/test/libregrtest/refleak.py
+++ b/Lib/test/libregrtest/refleak.py
@@ -7,36 +7,6 @@
from test import support
-try:
- MAXFD = os.sysconf("SC_OPEN_MAX")
-except Exception:
- MAXFD = 256
-
-
-def fd_count():
- """Count the number of open file descriptors"""
- if sys.platform.startswith(('linux', 'freebsd')):
- try:
- names = os.listdir("/proc/self/fd")
- return len(names)
- except FileNotFoundError:
- pass
-
- count = 0
- for fd in range(MAXFD):
- try:
- # Prefer dup() over fstat(). fstat() can require input/output
- # whereas dup() doesn't.
- fd2 = os.dup(fd)
- except OSError as e:
- if e.errno != errno.EBADF:
- raise
- else:
- os.close(fd2)
- count += 1
- return count
-
-
def dash_R(the_module, test, indirect_test, huntrleaks):
"""Run a test multiple times, looking for reference leaks.
@@ -174,7 +144,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs):
func1 = sys.getallocatedblocks
func2 = sys.gettotalrefcount
gc.collect()
- return func1(), func2(), fd_count()
+ return func1(), func2(), support.fd_count()
def clear_caches():
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 313c23004ed..3a4d27e6b88 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -107,7 +107,7 @@
"check_warnings", "check_no_resource_warning", "EnvironmentVarGuard",
"run_with_locale", "swap_item",
"swap_attr", "Matcher", "set_memlimit", "SuppressCrashReport", "sortdict",
- "run_with_tz", "PGO", "missing_compiler_executable",
+ "run_with_tz", "PGO", "missing_compiler_executable", "fd_count",
]
class Error(Exception):
@@ -2647,3 +2647,34 @@ def disable_faulthandler():
finally:
if is_enabled:
faulthandler.enable(file=fd, all_threads=True)
+
+
+try:
+ MAXFD = os.sysconf("SC_OPEN_MAX")
+except Exception:
+ MAXFD = 256
+
+
+def fd_count():
+ """Count the number of open file descriptors.
+ """
+ if sys.platform.startswith(('linux', 'freebsd')):
+ try:
+ names = os.listdir("/proc/self/fd")
+ return len(names)
+ except FileNotFoundError:
+ pass
+
+ count = 0
+ for fd in range(MAXFD):
+ try:
+ # Prefer dup() over fstat(). fstat() can require input/output
+ # whereas dup() doesn't.
+ fd2 = os.dup(fd)
+ except OSError as e:
+ if e.errno != errno.EBADF:
+ raise
+ else:
+ os.close(fd2)
+ count += 1
+ return count
diff --git a/Misc/NEWS.d/next/Library/2017-07-22-12-12-42.bpo-26732.lYLWBH.rst b/Misc/NEWS.d/next/Library/2017-07-22-12-12-42.bpo-26732.lYLWBH.rst
new file mode 100644
index 00000000000..cff0f9ad6ab
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2017-07-22-12-12-42.bpo-26732.lYLWBH.rst
@@ -0,0 +1,4 @@
+Fix too many fds in processes started with the "forkserver" method.
+
+A child process would inherit as many fds as the number of still-running
+children.
More information about the Python-checkins
mailing list