[Python-checkins] closes bpo-38692: Add a pidfd child process watcher to asyncio. (GH-17069)

[Benjamin Peterson]

https://github.com/python/cpython/commit/3ccdd9b180f9a3f29c8ddc8ad1b331fe8df26519
commit: 3ccdd9b180f9a3f29c8ddc8ad1b331fe8df26519
branch: master
author: Benjamin Peterson <benjamin at python.org>
committer: GitHub <noreply at github.com>
date: 2019-11-13T19:08:50-08:00
summary:

closes bpo-38692: Add a pidfd child process watcher to asyncio. (GH-17069)

files:
A Misc/NEWS.d/next/Library/2019-11-05-19-15-57.bpo-38692.2DCDA-.rst
M Doc/library/asyncio-policy.rst
M Doc/whatsnew/3.9.rst
M Lib/asyncio/unix_events.py
M Lib/test/test_asyncio/test_subprocess.py

diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst
index aa8f8f13eae02..d9d3232d2408b 100644
--- a/Doc/library/asyncio-policy.rst
+++ b/Doc/library/asyncio-policy.rst
@@ -257,6 +257,18 @@ implementation used by the asyncio event loop:
    This solution requires a running event loop in the main thread to work, as
    :class:`SafeChildWatcher`.
 
+.. class:: PidfdChildWatcher
+
+   This implementation polls process file descriptors (pidfds) to await child
+   process termination. In some respects, :class:`PidfdChildWatcher` is a
+   "Goldilocks" child watcher implementation. It doesn't require signals or
+   threads, doesn't interfere with any processes launched outside the event
+   loop, and scales linearly with the number of subprocesses launched by the
+   event loop. The main disadvantage is that pidfds are specific to Linux, and
+   only work on recent (5.3+) kernels.
+
+   .. versionadded:: 3.9
+
 
 Custom Policies
 ===============
diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst
index 1cd21c6ab8f71..b1beb0be090cf 100644
--- a/Doc/whatsnew/3.9.rst
+++ b/Doc/whatsnew/3.9.rst
@@ -130,6 +130,9 @@ that schedules a shutdown for the default executor that waits on the
 :func:`asyncio.run` has been updated to use the new :term:`coroutine`.
 (Contributed by Kyle Stanley in :issue:`34037`.)
 
+Added :class:`asyncio.PidfdChildWatcher`, a Linux-specific child watcher
+implementation that polls process file descriptors. (:issue:`38692`)
+
 curses
 ------
 
diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py
index d8f653045aee4..d02460c004376 100644
--- a/Lib/asyncio/unix_events.py
+++ b/Lib/asyncio/unix_events.py
@@ -878,6 +878,73 @@ def __exit__(self, a, b, c):
         raise NotImplementedError()
 
 
+class PidfdChildWatcher(AbstractChildWatcher):
+    """Child watcher implementation using Linux's pid file descriptors.
+
+    This child watcher polls process file descriptors (pidfds) to await child
+    process termination. In some respects, PidfdChildWatcher is a "Goldilocks"
+    child watcher implementation. It doesn't require signals or threads, doesn't
+    interfere with any processes launched outside the event loop, and scales
+    linearly with the number of subprocesses launched by the event loop. The
+    main disadvantage is that pidfds are specific to Linux, and only work on
+    recent (5.3+) kernels.
+    """
+
+    def __init__(self):
+        self._loop = None
+        self._callbacks = {}
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        pass
+
+    def is_active(self):
+        return self._loop is not None and self._loop.is_running()
+
+    def close(self):
+        self.attach_loop(None)
+
+    def attach_loop(self, loop):
+        if self._loop is not None and loop is None and self._callbacks:
+            warnings.warn(
+                'A loop is being detached '
+                'from a child watcher with pending handlers',
+                RuntimeWarning)
+        for pidfd, _, _ in self._callbacks.values():
+            self._loop._remove_reader(pidfd)
+            os.close(pidfd)
+        self._callbacks.clear()
+        self._loop = loop
+
+    def add_child_handler(self, pid, callback, *args):
+        existing = self._callbacks.get(pid)
+        if existing is not None:
+            self._callbacks[pid] = existing[0], callback, args
+        else:
+            pidfd = os.pidfd_open(pid)
+            self._loop._add_reader(pidfd, self._do_wait, pid)
+            self._callbacks[pid] = pidfd, callback, args
+
+    def _do_wait(self, pid):
+        pidfd, callback, args = self._callbacks.pop(pid)
+        self._loop._remove_reader(pidfd)
+        _, status = os.waitpid(pid, 0)
+        os.close(pidfd)
+        returncode = _compute_returncode(status)
+        callback(pid, returncode, *args)
+
+    def remove_child_handler(self, pid):
+        try:
+            pidfd, _, _ = self._callbacks.pop(pid)
+        except KeyError:
+            return False
+        self._loop._remove_reader(pidfd)
+        os.close(pidfd)
+        return True
+
+
 def _compute_returncode(status):
     if os.WIFSIGNALED(status):
         # The child process died because of a signal.
diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py
index 17552d03f5f04..a6c3acc420ac7 100644
--- a/Lib/test/test_asyncio/test_subprocess.py
+++ b/Lib/test/test_asyncio/test_subprocess.py
@@ -1,3 +1,4 @@
+import os
 import signal
 import sys
 import unittest
@@ -691,6 +692,23 @@ class SubprocessFastWatcherTests(SubprocessWatcherMixin,
 
         Watcher = unix_events.FastChildWatcher
 
+    def has_pidfd_support():
+        if not hasattr(os, 'pidfd_open'):
+            return False
+        try:
+            os.close(os.pidfd_open(os.getpid()))
+        except OSError:
+            return False
+        return True
+
+    @unittest.skipUnless(
+        has_pidfd_support(),
+        "operating system does not support pidfds",
+    )
+    class SubprocessPidfdWatcherTests(SubprocessWatcherMixin,
+                                      test_utils.TestCase):
+        Watcher = unix_events.PidfdChildWatcher
+
 else:
     # Windows
     class SubprocessProactorTests(SubprocessMixin, test_utils.TestCase):
diff --git a/Misc/NEWS.d/next/Library/2019-11-05-19-15-57.bpo-38692.2DCDA-.rst b/Misc/NEWS.d/next/Library/2019-11-05-19-15-57.bpo-38692.2DCDA-.rst
new file mode 100644
index 0000000000000..7c8b3e8d74ee7
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-11-05-19-15-57.bpo-38692.2DCDA-.rst
@@ -0,0 +1,2 @@
+Add :class:`asyncio.PidfdChildWatcher`, a Linux-specific child watcher
+implementation that polls process file descriptors.