From 3ccdd9b180f9a3f29c8ddc8ad1b331fe8df26519 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Wed, 13 Nov 2019 19:08:50 -0800 Subject: [PATCH] closes bpo-38692: Add a pidfd child process watcher to asyncio. (GH-17069) --- Doc/library/asyncio-policy.rst | 12 ++++ Doc/whatsnew/3.9.rst | 3 + Lib/asyncio/unix_events.py | 67 +++++++++++++++++++ Lib/test/test_asyncio/test_subprocess.py | 18 +++++ .../2019-11-05-19-15-57.bpo-38692.2DCDA-.rst | 2 + 5 files changed, 102 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2019-11-05-19-15-57.bpo-38692.2DCDA-.rst diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index aa8f8f13eae..d9d3232d240 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -257,6 +257,18 @@ implementation used by the asyncio event loop: This solution requires a running event loop in the main thread to work, as :class:`SafeChildWatcher`. +.. class:: PidfdChildWatcher + + This implementation polls process file descriptors (pidfds) to await child + process termination. In some respects, :class:`PidfdChildWatcher` is a + "Goldilocks" child watcher implementation. It doesn't require signals or + threads, doesn't interfere with any processes launched outside the event + loop, and scales linearly with the number of subprocesses launched by the + event loop. The main disadvantage is that pidfds are specific to Linux, and + only work on recent (5.3+) kernels. + + .. versionadded:: 3.9 + Custom Policies =============== diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 1cd21c6ab8f..b1beb0be090 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -130,6 +130,9 @@ that schedules a shutdown for the default executor that waits on the :func:`asyncio.run` has been updated to use the new :term:`coroutine`. (Contributed by Kyle Stanley in :issue:`34037`.) +Added :class:`asyncio.PidfdChildWatcher`, a Linux-specific child watcher +implementation that polls process file descriptors. (:issue:`38692`) + curses ------ diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index d8f653045ae..d02460c0043 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -878,6 +878,73 @@ class AbstractChildWatcher: raise NotImplementedError() +class PidfdChildWatcher(AbstractChildWatcher): + """Child watcher implementation using Linux's pid file descriptors. + + This child watcher polls process file descriptors (pidfds) to await child + process termination. In some respects, PidfdChildWatcher is a "Goldilocks" + child watcher implementation. It doesn't require signals or threads, doesn't + interfere with any processes launched outside the event loop, and scales + linearly with the number of subprocesses launched by the event loop. The + main disadvantage is that pidfds are specific to Linux, and only work on + recent (5.3+) kernels. + """ + + def __init__(self): + self._loop = None + self._callbacks = {} + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + pass + + def is_active(self): + return self._loop is not None and self._loop.is_running() + + def close(self): + self.attach_loop(None) + + def attach_loop(self, loop): + if self._loop is not None and loop is None and self._callbacks: + warnings.warn( + 'A loop is being detached ' + 'from a child watcher with pending handlers', + RuntimeWarning) + for pidfd, _, _ in self._callbacks.values(): + self._loop._remove_reader(pidfd) + os.close(pidfd) + self._callbacks.clear() + self._loop = loop + + def add_child_handler(self, pid, callback, *args): + existing = self._callbacks.get(pid) + if existing is not None: + self._callbacks[pid] = existing[0], callback, args + else: + pidfd = os.pidfd_open(pid) + self._loop._add_reader(pidfd, self._do_wait, pid) + self._callbacks[pid] = pidfd, callback, args + + def _do_wait(self, pid): + pidfd, callback, args = self._callbacks.pop(pid) + self._loop._remove_reader(pidfd) + _, status = os.waitpid(pid, 0) + os.close(pidfd) + returncode = _compute_returncode(status) + callback(pid, returncode, *args) + + def remove_child_handler(self, pid): + try: + pidfd, _, _ = self._callbacks.pop(pid) + except KeyError: + return False + self._loop._remove_reader(pidfd) + os.close(pidfd) + return True + + def _compute_returncode(status): if os.WIFSIGNALED(status): # The child process died because of a signal. diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 17552d03f5f..a6c3acc420a 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -1,3 +1,4 @@ +import os import signal import sys import unittest @@ -691,6 +692,23 @@ if sys.platform != 'win32': Watcher = unix_events.FastChildWatcher + def has_pidfd_support(): + if not hasattr(os, 'pidfd_open'): + return False + try: + os.close(os.pidfd_open(os.getpid())) + except OSError: + return False + return True + + @unittest.skipUnless( + has_pidfd_support(), + "operating system does not support pidfds", + ) + class SubprocessPidfdWatcherTests(SubprocessWatcherMixin, + test_utils.TestCase): + Watcher = unix_events.PidfdChildWatcher + else: # Windows class SubprocessProactorTests(SubprocessMixin, test_utils.TestCase): diff --git a/Misc/NEWS.d/next/Library/2019-11-05-19-15-57.bpo-38692.2DCDA-.rst b/Misc/NEWS.d/next/Library/2019-11-05-19-15-57.bpo-38692.2DCDA-.rst new file mode 100644 index 00000000000..7c8b3e8d74e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-11-05-19-15-57.bpo-38692.2DCDA-.rst @@ -0,0 +1,2 @@ +Add :class:`asyncio.PidfdChildWatcher`, a Linux-specific child watcher +implementation that polls process file descriptors.