[Python-checkins] GH-82448: Add thread timeout for loop.shutdown_default_executor (#97561)
gvanrossum
webhook-mailer at python.org
Wed Sep 28 13:39:53 EDT 2022
https://github.com/python/cpython/commit/575a253b5c203e8d2ebfd239ed5a613179f8984f
commit: 575a253b5c203e8d2ebfd239ed5a613179f8984f
branch: main
author: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com>
committer: gvanrossum <gvanrossum at gmail.com>
date: 2022-09-28T10:39:42-07:00
summary:
GH-82448: Add thread timeout for loop.shutdown_default_executor (#97561)
Co-authored-by: Kyle Stanley <aeros167 at gmail.com>
files:
A Misc/NEWS.d/next/Library/2019-09-25-00-37-51.bpo-38267.X9Jb5V.rst
M Doc/library/asyncio-eventloop.rst
M Doc/library/asyncio-runner.rst
M Lib/asyncio/base_events.py
M Lib/asyncio/constants.py
M Lib/asyncio/runners.py
diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
index b61ffd5a323e..0a960ab38b49 100644
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -180,18 +180,27 @@ Running and stopping the loop
.. versionadded:: 3.6
-.. coroutinemethod:: loop.shutdown_default_executor()
+.. coroutinemethod:: loop.shutdown_default_executor(timeout=None)
Schedule the closure of the default executor and wait for it to join all of
the threads in the :class:`ThreadPoolExecutor`. After calling this method, a
:exc:`RuntimeError` will be raised if :meth:`loop.run_in_executor` is called
while using the default executor.
+ The *timeout* parameter specifies the amount of time the executor will
+ be given to finish joining. The default value is ``None``, which means the
+ executor will be given an unlimited amount of time.
+
+ If the timeout duration is reached, a warning is emitted and executor is
+ terminated without waiting for its threads to finish joining.
+
Note that there is no need to call this function when
:func:`asyncio.run` is used.
.. versionadded:: 3.9
+ .. versionchanged:: 3.12
+ Added the *timeout* parameter.
Scheduling callbacks
^^^^^^^^^^^^^^^^^^^^
diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst
index 4abe7b6e087a..c43d664eba71 100644
--- a/Doc/library/asyncio-runner.rst
+++ b/Doc/library/asyncio-runner.rst
@@ -28,7 +28,7 @@ Running an asyncio Program
This function runs the passed coroutine, taking care of
managing the asyncio event loop, *finalizing asynchronous
- generators*, and closing the threadpool.
+ generators*, and closing the executor.
This function cannot be called when another asyncio event loop is
running in the same thread.
@@ -41,6 +41,10 @@ Running an asyncio Program
the end. It should be used as a main entry point for asyncio
programs, and should ideally only be called once.
+ The executor is given a timeout duration of 5 minutes to shutdown.
+ If the executor hasn't finished within that duration, a warning is
+ emitted and the executor is closed.
+
Example::
async def main():
diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py
index a675fff1688d..9c9d98dbb9c5 100644
--- a/Lib/asyncio/base_events.py
+++ b/Lib/asyncio/base_events.py
@@ -561,8 +561,13 @@ async def shutdown_asyncgens(self):
'asyncgen': agen
})
- async def shutdown_default_executor(self):
- """Schedule the shutdown of the default executor."""
+ async def shutdown_default_executor(self, timeout=None):
+ """Schedule the shutdown of the default executor.
+
+ The timeout parameter specifies the amount of time the executor will
+ be given to finish joining. The default value is None, which means
+ that the executor will be given an unlimited amount of time.
+ """
self._executor_shutdown_called = True
if self._default_executor is None:
return
@@ -572,7 +577,13 @@ async def shutdown_default_executor(self):
try:
await future
finally:
- thread.join()
+ thread.join(timeout)
+
+ if thread.is_alive():
+ warnings.warn("The executor did not finishing joining "
+ f"its threads within {timeout} seconds.",
+ RuntimeWarning, stacklevel=2)
+ self._default_executor.shutdown(wait=False)
def _do_shutdown(self, future):
try:
diff --git a/Lib/asyncio/constants.py b/Lib/asyncio/constants.py
index f171ead28fec..f0ce0433a7a8 100644
--- a/Lib/asyncio/constants.py
+++ b/Lib/asyncio/constants.py
@@ -26,6 +26,9 @@
FLOW_CONTROL_HIGH_WATER_SSL_READ = 256 # KiB
FLOW_CONTROL_HIGH_WATER_SSL_WRITE = 512 # KiB
+# Default timeout for joining the threads in the threadpool
+THREAD_JOIN_TIMEOUT = 300
+
# The enum should be here to break circular dependencies between
# base_events and sslproto
class _SendfileMode(enum.Enum):
diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py
index 840b133df83e..b1c4dbd76197 100644
--- a/Lib/asyncio/runners.py
+++ b/Lib/asyncio/runners.py
@@ -9,7 +9,7 @@
from . import events
from . import exceptions
from . import tasks
-
+from . import constants
class _State(enum.Enum):
CREATED = "created"
@@ -69,7 +69,8 @@ def close(self):
loop = self._loop
_cancel_all_tasks(loop)
loop.run_until_complete(loop.shutdown_asyncgens())
- loop.run_until_complete(loop.shutdown_default_executor())
+ loop.run_until_complete(
+ loop.shutdown_default_executor(constants.THREAD_JOIN_TIMEOUT))
finally:
if self._set_event_loop:
events.set_event_loop(None)
@@ -160,8 +161,8 @@ def run(main, *, debug=None):
"""Execute the coroutine and return the result.
This function runs the passed coroutine, taking care of
- managing the asyncio event loop and finalizing asynchronous
- generators.
+ managing the asyncio event loop, finalizing asynchronous
+ generators and closing the default executor.
This function cannot be called when another asyncio event loop is
running in the same thread.
@@ -172,6 +173,10 @@ def run(main, *, debug=None):
It should be used as a main entry point for asyncio programs, and should
ideally only be called once.
+ The executor is given a timeout duration of 5 minutes to shutdown.
+ If the executor hasn't finished within that duration, a warning is
+ emitted and the executor is closed.
+
Example:
async def main():
diff --git a/Misc/NEWS.d/next/Library/2019-09-25-00-37-51.bpo-38267.X9Jb5V.rst b/Misc/NEWS.d/next/Library/2019-09-25-00-37-51.bpo-38267.X9Jb5V.rst
new file mode 100644
index 000000000000..b842fdcb73e9
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-25-00-37-51.bpo-38267.X9Jb5V.rst
@@ -0,0 +1,3 @@
+Add *timeout* parameter to :meth:`asyncio.loop.shutdown_default_executor`.
+The default value is ``None``, which means the executor will be given an unlimited amount of time.
+When called from :class:`asyncio.Runner` or :func:`asyncio.run`, the default timeout is 5 minutes.
More information about the Python-checkins
mailing list