[Python-checkins] bpo-44962: Fix a race in WeakKeyDict, WeakValueDict and WeakSet when two threads attempt to commit the last pending removal (GH-27921)
ambv
webhook-mailer at python.org
Sat Aug 28 13:07:56 EDT 2021
https://github.com/python/cpython/commit/206b21ed9f64fedff67bfea7cf73e423e3e32393
commit: 206b21ed9f64fedff67bfea7cf73e423e3e32393
branch: main
author: Thomas Grainger <tagrain at gmail.com>
committer: ambv <lukasz at langa.pl>
date: 2021-08-28T19:07:37+02:00
summary:
bpo-44962: Fix a race in WeakKeyDict, WeakValueDict and WeakSet when two threads attempt to commit the last pending removal (GH-27921)
Fixes:
Traceback (most recent call last):
File "/home/graingert/projects/asyncio-demo/demo.py", line 36, in <module>
sys.exit(main())
File "/home/graingert/projects/asyncio-demo/demo.py", line 30, in main
test_all_tasks_threading()
File "/home/graingert/projects/asyncio-demo/demo.py", line 24, in test_all_tasks_threading
results.append(f.result())
File "/usr/lib/python3.10/concurrent/futures/_base.py", line 438, in result
return self.__get_result()
File "/usr/lib/python3.10/concurrent/futures/_base.py", line 390, in __get_result
raise self._exception
File "/usr/lib/python3.10/concurrent/futures/thread.py", line 52, in run
result = self.fn(*self.args, **self.kwargs)
File "/usr/lib/python3.10/asyncio/runners.py", line 47, in run
_cancel_all_tasks(loop)
File "/usr/lib/python3.10/asyncio/runners.py", line 56, in _cancel_all_tasks
to_cancel = tasks.all_tasks(loop)
File "/usr/lib/python3.10/asyncio/tasks.py", line 53, in all_tasks
tasks = list(_all_tasks)
File "/usr/lib/python3.10/_weakrefset.py", line 60, in __iter__
with _IterationGuard(self):
File "/usr/lib/python3.10/_weakrefset.py", line 33, in __exit__
w._commit_removals()
File "/usr/lib/python3.10/_weakrefset.py", line 57, in _commit_removals
discard(l.pop())
IndexError: pop from empty list
Also fixes:
Exception ignored in: weakref callback <function WeakKeyDictionary.__init__.<locals>.remove at 0x00007fe82245d2e0>
Traceback (most recent call last):
File "/usr/lib/pypy3/lib-python/3/weakref.py", line 390, in remove
del self.data[k]
KeyError: <weakref at 0x00007fe76e8d8180; dead>
Exception ignored in: weakref callback <function WeakKeyDictionary.__init__.<locals>.remove at 0x00007fe82245d2e0>
Traceback (most recent call last):
File "/usr/lib/pypy3/lib-python/3/weakref.py", line 390, in remove
del self.data[k]
KeyError: <weakref at 0x00007fe76e8d81a0; dead>
Exception ignored in: weakref callback <function WeakKeyDictionary.__init__.<locals>.remove at 0x00007fe82245d2e0>
Traceback (most recent call last):
File "/usr/lib/pypy3/lib-python/3/weakref.py", line 390, in remove
del self.data[k]
KeyError: <weakref at 0x000056548f1e24a0; dead>
See: https://github.com/agronholm/anyio/issues/362#issuecomment-904424310
See also: https://bugs.python.org/issue29519
Co-authored-by: Łukasz Langa <lukasz at langa.pl>
files:
A Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst
M Lib/_weakrefset.py
M Lib/weakref.py
diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py
index b267780f0ced7..2a27684324d80 100644
--- a/Lib/_weakrefset.py
+++ b/Lib/_weakrefset.py
@@ -51,10 +51,14 @@ def _remove(item, selfref=ref(self)):
self.update(data)
def _commit_removals(self):
- l = self._pending_removals
+ pop = self._pending_removals.pop
discard = self.data.discard
- while l:
- discard(l.pop())
+ while True:
+ try:
+ item = pop()
+ except IndexError:
+ return
+ discard(item)
def __iter__(self):
with _IterationGuard(self):
diff --git a/Lib/weakref.py b/Lib/weakref.py
index a968139f98631..994ea8aa37de5 100644
--- a/Lib/weakref.py
+++ b/Lib/weakref.py
@@ -119,14 +119,17 @@ def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref):
self.data = {}
self.update(other, **kw)
- def _commit_removals(self):
- l = self._pending_removals
+ def _commit_removals(self, _atomic_removal=_remove_dead_weakref):
+ pop = self._pending_removals.pop
d = self.data
# We shouldn't encounter any KeyError, because this method should
# always be called *before* mutating the dict.
- while l:
- key = l.pop()
- _remove_dead_weakref(d, key)
+ while True:
+ try:
+ key = pop()
+ except IndexError:
+ return
+ _atomic_removal(d, key)
def __getitem__(self, key):
if self._pending_removals:
@@ -370,7 +373,10 @@ def remove(k, selfref=ref(self)):
if self._iterating:
self._pending_removals.append(k)
else:
- del self.data[k]
+ try:
+ del self.data[k]
+ except KeyError:
+ pass
self._remove = remove
# A list of dead weakrefs (keys to be removed)
self._pending_removals = []
@@ -384,11 +390,16 @@ def _commit_removals(self):
# because a dead weakref never compares equal to a live weakref,
# even if they happened to refer to equal objects.
# However, it means keys may already have been removed.
- l = self._pending_removals
+ pop = self._pending_removals.pop
d = self.data
- while l:
+ while True:
+ try:
+ key = pop()
+ except IndexError:
+ return
+
try:
- del d[l.pop()]
+ del d[key]
except KeyError:
pass
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst
new file mode 100644
index 0000000000000..6b4b9dfd8bc3c
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst
@@ -0,0 +1 @@
+Fix a race in WeakKeyDictionary, WeakValueDictionary and WeakSet when two threads attempt to commit the last pending removal. This fixes asyncio.create_task and fixes a data loss in asyncio.run where shutdown_asyncgens is not run
\ No newline at end of file
More information about the Python-checkins
mailing list