[Python-checkins] bpo-40330: Fix utf-8 size check in ShareableList (GH-19606)
Antoine Pitrou
webhook-mailer at python.org
Mon Apr 20 14:55:10 EDT 2020
https://github.com/python/cpython/commit/eba9f6155df59c9beed97fb5764c9f01dd941af0
commit: eba9f6155df59c9beed97fb5764c9f01dd941af0
branch: master
author: Antoine Pitrou <antoine at python.org>
committer: GitHub <noreply at github.com>
date: 2020-04-20T11:54:55-07:00
summary:
bpo-40330: Fix utf-8 size check in ShareableList (GH-19606)
The item size must be checked after encoding to bytes, not before.
Automerge-Triggered-By: @pitrou
files:
A Misc/NEWS.d/next/Library/2020-04-19-17-31-29.bpo-40330.DGjoIS.rst
M Lib/multiprocessing/shared_memory.py
M Lib/test/_test_multiprocessing.py
diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 87e46cfbe526d..a3a5fcf4aba1e 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -445,11 +445,14 @@ def __setitem__(self, position, value):
if not isinstance(value, (str, bytes)):
new_format = self._types_mapping[type(value)]
+ encoded_value = value
else:
allocated_length = self._allocated_offsets[position + 1] - item_offset
- if len(value) > allocated_length:
- raise ValueError("exceeds available storage for existing str")
+ encoded_value = (value.encode(_encoding)
+ if isinstance(value, str) else value)
+ if len(encoded_value) > allocated_length:
+ raise ValueError("bytes/str item exceeds available storage")
if current_format[-1] == "s":
new_format = current_format
else:
@@ -462,8 +465,7 @@ def __setitem__(self, position, value):
new_format,
value
)
- value = value.encode(_encoding) if isinstance(value, str) else value
- struct.pack_into(new_format, self.shm.buf, offset, value)
+ struct.pack_into(new_format, self.shm.buf, offset, encoded_value)
def __reduce__(self):
return partial(self.__class__, name=self.shm.name), ()
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index d00e928c17790..d633e02d016fc 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -3995,9 +3995,21 @@ def test_shared_memory_ShareableList_basics(self):
sl[4] = 'some' # Change type at a given position.
self.assertEqual(sl[4], 'some')
self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q')
- with self.assertRaises(ValueError):
- sl[4] = 'far too many' # Exceeds available storage.
+ with self.assertRaisesRegex(ValueError,
+ "exceeds available storage"):
+ sl[4] = 'far too many'
self.assertEqual(sl[4], 'some')
+ sl[0] = 'encodés' # Exactly 8 bytes of UTF-8 data
+ self.assertEqual(sl[0], 'encodés')
+ self.assertEqual(sl[1], b'HoWdY') # no spillage
+ with self.assertRaisesRegex(ValueError,
+ "exceeds available storage"):
+ sl[0] = 'encodées' # Exactly 9 bytes of UTF-8 data
+ self.assertEqual(sl[1], b'HoWdY')
+ with self.assertRaisesRegex(ValueError,
+ "exceeds available storage"):
+ sl[1] = b'123456789'
+ self.assertEqual(sl[1], b'HoWdY')
# Exercise count().
with warnings.catch_warnings():
diff --git a/Misc/NEWS.d/next/Library/2020-04-19-17-31-29.bpo-40330.DGjoIS.rst b/Misc/NEWS.d/next/Library/2020-04-19-17-31-29.bpo-40330.DGjoIS.rst
new file mode 100644
index 0000000000000..98cb62f1b115e
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2020-04-19-17-31-29.bpo-40330.DGjoIS.rst
@@ -0,0 +1,2 @@
+In :meth:`ShareableList.__setitem__`, check the size of a new string item
+after encoding it to utf-8, not before.
More information about the Python-checkins
mailing list