[Python-checkins] bpo-35372: Fix the code page decoder for input > 2 GiB. (GH-10848)
Miss Islington (bot)
webhook-mailer at python.org
Mon Dec 3 04:15:06 EST 2018
https://github.com/python/cpython/commit/0f9b6687eb8b26dd804abcc6efd4d6430ae16f24
commit: 0f9b6687eb8b26dd804abcc6efd4d6430ae16f24
branch: 3.6
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2018-12-03T01:15:02-08:00
summary:
bpo-35372: Fix the code page decoder for input > 2 GiB. (GH-10848)
(cherry picked from commit 4013c179117754b039957db4730880bf3285919d)
Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>
files:
A Misc/NEWS.d/next/Core and Builtins/2018-12-01-19-20-53.bpo-35372.RwVJjZ.rst
M Lib/test/test_codecs.py
M Objects/unicodeobject.c
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index eb21a3915b93..56485de3f6e9 100644
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -3191,6 +3191,24 @@ def _get_fake_codepage(*a):
finally:
_bootlocale.getpreferredencoding = old_getpreferredencoding
+ @support.bigmemtest(size=2**31, memuse=7, dry_run=False)
+ def test_large_input(self):
+ # Test input longer than INT_MAX.
+ # Input should contain undecodable bytes before and after
+ # the INT_MAX limit.
+ encoded = (b'01234567' * (2**28-1) +
+ b'\x85\x86\xea\xeb\xec\xef\xfc\xfd\xfe\xff')
+ self.assertEqual(len(encoded), 2**31+2)
+ decoded = codecs.code_page_decode(932, encoded, 'surrogateescape', True)
+ self.assertEqual(decoded[1], len(encoded))
+ del encoded
+ self.assertEqual(len(decoded[0]), decoded[1])
+ self.assertEqual(decoded[0][:10], '0123456701')
+ self.assertEqual(decoded[0][-20:],
+ '6701234567'
+ '\udc85\udc86\udcea\udceb\udcec'
+ '\udcef\udcfc\udcfd\udcfe\udcff')
+
class ASCIITest(unittest.TestCase):
def test_encode(self):
diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-12-01-19-20-53.bpo-35372.RwVJjZ.rst b/Misc/NEWS.d/next/Core and Builtins/2018-12-01-19-20-53.bpo-35372.RwVJjZ.rst
new file mode 100644
index 000000000000..dc2de44b4f63
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2018-12-01-19-20-53.bpo-35372.RwVJjZ.rst
@@ -0,0 +1,2 @@
+Fixed the code page decoder for input longer than 2 GiB containing
+undecodable bytes.
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 29b019887ac0..0e64bf943db3 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -7285,7 +7285,7 @@ decode_code_page_errors(UINT code_page,
"in the target code page.";
/* each step cannot decode more than 1 character, but a character can be
represented as a surrogate pair */
- wchar_t buffer[2], *startout, *out;
+ wchar_t buffer[2], *out;
int insize;
Py_ssize_t outsize;
PyObject *errorHandler = NULL;
@@ -7322,7 +7322,7 @@ decode_code_page_errors(UINT code_page,
*v = (PyObject*)_PyUnicode_New(size * Py_ARRAY_LENGTH(buffer));
if (*v == NULL)
goto error;
- startout = PyUnicode_AS_UNICODE(*v);
+ out = PyUnicode_AS_UNICODE(*v);
}
else {
/* Extend unicode object */
@@ -7333,11 +7333,10 @@ decode_code_page_errors(UINT code_page,
}
if (unicode_resize(v, n + size * Py_ARRAY_LENGTH(buffer)) < 0)
goto error;
- startout = PyUnicode_AS_UNICODE(*v) + n;
+ out = PyUnicode_AS_UNICODE(*v) + n;
}
/* Decode the byte string character per character */
- out = startout;
while (in < endin)
{
/* Decode a character */
@@ -7392,7 +7391,7 @@ decode_code_page_errors(UINT code_page,
*out = 0;
/* Extend unicode object */
- outsize = out - startout;
+ outsize = out - PyUnicode_AS_UNICODE(*v);
assert(outsize <= PyUnicode_WSTR_LENGTH(*v));
if (unicode_resize(v, outsize) < 0)
goto error;
More information about the Python-checkins
mailing list