[Python-checkins] cpython (2.7): Issue #14579: Fix error handling bug in the utf-16 decoder.
antoine.pitrou
python-checkins at python.org
Sat Jul 21 00:54:04 CEST 2012
http://hg.python.org/cpython/rev/4cadf91aaddd
changeset: 78208:4cadf91aaddd
branch: 2.7
parent: 78187:204be25f24bd
user: Antoine Pitrou <solipsis at pitrou.net>
date: Sat Jul 21 00:52:06 2012 +0200
summary:
Issue #14579: Fix error handling bug in the utf-16 decoder.
Patch by Serhiy Storchaka.
files:
Lib/test/test_codecs.py | 28 ++++++++++++++++++++++++++--
Misc/NEWS | 3 +++
Objects/unicodeobject.c | 2 +-
3 files changed, 30 insertions(+), 3 deletions(-)
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -495,7 +495,19 @@
)
def test_errors(self):
- self.assertRaises(UnicodeDecodeError, codecs.utf_16_le_decode, "\xff", "strict", True)
+ tests = [
+ (b'\xff', u'\ufffd'),
+ (b'A\x00Z', u'A\ufffd'),
+ (b'A\x00B\x00C\x00D\x00Z', u'ABCD\ufffd'),
+ (b'\x00\xd8', u'\ufffd'),
+ (b'\x00\xd8A', u'\ufffd'),
+ (b'\x00\xd8A\x00', u'\ufffdA'),
+ (b'\x00\xdcA\x00', u'\ufffdA'),
+ ]
+ for raw, expected in tests:
+ self.assertRaises(UnicodeDecodeError, codecs.utf_16_le_decode,
+ raw, 'strict', True)
+ self.assertEqual(raw.decode('utf-16le', 'replace'), expected)
class UTF16BETest(ReadTest):
encoding = "utf-16-be"
@@ -516,7 +528,19 @@
)
def test_errors(self):
- self.assertRaises(UnicodeDecodeError, codecs.utf_16_be_decode, "\xff", "strict", True)
+ tests = [
+ (b'\xff', u'\ufffd'),
+ (b'\x00A\xff', u'A\ufffd'),
+ (b'\x00A\x00B\x00C\x00DZ', u'ABCD\ufffd'),
+ (b'\xd8\x00', u'\ufffd'),
+ (b'\xd8\x00\xdc', u'\ufffd'),
+ (b'\xd8\x00\x00A', u'\ufffdA'),
+ (b'\xdc\x00\x00A', u'\ufffdA'),
+ ]
+ for raw, expected in tests:
+ self.assertRaises(UnicodeDecodeError, codecs.utf_16_be_decode,
+ raw, 'strict', True)
+ self.assertEqual(raw.decode('utf-16be', 'replace'), expected)
class UTF8Test(ReadTest):
encoding = "utf-8"
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -9,6 +9,9 @@
Core and Builtins
-----------------
+- Issue #14579: Fix error handling bug in the utf-16 decoder. Patch by
+ Serhiy Storchaka.
+
- Issue #15368: An issue that caused bytecode generation to be
non-deterministic when using randomized hashing (-R) has been fixed.
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -2564,7 +2564,7 @@
}
/* UTF-16 code pair: */
- if (q >= e) {
+ if (e - q < 2) {
errmsg = "unexpected end of data";
startinpos = (((const char *)q)-2)-starts;
endinpos = ((const char *)e)-starts;
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list