[Python-checkins] cpython (merge 3.3 -> default): Issue #11461: Fix the incremental UTF-16 decoder. Original patch by
serhiy.storchaka
python-checkins at python.org
Tue Jan 8 22:47:44 CET 2013
http://hg.python.org/cpython/rev/5e84d020d001
changeset: 81333:5e84d020d001
parent: 81331:6d019606d948
parent: 81332:eed1883b1974
user: Serhiy Storchaka <storchaka at gmail.com>
date: Tue Jan 08 23:41:55 2013 +0200
summary:
Issue #11461: Fix the incremental UTF-16 decoder. Original patch by
Amaury Forgeot d'Arc. Added tests for partial decoding of non-BMP
characters.
files:
Lib/test/test_codecs.py | 48 ++++++++++++++++++++++++----
Misc/NEWS | 3 +
Objects/unicodeobject.c | 5 ++-
3 files changed, 47 insertions(+), 9 deletions(-)
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -330,7 +330,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"", # first byte of BOM read
"", # second byte of BOM read
@@ -352,6 +352,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -386,7 +390,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"",
"",
@@ -404,6 +408,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -426,7 +434,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"",
"",
@@ -444,6 +452,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -494,7 +506,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"", # first byte of BOM read
"", # second byte of BOM read => byteorder known
@@ -506,6 +518,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -543,7 +559,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"",
"\x00",
@@ -553,6 +569,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -582,7 +602,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"",
"\x00",
@@ -592,6 +612,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -621,7 +645,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff\U00010000",
[
"\x00",
"\x00",
@@ -634,6 +658,10 @@
"\x00\xff\u07ff\u0800",
"\x00\xff\u07ff\u0800",
"\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff\U00010000",
]
)
@@ -816,7 +844,7 @@
def test_partial(self):
self.check_partial(
- "\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
[
"",
"",
@@ -835,6 +863,10 @@
"\ufeff\x00\xff\u07ff\u0800",
"\ufeff\x00\xff\u07ff\u0800",
"\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
]
)
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
Core and Builtins
-----------------
+- Issue #11461: Fix the incremental UTF-16 decoder. Original patch by
+ Amaury Forgeot d'Arc.
+
- Issue #16881: Fix Py_ARRAY_LENGTH macro for GCC < 3.1.
- Issue #16856: Fix a segmentation fault from calling repr() on a dict with
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -5159,8 +5159,11 @@
/* The remaining input chars are ignored if the callback
chooses to skip the input */
case 1:
+ q -= 2;
+ if (consumed)
+ goto End;
errmsg = "unexpected end of data";
- startinpos = ((const char *)q) - 2 - starts;
+ startinpos = ((const char *)q) - starts;
endinpos = ((const char *)e) - starts;
break;
case 2:
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list