[issue15278] UnicodeDecodeError when readline in codecs.py
Serhiy Storchaka
report at bugs.python.org
Mon Oct 8 18:10:09 CEST 2012
Serhiy Storchaka added the comment:
Here are the patches.
----------
keywords: +patch
Added file: http://bugs.python.org/file27495/utf16_partial_decode-3.3.patch
Added file: http://bugs.python.org/file27496/utf16_partial_decode-3.2.patch
Added file: http://bugs.python.org/file27497/utf16_partial_decode-2.7.patch
_______________________________________
Python tracker <report at bugs.python.org>
<http://bugs.python.org/issue15278>
_______________________________________
-------------- next part --------------
diff -r 0d41a1b4c4fe Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py Mon Oct 08 07:46:11 2012 +0200
+++ b/Lib/test/test_codecs.py Mon Oct 08 18:26:34 2012 +0300
@@ -330,7 +330,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"", # first byte of BOM read
"", # second byte of BOM read
@@ -352,6 +352,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -386,7 +390,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"",
"",
@@ -404,6 +408,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -426,7 +434,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"",
"",
@@ -444,6 +452,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -494,7 +506,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"", # first byte of BOM read
"", # second byte of BOM read => byteorder known
@@ -506,6 +518,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -543,7 +559,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"",
"\x00",
@@ -553,6 +569,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -582,7 +602,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"",
"\x00",
@@ -592,6 +612,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -621,7 +645,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff\U00010000",
[
"\x00",
"\x00",
@@ -634,6 +658,10 @@
"\x00\xff\u07ff\u0800",
"\x00\xff\u07ff\u0800",
"\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff\U00010000",
]
)
@@ -812,7 +840,7 @@
def test_partial(self):
self.check_partial(
- "\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
[
"",
"",
@@ -831,6 +859,10 @@
"\ufeff\x00\xff\u07ff\u0800",
"\ufeff\x00\xff\u07ff\u0800",
"\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
]
)
diff -r 0d41a1b4c4fe Objects/unicodeobject.c
--- a/Objects/unicodeobject.c Mon Oct 08 07:46:11 2012 +0200
+++ b/Objects/unicodeobject.c Mon Oct 08 18:26:34 2012 +0300
@@ -5143,8 +5143,11 @@
/* The remaining input chars are ignored if the callback
chooses to skip the input */
case 1:
+ q -= 2;
+ if (consumed)
+ goto End;
errmsg = "unexpected end of data";
- startinpos = ((const char *)q) - 2 - starts;
+ startinpos = ((const char *)q) - starts;
endinpos = ((const char *)e) - starts;
break;
case 2:
-------------- next part --------------
diff -r 88a7b9c3b6c0 Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py Sun Oct 07 18:04:38 2012 +0200
+++ b/Lib/test/test_codecs.py Mon Oct 08 18:32:00 2012 +0300
@@ -313,7 +313,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"", # first byte of BOM read
"", # second byte of BOM read
@@ -335,6 +335,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -369,7 +373,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"",
"",
@@ -387,6 +391,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -409,7 +417,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"",
"",
@@ -427,6 +435,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -477,7 +489,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"", # first byte of BOM read
"", # second byte of BOM read => byteorder known
@@ -489,6 +501,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -526,7 +542,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"",
"\x00",
@@ -536,6 +552,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -565,7 +585,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"",
"\x00",
@@ -575,6 +595,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -604,7 +628,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff\U00010000",
[
"\x00",
"\x00",
@@ -617,6 +641,10 @@
"\x00\xff\u07ff\u0800",
"\x00\xff\u07ff\u0800",
"\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff\U00010000",
]
)
@@ -690,7 +718,7 @@
def test_partial(self):
self.check_partial(
- "\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
[
"",
"",
@@ -709,6 +737,10 @@
"\ufeff\x00\xff\u07ff\u0800",
"\ufeff\x00\xff\u07ff\u0800",
"\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
]
)
diff -r 88a7b9c3b6c0 Objects/unicodeobject.c
--- a/Objects/unicodeobject.c Sun Oct 07 18:04:38 2012 +0200
+++ b/Objects/unicodeobject.c Mon Oct 08 18:32:00 2012 +0300
@@ -3572,8 +3572,11 @@
/* UTF-16 code pair: */
if (e - q < 2) {
+ q -= 2;
+ if (consumed)
+ break;
errmsg = "unexpected end of data";
- startinpos = (((const char *)q) - 2) - starts;
+ startinpos = ((const char *)q) - starts;
endinpos = ((const char *)e) - starts;
goto utf16Error;
}
-------------- next part --------------
diff -r 3b484f53f91b Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py Sun Oct 07 20:37:54 2012 -0700
+++ b/Lib/test/test_codecs.py Mon Oct 08 18:46:38 2012 +0300
@@ -281,7 +281,7 @@
def test_partial(self):
self.check_partial(
- u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff\U00010000",
[
u"", # first byte of BOM read
u"", # second byte of BOM read
@@ -303,6 +303,10 @@
u"\x00\xff\u0100",
u"\x00\xff\u0100",
u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff\U00010000",
]
)
@@ -331,7 +335,7 @@
def test_partial(self):
self.check_partial(
- u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff\U00010000",
[
u"",
u"",
@@ -349,6 +353,10 @@
u"\x00\xff\u0100",
u"\x00\xff\u0100",
u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff\U00010000",
]
)
@@ -371,7 +379,7 @@
def test_partial(self):
self.check_partial(
- u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff\U00010000",
[
u"",
u"",
@@ -389,6 +397,10 @@
u"\x00\xff\u0100",
u"\x00\xff\u0100",
u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff\U00010000",
]
)
@@ -439,7 +451,7 @@
def test_partial(self):
self.check_partial(
- u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff\U00010000",
[
u"", # first byte of BOM read
u"", # second byte of BOM read => byteorder known
@@ -451,6 +463,10 @@
u"\x00\xff\u0100",
u"\x00\xff\u0100",
u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff\U00010000",
]
)
@@ -481,7 +497,7 @@
def test_partial(self):
self.check_partial(
- u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff\U00010000",
[
u"",
u"\x00",
@@ -491,6 +507,10 @@
u"\x00\xff\u0100",
u"\x00\xff\u0100",
u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff\U00010000",
]
)
@@ -514,7 +534,7 @@
def test_partial(self):
self.check_partial(
- u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff\U00010000",
[
u"",
u"\x00",
@@ -524,6 +544,10 @@
u"\x00\xff\u0100",
u"\x00\xff\u0100",
u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff",
+ u"\x00\xff\u0100\uffff\U00010000",
]
)
@@ -547,7 +571,7 @@
def test_partial(self):
self.check_partial(
- u"\x00\xff\u07ff\u0800\uffff",
+ u"\x00\xff\u07ff\u0800\uffff\U00010000",
[
u"\x00",
u"\x00",
@@ -560,6 +584,10 @@
u"\x00\xff\u07ff\u0800",
u"\x00\xff\u07ff\u0800",
u"\x00\xff\u07ff\u0800\uffff",
+ u"\x00\xff\u07ff\u0800\uffff",
+ u"\x00\xff\u07ff\u0800\uffff",
+ u"\x00\xff\u07ff\u0800\uffff",
+ u"\x00\xff\u07ff\u0800\uffff\U00010000",
]
)
@@ -619,7 +647,7 @@
def test_partial(self):
self.check_partial(
- u"\ufeff\x00\xff\u07ff\u0800\uffff",
+ u"\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
[
u"",
u"",
@@ -638,6 +666,10 @@
u"\ufeff\x00\xff\u07ff\u0800",
u"\ufeff\x00\xff\u07ff\u0800",
u"\ufeff\x00\xff\u07ff\u0800\uffff",
+ u"\ufeff\x00\xff\u07ff\u0800\uffff",
+ u"\ufeff\x00\xff\u07ff\u0800\uffff",
+ u"\ufeff\x00\xff\u07ff\u0800\uffff",
+ u"\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
]
)
diff -r 3b484f53f91b Objects/unicodeobject.c
--- a/Objects/unicodeobject.c Sun Oct 07 20:37:54 2012 -0700
+++ b/Objects/unicodeobject.c Mon Oct 08 18:46:38 2012 +0300
@@ -2565,8 +2565,11 @@
/* UTF-16 code pair: */
if (e - q < 2) {
+ q -= 2;
+ if (consumed)
+ break;
errmsg = "unexpected end of data";
- startinpos = (((const char *)q)-2)-starts;
+ startinpos = ((const char *)q)-starts;
endinpos = ((const char *)e)-starts;
goto utf16Error;
}
More information about the Python-bugs-list
mailing list