[issue15379] Charmap decoding of no-BMP characters
Serhiy Storchaka
report at bugs.python.org
Fri Sep 21 22:14:10 CEST 2012
Serhiy Storchaka added the comment:
Patches updated. Added a few new tests, used MAX_UNICODE, a little changed extrachars grow step.
----------
Added file: http://bugs.python.org/file27249/decode_charmap_maxchar-3.3_2.patch
Added file: http://bugs.python.org/file27250/decode_charmap_maxchar-3.2_2.patch
_______________________________________
Python tracker <report at bugs.python.org>
<http://bugs.python.org/issue15379>
_______________________________________
-------------- next part --------------
diff -r 3e677956eef4 Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py Fri Sep 21 10:12:14 2012 -0700
+++ b/Lib/test/test_codecs.py Fri Sep 21 23:08:31 2012 +0300
@@ -1693,6 +1693,15 @@
)
self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict", "\U0010FFFFbc"),
+ ("\U0010FFFFbc", 3)
+ )
+
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab"
+ )
+
+ self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "replace", "ab"),
("ab\ufffd", 3)
)
@@ -1718,6 +1727,113 @@
("", len(allbytes))
)
+ def test_decode_with_int2str_map(self):
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: 'a', 1: 'b', 2: 'c'}),
+ ("abc", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: 'Aa', 1: 'Bb', 2: 'Cc'}),
+ ("AaBbCc", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: '\U0010FFFF', 1: 'b', 2: 'c'}),
+ ("\U0010FFFFbc", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: 'a', 1: 'b', 2: ''}),
+ ("ab", 3)
+ )
+
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict",
+ {0: 'a', 1: 'b'}
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "replace",
+ {0: 'a', 1: 'b'}),
+ ("ab\ufffd", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "replace",
+ {0: 'a', 1: 'b', 2: None}),
+ ("ab\ufffd", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "ignore",
+ {0: 'a', 1: 'b'}),
+ ("ab", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "ignore",
+ {0: 'a', 1: 'b', 2: None}),
+ ("ab", 3)
+ )
+
+ allbytes = bytes(range(256))
+ self.assertEqual(
+ codecs.charmap_decode(allbytes, "ignore", {}),
+ ("", len(allbytes))
+ )
+
+ def test_decode_with_int2int_map(self):
+ a = ord('a')
+ b = ord('b')
+ c = ord('c')
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: a, 1: b, 2: c}),
+ ("abc", 3)
+ )
+
+ # Issue #15379
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: 0x10FFFF, 1: b, 2: c}),
+ ("\U0010FFFFbc", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: sys.maxunicode, 1: b, 2: c}),
+ (chr(sys.maxunicode) + "bc", 3)
+ )
+
+ self.assertRaises(TypeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict",
+ {0: sys.maxunicode + 1, 1: b, 2: c}
+ )
+
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict",
+ {0: a, 1: b},
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "replace",
+ {0: a, 1: b}),
+ ("ab\ufffd", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "ignore",
+ {0: a, 1: b}),
+ ("ab", 3)
+ )
+
+
class WithStmtTest(unittest.TestCase):
def test_encodedfile(self):
f = io.BytesIO(b"\xc3\xbc")
diff -r 3e677956eef4 Objects/unicodeobject.c
--- a/Objects/unicodeobject.c Fri Sep 21 10:12:14 2012 -0700
+++ b/Objects/unicodeobject.c Fri Sep 21 23:08:31 2012 +0300
@@ -7525,9 +7525,10 @@
/* Apply mapping */
if (PyLong_Check(x)) {
long value = PyLong_AS_LONG(x);
- if (value < 0 || value > 65535) {
- PyErr_SetString(PyExc_TypeError,
- "character mapping must be in range(65536)");
+ if (value < 0 || value > MAX_UNICODE) {
+ PyErr_Format(PyExc_TypeError,
+ "character mapping must be in range(0x%lx)",
+ (unsigned long)MAX_UNICODE + 1);
Py_DECREF(x);
goto onError;
}
-------------- next part --------------
diff -r aa73e60f65e9 Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py Fri Sep 21 17:26:35 2012 +0300
+++ b/Lib/test/test_codecs.py Fri Sep 21 23:08:15 2012 +0300
@@ -1546,6 +1546,10 @@
("abc", 3)
)
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab"
+ )
+
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "replace", "ab"),
("ab\ufffd", 3)
@@ -1572,6 +1576,107 @@
("", len(allbytes))
)
+ def test_decode_with_int2str_map(self):
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: 'a', 1: 'b', 2: 'c'}),
+ ("abc", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: 'Aa', 1: 'Bb', 2: 'Cc'}),
+ ("AaBbCc", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: '\U0010FFFF', 1: 'b', 2: 'c'}),
+ ("\U0010FFFFbc", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: 'a', 1: 'b', 2: ''}),
+ ("ab", 3)
+ )
+
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict",
+ {0: 'a', 1: 'b'}
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "replace",
+ {0: 'a', 1: 'b'}),
+ ("ab\ufffd", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "replace",
+ {0: 'a', 1: 'b', 2: None}),
+ ("ab\ufffd", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "ignore",
+ {0: 'a', 1: 'b'}),
+ ("ab", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "ignore",
+ {0: 'a', 1: 'b', 2: None}),
+ ("ab", 3)
+ )
+
+ allbytes = bytes(range(256))
+ self.assertEqual(
+ codecs.charmap_decode(allbytes, "ignore", {}),
+ ("", len(allbytes))
+ )
+
+ def test_decode_with_int2int_map(self):
+ a = ord('a')
+ b = ord('b')
+ c = ord('c')
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: a, 1: b, 2: c}),
+ ("abc", 3)
+ )
+
+ # Issue #15379
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: 0x10FFFF, 1: b, 2: c}),
+ ("\U0010FFFFbc", 3)
+ )
+
+ self.assertRaises(TypeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict",
+ {0: 0x110000, 1: b, 2: c}
+ )
+
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict",
+ {0: a, 1: b},
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "replace",
+ {0: a, 1: b}),
+ ("ab\ufffd", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "ignore",
+ {0: a, 1: b}),
+ ("ab", 3)
+ )
+
+
class WithStmtTest(unittest.TestCase):
def test_encodedfile(self):
f = io.BytesIO(b"\xc3\xbc")
diff -r aa73e60f65e9 Objects/unicodeobject.c
--- a/Objects/unicodeobject.c Fri Sep 21 17:26:35 2012 +0300
+++ b/Objects/unicodeobject.c Fri Sep 21 23:08:15 2012 +0300
@@ -5250,12 +5250,36 @@
/* Apply mapping */
if (PyLong_Check(x)) {
long value = PyLong_AS_LONG(x);
- if (value < 0 || value > 65535) {
+ if (value < 0 || value > 0x10FFFF) {
PyErr_SetString(PyExc_TypeError,
- "character mapping must be in range(65536)");
+ "character mapping must be in range(0x110000)");
Py_DECREF(x);
goto onError;
}
+
+#ifndef Py_UNICODE_WIDE
+ if (value > 0xFFFF) {
+ /* see the code for 1-n mapping below */
+ if (extrachars < 2) {
+ /* resize first */
+ Py_ssize_t oldpos = p - PyUnicode_AS_UNICODE(v);
+ Py_ssize_t needed = 10 - extrachars;
+ extrachars += needed;
+ /* XXX overflow detection missing */
+ if (_PyUnicode_Resize(&v,
+ PyUnicode_GET_SIZE(v) + needed) < 0) {
+ Py_DECREF(x);
+ goto onError;
+ }
+ p = PyUnicode_AS_UNICODE(v) + oldpos;
+ }
+ value -= 0x10000;
+ *p++ = 0xD800 | (value >> 10);
+ *p++ = 0xDC00 | (value & 0x3FF);
+ extrachars -= 2;
+ }
+ else
+#endif
*p++ = (Py_UNICODE)value;
}
else if (x == Py_None) {
More information about the Python-bugs-list
mailing list