[Python-checkins] cpython (merge 3.3 -> default): Issue #14850: Now a chamap decoder treates U+FFFE as "undefined mapping"
serhiy.storchaka
python-checkins at python.org
Tue Jan 15 14:40:04 CET 2013
http://hg.python.org/cpython/rev/03e22cc9407a
changeset: 81520:03e22cc9407a
parent: 81512:1f66fc397c8d
parent: 81519:6ac4f1609847
user: Serhiy Storchaka <storchaka at gmail.com>
date: Tue Jan 15 15:30:04 2013 +0200
summary:
Issue #14850: Now a chamap decoder treates U+FFFE as "undefined mapping"
in any mapping, not only in an unicode string.
files:
Lib/test/test_codecs.py | 46 +++++++++++++++++++++++++
Misc/NEWS | 3 +
Objects/unicodeobject.c | 52 ++++++++++++++++++----------
3 files changed, 82 insertions(+), 19 deletions(-)
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -1737,6 +1737,10 @@
codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab"
)
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab\ufffe"
+ )
+
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "replace", "ab"),
("ab\ufffd", 3)
@@ -1793,6 +1797,17 @@
{0: 'a', 1: 'b'}
)
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict",
+ {0: 'a', 1: 'b', 2: None}
+ )
+
+ # Issue #14850
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict",
+ {0: 'a', 1: 'b', 2: '\ufffe'}
+ )
+
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "replace",
{0: 'a', 1: 'b'}),
@@ -1805,6 +1820,13 @@
("ab\ufffd", 3)
)
+ # Issue #14850
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "replace",
+ {0: 'a', 1: 'b', 2: '\ufffe'}),
+ ("ab\ufffd", 3)
+ )
+
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "ignore",
{0: 'a', 1: 'b'}),
@@ -1817,6 +1839,13 @@
("ab", 3)
)
+ # Issue #14850
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "ignore",
+ {0: 'a', 1: 'b', 2: '\ufffe'}),
+ ("ab", 3)
+ )
+
allbytes = bytes(range(256))
self.assertEqual(
codecs.charmap_decode(allbytes, "ignore", {}),
@@ -1857,6 +1886,11 @@
{0: a, 1: b},
)
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict",
+ {0: a, 1: b, 2: 0xFFFE},
+ )
+
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "replace",
{0: a, 1: b}),
@@ -1864,11 +1898,23 @@
)
self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "replace",
+ {0: a, 1: b, 2: 0xFFFE}),
+ ("ab\ufffd", 3)
+ )
+
+ self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "ignore",
{0: a, 1: b}),
("ab", 3)
)
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "ignore",
+ {0: a, 1: b, 2: 0xFFFE}),
+ ("ab", 3)
+ )
+
class WithStmtTest(unittest.TestCase):
def test_encodedfile(self):
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
Core and Builtins
-----------------
+- Issue #14850: Now a chamap decoder treates U+FFFE as "undefined mapping"
+ in any mapping, not only in a string.
+
- Issue #16730: importlib.machinery.FileFinder now no longers raises an
exception when trying to populate its cache and it finds out the directory is
unreadable or has turned into a file. Reported and diagnosed by
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -7393,15 +7393,18 @@
if (PyErr_ExceptionMatches(PyExc_LookupError)) {
/* No mapping found means: mapping is undefined. */
PyErr_Clear();
- x = Py_None;
- Py_INCREF(x);
+ goto Undefined;
} else
goto onError;
}
/* Apply mapping */
+ if (x == Py_None)
+ goto Undefined;
if (PyLong_Check(x)) {
long value = PyLong_AS_LONG(x);
+ if (value == 0xFFFE)
+ goto Undefined;
if (value < 0 || value > MAX_UNICODE) {
PyErr_Format(PyExc_TypeError,
"character mapping must be in range(0x%lx)",
@@ -7415,25 +7418,23 @@
PyUnicode_WRITE(writer.kind, writer.data, writer.pos, value);
writer.pos++;
}
- else if (x == Py_None) {
- /* undefined mapping */
- startinpos = s-starts;
- endinpos = startinpos+1;
- if (unicode_decode_call_errorhandler_writer(
- errors, &errorHandler,
- "charmap", "character maps to <undefined>",
- &starts, &e, &startinpos, &endinpos, &exc, &s,
- &writer)) {
- Py_DECREF(x);
+ else if (PyUnicode_Check(x)) {
+ if (PyUnicode_READY(x) == -1)
goto onError;
+ if (PyUnicode_GET_LENGTH(x) == 1) {
+ Py_UCS4 value = PyUnicode_READ_CHAR(x, 0);
+ if (value == 0xFFFE)
+ goto Undefined;
+ if (_PyUnicodeWriter_Prepare(&writer, 1, value) == -1)
+ goto onError;
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, value);
+ writer.pos++;
}
- Py_DECREF(x);
- continue;
- }
- else if (PyUnicode_Check(x)) {
- writer.overallocate = 1;
- if (_PyUnicodeWriter_WriteStr(&writer, x) == -1)
- goto onError;
+ else {
+ writer.overallocate = 1;
+ if (_PyUnicodeWriter_WriteStr(&writer, x) == -1)
+ goto onError;
+ }
}
else {
/* wrong return value */
@@ -7444,6 +7445,19 @@
}
Py_DECREF(x);
++s;
+ continue;
+Undefined:
+ /* undefined mapping */
+ Py_XDECREF(x);
+ startinpos = s-starts;
+ endinpos = startinpos+1;
+ if (unicode_decode_call_errorhandler_writer(
+ errors, &errorHandler,
+ "charmap", "character maps to <undefined>",
+ &starts, &e, &startinpos, &endinpos, &exc, &s,
+ &writer)) {
+ goto onError;
+ }
}
}
Py_XDECREF(errorHandler);
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list