[Python-checkins] [3.9] gh-99612: Fix PyUnicode_DecodeUTF8Stateful() for ASCII-only data (GH-99613) (GH-107224) (#107231)

[ambv]

https://github.com/python/cpython/commit/4a793281956db0e4a1ca5fdc5f3a0e91f331a75d
commit: 4a793281956db0e4a1ca5fdc5f3a0e91f331a75d
branch: 3.9
author: Serhiy Storchaka <storchaka at gmail.com>
committer: ambv <lukasz at langa.pl>
date: 2023-08-22T20:25:15+02:00
summary:

[3.9] gh-99612: Fix PyUnicode_DecodeUTF8Stateful() for ASCII-only data (GH-99613) (GH-107224) (#107231)

Previously *consumed was not set in this case.
(cherry picked from commit f08e52ccb027f6f703302b8c1a82db9fd3934270).
(cherry picked from commit b8b3e6afc0a48c3cbb7c36d2f73e332edcd6058c)

files:
A Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst
M Lib/test/test_unicode.py
M Modules/_testcapimodule.c
M Objects/unicodeobject.c

diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
index 787f841573876..0dd18a637f9cc 100644
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -2870,6 +2870,53 @@ def test_asutf8andsize(self):
         self.assertEqual(unicode_asutf8andsize(nonbmp), (b'\xf4\x8f\xbf\xbf', 4))
         self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize, 'a\ud800b\udfffc')
 
+    @support.cpython_only
+    def test_decodeutf8(self):
+        """Test PyUnicode_DecodeUTF8()"""
+        import _testcapi
+        decodeutf8 = _testcapi.unicode_decodeutf8
+
+        for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
+            b = s.encode('utf-8')
+            self.assertEqual(decodeutf8(b), s)
+            self.assertEqual(decodeutf8(b, 'strict'), s)
+
+        self.assertRaises(UnicodeDecodeError, decodeutf8, b'\x80')
+        self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xc0')
+        self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xff')
+        self.assertRaises(UnicodeDecodeError, decodeutf8, b'a\xf0\x9f')
+        self.assertEqual(decodeutf8(b'a\xf0\x9f', 'replace'), 'a\ufffd')
+        self.assertEqual(decodeutf8(b'a\xf0\x9fb', 'replace'), 'a\ufffdb')
+
+        self.assertRaises(LookupError, decodeutf8, b'a\x80', 'foo')
+        # TODO: Test PyUnicode_DecodeUTF8() with NULL as data and
+        # negative size.
+
+    @support.cpython_only
+    def test_decodeutf8stateful(self):
+        """Test PyUnicode_DecodeUTF8Stateful()"""
+        import _testcapi
+        decodeutf8stateful = _testcapi.unicode_decodeutf8stateful
+
+        for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
+            b = s.encode('utf-8')
+            self.assertEqual(decodeutf8stateful(b), (s, len(b)))
+            self.assertEqual(decodeutf8stateful(b, 'strict'), (s, len(b)))
+
+        self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\x80')
+        self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xc0')
+        self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xff')
+        self.assertEqual(decodeutf8stateful(b'a\xf0\x9f'), ('a', 1))
+        self.assertEqual(decodeutf8stateful(b'a\xf0\x9f', 'replace'), ('a', 1))
+        self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'a\xf0\x9fb')
+        self.assertEqual(decodeutf8stateful(b'a\xf0\x9fb', 'replace'), ('a\ufffdb', 4))
+
+        self.assertRaises(LookupError, decodeutf8stateful, b'a\x80', 'foo')
+        # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as data and
+        # negative size.
+        # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as the address of
+        # "consumed".
+
     # Test PyUnicode_FindChar()
     @support.cpython_only
     def test_findchar(self):
diff --git a/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst b/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst
new file mode 100644
index 0000000000000..40e3c8db5403c
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst	
@@ -0,0 +1,2 @@
+Fix :c:func:`PyUnicode_DecodeUTF8Stateful` for ASCII-only data:
+``*consumed`` was not set.
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index 361ce7b55a26d..8380ad9a34cec 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -1987,6 +1987,40 @@ unicode_asutf8andsize(PyObject *self, PyObject *args)
     return Py_BuildValue("(Nn)", result, utf8_len);
 }
 
+/* Test PyUnicode_DecodeUTF8() */
+static PyObject *
+unicode_decodeutf8(PyObject *self, PyObject *args)
+{
+    const char *data;
+    Py_ssize_t size;
+    const char *errors = NULL;
+
+    if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors))
+        return NULL;
+
+    return PyUnicode_DecodeUTF8(data, size, errors);
+}
+
+/* Test PyUnicode_DecodeUTF8Stateful() */
+static PyObject *
+unicode_decodeutf8stateful(PyObject *self, PyObject *args)
+{
+    const char *data;
+    Py_ssize_t size;
+    const char *errors = NULL;
+    Py_ssize_t consumed = 123456789;
+    PyObject *result;
+
+    if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors))
+        return NULL;
+
+    result = PyUnicode_DecodeUTF8Stateful(data, size, errors, &consumed);
+    if (!result) {
+        return NULL;
+    }
+    return Py_BuildValue("(Nn)", result, consumed);
+}
+
 static PyObject *
 unicode_findchar(PyObject *self, PyObject *args)
 {
@@ -5502,7 +5536,8 @@ static PyMethodDef TestMethods[] = {
     {"unicode_asucs4",          unicode_asucs4,                  METH_VARARGS},
     {"unicode_asutf8",          unicode_asutf8,                  METH_VARARGS},
     {"unicode_asutf8andsize",   unicode_asutf8andsize,           METH_VARARGS},
-    {"unicode_findchar",        unicode_findchar,                METH_VARARGS},
+    {"unicode_decodeutf8",       unicode_decodeutf8,             METH_VARARGS},
+    {"unicode_decodeutf8stateful",unicode_decodeutf8stateful,    METH_VARARGS},    {"unicode_findchar",        unicode_findchar,                METH_VARARGS},
     {"unicode_copycharacters",  unicode_copycharacters,          METH_VARARGS},
     {"unicode_encodedecimal",   unicode_encodedecimal,           METH_VARARGS},
     {"unicode_transformdecimaltoascii", unicode_transformdecimaltoascii, METH_VARARGS},
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index e5e9ba20301c0..bd08b198781ae 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -5052,6 +5052,9 @@ unicode_decode_utf8(const char *s, Py_ssize_t size,
     }
     s += ascii_decode(s, end, PyUnicode_1BYTE_DATA(u));
     if (s == end) {
+        if (consumed) {
+            *consumed = size;
+        }
         return u;
     }