[Python-checkins] bpo-37751: Fix codecs.lookup() normalization (GH-15092)

[Victor Stinner]

https://github.com/python/cpython/commit/20f59fe1f7748ae899aceee4cb560e5e1f528a1f
commit: 20f59fe1f7748ae899aceee4cb560e5e1f528a1f
branch: master
author: Jordon Xu <46997731+qigangxu at users.noreply.github.com>
committer: Victor Stinner <vstinner at redhat.com>
date: 2019-08-21T14:26:20+01:00
summary:

bpo-37751: Fix codecs.lookup() normalization (GH-15092)

Fix codecs.lookup() to normalize the encoding name the same way
than encodings.normalize_encoding(), except that codecs.lookup()
also converts the name to lower case.

files:
A Misc/NEWS.d/next/Core and Builtins/2019-08-20-04-36-37.bpo-37751.CSFzUd.rst
M Python/codecs.c

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-08-20-04-36-37.bpo-37751.CSFzUd.rst b/Misc/NEWS.d/next/Core and Builtins/2019-08-20-04-36-37.bpo-37751.CSFzUd.rst
new file mode 100644
index 000000000000..4da59ff4c1c9
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-08-20-04-36-37.bpo-37751.CSFzUd.rst	
@@ -0,0 +1 @@
+Fix :func:`codecs.lookup` to normalize the encoding name the same way than :func:`encodings.normalize_encoding`, except that :func:`codecs.lookup` also converts the name to lower case.
diff --git a/Python/codecs.c b/Python/codecs.c
index 4f38b33e0b76..08e9b916f201 100644
--- a/Python/codecs.c
+++ b/Python/codecs.c
@@ -49,15 +49,16 @@ int PyCodec_Register(PyObject *search_function)
     return -1;
 }
 
-/* Convert a string to a normalized Python string: all characters are
-   converted to lower case, spaces are replaced with underscores. */
+extern int _Py_normalize_encoding(const char *, char *, size_t);
+
+/* Convert a string to a normalized Python string(decoded from UTF-8): all characters are
+   converted to lower case, spaces and hyphens are replaced with underscores. */
 
 static
 PyObject *normalizestring(const char *string)
 {
-    size_t i;
     size_t len = strlen(string);
-    char *p;
+    char *encoding;
     PyObject *v;
 
     if (len > PY_SSIZE_T_MAX) {
@@ -65,20 +66,19 @@ PyObject *normalizestring(const char *string)
         return NULL;
     }
 
-    p = PyMem_Malloc(len + 1);
-    if (p == NULL)
+    encoding = PyMem_Malloc(len + 1);
+    if (encoding == NULL)
         return PyErr_NoMemory();
-    for (i = 0; i < len; i++) {
-        char ch = string[i];
-        if (ch == ' ')
-            ch = '-';
-        else
-            ch = Py_TOLOWER(Py_CHARMASK(ch));
-        p[i] = ch;
+
+    if (!_Py_normalize_encoding(string, encoding, len + 1))
+    {
+        PyErr_SetString(PyExc_RuntimeError, "_Py_normalize_encoding() failed");
+        PyMem_Free(encoding);
+        return NULL;
     }
-    p[i] = '\0';
-    v = PyUnicode_FromString(p);
-    PyMem_Free(p);
+
+    v = PyUnicode_FromString(encoding);
+    PyMem_Free(encoding);
     return v;
 }