[Python-checkins] cpython: Issue #25267: The UTF-8 encoder is now up to 75 times as fast for error
victor.stinner
python-checkins at python.org
Thu Oct 1 17:20:08 EDT 2015
https://hg.python.org/cpython/rev/2b5357b38366
changeset: 98469:2b5357b38366
user: Victor Stinner <victor.stinner at gmail.com>
date: Thu Oct 01 21:54:51 2015 +0200
summary:
Issue #25267: The UTF-8 encoder is now up to 75 times as fast for error
handlers: ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``.
Patch co-written with Serhiy Storchaka.
files:
Doc/whatsnew/3.6.rst | 3 +
Lib/test/test_codecs.py | 37 ++++-
Misc/NEWS | 4 +
Objects/stringlib/codecs.h | 149 ++++++++++++++++--------
Objects/unicodeobject.c | 7 +-
5 files changed, 136 insertions(+), 64 deletions(-)
diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst
--- a/Doc/whatsnew/3.6.rst
+++ b/Doc/whatsnew/3.6.rst
@@ -120,6 +120,9 @@
* The ASCII and the Latin1 encoders are now up to 3 times as fast for the error
error ``surrogateescape``.
+* The UTF-8 encoder is now up to 75 times as fast for error handlers:
+ ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``.
+
Build and C API Changes
=======================
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -361,6 +361,12 @@
self.assertEqual("[\uDC80]".encode(self.encoding, "replace"),
"[?]".encode(self.encoding))
+ # sequential surrogate characters
+ self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "ignore"),
+ "[]".encode(self.encoding))
+ self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "replace"),
+ "[??]".encode(self.encoding))
+
bom = "".encode(self.encoding)
for before, after in [("\U00010fff", "A"), ("[", "]"),
("A", "\U00010fff")]:
@@ -753,6 +759,7 @@
encoding = "utf-8"
ill_formed_sequence = b"\xed\xb2\x80"
ill_formed_sequence_replace = "\ufffd" * 3
+ BOM = b''
def test_partial(self):
self.check_partial(
@@ -785,23 +792,32 @@
super().test_lone_surrogates()
# not sure if this is making sense for
# UTF-16 and UTF-32
- self.assertEqual("[\uDC80]".encode('utf-8', "surrogateescape"),
- b'[\x80]')
+ self.assertEqual("[\uDC80]".encode(self.encoding, "surrogateescape"),
+ self.BOM + b'[\x80]')
+
+ with self.assertRaises(UnicodeEncodeError) as cm:
+ "[\uDC80\uD800\uDFFF]".encode(self.encoding, "surrogateescape")
+ exc = cm.exception
+ self.assertEqual(exc.object[exc.start:exc.end], '\uD800\uDFFF')
def test_surrogatepass_handler(self):
- self.assertEqual("abc\ud800def".encode("utf-8", "surrogatepass"),
- b"abc\xed\xa0\x80def")
- self.assertEqual(b"abc\xed\xa0\x80def".decode("utf-8", "surrogatepass"),
+ self.assertEqual("abc\ud800def".encode(self.encoding, "surrogatepass"),
+ self.BOM + b"abc\xed\xa0\x80def")
+ self.assertEqual("\U00010fff\uD800".encode(self.encoding, "surrogatepass"),
+ self.BOM + b"\xf0\x90\xbf\xbf\xed\xa0\x80")
+ self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "surrogatepass"),
+ self.BOM + b'[\xed\xa0\x80\xed\xb2\x80]')
+
+ self.assertEqual(b"abc\xed\xa0\x80def".decode(self.encoding, "surrogatepass"),
"abc\ud800def")
- self.assertEqual("\U00010fff\uD800".encode("utf-8", "surrogatepass"),
- b"\xf0\x90\xbf\xbf\xed\xa0\x80")
- self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode("utf-8", "surrogatepass"),
+ self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode(self.encoding, "surrogatepass"),
"\U00010fff\uD800")
+
self.assertTrue(codecs.lookup_error("surrogatepass"))
with self.assertRaises(UnicodeDecodeError):
- b"abc\xed\xa0".decode("utf-8", "surrogatepass")
+ b"abc\xed\xa0".decode(self.encoding, "surrogatepass")
with self.assertRaises(UnicodeDecodeError):
- b"abc\xed\xa0z".decode("utf-8", "surrogatepass")
+ b"abc\xed\xa0z".decode(self.encoding, "surrogatepass")
@unittest.skipUnless(sys.platform == 'win32',
@@ -1008,6 +1024,7 @@
class UTF8SigTest(UTF8Test, unittest.TestCase):
encoding = "utf-8-sig"
+ BOM = codecs.BOM_UTF8
def test_partial(self):
self.check_partial(
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,10 @@
Core and Builtins
-----------------
+- Issue #25267: The UTF-8 encoder is now up to 75 times as fast for error
+ handlers: ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``.
+ Patch co-written with Serhiy Storchaka.
+
- Issue #25280: Import trace messages emitted in verbose (-v) mode are no
longer formatted twice.
diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h
--- a/Objects/stringlib/codecs.h
+++ b/Objects/stringlib/codecs.h
@@ -268,9 +268,10 @@
Py_ssize_t nallocated; /* number of result bytes allocated */
Py_ssize_t nneeded; /* number of result bytes needed */
#if STRINGLIB_SIZEOF_CHAR > 1
- PyObject *errorHandler = NULL;
+ PyObject *error_handler_obj = NULL;
PyObject *exc = NULL;
PyObject *rep = NULL;
+ _Py_error_handler error_handler = _Py_ERROR_UNKNOWN;
#endif
#if STRINGLIB_SIZEOF_CHAR == 1
const Py_ssize_t max_char_size = 2;
@@ -326,72 +327,116 @@
}
#if STRINGLIB_SIZEOF_CHAR > 1
else if (Py_UNICODE_IS_SURROGATE(ch)) {
- Py_ssize_t newpos;
- Py_ssize_t repsize, k, startpos;
+ Py_ssize_t startpos, endpos, newpos;
+ Py_ssize_t repsize, k;
+ if (error_handler == _Py_ERROR_UNKNOWN)
+ error_handler = get_error_handler(errors);
+
startpos = i-1;
- rep = unicode_encode_call_errorhandler(
- errors, &errorHandler, "utf-8", "surrogates not allowed",
- unicode, &exc, startpos, startpos+1, &newpos);
- if (!rep)
- goto error;
+ endpos = startpos+1;
- if (PyBytes_Check(rep))
- repsize = PyBytes_GET_SIZE(rep);
- else
- repsize = PyUnicode_GET_LENGTH(rep);
+ while ((endpos < size) && Py_UNICODE_IS_SURROGATE(data[endpos]))
+ endpos++;
- if (repsize > max_char_size) {
- Py_ssize_t offset;
+ switch (error_handler)
+ {
+ case _Py_ERROR_REPLACE:
+ memset(p, '?', endpos - startpos);
+ p += (endpos - startpos);
+ /* fall through the ignore handler */
+ case _Py_ERROR_IGNORE:
+ i += (endpos - startpos - 1);
+ break;
- if (result == NULL)
- offset = p - stackbuf;
+
+ case _Py_ERROR_SURROGATEPASS:
+ for (k=startpos; k<endpos; k++) {
+ ch = data[k];
+ *p++ = (char)(0xe0 | (ch >> 12));
+ *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
+ *p++ = (char)(0x80 | (ch & 0x3f));
+ }
+ i += (endpos - startpos - 1);
+ break;
+
+ case _Py_ERROR_SURROGATEESCAPE:
+ for (k=startpos; k<endpos; k++) {
+ ch = data[k];
+ if (!(0xDC80 <= ch && ch <= 0xDCFF))
+ break;
+ *p++ = (char)(ch & 0xff);
+ }
+ if (k >= endpos) {
+ i += (endpos - startpos - 1);
+ break;
+ }
+ startpos = k;
+ assert(startpos < endpos);
+ /* fall through the default handler */
+
+ default:
+ rep = unicode_encode_call_errorhandler(
+ errors, &error_handler_obj, "utf-8", "surrogates not allowed",
+ unicode, &exc, startpos, endpos, &newpos);
+ if (!rep)
+ goto error;
+
+ if (PyBytes_Check(rep))
+ repsize = PyBytes_GET_SIZE(rep);
else
- offset = p - PyBytes_AS_STRING(result);
+ repsize = PyUnicode_GET_LENGTH(rep);
- if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) {
- /* integer overflow */
- PyErr_NoMemory();
- goto error;
+ if (repsize > max_char_size) {
+ Py_ssize_t offset;
+
+ if (result == NULL)
+ offset = p - stackbuf;
+ else
+ offset = p - PyBytes_AS_STRING(result);
+
+ if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) {
+ /* integer overflow */
+ PyErr_NoMemory();
+ goto error;
+ }
+ nallocated += repsize - max_char_size;
+ if (result != NULL) {
+ if (_PyBytes_Resize(&result, nallocated) < 0)
+ goto error;
+ } else {
+ result = PyBytes_FromStringAndSize(NULL, nallocated);
+ if (result == NULL)
+ goto error;
+ Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset);
+ }
+ p = PyBytes_AS_STRING(result) + offset;
}
- nallocated += repsize - max_char_size;
- if (result != NULL) {
- if (_PyBytes_Resize(&result, nallocated) < 0)
+
+ if (PyBytes_Check(rep)) {
+ memcpy(p, PyBytes_AS_STRING(rep), repsize);
+ p += repsize;
+ }
+ else {
+ /* rep is unicode */
+ if (PyUnicode_READY(rep) < 0)
goto error;
- } else {
- result = PyBytes_FromStringAndSize(NULL, nallocated);
- if (result == NULL)
- goto error;
- Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset);
- }
- p = PyBytes_AS_STRING(result) + offset;
- }
- if (PyBytes_Check(rep)) {
- char *prep = PyBytes_AS_STRING(rep);
- for(k = repsize; k > 0; k--)
- *p++ = *prep++;
- } else /* rep is unicode */ {
- enum PyUnicode_Kind repkind;
- void *repdata;
-
- if (PyUnicode_READY(rep) < 0)
- goto error;
- repkind = PyUnicode_KIND(rep);
- repdata = PyUnicode_DATA(rep);
-
- for(k=0; k<repsize; k++) {
- Py_UCS4 c = PyUnicode_READ(repkind, repdata, k);
- if (0x80 <= c) {
+ if (!PyUnicode_IS_ASCII(rep)) {
raise_encode_exception(&exc, "utf-8",
unicode,
i-1, i,
"surrogates not allowed");
goto error;
}
- *p++ = (char)c;
+
+ assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND);
+ memcpy(p, PyUnicode_DATA(rep), repsize);
+ p += repsize;
}
+ Py_CLEAR(rep);
+
+ i = newpos;
}
- Py_CLEAR(rep);
}
else
#if STRINGLIB_SIZEOF_CHAR > 2
@@ -430,7 +475,7 @@
}
#if STRINGLIB_SIZEOF_CHAR > 1
- Py_XDECREF(errorHandler);
+ Py_XDECREF(error_handler_obj);
Py_XDECREF(exc);
#endif
return result;
@@ -438,7 +483,7 @@
#if STRINGLIB_SIZEOF_CHAR > 1
error:
Py_XDECREF(rep);
- Py_XDECREF(errorHandler);
+ Py_XDECREF(error_handler_obj);
Py_XDECREF(exc);
Py_XDECREF(result);
return NULL;
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -297,6 +297,7 @@
_Py_ERROR_UNKNOWN=0,
_Py_ERROR_STRICT,
_Py_ERROR_SURROGATEESCAPE,
+ _Py_ERROR_SURROGATEPASS,
_Py_ERROR_REPLACE,
_Py_ERROR_IGNORE,
_Py_ERROR_XMLCHARREFREPLACE,
@@ -312,6 +313,8 @@
return _Py_ERROR_STRICT;
if (strcmp(errors, "surrogateescape") == 0)
return _Py_ERROR_SURROGATEESCAPE;
+ if (strcmp(errors, "surrogatepass") == 0)
+ return _Py_ERROR_SURROGATEPASS;
if (strcmp(errors, "ignore") == 0)
return _Py_ERROR_IGNORE;
if (strcmp(errors, "replace") == 0)
@@ -6479,8 +6482,8 @@
goto onError;
case _Py_ERROR_REPLACE:
- while (collstart++ < collend)
- *str++ = '?';
+ memset(str, '?', collend - collstart);
+ str += (collend - collstart);
/* fall through ignore error handler */
case _Py_ERROR_IGNORE:
pos = collend;
--
Repository URL: https://hg.python.org/cpython
More information about the Python-checkins
mailing list