Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Issue #25267: The UTF-8 encoder is now up to 75 times as fast for error 

handlers: ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``.
Patch co-written with Serhiy Storchaka.
This commit is contained in:
Victor Stinner 2015-10-01 21:54:51 +02:00
parent 29a1445136
commit 01ada3996b
5 changed files with 134 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Doc/whatsnew/3.6.rst
View File

@ -120,6 +120,9 @@ Optimizations
* The ASCII and the Latin1 encoders are now up to 3 times as fast for the error
error ``surrogateescape``.
* The UTF-8 encoder is now up to 75 times as fast for error handlers:
``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``.
Build and C API Changes
=======================

37
Lib/test/test_codecs.py
View File

@ -361,6 +361,12 @@ class ReadTest(MixInCheckStateHandling):
self.assertEqual("[\uDC80]".encode(self.encoding, "replace"),
"[?]".encode(self.encoding))
# sequential surrogate characters
self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "ignore"),
"[]".encode(self.encoding))
self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "replace"),
"[??]".encode(self.encoding))
bom = "".encode(self.encoding)
for before, after in [("\U00010fff", "A"), ("[", "]"),
("A", "\U00010fff")]:
@ -753,6 +759,7 @@ class UTF8Test(ReadTest, unittest.TestCase):
encoding = "utf-8"
ill_formed_sequence = b"\xed\xb2\x80"
ill_formed_sequence_replace = "\ufffd" * 3
BOM = b''
def test_partial(self):
self.check_partial(
@ -785,23 +792,32 @@ class UTF8Test(ReadTest, unittest.TestCase):
super().test_lone_surrogates()
# not sure if this is making sense for
# UTF-16 and UTF-32
self.assertEqual("[\uDC80]".encode('utf-8', "surrogateescape"),
b'[\x80]')
self.assertEqual("[\uDC80]".encode(self.encoding, "surrogateescape"),
self.BOM + b'[\x80]')
with self.assertRaises(UnicodeEncodeError) as cm:
"[\uDC80\uD800\uDFFF]".encode(self.encoding, "surrogateescape")
exc = cm.exception
self.assertEqual(exc.object[exc.start:exc.end], '\uD800\uDFFF')
def test_surrogatepass_handler(self):
self.assertEqual("abc\ud800def".encode("utf-8", "surrogatepass"),
b"abc\xed\xa0\x80def")
self.assertEqual(b"abc\xed\xa0\x80def".decode("utf-8", "surrogatepass"),
self.assertEqual("abc\ud800def".encode(self.encoding, "surrogatepass"),
self.BOM + b"abc\xed\xa0\x80def")
self.assertEqual("\U00010fff\uD800".encode(self.encoding, "surrogatepass"),
self.BOM + b"\xf0\x90\xbf\xbf\xed\xa0\x80")
self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "surrogatepass"),
self.BOM + b'[\xed\xa0\x80\xed\xb2\x80]')
self.assertEqual(b"abc\xed\xa0\x80def".decode(self.encoding, "surrogatepass"),
"abc\ud800def")
self.assertEqual("\U00010fff\uD800".encode("utf-8", "surrogatepass"),
b"\xf0\x90\xbf\xbf\xed\xa0\x80")
self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode("utf-8", "surrogatepass"),
self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode(self.encoding, "surrogatepass"),
"\U00010fff\uD800")
self.assertTrue(codecs.lookup_error("surrogatepass"))
with self.assertRaises(UnicodeDecodeError):
b"abc\xed\xa0".decode("utf-8", "surrogatepass")
b"abc\xed\xa0".decode(self.encoding, "surrogatepass")
with self.assertRaises(UnicodeDecodeError):
b"abc\xed\xa0z".decode("utf-8", "surrogatepass")
b"abc\xed\xa0z".decode(self.encoding, "surrogatepass")
@unittest.skipUnless(sys.platform == 'win32',
@ -1008,6 +1024,7 @@ class ReadBufferTest(unittest.TestCase):
class UTF8SigTest(UTF8Test, unittest.TestCase):
encoding = "utf-8-sig"
BOM = codecs.BOM_UTF8
def test_partial(self):
self.check_partial(

4
Misc/NEWS
View File

@ -10,6 +10,10 @@ Release date: XXXX-XX-XX
Core and Builtins
-----------------
- Issue #25267: The UTF-8 encoder is now up to 75 times as fast for error
handlers: ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``.
Patch co-written with Serhiy Storchaka.
- Issue #25280: Import trace messages emitted in verbose (-v) mode are no
longer formatted twice.

145
Objects/stringlib/codecs.h
View File

@ -268,9 +268,10 @@ STRINGLIB(utf8_encoder)(PyObject *unicode,
Py_ssize_t nallocated; /* number of result bytes allocated */
Py_ssize_t nneeded; /* number of result bytes needed */
#if STRINGLIB_SIZEOF_CHAR > 1
PyObject *errorHandler = NULL;
PyObject *error_handler_obj = NULL;
PyObject *exc = NULL;
PyObject *rep = NULL;
_Py_error_handler error_handler = _Py_ERROR_UNKNOWN;
#endif
#if STRINGLIB_SIZEOF_CHAR == 1
const Py_ssize_t max_char_size = 2;
@ -326,72 +327,116 @@ STRINGLIB(utf8_encoder)(PyObject *unicode,
}
#if STRINGLIB_SIZEOF_CHAR > 1
else if (Py_UNICODE_IS_SURROGATE(ch)) {
Py_ssize_t newpos;
Py_ssize_t repsize, k, startpos;
Py_ssize_t startpos, endpos, newpos;
Py_ssize_t repsize, k;
if (error_handler == _Py_ERROR_UNKNOWN)
error_handler = get_error_handler(errors);
startpos = i-1;
rep = unicode_encode_call_errorhandler(
errors, &errorHandler, "utf-8", "surrogates not allowed",
unicode, &exc, startpos, startpos+1, &newpos);
if (!rep)
goto error;
endpos = startpos+1;
if (PyBytes_Check(rep))
repsize = PyBytes_GET_SIZE(rep);
else
repsize = PyUnicode_GET_LENGTH(rep);
while ((endpos < size) && Py_UNICODE_IS_SURROGATE(data[endpos]))
endpos++;
if (repsize > max_char_size) {
Py_ssize_t offset;
switch (error_handler)
{
case _Py_ERROR_REPLACE:
memset(p, '?', endpos - startpos);
p += (endpos - startpos);
/* fall through the ignore handler */
case _Py_ERROR_IGNORE:
i += (endpos - startpos - 1);
break;
if (result == NULL)
offset = p - stackbuf;
case _Py_ERROR_SURROGATEPASS:
for (k=startpos; k<endpos; k++) {
ch = data[k];
*p++ = (char)(0xe0 | (ch >> 12));
*p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
*p++ = (char)(0x80 | (ch & 0x3f));
}
i += (endpos - startpos - 1);
break;
case _Py_ERROR_SURROGATEESCAPE:
for (k=startpos; k<endpos; k++) {
ch = data[k];
if (!(0xDC80 <= ch && ch <= 0xDCFF))
break;
*p++ = (char)(ch & 0xff);
}
if (k >= endpos) {
i += (endpos - startpos - 1);
break;
}
startpos = k;
assert(startpos < endpos);
/* fall through the default handler */
default:
rep = unicode_encode_call_errorhandler(
errors, &error_handler_obj, "utf-8", "surrogates not allowed",
unicode, &exc, startpos, endpos, &newpos);
if (!rep)
goto error;
if (PyBytes_Check(rep))
repsize = PyBytes_GET_SIZE(rep);
else
offset = p - PyBytes_AS_STRING(result);
repsize = PyUnicode_GET_LENGTH(rep);
if (repsize > max_char_size) {
Py_ssize_t offset;
if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) {
/* integer overflow */
PyErr_NoMemory();
goto error;
}
nallocated += repsize - max_char_size;
if (result != NULL) {
if (_PyBytes_Resize(&result, nallocated) < 0)
goto error;
} else {
result = PyBytes_FromStringAndSize(NULL, nallocated);
if (result == NULL)
offset = p - stackbuf;
else
offset = p - PyBytes_AS_STRING(result);
if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) {
/* integer overflow */
PyErr_NoMemory();
goto error;
Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset);
}
nallocated += repsize - max_char_size;
if (result != NULL) {
if (_PyBytes_Resize(&result, nallocated) < 0)
goto error;
} else {
result = PyBytes_FromStringAndSize(NULL, nallocated);
if (result == NULL)
goto error;
Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset);
}
p = PyBytes_AS_STRING(result) + offset;
}
p = PyBytes_AS_STRING(result) + offset;
}
if (PyBytes_Check(rep)) {
char *prep = PyBytes_AS_STRING(rep);
for(k = repsize; k > 0; k--)
*p++ = *prep++;
} else /* rep is unicode */ {
enum PyUnicode_Kind repkind;
void *repdata;
if (PyBytes_Check(rep)) {
memcpy(p, PyBytes_AS_STRING(rep), repsize);
p += repsize;
}
else {
/* rep is unicode */
if (PyUnicode_READY(rep) < 0)
goto error;
if (PyUnicode_READY(rep) < 0)
goto error;
repkind = PyUnicode_KIND(rep);
repdata = PyUnicode_DATA(rep);
for(k=0; k<repsize; k++) {
Py_UCS4 c = PyUnicode_READ(repkind, repdata, k);
if (0x80 <= c) {
if (!PyUnicode_IS_ASCII(rep)) {
raise_encode_exception(&exc, "utf-8",
unicode,
i-1, i,
"surrogates not allowed");
goto error;
}
*p++ = (char)c;
assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND);
memcpy(p, PyUnicode_DATA(rep), repsize);
p += repsize;
}
Py_CLEAR(rep);
i = newpos;
}
Py_CLEAR(rep);
}
else
#if STRINGLIB_SIZEOF_CHAR > 2
@ -430,7 +475,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicode,
}
#if STRINGLIB_SIZEOF_CHAR > 1
Py_XDECREF(errorHandler);
Py_XDECREF(error_handler_obj);
Py_XDECREF(exc);
#endif
return result;
@ -438,7 +483,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicode,
#if STRINGLIB_SIZEOF_CHAR > 1
error:
Py_XDECREF(rep);
Py_XDECREF(errorHandler);
Py_XDECREF(error_handler_obj);
Py_XDECREF(exc);
Py_XDECREF(result);
return NULL;

7
Objects/unicodeobject.c
View File

@ -297,6 +297,7 @@ typedef enum {
_Py_ERROR_UNKNOWN=0,
_Py_ERROR_STRICT,
_Py_ERROR_SURROGATEESCAPE,
_Py_ERROR_SURROGATEPASS,
_Py_ERROR_REPLACE,
_Py_ERROR_IGNORE,
_Py_ERROR_XMLCHARREFREPLACE,
@ -312,6 +313,8 @@ get_error_handler(const char *errors)
return _Py_ERROR_STRICT;
if (strcmp(errors, "surrogateescape") == 0)
return _Py_ERROR_SURROGATEESCAPE;
if (strcmp(errors, "surrogatepass") == 0)
return _Py_ERROR_SURROGATEPASS;
if (strcmp(errors, "ignore") == 0)
return _Py_ERROR_IGNORE;
if (strcmp(errors, "replace") == 0)
@ -6479,8 +6482,8 @@ unicode_encode_ucs1(PyObject *unicode,
goto onError;
case _Py_ERROR_REPLACE:
while (collstart++ < collend)
*str++ = '?';
memset(str, '?', collend - collstart);
str += (collend - collstart);
/* fall through ignore error handler */
case _Py_ERROR_IGNORE:
pos = collend;