Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Call directly PyUnicode_DecodeUTF8Stateful() instead of PyUnicode_DecodeUTF8() 

* Remove micro-optimization from PyUnicode_FromStringAndSize():
   PyUnicode_DecodeUTF8Stateful() has already these optimizations (for size=0
   and one ascii char).
 * Rename utf8_max_char_size_and_char_count() to utf8_scanner(), and remove an
   useless variable
This commit is contained in:
Victor Stinner 2011-12-11 21:53:09 +01:00
parent 382955ff4e
commit a1d12bb119
1 changed files with 14 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
Objects/unicodeobject.c
View File

@ -1717,28 +1717,10 @@ PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size)
"Negative size passed to PyUnicode_FromStringAndSize");
return NULL;
}
/* If the Unicode data is known at construction time, we can apply
some optimizations which share commonly used objects.
Also, this means the input must be UTF-8, so fall back to the
UTF-8 decoder at the end. */
if (u != NULL) {
/* Optimization for empty strings */
if (size == 0 && unicode_empty != NULL) {
Py_INCREF(unicode_empty);
return unicode_empty;
}
/* Single characters are shared when using this constructor.
Restrict to ASCII, since the input must be UTF-8. */
if (size == 1 && (unsigned char)*u < 128)
return get_latin1_char((unsigned char)*u);
return PyUnicode_DecodeUTF8(u, size, NULL);
}
return (PyObject *)_PyUnicode_New(size);
if (u != NULL)
return PyUnicode_DecodeUTF8Stateful(u, size, NULL, NULL);
else
return (PyObject *)_PyUnicode_New(size);
}
PyObject *
@ -1749,15 +1731,16 @@ PyUnicode_FromString(const char *u)
PyErr_SetString(PyExc_OverflowError, "input too long");
return NULL;
}
return PyUnicode_FromStringAndSize(u, size);
return PyUnicode_DecodeUTF8Stateful(u, (Py_ssize_t)size, NULL, NULL);
}
PyObject *
_PyUnicode_FromId(_Py_Identifier *id)
{
if (!id->object) {
id->object = PyUnicode_FromString(id->string);
id->object = PyUnicode_DecodeUTF8Stateful(id->string,
strlen(id->string),
NULL, NULL);
if (!id->object)
return NULL;
PyUnicode_InternInPlace(&id->object);
@ -2443,7 +2426,7 @@ PyUnicode_FromFormatV(const char *format, va_list vargs)
{
/* UTF-8 */
const char *s = va_arg(count, const char*);
PyObject *str = PyUnicode_DecodeUTF8(s, strlen(s), "replace");
PyObject *str = PyUnicode_DecodeUTF8Stateful(s, strlen(s), "replace", NULL);
if (!str)
goto fail;
/* since PyUnicode_DecodeUTF8 returns already flexible
@ -2482,7 +2465,7 @@ PyUnicode_FromFormatV(const char *format, va_list vargs)
*callresult++ = NULL;
}
else {
str_obj = PyUnicode_DecodeUTF8(str, strlen(str), "replace");
str_obj = PyUnicode_DecodeUTF8Stateful(str, strlen(str), "replace", NULL);
if (!str_obj)
goto fail;
if (PyUnicode_READY(str_obj)) {
@ -2947,7 +2930,7 @@ PyUnicode_Decode(const char *s,
if (normalize_encoding(encoding, lower, sizeof(lower))) {
if ((strcmp(lower, "utf-8") == 0) ||
(strcmp(lower, "utf8") == 0))
return PyUnicode_DecodeUTF8(s, size, errors);
return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL);
else if ((strcmp(lower, "latin-1") == 0) ||
(strcmp(lower, "latin1") == 0) ||
(strcmp(lower, "iso-8859-1") == 0))
@ -3260,7 +3243,7 @@ PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size)
#ifdef HAVE_MBCS
return PyUnicode_DecodeMBCS(s, size, NULL);
#elif defined(__APPLE__)
return PyUnicode_DecodeUTF8(s, size, "surrogateescape");
return PyUnicode_DecodeUTF8Stateful(s, size, "surrogateescape", NULL);
#else
PyInterpreterState *interp = PyThreadState_GET()->interp;
/* Bootstrap check: if the filesystem codec is implemented in Python, we
@ -4240,11 +4223,9 @@ PyUnicode_DecodeUTF8(const char *s,
PyUnicode_DecodeUTF8Stateful.
*/
static Py_UCS4
utf8_max_char_size_and_char_count(const char *s, Py_ssize_t string_size,
Py_ssize_t *unicode_size)
utf8_scanner(const unsigned char *p, Py_ssize_t string_size, Py_ssize_t *unicode_size)
{
Py_ssize_t char_count = 0;
const unsigned char *p = (const unsigned char *)s;
const unsigned char *end = p + string_size;
const unsigned char *aligned_end = (const unsigned char *) ((size_t) end & ~LONG_PTR_MASK);
@ -4563,7 +4544,7 @@ PyUnicode_DecodeUTF8Stateful(const char *s,
return unicode_empty;
}
maxchar = utf8_max_char_size_and_char_count(s, size, &unicode_size);
maxchar = utf8_scanner((const unsigned char *)s, size, &unicode_size);
/* When the string is ASCII only, just use memcpy and return.
unicode_size may be != size if there is an incomplete UTF-8