Optimize built-in unicode codecs by avoiding unnecessary copying.
The approach used is similiar to what is currently used in the version of unicodeobject.c in Python 2.x. The only difference is we use _PyBytes_Resize instead of _PyString_Resize.
This commit is contained in:
Alexandre Vassalotti
parent
9cb6f7f7a5
commit
44531cb2db
|
|
@ -1873,7 +1873,7 @@ PyObject *PyUnicode_EncodeUTF7(const Py_UNICODE *s,
|
|||
int encodeWhiteSpace,
|
||||
const char *errors)
|
||||
{
|
||||
PyObject *v, *result;
|
||||
PyObject *v;
|
||||
/* It might be possible to tighten this worst case */
|
||||
Py_ssize_t cbAllocated = 5 * size;
|
||||
int inShift = 0;
|
||||
|
|
@ -1889,11 +1889,11 @@ PyObject *PyUnicode_EncodeUTF7(const Py_UNICODE *s,
|
|||
if (cbAllocated / 5 != size)
|
||||
return PyErr_NoMemory();
|
||||
|
||||
v = PyByteArray_FromStringAndSize(NULL, cbAllocated);
|
||||
v = PyBytes_FromStringAndSize(NULL, cbAllocated);
|
||||
if (v == NULL)
|
||||
return NULL;
|
||||
|
||||
start = out = PyByteArray_AS_STRING(v);
|
||||
start = out = PyBytes_AS_STRING(v);
|
||||
for (;i < size; ++i) {
|
||||
Py_UNICODE ch = s[i];
|
||||
|
||||
|
|
@ -1958,10 +1958,9 @@ PyObject *PyUnicode_EncodeUTF7(const Py_UNICODE *s,
|
|||
*out++= B64(charsleft << (6-bitsleft) );
|
||||
*out++ = '-';
|
||||
}
|
||||
|
||||
result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(v), out - start);
|
||||
Py_DECREF(v);
|
||||
return result;
|
||||
if (_PyBytes_Resize(&v, out - start) < 0)
|
||||
return NULL;
|
||||
return v;
|
||||
}
|
||||
|
||||
#undef SPECIAL
|
||||
|
|
@ -2479,7 +2478,7 @@ PyUnicode_EncodeUTF32(const Py_UNICODE *s,
|
|||
const char *errors,
|
||||
int byteorder)
|
||||
{
|
||||
PyObject *v, *result;
|
||||
PyObject *v;
|
||||
unsigned char *p;
|
||||
Py_ssize_t nsize, bytesize;
|
||||
#ifndef Py_UNICODE_WIDE
|
||||
|
|
@ -2515,11 +2514,11 @@ PyUnicode_EncodeUTF32(const Py_UNICODE *s,
|
|||
bytesize = nsize * 4;
|
||||
if (bytesize / 4 != nsize)
|
||||
return PyErr_NoMemory();
|
||||
v = PyByteArray_FromStringAndSize(NULL, bytesize);
|
||||
v = PyBytes_FromStringAndSize(NULL, bytesize);
|
||||
if (v == NULL)
|
||||
return NULL;
|
||||
|
||||
p = (unsigned char *)PyByteArray_AS_STRING(v);
|
||||
p = (unsigned char *)PyBytes_AS_STRING(v);
|
||||
if (byteorder == 0)
|
||||
STORECHAR(0xFEFF);
|
||||
if (size == 0)
|
||||
|
|
@ -2556,9 +2555,7 @@ PyUnicode_EncodeUTF32(const Py_UNICODE *s,
|
|||
}
|
||||
|
||||
done:
|
||||
result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(v), Py_SIZE(v));
|
||||
Py_DECREF(v);
|
||||
return result;
|
||||
return v;
|
||||
#undef STORECHAR
|
||||
}
|
||||
|
||||
|
|
@ -2757,7 +2754,7 @@ PyUnicode_EncodeUTF16(const Py_UNICODE *s,
|
|||
const char *errors,
|
||||
int byteorder)
|
||||
{
|
||||
PyObject *v, *result;
|
||||
PyObject *v;
|
||||
unsigned char *p;
|
||||
Py_ssize_t nsize, bytesize;
|
||||
#ifdef Py_UNICODE_WIDE
|
||||
|
|
@ -2792,11 +2789,11 @@ PyUnicode_EncodeUTF16(const Py_UNICODE *s,
|
|||
bytesize = nsize * 2;
|
||||
if (bytesize / 2 != nsize)
|
||||
return PyErr_NoMemory();
|
||||
v = PyByteArray_FromStringAndSize(NULL, bytesize);
|
||||
v = PyBytes_FromStringAndSize(NULL, bytesize);
|
||||
if (v == NULL)
|
||||
return NULL;
|
||||
|
||||
p = (unsigned char *)PyByteArray_AS_STRING(v);
|
||||
p = (unsigned char *)PyBytes_AS_STRING(v);
|
||||
if (byteorder == 0)
|
||||
STORECHAR(0xFEFF);
|
||||
if (size == 0)
|
||||
|
|
@ -2828,9 +2825,7 @@ PyUnicode_EncodeUTF16(const Py_UNICODE *s,
|
|||
}
|
||||
|
||||
done:
|
||||
result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(v), Py_SIZE(v));
|
||||
Py_DECREF(v);
|
||||
return result;
|
||||
return v;
|
||||
#undef STORECHAR
|
||||
}
|
||||
|
||||
|
|
@ -3120,7 +3115,7 @@ static const char *hexdigits = "0123456789abcdef";
|
|||
PyObject *PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s,
|
||||
Py_ssize_t size)
|
||||
{
|
||||
PyObject *repr, *result;
|
||||
PyObject *repr;
|
||||
char *p;
|
||||
|
||||
#ifdef Py_UNICODE_WIDE
|
||||
|
|
@ -3147,17 +3142,20 @@ PyObject *PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s,
|
|||
escape.
|
||||
*/
|
||||
|
||||
if (size == 0)
|
||||
return PyBytes_FromStringAndSize(NULL, 0);
|
||||
|
||||
if (size > (PY_SSIZE_T_MAX - 2 - 1) / expandsize)
|
||||
return PyErr_NoMemory();
|
||||
|
||||
repr = PyByteArray_FromStringAndSize(NULL,
|
||||
repr = PyBytes_FromStringAndSize(NULL,
|
||||
2
|
||||
+ expandsize*size
|
||||
+ 1);
|
||||
if (repr == NULL)
|
||||
return NULL;
|
||||
|
||||
p = PyByteArray_AS_STRING(repr);
|
||||
p = PyBytes_AS_STRING(repr);
|
||||
|
||||
while (size-- > 0) {
|
||||
Py_UNICODE ch = *s++;
|
||||
|
|
@ -3249,13 +3247,13 @@ PyObject *PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s,
|
|||
*p++ = (char) ch;
|
||||
}
|
||||
|
||||
result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(repr),
|
||||
p - PyByteArray_AS_STRING(repr));
|
||||
Py_DECREF(repr);
|
||||
return result;
|
||||
assert(p - PyBytes_AS_STRING(repr) > 0);
|
||||
if (_PyBytes_Resize(&repr, p - PyBytes_AS_STRING(repr)) < 0)
|
||||
return NULL;
|
||||
return repr;
|
||||
}
|
||||
|
||||
PyObject *PyUnicode_AsUnicodeEscapeString(PyObject *unicode)
|
||||
PyObject *PyUnicodeAsUnicodeEscapeString(PyObject *unicode)
|
||||
{
|
||||
PyObject *s;
|
||||
if (!PyUnicode_Check(unicode)) {
|
||||
|
|
@ -3389,7 +3387,7 @@ PyObject *PyUnicode_DecodeRawUnicodeEscape(const char *s,
|
|||
PyObject *PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s,
|
||||
Py_ssize_t size)
|
||||
{
|
||||
PyObject *repr, *result;
|
||||
PyObject *repr;
|
||||
char *p;
|
||||
char *q;
|
||||
|
||||
|
|
@ -3402,13 +3400,13 @@ PyObject *PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s,
|
|||
if (size > PY_SSIZE_T_MAX / expandsize)
|
||||
return PyErr_NoMemory();
|
||||
|
||||
repr = PyByteArray_FromStringAndSize(NULL, expandsize * size);
|
||||
repr = PyBytes_FromStringAndSize(NULL, expandsize * size);
|
||||
if (repr == NULL)
|
||||
return NULL;
|
||||
if (size == 0)
|
||||
goto done;
|
||||
return repr;
|
||||
|
||||
p = q = PyByteArray_AS_STRING(repr);
|
||||
p = q = PyBytes_AS_STRING(repr);
|
||||
while (size-- > 0) {
|
||||
Py_UNICODE ch = *s++;
|
||||
#ifdef Py_UNICODE_WIDE
|
||||
|
|
@ -3468,10 +3466,10 @@ PyObject *PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s,
|
|||
}
|
||||
size = p - q;
|
||||
|
||||
done:
|
||||
result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(repr), size);
|
||||
Py_DECREF(repr);
|
||||
return result;
|
||||
assert(size > 0);
|
||||
if (_PyBytes_Resize(&repr, size) < 0)
|
||||
return NULL;
|
||||
return repr;
|
||||
}
|
||||
|
||||
PyObject *PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode)
|
||||
|
|
@ -3706,7 +3704,6 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
|
|||
const char *reason = (limit == 256) ? "ordinal not in range(256)" : "ordinal not in range(128)";
|
||||
PyObject *errorHandler = NULL;
|
||||
PyObject *exc = NULL;
|
||||
PyObject *result = NULL;
|
||||
/* the following variable is used for caching string comparisons
|
||||
* -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */
|
||||
int known_errorHandler = -1;
|
||||
|
|
@ -3715,10 +3712,10 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
|
|||
replacements, if we need more, we'll resize */
|
||||
if (size == 0)
|
||||
return PyBytes_FromStringAndSize(NULL, 0);
|
||||
res = PyByteArray_FromStringAndSize(NULL, size);
|
||||
res = PyBytes_FromStringAndSize(NULL, size);
|
||||
if (res == NULL)
|
||||
return NULL;
|
||||
str = PyByteArray_AS_STRING(res);
|
||||
str = PyBytes_AS_STRING(res);
|
||||
ressize = size;
|
||||
|
||||
while (p<endp) {
|
||||
|
|
@ -3768,7 +3765,7 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
|
|||
p = collend;
|
||||
break;
|
||||
case 4: /* xmlcharrefreplace */
|
||||
respos = str - PyByteArray_AS_STRING(res);
|
||||
respos = str - PyBytes_AS_STRING(res);
|
||||
/* determine replacement size (temporarily (mis)uses p) */
|
||||
for (p = collstart, repsize = 0; p < collend; ++p) {
|
||||
if (*p<10)
|
||||
|
|
@ -3795,9 +3792,9 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
|
|||
if (requiredsize > ressize) {
|
||||
if (requiredsize<2*ressize)
|
||||
requiredsize = 2*ressize;
|
||||
if (PyByteArray_Resize(res, requiredsize))
|
||||
if (_PyBytes_Resize(&res, requiredsize))
|
||||
goto onError;
|
||||
str = PyByteArray_AS_STRING(res) + respos;
|
||||
str = PyBytes_AS_STRING(res) + respos;
|
||||
ressize = requiredsize;
|
||||
}
|
||||
/* generate replacement (temporarily (mis)uses p) */
|
||||
|
|
@ -3815,17 +3812,17 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
|
|||
/* need more space? (at least enough for what we
|
||||
have+the replacement+the rest of the string, so
|
||||
we won't have to check space for encodable characters) */
|
||||
respos = str - PyByteArray_AS_STRING(res);
|
||||
respos = str - PyBytes_AS_STRING(res);
|
||||
repsize = PyUnicode_GET_SIZE(repunicode);
|
||||
requiredsize = respos+repsize+(endp-collend);
|
||||
if (requiredsize > ressize) {
|
||||
if (requiredsize<2*ressize)
|
||||
requiredsize = 2*ressize;
|
||||
if (PyByteArray_Resize(res, requiredsize)) {
|
||||
if (_PyBytes_Resize(&res, requiredsize)) {
|
||||
Py_DECREF(repunicode);
|
||||
goto onError;
|
||||
}
|
||||
str = PyByteArray_AS_STRING(res) + respos;
|
||||
str = PyBytes_AS_STRING(res) + respos;
|
||||
ressize = requiredsize;
|
||||
}
|
||||
/* check if there is anything unencodable in the replacement
|
||||
|
|
@ -3845,13 +3842,23 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
|
|||
}
|
||||
}
|
||||
}
|
||||
result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(res),
|
||||
str - PyByteArray_AS_STRING(res));
|
||||
onError:
|
||||
Py_DECREF(res);
|
||||
/* Resize if we allocated to much */
|
||||
size = str - PyBytes_AS_STRING(res);
|
||||
if (size < ressize) { /* If this falls res will be NULL */
|
||||
assert(size > 0);
|
||||
if (_PyBytes_Resize(&res, size) < 0)
|
||||
goto onError;
|
||||
}
|
||||
|
||||
Py_XDECREF(errorHandler);
|
||||
Py_XDECREF(exc);
|
||||
return result;
|
||||
return res;
|
||||
|
||||
onError:
|
||||
Py_XDECREF(res);
|
||||
Py_XDECREF(errorHandler);
|
||||
Py_XDECREF(exc);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
PyObject *PyUnicode_EncodeLatin1(const Py_UNICODE *p,
|
||||
|
|
@ -4104,7 +4111,7 @@ static int encode_mbcs(PyObject **repr,
|
|||
else {
|
||||
/* Extend string object */
|
||||
n = PyBytes_Size(*repr);
|
||||
if (_PyBytes_Resize(repr, n + mbcssize) < 0)
|
||||
if (_PyBytes_Resize(&repr, n + mbcssize) < 0)
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -4834,7 +4841,8 @@ PyObject *PyUnicode_EncodeCharmap(const Py_UNICODE *p,
|
|||
|
||||
/* Resize if we allocated to much */
|
||||
if (respos<PyBytes_GET_SIZE(res))
|
||||
_PyBytes_Resize(&res, respos);
|
||||
if (_PyBytes_Resize(&res, respos) < 0)
|
||||
goto onError;
|
||||
|
||||
Py_XDECREF(exc);
|
||||
Py_XDECREF(errorHandler);
|
||||
|
|
|
|||
Loading…
Reference in New Issue