Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Back out 2.140.

This commit is contained in:
Martin v. Löwis 2002-04-21 09:59:45 +00:00
parent 639295f0a5
commit 2a7ff35a07
1 changed files with 55 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
Objects/unicodeobject.c
View File

@ -1172,6 +1172,12 @@ int utf8_encoding_error(const Py_UNICODE **source,
} }
#endif #endif
/* Allocation strategy: we default to Latin-1, then do one resize
whenever we hit an order boundary. The assumption is that
characters from higher orders usually occur often enough to warrant
this.
*/
PyObject * PyObject *
PyUnicode_EncodeUTF8(const Py_UNICODE *s, PyUnicode_EncodeUTF8(const Py_UNICODE *s,
int size, int size,
@ -1179,81 +1185,87 @@ PyUnicode_EncodeUTF8(const Py_UNICODE *s,
{ {
PyObject *v; PyObject *v;
char *p; char *p;
int allocated = 0; int i = 0;
int i; int overalloc = 2;
int len;
/* Short-cut for emtpy strings */ /* Short-cut for emtpy strings */
if (size == 0) if (size == 0)
return PyString_FromStringAndSize(NULL, 0); return PyString_FromStringAndSize(NULL, 0);
for (i = 0; i < size; ) { v = PyString_FromStringAndSize(NULL, overalloc * size);
Py_UCS4 ch = s[i++];
if (ch < 0x80)
allocated += 1;
else if (ch < 0x0800)
allocated += 2;
else if (ch < 0x10000) {
/* Check for high surrogate */
if (0xD800 <= ch && ch <= 0xDBFF &&
i != size &&
0xDC00 <= s[i] && s[i] <= 0xDFFF) {
allocated += 1;
i++;
}
allocated += 3;
} else
allocated += 4;
}
v = PyString_FromStringAndSize(NULL, allocated);
if (v == NULL) if (v == NULL)
return NULL; return NULL;
p = PyString_AS_STRING(v); p = PyString_AS_STRING(v);
for (i = 0; i < size; ) {
while (i < size) {
Py_UCS4 ch = s[i++]; Py_UCS4 ch = s[i++];
if (ch < 0x80) { if (ch < 0x80)
/* Encode ASCII */
*p++ = (char) ch; *p++ = (char) ch;
}
else if (ch < 0x0800) { else if (ch < 0x0800) {
/* Encode Latin-1 */
*p++ = (char)(0xc0 | (ch >> 6)); *p++ = (char)(0xc0 | (ch >> 6));
*p++ = (char)(0x80 | (ch & 0x3f)); *p++ = (char)(0x80 | (ch & 0x3f));
} }
else { else {
/* Encode UCS2 Unicode ordinals */
if (ch < 0x10000) { if (ch < 0x10000) {
/* Check for high surrogate */
/* Special case: check for high surrogate */
if (0xD800 <= ch && ch <= 0xDBFF && i != size) { if (0xD800 <= ch && ch <= 0xDBFF && i != size) {
Py_UCS4 ch2 = s[i]; Py_UCS4 ch2 = s[i];
/* Check for low surrogate */ /* Check for low surrogate and combine the two to
form a UCS4 value */
if (0xDC00 <= ch2 && ch2 <= 0xDFFF) { if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {
ch = ((ch - 0xD800)<<10 | (ch2-0xDC00))+0x10000; ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000;
*p++ = (char)((ch >> 18) | 0xf0); i++;
*p++ = (char)(0x80 | ((ch >> 12) & 0x3f)); goto encodeUCS4;
*p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
*p++ = (char)(0x80 | (ch & 0x3f));
i++;
continue;
} }
/* Fall through: handles isolated high surrogates */ /* Fall through: handles isolated high surrogates */
} }
if (overalloc < 3) {
len = (int)(p - PyString_AS_STRING(v));
overalloc = 3;
if (_PyString_Resize(&v, overalloc * size))
goto onError;
p = PyString_AS_STRING(v) + len;
}
*p++ = (char)(0xe0 | (ch >> 12)); *p++ = (char)(0xe0 | (ch >> 12));
*p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
*p++ = (char)(0x80 | (ch & 0x3f)); *p++ = (char)(0x80 | (ch & 0x3f));
continue;
} else {
*p++ = (char)(0xf0 | (ch>>18));
*p++ = (char)(0x80 | ((ch>>12) & 0x3f));
*p++ = (char)(0x80 | ((ch>>6) & 0x3f));
*p++ = (char)(0x80 | (ch & 0x3f));
} }
/* Encode UCS4 Unicode ordinals */
encodeUCS4:
if (overalloc < 4) {
len = (int)(p - PyString_AS_STRING(v));
overalloc = 4;
if (_PyString_Resize(&v, overalloc * size))
goto onError;
p = PyString_AS_STRING(v) + len;
}
*p++ = (char)(0xf0 | (ch >> 18));
*p++ = (char)(0x80 | ((ch >> 12) & 0x3f));
*p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
*p++ = (char)(0x80 | (ch & 0x3f));
} }
} }
assert(p - PyString_AS_STRING(v) == allocated); *p = '\0';
assert((p - PyString_AS_STRING(v)) <= overalloc*size);
if (_PyString_Resize(&v, (int)(p - PyString_AS_STRING(v))))
goto onError;
return v; return v;
onError:
Py_DECREF(v);
return NULL;
} }
PyObject *PyUnicode_AsUTF8String(PyObject *unicode) PyObject *PyUnicode_AsUTF8String(PyObject *unicode)