Fix for the UTF-8 memory allocation bug and the UTF-8 encoding
bug related to lone high surrogates.
This commit is contained in:
parent
6a60915843
commit
3688a882d3
|
@ -495,19 +495,18 @@ else:
|
|||
verify(unicode('+3ADYAA-', 'utf-7', 'replace') == u'\ufffd')
|
||||
|
||||
# UTF-8 specific encoding tests:
|
||||
verify(u'\u20ac'.encode('utf-8') == \
|
||||
''.join((chr(0xe2), chr(0x82), chr(0xac))) )
|
||||
verify(u'\ud800\udc02'.encode('utf-8') == \
|
||||
''.join((chr(0xf0), chr(0x90), chr(0x80), chr(0x82))) )
|
||||
verify(u'\ud84d\udc56'.encode('utf-8') == \
|
||||
''.join((chr(0xf0), chr(0xa3), chr(0x91), chr(0x96))) )
|
||||
verify(u'\u20ac'.encode('utf-8') == '\xe2\x82\xac')
|
||||
verify(u'\ud800\udc02'.encode('utf-8') == '\xf0\x90\x80\x82')
|
||||
verify(u'\ud84d\udc56'.encode('utf-8') == '\xf0\xa3\x91\x96')
|
||||
verify(u'\ud800'.encode('utf-8') == '\xed\xa0\x80')
|
||||
verify(u'\udc00'.encode('utf-8') == '\xed\xb0\x80')
|
||||
verify((u'\ud800\udc02'*1000).encode('utf-8') ==
|
||||
'\xf0\x90\x80\x82'*1000)
|
||||
|
||||
# UTF-8 specific decoding tests
|
||||
verify(unicode(''.join((chr(0xf0), chr(0xa3), chr(0x91), chr(0x96))),
|
||||
'utf-8') == u'\U00023456' )
|
||||
verify(unicode(''.join((chr(0xf0), chr(0x90), chr(0x80), chr(0x82))),
|
||||
'utf-8') == u'\U00010002' )
|
||||
verify(unicode(''.join((chr(0xe2), chr(0x82), chr(0xac))),
|
||||
'utf-8') == u'\u20ac' )
|
||||
verify(unicode('\xf0\xa3\x91\x96', 'utf-8') == u'\U00023456' )
|
||||
verify(unicode('\xf0\x90\x80\x82', 'utf-8') == u'\U00010002' )
|
||||
verify(unicode('\xe2\x82\xac', 'utf-8') == u'\u20ac' )
|
||||
|
||||
# Other possible utf-8 test cases:
|
||||
# * strict decoding testing for all of the
|
||||
|
|
|
@ -1171,61 +1171,64 @@ PyObject *PyUnicode_EncodeUTF8(const Py_UNICODE *s,
|
|||
{
|
||||
PyObject *v;
|
||||
char *p;
|
||||
char *q;
|
||||
Py_UCS4 ch2;
|
||||
unsigned int cbAllocated = 3 * size;
|
||||
unsigned int cbAllocated = 2 * size;
|
||||
unsigned int cbWritten = 0;
|
||||
int i = 0;
|
||||
|
||||
v = PyString_FromStringAndSize(NULL, cbAllocated);
|
||||
v = PyString_FromStringAndSize(NULL, cbAllocated + 4);
|
||||
if (v == NULL)
|
||||
return NULL;
|
||||
if (size == 0)
|
||||
return v;
|
||||
|
||||
p = q = PyString_AS_STRING(v);
|
||||
p = PyString_AS_STRING(v);
|
||||
while (i < size) {
|
||||
Py_UCS4 ch = s[i++];
|
||||
|
||||
if (ch < 0x80) {
|
||||
*p++ = (char) ch;
|
||||
cbWritten++;
|
||||
}
|
||||
|
||||
else if (ch < 0x0800) {
|
||||
*p++ = 0xc0 | (ch >> 6);
|
||||
*p++ = 0x80 | (ch & 0x3f);
|
||||
cbWritten += 2;
|
||||
}
|
||||
else if (ch < 0x10000) {
|
||||
/* Check for high surrogate */
|
||||
if (0xD800 <= ch && ch <= 0xDBFF) {
|
||||
if (i != size) {
|
||||
ch2 = s[i];
|
||||
if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {
|
||||
|
||||
if (cbWritten >= (cbAllocated - 4)) {
|
||||
/* Provide enough room for some more
|
||||
surrogates */
|
||||
cbAllocated += 4*10;
|
||||
if (_PyString_Resize(&v, cbAllocated))
|
||||
else {
|
||||
|
||||
/* Assure that we have enough room for high order Unicode
|
||||
ordinals */
|
||||
if (cbWritten >= cbAllocated) {
|
||||
cbAllocated += 4 * 10;
|
||||
if (_PyString_Resize(&v, cbAllocated + 4))
|
||||
goto onError;
|
||||
p = PyString_AS_STRING(v) + cbWritten;
|
||||
}
|
||||
|
||||
/* combine the two values */
|
||||
if (ch < 0x10000) {
|
||||
/* Check for high surrogate */
|
||||
if (0xD800 <= ch && ch <= 0xDBFF && i != size) {
|
||||
Py_UCS4 ch2 = s[i];
|
||||
/* Check for low surrogate */
|
||||
if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {
|
||||
ch = ((ch - 0xD800)<<10 | (ch2-0xDC00))+0x10000;
|
||||
|
||||
*p++ = (char)((ch >> 18) | 0xf0);
|
||||
*p++ = (char)(0x80 | ((ch >> 12) & 0x3f));
|
||||
*p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
|
||||
*p++ = (char)(0x80 | (ch & 0x3f));
|
||||
i++;
|
||||
cbWritten += 4;
|
||||
continue;
|
||||
}
|
||||
/* Fall through: handles isolated high surrogates */
|
||||
}
|
||||
}
|
||||
else {
|
||||
*p++ = (char)(0xe0 | (ch >> 12));
|
||||
cbWritten += 3;
|
||||
}
|
||||
*p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
|
||||
*p++ = (char)(0x80 | (ch & 0x3f));
|
||||
cbWritten += 3;
|
||||
|
||||
} else {
|
||||
*p++ = 0xf0 | (ch>>18);
|
||||
*p++ = 0x80 | ((ch>>12) & 0x3f);
|
||||
|
@ -1234,8 +1237,9 @@ PyObject *PyUnicode_EncodeUTF8(const Py_UNICODE *s,
|
|||
cbWritten += 4;
|
||||
}
|
||||
}
|
||||
}
|
||||
*p = '\0';
|
||||
if (_PyString_Resize(&v, p - q))
|
||||
if (_PyString_Resize(&v, cbWritten))
|
||||
goto onError;
|
||||
return v;
|
||||
|
||||
|
|
Loading…
Reference in New Issue