Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Port code page codec to Unicode API.

This commit is contained in:
Martin v. Löwis 2011-11-04 18:23:06 +01:00
parent 8ba79306d1
commit 3d325191bf
2 changed files with 73 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Lib/test/test_codeccallbacks.py
View File

@ -577,22 +577,18 @@ class CodecCallbackTest(unittest.TestCase):
UnicodeEncodeError("ascii", "\uffff", 0, 1, "ouch")),
("\\uffff", 1)
)
if SIZEOF_WCHAR_T == 2:
len_wide = 2
else:
len_wide = 1
if SIZEOF_WCHAR_T > 0:
self.assertEqual(
codecs.backslashreplace_errors(
UnicodeEncodeError("ascii", "\U00010000",
0, len_wide, "ouch")),
("\\U00010000", len_wide)
0, 1, "ouch")),
("\\U00010000", 1)
)
self.assertEqual(
codecs.backslashreplace_errors(
UnicodeEncodeError("ascii", "\U0010ffff",
0, len_wide, "ouch")),
("\\U0010ffff", len_wide)
0, 1, "ouch")),
("\\U0010ffff", 1)
)
# Lone surrogates (regardless of unicode width)
self.assertEqual(

123
Objects/unicodeobject.c
View File

@ -4680,9 +4680,6 @@ _PyUnicode_AsUTF8String(PyObject *unicode, const char *errors)
int kind;
void *data;
Py_ssize_t size;
#if SIZEOF_WCHAR_T == 2
Py_ssize_t wchar_offset = 0;
#endif
if (!PyUnicode_Check(unicode)) {
PyErr_BadArgument();
@ -4738,9 +4735,6 @@ _PyUnicode_AsUTF8String(PyObject *unicode, const char *errors)
PyObject *rep;
Py_ssize_t repsize, k, startpos;
startpos = i-1;
#if SIZEOF_WCHAR_T == 2
startpos += wchar_offset;
#endif
rep = unicode_encode_call_errorhandler(
errors, &errorHandler, "utf-8", "surrogates not allowed",
unicode, &exc, startpos, startpos+1, &newpos);
@ -4809,9 +4803,6 @@ _PyUnicode_AsUTF8String(PyObject *unicode, const char *errors)
*p++ = (char)(0x80 | ((ch >> 12) & 0x3f));
*p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
*p++ = (char)(0x80 | (ch & 0x3f));
#if SIZEOF_WCHAR_T == 2
wchar_offset++;
#endif
}
}
@ -7315,23 +7306,37 @@ encode_code_page_flags(UINT code_page, const char *errors)
*/
static int
encode_code_page_strict(UINT code_page, PyObject **outbytes,
const Py_UNICODE *p, const int size,
PyObject *unicode, Py_ssize_t offset, int len,
const char* errors)
{
BOOL usedDefaultChar = FALSE;
BOOL *pusedDefaultChar = &usedDefaultChar;
int outsize;
PyObject *exc = NULL;
Py_UNICODE *p;
Py_ssize_t size;
const DWORD flags = encode_code_page_flags(code_page, NULL);
char *out;
/* Create a substring so that we can get the UTF-16 representation
of just the slice under consideration. */
PyObject *substring;
assert(size > 0);
assert(len > 0);
if (code_page != CP_UTF8 && code_page != CP_UTF7)
pusedDefaultChar = &usedDefaultChar;
else
pusedDefaultChar = NULL;
substring = PyUnicode_Substring(unicode, offset, offset+len);
if (substring == NULL)
return -1;
p = PyUnicode_AsUnicodeAndSize(substring, &size);
if (p == NULL) {
Py_DECREF(substring);
return -1;
}
/* First get the size of the result */
outsize = WideCharToMultiByte(code_page, flags,
p, size,
@ -7340,14 +7345,18 @@ encode_code_page_strict(UINT code_page, PyObject **outbytes,
if (outsize <= 0)
goto error;
/* If we used a default char, then we failed! */
if (pusedDefaultChar && *pusedDefaultChar)
if (pusedDefaultChar && *pusedDefaultChar) {
Py_DECREF(substring);
return -2;
}
if (*outbytes == NULL) {
/* Create string object */
*outbytes = PyBytes_FromStringAndSize(NULL, outsize);
if (*outbytes == NULL)
if (*outbytes == NULL) {
Py_DECREF(substring);
return -1;
}
out = PyBytes_AS_STRING(*outbytes);
}
else {
@ -7355,10 +7364,13 @@ encode_code_page_strict(UINT code_page, PyObject **outbytes,
const Py_ssize_t n = PyBytes_Size(*outbytes);
if (outsize > PY_SSIZE_T_MAX - n) {
PyErr_NoMemory();
Py_DECREF(substring);
return -1;
}
if (_PyBytes_Resize(outbytes, n + outsize) < 0)
if (_PyBytes_Resize(outbytes, n + outsize) < 0) {
Py_DECREF(substring);
return -1;
}
out = PyBytes_AS_STRING(*outbytes) + n;
}
@ -7367,6 +7379,7 @@ encode_code_page_strict(UINT code_page, PyObject **outbytes,
p, size,
out, outsize,
NULL, pusedDefaultChar);
Py_CLEAR(substring);
if (outsize <= 0)
goto error;
if (pusedDefaultChar && *pusedDefaultChar)
@ -7374,6 +7387,7 @@ encode_code_page_strict(UINT code_page, PyObject **outbytes,
return 0;
error:
Py_XDECREF(substring);
if (GetLastError() == ERROR_NO_UNICODE_TRANSLATION)
return -2;
PyErr_SetFromWindowsErr(0);
@ -7390,12 +7404,11 @@ error:
static int
encode_code_page_errors(UINT code_page, PyObject **outbytes,
PyObject *unicode, Py_ssize_t unicode_offset,
const Py_UNICODE *in, const int insize,
const char* errors)
Py_ssize_t insize, const char* errors)
{
const DWORD flags = encode_code_page_flags(code_page, errors);
const Py_UNICODE *startin = in;
const Py_UNICODE *endin = in + insize;
Py_ssize_t pos = unicode_offset;
Py_ssize_t endin = unicode_offset + insize;
/* Ideally, we should get reason from FormatMessage. This is the Windows
2000 English version of the message. */
const char *reason = "invalid character";
@ -7404,12 +7417,11 @@ encode_code_page_errors(UINT code_page, PyObject **outbytes,
BOOL usedDefaultChar = FALSE, *pusedDefaultChar;
Py_ssize_t outsize;
char *out;
int charsize;
PyObject *errorHandler = NULL;
PyObject *exc = NULL;
PyObject *encoding_obj = NULL;
char *encoding;
Py_ssize_t startpos, newpos, newoutsize;
Py_ssize_t newpos, newoutsize;
PyObject *rep;
int ret = -1;
@ -7422,7 +7434,7 @@ encode_code_page_errors(UINT code_page, PyObject **outbytes,
if (errors == NULL || strcmp(errors, "strict") == 0) {
/* The last error was ERROR_NO_UNICODE_TRANSLATION,
then we raise a UnicodeEncodeError. */
make_encode_exception(&exc, encoding, in, insize, 0, 0, reason);
make_encode_exception_obj(&exc, encoding, unicode, 0, 0, reason);
if (exc != NULL) {
PyCodec_StrictErrors(exc);
Py_DECREF(exc);
@ -7462,23 +7474,30 @@ encode_code_page_errors(UINT code_page, PyObject **outbytes,
}
/* Encode the string character per character */
while (in < endin)
while (pos < endin)
{
if ((in + 2) <= endin
&& 0xD800 <= in[0] && in[0] <= 0xDBFF
&& 0xDC00 <= in[1] && in[1] <= 0xDFFF)
charsize = 2;
else
charsize = 1;
Py_UCS4 ch = PyUnicode_READ_CHAR(unicode, pos);
wchar_t chars[2];
int charsize;
if (ch < 0x10000) {
chars[0] = (wchar_t)ch;
charsize = 1;
}
else {
ch -= 0x10000;
chars[0] = 0xd800 + (ch >> 10);
chars[1] = 0xdc00 + (ch & 0x3ff);
charsize = 2;
}
outsize = WideCharToMultiByte(code_page, flags,
in, charsize,
chars, charsize,
buffer, Py_ARRAY_LENGTH(buffer),
NULL, pusedDefaultChar);
if (outsize > 0) {
if (pusedDefaultChar == NULL || !(*pusedDefaultChar))
{
in += charsize;
pos++;
memcpy(out, buffer, outsize);
out += outsize;
continue;
@ -7489,15 +7508,13 @@ encode_code_page_errors(UINT code_page, PyObject **outbytes,
goto error;
}
charsize = Py_MAX(charsize - 1, 1);
startpos = unicode_offset + in - startin;
rep = unicode_encode_call_errorhandler(
errors, &errorHandler, encoding, reason,
unicode, &exc,
startpos, startpos + charsize, &newpos);
pos, pos + 1, &newpos);
if (rep == NULL)
goto error;
in += (newpos - startpos);
pos = newpos;
if (PyBytes_Check(rep)) {
outsize = PyBytes_GET_SIZE(rep);
@ -7538,10 +7555,9 @@ encode_code_page_errors(UINT code_page, PyObject **outbytes,
for (i=0; i < outsize; i++) {
Py_UCS4 ch = PyUnicode_READ(kind, data, i);
if (ch > 127) {
raise_encode_exception(&exc,
encoding,
startin, insize,
startpos, startpos + charsize,
raise_encode_exception_obj(&exc,
encoding, unicode,
pos, pos + 1,
"unable to encode error handler result to ASCII");
Py_DECREF(rep);
goto error;
@ -7572,55 +7588,54 @@ encode_code_page(int code_page,
PyObject *unicode,
const char *errors)
{
const Py_UNICODE *p;
Py_ssize_t size;
Py_ssize_t len;
PyObject *outbytes = NULL;
Py_ssize_t offset;
int chunk_len, ret, done;
p = PyUnicode_AsUnicodeAndSize(unicode, &size);
if (p == NULL)
return NULL;
if (PyUnicode_READY(unicode) < 0)
return NULL;
len = PyUnicode_GET_LENGTH(unicode);
if (code_page < 0) {
PyErr_SetString(PyExc_ValueError, "invalid code page number");
return NULL;
}
if (size == 0)
if (len == 0)
return PyBytes_FromStringAndSize(NULL, 0);
offset = 0;
do
{
#ifdef NEED_RETRY
if (size > INT_MAX) {
chunk_len = INT_MAX;
/* UTF-16 encoding may double the size, so use only INT_MAX/2
chunks. */
if (len > INT_MAX/2) {
chunk_len = INT_MAX/2;
done = 0;
}
else
#endif
{
chunk_len = (int)size;
chunk_len = (int)len;
done = 1;
}
ret = encode_code_page_strict(code_page, &outbytes,
p, chunk_len,
unicode, offset, chunk_len,
errors);
if (ret == -2)
ret = encode_code_page_errors(code_page, &outbytes,
unicode, offset,
p, chunk_len,
errors);
chunk_len, errors);
if (ret < 0) {
Py_XDECREF(outbytes);
return NULL;
}
p += chunk_len;
offset += chunk_len;
size -= chunk_len;
len -= chunk_len;
} while (!done);
return outbytes;