traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-35372: Fix the code page decoder for input > 2 GiB. (GH-10848)

This commit is contained in:
Serhiy Storchaka 2018-12-03 10:36:45 +02:00 committed by GitHub
parent 062cbb6772
commit 4013c17911
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 24 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
Lib/test/test_codecs.py
View File

@ -3190,6 +3190,24 @@ class CodePageTest(unittest.TestCase):
codec = codecs.lookup('cp123')
self.assertEqual(codec.name, 'mbcs')
@support.bigmemtest(size=2**31, memuse=7, dry_run=False)
def test_large_input(self):
# Test input longer than INT_MAX.
# Input should contain undecodable bytes before and after
# the INT_MAX limit.
encoded = (b'01234567' * (2**28-1) +
b'\x85\x86\xea\xeb\xec\xef\xfc\xfd\xfe\xff')
self.assertEqual(len(encoded), 2**31+2)
decoded = codecs.code_page_decode(932, encoded, 'surrogateescape', True)
self.assertEqual(decoded[1], len(encoded))
del encoded
self.assertEqual(len(decoded[0]), decoded[1])
self.assertEqual(decoded[0][:10], '0123456701')
self.assertEqual(decoded[0][-20:],
'6701234567'
'\udc85\udc86\udcea\udceb\udcec'
'\udcef\udcfc\udcfd\udcfe\udcff')
class ASCIITest(unittest.TestCase):
def test_encode(self):

2
Misc/NEWS.d/next/Core and Builtins/2018-12-01-19-20-53.bpo-35372.RwVJjZ.rst Normal file
View File

@ -0,0 +1,2 @@
Fixed the code page decoder for input longer than 2 GiB containing
undecodable bytes.

9
Objects/unicodeobject.c
View File

@ -7211,7 +7211,7 @@ decode_code_page_errors(UINT code_page,
"in the target code page.";
/* each step cannot decode more than 1 character, but a character can be
represented as a surrogate pair */
wchar_t buffer[2], *startout, *out;
wchar_t buffer[2], *out;
int insize;
Py_ssize_t outsize;
PyObject *errorHandler = NULL;
@ -7248,7 +7248,7 @@ decode_code_page_errors(UINT code_page,
*v = (PyObject*)_PyUnicode_New(size * Py_ARRAY_LENGTH(buffer));
if (*v == NULL)
goto error;
startout = PyUnicode_AS_UNICODE(*v);
out = PyUnicode_AS_UNICODE(*v);
}
else {
/* Extend unicode object */
@ -7259,11 +7259,10 @@ decode_code_page_errors(UINT code_page,
}
if (unicode_resize(v, n + size * Py_ARRAY_LENGTH(buffer)) < 0)
goto error;
startout = PyUnicode_AS_UNICODE(*v) + n;
out = PyUnicode_AS_UNICODE(*v) + n;
}
/* Decode the byte string character per character */
out = startout;
while (in < endin)
{
/* Decode a character */
@ -7318,7 +7317,7 @@ decode_code_page_errors(UINT code_page,
*out = 0;
/* Extend unicode object */
outsize = out - startout;
outsize = out - PyUnicode_AS_UNICODE(*v);
assert(outsize <= PyUnicode_WSTR_LENGTH(*v));
if (unicode_resize(v, outsize) < 0)
goto error;