Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Issue #11461: Fix the incremental UTF-16 decoder. Original patch by 

Amaury Forgeot d'Arc. Added tests for partial decoding of non-BMP
characters.
This commit is contained in:
Serhiy Storchaka 2013-01-08 23:14:24 +02:00
parent dec798eb46
commit 48e188e573
3 changed files with 47 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
Lib/test/test_codecs.py
View File

@ -313,7 +313,7 @@ class UTF32Test(ReadTest):
def test_partial(self):
self.check_partial(
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff\U00010000",
[
"", # first byte of BOM read
"", # second byte of BOM read
@ -335,6 +335,10 @@ class UTF32Test(ReadTest):
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff\U00010000",
]
)
@ -369,7 +373,7 @@ class UTF32LETest(ReadTest):
def test_partial(self):
self.check_partial(
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff\U00010000",
[
"",
"",
@ -387,6 +391,10 @@ class UTF32LETest(ReadTest):
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff\U00010000",
]
)
@ -409,7 +417,7 @@ class UTF32BETest(ReadTest):
def test_partial(self):
self.check_partial(
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff\U00010000",
[
"",
"",
@ -427,6 +435,10 @@ class UTF32BETest(ReadTest):
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff\U00010000",
]
)
@ -477,7 +489,7 @@ class UTF16Test(ReadTest):
def test_partial(self):
self.check_partial(
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff\U00010000",
[
"", # first byte of BOM read
"", # second byte of BOM read => byteorder known
@ -489,6 +501,10 @@ class UTF16Test(ReadTest):
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff\U00010000",
]
)
@ -526,7 +542,7 @@ class UTF16LETest(ReadTest):
def test_partial(self):
self.check_partial(
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff\U00010000",
[
"",
"\x00",
@ -536,6 +552,10 @@ class UTF16LETest(ReadTest):
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff\U00010000",
]
)
@ -565,7 +585,7 @@ class UTF16BETest(ReadTest):
def test_partial(self):
self.check_partial(
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff\U00010000",
[
"",
"\x00",
@ -575,6 +595,10 @@ class UTF16BETest(ReadTest):
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff",
"\x00\xff\u0100\uffff\U00010000",
]
)
@ -604,7 +628,7 @@ class UTF8Test(ReadTest):
def test_partial(self):
self.check_partial(
"\x00\xff\u07ff\u0800\uffff",
"\x00\xff\u07ff\u0800\uffff\U00010000",
[
"\x00",
"\x00",
@ -617,6 +641,10 @@ class UTF8Test(ReadTest):
"\x00\xff\u07ff\u0800",
"\x00\xff\u07ff\u0800",
"\x00\xff\u07ff\u0800\uffff",
"\x00\xff\u07ff\u0800\uffff",
"\x00\xff\u07ff\u0800\uffff",
"\x00\xff\u07ff\u0800\uffff",
"\x00\xff\u07ff\u0800\uffff\U00010000",
]
)
@ -694,7 +722,7 @@ class UTF8SigTest(ReadTest):
def test_partial(self):
self.check_partial(
"\ufeff\x00\xff\u07ff\u0800\uffff",
"\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
[
"",
"",
@ -713,6 +741,10 @@ class UTF8SigTest(ReadTest):
"\ufeff\x00\xff\u07ff\u0800",
"\ufeff\x00\xff\u07ff\u0800",
"\ufeff\x00\xff\u07ff\u0800\uffff",
"\ufeff\x00\xff\u07ff\u0800\uffff",
"\ufeff\x00\xff\u07ff\u0800\uffff",
"\ufeff\x00\xff\u07ff\u0800\uffff",
"\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
]
)

3
Misc/NEWS
View File

@ -10,6 +10,9 @@ What's New in Python 3.2.4
Core and Builtins
-----------------
- Issue #11461: Fix the incremental UTF-16 decoder. Original patch by
Amaury Forgeot d'Arc.
- Issue #16367: Fix FileIO.readall() on Windows for files larger than 2 GB.
- Issue #16455: On FreeBSD and Solaris, if the locale is C, the

5
Objects/unicodeobject.c
View File

@ -3573,8 +3573,11 @@ PyUnicode_DecodeUTF16Stateful(const char *s,
/* UTF-16 code pair: */
if (e - q < 2) {
q -= 2;
if (consumed)
break;
errmsg = "unexpected end of data";
startinpos = (((const char *)q) - 2) - starts;
startinpos = ((const char *)q) - starts;
endinpos = ((const char *)e) - starts;
goto utf16Error;
}