diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 0f7c23efccb..4c58b2df2b0 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -313,7 +313,7 @@ class UTF32Test(ReadTest): def test_partial(self): self.check_partial( - "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", [ "", # first byte of BOM read "", # second byte of BOM read @@ -335,6 +335,10 @@ class UTF32Test(ReadTest): "\x00\xff\u0100", "\x00\xff\u0100", "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", ] ) @@ -369,7 +373,7 @@ class UTF32LETest(ReadTest): def test_partial(self): self.check_partial( - "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", [ "", "", @@ -387,6 +391,10 @@ class UTF32LETest(ReadTest): "\x00\xff\u0100", "\x00\xff\u0100", "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", ] ) @@ -409,7 +417,7 @@ class UTF32BETest(ReadTest): def test_partial(self): self.check_partial( - "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", [ "", "", @@ -427,6 +435,10 @@ class UTF32BETest(ReadTest): "\x00\xff\u0100", "\x00\xff\u0100", "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", ] ) @@ -477,7 +489,7 @@ class UTF16Test(ReadTest): def test_partial(self): self.check_partial( - "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", [ "", # first byte of BOM read "", # second byte of BOM read => byteorder known @@ -489,6 +501,10 @@ class UTF16Test(ReadTest): "\x00\xff\u0100", "\x00\xff\u0100", "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", ] ) @@ -526,7 +542,7 @@ class UTF16LETest(ReadTest): def test_partial(self): self.check_partial( - "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", [ "", "\x00", @@ -536,6 +552,10 @@ class UTF16LETest(ReadTest): "\x00\xff\u0100", "\x00\xff\u0100", "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", ] ) @@ -565,7 +585,7 @@ class UTF16BETest(ReadTest): def test_partial(self): self.check_partial( - "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", [ "", "\x00", @@ -575,6 +595,10 @@ class UTF16BETest(ReadTest): "\x00\xff\u0100", "\x00\xff\u0100", "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff", + "\x00\xff\u0100\uffff\U00010000", ] ) @@ -604,7 +628,7 @@ class UTF8Test(ReadTest): def test_partial(self): self.check_partial( - "\x00\xff\u07ff\u0800\uffff", + "\x00\xff\u07ff\u0800\uffff\U00010000", [ "\x00", "\x00", @@ -617,6 +641,10 @@ class UTF8Test(ReadTest): "\x00\xff\u07ff\u0800", "\x00\xff\u07ff\u0800", "\x00\xff\u07ff\u0800\uffff", + "\x00\xff\u07ff\u0800\uffff", + "\x00\xff\u07ff\u0800\uffff", + "\x00\xff\u07ff\u0800\uffff", + "\x00\xff\u07ff\u0800\uffff\U00010000", ] ) @@ -694,7 +722,7 @@ class UTF8SigTest(ReadTest): def test_partial(self): self.check_partial( - "\ufeff\x00\xff\u07ff\u0800\uffff", + "\ufeff\x00\xff\u07ff\u0800\uffff\U00010000", [ "", "", @@ -713,6 +741,10 @@ class UTF8SigTest(ReadTest): "\ufeff\x00\xff\u07ff\u0800", "\ufeff\x00\xff\u07ff\u0800", "\ufeff\x00\xff\u07ff\u0800\uffff", + "\ufeff\x00\xff\u07ff\u0800\uffff", + "\ufeff\x00\xff\u07ff\u0800\uffff", + "\ufeff\x00\xff\u07ff\u0800\uffff", + "\ufeff\x00\xff\u07ff\u0800\uffff\U00010000", ] ) diff --git a/Misc/NEWS b/Misc/NEWS index 8ada15ad709..f51476aab4b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.2.4 Core and Builtins ----------------- +- Issue #11461: Fix the incremental UTF-16 decoder. Original patch by + Amaury Forgeot d'Arc. + - Issue #16367: Fix FileIO.readall() on Windows for files larger than 2 GB. - Issue #16455: On FreeBSD and Solaris, if the locale is C, the diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 7cd039990c7..7f86bfd6df9 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3573,8 +3573,11 @@ PyUnicode_DecodeUTF16Stateful(const char *s, /* UTF-16 code pair: */ if (e - q < 2) { + q -= 2; + if (consumed) + break; errmsg = "unexpected end of data"; - startinpos = (((const char *)q) - 2) - starts; + startinpos = ((const char *)q) - starts; endinpos = ((const char *)e) - starts; goto utf16Error; }