#8271: the utf-8 decoder now outputs the correct number of U+FFFD characters when used with the "replace" error handler on invalid utf-8 sequences. Patch by Serhiy Storchaka, tests by Ezio Melotti.

Lib/test/test_unicode.py

@@ -1349,7 +1349,7 @@ class UnicodeTest(string_tests.CommonTest,
             # with start byte of a 2-byte sequence
             (b'\xc2', FFFD), # only the start byte
             (b'\xc2\xc2', FFFD*2), # 2 start bytes
-            (b'\xc2\xc2\xc2', FFFD*3), # 2 start bytes
+            (b'\xc2\xc2\xc2', FFFD*3), # 3 start bytes
             (b'\xc2\x41', FFFD+'A'), # invalid continuation byte
             # with start byte of a 3-byte sequence
             (b'\xe1', FFFD), # only the start byte
@@ -1419,6 +1419,226 @@ class UnicodeTest(string_tests.CommonTest,
             self.assertEqual(seq.decode('utf-8', 'ignore'),
                              res.replace('\uFFFD', ''))
 
+    def to_bytestring(self, seq):
+        return bytes(int(c, 16) for c in seq.split())
+
+    def assertCorrectUTF8Decoding(self, seq, res, err):
+        """
+        Check that an invalid UTF-8 sequence raises an UnicodeDecodeError when
+        'strict' is used, returns res when 'replace' is used, and that doesn't
+        return anything when 'ignore' is used.
+        """
+        with self.assertRaises(UnicodeDecodeError) as cm:
+            seq.decode('utf-8')
+        exc = cm.exception
+
+        self.assertIn(err, str(exc))
+        self.assertEqual(seq.decode('utf-8', 'replace'), res)
+        self.assertEqual((b'aaaa' + seq + b'bbbb').decode('utf-8', 'replace'),
+                         'aaaa' + res + 'bbbb')
+        res = res.replace('\ufffd', '')
+        self.assertEqual(seq.decode('utf-8', 'ignore'), res)
+        self.assertEqual((b'aaaa' + seq + b'bbbb').decode('utf-8', 'ignore'),
+                          'aaaa' + res + 'bbbb')
+
+    def test_invalid_start_byte(self):
+        """
+        Test that an 'invalid start byte' error is raised when the first byte
+        is not in the ASCII range or is not a valid start byte of a 2-, 3-, or
+        4-bytes sequence. The invalid start byte is replaced with a single
+        U+FFFD when errors='replace'.
+        E.g. <80> is a continuation byte and can appear only after a start byte.
+        """
+        FFFD = '\ufffd'
+        for byte in b'\x80\xA0\x9F\xBF\xC0\xC1\xF5\xFF':
+            self.assertCorrectUTF8Decoding(bytes([byte]), '\ufffd',
+                                           'invalid start byte')
+
+    def test_unexpected_end_of_data(self):
+        """
+        Test that an 'unexpected end of data' error is raised when the string
+        ends after a start byte of a 2-, 3-, or 4-bytes sequence without having
+        enough continuation bytes.  The incomplete sequence is replaced with a
+        single U+FFFD when errors='replace'.
+        E.g. in the sequence <F3 80 80>, F3 is the start byte of a 4-bytes
+        sequence, but it's followed by only 2 valid continuation bytes and the
+        last continuation bytes is missing.
+        Note: the continuation bytes must be all valid, if one of them is
+        invalid another error will be raised.
+        """
+        sequences = [
+            'C2', 'DF',
+            'E0 A0', 'E0 BF', 'E1 80', 'E1 BF', 'EC 80', 'EC BF',
+            'ED 80', 'ED 9F', 'EE 80', 'EE BF', 'EF 80', 'EF BF',
+            'F0 90', 'F0 BF', 'F0 90 80', 'F0 90 BF', 'F0 BF 80', 'F0 BF BF',
+            'F1 80', 'F1 BF', 'F1 80 80', 'F1 80 BF', 'F1 BF 80', 'F1 BF BF',
+            'F3 80', 'F3 BF', 'F3 80 80', 'F3 80 BF', 'F3 BF 80', 'F3 BF BF',
+            'F4 80', 'F4 8F', 'F4 80 80', 'F4 80 BF', 'F4 8F 80', 'F4 8F BF'
+        ]
+        FFFD = '\ufffd'
+        for seq in sequences:
+            self.assertCorrectUTF8Decoding(self.to_bytestring(seq), '\ufffd',
+                                           'unexpected end of data')
+
+    def test_invalid_cb_for_2bytes_seq(self):
+        """
+        Test that an 'invalid continuation byte' error is raised when the
+        continuation byte of a 2-bytes sequence is invalid.  The start byte
+        is replaced by a single U+FFFD and the second byte is handled
+        separately when errors='replace'.
+        E.g. in the sequence <C2 41>, C2 is the start byte of a 2-bytes
+        sequence, but 41 is not a valid continuation byte because it's the
+        ASCII letter 'A'.
+        """
+        FFFD = '\ufffd'
+        FFFDx2 = FFFD * 2
+        sequences = [
+            ('C2 00', FFFD+'\x00'), ('C2 7F', FFFD+'\x7f'),
+            ('C2 C0', FFFDx2), ('C2 FF', FFFDx2),
+            ('DF 00', FFFD+'\x00'), ('DF 7F', FFFD+'\x7f'),
+            ('DF C0', FFFDx2), ('DF FF', FFFDx2),
+        ]
+        for seq, res in sequences:
+            self.assertCorrectUTF8Decoding(self.to_bytestring(seq), res,
+                                           'invalid continuation byte')
+
+    def test_invalid_cb_for_3bytes_seq(self):
+        """
+        Test that an 'invalid continuation byte' error is raised when the
+        continuation byte(s) of a 3-bytes sequence are invalid.  When
+        errors='replace', if the first continuation byte is valid, the first
+        two bytes (start byte + 1st cb) are replaced by a single U+FFFD and the
+        third byte is handled separately, otherwise only the start byte is
+        replaced with a U+FFFD and the other continuation bytes are handled
+        separately.
+        E.g. in the sequence <E1 80 41>, E1 is the start byte of a 3-bytes
+        sequence, 80 is a valid continuation byte, but 41 is not a valid cb
+        because it's the ASCII letter 'A'.
+        Note: when the start byte is E0 or ED, the valid ranges for the first
+        continuation byte are limited to A0..BF and 80..9F respectively.
+        Python 2 used to consider all the bytes in range 80..BF valid when the
+        start byte was ED.  This is fixed in Python 3.
+        """
+        FFFD = '\ufffd'
+        FFFDx2 = FFFD * 2
+        sequences = [
+            ('E0 00', FFFD+'\x00'), ('E0 7F', FFFD+'\x7f'), ('E0 80', FFFDx2),
+            ('E0 9F', FFFDx2), ('E0 C0', FFFDx2), ('E0 FF', FFFDx2),
+            ('E0 A0 00', FFFD+'\x00'), ('E0 A0 7F', FFFD+'\x7f'),
+            ('E0 A0 C0', FFFDx2), ('E0 A0 FF', FFFDx2),
+            ('E0 BF 00', FFFD+'\x00'), ('E0 BF 7F', FFFD+'\x7f'),
+            ('E0 BF C0', FFFDx2), ('E0 BF FF', FFFDx2), ('E1 00', FFFD+'\x00'),
+            ('E1 7F', FFFD+'\x7f'), ('E1 C0', FFFDx2), ('E1 FF', FFFDx2),
+            ('E1 80 00', FFFD+'\x00'), ('E1 80 7F', FFFD+'\x7f'),
+            ('E1 80 C0', FFFDx2), ('E1 80 FF', FFFDx2),
+            ('E1 BF 00', FFFD+'\x00'), ('E1 BF 7F', FFFD+'\x7f'),
+            ('E1 BF C0', FFFDx2), ('E1 BF FF', FFFDx2), ('EC 00', FFFD+'\x00'),
+            ('EC 7F', FFFD+'\x7f'), ('EC C0', FFFDx2), ('EC FF', FFFDx2),
+            ('EC 80 00', FFFD+'\x00'), ('EC 80 7F', FFFD+'\x7f'),
+            ('EC 80 C0', FFFDx2), ('EC 80 FF', FFFDx2),
+            ('EC BF 00', FFFD+'\x00'), ('EC BF 7F', FFFD+'\x7f'),
+            ('EC BF C0', FFFDx2), ('EC BF FF', FFFDx2), ('ED 00', FFFD+'\x00'),
+            ('ED 7F', FFFD+'\x7f'),
+            ('ED A0', FFFDx2), ('ED BF', FFFDx2), # see note ^
+            ('ED C0', FFFDx2), ('ED FF', FFFDx2), ('ED 80 00', FFFD+'\x00'),
+            ('ED 80 7F', FFFD+'\x7f'), ('ED 80 C0', FFFDx2),
+            ('ED 80 FF', FFFDx2), ('ED 9F 00', FFFD+'\x00'),
+            ('ED 9F 7F', FFFD+'\x7f'), ('ED 9F C0', FFFDx2),
+            ('ED 9F FF', FFFDx2), ('EE 00', FFFD+'\x00'),
+            ('EE 7F', FFFD+'\x7f'), ('EE C0', FFFDx2), ('EE FF', FFFDx2),
+            ('EE 80 00', FFFD+'\x00'), ('EE 80 7F', FFFD+'\x7f'),
+            ('EE 80 C0', FFFDx2), ('EE 80 FF', FFFDx2),
+            ('EE BF 00', FFFD+'\x00'), ('EE BF 7F', FFFD+'\x7f'),
+            ('EE BF C0', FFFDx2), ('EE BF FF', FFFDx2), ('EF 00', FFFD+'\x00'),
+            ('EF 7F', FFFD+'\x7f'), ('EF C0', FFFDx2), ('EF FF', FFFDx2),
+            ('EF 80 00', FFFD+'\x00'), ('EF 80 7F', FFFD+'\x7f'),
+            ('EF 80 C0', FFFDx2), ('EF 80 FF', FFFDx2),
+            ('EF BF 00', FFFD+'\x00'), ('EF BF 7F', FFFD+'\x7f'),
+            ('EF BF C0', FFFDx2), ('EF BF FF', FFFDx2),
+        ]
+        for seq, res in sequences:
+            self.assertCorrectUTF8Decoding(self.to_bytestring(seq), res,
+                                           'invalid continuation byte')
+
+    def test_invalid_cb_for_4bytes_seq(self):
+        """
+        Test that an 'invalid continuation byte' error is raised when the
+        continuation byte(s) of a 4-bytes sequence are invalid.  When
+        errors='replace',the start byte and all the following valid
+        continuation bytes are replaced with a single U+FFFD, and all the bytes
+        starting from the first invalid continuation bytes (included) are
+        handled separately.
+        E.g. in the sequence <E1 80 41>, E1 is the start byte of a 3-bytes
+        sequence, 80 is a valid continuation byte, but 41 is not a valid cb
+        because it's the ASCII letter 'A'.
+        Note: when the start byte is E0 or ED, the valid ranges for the first
+        continuation byte are limited to A0..BF and 80..9F respectively.
+        However, when the start byte is ED, Python 2 considers all the bytes
+        in range 80..BF valid.  This is fixed in Python 3.
+        """
+        FFFD = '\ufffd'
+        FFFDx2 = FFFD * 2
+        sequences = [
+            ('F0 00', FFFD+'\x00'), ('F0 7F', FFFD+'\x7f'), ('F0 80', FFFDx2),
+            ('F0 8F', FFFDx2), ('F0 C0', FFFDx2), ('F0 FF', FFFDx2),
+            ('F0 90 00', FFFD+'\x00'), ('F0 90 7F', FFFD+'\x7f'),
+            ('F0 90 C0', FFFDx2), ('F0 90 FF', FFFDx2),
+            ('F0 BF 00', FFFD+'\x00'), ('F0 BF 7F', FFFD+'\x7f'),
+            ('F0 BF C0', FFFDx2), ('F0 BF FF', FFFDx2),
+            ('F0 90 80 00', FFFD+'\x00'), ('F0 90 80 7F', FFFD+'\x7f'),
+            ('F0 90 80 C0', FFFDx2), ('F0 90 80 FF', FFFDx2),
+            ('F0 90 BF 00', FFFD+'\x00'), ('F0 90 BF 7F', FFFD+'\x7f'),
+            ('F0 90 BF C0', FFFDx2), ('F0 90 BF FF', FFFDx2),
+            ('F0 BF 80 00', FFFD+'\x00'), ('F0 BF 80 7F', FFFD+'\x7f'),
+            ('F0 BF 80 C0', FFFDx2), ('F0 BF 80 FF', FFFDx2),
+            ('F0 BF BF 00', FFFD+'\x00'), ('F0 BF BF 7F', FFFD+'\x7f'),
+            ('F0 BF BF C0', FFFDx2), ('F0 BF BF FF', FFFDx2),
+            ('F1 00', FFFD+'\x00'), ('F1 7F', FFFD+'\x7f'), ('F1 C0', FFFDx2),
+            ('F1 FF', FFFDx2), ('F1 80 00', FFFD+'\x00'),
+            ('F1 80 7F', FFFD+'\x7f'), ('F1 80 C0', FFFDx2),
+            ('F1 80 FF', FFFDx2), ('F1 BF 00', FFFD+'\x00'),
+            ('F1 BF 7F', FFFD+'\x7f'), ('F1 BF C0', FFFDx2),
+            ('F1 BF FF', FFFDx2), ('F1 80 80 00', FFFD+'\x00'),
+            ('F1 80 80 7F', FFFD+'\x7f'), ('F1 80 80 C0', FFFDx2),
+            ('F1 80 80 FF', FFFDx2), ('F1 80 BF 00', FFFD+'\x00'),
+            ('F1 80 BF 7F', FFFD+'\x7f'), ('F1 80 BF C0', FFFDx2),
+            ('F1 80 BF FF', FFFDx2), ('F1 BF 80 00', FFFD+'\x00'),
+            ('F1 BF 80 7F', FFFD+'\x7f'), ('F1 BF 80 C0', FFFDx2),
+            ('F1 BF 80 FF', FFFDx2), ('F1 BF BF 00', FFFD+'\x00'),
+            ('F1 BF BF 7F', FFFD+'\x7f'), ('F1 BF BF C0', FFFDx2),
+            ('F1 BF BF FF', FFFDx2), ('F3 00', FFFD+'\x00'),
+            ('F3 7F', FFFD+'\x7f'), ('F3 C0', FFFDx2), ('F3 FF', FFFDx2),
+            ('F3 80 00', FFFD+'\x00'), ('F3 80 7F', FFFD+'\x7f'),
+            ('F3 80 C0', FFFDx2), ('F3 80 FF', FFFDx2),
+            ('F3 BF 00', FFFD+'\x00'), ('F3 BF 7F', FFFD+'\x7f'),
+            ('F3 BF C0', FFFDx2), ('F3 BF FF', FFFDx2),
+            ('F3 80 80 00', FFFD+'\x00'), ('F3 80 80 7F', FFFD+'\x7f'),
+            ('F3 80 80 C0', FFFDx2), ('F3 80 80 FF', FFFDx2),
+            ('F3 80 BF 00', FFFD+'\x00'), ('F3 80 BF 7F', FFFD+'\x7f'),
+            ('F3 80 BF C0', FFFDx2), ('F3 80 BF FF', FFFDx2),
+            ('F3 BF 80 00', FFFD+'\x00'), ('F3 BF 80 7F', FFFD+'\x7f'),
+            ('F3 BF 80 C0', FFFDx2), ('F3 BF 80 FF', FFFDx2),
+            ('F3 BF BF 00', FFFD+'\x00'), ('F3 BF BF 7F', FFFD+'\x7f'),
+            ('F3 BF BF C0', FFFDx2), ('F3 BF BF FF', FFFDx2),
+            ('F4 00', FFFD+'\x00'), ('F4 7F', FFFD+'\x7f'), ('F4 90', FFFDx2),
+            ('F4 BF', FFFDx2), ('F4 C0', FFFDx2), ('F4 FF', FFFDx2),
+            ('F4 80 00', FFFD+'\x00'), ('F4 80 7F', FFFD+'\x7f'),
+            ('F4 80 C0', FFFDx2), ('F4 80 FF', FFFDx2),
+            ('F4 8F 00', FFFD+'\x00'), ('F4 8F 7F', FFFD+'\x7f'),
+            ('F4 8F C0', FFFDx2), ('F4 8F FF', FFFDx2),
+            ('F4 80 80 00', FFFD+'\x00'), ('F4 80 80 7F', FFFD+'\x7f'),
+            ('F4 80 80 C0', FFFDx2), ('F4 80 80 FF', FFFDx2),
+            ('F4 80 BF 00', FFFD+'\x00'), ('F4 80 BF 7F', FFFD+'\x7f'),
+            ('F4 80 BF C0', FFFDx2), ('F4 80 BF FF', FFFDx2),
+            ('F4 8F 80 00', FFFD+'\x00'), ('F4 8F 80 7F', FFFD+'\x7f'),
+            ('F4 8F 80 C0', FFFDx2), ('F4 8F 80 FF', FFFDx2),
+            ('F4 8F BF 00', FFFD+'\x00'), ('F4 8F BF 7F', FFFD+'\x7f'),
+            ('F4 8F BF C0', FFFDx2), ('F4 8F BF FF', FFFDx2)
+        ]
+        for seq, res in sequences:
+            self.assertCorrectUTF8Decoding(self.to_bytestring(seq), res,
+                                           'invalid continuation byte')
+
     def test_codecs_idna(self):
         # Test whether trailing dot is preserved
         self.assertEqual("www.python.org.".encode("idna"), b"www.python.org.")

Misc/NEWS

@@ -12,6 +12,10 @@ What's New in Python 3.3.1?
 Core and Builtins
 -----------------
 
+- Issue #8271: the utf-8 decoder now outputs the correct number of U+FFFD
+  characters when used with the 'replace' error handler on invalid utf-8
+  sequences.  Patch by Serhiy Storchaka, tests by Ezio Melotti.
+
 - Issue #5765: Apply a hard recursion limit in the compiler instead of
   blowing the stack and segfaulting. Initial patch by Andrea Griffini.
 
@@ -33,7 +37,7 @@ Core and Builtins
 - Issue #16271: Fix strange bugs that resulted from __qualname__ appearing in a
   class's __dict__ and on type.
 
-- Issue #16197: Update winreg docstrings and documentation to match code.      
+- Issue #16197: Update winreg docstrings and documentation to match code.
   Patch by Zachary Ware.
 
 - Issue #16241: Document -X faulthandler command line option.

Objects/stringlib/codecs.h

@@ -91,15 +91,14 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end,
             }
         }
 
-        if (ch < 0xC2) {
-            /* invalid sequence
-               \x80-\xBF -- continuation byte
-               \xC0-\xC1 -- fake 0000-007F */
-            goto InvalidStart;
-        }
-
         if (ch < 0xE0) {
             /* \xC2\x80-\xDF\xBF -- 0080-07FF */
+            if (ch < 0xC2) {
+                /* invalid sequence
+                \x80-\xBF -- continuation byte
+                \xC0-\xC1 -- fake 0000-007F */
+                goto InvalidStart;
+            }
             Py_UCS4 ch2;
             if (end - s < 2) {
                 /* unexpected end of data: the caller will decide whether
@@ -109,14 +108,15 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end,
             ch2 = (unsigned char)s[1];
             if (!IS_CONTINUATION_BYTE(ch2))
                 /* invalid continuation byte */
-                goto InvalidContinuation;
+                goto InvalidContinuation1;
             ch = (ch << 6) + ch2 -
                  ((0xC0 << 6) + 0x80);
             assert ((ch > 0x007F) && (ch <= 0x07FF));
             s += 2;
             if (STRINGLIB_MAX_CHAR <= 0x007F ||
                 (STRINGLIB_MAX_CHAR < 0x07FF && ch > STRINGLIB_MAX_CHAR))
-                goto Overflow;
+                /* Out-of-range */
+                goto Return;
             *p++ = ch;
             continue;
         }
@@ -127,28 +127,37 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end,
             if (end - s < 3) {
                 /* unexpected end of data: the caller will decide whether
                    it's an error or not */
+                if (end - s < 2)
+                    break;
+                ch2 = (unsigned char)s[1];
+                if (!IS_CONTINUATION_BYTE(ch2) ||
+                    (ch2 < 0xA0 ? ch == 0xE0 : ch == 0xED))
+                    /* for clarification see comments below */
+                    goto InvalidContinuation1;
                 break;
             }
             ch2 = (unsigned char)s[1];
             ch3 = (unsigned char)s[2];
-            if (!IS_CONTINUATION_BYTE(ch2) ||
+            if (!IS_CONTINUATION_BYTE(ch2)) {
-                !IS_CONTINUATION_BYTE(ch3)) {
                 /* invalid continuation byte */
-                goto InvalidContinuation;
+                goto InvalidContinuation1;
             }
             if (ch == 0xE0) {
                 if (ch2 < 0xA0)
                     /* invalid sequence
                        \xE0\x80\x80-\xE0\x9F\xBF -- fake 0000-0800 */
-                    goto InvalidContinuation;
+                    goto InvalidContinuation1;
-            }
+            } else if (ch == 0xED && ch2 >= 0xA0) {
-            else if (ch == 0xED && ch2 > 0x9F) {
                 /* Decoding UTF-8 sequences in range \xED\xA0\x80-\xED\xBF\xBF
                    will result in surrogates in range D800-DFFF. Surrogates are
                    not valid UTF-8 so they are rejected.
                    See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf
                    (table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt */
-                goto InvalidContinuation;
+                goto InvalidContinuation1;
+            }
+            if (!IS_CONTINUATION_BYTE(ch3)) {
+                /* invalid continuation byte */
+                goto InvalidContinuation2;
             }
             ch = (ch << 12) + (ch2 << 6) + ch3 -
                  ((0xE0 << 12) + (0x80 << 6) + 0x80);
@@ -156,7 +165,8 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end,
             s += 3;
             if (STRINGLIB_MAX_CHAR <= 0x07FF ||
                 (STRINGLIB_MAX_CHAR < 0xFFFF && ch > STRINGLIB_MAX_CHAR))
-                goto Overflow;
+                /* Out-of-range */
+                goto Return;
             *p++ = ch;
             continue;
         }
@@ -167,27 +177,44 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end,
             if (end - s < 4) {
                 /* unexpected end of data: the caller will decide whether
                    it's an error or not */
+                if (end - s < 2)
+                    break;
+                ch2 = (unsigned char)s[1];
+                if (!IS_CONTINUATION_BYTE(ch2) ||
+                    (ch2 < 0x90 ? ch == 0xF0 : ch == 0xF4))
+                    /* for clarification see comments below */
+                    goto InvalidContinuation1;
+                if (end - s < 3)
+                    break;
+                ch3 = (unsigned char)s[2];
+                if (!IS_CONTINUATION_BYTE(ch3))
+                    goto InvalidContinuation2;
                 break;
             }
             ch2 = (unsigned char)s[1];
             ch3 = (unsigned char)s[2];
             ch4 = (unsigned char)s[3];
-            if (!IS_CONTINUATION_BYTE(ch2) ||
+            if (!IS_CONTINUATION_BYTE(ch2)) {
-                !IS_CONTINUATION_BYTE(ch3) ||
-                !IS_CONTINUATION_BYTE(ch4)) {
                 /* invalid continuation byte */
-                goto InvalidContinuation;
+                goto InvalidContinuation1;
             }
             if (ch == 0xF0) {
                 if (ch2 < 0x90)
                     /* invalid sequence
-                       \xF0\x80\x80\x80-\xF0\x80\xBF\xBF -- fake 0000-FFFF */
+                       \xF0\x80\x80\x80-\xF0\x8F\xBF\xBF -- fake 0000-FFFF */
-                    goto InvalidContinuation;
+                    goto InvalidContinuation1;
-            }
+            } else if (ch == 0xF4 && ch2 >= 0x90) {
-            else if (ch == 0xF4 && ch2 > 0x8F) {
                 /* invalid sequence
                    \xF4\x90\x80\80- -- 110000- overflow */
-                goto InvalidContinuation;
+                goto InvalidContinuation1;
+            }
+            if (!IS_CONTINUATION_BYTE(ch3)) {
+                /* invalid continuation byte */
+                goto InvalidContinuation2;
+            }
+            if (!IS_CONTINUATION_BYTE(ch4)) {
+                /* invalid continuation byte */
+                goto InvalidContinuation3;
             }
             ch = (ch << 18) + (ch2 << 12) + (ch3 << 6) + ch4 -
                  ((0xF0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80);
@@ -195,14 +222,14 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end,
             s += 4;
             if (STRINGLIB_MAX_CHAR <= 0xFFFF ||
                 (STRINGLIB_MAX_CHAR < 0x10FFFF && ch > STRINGLIB_MAX_CHAR))
-                goto Overflow;
+                /* Out-of-range */
+                goto Return;
             *p++ = ch;
             continue;
         }
         goto InvalidStart;
     }
     ch = 0;
-Overflow:
 Return:
     *inptr = s;
     *outpos = p - dest;
@@ -210,13 +237,18 @@ Return:
 InvalidStart:
     ch = 1;
     goto Return;
-InvalidContinuation:
+InvalidContinuation1:
     ch = 2;
     goto Return;
+InvalidContinuation2:
+    ch = 3;
+    goto Return;
+InvalidContinuation3:
+    ch = 4;
+    goto Return;
 }
 
 #undef ASCII_CHAR_MASK
-#undef IS_CONTINUATION_BYTE
 
 
 /* UTF-8 encoder specialized for a Unicode kind to avoid the slow

Objects/unicodeobject.c

@@ -4759,9 +4759,7 @@ PyUnicode_DecodeUTF8Stateful(const char *s,
                 goto End;
             errmsg = "unexpected end of data";
             startinpos = s - starts;
-            endinpos = startinpos + 1;
+            endinpos = end - starts;
-            while (endinpos < size && (starts[endinpos] & 0xC0) == 0x80)
-                endinpos++;
             break;
         case 1:
             errmsg = "invalid start byte";
@@ -4769,11 +4767,11 @@ PyUnicode_DecodeUTF8Stateful(const char *s,
             endinpos = startinpos + 1;
             break;
         case 2:
+        case 3:
+        case 4:
             errmsg = "invalid continuation byte";
             startinpos = s - starts;
-            endinpos = startinpos + 1;
+            endinpos = startinpos + ch - 1;
-            while (endinpos < size && (starts[endinpos] & 0xC0) == 0x80)
-                endinpos++;
             break;
         default:
             if (unicode_putchar(&unicode, &outpos, ch) < 0)