Add tests for the C tokenizer and expose it as a private module (GH-27924)

Lib/test/test_tokenize.py

@@ -3,7 +3,7 @@ from test.support import os_helper
 from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP,
                      STRING, ENDMARKER, ENCODING, tok_name, detect_encoding,
                      open as tokenize_open, Untokenizer, generate_tokens,
-                     NEWLINE)
+                     NEWLINE, _generate_tokens_from_c_tokenizer)
 from io import BytesIO, StringIO
 import unittest
 from unittest import TestCase, mock
@@ -12,7 +12,6 @@ from test.test_grammar import (VALID_UNDERSCORE_LITERALS,
 import os
 import token
 
-
 # Converts a source string into a list of textual representation
 # of the tokens such as:
 # `    NAME       'if'          (1, 0) (1, 2)`
@@ -1654,5 +1653,865 @@ class TestRoundtrip(TestCase):
         self.check_roundtrip(code)
 
 
+class CTokenizeTest(TestCase):
+    def check_tokenize(self, s, expected):
+        # Format the tokens in s in a table format.
+        # The ENDMARKER and final NEWLINE are omitted.
+        with self.subTest(source=s):
+            result = stringify_tokens_from_source(
+                _generate_tokens_from_c_tokenizer(s), s
+            )
+            self.assertEqual(result, expected.rstrip().splitlines())
+
+    def test_int(self):
+
+        self.check_tokenize('0xff <= 255', """\
+    NUMBER     '0xff'        (1, 0) (1, 4)
+    LESSEQUAL  '<='          (1, 5) (1, 7)
+    NUMBER     '255'         (1, 8) (1, 11)
+    """)
+
+        self.check_tokenize('0b10 <= 255', """\
+    NUMBER     '0b10'        (1, 0) (1, 4)
+    LESSEQUAL  '<='          (1, 5) (1, 7)
+    NUMBER     '255'         (1, 8) (1, 11)
+    """)
+
+        self.check_tokenize('0o123 <= 0O123', """\
+    NUMBER     '0o123'       (1, 0) (1, 5)
+    LESSEQUAL  '<='          (1, 6) (1, 8)
+    NUMBER     '0O123'       (1, 9) (1, 14)
+    """)
+
+        self.check_tokenize('1234567 > ~0x15', """\
+    NUMBER     '1234567'     (1, 0) (1, 7)
+    GREATER    '>'           (1, 8) (1, 9)
+    TILDE      '~'           (1, 10) (1, 11)
+    NUMBER     '0x15'        (1, 11) (1, 15)
+    """)
+
+        self.check_tokenize('2134568 != 1231515', """\
+    NUMBER     '2134568'     (1, 0) (1, 7)
+    NOTEQUAL   '!='          (1, 8) (1, 10)
+    NUMBER     '1231515'     (1, 11) (1, 18)
+    """)
+
+        self.check_tokenize('(-124561-1) & 200000000', """\
+    LPAR       '('           (1, 0) (1, 1)
+    MINUS      '-'           (1, 1) (1, 2)
+    NUMBER     '124561'      (1, 2) (1, 8)
+    MINUS      '-'           (1, 8) (1, 9)
+    NUMBER     '1'           (1, 9) (1, 10)
+    RPAR       ')'           (1, 10) (1, 11)
+    AMPER      '&'           (1, 12) (1, 13)
+    NUMBER     '200000000'   (1, 14) (1, 23)
+    """)
+
+        self.check_tokenize('0xdeadbeef != -1', """\
+    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
+    NOTEQUAL   '!='          (1, 11) (1, 13)
+    MINUS      '-'           (1, 14) (1, 15)
+    NUMBER     '1'           (1, 15) (1, 16)
+    """)
+
+        self.check_tokenize('0xdeadc0de & 12345', """\
+    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
+    AMPER      '&'           (1, 11) (1, 12)
+    NUMBER     '12345'       (1, 13) (1, 18)
+    """)
+
+        self.check_tokenize('0xFF & 0x15 | 1234', """\
+    NUMBER     '0xFF'        (1, 0) (1, 4)
+    AMPER      '&'           (1, 5) (1, 6)
+    NUMBER     '0x15'        (1, 7) (1, 11)
+    VBAR       '|'           (1, 12) (1, 13)
+    NUMBER     '1234'        (1, 14) (1, 18)
+    """)
+
+    def test_float(self):
+
+        self.check_tokenize('x = 3.14159', """\
+    NAME       'x'           (1, 0) (1, 1)
+    EQUAL      '='           (1, 2) (1, 3)
+    NUMBER     '3.14159'     (1, 4) (1, 11)
+    """)
+
+        self.check_tokenize('x = 314159.', """\
+    NAME       'x'           (1, 0) (1, 1)
+    EQUAL      '='           (1, 2) (1, 3)
+    NUMBER     '314159.'     (1, 4) (1, 11)
+    """)
+
+        self.check_tokenize('x = .314159', """\
+    NAME       'x'           (1, 0) (1, 1)
+    EQUAL      '='           (1, 2) (1, 3)
+    NUMBER     '.314159'     (1, 4) (1, 11)
+    """)
+
+        self.check_tokenize('x = 3e14159', """\
+    NAME       'x'           (1, 0) (1, 1)
+    EQUAL      '='           (1, 2) (1, 3)
+    NUMBER     '3e14159'     (1, 4) (1, 11)
+    """)
+
+        self.check_tokenize('x = 3E123', """\
+    NAME       'x'           (1, 0) (1, 1)
+    EQUAL      '='           (1, 2) (1, 3)
+    NUMBER     '3E123'       (1, 4) (1, 9)
+    """)
+
+        self.check_tokenize('x+y = 3e-1230', """\
+    NAME       'x'           (1, 0) (1, 1)
+    PLUS       '+'           (1, 1) (1, 2)
+    NAME       'y'           (1, 2) (1, 3)
+    EQUAL      '='           (1, 4) (1, 5)
+    NUMBER     '3e-1230'     (1, 6) (1, 13)
+    """)
+
+        self.check_tokenize('x = 3.14e159', """\
+    NAME       'x'           (1, 0) (1, 1)
+    EQUAL      '='           (1, 2) (1, 3)
+    NUMBER     '3.14e159'    (1, 4) (1, 12)
+    """)
+
+    def test_string(self):
+
+        self.check_tokenize('x = \'\'; y = ""', """\
+    NAME       'x'           (1, 0) (1, 1)
+    EQUAL      '='           (1, 2) (1, 3)
+    STRING     "''"          (1, 4) (1, 6)
+    SEMI       ';'           (1, 6) (1, 7)
+    NAME       'y'           (1, 8) (1, 9)
+    EQUAL      '='           (1, 10) (1, 11)
+    STRING     '""'          (1, 12) (1, 14)
+    """)
+
+        self.check_tokenize('x = \'"\'; y = "\'"', """\
+    NAME       'x'           (1, 0) (1, 1)
+    EQUAL      '='           (1, 2) (1, 3)
+    STRING     '\\'"\\''       (1, 4) (1, 7)
+    SEMI       ';'           (1, 7) (1, 8)
+    NAME       'y'           (1, 9) (1, 10)
+    EQUAL      '='           (1, 11) (1, 12)
+    STRING     '"\\'"'        (1, 13) (1, 16)
+    """)
+
+        self.check_tokenize('x = "doesn\'t "shrink", does it"', """\
+    NAME       'x'           (1, 0) (1, 1)
+    EQUAL      '='           (1, 2) (1, 3)
+    STRING     '"doesn\\'t "' (1, 4) (1, 14)
+    NAME       'shrink'      (1, 14) (1, 20)
+    STRING     '", does it"' (1, 20) (1, 31)
+    """)
+
+        self.check_tokenize("x = 'abc' + 'ABC'", """\
+    NAME       'x'           (1, 0) (1, 1)
+    EQUAL      '='           (1, 2) (1, 3)
+    STRING     "'abc'"       (1, 4) (1, 9)
+    PLUS       '+'           (1, 10) (1, 11)
+    STRING     "'ABC'"       (1, 12) (1, 17)
+    """)
+
+        self.check_tokenize('y = "ABC" + "ABC"', """\
+    NAME       'y'           (1, 0) (1, 1)
+    EQUAL      '='           (1, 2) (1, 3)
+    STRING     '"ABC"'       (1, 4) (1, 9)
+    PLUS       '+'           (1, 10) (1, 11)
+    STRING     '"ABC"'       (1, 12) (1, 17)
+    """)
+
+        self.check_tokenize("x = r'abc' + r'ABC' + R'ABC' + R'ABC'", """\
+    NAME       'x'           (1, 0) (1, 1)
+    EQUAL      '='           (1, 2) (1, 3)
+    STRING     "r'abc'"      (1, 4) (1, 10)
+    PLUS       '+'           (1, 11) (1, 12)
+    STRING     "r'ABC'"      (1, 13) (1, 19)
+    PLUS       '+'           (1, 20) (1, 21)
+    STRING     "R'ABC'"      (1, 22) (1, 28)
+    PLUS       '+'           (1, 29) (1, 30)
+    STRING     "R'ABC'"      (1, 31) (1, 37)
+    """)
+
+        self.check_tokenize('y = r"abc" + r"ABC" + R"ABC" + R"ABC"', """\
+    NAME       'y'           (1, 0) (1, 1)
+    EQUAL      '='           (1, 2) (1, 3)
+    STRING     'r"abc"'      (1, 4) (1, 10)
+    PLUS       '+'           (1, 11) (1, 12)
+    STRING     'r"ABC"'      (1, 13) (1, 19)
+    PLUS       '+'           (1, 20) (1, 21)
+    STRING     'R"ABC"'      (1, 22) (1, 28)
+    PLUS       '+'           (1, 29) (1, 30)
+    STRING     'R"ABC"'      (1, 31) (1, 37)
+    """)
+
+        self.check_tokenize("u'abc' + U'abc'", """\
+    STRING     "u'abc'"      (1, 0) (1, 6)
+    PLUS       '+'           (1, 7) (1, 8)
+    STRING     "U'abc'"      (1, 9) (1, 15)
+    """)
+
+        self.check_tokenize('u"abc" + U"abc"', """\
+    STRING     'u"abc"'      (1, 0) (1, 6)
+    PLUS       '+'           (1, 7) (1, 8)
+    STRING     'U"abc"'      (1, 9) (1, 15)
+    """)
+
+        self.check_tokenize("b'abc' + B'abc'", """\
+    STRING     "b'abc'"      (1, 0) (1, 6)
+    PLUS       '+'           (1, 7) (1, 8)
+    STRING     "B'abc'"      (1, 9) (1, 15)
+    """)
+
+        self.check_tokenize('b"abc" + B"abc"', """\
+    STRING     'b"abc"'      (1, 0) (1, 6)
+    PLUS       '+'           (1, 7) (1, 8)
+    STRING     'B"abc"'      (1, 9) (1, 15)
+    """)
+
+        self.check_tokenize("br'abc' + bR'abc' + Br'abc' + BR'abc'", """\
+    STRING     "br'abc'"     (1, 0) (1, 7)
+    PLUS       '+'           (1, 8) (1, 9)
+    STRING     "bR'abc'"     (1, 10) (1, 17)
+    PLUS       '+'           (1, 18) (1, 19)
+    STRING     "Br'abc'"     (1, 20) (1, 27)
+    PLUS       '+'           (1, 28) (1, 29)
+    STRING     "BR'abc'"     (1, 30) (1, 37)
+    """)
+
+        self.check_tokenize('br"abc" + bR"abc" + Br"abc" + BR"abc"', """\
+    STRING     'br"abc"'     (1, 0) (1, 7)
+    PLUS       '+'           (1, 8) (1, 9)
+    STRING     'bR"abc"'     (1, 10) (1, 17)
+    PLUS       '+'           (1, 18) (1, 19)
+    STRING     'Br"abc"'     (1, 20) (1, 27)
+    PLUS       '+'           (1, 28) (1, 29)
+    STRING     'BR"abc"'     (1, 30) (1, 37)
+    """)
+
+        self.check_tokenize("rb'abc' + rB'abc' + Rb'abc' + RB'abc'", """\
+    STRING     "rb'abc'"     (1, 0) (1, 7)
+    PLUS       '+'           (1, 8) (1, 9)
+    STRING     "rB'abc'"     (1, 10) (1, 17)
+    PLUS       '+'           (1, 18) (1, 19)
+    STRING     "Rb'abc'"     (1, 20) (1, 27)
+    PLUS       '+'           (1, 28) (1, 29)
+    STRING     "RB'abc'"     (1, 30) (1, 37)
+    """)
+
+        self.check_tokenize('rb"abc" + rB"abc" + Rb"abc" + RB"abc"', """\
+    STRING     'rb"abc"'     (1, 0) (1, 7)
+    PLUS       '+'           (1, 8) (1, 9)
+    STRING     'rB"abc"'     (1, 10) (1, 17)
+    PLUS       '+'           (1, 18) (1, 19)
+    STRING     'Rb"abc"'     (1, 20) (1, 27)
+    PLUS       '+'           (1, 28) (1, 29)
+    STRING     'RB"abc"'     (1, 30) (1, 37)
+    """)
+
+        self.check_tokenize('"a\\\nde\\\nfg"', """\
+    STRING     '"a\\\\\\nde\\\\\\nfg"\' (1, 0) (3, 3)
+    """)
+
+        self.check_tokenize('u"a\\\nde"', """\
+    STRING     'u"a\\\\\\nde"\'  (1, 0) (2, 3)
+    """)
+
+        self.check_tokenize('rb"a\\\nd"', """\
+    STRING     'rb"a\\\\\\nd"\'  (1, 0) (2, 2)
+    """)
+
+        self.check_tokenize(r'"""a\
+b"""', """\
+    STRING     '\"\""a\\\\\\nb\"\""' (1, 0) (2, 4)
+    """)
+        self.check_tokenize(r'u"""a\
+b"""', """\
+    STRING     'u\"\""a\\\\\\nb\"\""' (1, 0) (2, 4)
+    """)
+        self.check_tokenize(r'rb"""a\
+b\
+c"""', """\
+    STRING     'rb"\""a\\\\\\nb\\\\\\nc"\""' (1, 0) (3, 4)
+    """)
+ 
+        self.check_tokenize('f"abc"', """\
+    STRING     'f"abc"'      (1, 0) (1, 6)
+    """)
+
+        self.check_tokenize('fR"a{b}c"', """\
+    STRING     'fR"a{b}c"'   (1, 0) (1, 9)
+    """)
+
+        self.check_tokenize('f"""abc"""', """\
+    STRING     'f\"\"\"abc\"\"\"'  (1, 0) (1, 10)
+    """)
+
+        self.check_tokenize(r'f"abc\
+def"', """\
+    STRING     'f"abc\\\\\\ndef"' (1, 0) (2, 4)
+    """)
+
+        self.check_tokenize(r'Rf"abc\
+def"', """\
+    STRING     'Rf"abc\\\\\\ndef"' (1, 0) (2, 4)
+    """)
+
+    def test_function(self):
+
+        self.check_tokenize('def d22(a, b, c=2, d=2, *k): pass', """\
+    NAME       'def'         (1, 0) (1, 3)
+    NAME       'd22'         (1, 4) (1, 7)
+    LPAR       '('           (1, 7) (1, 8)
+    NAME       'a'           (1, 8) (1, 9)
+    COMMA      ','           (1, 9) (1, 10)
+    NAME       'b'           (1, 11) (1, 12)
+    COMMA      ','           (1, 12) (1, 13)
+    NAME       'c'           (1, 14) (1, 15)
+    EQUAL      '='           (1, 15) (1, 16)
+    NUMBER     '2'           (1, 16) (1, 17)
+    COMMA      ','           (1, 17) (1, 18)
+    NAME       'd'           (1, 19) (1, 20)
+    EQUAL      '='           (1, 20) (1, 21)
+    NUMBER     '2'           (1, 21) (1, 22)
+    COMMA      ','           (1, 22) (1, 23)
+    STAR       '*'           (1, 24) (1, 25)
+    NAME       'k'           (1, 25) (1, 26)
+    RPAR       ')'           (1, 26) (1, 27)
+    COLON      ':'           (1, 27) (1, 28)
+    NAME       'pass'        (1, 29) (1, 33)
+    """)
+
+        self.check_tokenize('def d01v_(a=1, *k, **w): pass', """\
+    NAME       'def'         (1, 0) (1, 3)
+    NAME       'd01v_'       (1, 4) (1, 9)
+    LPAR       '('           (1, 9) (1, 10)
+    NAME       'a'           (1, 10) (1, 11)
+    EQUAL      '='           (1, 11) (1, 12)
+    NUMBER     '1'           (1, 12) (1, 13)
+    COMMA      ','           (1, 13) (1, 14)
+    STAR       '*'           (1, 15) (1, 16)
+    NAME       'k'           (1, 16) (1, 17)
+    COMMA      ','           (1, 17) (1, 18)
+    DOUBLESTAR '**'          (1, 19) (1, 21)
+    NAME       'w'           (1, 21) (1, 22)
+    RPAR       ')'           (1, 22) (1, 23)
+    COLON      ':'           (1, 23) (1, 24)
+    NAME       'pass'        (1, 25) (1, 29)
+    """)
+
+        self.check_tokenize('def d23(a: str, b: int=3) -> int: pass', """\
+    NAME       'def'         (1, 0) (1, 3)
+    NAME       'd23'         (1, 4) (1, 7)
+    LPAR       '('           (1, 7) (1, 8)
+    NAME       'a'           (1, 8) (1, 9)
+    COLON      ':'           (1, 9) (1, 10)
+    NAME       'str'         (1, 11) (1, 14)
+    COMMA      ','           (1, 14) (1, 15)
+    NAME       'b'           (1, 16) (1, 17)
+    COLON      ':'           (1, 17) (1, 18)
+    NAME       'int'         (1, 19) (1, 22)
+    EQUAL      '='           (1, 22) (1, 23)
+    NUMBER     '3'           (1, 23) (1, 24)
+    RPAR       ')'           (1, 24) (1, 25)
+    RARROW     '->'          (1, 26) (1, 28)
+    NAME       'int'         (1, 29) (1, 32)
+    COLON      ':'           (1, 32) (1, 33)
+    NAME       'pass'        (1, 34) (1, 38)
+    """)
+
+    def test_comparison(self):
+
+        self.check_tokenize("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != "
+                            "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass", """\
+    NAME       'if'          (1, 0) (1, 2)
+    NUMBER     '1'           (1, 3) (1, 4)
+    LESS       '<'           (1, 5) (1, 6)
+    NUMBER     '1'           (1, 7) (1, 8)
+    GREATER    '>'           (1, 9) (1, 10)
+    NUMBER     '1'           (1, 11) (1, 12)
+    EQEQUAL    '=='          (1, 13) (1, 15)
+    NUMBER     '1'           (1, 16) (1, 17)
+    GREATEREQUAL '>='          (1, 18) (1, 20)
+    NUMBER     '5'           (1, 21) (1, 22)
+    LESSEQUAL  '<='          (1, 23) (1, 25)
+    NUMBER     '0x15'        (1, 26) (1, 30)
+    LESSEQUAL  '<='          (1, 31) (1, 33)
+    NUMBER     '0x12'        (1, 34) (1, 38)
+    NOTEQUAL   '!='          (1, 39) (1, 41)
+    NUMBER     '1'           (1, 42) (1, 43)
+    NAME       'and'         (1, 44) (1, 47)
+    NUMBER     '5'           (1, 48) (1, 49)
+    NAME       'in'          (1, 50) (1, 52)
+    NUMBER     '1'           (1, 53) (1, 54)
+    NAME       'not'         (1, 55) (1, 58)
+    NAME       'in'          (1, 59) (1, 61)
+    NUMBER     '1'           (1, 62) (1, 63)
+    NAME       'is'          (1, 64) (1, 66)
+    NUMBER     '1'           (1, 67) (1, 68)
+    NAME       'or'          (1, 69) (1, 71)
+    NUMBER     '5'           (1, 72) (1, 73)
+    NAME       'is'          (1, 74) (1, 76)
+    NAME       'not'         (1, 77) (1, 80)
+    NUMBER     '1'           (1, 81) (1, 82)
+    COLON      ':'           (1, 82) (1, 83)
+    NAME       'pass'        (1, 84) (1, 88)
+    """)
+
+    def test_additive(self):
+
+        self.check_tokenize('x = 1 - y + 15 - 1 + 0x124 + z + a[5]', """\
+    NAME       'x'           (1, 0) (1, 1)
+    EQUAL      '='           (1, 2) (1, 3)
+    NUMBER     '1'           (1, 4) (1, 5)
+    MINUS      '-'           (1, 6) (1, 7)
+    NAME       'y'           (1, 8) (1, 9)
+    PLUS       '+'           (1, 10) (1, 11)
+    NUMBER     '15'          (1, 12) (1, 14)
+    MINUS      '-'           (1, 15) (1, 16)
+    NUMBER     '1'           (1, 17) (1, 18)
+    PLUS       '+'           (1, 19) (1, 20)
+    NUMBER     '0x124'       (1, 21) (1, 26)
+    PLUS       '+'           (1, 27) (1, 28)
+    NAME       'z'           (1, 29) (1, 30)
+    PLUS       '+'           (1, 31) (1, 32)
+    NAME       'a'           (1, 33) (1, 34)
+    LSQB       '['           (1, 34) (1, 35)
+    NUMBER     '5'           (1, 35) (1, 36)
+    RSQB       ']'           (1, 36) (1, 37)
+    """)
+
+    def test_multiplicative(self):
+
+        self.check_tokenize('x = 1//1*1/5*12%0x12@42', """\
+    NAME       'x'           (1, 0) (1, 1)
+    EQUAL      '='           (1, 2) (1, 3)
+    NUMBER     '1'           (1, 4) (1, 5)
+    DOUBLESLASH '//'          (1, 5) (1, 7)
+    NUMBER     '1'           (1, 7) (1, 8)
+    STAR       '*'           (1, 8) (1, 9)
+    NUMBER     '1'           (1, 9) (1, 10)
+    SLASH      '/'           (1, 10) (1, 11)
+    NUMBER     '5'           (1, 11) (1, 12)
+    STAR       '*'           (1, 12) (1, 13)
+    NUMBER     '12'          (1, 13) (1, 15)
+    PERCENT    '%'           (1, 15) (1, 16)
+    NUMBER     '0x12'        (1, 16) (1, 20)
+    AT         '@'           (1, 20) (1, 21)
+    NUMBER     '42'          (1, 21) (1, 23)
+    """)
+
+    def test_unary(self):
+
+        self.check_tokenize('~1 ^ 1 & 1 |1 ^ -1', """\
+    TILDE      '~'           (1, 0) (1, 1)
+    NUMBER     '1'           (1, 1) (1, 2)
+    CIRCUMFLEX '^'           (1, 3) (1, 4)
+    NUMBER     '1'           (1, 5) (1, 6)
+    AMPER      '&'           (1, 7) (1, 8)
+    NUMBER     '1'           (1, 9) (1, 10)
+    VBAR       '|'           (1, 11) (1, 12)
+    NUMBER     '1'           (1, 12) (1, 13)
+    CIRCUMFLEX '^'           (1, 14) (1, 15)
+    MINUS      '-'           (1, 16) (1, 17)
+    NUMBER     '1'           (1, 17) (1, 18)
+    """)
+
+        self.check_tokenize('-1*1/1+1*1//1 - ---1**1', """\
+    MINUS      '-'           (1, 0) (1, 1)
+    NUMBER     '1'           (1, 1) (1, 2)
+    STAR       '*'           (1, 2) (1, 3)
+    NUMBER     '1'           (1, 3) (1, 4)
+    SLASH      '/'           (1, 4) (1, 5)
+    NUMBER     '1'           (1, 5) (1, 6)
+    PLUS       '+'           (1, 6) (1, 7)
+    NUMBER     '1'           (1, 7) (1, 8)
+    STAR       '*'           (1, 8) (1, 9)
+    NUMBER     '1'           (1, 9) (1, 10)
+    DOUBLESLASH '//'          (1, 10) (1, 12)
+    NUMBER     '1'           (1, 12) (1, 13)
+    MINUS      '-'           (1, 14) (1, 15)
+    MINUS      '-'           (1, 16) (1, 17)
+    MINUS      '-'           (1, 17) (1, 18)
+    MINUS      '-'           (1, 18) (1, 19)
+    NUMBER     '1'           (1, 19) (1, 20)
+    DOUBLESTAR '**'          (1, 20) (1, 22)
+    NUMBER     '1'           (1, 22) (1, 23)
+    """)
+
+    def test_selector(self):
+
+        self.check_tokenize("import sys, time\nx = sys.modules['time'].time()", """\
+    NAME       'import'      (1, 0) (1, 6)
+    NAME       'sys'         (1, 7) (1, 10)
+    COMMA      ','           (1, 10) (1, 11)
+    NAME       'time'        (1, 12) (1, 16)
+    NEWLINE    ''            (1, 16) (1, 16)
+    NAME       'x'           (2, 0) (2, 1)
+    EQUAL      '='           (2, 2) (2, 3)
+    NAME       'sys'         (2, 4) (2, 7)
+    DOT        '.'           (2, 7) (2, 8)
+    NAME       'modules'     (2, 8) (2, 15)
+    LSQB       '['           (2, 15) (2, 16)
+    STRING     "'time'"      (2, 16) (2, 22)
+    RSQB       ']'           (2, 22) (2, 23)
+    DOT        '.'           (2, 23) (2, 24)
+    NAME       'time'        (2, 24) (2, 28)
+    LPAR       '('           (2, 28) (2, 29)
+    RPAR       ')'           (2, 29) (2, 30)
+    """)
+
+    def test_method(self):
+
+        self.check_tokenize('@staticmethod\ndef foo(x,y): pass', """\
+    AT         '@'           (1, 0) (1, 1)
+    NAME       'staticmethod' (1, 1) (1, 13)
+    NEWLINE    ''            (1, 13) (1, 13)
+    NAME       'def'         (2, 0) (2, 3)
+    NAME       'foo'         (2, 4) (2, 7)
+    LPAR       '('           (2, 7) (2, 8)
+    NAME       'x'           (2, 8) (2, 9)
+    COMMA      ','           (2, 9) (2, 10)
+    NAME       'y'           (2, 10) (2, 11)
+    RPAR       ')'           (2, 11) (2, 12)
+    COLON      ':'           (2, 12) (2, 13)
+    NAME       'pass'        (2, 14) (2, 18)
+    """)
+
+    def test_tabs(self):
+
+        self.check_tokenize('@staticmethod\ndef foo(x,y): pass', """\
+    AT         '@'           (1, 0) (1, 1)
+    NAME       'staticmethod' (1, 1) (1, 13)
+    NEWLINE    ''            (1, 13) (1, 13)
+    NAME       'def'         (2, 0) (2, 3)
+    NAME       'foo'         (2, 4) (2, 7)
+    LPAR       '('           (2, 7) (2, 8)
+    NAME       'x'           (2, 8) (2, 9)
+    COMMA      ','           (2, 9) (2, 10)
+    NAME       'y'           (2, 10) (2, 11)
+    RPAR       ')'           (2, 11) (2, 12)
+    COLON      ':'           (2, 12) (2, 13)
+    NAME       'pass'        (2, 14) (2, 18)
+    """)
+
+    def test_async(self):
+
+        self.check_tokenize('async = 1', """\
+    ASYNC      'async'       (1, 0) (1, 5)
+    EQUAL      '='           (1, 6) (1, 7)
+    NUMBER     '1'           (1, 8) (1, 9)
+    """)
+
+        self.check_tokenize('a = (async = 1)', """\
+    NAME       'a'           (1, 0) (1, 1)
+    EQUAL      '='           (1, 2) (1, 3)
+    LPAR       '('           (1, 4) (1, 5)
+    ASYNC      'async'       (1, 5) (1, 10)
+    EQUAL      '='           (1, 11) (1, 12)
+    NUMBER     '1'           (1, 13) (1, 14)
+    RPAR       ')'           (1, 14) (1, 15)
+    """)
+
+        self.check_tokenize('async()', """\
+    ASYNC      'async'       (1, 0) (1, 5)
+    LPAR       '('           (1, 5) (1, 6)
+    RPAR       ')'           (1, 6) (1, 7)
+    """)
+
+        self.check_tokenize('class async(Bar):pass', """\
+    NAME       'class'       (1, 0) (1, 5)
+    ASYNC      'async'       (1, 6) (1, 11)
+    LPAR       '('           (1, 11) (1, 12)
+    NAME       'Bar'         (1, 12) (1, 15)
+    RPAR       ')'           (1, 15) (1, 16)
+    COLON      ':'           (1, 16) (1, 17)
+    NAME       'pass'        (1, 17) (1, 21)
+    """)
+
+        self.check_tokenize('class async:pass', """\
+    NAME       'class'       (1, 0) (1, 5)
+    ASYNC      'async'       (1, 6) (1, 11)
+    COLON      ':'           (1, 11) (1, 12)
+    NAME       'pass'        (1, 12) (1, 16)
+    """)
+
+        self.check_tokenize('await = 1', """\
+    AWAIT      'await'       (1, 0) (1, 5)
+    EQUAL      '='           (1, 6) (1, 7)
+    NUMBER     '1'           (1, 8) (1, 9)
+    """)
+
+        self.check_tokenize('foo.async', """\
+    NAME       'foo'         (1, 0) (1, 3)
+    DOT        '.'           (1, 3) (1, 4)
+    ASYNC      'async'       (1, 4) (1, 9)
+    """)
+
+        self.check_tokenize('async for a in b: pass', """\
+    ASYNC      'async'       (1, 0) (1, 5)
+    NAME       'for'         (1, 6) (1, 9)
+    NAME       'a'           (1, 10) (1, 11)
+    NAME       'in'          (1, 12) (1, 14)
+    NAME       'b'           (1, 15) (1, 16)
+    COLON      ':'           (1, 16) (1, 17)
+    NAME       'pass'        (1, 18) (1, 22)
+    """)
+
+        self.check_tokenize('async with a as b: pass', """\
+    ASYNC      'async'       (1, 0) (1, 5)
+    NAME       'with'        (1, 6) (1, 10)
+    NAME       'a'           (1, 11) (1, 12)
+    NAME       'as'          (1, 13) (1, 15)
+    NAME       'b'           (1, 16) (1, 17)
+    COLON      ':'           (1, 17) (1, 18)
+    NAME       'pass'        (1, 19) (1, 23)
+    """)
+
+        self.check_tokenize('async.foo', """\
+    ASYNC      'async'       (1, 0) (1, 5)
+    DOT        '.'           (1, 5) (1, 6)
+    NAME       'foo'         (1, 6) (1, 9)
+    """)
+
+        self.check_tokenize('async', """\
+    ASYNC      'async'       (1, 0) (1, 5)
+    """)
+
+        self.check_tokenize('async\n#comment\nawait', """\
+    ASYNC      'async'       (1, 0) (1, 5)
+    NEWLINE    ''            (1, 5) (1, 5)
+    AWAIT      'await'       (3, 0) (3, 5)
+    """)
+
+        self.check_tokenize('async\n...\nawait', """\
+    ASYNC      'async'       (1, 0) (1, 5)
+    NEWLINE    ''            (1, 5) (1, 5)
+    ELLIPSIS   '...'         (2, 0) (2, 3)
+    NEWLINE    ''            (2, 3) (2, 3)
+    AWAIT      'await'       (3, 0) (3, 5)
+    """)
+
+        self.check_tokenize('async\nawait', """\
+    ASYNC      'async'       (1, 0) (1, 5)
+    NEWLINE    ''            (1, 5) (1, 5)
+    AWAIT      'await'       (2, 0) (2, 5)
+    """)
+
+        self.check_tokenize('foo.async + 1', """\
+    NAME       'foo'         (1, 0) (1, 3)
+    DOT        '.'           (1, 3) (1, 4)
+    ASYNC      'async'       (1, 4) (1, 9)
+    PLUS       '+'           (1, 10) (1, 11)
+    NUMBER     '1'           (1, 12) (1, 13)
+    """)
+
+        self.check_tokenize('async def foo(): pass', """\
+    ASYNC      'async'       (1, 0) (1, 5)
+    NAME       'def'         (1, 6) (1, 9)
+    NAME       'foo'         (1, 10) (1, 13)
+    LPAR       '('           (1, 13) (1, 14)
+    RPAR       ')'           (1, 14) (1, 15)
+    COLON      ':'           (1, 15) (1, 16)
+    NAME       'pass'        (1, 17) (1, 21)
+    """)
+
+        self.check_tokenize('''\
+async def foo():
+  def foo(await):
+    await = 1
+  if 1:
+    await
+async += 1
+''', """\
+    ASYNC      'async'       (1, 0) (1, 5)
+    NAME       'def'         (1, 6) (1, 9)
+    NAME       'foo'         (1, 10) (1, 13)
+    LPAR       '('           (1, 13) (1, 14)
+    RPAR       ')'           (1, 14) (1, 15)
+    COLON      ':'           (1, 15) (1, 16)
+    NEWLINE    ''            (1, 16) (1, 16)
+    INDENT     ''            (2, -1) (2, -1)
+    NAME       'def'         (2, 2) (2, 5)
+    NAME       'foo'         (2, 6) (2, 9)
+    LPAR       '('           (2, 9) (2, 10)
+    AWAIT      'await'       (2, 10) (2, 15)
+    RPAR       ')'           (2, 15) (2, 16)
+    COLON      ':'           (2, 16) (2, 17)
+    NEWLINE    ''            (2, 17) (2, 17)
+    INDENT     ''            (3, -1) (3, -1)
+    AWAIT      'await'       (3, 4) (3, 9)
+    EQUAL      '='           (3, 10) (3, 11)
+    NUMBER     '1'           (3, 12) (3, 13)
+    NEWLINE    ''            (3, 13) (3, 13)
+    DEDENT     ''            (4, -1) (4, -1)
+    NAME       'if'          (4, 2) (4, 4)
+    NUMBER     '1'           (4, 5) (4, 6)
+    COLON      ':'           (4, 6) (4, 7)
+    NEWLINE    ''            (4, 7) (4, 7)
+    INDENT     ''            (5, -1) (5, -1)
+    AWAIT      'await'       (5, 4) (5, 9)
+    NEWLINE    ''            (5, 9) (5, 9)
+    DEDENT     ''            (6, -1) (6, -1)
+    DEDENT     ''            (6, -1) (6, -1)
+    ASYNC      'async'       (6, 0) (6, 5)
+    PLUSEQUAL  '+='          (6, 6) (6, 8)
+    NUMBER     '1'           (6, 9) (6, 10)
+    NEWLINE    ''            (6, 10) (6, 10)
+    """)
+
+        self.check_tokenize('async def foo():\n  async for i in 1: pass', """\
+    ASYNC      'async'       (1, 0) (1, 5)
+    NAME       'def'         (1, 6) (1, 9)
+    NAME       'foo'         (1, 10) (1, 13)
+    LPAR       '('           (1, 13) (1, 14)
+    RPAR       ')'           (1, 14) (1, 15)
+    COLON      ':'           (1, 15) (1, 16)
+    NEWLINE    ''            (1, 16) (1, 16)
+    INDENT     ''            (2, -1) (2, -1)
+    ASYNC      'async'       (2, 2) (2, 7)
+    NAME       'for'         (2, 8) (2, 11)
+    NAME       'i'           (2, 12) (2, 13)
+    NAME       'in'          (2, 14) (2, 16)
+    NUMBER     '1'           (2, 17) (2, 18)
+    COLON      ':'           (2, 18) (2, 19)
+    NAME       'pass'        (2, 20) (2, 24)
+    DEDENT     ''            (2, -1) (2, -1)
+    """)
+
+        self.check_tokenize('async def foo(async): await', """\
+    ASYNC      'async'       (1, 0) (1, 5)
+    NAME       'def'         (1, 6) (1, 9)
+    NAME       'foo'         (1, 10) (1, 13)
+    LPAR       '('           (1, 13) (1, 14)
+    ASYNC      'async'       (1, 14) (1, 19)
+    RPAR       ')'           (1, 19) (1, 20)
+    COLON      ':'           (1, 20) (1, 21)
+    AWAIT      'await'       (1, 22) (1, 27)
+    """)
+
+        self.check_tokenize('''\
+def f():
+
+  def baz(): pass
+  async def bar(): pass
+
+  await = 2''', """\
+    NAME       'def'         (1, 0) (1, 3)
+    NAME       'f'           (1, 4) (1, 5)
+    LPAR       '('           (1, 5) (1, 6)
+    RPAR       ')'           (1, 6) (1, 7)
+    COLON      ':'           (1, 7) (1, 8)
+    NEWLINE    ''            (1, 8) (1, 8)
+    INDENT     ''            (3, -1) (3, -1)
+    NAME       'def'         (3, 2) (3, 5)
+    NAME       'baz'         (3, 6) (3, 9)
+    LPAR       '('           (3, 9) (3, 10)
+    RPAR       ')'           (3, 10) (3, 11)
+    COLON      ':'           (3, 11) (3, 12)
+    NAME       'pass'        (3, 13) (3, 17)
+    NEWLINE    ''            (3, 17) (3, 17)
+    ASYNC      'async'       (4, 2) (4, 7)
+    NAME       'def'         (4, 8) (4, 11)
+    NAME       'bar'         (4, 12) (4, 15)
+    LPAR       '('           (4, 15) (4, 16)
+    RPAR       ')'           (4, 16) (4, 17)
+    COLON      ':'           (4, 17) (4, 18)
+    NAME       'pass'        (4, 19) (4, 23)
+    NEWLINE    ''            (4, 23) (4, 23)
+    AWAIT      'await'       (6, 2) (6, 7)
+    EQUAL      '='           (6, 8) (6, 9)
+    NUMBER     '2'           (6, 10) (6, 11)
+    DEDENT     ''            (6, -1) (6, -1)
+    """)
+
+        self.check_tokenize('''\
+async def f():
+
+  def baz(): pass
+  async def bar(): pass
+
+  await = 2''', """\
+    ASYNC      'async'       (1, 0) (1, 5)
+    NAME       'def'         (1, 6) (1, 9)
+    NAME       'f'           (1, 10) (1, 11)
+    LPAR       '('           (1, 11) (1, 12)
+    RPAR       ')'           (1, 12) (1, 13)
+    COLON      ':'           (1, 13) (1, 14)
+    NEWLINE    ''            (1, 14) (1, 14)
+    INDENT     ''            (3, -1) (3, -1)
+    NAME       'def'         (3, 2) (3, 5)
+    NAME       'baz'         (3, 6) (3, 9)
+    LPAR       '('           (3, 9) (3, 10)
+    RPAR       ')'           (3, 10) (3, 11)
+    COLON      ':'           (3, 11) (3, 12)
+    NAME       'pass'        (3, 13) (3, 17)
+    NEWLINE    ''            (3, 17) (3, 17)
+    ASYNC      'async'       (4, 2) (4, 7)
+    NAME       'def'         (4, 8) (4, 11)
+    NAME       'bar'         (4, 12) (4, 15)
+    LPAR       '('           (4, 15) (4, 16)
+    RPAR       ')'           (4, 16) (4, 17)
+    COLON      ':'           (4, 17) (4, 18)
+    NAME       'pass'        (4, 19) (4, 23)
+    NEWLINE    ''            (4, 23) (4, 23)
+    AWAIT      'await'       (6, 2) (6, 7)
+    EQUAL      '='           (6, 8) (6, 9)
+    NUMBER     '2'           (6, 10) (6, 11)
+    DEDENT     ''            (6, -1) (6, -1)
+    """)
+
+    def test_unicode(self):
+
+        self.check_tokenize("Örter = u'places'\ngrün = U'green'", """\
+    NAME       'Örter'       (1, 0) (1, 6)
+    EQUAL      '='           (1, 7) (1, 8)
+    STRING     "u'places'"   (1, 9) (1, 18)
+    NEWLINE    ''            (1, 18) (1, 18)
+    NAME       'grün'        (2, 0) (2, 5)
+    EQUAL      '='           (2, 6) (2, 7)
+    STRING     "U'green'"    (2, 8) (2, 16)
+    """)
+
+    def test_invalid_syntax(self):
+        def get_tokens(string):
+            return list(_generate_tokens_from_c_tokenizer(string))
+
+        self.assertRaises(SyntaxError, get_tokens, "(1+2]")
+        self.assertRaises(SyntaxError, get_tokens, "(1+2}")
+        self.assertRaises(SyntaxError, get_tokens, "{1+2]")
+
+        self.assertRaises(SyntaxError, get_tokens, "1_")
+        self.assertRaises(SyntaxError, get_tokens, "1.2_")
+        self.assertRaises(SyntaxError, get_tokens, "1e2_")
+        self.assertRaises(SyntaxError, get_tokens, "1e+")
+
+        self.assertRaises(SyntaxError, get_tokens, "\xa0")
+        self.assertRaises(SyntaxError, get_tokens, "€")
+
+        self.assertRaises(SyntaxError, get_tokens, "0b12")
+        self.assertRaises(SyntaxError, get_tokens, "0b1_2")
+        self.assertRaises(SyntaxError, get_tokens, "0b2")
+        self.assertRaises(SyntaxError, get_tokens, "0b1_")
+        self.assertRaises(SyntaxError, get_tokens, "0b")
+        self.assertRaises(SyntaxError, get_tokens, "0o18")
+        self.assertRaises(SyntaxError, get_tokens, "0o1_8")
+        self.assertRaises(SyntaxError, get_tokens, "0o8")
+        self.assertRaises(SyntaxError, get_tokens, "0o1_")
+        self.assertRaises(SyntaxError, get_tokens, "0o")
+        self.assertRaises(SyntaxError, get_tokens, "0x1_")
+        self.assertRaises(SyntaxError, get_tokens, "0x")
+        self.assertRaises(SyntaxError, get_tokens, "1_")
+        self.assertRaises(SyntaxError, get_tokens, "012")
+        self.assertRaises(SyntaxError, get_tokens, "1.2_")
+        self.assertRaises(SyntaxError, get_tokens, "1e2_")
+        self.assertRaises(SyntaxError, get_tokens, "1e+")
+
+        self.assertRaises(SyntaxError, get_tokens, "'sdfsdf")
+        self.assertRaises(SyntaxError, get_tokens, "'''sdfsdf''")
+
+        self.assertRaises(SyntaxError, get_tokens, "("*1000+"a"+")"*1000)
+        self.assertRaises(SyntaxError, get_tokens, "]")
+
+
 if __name__ == "__main__":
     unittest.main()

Lib/tokenize.py

@@ -680,5 +680,13 @@ def main():
         perror("unexpected error: %s" % err)
         raise
 
+def _generate_tokens_from_c_tokenizer(source):
+    """Tokenize a source reading Python code as unicode strings using the internal C tokenizer"""
+    import _tokenize as c_tokenizer
+    for info in c_tokenizer.TokenizerIter(source):
+        tok, type, lineno, end_lineno, col_off, end_col_off, line = info
+        yield TokenInfo(type, tok, (lineno, col_off), (end_lineno, end_col_off), line)
+
+
 if __name__ == "__main__":
     main()

Makefile.pre.in

@@ -339,6 +339,7 @@ PARSER_HEADERS= \
 PYTHON_OBJS=	\
 		Python/_warnings.o \
 		Python/Python-ast.o \
+		Python/Python-tokenize.o \
 		Python/asdl.o \
 		Python/ast.o \
 		Python/ast_opt.o \

Modules/config.c.in

@@ -28,6 +28,7 @@ extern PyObject* PyMarshal_Init(void);
 extern PyObject* PyInit__imp(void);
 extern PyObject* PyInit_gc(void);
 extern PyObject* PyInit__ast(void);
+extern PyObject* PyInit__tokenize(void);
 extern PyObject* _PyWarnings_Init(void);
 extern PyObject* PyInit__string(void);
 
@@ -44,6 +45,9 @@ struct _inittab _PyImport_Inittab[] = {
     /* This lives in Python/Python-ast.c */
     {"_ast", PyInit__ast},
 
+    /* This lives in Python/Python-tokenizer.c */
+    {"_tokenize", PyInit__tokenize},
+
     /* These entries are here for sys.builtin_module_names */
     {"builtins", NULL},
     {"sys", NULL},

PC/config.c

@@ -72,9 +72,8 @@ extern PyObject* _PyWarnings_Init(void);
 extern PyObject* PyInit__string(void);
 extern PyObject* PyInit__stat(void);
 extern PyObject* PyInit__opcode(void);
-
 extern PyObject* PyInit__contextvars(void);
-
+extern PyObject* PyInit__tokenize(void);
 
 /* tools/freeze/makeconfig.py marker for additional "extern" */
 /* -- ADDMODULE MARKER 1 -- */
@@ -83,7 +82,6 @@ extern PyObject* PyMarshal_Init(void);
 extern PyObject* PyInit__imp(void);
 
 struct _inittab _PyImport_Inittab[] = {
-
     {"_abc", PyInit__abc},
     {"array", PyInit_array},
     {"_ast", PyInit__ast},
@@ -105,6 +103,7 @@ struct _inittab _PyImport_Inittab[] = {
     {"_blake2", PyInit__blake2},
     {"time", PyInit_time},
     {"_thread", PyInit__thread},
+    {"_tokenize", PyInit__tokenize},
     {"_typing", PyInit__typing},
     {"_statistics", PyInit__statistics},
 #ifdef WIN32

PCbuild/pythoncore.vcxproj

@@ -488,6 +488,7 @@
     <ClCompile Include="..\Python\pystrtod.c" />
     <ClCompile Include="..\Python\dtoa.c" />
     <ClCompile Include="..\Python\Python-ast.c" />
+    <ClCompile Include="..\Python\Python-tokenize.c" />
     <ClCompile Include="..\Python\pythonrun.c" />
     <ClCompile Include="..\Python\specialize.c" />
     <ClCompile Include="..\Python\suggestions.c" />

Python/Python-tokenize.c

@@ -0,0 +1,195 @@
+#include "Python.h"
+#include "../Parser/tokenizer.h"
+
+static struct PyModuleDef _tokenizemodule;
+
+typedef struct {
+    PyTypeObject* TokenizerIter;
+} tokenize_state;
+
+static tokenize_state*
+get_tokenize_state(PyObject* module)
+{
+    return (tokenize_state*)PyModule_GetState(module);
+}
+
+#define _tokenize_get_state_by_type(type)                                                               \
+    get_tokenize_state(_PyType_GetModuleByDef(type, &_tokenizemodule))
+
+#include "clinic/Python-tokenize.c.h"
+
+/*[clinic input]
+module _tokenizer
+class _tokenizer.tokenizeriter "tokenizeriterobject *" "_tokenize_get_state_by_type(type)->TokenizerIter"
+[clinic start generated code]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=96d98ee2fef7a8bc]*/
+
+typedef struct {
+    PyObject_HEAD
+    struct tok_state* tok;
+} tokenizeriterobject;
+
+/*[clinic input]
+@classmethod
+_tokenizer.tokenizeriter.__new__ as tokenizeriter_new
+
+    source: str
+[clinic start generated code]*/
+
+static PyObject *
+tokenizeriter_new_impl(PyTypeObject *type, const char *source)
+/*[clinic end generated code: output=7fd9f46cf9263cbb input=4384b368407375c6]*/
+{
+    tokenizeriterobject* self = (tokenizeriterobject*)type->tp_alloc(type, 0);
+    if (self == NULL) {
+        return NULL;
+    }
+    PyObject* filename = PyUnicode_FromString("<string>");
+    if (filename == NULL) {
+        return NULL;
+    }
+    self->tok = PyTokenizer_FromUTF8(source, 1);
+    if (self->tok == NULL) {
+        return NULL;
+    }
+    self->tok->filename = filename;
+    return (PyObject*)self;
+}
+
+static PyObject*
+tokenizeriter_next(tokenizeriterobject* it)
+{
+    const char* start;
+    const char* end;
+    int type = PyTokenizer_Get(it->tok, &start, &end);
+    if (type == ERRORTOKEN && PyErr_Occurred()) {
+        return NULL;
+    }
+    if (type == ERRORTOKEN || type == ENDMARKER) {
+        PyErr_SetString(PyExc_StopIteration, "EOF");
+        return NULL;
+    }
+    PyObject* str = NULL;
+    if (start == NULL || end == NULL) {
+        str = PyUnicode_FromString("");
+    } else {
+        str = PyUnicode_FromStringAndSize(start, end - start);
+    }
+    if (str == NULL) {
+        return NULL;
+    }
+
+    Py_ssize_t size = it->tok->inp - it->tok->buf;
+    PyObject* line = PyUnicode_DecodeUTF8(it->tok->buf, size, "replace");
+    if (line == NULL) {
+        Py_DECREF(str);
+        return NULL;
+    }
+    const char* line_start = type == STRING ? it->tok->multi_line_start : it->tok->line_start;
+    int lineno = type == STRING ? it->tok->first_lineno : it->tok->lineno;
+    int end_lineno = it->tok->lineno;
+    int col_offset = -1;
+    int end_col_offset = -1;
+    if (start != NULL && start >= line_start) {
+        col_offset = (int)(start - line_start);
+    }
+    if (end != NULL && end >= it->tok->line_start) {
+        end_col_offset = (int)(end - it->tok->line_start);
+    }
+
+    return Py_BuildValue("(NiiiiiN)", str, type, lineno, end_lineno, col_offset, end_col_offset, line);
+}
+
+static void
+tokenizeriter_dealloc(tokenizeriterobject* it)
+{
+    PyTypeObject* tp = Py_TYPE(it);
+    PyTokenizer_Free(it->tok);
+    tp->tp_free(it);
+    Py_DECREF(tp);
+}
+
+static PyType_Slot tokenizeriter_slots[] = {
+        {Py_tp_new, tokenizeriter_new},
+        {Py_tp_dealloc, tokenizeriter_dealloc},
+        {Py_tp_getattro, PyObject_GenericGetAttr},
+        {Py_tp_iter, PyObject_SelfIter},
+        {Py_tp_iternext, tokenizeriter_next},
+        {0, NULL},
+};
+
+static PyType_Spec tokenizeriter_spec = {
+        .name = "_tokenize.TokenizerIter",
+        .basicsize = sizeof(tokenizeriterobject),
+        .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE),
+        .slots = tokenizeriter_slots,
+};
+
+
+static int
+tokenizemodule_exec(PyObject* m)
+{
+    tokenize_state* state = get_tokenize_state(m);
+    if (state == NULL) {
+        return -1;
+    }
+
+    state->TokenizerIter = (PyTypeObject *)PyType_FromModuleAndSpec(
+        m, &tokenizeriter_spec, NULL);
+    if (state->TokenizerIter == NULL) {
+        return -1;
+    }
+    if (PyModule_AddType(m, state->TokenizerIter) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static PyMethodDef tokenize_methods[] = {
+        {NULL, NULL, 0, NULL} /* Sentinel */
+};
+
+static PyModuleDef_Slot tokenizemodule_slots[] = {
+    {Py_mod_exec, tokenizemodule_exec},
+    {0, NULL}
+};
+
+static int
+tokenizemodule_traverse(PyObject *m, visitproc visit, void *arg)
+{
+    tokenize_state *state = get_tokenize_state(m);
+    Py_VISIT(state->TokenizerIter);
+    return 0;
+}
+
+static int
+tokenizemodule_clear(PyObject *m)
+{
+    tokenize_state *state = get_tokenize_state(m);
+    Py_CLEAR(state->TokenizerIter);
+    return 0;
+}
+
+static void
+tokenizemodule_free(void *m)
+{
+    tokenizemodule_clear((PyObject *)m);
+}
+
+static struct PyModuleDef _tokenizemodule = {
+        PyModuleDef_HEAD_INIT,
+        .m_name = "_tokenize",
+        .m_size = sizeof(tokenize_state),
+        .m_slots = tokenizemodule_slots,
+        .m_methods = tokenize_methods,
+        .m_traverse = tokenizemodule_traverse,
+        .m_clear = tokenizemodule_clear,
+        .m_free = tokenizemodule_free,
+};
+
+PyMODINIT_FUNC
+PyInit__tokenize(void)
+{
+    return PyModuleDef_Init(&_tokenizemodule);
+}

Python/clinic/Python-tokenize.c.h

@@ -0,0 +1,41 @@
+/*[clinic input]
+preserve
+[clinic start generated code]*/
+
+static PyObject *
+tokenizeriter_new_impl(PyTypeObject *type, const char *source);
+
+static PyObject *
+tokenizeriter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
+{
+    PyObject *return_value = NULL;
+    static const char * const _keywords[] = {"source", NULL};
+    static _PyArg_Parser _parser = {NULL, _keywords, "tokenizeriter", 0};
+    PyObject *argsbuf[1];
+    PyObject * const *fastargs;
+    Py_ssize_t nargs = PyTuple_GET_SIZE(args);
+    const char *source;
+
+    fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf);
+    if (!fastargs) {
+        goto exit;
+    }
+    if (!PyUnicode_Check(fastargs[0])) {
+        _PyArg_BadArgument("tokenizeriter", "argument 'source'", "str", fastargs[0]);
+        goto exit;
+    }
+    Py_ssize_t source_length;
+    source = PyUnicode_AsUTF8AndSize(fastargs[0], &source_length);
+    if (source == NULL) {
+        goto exit;
+    }
+    if (strlen(source) != (size_t)source_length) {
+        PyErr_SetString(PyExc_ValueError, "embedded null character");
+        goto exit;
+    }
+    return_value = tokenizeriter_new_impl(type, source);
+
+exit:
+    return return_value;
+}
+/*[clinic end generated code: output=dfcd64774e01bfe6 input=a9049054013a1b77]*/

Python/stdlib_module_names.h

@@ -80,6 +80,7 @@ static const char* _Py_stdlib_module_names[] = {
 "_thread",
 "_threading_local",
 "_tkinter",
+"_tokenize",
 "_tracemalloc",
 "_typing",
 "_uuid",