traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add ELLIPSIS and RARROW. Add tests (#666)

This commit is contained in:
Jim Fasarakis-Hilliard 2017-03-14 22:16:15 +02:00 committed by Stefan Krah
parent 9135275cba
commit d4914e9041
2 changed files with 24 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
Lib/test/test_tokenize.py
View File

@ -407,6 +407,25 @@ def"', """\
OP ')' (1, 22) (1, 23)
OP ':' (1, 23) (1, 24)
NAME 'pass' (1, 25) (1, 29)
""")
self.check_tokenize("def d23(a: str, b: int=3) -> int: pass", """\
NAME 'def' (1, 0) (1, 3)
NAME 'd23' (1, 4) (1, 7)
OP '(' (1, 7) (1, 8)
NAME 'a' (1, 8) (1, 9)
OP ':' (1, 9) (1, 10)
NAME 'str' (1, 11) (1, 14)
OP ',' (1, 14) (1, 15)
NAME 'b' (1, 16) (1, 17)
OP ':' (1, 17) (1, 18)
NAME 'int' (1, 19) (1, 22)
OP '=' (1, 22) (1, 23)
NUMBER '3' (1, 23) (1, 24)
OP ')' (1, 24) (1, 25)
OP '->' (1, 26) (1, 28)
NAME 'int' (1, 29) (1, 32)
OP ':' (1, 32) (1, 33)
NAME 'pass' (1, 34) (1, 38)
""")
def test_comparison(self):
@ -1371,6 +1390,8 @@ class TestTokenize(TestCase):
self.assertExactTypeEqual('**=', token.DOUBLESTAREQUAL)
self.assertExactTypeEqual('//', token.DOUBLESLASH)
self.assertExactTypeEqual('//=', token.DOUBLESLASHEQUAL)
self.assertExactTypeEqual('...', token.ELLIPSIS)
self.assertExactTypeEqual('->', token.RARROW)
self.assertExactTypeEqual('@', token.AT)
self.assertExactTypeEqual('@=', token.ATEQUAL)

4
Lib/tokenize.py
View File

@ -86,12 +86,14 @@ EXACT_TOKEN_TYPES = {
'%=': PERCENTEQUAL,
'&=': AMPEREQUAL,
'|=': VBAREQUAL,
'^=': CIRCUMFLEXEQUAL,
'^=': CIRCUMFLEXEQUAL,
'<<=': LEFTSHIFTEQUAL,
'>>=': RIGHTSHIFTEQUAL,
'**=': DOUBLESTAREQUAL,
'//': DOUBLESLASH,
'//=': DOUBLESLASHEQUAL,
'...': ELLIPSIS,
'->': RARROW,
'@': AT,
'@=': ATEQUAL,
}