63 lines
1.4 KiB
Plaintext
63 lines
1.4 KiB
Plaintext
ENDMARKER
|
|
NAME
|
|
NUMBER
|
|
STRING
|
|
NEWLINE
|
|
INDENT
|
|
DEDENT
|
|
|
|
LPAR '('
|
|
RPAR ')'
|
|
LSQB '['
|
|
RSQB ']'
|
|
COLON ':'
|
|
COMMA ','
|
|
SEMI ';'
|
|
PLUS '+'
|
|
MINUS '-'
|
|
STAR '*'
|
|
SLASH '/'
|
|
VBAR '|'
|
|
AMPER '&'
|
|
LESS '<'
|
|
GREATER '>'
|
|
EQUAL '='
|
|
DOT '.'
|
|
PERCENT '%'
|
|
LBRACE '{'
|
|
RBRACE '}'
|
|
EQEQUAL '=='
|
|
NOTEQUAL '!='
|
|
LESSEQUAL '<='
|
|
GREATEREQUAL '>='
|
|
TILDE '~'
|
|
CIRCUMFLEX '^'
|
|
LEFTSHIFT '<<'
|
|
RIGHTSHIFT '>>'
|
|
DOUBLESTAR '**'
|
|
PLUSEQUAL '+='
|
|
MINEQUAL '-='
|
|
STAREQUAL '*='
|
|
SLASHEQUAL '/='
|
|
PERCENTEQUAL '%='
|
|
AMPEREQUAL '&='
|
|
VBAREQUAL '|='
|
|
CIRCUMFLEXEQUAL '^='
|
|
LEFTSHIFTEQUAL '<<='
|
|
RIGHTSHIFTEQUAL '>>='
|
|
DOUBLESTAREQUAL '**='
|
|
DOUBLESLASH '//'
|
|
DOUBLESLASHEQUAL '//='
|
|
AT '@'
|
|
ATEQUAL '@='
|
|
RARROW '->'
|
|
ELLIPSIS '...'
|
|
|
|
OP
|
|
ERRORTOKEN
|
|
|
|
# These aren't used by the C tokenizer but are needed for tokenize.py
|
|
COMMENT
|
|
NL
|
|
ENCODING
|