traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added 2-char tokens and new versions of comparisons

This commit is contained in:
Guido van Rossum 1991-10-20 20:25:03 +00:00
parent 8883aaa5eb
commit fbab905ae1
1 changed files with 47 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
Parser/tokenizer.c
View File

@ -81,6 +81,11 @@ char *tok_name[] = {
"BACKQUOTE",
"LBRACE",
"RBRACE",
"EQEQUAL",
"NOTEQUAL",
"LESSEQUAL",
"GREATEREQUAL",
/* This table must match the #defines in token.h! */
"OP",
"<ERRORTOKEN>",
"<N_TOKENS>"
@ -301,6 +306,37 @@ tok_1char(c)
}
int
tok_2char(c1, c2)
int c1, c2;
{
switch (c1) {
case '=':
switch (c2) {
case '=': return EQEQUAL;
}
break;
case '!':
switch (c2) {
case '=': return NOTEQUAL;
}
break;
case '<':
switch (c2) {
case '>': return NOTEQUAL;
case '=': return LESSEQUAL;
}
break;
case '>':
switch (c2) {
case '=': return GREATEREQUAL;
}
break;
}
return OP;
}
/* Get next token, after space stripping etc. */
int
@ -531,6 +567,17 @@ tok_get(tok, p_start, p_end)
goto again; /* Read next line */
}
/* Check for two-character token */
{
int c2 = tok_nextc(tok);
int token = tok_2char(c, c2);
if (token != OP) {
*p_end = tok->cur;
return token;
}
tok_backup(tok, c2);
}
/* Punctuation character */
*p_end = tok->cur;
return tok_1char(c);