traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added shift and mask ops. 

Allow numbers starting with a period.
This commit is contained in:
Guido van Rossum 1991-10-24 14:59:40 +00:00
parent 7928cd7636
commit baf0ebf43c
1 changed files with 25 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
Parser/tokenizer.c
View File

@ -85,6 +85,10 @@ char *tok_name[] = {
"NOTEQUAL",
"LESSEQUAL",
"GREATEREQUAL",
"TILDE",
"CIRCUMFLEX",
"LEFTSHIFT",
"RIGHTSHIFT",
/* This table must match the #defines in token.h! */
"OP",
"<ERRORTOKEN>",
@ -301,6 +305,8 @@ tok_1char(c)
case '`': return BACKQUOTE;
case '{': return LBRACE;
case '}': return RBRACE;
case '^': return CIRCUMFLEX;
case '~': return TILDE;
default: return OP;
}
}
@ -325,11 +331,13 @@ tok_2char(c1, c2)
switch (c2) {
case '>': return NOTEQUAL;
case '=': return LESSEQUAL;
case '<': return LEFTSHIFT;
}
break;
case '>':
switch (c2) {
case '=': return GREATEREQUAL;
case '>': return RIGHTSHIFT;
}
break;
}
@ -438,7 +446,7 @@ tok_get(tok, p_start, p_end)
beginning or end of the file. (Will vi never die...?)
For Python it must be at the beginning of the file! */
int x;
/* XXX The case to (unsigned char *) is needed by THINK C 3.0 */
/* XXX The cast to (unsigned char *) is needed by THINK C 3.0 */
if (sscanf(/*(unsigned char *)*/tok->cur,
" vi:set tabsize=%d:", &x) == 1 &&
x >= 1 && x <= 40) {
@ -451,8 +459,10 @@ tok_get(tok, p_start, p_end)
}
/* Check for EOF and errors now */
if (c == EOF)
if (c == EOF) {
*p_start = *p_end = tok->cur;
return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
}
/* Identifier (most frequent token!) */
if (isalpha(c) || c == '_') {
@ -473,6 +483,19 @@ tok_get(tok, p_start, p_end)
return NEWLINE;
}
/* Period or number starting with period? */
if (c == '.') {
c = tok_nextc(tok);
if (isdigit(c)) {
goto fraction;
}
else {
tok_backup(tok, c);
*p_end = tok->cur;
return DOT;
}
}
/* Number */
if (isdigit(c)) {
if (c == '0') {