Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Completely ignore lines with only a newline token on them, except 

wholly empty lines interactively.
This commit is contained in:
Guido van Rossum 1991-07-27 21:42:56 +00:00
parent 8dcbbac15a
commit 8c11a5c759
1 changed files with 47 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
Parser/tokenizer.c
View File

@ -124,7 +124,7 @@ tok_setups(str)
} }
/* Set up tokenizer for string */ /* Set up tokenizer for file */
struct tok_state * struct tok_state *
tok_setupf(fp, ps1, ps2) tok_setupf(fp, ps1, ps2)
@ -309,7 +309,11 @@ tok_get(tok, p_start, p_end)
char **p_start, **p_end; /* Out: point to start/end of token */ char **p_start, **p_end; /* Out: point to start/end of token */
{ {
register int c; register int c;
int blankline;
nextline:
blankline = 0;
/* Get indentation level */ /* Get indentation level */
if (tok->atbol) { if (tok->atbol) {
register int col = 0; register int col = 0;
@ -325,30 +329,45 @@ tok_get(tok, p_start, p_end)
break; break;
} }
tok_backup(tok, c); tok_backup(tok, c);
if (col == tok->indstack[tok->indent]) { if (c == '#' || c == '\n') {
/* No change */ /* Lines with only whitespace and/or comments
shouldn't affect the indentation and are
not passed to the parser as NEWLINE tokens,
except *totally* empty lines in interactive
mode, which signal the end of a command group. */
if (col == 0 && c == '\n' && tok->prompt != NULL)
blankline = 0; /* Let it through */
else
blankline = 1; /* Ignore completely */
/* We can't jump back right here since we still
may need to skip to the end of a comment */
} }
else if (col > tok->indstack[tok->indent]) { if (!blankline) {
/* Indent -- always one */ if (col == tok->indstack[tok->indent]) {
if (tok->indent+1 >= MAXINDENT) { /* No change */
fprintf(stderr, "excessive indent\n");
tok->done = E_TOKEN;
return ERRORTOKEN;
} }
tok->pendin++; else if (col > tok->indstack[tok->indent]) {
tok->indstack[++tok->indent] = col; /* Indent -- always one */
} if (tok->indent+1 >= MAXINDENT) {
else /* col < tok->indstack[tok->indent] */ { fprintf(stderr, "excessive indent\n");
/* Dedent -- any number, must be consistent */ tok->done = E_TOKEN;
while (tok->indent > 0 && return ERRORTOKEN;
col < tok->indstack[tok->indent]) { }
tok->indent--; tok->pendin++;
tok->pendin--; tok->indstack[++tok->indent] = col;
} }
if (col != tok->indstack[tok->indent]) { else /* col < tok->indstack[tok->indent] */ {
fprintf(stderr, "inconsistent dedent\n"); /* Dedent -- any number, must be consistent */
tok->done = E_TOKEN; while (tok->indent > 0 &&
return ERRORTOKEN; col < tok->indstack[tok->indent]) {
tok->indent--;
tok->pendin--;
}
if (col != tok->indstack[tok->indent]) {
fprintf(stderr, "inconsistent dedent\n");
tok->done = E_TOKEN;
return ERRORTOKEN;
}
} }
} }
} }
@ -380,13 +399,14 @@ tok_get(tok, p_start, p_end)
if (c == '#') { if (c == '#') {
/* Hack to allow overriding the tabsize in the file. /* Hack to allow overriding the tabsize in the file.
This is also recognized by vi, when it occurs near the This is also recognized by vi, when it occurs near the
beginning or end of the file. (Will vi never die...?) */ beginning or end of the file. (Will vi never die...?)
For Python it must be at the beginning of the file! */
int x; int x;
/* XXX The case to (unsigned char *) is needed by THINK C 3.0 */ /* XXX The case to (unsigned char *) is needed by THINK C 3.0 */
if (sscanf(/*(unsigned char *)*/tok->cur, if (sscanf(/*(unsigned char *)*/tok->cur,
" vi:set tabsize=%d:", &x) == 1 && " vi:set tabsize=%d:", &x) == 1 &&
x >= 1 && x <= 40) { x >= 1 && x <= 40) {
fprintf(stderr, "# vi:set tabsize=%d:\n", x); /* fprintf(stderr, "# vi:set tabsize=%d:\n", x); */
tok->tabsize = x; tok->tabsize = x;
} }
do { do {
@ -411,6 +431,8 @@ tok_get(tok, p_start, p_end)
/* Newline */ /* Newline */
if (c == '\n') { if (c == '\n') {
tok->atbol = 1; tok->atbol = 1;
if (blankline)
goto nextline;
*p_end = tok->cur - 1; /* Leave '\n' out of the string */ *p_end = tok->cur - 1; /* Leave '\n' out of the string */
return NEWLINE; return NEWLINE;
} }