Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

bpo-42864: Simplify the tokenizer exceptions after generic SyntaxError (GH-24273) 

Automerge-Triggered-By: GH:pablogsal
This commit is contained in:
Pablo Galindo 2021-01-20 19:11:56 +00:00 committed by GitHub
parent 75e59a97f5
commit c3f167d7b2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Parser/pegen.c
View File

@ -1177,14 +1177,9 @@ _PyPegen_check_tokenizer_errors(Parser *p) {
return 0;
}
Token *current_token = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1];
Py_ssize_t current_err_line = current_token->lineno;
// Save the tokenizer state to restore them later in case we found nothing
struct tok_state saved_tok;
memcpy(&saved_tok, p->tok, sizeof(struct tok_state));
for (;;) {
const char *start;
const char *end;
@ -1206,8 +1201,6 @@ _PyPegen_check_tokenizer_errors(Parser *p) {
break;
}
// Restore the tokenizer state
memcpy(p->tok, &saved_tok, sizeof(struct tok_state));
return 0;
}
@ -1239,10 +1232,10 @@ _PyPegen_run_parser(Parser *p)
RAISE_INDENTATION_ERROR("unexpected unindent");
}
else {
if (_PyPegen_check_tokenizer_errors(p)) {
return NULL;
}
RAISE_SYNTAX_ERROR("invalid syntax");
// _PyPegen_check_tokenizer_errors will override the existing
// generic SyntaxError we just raised if errors are found.
_PyPegen_check_tokenizer_errors(p);
}
}
return NULL;