traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-40334: Catch E_EOF error, when the tokenizer returns ERRORTOKEN (GH-19743) 

An E_EOF error was only being caught after the parser exited before this commit. There are some cases though, where the tokenizer returns ERRORTOKEN *and* has set an E_EOF error (like when EOF directly follows a line continuation character) which weren't correctly handled before.
This commit is contained in:
Lysandros Nikolaou 2020-04-28 03:23:35 +03:00 committed by GitHub
parent 5d1f32d33b
commit d55133f49f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Lib/test/test_eof.py
View File

@ -26,7 +26,6 @@ class EOFTestCase(unittest.TestCase):
else:
raise support.TestFailed
@support.skip_if_new_parser("TODO for PEG -- fails with new parser")
def test_line_continuation_EOF(self):
"""A continuation at the end of input must be an error; bpo2180."""
expect = 'unexpected EOF while parsing (<string>, line 1)'
@ -37,7 +36,6 @@ class EOFTestCase(unittest.TestCase):
exec('\\')
self.assertEqual(str(excinfo.exception), expect)
@unittest.skip("TODO for PEG -- fails even with old parser now")
@unittest.skipIf(not sys.executable, "sys.executable required")
def test_line_continuation_EOF_from_file_bpo2180(self):
"""Ensure tok_nextc() does not add too many ending newlines."""

9
Parser/pegen/pegen.c
View File

@ -344,13 +344,16 @@ tokenizer_error(Parser *p)
break;
case E_BADPREFIX:
return tokenizer_error_with_col_offset(p,
PyExc_SyntaxError, "invalid string prefix");
errtype, "invalid string prefix");
case E_EOFS:
return tokenizer_error_with_col_offset(p,
PyExc_SyntaxError, "EOF while scanning triple-quoted string literal");
errtype, "EOF while scanning triple-quoted string literal");
case E_EOLS:
return tokenizer_error_with_col_offset(p,
PyExc_SyntaxError, "EOL while scanning string literal");
errtype, "EOL while scanning string literal");
case E_EOF:
return tokenizer_error_with_col_offset(p,
errtype, "unexpected EOF while parsing");
case E_DEDENT:
return tokenizer_error_with_col_offset(p,
PyExc_IndentationError, "unindent does not match any outer indentation level");