bpo-40267: Fix message when last input character produces a SyntaxError (GH-19521)

When there is a SyntaxError after reading the last input character from
the tokenizer and if no newline follows it, the error message used to be
`unexpected EOF while parsing`, which is wrong.

Include/token.h

@@ -78,6 +78,10 @@ extern "C" {
 #define ISTERMINAL(x)           ((x) < NT_OFFSET)
 #define ISNONTERMINAL(x)        ((x) >= NT_OFFSET)
 #define ISEOF(x)                ((x) == ENDMARKER)
+#define ISWHITESPACE(x)         ((x) == ENDMARKER || \
+                                 (x) == NEWLINE   || \
+                                 (x) == INDENT    || \
+                                 (x) == DEDENT)
 
 
 PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */

Lib/test/test_fstring.py

@@ -713,7 +713,7 @@ non-important content
 
         # lambda doesn't work without parens, because the colon
         #  makes the parser think it's a format_spec
-        self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing',
+        self.assertAllRaise(SyntaxError, 'invalid syntax',
                             ["f'{lambda x:x}'",
                              ])
 

Misc/NEWS.d/next/Core and Builtins/2020-04-14-18-54-50.bpo-40267.Q2N6Bw.rst

@@ -0,0 +1 @@
+Fix the tokenizer to display the correct error message, when there is a SyntaxError on the last input character and no newline follows. It used to be `unexpected EOF while parsing`, while it should be `invalid syntax`.

Parser/parsetok.c

@@ -332,6 +332,9 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
              PyParser_AddToken(ps, (int)type, str,
                                lineno, col_offset, tok->lineno, end_col_offset,
                                &(err_ret->expected))) != E_OK) {
+            if (tok->done == E_EOF && !ISWHITESPACE(type)) {
+                tok->done = E_SYNTAX;
+            }
             if (err_ret->error != E_DONE) {
                 PyObject_FREE(str);
                 err_ret->token = type;

Tools/scripts/generate_token.py

@@ -69,6 +69,10 @@ extern "C" {
 #define ISTERMINAL(x)           ((x) < NT_OFFSET)
 #define ISNONTERMINAL(x)        ((x) >= NT_OFFSET)
 #define ISEOF(x)                ((x) == ENDMARKER)
+#define ISWHITESPACE(x)         ((x) == ENDMARKER || \\
+                                 (x) == NEWLINE   || \\
+                                 (x) == INDENT    || \\
+                                 (x) == DEDENT)
 
 
 PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */