bpo-43797: Improve syntax error for invalid comparisons (#25317)

* bpo-43797: Improve syntax error for invalid comparisons

* Update Lib/test/test_fstring.py

Co-authored-by: Guido van Rossum <gvanrossum@gmail.com>

* Apply review comments

* can't -> cannot

Co-authored-by: Guido van Rossum <gvanrossum@gmail.com>

Grammar/python.gram

@@ -163,17 +163,20 @@ dotted_name[expr_ty]:
     | NAME
 
 if_stmt[stmt_ty]:
-    | 'if' a=named_expression &&':' b=block c=elif_stmt {
+    | 'if' a=named_expression ':' b=block c=elif_stmt {
         _PyAST_If(a, b, CHECK(asdl_stmt_seq*, _PyPegen_singleton_seq(p, c)), EXTRA) }
-    | 'if' a=named_expression &&':' b=block c=[else_block] { _PyAST_If(a, b, c, EXTRA) }
+    | 'if' a=named_expression ':' b=block c=[else_block] { _PyAST_If(a, b, c, EXTRA) }
+    | invalid_if_stmt
 elif_stmt[stmt_ty]:
-    | 'elif' a=named_expression &&':' b=block c=elif_stmt {
+    | 'elif' a=named_expression ':' b=block c=elif_stmt {
         _PyAST_If(a, b, CHECK(asdl_stmt_seq*, _PyPegen_singleton_seq(p, c)), EXTRA) }
-    | 'elif' a=named_expression &&':' b=block c=[else_block] { _PyAST_If(a, b, c, EXTRA) }
+    | 'elif' a=named_expression ':' b=block c=[else_block] { _PyAST_If(a, b, c, EXTRA) }
+    | invalid_elif_stmt
 else_block[asdl_stmt_seq*]: 'else' &&':' b=block { b }
 
 while_stmt[stmt_ty]:
-    | 'while' a=named_expression &&':' b=block c=[else_block] { _PyAST_While(a, b, c, EXTRA) }
+    | 'while' a=named_expression ':' b=block c=[else_block] { _PyAST_While(a, b, c, EXTRA) }
+    | invalid_while_stmt
 
 for_stmt[stmt_ty]:
     | 'for' t=star_targets 'in' ~ ex=star_expressions &&':' tc=[TYPE_COMMENT] b=block el=[else_block] {
@@ -438,10 +441,11 @@ star_named_expressions[asdl_expr_seq*]: a[asdl_expr_seq*]=','.star_named_express
 star_named_expression[expr_ty]:
     | '*' a=bitwise_or { _PyAST_Starred(a, Load, EXTRA) }
     | named_expression
+
 named_expression[expr_ty]:
     | a=NAME ':=' ~ b=expression { _PyAST_NamedExpr(CHECK(expr_ty, _PyPegen_set_expr_context(p, a, Store)), b, EXTRA) }
-    | expression !':='
     | invalid_named_expression
+    | expression !':='
 
 annotated_rhs[expr_ty]: yield_expr | star_expressions
 
@@ -772,6 +776,12 @@ invalid_named_expression:
     | a=expression ':=' expression {
         RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
             a, "cannot use assignment expressions with %s", _PyPegen_get_expr_name(a)) }
+    | a=NAME b='=' bitwise_or !('='|':='|',') {
+        RAISE_SYNTAX_ERROR_KNOWN_LOCATION(b, "invalid syntax. Maybe you meant '==' or ':=' instead of '='?") }
+    | !(list|tuple|genexp|'True'|'None'|'False') a=bitwise_or b='=' bitwise_or !('='|':='|',') {
+        RAISE_SYNTAX_ERROR_KNOWN_LOCATION(b, "cannot assign to %s here. Maybe you meant '==' instead of '='?",
+                                          _PyPegen_get_expr_name(a)) }
+
 invalid_assignment:
     | a=invalid_ann_assign_target ':' expression {
         RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
@@ -841,9 +851,9 @@ invalid_for_target:
 
 invalid_group:
     | '(' a=starred_expression ')' {
-        RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "can't use starred expression here") }
+        RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use starred expression here") }
     | '(' a='**' expression ')' {
-        RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "can't use double starred expression here") }
+        RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use double starred expression here") }
 invalid_import_from_targets:
     | import_from_as_names ',' {
         RAISE_SYNTAX_ERROR("trailing comma not allowed without surrounding parentheses") }
@@ -860,6 +870,11 @@ invalid_except_block:
 
 invalid_match_stmt:
     | "match" subject_expr !':' { CHECK_VERSION(void*, 10, "Pattern matching is", RAISE_SYNTAX_ERROR("expected ':'") ) }
-
 invalid_case_block:
     | "case" patterns guard? !':' { RAISE_SYNTAX_ERROR("expected ':'") }
+invalid_if_stmt:
+    | 'if' named_expression NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
+invalid_elif_stmt:
+    | 'elif' named_expression NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
+invalid_while_stmt:
+    | 'while' named_expression NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }

Lib/test/test_cmd_line_script.py

@@ -600,8 +600,8 @@ class CmdLineTest(unittest.TestCase):
             script_name = _make_test_script(script_dir, 'script', script)
             exitcode, stdout, stderr = assert_python_failure(script_name)
             text = io.TextIOWrapper(io.BytesIO(stderr), 'ascii').read()
-            # Confirm that the caret is located under the first 1 character
+            # Confirm that the caret is located under the '=' sign
-            self.assertIn("\n    1 + 1 = 2\n    ^", text)
+            self.assertIn("\n    1 + 1 = 2\n          ^\n", text)
 
     def test_syntaxerror_indented_caret_position(self):
         script = textwrap.dedent("""\
@@ -613,7 +613,7 @@ class CmdLineTest(unittest.TestCase):
             exitcode, stdout, stderr = assert_python_failure(script_name)
             text = io.TextIOWrapper(io.BytesIO(stderr), 'ascii').read()
             # Confirm that the caret is located under the first 1 character
-            self.assertIn("\n    1 + 1 = 2\n    ^", text)
+            self.assertIn("\n    1 + 1 = 2\n          ^\n", text)
 
             # Try the same with a form feed at the start of the indented line
             script = (
@@ -624,7 +624,7 @@ class CmdLineTest(unittest.TestCase):
             exitcode, stdout, stderr = assert_python_failure(script_name)
             text = io.TextIOWrapper(io.BytesIO(stderr), "ascii").read()
             self.assertNotIn("\f", text)
-            self.assertIn("\n    1 + 1 = 2\n    ^", text)
+            self.assertIn("\n    1 + 1 = 2\n          ^\n", text)
 
     def test_syntaxerror_multi_line_fstring(self):
         script = 'foo = f"""{}\nfoo"""\n'

Lib/test/test_codeop.py

@@ -275,7 +275,6 @@ class CodeopTests(unittest.TestCase):
         ai("a = 'a\\\n")
 
         ai("a = 1","eval")
-        ai("a = (","eval")
         ai("]","eval")
         ai("())","eval")
         ai("[}","eval")

Lib/test/test_exceptions.py

@@ -260,7 +260,7 @@ class ExceptionTests(unittest.TestCase):
         check('[*x for x in xs]', 1, 2)
         check('foo(x for x in range(10), 100)', 1, 5)
         check('for 1 in []: pass', 1, 5)
-        check('(yield i) = 2', 1, 2)
+        check('(yield i) = 2', 1, 11)
         check('def f(*):\n  pass', 1, 8)
 
     @cpython_only

Lib/test/test_fstring.py

@@ -990,7 +990,7 @@ x = (
                              ])
 
     def test_assignment(self):
-        self.assertAllRaise(SyntaxError, 'invalid syntax',
+        self.assertAllRaise(SyntaxError, r'invalid syntax',
                             ["f'' = 3",
                              "f'{0}' = x",
                              "f'{x}' = x",
@@ -1276,11 +1276,11 @@ x = (
             f'{1:_,}'
 
     def test_syntax_error_for_starred_expressions(self):
-        error_msg = re.escape("can't use starred expression here")
+        error_msg = re.escape("cannot use starred expression here")
         with self.assertRaisesRegex(SyntaxError, error_msg):
             compile("f'{*a}'", "?", "exec")
 
-        error_msg = re.escape("can't use double starred expression here")
+        error_msg = re.escape("cannot use double starred expression here")
         with self.assertRaisesRegex(SyntaxError, error_msg):
             compile("f'{**a}'", "?", "exec")
 

Lib/test/test_generators.py

@@ -2013,7 +2013,7 @@ SyntaxError: 'yield' outside function
 >>> def f(): (yield bar) = y
 Traceback (most recent call last):
   ...
-SyntaxError: cannot assign to yield expression
+SyntaxError: cannot assign to yield expression here. Maybe you meant '==' instead of '='?
 
 >>> def f(): (yield bar) += y
 Traceback (most recent call last):

Lib/test/test_genexps.py

@@ -103,7 +103,7 @@ Verify that parenthesis are required when used as a keyword argument value
     >>> dict(a = i for i in range(10))
     Traceback (most recent call last):
        ...
-    SyntaxError: invalid syntax
+    SyntaxError: invalid syntax. Maybe you meant '==' or ':=' instead of '='?
 
 Verify that parenthesis are required when used as a keyword argument value
 

Lib/test/test_syntax.py

@@ -61,7 +61,7 @@ SyntaxError: cannot assign to __debug__
 
 >>> f() = 1
 Traceback (most recent call last):
-SyntaxError: cannot assign to function call
+SyntaxError: cannot assign to function call here. Maybe you meant '==' instead of '='?
 
 >>> yield = 1
 Traceback (most recent call last):
@@ -73,7 +73,7 @@ SyntaxError: cannot delete function call
 
 >>> a + 1 = 2
 Traceback (most recent call last):
-SyntaxError: cannot assign to operator
+SyntaxError: cannot assign to expression here. Maybe you meant '==' instead of '='?
 
 >>> (x for x in x) = 1
 Traceback (most recent call last):
@@ -81,19 +81,19 @@ SyntaxError: cannot assign to generator expression
 
 >>> 1 = 1
 Traceback (most recent call last):
-SyntaxError: cannot assign to literal
+SyntaxError: cannot assign to literal here. Maybe you meant '==' instead of '='?
 
 >>> "abc" = 1
 Traceback (most recent call last):
-SyntaxError: cannot assign to literal
+SyntaxError: cannot assign to literal here. Maybe you meant '==' instead of '='?
 
 >>> b"" = 1
 Traceback (most recent call last):
-SyntaxError: cannot assign to literal
+SyntaxError: cannot assign to literal here. Maybe you meant '==' instead of '='?
 
 >>> ... = 1
 Traceback (most recent call last):
-SyntaxError: cannot assign to Ellipsis
+SyntaxError: cannot assign to Ellipsis here. Maybe you meant '==' instead of '='?
 
 >>> `1` = 1
 Traceback (most recent call last):
@@ -126,15 +126,15 @@ SyntaxError: cannot assign to __debug__
 
 >>> [a, b, c + 1] = [1, 2, 3]
 Traceback (most recent call last):
-SyntaxError: cannot assign to operator
+SyntaxError: cannot assign to expression
 
 >>> [a, b[1], c + 1] = [1, 2, 3]
 Traceback (most recent call last):
-SyntaxError: cannot assign to operator
+SyntaxError: cannot assign to expression
 
 >>> [a, b.c.d, c + 1] = [1, 2, 3]
 Traceback (most recent call last):
-SyntaxError: cannot assign to operator
+SyntaxError: cannot assign to expression
 
 >>> a if 1 else b = 1
 Traceback (most recent call last):
@@ -181,7 +181,7 @@ SyntaxError: cannot assign to function call
 
 >>> for (*a, b, c+1) in b: pass
 Traceback (most recent call last):
-SyntaxError: cannot assign to operator
+SyntaxError: cannot assign to expression
 
 >>> for (x, *(y, z.d())) in b: pass
 Traceback (most recent call last):
@@ -193,7 +193,7 @@ SyntaxError: cannot assign to function call
 
 >>> for a, b, (c + 1, d()): pass
 Traceback (most recent call last):
-SyntaxError: cannot assign to operator
+SyntaxError: cannot assign to expression
 
 >>> for i < (): pass
 Traceback (most recent call last):
@@ -217,7 +217,7 @@ SyntaxError: cannot assign to function call
 
 >>> with a as (*b, c, d+1): pass
 Traceback (most recent call last):
-SyntaxError: cannot assign to operator
+SyntaxError: cannot assign to expression
 
 >>> with a as (x, *(y, z.d())): pass
 Traceback (most recent call last):
@@ -465,7 +465,7 @@ keyword slot of a call site.  Test a few different options.
 # SyntaxError: expression cannot contain assignment, perhaps you meant "=="?
 # >>> f(True=2)
 # Traceback (most recent call last):
-# SyntaxError: cannot assign to True
+# SyntaxError: cannot assign to True here. Maybe you meant '==' instead of '='?
 >>> f(__debug__=1)
 Traceback (most recent call last):
 SyntaxError: cannot assign to __debug__
@@ -684,7 +684,7 @@ leading to spurious errors.
    ...   pass
    Traceback (most recent call last):
      ...
-   SyntaxError: cannot assign to function call
+   SyntaxError: cannot assign to function call here. Maybe you meant '==' instead of '='?
 
    >>> if 1:
    ...   pass
@@ -692,7 +692,7 @@ leading to spurious errors.
    ...   x() = 1
    Traceback (most recent call last):
      ...
-   SyntaxError: cannot assign to function call
+   SyntaxError: cannot assign to function call here. Maybe you meant '==' instead of '='?
 
    >>> if 1:
    ...   x() = 1
@@ -702,7 +702,7 @@ leading to spurious errors.
    ...   pass
    Traceback (most recent call last):
      ...
-   SyntaxError: cannot assign to function call
+   SyntaxError: cannot assign to function call here. Maybe you meant '==' instead of '='?
 
    >>> if 1:
    ...   pass
@@ -712,7 +712,7 @@ leading to spurious errors.
    ...   pass
    Traceback (most recent call last):
      ...
-   SyntaxError: cannot assign to function call
+   SyntaxError: cannot assign to function call here. Maybe you meant '==' instead of '='?
 
    >>> if 1:
    ...   pass
@@ -722,7 +722,7 @@ leading to spurious errors.
    ...   x() = 1
    Traceback (most recent call last):
      ...
-   SyntaxError: cannot assign to function call
+   SyntaxError: cannot assign to function call here. Maybe you meant '==' instead of '='?
 
     Missing ':' before suites:
 
@@ -843,6 +843,26 @@ leading to spurious errors.
    Traceback (most recent call last):
    SyntaxError: expected ':'
 
+   >>> if x = 3:
+   ...    pass
+   Traceback (most recent call last):
+   SyntaxError: invalid syntax. Maybe you meant '==' or ':=' instead of '='?
+
+   >>> while x = 3:
+   ...    pass
+   Traceback (most recent call last):
+   SyntaxError: invalid syntax. Maybe you meant '==' or ':=' instead of '='?
+
+   >>> if x.a = 3:
+   ...    pass
+   Traceback (most recent call last):
+   SyntaxError: cannot assign to attribute here. Maybe you meant '==' instead of '='?
+
+   >>> while x.a = 3:
+   ...    pass
+   Traceback (most recent call last):
+   SyntaxError: cannot assign to attribute here. Maybe you meant '==' instead of '='?
+
 Make sure that the old "raise X, Y[, Z]" form is gone:
    >>> raise X, Y
    Traceback (most recent call last):
@@ -894,19 +914,19 @@ SyntaxError: keyword argument repeated: a
 
 >>> {1, 2, 3} = 42
 Traceback (most recent call last):
-SyntaxError: cannot assign to set display
+SyntaxError: cannot assign to set display here. Maybe you meant '==' instead of '='?
 
 >>> {1: 2, 3: 4} = 42
 Traceback (most recent call last):
-SyntaxError: cannot assign to dict display
+SyntaxError: cannot assign to dict literal here. Maybe you meant '==' instead of '='?
 
 >>> f'{x}' = 42
 Traceback (most recent call last):
-SyntaxError: cannot assign to f-string expression
+SyntaxError: cannot assign to f-string expression here. Maybe you meant '==' instead of '='?
 
 >>> f'{x}-{y}' = 42
 Traceback (most recent call last):
-SyntaxError: cannot assign to f-string expression
+SyntaxError: cannot assign to f-string expression here. Maybe you meant '==' instead of '='?
 
 >>> from t import x,
 Traceback (most recent call last):
@@ -988,7 +1008,7 @@ class SyntaxTestCase(unittest.TestCase):
     def test_expression_with_assignment(self):
         self._check_error(
             "print(end1 + end2 = ' ')",
-            'expression cannot contain assignment, perhaps you meant "=="?',
+            "cannot assign to expression here. Maybe you meant '==' instead of '='?",
             offset=19
         )
 
@@ -1000,31 +1020,31 @@ class SyntaxTestCase(unittest.TestCase):
 
     def test_assign_del(self):
         self._check_error("del (,)", "invalid syntax")
-        self._check_error("del 1", "delete literal")
+        self._check_error("del 1", "cannot delete literal")
-        self._check_error("del (1, 2)", "delete literal")
+        self._check_error("del (1, 2)", "cannot delete literal")
-        self._check_error("del None", "delete None")
+        self._check_error("del None", "cannot delete None")
-        self._check_error("del *x", "delete starred")
+        self._check_error("del *x", "cannot delete starred")
-        self._check_error("del (*x)", "use starred expression")
+        self._check_error("del (*x)", "cannot use starred expression")
-        self._check_error("del (*x,)", "delete starred")
+        self._check_error("del (*x,)", "cannot delete starred")
-        self._check_error("del [*x,]", "delete starred")
+        self._check_error("del [*x,]", "cannot delete starred")
-        self._check_error("del f()", "delete function call")
+        self._check_error("del f()", "cannot delete function call")
-        self._check_error("del f(a, b)", "delete function call")
+        self._check_error("del f(a, b)", "cannot delete function call")
-        self._check_error("del o.f()", "delete function call")
+        self._check_error("del o.f()", "cannot delete function call")
-        self._check_error("del a[0]()", "delete function call")
+        self._check_error("del a[0]()", "cannot delete function call")
-        self._check_error("del x, f()", "delete function call")
+        self._check_error("del x, f()", "cannot delete function call")
-        self._check_error("del f(), x", "delete function call")
+        self._check_error("del f(), x", "cannot delete function call")
-        self._check_error("del [a, b, ((c), (d,), e.f())]", "delete function call")
+        self._check_error("del [a, b, ((c), (d,), e.f())]", "cannot delete function call")
-        self._check_error("del (a if True else b)", "delete conditional")
+        self._check_error("del (a if True else b)", "cannot delete conditional")
-        self._check_error("del +a", "delete operator")
+        self._check_error("del +a", "cannot delete expression")
-        self._check_error("del a, +b", "delete operator")
+        self._check_error("del a, +b", "cannot delete expression")
-        self._check_error("del a + b", "delete operator")
+        self._check_error("del a + b", "cannot delete expression")
-        self._check_error("del (a + b, c)", "delete operator")
+        self._check_error("del (a + b, c)", "cannot delete expression")
-        self._check_error("del (c[0], a + b)", "delete operator")
+        self._check_error("del (c[0], a + b)", "cannot delete expression")
-        self._check_error("del a.b.c + 2", "delete operator")
+        self._check_error("del a.b.c + 2", "cannot delete expression")
-        self._check_error("del a.b.c[0] + 2", "delete operator")
+        self._check_error("del a.b.c[0] + 2", "cannot delete expression")
-        self._check_error("del (a, b, (c, d.e.f + 2))", "delete operator")
+        self._check_error("del (a, b, (c, d.e.f + 2))", "cannot delete expression")
-        self._check_error("del [a, b, (c, d.e.f[0] + 2)]", "delete operator")
+        self._check_error("del [a, b, (c, d.e.f[0] + 2)]", "cannot delete expression")
-        self._check_error("del (a := 5)", "delete named expression")
+        self._check_error("del (a := 5)", "cannot delete named expression")
         # We don't have a special message for this, but make sure we don't
         # report "cannot delete name"
         self._check_error("del a += b", "invalid syntax")

Lib/test/test_unpack_ex.py

@@ -349,27 +349,27 @@ Now some general starred expressions (all fail).
     >>> (*x),y = 1, 2 # doctest:+ELLIPSIS
     Traceback (most recent call last):
       ...
-    SyntaxError: can't use starred expression here
+    SyntaxError: cannot use starred expression here
 
     >>> (((*x))),y = 1, 2 # doctest:+ELLIPSIS
     Traceback (most recent call last):
       ...
-    SyntaxError: can't use starred expression here
+    SyntaxError: cannot use starred expression here
 
     >>> z,(*x),y = 1, 2, 4 # doctest:+ELLIPSIS
     Traceback (most recent call last):
       ...
-    SyntaxError: can't use starred expression here
+    SyntaxError: cannot use starred expression here
 
     >>> z,(*x) = 1, 2 # doctest:+ELLIPSIS
     Traceback (most recent call last):
       ...
-    SyntaxError: can't use starred expression here
+    SyntaxError: cannot use starred expression here
 
     >>> ((*x),y) = 1, 2 # doctest:+ELLIPSIS
     Traceback (most recent call last):
       ...
-    SyntaxError: can't use starred expression here
+    SyntaxError: cannot use starred expression here
 
 Some size constraints (all fail.)
 

Misc/NEWS.d/next/Core and Builtins/2021-04-09-19-12-48.bpo-43797.HfRqNP.rst

@@ -0,0 +1,2 @@
+Improve ``SyntaxError`` error messages for invalid comparisons. Patch by
+Pablo Galindo.

Parser/pegen.c

@@ -147,8 +147,8 @@ byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset)
         return 0;
     }
     Py_ssize_t len = strlen(str);
-    if (col_offset > len) {
+    if (col_offset > len + 1) {
-        col_offset = len;
+        col_offset = len + 1;
     }
     assert(col_offset >= 0);
     PyObject *text = PyUnicode_DecodeUTF8(str, col_offset, "replace");
@@ -184,7 +184,7 @@ _PyPegen_get_expr_name(expr_ty e)
         case BoolOp_kind:
         case BinOp_kind:
         case UnaryOp_kind:
-            return "operator";
+            return "expression";
         case GeneratorExp_kind:
             return "generator expression";
         case Yield_kind:
@@ -199,7 +199,7 @@ _PyPegen_get_expr_name(expr_ty e)
         case DictComp_kind:
             return "dict comprehension";
         case Dict_kind:
-            return "dict display";
+            return "dict literal";
         case Set_kind:
             return "set display";
         case JoinedStr_kind: