From 3c3aa4516c70753de06bb142b6793d01330fcf0f Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Thu, 2 Apr 2020 15:34:54 -0700 Subject: [PATCH] lib2to3: Support named assignment expressions (GH-12702) There are two copies of the grammar -- the one used by Python itself as Grammar/Grammar, and the one used by lib2to3 which has necessarily diverged at Lib/lib2to3/Grammar.txt because it needs to support older syntax an we want it to be reasonable stable to avoid requiring fixer rewrites. This brings suport for syntax like `if x:= foo():` to match what the live Python grammar does. This should've been added at the time of the walrus operator itself, but lib2to3 being independent is often overlooked. So we do consider this a bugfix rather than enhancement. --- Lib/lib2to3/Grammar.txt | 10 ++++++---- Lib/lib2to3/pgen2/grammar.py | 1 + Lib/lib2to3/pgen2/token.py | 3 ++- Lib/lib2to3/pgen2/tokenize.py | 2 +- Lib/lib2to3/tests/test_parser.py | 15 +++++++++++++++ .../2019-06-18-19-38-27.bpo-36541.XI8mi1.rst | 2 ++ 6 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2019-06-18-19-38-27.bpo-36541.XI8mi1.rst diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index 51f58209f03..e007dc188af 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -67,8 +67,8 @@ assert_stmt: 'assert' test [',' test] compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt async_stmt: ASYNC (funcdef | with_stmt | for_stmt) -if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] -while_stmt: 'while' test ':' suite ['else' ':' suite] +if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite] +while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] try_stmt: ('try' ':' suite ((except_clause ':' suite)+ @@ -91,6 +91,7 @@ testlist_safe: old_test [(',' old_test)+ [',']] old_test: or_test | old_lambdef old_lambdef: 'lambda' [varargslist] ':' old_test +namedexpr_test: test [':=' test] test: or_test ['if' or_test 'else' test] | lambdef or_test: and_test ('or' and_test)* and_test: not_test ('and' not_test)* @@ -111,8 +112,8 @@ atom: ('(' [yield_expr|testlist_gexp] ')' | '{' [dictsetmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+ | '.' '.' '.') -listmaker: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) -testlist_gexp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) +listmaker: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] ) +testlist_gexp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] ) lambdef: 'lambda' [varargslist] ':' test trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME subscriptlist: subscript (',' subscript)* [','] @@ -137,6 +138,7 @@ arglist: argument (',' argument)* [','] # multiple (test comp_for) arguments are blocked; keyword unpackings # that precede iterable unpackings are blocked; etc. argument: ( test [comp_for] | + test ':=' test | test '=' test | '**' test | '*' test ) diff --git a/Lib/lib2to3/pgen2/grammar.py b/Lib/lib2to3/pgen2/grammar.py index a1da546eee2..6a4d575ac2c 100644 --- a/Lib/lib2to3/pgen2/grammar.py +++ b/Lib/lib2to3/pgen2/grammar.py @@ -178,6 +178,7 @@ opmap_raw = """ // DOUBLESLASH //= DOUBLESLASHEQUAL -> RARROW +:= COLONEQUAL """ opmap = {} diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py index 1a679554d2d..5f6612f5b30 100755 --- a/Lib/lib2to3/pgen2/token.py +++ b/Lib/lib2to3/pgen2/token.py @@ -65,7 +65,8 @@ RARROW = 55 AWAIT = 56 ASYNC = 57 ERRORTOKEN = 58 -N_TOKENS = 59 +COLONEQUAL = 59 +N_TOKENS = 60 NT_OFFSET = 256 #--end constants-- diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index 7924ff3cd58..0e2685d4043 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -93,7 +93,7 @@ Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=", r"~") Bracket = '[][(){}]' -Special = group(r'\r?\n', r'[:;.,`@]') +Special = group(r'\r?\n', r':=', r'[:;.,`@]') Funny = group(Operator, Bracket, Special) PlainToken = group(Number, Funny, String, Name) diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index a0c31e8f530..ba2bb787332 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -629,6 +629,21 @@ class TestLiterals(GrammarTest): self.validate(s) +class TestNamedAssignments(GrammarTest): + + def test_named_assignment_if(self): + driver.parse_string("if f := x(): pass\n") + + def test_named_assignment_while(self): + driver.parse_string("while f := x(): pass\n") + + def test_named_assignment_generator(self): + driver.parse_string("any((lastNum := num) == 1 for num in [1, 2, 3])\n") + + def test_named_assignment_listcomp(self): + driver.parse_string("[(lastNum := num) == 1 for num in [1, 2, 3]]\n") + + class TestPickleableException(unittest.TestCase): def test_ParseError(self): err = ParseError('msg', 2, None, (1, 'context')) diff --git a/Misc/NEWS.d/next/Library/2019-06-18-19-38-27.bpo-36541.XI8mi1.rst b/Misc/NEWS.d/next/Library/2019-06-18-19-38-27.bpo-36541.XI8mi1.rst new file mode 100644 index 00000000000..e7b9dd648b4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-06-18-19-38-27.bpo-36541.XI8mi1.rst @@ -0,0 +1,2 @@ +lib2to3 now recognizes named assignment expressions (the walrus operator, +``:=``)