From 28325749c01815097aa2bc06508004a1f894c279 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)" Date: Fri, 9 Sep 2016 18:18:52 -0700 Subject: [PATCH] Issue #25969: Update the lib2to3 grammar to handle the unpacking generalizations added in 3.5. --- Lib/lib2to3/Grammar.txt | 23 +++++++++++++++------ Lib/lib2to3/fixes/fix_apply.py | 11 ++++++++++ Lib/lib2to3/fixes/fix_intern.py | 10 +++++++++ Lib/lib2to3/fixes/fix_reload.py | 11 ++++++++++ Lib/lib2to3/tests/test_fixers.py | 4 ++++ Lib/lib2to3/tests/test_parser.py | 35 ++++++++++++++++++++++++++++++++ Misc/NEWS | 3 +++ 7 files changed, 91 insertions(+), 6 deletions(-) diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index c954669e8a7..88f7884b5a5 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -138,15 +138,26 @@ subscript: test | [test] ':' [test] [sliceop] sliceop: ':' [test] exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] testlist: test (',' test)* [','] -dictsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | - (test (comp_for | (',' test)* [','])) ) +dictsetmaker: ( ((test ':' test | '**' expr) + (comp_for | (',' (test ':' test | '**' expr))* [','])) | + ((test | star_expr) + (comp_for | (',' (test | star_expr))* [','])) ) classdef: 'class' NAME ['(' [arglist] ')'] ':' suite -arglist: (argument ',')* (argument [','] - |'*' test (',' argument)* [',' '**' test] - |'**' test) -argument: test [comp_for] | test '=' test # Really [keyword '='] test +arglist: argument (',' argument)* [','] + +# "test '=' test" is really "keyword '=' test", but we have no such token. +# These need to be in a single rule to avoid grammar that is ambiguous +# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr, +# we explicitly match '*' here, too, to give it proper precedence. +# Illegal combinations and orderings are blocked in ast.c: +# multiple (test comp_for) arguements are blocked; keyword unpackings +# that precede iterable unpackings are blocked; etc. +argument: ( test [comp_for] | + test '=' test | + '**' expr | + star_expr ) comp_iter: comp_for | comp_if comp_for: 'for' exprlist 'in' testlist_safe [comp_iter] diff --git a/Lib/lib2to3/fixes/fix_apply.py b/Lib/lib2to3/fixes/fix_apply.py index 81a46dc43fa..826ec8c9b62 100644 --- a/Lib/lib2to3/fixes/fix_apply.py +++ b/Lib/lib2to3/fixes/fix_apply.py @@ -34,6 +34,17 @@ class FixApply(fixer_base.BaseFix): func = results["func"] args = results["args"] kwds = results.get("kwds") + # I feel like we should be able to express this logic in the + # PATTERN above but I don't know how to do it so... + if args: + if args.type == self.syms.star_expr: + return # Make no change. + if (args.type == self.syms.argument and + args.children[0].value == '**'): + return # Make no change. + if kwds and (kwds.type == self.syms.argument and + kwds.children[0].value == '**'): + return # Make no change. prefix = node.prefix func = func.clone() if (func.type not in (token.NAME, syms.atom) and diff --git a/Lib/lib2to3/fixes/fix_intern.py b/Lib/lib2to3/fixes/fix_intern.py index fb2973c2425..a852330908b 100644 --- a/Lib/lib2to3/fixes/fix_intern.py +++ b/Lib/lib2to3/fixes/fix_intern.py @@ -25,6 +25,16 @@ class FixIntern(fixer_base.BaseFix): """ def transform(self, node, results): + if results: + # I feel like we should be able to express this logic in the + # PATTERN above but I don't know how to do it so... + obj = results['obj'] + if obj: + if obj.type == self.syms.star_expr: + return # Make no change. + if (obj.type == self.syms.argument and + obj.children[0].value == '**'): + return # Make no change. names = ('sys', 'intern') new = ImportAndCall(node, results, names) touch_import(None, 'sys', node) diff --git a/Lib/lib2to3/fixes/fix_reload.py b/Lib/lib2to3/fixes/fix_reload.py index 18553575885..b71727377b0 100644 --- a/Lib/lib2to3/fixes/fix_reload.py +++ b/Lib/lib2to3/fixes/fix_reload.py @@ -22,6 +22,17 @@ class FixReload(fixer_base.BaseFix): """ def transform(self, node, results): + if results: + # I feel like we should be able to express this logic in the + # PATTERN above but I don't know how to do it so... + obj = results['obj'] + print('obj:', repr(obj)) + if obj: + if obj.type == self.syms.star_expr: + return # Make no change. + if (obj.type == self.syms.argument and + obj.children[0].value == '**'): + return # Make no change. names = ('imp', 'reload') new = ImportAndCall(node, results, names) touch_import(None, 'imp', node) diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index e69cf321c15..640dcefd94e 100644 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -260,6 +260,10 @@ class Test_apply(FixerTestCase): s = """apply(f, *args)""" self.unchanged(s) + def test_unchanged_6b(self): + s = """apply(f, **kwds)""" + self.unchanged(s) + def test_unchanged_7(self): s = """apply(func=f, args=args, kwds=kwds)""" self.unchanged(s) diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 0b2ca8b384a..e4a0194b822 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -208,6 +208,41 @@ class TestRaiseChanges(GrammarTest): self.invalid_syntax("raise E from") +# Modelled after Lib/test/test_grammar.py:TokenTests.test_funcdef issue2292 +# and Lib/test/text_parser.py test_list_displays, test_set_displays, +# test_dict_displays, test_argument_unpacking, ... changes. +class TestUnpackingGeneralizations(GrammarTest): + def test_mid_positional_star(self): + self.validate("""func(1, *(2, 3), 4)""") + + def test_double_star_dict_literal(self): + self.validate("""func(**{'eggs':'scrambled', 'spam':'fried'})""") + + def test_double_star_dict_literal_after_keywords(self): + self.validate("""func(spam='fried', **{'eggs':'scrambled'})""") + + def test_list_display(self): + self.validate("""[*{2}, 3, *[4]]""") + + def test_set_display(self): + self.validate("""{*{2}, 3, *[4]}""") + + def test_dict_display_1(self): + self.validate("""{**{}}""") + + def test_dict_display_2(self): + self.validate("""{**{}, 3:4, **{5:6, 7:8}}""") + + def test_argument_unpacking_1(self): + self.validate("""f(a, *b, *c, d)""") + + def test_argument_unpacking_2(self): + self.validate("""f(**a, **b)""") + + def test_argument_unpacking_3(self): + self.validate("""f(2, *a, *b, **b, **c, **d)""") + + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef class TestFunctionAnnotations(GrammarTest): def test_1(self): diff --git a/Misc/NEWS b/Misc/NEWS index fcc27ba1cda..bce1688118d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -65,6 +65,9 @@ Core and Builtins Library ------- +- Issue #25969: Update the lib2to3 grammar to handle the unpacking + generalizations added in 3.5. + - Issue #27932: Fixes memory leak in platform.win32_ver() - Issue #14977: mailcap now respects the order of the lines in the mailcap