bpo-33064: lib2to3: support trailing comma after *args and **kwargs (#6096)

New tests also added.

I also made the comments in line with the builtin Grammar/Grammar. PEP 306 was
withdrawn, Kees Blom's railroad program has been lost to the sands of time for
at least 16 years now (I found a python-dev post from people looking for it).

Lib/lib2to3/Grammar.txt

@@ -1,26 +1,7 @@
 # Grammar for 2to3. This grammar supports Python 2.x and 3.x.
 
-# Note:  Changing the grammar specified in this file will most likely
-#        require corresponding changes in the parser module
-#        (../Modules/parsermodule.c).  If you can't make the changes to
-#        that module yourself, please co-ordinate the required changes
-#        with someone who can; ask around on python-dev for help.  Fred
-#        Drake <fdrake@acm.org> will probably be listening there.
-
-# NOTE WELL: You should also follow all the steps listed in PEP 306,
-# "How to Change Python's Grammar"
-
-# Commands for Kees Blom's railroad program
-#diagram:token NAME
-#diagram:token NUMBER
-#diagram:token STRING
-#diagram:token NEWLINE
-#diagram:token ENDMARKER
-#diagram:token INDENT
-#diagram:output\input python.bla
-#diagram:token DEDENT
-#diagram:output\textwidth 20.04cm\oddsidemargin  0.0cm\evensidemargin 0.0cm
-#diagram:rules
+# NOTE WELL: You should also follow all the steps listed at
+# https://devguide.python.org/grammar/
 
 # Start symbols for the grammar:
 #	file_input is a module or sequence of commands read from an input file;
@@ -38,13 +19,13 @@ async_funcdef: 'async' funcdef
 funcdef: 'def' NAME parameters ['->' test] ':' suite
 parameters: '(' [typedargslist] ')'
 typedargslist: ((tfpdef ['=' test] ',')*
-                ('*' [tname] (',' tname ['=' test])* [',' '**' tname] | '**' tname)
+                ('*' [tname] (',' tname ['=' test])* [',' ['**' tname [',']]] | '**' tname [','])
                 | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
 tname: NAME [':' test]
 tfpdef: tname | '(' tfplist ')'
 tfplist: tfpdef (',' tfpdef)* [',']
 varargslist: ((vfpdef ['=' test] ',')*
-              ('*' [vname] (',' vname ['=' test])*  [',' '**' vname] | '**' vname)
+              ('*' [vname] (',' vname ['=' test])*  [',' ['**' vname [',']]] | '**' vname [','])
               | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
 vname: NAME
 vfpdef: vname | '(' vfplist ')'

Lib/lib2to3/tests/test_parser.py

@@ -9,7 +9,6 @@ test_grammar.py files from both Python 2 and Python 3.
 # Testing imports
 from . import support
 from .support import driver, driver_no_print_statement
-from test.support import verbose
 
 # Python imports
 import difflib
@@ -22,7 +21,6 @@ import subprocess
 import sys
 import tempfile
 import unittest
-import warnings
 
 # Local imports
 from lib2to3.pgen2 import driver as pgen2_driver
@@ -305,6 +303,38 @@ class TestFunctionAnnotations(GrammarTest):
                         *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass"""
         self.validate(s)
 
+    def test_9(self):
+        s = """def f(
+          a: str,
+          b: int,
+          *,
+          c: bool = False,
+          **kwargs,
+        ) -> None:
+            call(c=c, **kwargs,)"""
+        self.validate(s)
+
+    def test_10(self):
+        s = """def f(
+          a: str,
+        ) -> None:
+            call(a,)"""
+        self.validate(s)
+
+    def test_11(self):
+        s = """def f(
+          a: str = '',
+        ) -> None:
+            call(a=a,)"""
+        self.validate(s)
+
+    def test_12(self):
+        s = """def f(
+          *args: str,
+        ) -> None:
+            call(*args,)"""
+        self.validate(s)
+
 
 # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot
 class TestVarAnnotations(GrammarTest):
@@ -407,7 +437,7 @@ class TestClassDef(GrammarTest):
         self.validate("class B(t, *args): pass")
         self.validate("class B(t, **kwargs): pass")
         self.validate("class B(t, *args, **kwargs): pass")
-        self.validate("class B(t, y=9, *args, **kwargs): pass")
+        self.validate("class B(t, y=9, *args, **kwargs,): pass")
 
 
 class TestParserIdempotency(support.TestCase):

Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst

@@ -0,0 +1,2 @@
+lib2to3 now properly supports trailing commas after ``*args`` and
+``**kwargs`` in function signatures.