Issue #9154: Fix parser module to understand function annotations.

Lib/test/test_parser.py

@@ -146,6 +146,27 @@ class RoundtripLegalSyntaxTestCase(unittest.TestCase):
         self.check_suite("@funcattrs()\n"
                          "def f(): pass")
 
+        # keyword-only arguments
+        self.check_suite("def f(*, a): pass")
+        self.check_suite("def f(*, a = 5): pass")
+        self.check_suite("def f(*, a = 5, b): pass")
+        self.check_suite("def f(*, a, b = 5): pass")
+        self.check_suite("def f(*, a, b = 5, **kwds): pass")
+        self.check_suite("def f(*args, a): pass")
+        self.check_suite("def f(*args, a = 5): pass")
+        self.check_suite("def f(*args, a = 5, b): pass")
+        self.check_suite("def f(*args, a, b = 5): pass")
+        self.check_suite("def f(*args, a, b = 5, **kwds): pass")
+
+        # function annotations
+        self.check_suite("def f(a: int): pass")
+        self.check_suite("def f(a: int = 5): pass")
+        self.check_suite("def f(*args: list): pass")
+        self.check_suite("def f(**kwds: dict): pass")
+        self.check_suite("def f(*, a: int): pass")
+        self.check_suite("def f(*, a: int = 5): pass")
+        self.check_suite("def f() -> int: pass")
+
     def test_class_defs(self):
         self.check_suite("class foo():pass")
         self.check_suite("class foo(object):pass")

Misc/NEWS

@@ -56,6 +56,8 @@ Core and Builtins
 Library
 -------
 
+- Issue #9154: Fix parser module to understand function annotations.
+
 - Issue #14664: It is now possible to use @unittest.skip{If,Unless} on a
   test class that doesn't inherit from TestCase (i.e. a mixin).
 

Modules/parsermodule.c

@@ -938,6 +938,7 @@ static int validate_terminal(node *terminal, int type, char *string);
 #define validate_doublestar(ch) validate_terminal(ch, DOUBLESTAR, "**")
 #define validate_dot(ch)        validate_terminal(ch,        DOT, ".")
 #define validate_at(ch)         validate_terminal(ch,         AT, "@")
+#define validate_rarrow(ch)     validate_terminal(ch,     RARROW, "->")
 #define validate_name(ch, str)  validate_terminal(ch,       NAME, str)
 
 #define VALIDATER(n)    static int validate_##n(node *tree)
@@ -1226,68 +1227,68 @@ validate_vfpdef(node *tree)
     return 0;
 }
 
-/* '*' vfpdef (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef
+/* '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef
  * ..or tfpdef in place of vfpdef. vfpdef: NAME; tfpdef: NAME [':' test]
  */
 static int
 validate_varargslist_trailer(node *tree, int start)
 {
     int nch = NCH(tree);
-    int res = 0, i;
+    int res = 0;
-    int sym;
 
     if (nch <= start) {
         err_string("expected variable argument trailer for varargslist");
         return 0;
     }
-    sym = TYPE(CHILD(tree, start));
+    if (TYPE(CHILD(tree, start)) == STAR) {
-    if (sym == STAR) {
         /*
-         * '*' vfpdef (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef
+         * '*' [vfpdef]
          */
-        if (nch-start == 2)
+        res = validate_star(CHILD(tree, start++));
-            res = validate_vfpdef(CHILD(tree, start+1));
+        if (res && start < nch && (TYPE(CHILD(tree, start)) == vfpdef ||
-        else if (nch-start == 5 && TYPE(CHILD(tree, start+2)) == COMMA)
+                                   TYPE(CHILD(tree, start)) == tfpdef))
-            res = (validate_vfpdef(CHILD(tree, start+1))
+            res = validate_vfpdef(CHILD(tree, start++));
-                   && validate_comma(CHILD(tree, start+2))
+        /*
-                   && validate_doublestar(CHILD(tree, start+3))
+         * (',' vfpdef ['=' test])*
-                   && validate_vfpdef(CHILD(tree, start+4)));
+         */
+        while (res && start + 1 < nch && (
+                   TYPE(CHILD(tree, start + 1)) == vfpdef ||
+                   TYPE(CHILD(tree, start + 1)) == tfpdef)) {
+            res = (validate_comma(CHILD(tree, start++))
+                   && validate_vfpdef(CHILD(tree, start++)));
+            if (res && start + 1 < nch && TYPE(CHILD(tree, start)) == EQUAL)
+                res = (validate_equal(CHILD(tree, start++))
+                       && validate_test(CHILD(tree, start++)));
+        }
+        /*
+         * [',' '**' vfpdef]
+         */
+        if (res && start + 2 < nch && TYPE(CHILD(tree, start+1)) == DOUBLESTAR)
+            res = (validate_comma(CHILD(tree, start++))
+                   && validate_doublestar(CHILD(tree, start++))
+                   && validate_vfpdef(CHILD(tree, start++)));
+    }
+    else if (TYPE(CHILD(tree, start)) == DOUBLESTAR) {
+        /*
+         * '**' vfpdef
+         */
+        if (start + 1 < nch)
+            res = (validate_doublestar(CHILD(tree, start++))
+                   && validate_vfpdef(CHILD(tree, start++)));
         else {
-            /* skip over vfpdef (',' vfpdef ['=' test])*  */
+            res = 0;
-            i = start + 1;
+            err_string("expected vfpdef after ** in varargslist trailer");
-            if (TYPE(CHILD(tree, i)) == vfpdef ||
-                TYPE(CHILD(tree, i)) == tfpdef) { /* skip over vfpdef or tfpdef */
-                i += 1;
-            }
-            while (res && i+1 < nch) { /* validate  (',' vfpdef ['=' test])* */
-                res = validate_comma(CHILD(tree, i));
-                if (TYPE(CHILD(tree, i+1)) == DOUBLESTAR)
-                    break;
-                res = res && validate_vfpdef(CHILD(tree, i+1));
-                if (res && i+2 < nch && TYPE(CHILD(tree, i+2)) == EQUAL) {
-                    res = res && (i+3 < nch)
-                          && validate_test(CHILD(tree, i+3));
-                    i += 4;
-                }
-                else {
-                    i += 2;
-                }
-            }
-            /* [',' '**' vfpdef] */
-            if (res && i+1 < nch && TYPE(CHILD(tree, i+1)) == DOUBLESTAR) {
-                res = validate_vfpdef(CHILD(tree, i+2));
-            }
         }
     }
-    else if (sym == DOUBLESTAR) {
+    else {
-        /*
+        res = 0;
-         *  '**' NAME
+        err_string("expected * or ** in varargslist trailer");
-         */
+    }
-        if (nch-start == 2)
+
-            res = validate_vfpdef(CHILD(tree, start+1));
+    if (res && start != nch) {
+        res = 0;
+        err_string("unexpected extra children in varargslist trailer");
     }
-    if (!res)
-        err_string("illegal variable argument trailer for varargslist");
     return res;
 }
 
@@ -2495,23 +2496,36 @@ validate_with_stmt(node *tree)
     return ok;
 }
 
-/*  funcdef:
+/* funcdef: 'def' NAME parameters ['->' test] ':' suite */
- *
+
- *     -5   -4         -3  -2    -1
- *  'def' NAME parameters ':' suite
- */
 static int
 validate_funcdef(node *tree)
 {
     int nch = NCH(tree);
-    int ok = (validate_ntype(tree, funcdef)
+    int res = validate_ntype(tree, funcdef);
-               && (nch == 5)
+    if (res) {
-               && validate_name(RCHILD(tree, -5), "def")
+        if (nch == 5) {
-               && validate_ntype(RCHILD(tree, -4), NAME)
+            res = (validate_name(CHILD(tree, 0), "def")
-               && validate_colon(RCHILD(tree, -2))
+                   && validate_ntype(CHILD(tree, 1), NAME)
-               && validate_parameters(RCHILD(tree, -3))
+                   && validate_parameters(CHILD(tree, 2))
-               && validate_suite(RCHILD(tree, -1)));
+                   && validate_colon(CHILD(tree, 3))
-    return ok;
+                   && validate_suite(CHILD(tree, 4)));
+        }
+        else if (nch == 7) {
+            res = (validate_name(CHILD(tree, 0), "def")
+                   && validate_ntype(CHILD(tree, 1), NAME)
+                   && validate_parameters(CHILD(tree, 2))
+                   && validate_rarrow(CHILD(tree, 3))
+                   && validate_test(CHILD(tree, 4))
+                   && validate_colon(CHILD(tree, 5))
+                   && validate_suite(CHILD(tree, 6)));
+        }
+        else {
+            res = 0;
+            err_string("illegal number of children for funcdef");
+        }
+    }
+    return res;
 }