bpo-29463: Add docstring field to some AST nodes. (#46)

* bpo-29463: Add docstring field to some AST nodes.

ClassDef, ModuleDef, FunctionDef, and AsyncFunctionDef has docstring
field for now.  It was first statement of there body.

* fix document.  thanks travis!

* doc fixes

Doc/library/ast.rst

@@ -133,9 +133,17 @@ and classes for traversing abstract syntax trees:
 .. function:: get_docstring(node, clean=True)
 
    Return the docstring of the given *node* (which must be a
-   :class:`FunctionDef`, :class:`ClassDef` or :class:`Module` node), or ``None``
+   :class:`FunctionDef`, :class:`AsyncFunctionDef`, :class:`ClassDef`,
-   if it has no docstring.  If *clean* is true, clean up the docstring's
+   or :class:`Module` node), or ``None`` if it has no docstring.
-   indentation with :func:`inspect.cleandoc`.
+   If *clean* is true, clean up the docstring's indentation with
+   :func:`inspect.cleandoc`.
+
+   .. versionchanged:: 3.5
+      :class:`AsyncFunctionDef` is now supported.
+
+   .. versionchanged:: 3.7
+      The docstring is now exported from the node docstring field, instead of
+      the first body statement.
 
 
 .. function:: fix_missing_locations(node)

Doc/whatsnew/3.7.rst

@@ -196,6 +196,12 @@ Changes in the Python API
   Use the :meth:`~http.cookies.Morsel.set` method for setting them.
   (Contributed by Serhiy Storchaka in :issue:`29192`.)
 
+* ``Module``, ``FunctionDef``, ``AsyncFunctionDef``, and
+  ``ClassDef`` AST nodes now have a new ``docstring`` field.
+  The first statement in their body is not considered as a docstring
+  anymore.  ``co_firstlineno`` and ``co_lnotab`` of code object for class
+  and module are affected by this change.
+  (Contributed by INADA Naoki and Eugene Toder in :issue:`29463`.)
 
 CPython bytecode changes
 ------------------------

Include/Python-ast.h

@@ -46,6 +46,7 @@ struct _mod {
     union {
         struct {
             asdl_seq *body;
+            string docstring;
         } Module;
         
         struct {
@@ -80,6 +81,7 @@ struct _stmt {
             asdl_seq *body;
             asdl_seq *decorator_list;
             expr_ty returns;
+            string docstring;
         } FunctionDef;
         
         struct {
@@ -88,6 +90,7 @@ struct _stmt {
             asdl_seq *body;
             asdl_seq *decorator_list;
             expr_ty returns;
+            string docstring;
         } AsyncFunctionDef;
         
         struct {
@@ -96,6 +99,7 @@ struct _stmt {
             asdl_seq *keywords;
             asdl_seq *body;
             asdl_seq *decorator_list;
+            string docstring;
         } ClassDef;
         
         struct {
@@ -439,26 +443,27 @@ struct _withitem {
 };
 
 
-#define Module(a0, a1) _Py_Module(a0, a1)
+#define Module(a0, a1, a2) _Py_Module(a0, a1, a2)
-mod_ty _Py_Module(asdl_seq * body, PyArena *arena);
+mod_ty _Py_Module(asdl_seq * body, string docstring, PyArena *arena);
 #define Interactive(a0, a1) _Py_Interactive(a0, a1)
 mod_ty _Py_Interactive(asdl_seq * body, PyArena *arena);
 #define Expression(a0, a1) _Py_Expression(a0, a1)
 mod_ty _Py_Expression(expr_ty body, PyArena *arena);
 #define Suite(a0, a1) _Py_Suite(a0, a1)
 mod_ty _Py_Suite(asdl_seq * body, PyArena *arena);
-#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
+#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8)
 stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
-                        asdl_seq * decorator_list, expr_ty returns, int lineno,
+                        asdl_seq * decorator_list, expr_ty returns, string
-                        int col_offset, PyArena *arena);
+                        docstring, int lineno, int col_offset, PyArena *arena);
-#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
+#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8)
 stmt_ty _Py_AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq *
                              body, asdl_seq * decorator_list, expr_ty returns,
-                             int lineno, int col_offset, PyArena *arena);
+                             string docstring, int lineno, int col_offset,
-#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7)
+                             PyArena *arena);
+#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8)
 stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords,
-                     asdl_seq * body, asdl_seq * decorator_list, int lineno,
+                     asdl_seq * body, asdl_seq * decorator_list, string
-                     int col_offset, PyArena *arena);
+                     docstring, int lineno, int col_offset, PyArena *arena);
 #define Return(a0, a1, a2, a3) _Py_Return(a0, a1, a2, a3)
 stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
 #define Delete(a0, a1, a2, a3) _Py_Delete(a0, a1, a2, a3)

Lib/ast.py

@@ -197,15 +197,7 @@ def get_docstring(node, clean=True):
     """
     if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)):
         raise TypeError("%r can't have docstrings" % node.__class__.__name__)
-    if not(node.body and isinstance(node.body[0], Expr)):
+    text = node.docstring
-        return
-    node = node.body[0].value
-    if isinstance(node, Str):
-        text = node.s
-    elif isinstance(node, Constant) and isinstance(node.value, str):
-        text = node.value
-    else:
-        return
     if clean:
         import inspect
         text = inspect.cleandoc(text)

Lib/test/test_ast.py

@@ -27,8 +27,12 @@ def to_tuple(t):
 exec_tests = [
     # None
     "None",
+    # Module docstring
+    "'module docstring'",
     # FunctionDef
     "def f(): pass",
+    # FunctionDef with docstring
+    "def f(): 'function docstring'",
     # FunctionDef with arg
     "def f(a): pass",
     # FunctionDef with arg and default value
@@ -37,10 +41,12 @@ exec_tests = [
     "def f(*args): pass",
     # FunctionDef with kwargs
     "def f(**kwargs): pass",
-    # FunctionDef with all kind of args
+    # FunctionDef with all kind of args and docstring
-    "def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): pass",
+    "def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): 'doc for f()'",
     # ClassDef
     "class C:pass",
+    # ClassDef with docstring
+    "class C: 'docstring for class C'",
     # ClassDef, new style class
     "class C(object): pass",
     # Return
@@ -108,7 +114,7 @@ exec_tests = [
     # setcomp with naked tuple
     "{r for l,m in x}",
     # AsyncFunctionDef
-    "async def f():\n await something()",
+    "async def f():\n 'async function'\n await something()",
     # AsyncFor
     "async def f():\n async for e in i: 1\n else: 2",
     # AsyncWith
@@ -321,9 +327,24 @@ class AST_Tests(unittest.TestCase):
 
     def test_module(self):
         body = [ast.Num(42)]
-        x = ast.Module(body)
+        x = ast.Module(body, None)
         self.assertEqual(x.body, body)
 
+    def test_docstring(self):
+        body = []  # AST nodes having docstring must accept empty body
+        x = ast.Module(body, "module docstring")
+        self.assertEqual(x.docstring, "module docstring")
+
+        a = ast.arguments()
+        x = ast.FunctionDef("x", a, body, [], None, "func docstring")
+        self.assertEqual(x.docstring, "func docstring")
+
+        x = ast.AsyncFunctionDef("x", a, body, [], None, "async func docstring")
+        self.assertEqual(x.docstring, "async func docstring")
+
+        x = ast.ClassDef("x", [], [], body, [], "class docstring")
+        self.assertEqual(x.docstring, "class docstring")
+
     def test_nodeclasses(self):
         # Zero arguments constructor explicitly allowed
         x = ast.BinOp()
@@ -394,25 +415,18 @@ class AST_Tests(unittest.TestCase):
 
     def test_invalid_sum(self):
         pos = dict(lineno=2, col_offset=3)
-        m = ast.Module([ast.Expr(ast.expr(**pos), **pos)])
+        m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], "doc")
         with self.assertRaises(TypeError) as cm:
             compile(m, "<test>", "exec")
         self.assertIn("but got <_ast.expr", str(cm.exception))
 
     def test_invalid_identitifer(self):
-        m = ast.Module([ast.Expr(ast.Name(42, ast.Load()))])
+        m = ast.Module([ast.Expr(ast.Name(42, ast.Load()))], None)
         ast.fix_missing_locations(m)
         with self.assertRaises(TypeError) as cm:
             compile(m, "<test>", "exec")
         self.assertIn("identifier must be of type str", str(cm.exception))
 
-    def test_invalid_string(self):
-        m = ast.Module([ast.Expr(ast.Str(42))])
-        ast.fix_missing_locations(m)
-        with self.assertRaises(TypeError) as cm:
-            compile(m, "<test>", "exec")
-        self.assertIn("string must be of type str", str(cm.exception))
-
     def test_empty_yield_from(self):
         # Issue 16546: yield from value is not optional.
         empty_yield_from = ast.parse("def f():\n yield from g()")
@@ -442,18 +456,18 @@ class ASTHelpers_Test(unittest.TestCase):
         self.assertEqual(ast.dump(node),
             "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), "
             "args=[Name(id='eggs', ctx=Load()), Str(s='and cheese')], "
-            "keywords=[]))])"
+            "keywords=[]))], docstring=None)"
         )
         self.assertEqual(ast.dump(node, annotate_fields=False),
             "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), "
-            "Str('and cheese')], []))])"
+            "Str('and cheese')], []))], None)"
         )
         self.assertEqual(ast.dump(node, include_attributes=True),
             "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), "
             "lineno=1, col_offset=0), args=[Name(id='eggs', ctx=Load(), "
             "lineno=1, col_offset=5), Str(s='and cheese', lineno=1, "
             "col_offset=11)], keywords=[], "
-            "lineno=1, col_offset=0), lineno=1, col_offset=0)])"
+            "lineno=1, col_offset=0), lineno=1, col_offset=0)], docstring=None)"
         )
 
     def test_copy_location(self):
@@ -478,7 +492,7 @@ class ASTHelpers_Test(unittest.TestCase):
             "Expr(value=Call(func=Name(id='spam', ctx=Load(), lineno=1, "
             "col_offset=0), args=[Str(s='eggs', lineno=1, col_offset=0)], "
             "keywords=[], lineno=1, "
-            "col_offset=0), lineno=1, col_offset=0)])"
+            "col_offset=0), lineno=1, col_offset=0)], docstring=None)"
         )
 
     def test_increment_lineno(self):
@@ -545,7 +559,7 @@ class ASTHelpers_Test(unittest.TestCase):
                                names=[ast.alias(name='sleep')],
                                level=None,
                                lineno=None, col_offset=None)]
-        mod = ast.Module(body)
+        mod = ast.Module(body, None)
         with self.assertRaises(ValueError) as cm:
             compile(mod, 'test', 'exec')
         self.assertIn("invalid integer value: None", str(cm.exception))
@@ -555,7 +569,7 @@ class ASTHelpers_Test(unittest.TestCase):
                                names=[ast.alias(name='sleep')],
                                level=None,
                                lineno=0, col_offset=0)]
-        mod = ast.Module(body)
+        mod = ast.Module(body, None)
         code = compile(mod, 'test', 'exec')
         ns = {}
         exec(code, ns)
@@ -573,11 +587,11 @@ class ASTValidatorTests(unittest.TestCase):
             self.assertIn(msg, str(cm.exception))
 
     def expr(self, node, msg=None, *, exc=ValueError):
-        mod = ast.Module([ast.Expr(node)])
+        mod = ast.Module([ast.Expr(node)], None)
         self.mod(mod, msg, exc=exc)
 
     def stmt(self, stmt, msg=None):
-        mod = ast.Module([stmt])
+        mod = ast.Module([stmt], None)
         self.mod(mod, msg)
 
     def test_module(self):
@@ -619,16 +633,16 @@ class ASTValidatorTests(unittest.TestCase):
 
     def test_funcdef(self):
         a = ast.arguments([], None, [], [], None, [])
-        f = ast.FunctionDef("x", a, [], [], None)
+        f = ast.FunctionDef("x", a, [], [], None, None)
         self.stmt(f, "empty body on FunctionDef")
         f = ast.FunctionDef("x", a, [ast.Pass()], [ast.Name("x", ast.Store())],
-                            None)
+                            None, None)
         self.stmt(f, "must have Load context")
         f = ast.FunctionDef("x", a, [ast.Pass()], [],
-                            ast.Name("x", ast.Store()))
+                            ast.Name("x", ast.Store()), None)
         self.stmt(f, "must have Load context")
         def fac(args):
-            return ast.FunctionDef("x", args, [ast.Pass()], [], None)
+            return ast.FunctionDef("x", args, [ast.Pass()], [], None, None)
         self._check_arguments(fac, self.stmt)
 
     def test_classdef(self):
@@ -642,7 +656,7 @@ class ASTValidatorTests(unittest.TestCase):
             if decorator_list is None:
                 decorator_list = []
             return ast.ClassDef("myclass", bases, keywords,
-                                body, decorator_list)
+                                body, decorator_list, None)
         self.stmt(cls(bases=[ast.Name("x", ast.Store())]),
                   "must have Load context")
         self.stmt(cls(keywords=[ast.keyword("x", ast.Name("x", ast.Store()))]),
@@ -1009,9 +1023,6 @@ class ConstantTests(unittest.TestCase):
         tree = ast.parse("'docstring'\nx = 1")
         self.assertEqual(ast.get_docstring(tree), 'docstring')
 
-        tree.body[0].value = ast.Constant(value='constant docstring')
-        self.assertEqual(ast.get_docstring(tree), 'constant docstring')
-
     def get_load_const(self, tree):
         # Compile to bytecode, disassemble and get parameter of LOAD_CONST
         # instructions
@@ -1083,50 +1094,53 @@ def main():
 
 #### EVERYTHING BELOW IS GENERATED #####
 exec_results = [
-('Module', [('Expr', (1, 0), ('NameConstant', (1, 0), None))]),
+('Module', [('Expr', (1, 0), ('NameConstant', (1, 0), None))], None),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (1, 9))], [], None)]),
+('Module', [], 'module docstring'),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, []), [('Pass', (1, 10))], [], None)]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (1, 9))], [], None, None)], None),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, [('Num', (1, 8), 0)]), [('Pass', (1, 12))], [], None)]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [], [], None, 'function docstring')], None),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], ('arg', (1, 7), 'args', None), [], [], None, []), [('Pass', (1, 14))], [], None)]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, []), [('Pass', (1, 10))], [], None, None)], None),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], ('arg', (1, 8), 'kwargs', None), []), [('Pass', (1, 17))], [], None)]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, [('Num', (1, 8), 0)]), [('Pass', (1, 12))], [], None, None)], None),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None), ('arg', (1, 9), 'b', None), ('arg', (1, 14), 'c', None), ('arg', (1, 22), 'd', None), ('arg', (1, 28), 'e', None)], ('arg', (1, 35), 'args', None), [('arg', (1, 41), 'f', None)], [('Num', (1, 43), 42)], ('arg', (1, 49), 'kwargs', None), [('Num', (1, 11), 1), ('NameConstant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('Pass', (1, 58))], [], None)]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], ('arg', (1, 7), 'args', None), [], [], None, []), [('Pass', (1, 14))], [], None, None)], None),
-('Module', [('ClassDef', (1, 0), 'C', [], [], [('Pass', (1, 8))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], ('arg', (1, 8), 'kwargs', None), []), [('Pass', (1, 17))], [], None, None)], None),
-('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], [('Pass', (1, 17))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None), ('arg', (1, 9), 'b', None), ('arg', (1, 14), 'c', None), ('arg', (1, 22), 'd', None), ('arg', (1, 28), 'e', None)], ('arg', (1, 35), 'args', None), [('arg', (1, 41), 'f', None)], [('Num', (1, 43), 42)], ('arg', (1, 49), 'kwargs', None), [('Num', (1, 11), 1), ('NameConstant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [], [], None, 'doc for f()')], None),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None)]),
+('Module', [('ClassDef', (1, 0), 'C', [], [], [('Pass', (1, 8))], [], None)], None),
-('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
+('Module', [('ClassDef', (1, 0), 'C', [], [], [], [], 'docstring for class C')], None),
-('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]),
+('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], [('Pass', (1, 17))], [], None)], None),
-('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Num', (1, 5), 1))]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None, None)], None),
-('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]),
+('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])], None),
-('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]),
+('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))], None),
-('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]),
+('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Num', (1, 5), 1))], None),
-('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))])]),
+('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])], None),
-('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))])]),
+('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])], None),
-('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], []), None)]),
+('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])], None),
-('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])]),
+('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))])], None),
-('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])]),
+('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))])], None),
-('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]),
+('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], []), None)], None),
-('Module', [('Import', (1, 0), [('alias', 'sys', None)])]),
+('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])], None),
-('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)]),
+('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])], None),
-('Module', [('Global', (1, 0), ['v'])]),
+('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)], None),
-('Module', [('Expr', (1, 0), ('Num', (1, 0), 1))]),
+('Module', [('Import', (1, 0), [('alias', 'sys', None)])], None),
-('Module', [('Pass', (1, 0))]),
+('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)], None),
-('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Break', (1, 11))], [])]),
+('Module', [('Global', (1, 0), ['v'])], None),
-('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Continue', (1, 11))], [])]),
+('Module', [('Expr', (1, 0), ('Num', (1, 0), 1))], None),
-('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 4), 'a', ('Store',)), ('Name', (1, 6), 'b', ('Store',))], ('Store',)), ('Name', (1, 11), 'c', ('Load',)), [('Pass', (1, 14))], [])]),
+('Module', [('Pass', (1, 0))], None),
-('Module', [('Expr', (1, 0), ('ListComp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [], 0)]))]),
+('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Break', (1, 11))], [])], None),
-('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [], 0)]))]),
+('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Continue', (1, 11))], [])], None),
-('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 12), [('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 20), 'c', ('Load',)), [], 0)]))]),
+('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 4), 'a', ('Store',)), ('Name', (1, 6), 'b', ('Store',))], ('Store',)), ('Name', (1, 11), 'c', ('Load',)), [('Pass', (1, 14))], [])], None),
-('Module', [('Expr', (1, 0), ('GeneratorExp', (2, 4), ('Tuple', (3, 4), [('Name', (3, 4), 'Aa', ('Load',)), ('Name', (5, 7), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4), [('Name', (8, 4), 'Aa', ('Store',)), ('Name', (10, 4), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10), 'Cc', ('Load',)), [], 0)]))]),
+('Module', [('Expr', (1, 0), ('ListComp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [], 0)]))], None),
-('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Name', (1, 11), 'w', ('Store',)), ('Name', (1, 16), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22), 'm', ('Store',)), ('Name', (1, 27), 'p', ('Load',)), [('Name', (1, 32), 'g', ('Load',))], 0)]))]),
+('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [], 0)]))], None),
-('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [], 0)]))]),
+('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 12), [('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 20), 'c', ('Load',)), [], 0)]))], None),
-('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))], 0)]))]),
+('Module', [('Expr', (1, 0), ('GeneratorExp', (2, 4), ('Tuple', (3, 4), [('Name', (3, 4), 'Aa', ('Load',)), ('Name', (5, 7), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4), [('Name', (8, 4), 'Aa', ('Store',)), ('Name', (10, 4), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10), 'Cc', ('Load',)), [], 0)]))], None),
-('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [], 0)]))]),
+('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Name', (1, 11), 'w', ('Store',)), ('Name', (1, 16), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22), 'm', ('Store',)), ('Name', (1, 27), 'p', ('Load',)), [('Name', (1, 32), 'g', ('Load',))], 0)]))], None),
-('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('Await', (2, 1), ('Call', (2, 7), ('Name', (2, 7), 'something', ('Load',)), [], [])))], [], None)]),
+('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [], 0)]))], None),
-('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('AsyncFor', (2, 7), ('Name', (2, 11), 'e', ('Store',)), ('Name', (2, 16), 'i', ('Load',)), [('Expr', (2, 19), ('Num', (2, 19), 1))], [('Expr', (3, 7), ('Num', (3, 7), 2))])], [], None)]),
+('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))], 0)]))], None),
-('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('AsyncWith', (2, 7), [('withitem', ('Name', (2, 12), 'a', ('Load',)), ('Name', (2, 17), 'b', ('Store',)))], [('Expr', (2, 20), ('Num', (2, 20), 1))])], [], None)]),
+('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [], 0)]))], None),
-('Module', [('Expr', (1, 0), ('Dict', (1, 0), [None, ('Num', (1, 10), 2)], [('Dict', (1, 3), [('Num', (1, 4), 1)], [('Num', (1, 6), 2)]), ('Num', (1, 12), 3)]))]),
+('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (3, 1), ('Await', (3, 1), ('Call', (3, 7), ('Name', (3, 7), 'something', ('Load',)), [], [])))], [], None, 'async function')], None),
-('Module', [('Expr', (1, 0), ('Set', (1, 0), [('Starred', (1, 1), ('Set', (1, 2), [('Num', (1, 3), 1), ('Num', (1, 6), 2)]), ('Load',)), ('Num', (1, 10), 3)]))]),
+('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('AsyncFor', (2, 7), ('Name', (2, 11), 'e', ('Store',)), ('Name', (2, 16), 'i', ('Load',)), [('Expr', (2, 19), ('Num', (2, 19), 1))], [('Expr', (3, 7), ('Num', (3, 7), 2))])], [], None, None)], None),
-('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('ListComp', (2, 2), ('Name', (2, 2), 'i', ('Load',)), [('comprehension', ('Name', (2, 14), 'b', ('Store',)), ('Name', (2, 19), 'c', ('Load',)), [], 1)]))], [], None)]),
+('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('AsyncWith', (2, 7), [('withitem', ('Name', (2, 12), 'a', ('Load',)), ('Name', (2, 17), 'b', ('Store',)))], [('Expr', (2, 20), ('Num', (2, 20), 1))])], [], None, None)], None),
+('Module', [('Expr', (1, 0), ('Dict', (1, 0), [None, ('Num', (1, 10), 2)], [('Dict', (1, 3), [('Num', (1, 4), 1)], [('Num', (1, 6), 2)]), ('Num', (1, 12), 3)]))], None),
+('Module', [('Expr', (1, 0), ('Set', (1, 0), [('Starred', (1, 1), ('Set', (1, 2), [('Num', (1, 3), 1), ('Num', (1, 6), 2)]), ('Load',)), ('Num', (1, 10), 3)]))], None),
+('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('ListComp', (2, 2), ('Name', (2, 2), 'i', ('Load',)), [('comprehension', ('Name', (2, 14), 'b', ('Store',)), ('Name', (2, 19), 'c', ('Load',)), [], 1)]))], [], None, None)], None),
 ]
 single_results = [
 ('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),

Lib/test/test_opcodes.py

@@ -27,7 +27,7 @@ class OpcodeTest(unittest.TestCase):
             with open(ann_module.__file__) as f:
                 txt = f.read()
             co = compile(txt, ann_module.__file__, 'exec')
-            self.assertEqual(co.co_firstlineno, 6)
+            self.assertEqual(co.co_firstlineno, 8)
         except OSError:
             pass
 

Misc/NEWS

@@ -19,6 +19,10 @@ Core and Builtins
 
 - bpo-29347: Fixed possibly dereferencing undefined pointers
   when creating weakref objects.
+- bpo-29463: Add ``docstring`` field to Module, ClassDef, FunctionDef,
+  and AsyncFunctionDef ast nodes.  docstring is not first stmt in their body
+  anymore.  It affects ``co_firstlineno`` and ``co_lnotab`` of code object
+  for module and class.
 
 - bpo-29438: Fixed use-after-free problem in key sharing dict.
 

Parser/Python.asdl

@@ -6,7 +6,7 @@
 
 module Python
 {
-    mod = Module(stmt* body)
+    mod = Module(stmt* body, string? docstring)
         | Interactive(stmt* body)
         | Expression(expr body)
 
@@ -14,15 +14,18 @@ module Python
         | Suite(stmt* body)
 
     stmt = FunctionDef(identifier name, arguments args,
-                       stmt* body, expr* decorator_list, expr? returns)
+                       stmt* body, expr* decorator_list, expr? returns,
+                       string? docstring)
           | AsyncFunctionDef(identifier name, arguments args,
-                             stmt* body, expr* decorator_list, expr? returns)
+                             stmt* body, expr* decorator_list, expr? returns,
+                             string? docstring)
 
           | ClassDef(identifier name,
              expr* bases,
              keyword* keywords,
              stmt* body,
-             expr* decorator_list)
+             expr* decorator_list,
+             string? docstring)
           | Return(expr? value)
 
           | Delete(expr* targets)

Python/Python-ast.c

@@ -10,8 +10,10 @@ static PyTypeObject *mod_type;
 static PyObject* ast2obj_mod(void*);
 static PyTypeObject *Module_type;
 _Py_IDENTIFIER(body);
+_Py_IDENTIFIER(docstring);
 static char *Module_fields[]={
     "body",
+    "docstring",
 };
 static PyTypeObject *Interactive_type;
 static char *Interactive_fields[]={
@@ -44,6 +46,7 @@ static char *FunctionDef_fields[]={
     "body",
     "decorator_list",
     "returns",
+    "docstring",
 };
 static PyTypeObject *AsyncFunctionDef_type;
 static char *AsyncFunctionDef_fields[]={
@@ -52,6 +55,7 @@ static char *AsyncFunctionDef_fields[]={
     "body",
     "decorator_list",
     "returns",
+    "docstring",
 };
 static PyTypeObject *ClassDef_type;
 _Py_IDENTIFIER(bases);
@@ -62,6 +66,7 @@ static char *ClassDef_fields[]={
     "keywords",
     "body",
     "decorator_list",
+    "docstring",
 };
 static PyTypeObject *Return_type;
 _Py_IDENTIFIER(value);
@@ -861,7 +866,7 @@ static int init_types(void)
     mod_type = make_type("mod", &AST_type, NULL, 0);
     if (!mod_type) return 0;
     if (!add_attributes(mod_type, NULL, 0)) return 0;
-    Module_type = make_type("Module", mod_type, Module_fields, 1);
+    Module_type = make_type("Module", mod_type, Module_fields, 2);
     if (!Module_type) return 0;
     Interactive_type = make_type("Interactive", mod_type, Interactive_fields,
                                  1);
@@ -874,12 +879,12 @@ static int init_types(void)
     if (!stmt_type) return 0;
     if (!add_attributes(stmt_type, stmt_attributes, 2)) return 0;
     FunctionDef_type = make_type("FunctionDef", stmt_type, FunctionDef_fields,
-                                 5);
+                                 6);
     if (!FunctionDef_type) return 0;
     AsyncFunctionDef_type = make_type("AsyncFunctionDef", stmt_type,
-                                      AsyncFunctionDef_fields, 5);
+                                      AsyncFunctionDef_fields, 6);
     if (!AsyncFunctionDef_type) return 0;
-    ClassDef_type = make_type("ClassDef", stmt_type, ClassDef_fields, 5);
+    ClassDef_type = make_type("ClassDef", stmt_type, ClassDef_fields, 6);
     if (!ClassDef_type) return 0;
     Return_type = make_type("Return", stmt_type, Return_fields, 1);
     if (!Return_type) return 0;
@@ -1206,7 +1211,7 @@ static int obj2ast_alias(PyObject* obj, alias_ty* out, PyArena* arena);
 static int obj2ast_withitem(PyObject* obj, withitem_ty* out, PyArena* arena);
 
 mod_ty
-Module(asdl_seq * body, PyArena *arena)
+Module(asdl_seq * body, string docstring, PyArena *arena)
 {
     mod_ty p;
     p = (mod_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1214,6 +1219,7 @@ Module(asdl_seq * body, PyArena *arena)
         return NULL;
     p->kind = Module_kind;
     p->v.Module.body = body;
+    p->v.Module.docstring = docstring;
     return p;
 }
 
@@ -1260,8 +1266,8 @@ Suite(asdl_seq * body, PyArena *arena)
 
 stmt_ty
 FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
-            decorator_list, expr_ty returns, int lineno, int col_offset,
+            decorator_list, expr_ty returns, string docstring, int lineno, int
-            PyArena *arena)
+            col_offset, PyArena *arena)
 {
     stmt_ty p;
     if (!name) {
@@ -1283,6 +1289,7 @@ FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
     p->v.FunctionDef.body = body;
     p->v.FunctionDef.decorator_list = decorator_list;
     p->v.FunctionDef.returns = returns;
+    p->v.FunctionDef.docstring = docstring;
     p->lineno = lineno;
     p->col_offset = col_offset;
     return p;
@@ -1290,8 +1297,8 @@ FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
 
 stmt_ty
 AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq
-                 * decorator_list, expr_ty returns, int lineno, int col_offset,
+                 * decorator_list, expr_ty returns, string docstring, int
-                 PyArena *arena)
+                 lineno, int col_offset, PyArena *arena)
 {
     stmt_ty p;
     if (!name) {
@@ -1313,6 +1320,7 @@ AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq
     p->v.AsyncFunctionDef.body = body;
     p->v.AsyncFunctionDef.decorator_list = decorator_list;
     p->v.AsyncFunctionDef.returns = returns;
+    p->v.AsyncFunctionDef.docstring = docstring;
     p->lineno = lineno;
     p->col_offset = col_offset;
     return p;
@@ -1320,8 +1328,8 @@ AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq
 
 stmt_ty
 ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, asdl_seq *
-         body, asdl_seq * decorator_list, int lineno, int col_offset, PyArena
+         body, asdl_seq * decorator_list, string docstring, int lineno, int
-         *arena)
+         col_offset, PyArena *arena)
 {
     stmt_ty p;
     if (!name) {
@@ -1338,6 +1346,7 @@ ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, asdl_seq *
     p->v.ClassDef.keywords = keywords;
     p->v.ClassDef.body = body;
     p->v.ClassDef.decorator_list = decorator_list;
+    p->v.ClassDef.docstring = docstring;
     p->lineno = lineno;
     p->col_offset = col_offset;
     return p;
@@ -2601,6 +2610,11 @@ ast2obj_mod(void* _o)
         if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
             goto failed;
         Py_DECREF(value);
+        value = ast2obj_string(o->v.Module.docstring);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_docstring, value) == -1)
+            goto failed;
+        Py_DECREF(value);
         break;
     case Interactive_kind:
         result = PyType_GenericNew(Interactive_type, NULL, NULL);
@@ -2675,6 +2689,11 @@ ast2obj_stmt(void* _o)
         if (_PyObject_SetAttrId(result, &PyId_returns, value) == -1)
             goto failed;
         Py_DECREF(value);
+        value = ast2obj_string(o->v.FunctionDef.docstring);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_docstring, value) == -1)
+            goto failed;
+        Py_DECREF(value);
         break;
     case AsyncFunctionDef_kind:
         result = PyType_GenericNew(AsyncFunctionDef_type, NULL, NULL);
@@ -2705,6 +2724,11 @@ ast2obj_stmt(void* _o)
         if (_PyObject_SetAttrId(result, &PyId_returns, value) == -1)
             goto failed;
         Py_DECREF(value);
+        value = ast2obj_string(o->v.AsyncFunctionDef.docstring);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_docstring, value) == -1)
+            goto failed;
+        Py_DECREF(value);
         break;
     case ClassDef_kind:
         result = PyType_GenericNew(ClassDef_type, NULL, NULL);
@@ -2734,6 +2758,11 @@ ast2obj_stmt(void* _o)
         if (_PyObject_SetAttrId(result, &PyId_decorator_list, value) == -1)
             goto failed;
         Py_DECREF(value);
+        value = ast2obj_string(o->v.ClassDef.docstring);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_docstring, value) == -1)
+            goto failed;
+        Py_DECREF(value);
         break;
     case Return_kind:
         result = PyType_GenericNew(Return_type, NULL, NULL);
@@ -3974,6 +4003,7 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
     }
     if (isinstance) {
         asdl_seq* body;
+        string docstring;
 
         if (_PyObject_HasAttrId(obj, &PyId_body)) {
             int res;
@@ -4003,7 +4033,17 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
             PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Module");
             return 1;
         }
-        *out = Module(body, arena);
+        if (exists_not_none(obj, &PyId_docstring)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_docstring);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_string(tmp, &docstring, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            docstring = NULL;
+        }
+        *out = Module(body, docstring, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -4159,6 +4199,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         asdl_seq* body;
         asdl_seq* decorator_list;
         expr_ty returns;
+        string docstring;
 
         if (_PyObject_HasAttrId(obj, &PyId_name)) {
             int res;
@@ -4248,8 +4289,18 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         } else {
             returns = NULL;
         }
-        *out = FunctionDef(name, args, body, decorator_list, returns, lineno,
+        if (exists_not_none(obj, &PyId_docstring)) {
-                           col_offset, arena);
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_docstring);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_string(tmp, &docstring, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            docstring = NULL;
+        }
+        *out = FunctionDef(name, args, body, decorator_list, returns,
+                           docstring, lineno, col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -4263,6 +4314,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         asdl_seq* body;
         asdl_seq* decorator_list;
         expr_ty returns;
+        string docstring;
 
         if (_PyObject_HasAttrId(obj, &PyId_name)) {
             int res;
@@ -4352,8 +4404,18 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         } else {
             returns = NULL;
         }
+        if (exists_not_none(obj, &PyId_docstring)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_docstring);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_string(tmp, &docstring, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            docstring = NULL;
+        }
         *out = AsyncFunctionDef(name, args, body, decorator_list, returns,
-                                lineno, col_offset, arena);
+                                docstring, lineno, col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -4367,6 +4429,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         asdl_seq* keywords;
         asdl_seq* body;
         asdl_seq* decorator_list;
+        string docstring;
 
         if (_PyObject_HasAttrId(obj, &PyId_name)) {
             int res;
@@ -4491,8 +4554,18 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             PyErr_SetString(PyExc_TypeError, "required field \"decorator_list\" missing from ClassDef");
             return 1;
         }
-        *out = ClassDef(name, bases, keywords, body, decorator_list, lineno,
+        if (exists_not_none(obj, &PyId_docstring)) {
-                        col_offset, arena);
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_docstring);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_string(tmp, &docstring, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            docstring = NULL;
+        }
+        *out = ClassDef(name, bases, keywords, body, decorator_list, docstring,
+                        lineno, col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }

Python/ast.c

@@ -366,9 +366,12 @@ validate_assignlist(asdl_seq *targets, expr_context_ty ctx)
 }
 
 static int
-validate_body(asdl_seq *body, const char *owner)
+validate_body(asdl_seq *body, const char *owner, int allowempty)
 {
-    return validate_nonempty_seq(body, "body", owner) && validate_stmts(body);
+    if (!allowempty && !validate_nonempty_seq(body, "body", owner)) {
+        return 0;
+    }
+    return validate_stmts(body);
 }
 
 static int
@@ -377,13 +380,15 @@ validate_stmt(stmt_ty stmt)
     int i;
     switch (stmt->kind) {
     case FunctionDef_kind:
-        return validate_body(stmt->v.FunctionDef.body, "FunctionDef") &&
+        return validate_body(stmt->v.FunctionDef.body, "FunctionDef",
+                             stmt->v.FunctionDef.docstring != NULL) &&
             validate_arguments(stmt->v.FunctionDef.args) &&
             validate_exprs(stmt->v.FunctionDef.decorator_list, Load, 0) &&
             (!stmt->v.FunctionDef.returns ||
              validate_expr(stmt->v.FunctionDef.returns, Load));
     case ClassDef_kind:
-        return validate_body(stmt->v.ClassDef.body, "ClassDef") &&
+        return validate_body(stmt->v.ClassDef.body, "ClassDef",
+                             stmt->v.ClassDef.docstring != NULL) &&
             validate_exprs(stmt->v.ClassDef.bases, Load, 0) &&
             validate_keywords(stmt->v.ClassDef.keywords) &&
             validate_exprs(stmt->v.ClassDef.decorator_list, Load, 0);
@@ -411,20 +416,20 @@ validate_stmt(stmt_ty stmt)
     case For_kind:
         return validate_expr(stmt->v.For.target, Store) &&
             validate_expr(stmt->v.For.iter, Load) &&
-            validate_body(stmt->v.For.body, "For") &&
+            validate_body(stmt->v.For.body, "For", 0) &&
             validate_stmts(stmt->v.For.orelse);
     case AsyncFor_kind:
         return validate_expr(stmt->v.AsyncFor.target, Store) &&
             validate_expr(stmt->v.AsyncFor.iter, Load) &&
-            validate_body(stmt->v.AsyncFor.body, "AsyncFor") &&
+            validate_body(stmt->v.AsyncFor.body, "AsyncFor", 0) &&
             validate_stmts(stmt->v.AsyncFor.orelse);
     case While_kind:
         return validate_expr(stmt->v.While.test, Load) &&
-            validate_body(stmt->v.While.body, "While") &&
+            validate_body(stmt->v.While.body, "While", 0) &&
             validate_stmts(stmt->v.While.orelse);
     case If_kind:
         return validate_expr(stmt->v.If.test, Load) &&
-            validate_body(stmt->v.If.body, "If") &&
+            validate_body(stmt->v.If.body, "If", 0) &&
             validate_stmts(stmt->v.If.orelse);
     case With_kind:
         if (!validate_nonempty_seq(stmt->v.With.items, "items", "With"))
@@ -435,7 +440,7 @@ validate_stmt(stmt_ty stmt)
                 (item->optional_vars && !validate_expr(item->optional_vars, Store)))
                 return 0;
         }
-        return validate_body(stmt->v.With.body, "With");
+        return validate_body(stmt->v.With.body, "With", 0);
     case AsyncWith_kind:
         if (!validate_nonempty_seq(stmt->v.AsyncWith.items, "items", "AsyncWith"))
             return 0;
@@ -445,7 +450,7 @@ validate_stmt(stmt_ty stmt)
                 (item->optional_vars && !validate_expr(item->optional_vars, Store)))
                 return 0;
         }
-        return validate_body(stmt->v.AsyncWith.body, "AsyncWith");
+        return validate_body(stmt->v.AsyncWith.body, "AsyncWith", 0);
     case Raise_kind:
         if (stmt->v.Raise.exc) {
             return validate_expr(stmt->v.Raise.exc, Load) &&
@@ -457,7 +462,7 @@ validate_stmt(stmt_ty stmt)
         }
         return 1;
     case Try_kind:
-        if (!validate_body(stmt->v.Try.body, "Try"))
+        if (!validate_body(stmt->v.Try.body, "Try", 0))
             return 0;
         if (!asdl_seq_LEN(stmt->v.Try.handlers) &&
             !asdl_seq_LEN(stmt->v.Try.finalbody)) {
@@ -473,7 +478,7 @@ validate_stmt(stmt_ty stmt)
             excepthandler_ty handler = asdl_seq_GET(stmt->v.Try.handlers, i);
             if ((handler->v.ExceptHandler.type &&
                  !validate_expr(handler->v.ExceptHandler.type, Load)) ||
-                !validate_body(handler->v.ExceptHandler.body, "ExceptHandler"))
+                !validate_body(handler->v.ExceptHandler.body, "ExceptHandler", 0))
                 return 0;
         }
         return (!asdl_seq_LEN(stmt->v.Try.finalbody) ||
@@ -498,7 +503,8 @@ validate_stmt(stmt_ty stmt)
     case Expr_kind:
         return validate_expr(stmt->v.Expr.value, Load);
     case AsyncFunctionDef_kind:
-        return validate_body(stmt->v.AsyncFunctionDef.body, "AsyncFunctionDef") &&
+        return validate_body(stmt->v.AsyncFunctionDef.body, "AsyncFunctionDef",
+                             stmt->v.AsyncFunctionDef.docstring != NULL) &&
             validate_arguments(stmt->v.AsyncFunctionDef.args) &&
             validate_exprs(stmt->v.AsyncFunctionDef.decorator_list, Load, 0) &&
             (!stmt->v.AsyncFunctionDef.returns ||
@@ -594,7 +600,9 @@ struct compiling {
 static asdl_seq *seq_for_testlist(struct compiling *, const node *);
 static expr_ty ast_for_expr(struct compiling *, const node *);
 static stmt_ty ast_for_stmt(struct compiling *, const node *);
-static asdl_seq *ast_for_suite(struct compiling *, const node *);
+static asdl_seq *ast_for_body(struct compiling *c, const node *n,
+                              string *docstring);
+static string docstring_from_stmts(asdl_seq *stmts);
 static asdl_seq *ast_for_exprlist(struct compiling *, const node *,
                                   expr_context_ty);
 static expr_ty ast_for_testlist(struct compiling *, const node *);
@@ -806,7 +814,7 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags,
                     }
                 }
             }
-            res = Module(stmts, arena);
+            res = Module(stmts, docstring_from_stmts(stmts), arena);
             break;
         case eval_input: {
             expr_ty testlist_ast;
@@ -1593,6 +1601,7 @@ ast_for_funcdef_impl(struct compiling *c, const node *n,
     arguments_ty args;
     asdl_seq *body;
     expr_ty returns = NULL;
+    string docstring;
     int name_i = 1;
 
     REQ(n, funcdef);
@@ -1611,17 +1620,17 @@ ast_for_funcdef_impl(struct compiling *c, const node *n,
             return NULL;
         name_i += 2;
     }
-    body = ast_for_suite(c, CHILD(n, name_i + 3));
+    body = ast_for_body(c, CHILD(n, name_i + 3), &docstring);
     if (!body)
         return NULL;
 
     if (is_async)
         return AsyncFunctionDef(name, args, body, decorator_seq, returns,
-                                LINENO(n),
+                                docstring, LINENO(n),
                                 n->n_col_offset, c->c_arena);
     else
         return FunctionDef(name, args, body, decorator_seq, returns,
-                           LINENO(n),
+                           docstring, LINENO(n),
                            n->n_col_offset, c->c_arena);
 }
 
@@ -3514,6 +3523,32 @@ ast_for_suite(struct compiling *c, const node *n)
     return seq;
 }
 
+static string
+docstring_from_stmts(asdl_seq *stmts)
+{
+    if (stmts && stmts->size) {
+        stmt_ty s = (stmt_ty)asdl_seq_GET(stmts, 0);
+        /* If first statement is a literal string, it's the doc string. */
+        if (s->kind == Expr_kind && s->v.Expr.value->kind == Str_kind) {
+            string doc = s->v.Expr.value->v.Str.s;
+            /* not very efficient, but simple */
+            memmove(&asdl_seq_GET(stmts, 0), &asdl_seq_GET(stmts, 1),
+                    (stmts->size - 1) * sizeof(void*));
+            stmts->size--;
+            return doc;
+        }
+    }
+    return NULL;
+}
+
+static asdl_seq *
+ast_for_body(struct compiling *c, const node *n, string *docstring)
+{
+    asdl_seq *stmts = ast_for_suite(c, n);
+    *docstring = docstring_from_stmts(stmts);
+    return stmts;
+}
+
 static stmt_ty
 ast_for_if_stmt(struct compiling *c, const node *n)
 {
@@ -3898,12 +3933,13 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
     /* classdef: 'class' NAME ['(' arglist ')'] ':' suite */
     PyObject *classname;
     asdl_seq *s;
+    string docstring;
     expr_ty call;
 
     REQ(n, classdef);
 
     if (NCH(n) == 4) { /* class NAME ':' suite */
-        s = ast_for_suite(c, CHILD(n, 3));
+        s = ast_for_body(c, CHILD(n, 3), &docstring);
         if (!s)
             return NULL;
         classname = NEW_IDENTIFIER(CHILD(n, 1));
@@ -3911,12 +3947,12 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
             return NULL;
         if (forbidden_name(c, classname, CHILD(n, 3), 0))
             return NULL;
-        return ClassDef(classname, NULL, NULL, s, decorator_seq, LINENO(n),
+        return ClassDef(classname, NULL, NULL, s, decorator_seq, docstring,
-                        n->n_col_offset, c->c_arena);
+                        LINENO(n), n->n_col_offset, c->c_arena);
     }
 
     if (TYPE(CHILD(n, 3)) == RPAR) { /* class NAME '(' ')' ':' suite */
-        s = ast_for_suite(c, CHILD(n,5));
+        s = ast_for_body(c, CHILD(n, 5), &docstring);
         if (!s)
             return NULL;
         classname = NEW_IDENTIFIER(CHILD(n, 1));
@@ -3924,8 +3960,8 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
             return NULL;
         if (forbidden_name(c, classname, CHILD(n, 3), 0))
             return NULL;
-        return ClassDef(classname, NULL, NULL, s, decorator_seq, LINENO(n),
+        return ClassDef(classname, NULL, NULL, s, decorator_seq, docstring,
-                        n->n_col_offset, c->c_arena);
+                        LINENO(n), n->n_col_offset, c->c_arena);
     }
 
     /* class NAME '(' arglist ')' ':' suite */
@@ -3941,7 +3977,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
         if (!call)
             return NULL;
     }
-    s = ast_for_suite(c, CHILD(n, 6));
+    s = ast_for_body(c, CHILD(n, 6), &docstring);
     if (!s)
         return NULL;
     classname = NEW_IDENTIFIER(CHILD(n, 1));
@@ -3951,7 +3987,8 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
         return NULL;
 
     return ClassDef(classname, call->v.Call.args, call->v.Call.keywords, s,
-                    decorator_seq, LINENO(n), n->n_col_offset, c->c_arena);
+                    decorator_seq, docstring, LINENO(n), n->n_col_offset,
+                    c->c_arena);
 }
 
 static stmt_ty

Python/compile.c

@@ -1324,18 +1324,6 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute)
     } \
 }
 
-static int
-compiler_isdocstring(stmt_ty s)
-{
-    if (s->kind != Expr_kind)
-        return 0;
-    if (s->v.Expr.value->kind == Str_kind)
-        return 1;
-    if (s->v.Expr.value->kind == Constant_kind)
-        return PyUnicode_CheckExact(s->v.Expr.value->v.Constant.value);
-    return 0;
-}
-
 static int
 is_const(expr_ty e)
 {
@@ -1435,36 +1423,27 @@ find_ann(asdl_seq *stmts)
    and for annotations. */
 
 static int
-compiler_body(struct compiler *c, asdl_seq *stmts)
+compiler_body(struct compiler *c, asdl_seq *stmts, string docstring)
 {
-    int i = 0;
-    stmt_ty st;
-
     /* Set current line number to the line number of first statement.
        This way line number for SETUP_ANNOTATIONS will always
        coincide with the line number of first "real" statement in module.
        If body is empy, then lineno will be set later in assemble. */
     if (c->u->u_scope_type == COMPILER_SCOPE_MODULE &&
         !c->u->u_lineno && asdl_seq_LEN(stmts)) {
-        st = (stmt_ty)asdl_seq_GET(stmts, 0);
+        stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0);
         c->u->u_lineno = st->lineno;
     }
     /* Every annotated class and module should have __annotations__. */
     if (find_ann(stmts)) {
         ADDOP(c, SETUP_ANNOTATIONS);
     }
-    if (!asdl_seq_LEN(stmts))
+    /* if not -OO mode, set docstring */
-        return 1;
+    if (c->c_optimize < 2 && docstring) {
-    st = (stmt_ty)asdl_seq_GET(stmts, 0);
+        ADDOP_O(c, LOAD_CONST, docstring, consts);
-    if (compiler_isdocstring(st) && c->c_optimize < 2) {
+        ADDOP_NAME(c, STORE_NAME, __doc__, names);
-        /* don't generate docstrings if -OO */
-        i = 1;
-        VISIT(c, expr, st->v.Expr.value);
-        if (!compiler_nameop(c, __doc__, Store))
-            return 0;
     }
-    for (; i < asdl_seq_LEN(stmts); i++)
+    VISIT_SEQ(c, stmt, stmts);
-        VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i));
     return 1;
 }
 
@@ -1484,7 +1463,7 @@ compiler_mod(struct compiler *c, mod_ty mod)
         return NULL;
     switch (mod->kind) {
     case Module_kind:
-        if (!compiler_body(c, mod->v.Module.body)) {
+        if (!compiler_body(c, mod->v.Module.body, mod->v.Module.docstring)) {
             compiler_exit_scope(c);
             return 0;
         }
@@ -1812,15 +1791,13 @@ static int
 compiler_function(struct compiler *c, stmt_ty s, int is_async)
 {
     PyCodeObject *co;
-    PyObject *qualname, *first_const = Py_None;
+    PyObject *qualname, *docstring = Py_None;
     arguments_ty args;
     expr_ty returns;
     identifier name;
     asdl_seq* decos;
     asdl_seq *body;
-    stmt_ty st;
+    Py_ssize_t i, funcflags;
-    Py_ssize_t i, n, funcflags;
-    int docstring;
     int annotations;
     int scope_type;
 
@@ -1866,27 +1843,18 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
         return 0;
     }
 
-    st = (stmt_ty)asdl_seq_GET(body, 0);
+    /* if not -OO mode, add docstring */
-    docstring = compiler_isdocstring(st);
+    if (c->c_optimize < 2 && s->v.FunctionDef.docstring)
-    if (docstring && c->c_optimize < 2) {
+        docstring = s->v.FunctionDef.docstring;
-        if (st->v.Expr.value->kind == Constant_kind)
+    if (compiler_add_o(c, c->u->u_consts, docstring) < 0) {
-            first_const = st->v.Expr.value->v.Constant.value;
-        else
-            first_const = st->v.Expr.value->v.Str.s;
-    }
-    if (compiler_add_o(c, c->u->u_consts, first_const) < 0)      {
         compiler_exit_scope(c);
         return 0;
     }
 
     c->u->u_argcount = asdl_seq_LEN(args->args);
     c->u->u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs);
-    n = asdl_seq_LEN(body);
     /* if there was a docstring, we need to skip the first statement */
-    for (i = docstring; i < n; i++) {
+    VISIT_SEQ_IN_SCOPE(c, stmt, body);
-        st = (stmt_ty)asdl_seq_GET(body, i);
-        VISIT_IN_SCOPE(c, stmt, st);
-    }
     co = assemble(c, 1);
     qualname = c->u->u_qualname;
     Py_INCREF(qualname);
@@ -1967,7 +1935,7 @@ compiler_class(struct compiler *c, stmt_ty s)
         }
         Py_DECREF(str);
         /* compile the body proper */
-        if (!compiler_body(c, s->v.ClassDef.body)) {
+        if (!compiler_body(c, s->v.ClassDef.body, s->v.ClassDef.docstring)) {
             compiler_exit_scope(c);
             return 0;
         }

Python/future.c

@@ -61,7 +61,6 @@ static int
 future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename)
 {
     int i, done = 0, prev_line = 0;
-    stmt_ty first;
 
     if (!(mod->kind == Module_kind || mod->kind == Interactive_kind))
         return 1;
@@ -77,16 +76,7 @@ future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename)
        but is preceded by a regular import.
     */
 
-    i = 0;
+    for (i = 0; i < asdl_seq_LEN(mod->v.Module.body); i++) {
-    first = (stmt_ty)asdl_seq_GET(mod->v.Module.body, i);
-    if (first->kind == Expr_kind
-        && (first->v.Expr.value->kind == Str_kind
-            || (first->v.Expr.value->kind == Constant_kind
-                && PyUnicode_CheckExact(first->v.Expr.value->v.Constant.value))))
-        i++;
-
-
-    for (; i < asdl_seq_LEN(mod->v.Module.body); i++) {
         stmt_ty s = (stmt_ty)asdl_seq_GET(mod->v.Module.body, i);
 
         if (done && s->lineno > prev_line)

Tools/parser/unparse.py

@@ -71,6 +71,8 @@ class Unparser:
     ########################################################
 
     def _Module(self, tree):
+        if tree.docstring is not None:
+            self.fill(repr(tree.docstring))
         for stmt in tree.body:
             self.dispatch(stmt)
 
@@ -235,6 +237,8 @@ class Unparser:
         self.write(")")
 
         self.enter()
+        if t.docstring is not None:
+            self.fill(repr(t.docstring))
         self.dispatch(t.body)
         self.leave()
 
@@ -257,6 +261,8 @@ class Unparser:
             self.write(" -> ")
             self.dispatch(t.returns)
         self.enter()
+        if t.docstring is not None:
+            self.fill(repr(t.docstring))
         self.dispatch(t.body)
         self.leave()