traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-29463: Add docstring field to some AST nodes. (#46) 

* bpo-29463: Add docstring field to some AST nodes.

ClassDef, ModuleDef, FunctionDef, and AsyncFunctionDef has docstring
field for now.  It was first statement of there body.

* fix document.  thanks travis!

* doc fixes
This commit is contained in:
INADA Naoki 2017-02-23 00:31:59 +09:00 committed by Victor Stinner
parent 1bc156430b
commit cb41b2766d
15 changed files with 3153 additions and 3048 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Doc/library/ast.rst
View File

@ -133,9 +133,17 @@ and classes for traversing abstract syntax trees:
.. function:: get_docstring(node, clean=True)
Return the docstring of the given *node* (which must be a
:class:`FunctionDef`, :class:`ClassDef` or :class:`Module` node), or ``None``
if it has no docstring. If *clean* is true, clean up the docstring's
indentation with :func:`inspect.cleandoc`.
:class:`FunctionDef`, :class:`AsyncFunctionDef`, :class:`ClassDef`,
or :class:`Module` node), or ``None`` if it has no docstring.
If *clean* is true, clean up the docstring's indentation with
:func:`inspect.cleandoc`.
.. versionchanged:: 3.5
:class:`AsyncFunctionDef` is now supported.
.. versionchanged:: 3.7
The docstring is now exported from the node docstring field, instead of
the first body statement.
.. function:: fix_missing_locations(node)

6
Doc/whatsnew/3.7.rst
View File

@ -196,6 +196,12 @@ Changes in the Python API
Use the :meth:`~http.cookies.Morsel.set` method for setting them.
(Contributed by Serhiy Storchaka in :issue:`29192`.)
* ``Module``, ``FunctionDef``, ``AsyncFunctionDef``, and
``ClassDef`` AST nodes now have a new ``docstring`` field.
The first statement in their body is not considered as a docstring
anymore. ``co_firstlineno`` and ``co_lnotab`` of code object for class
and module are affected by this change.
(Contributed by INADA Naoki and Eugene Toder in :issue:`29463`.)
CPython bytecode changes
------------------------

25
Include/Python-ast.h
View File

@ -46,6 +46,7 @@ struct _mod {
union {
struct {
asdl_seq *body;
string docstring;
} Module;
struct {
@ -80,6 +81,7 @@ struct _stmt {
asdl_seq *body;
asdl_seq *decorator_list;
expr_ty returns;
string docstring;
} FunctionDef;
struct {
@ -88,6 +90,7 @@ struct _stmt {
asdl_seq *body;
asdl_seq *decorator_list;
expr_ty returns;
string docstring;
} AsyncFunctionDef;
struct {
@ -96,6 +99,7 @@ struct _stmt {
asdl_seq *keywords;
asdl_seq *body;
asdl_seq *decorator_list;
string docstring;
} ClassDef;
struct {
@ -439,26 +443,27 @@ struct _withitem {
};
#define Module(a0, a1) _Py_Module(a0, a1)
mod_ty _Py_Module(asdl_seq * body, PyArena *arena);
#define Module(a0, a1, a2) _Py_Module(a0, a1, a2)
mod_ty _Py_Module(asdl_seq * body, string docstring, PyArena *arena);
#define Interactive(a0, a1) _Py_Interactive(a0, a1)
mod_ty _Py_Interactive(asdl_seq * body, PyArena *arena);
#define Expression(a0, a1) _Py_Expression(a0, a1)
mod_ty _Py_Expression(expr_ty body, PyArena *arena);
#define Suite(a0, a1) _Py_Suite(a0, a1)
mod_ty _Py_Suite(asdl_seq * body, PyArena *arena);
#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8)
stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
asdl_seq * decorator_list, expr_ty returns, int lineno,
int col_offset, PyArena *arena);
#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
asdl_seq * decorator_list, expr_ty returns, string
docstring, int lineno, int col_offset, PyArena *arena);
#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8)
stmt_ty _Py_AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq *
body, asdl_seq * decorator_list, expr_ty returns,
int lineno, int col_offset, PyArena *arena);
#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7)
string docstring, int lineno, int col_offset,
PyArena *arena);
#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8)
stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords,
asdl_seq * body, asdl_seq * decorator_list, int lineno,
int col_offset, PyArena *arena);
asdl_seq * body, asdl_seq * decorator_list, string
docstring, int lineno, int col_offset, PyArena *arena);
#define Return(a0, a1, a2, a3) _Py_Return(a0, a1, a2, a3)
stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
#define Delete(a0, a1, a2, a3) _Py_Delete(a0, a1, a2, a3)

10
Lib/ast.py
View File

@ -197,15 +197,7 @@ def get_docstring(node, clean=True):
"""
if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)):
raise TypeError("%r can't have docstrings" % node.__class__.__name__)
if not(node.body and isinstance(node.body[0], Expr)):
return
node = node.body[0].value
if isinstance(node, Str):
text = node.s
elif isinstance(node, Constant) and isinstance(node.value, str):
text = node.value
else:
return
text = node.docstring
if clean:
import inspect
text = inspect.cleandoc(text)

160
Lib/test/test_ast.py
View File

@ -27,8 +27,12 @@ def to_tuple(t):
exec_tests = [
# None
"None",
# Module docstring
"'module docstring'",
# FunctionDef
"def f(): pass",
# FunctionDef with docstring
"def f(): 'function docstring'",
# FunctionDef with arg
"def f(a): pass",
# FunctionDef with arg and default value
@ -37,10 +41,12 @@ exec_tests = [
"def f(*args): pass",
# FunctionDef with kwargs
"def f(**kwargs): pass",
# FunctionDef with all kind of args
"def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): pass",
# FunctionDef with all kind of args and docstring
"def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): 'doc for f()'",
# ClassDef
"class C:pass",
# ClassDef with docstring
"class C: 'docstring for class C'",
# ClassDef, new style class
"class C(object): pass",
# Return
@ -108,7 +114,7 @@ exec_tests = [
# setcomp with naked tuple
"{r for l,m in x}",
# AsyncFunctionDef
"async def f():\n await something()",
"async def f():\n 'async function'\n await something()",
# AsyncFor
"async def f():\n async for e in i: 1\n else: 2",
# AsyncWith
@ -321,9 +327,24 @@ class AST_Tests(unittest.TestCase):
def test_module(self):
body = [ast.Num(42)]
x = ast.Module(body)
x = ast.Module(body, None)
self.assertEqual(x.body, body)
def test_docstring(self):
body = [] # AST nodes having docstring must accept empty body
x = ast.Module(body, "module docstring")
self.assertEqual(x.docstring, "module docstring")
a = ast.arguments()
x = ast.FunctionDef("x", a, body, [], None, "func docstring")
self.assertEqual(x.docstring, "func docstring")
x = ast.AsyncFunctionDef("x", a, body, [], None, "async func docstring")
self.assertEqual(x.docstring, "async func docstring")
x = ast.ClassDef("x", [], [], body, [], "class docstring")
self.assertEqual(x.docstring, "class docstring")
def test_nodeclasses(self):
# Zero arguments constructor explicitly allowed
x = ast.BinOp()
@ -394,25 +415,18 @@ class AST_Tests(unittest.TestCase):
def test_invalid_sum(self):
pos = dict(lineno=2, col_offset=3)
m = ast.Module([ast.Expr(ast.expr(**pos), **pos)])
m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], "doc")
with self.assertRaises(TypeError) as cm:
compile(m, "<test>", "exec")
self.assertIn("but got <_ast.expr", str(cm.exception))
def test_invalid_identitifer(self):
m = ast.Module([ast.Expr(ast.Name(42, ast.Load()))])
m = ast.Module([ast.Expr(ast.Name(42, ast.Load()))], None)
ast.fix_missing_locations(m)
with self.assertRaises(TypeError) as cm:
compile(m, "<test>", "exec")
self.assertIn("identifier must be of type str", str(cm.exception))
def test_invalid_string(self):
m = ast.Module([ast.Expr(ast.Str(42))])
ast.fix_missing_locations(m)
with self.assertRaises(TypeError) as cm:
compile(m, "<test>", "exec")
self.assertIn("string must be of type str", str(cm.exception))
def test_empty_yield_from(self):
# Issue 16546: yield from value is not optional.
empty_yield_from = ast.parse("def f():\n yield from g()")
@ -442,18 +456,18 @@ class ASTHelpers_Test(unittest.TestCase):
self.assertEqual(ast.dump(node),
"Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), "
"args=[Name(id='eggs', ctx=Load()), Str(s='and cheese')], "
"keywords=[]))])"
"keywords=[]))], docstring=None)"
)
self.assertEqual(ast.dump(node, annotate_fields=False),
"Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), "
"Str('and cheese')], []))])"
"Str('and cheese')], []))], None)"
)
self.assertEqual(ast.dump(node, include_attributes=True),
"Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), "
"lineno=1, col_offset=0), args=[Name(id='eggs', ctx=Load(), "
"lineno=1, col_offset=5), Str(s='and cheese', lineno=1, "
"col_offset=11)], keywords=[], "
"lineno=1, col_offset=0), lineno=1, col_offset=0)])"
"lineno=1, col_offset=0), lineno=1, col_offset=0)], docstring=None)"
)
def test_copy_location(self):
@ -478,7 +492,7 @@ class ASTHelpers_Test(unittest.TestCase):
"Expr(value=Call(func=Name(id='spam', ctx=Load(), lineno=1, "
"col_offset=0), args=[Str(s='eggs', lineno=1, col_offset=0)], "
"keywords=[], lineno=1, "
"col_offset=0), lineno=1, col_offset=0)])"
"col_offset=0), lineno=1, col_offset=0)], docstring=None)"
)
def test_increment_lineno(self):
@ -545,7 +559,7 @@ class ASTHelpers_Test(unittest.TestCase):
names=[ast.alias(name='sleep')],
level=None,
lineno=None, col_offset=None)]
mod = ast.Module(body)
mod = ast.Module(body, None)
with self.assertRaises(ValueError) as cm:
compile(mod, 'test', 'exec')
self.assertIn("invalid integer value: None", str(cm.exception))
@ -555,7 +569,7 @@ class ASTHelpers_Test(unittest.TestCase):
names=[ast.alias(name='sleep')],
level=None,
lineno=0, col_offset=0)]
mod = ast.Module(body)
mod = ast.Module(body, None)
code = compile(mod, 'test', 'exec')
ns = {}
exec(code, ns)
@ -573,11 +587,11 @@ class ASTValidatorTests(unittest.TestCase):
self.assertIn(msg, str(cm.exception))
def expr(self, node, msg=None, *, exc=ValueError):
mod = ast.Module([ast.Expr(node)])
mod = ast.Module([ast.Expr(node)], None)
self.mod(mod, msg, exc=exc)
def stmt(self, stmt, msg=None):
mod = ast.Module([stmt])
mod = ast.Module([stmt], None)
self.mod(mod, msg)
def test_module(self):
@ -619,16 +633,16 @@ class ASTValidatorTests(unittest.TestCase):
def test_funcdef(self):
a = ast.arguments([], None, [], [], None, [])
f = ast.FunctionDef("x", a, [], [], None)
f = ast.FunctionDef("x", a, [], [], None, None)
self.stmt(f, "empty body on FunctionDef")
f = ast.FunctionDef("x", a, [ast.Pass()], [ast.Name("x", ast.Store())],
None)
None, None)
self.stmt(f, "must have Load context")
f = ast.FunctionDef("x", a, [ast.Pass()], [],
ast.Name("x", ast.Store()))
ast.Name("x", ast.Store()), None)
self.stmt(f, "must have Load context")
def fac(args):
return ast.FunctionDef("x", args, [ast.Pass()], [], None)
return ast.FunctionDef("x", args, [ast.Pass()], [], None, None)
self._check_arguments(fac, self.stmt)
def test_classdef(self):
@ -642,7 +656,7 @@ class ASTValidatorTests(unittest.TestCase):
if decorator_list is None:
decorator_list = []
return ast.ClassDef("myclass", bases, keywords,
body, decorator_list)
body, decorator_list, None)
self.stmt(cls(bases=[ast.Name("x", ast.Store())]),
"must have Load context")
self.stmt(cls(keywords=[ast.keyword("x", ast.Name("x", ast.Store()))]),
@ -1009,9 +1023,6 @@ class ConstantTests(unittest.TestCase):
tree = ast.parse("'docstring'\nx = 1")
self.assertEqual(ast.get_docstring(tree), 'docstring')
tree.body[0].value = ast.Constant(value='constant docstring')
self.assertEqual(ast.get_docstring(tree), 'constant docstring')
def get_load_const(self, tree):
# Compile to bytecode, disassemble and get parameter of LOAD_CONST
# instructions
@ -1083,50 +1094,53 @@ def main():
#### EVERYTHING BELOW IS GENERATED #####
exec_results = [
('Module', [('Expr', (1, 0), ('NameConstant', (1, 0), None))]),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (1, 9))], [], None)]),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, []), [('Pass', (1, 10))], [], None)]),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, [('Num', (1, 8), 0)]), [('Pass', (1, 12))], [], None)]),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], ('arg', (1, 7), 'args', None), [], [], None, []), [('Pass', (1, 14))], [], None)]),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], ('arg', (1, 8), 'kwargs', None), []), [('Pass', (1, 17))], [], None)]),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None), ('arg', (1, 9), 'b', None), ('arg', (1, 14), 'c', None), ('arg', (1, 22), 'd', None), ('arg', (1, 28), 'e', None)], ('arg', (1, 35), 'args', None), [('arg', (1, 41), 'f', None)], [('Num', (1, 43), 42)], ('arg', (1, 49), 'kwargs', None), [('Num', (1, 11), 1), ('NameConstant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('Pass', (1, 58))], [], None)]),
('Module', [('ClassDef', (1, 0), 'C', [], [], [('Pass', (1, 8))], [])]),
('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], [('Pass', (1, 17))], [])]),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None)]),
('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]),
('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Num', (1, 5), 1))]),
('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]),
('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]),
('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]),
('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))])]),
('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))])]),
('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], []), None)]),
('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])]),
('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])]),
('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]),
('Module', [('Import', (1, 0), [('alias', 'sys', None)])]),
('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)]),
('Module', [('Global', (1, 0), ['v'])]),
('Module', [('Expr', (1, 0), ('Num', (1, 0), 1))]),
('Module', [('Pass', (1, 0))]),
('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Break', (1, 11))], [])]),
('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Continue', (1, 11))], [])]),
('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 4), 'a', ('Store',)), ('Name', (1, 6), 'b', ('Store',))], ('Store',)), ('Name', (1, 11), 'c', ('Load',)), [('Pass', (1, 14))], [])]),
('Module', [('Expr', (1, 0), ('ListComp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [], 0)]))]),
('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [], 0)]))]),
('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 12), [('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 20), 'c', ('Load',)), [], 0)]))]),
('Module', [('Expr', (1, 0), ('GeneratorExp', (2, 4), ('Tuple', (3, 4), [('Name', (3, 4), 'Aa', ('Load',)), ('Name', (5, 7), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4), [('Name', (8, 4), 'Aa', ('Store',)), ('Name', (10, 4), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10), 'Cc', ('Load',)), [], 0)]))]),
('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Name', (1, 11), 'w', ('Store',)), ('Name', (1, 16), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22), 'm', ('Store',)), ('Name', (1, 27), 'p', ('Load',)), [('Name', (1, 32), 'g', ('Load',))], 0)]))]),
('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [], 0)]))]),
('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))], 0)]))]),
('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [], 0)]))]),
('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('Await', (2, 1), ('Call', (2, 7), ('Name', (2, 7), 'something', ('Load',)), [], [])))], [], None)]),
('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('AsyncFor', (2, 7), ('Name', (2, 11), 'e', ('Store',)), ('Name', (2, 16), 'i', ('Load',)), [('Expr', (2, 19), ('Num', (2, 19), 1))], [('Expr', (3, 7), ('Num', (3, 7), 2))])], [], None)]),
('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('AsyncWith', (2, 7), [('withitem', ('Name', (2, 12), 'a', ('Load',)), ('Name', (2, 17), 'b', ('Store',)))], [('Expr', (2, 20), ('Num', (2, 20), 1))])], [], None)]),
('Module', [('Expr', (1, 0), ('Dict', (1, 0), [None, ('Num', (1, 10), 2)], [('Dict', (1, 3), [('Num', (1, 4), 1)], [('Num', (1, 6), 2)]), ('Num', (1, 12), 3)]))]),
('Module', [('Expr', (1, 0), ('Set', (1, 0), [('Starred', (1, 1), ('Set', (1, 2), [('Num', (1, 3), 1), ('Num', (1, 6), 2)]), ('Load',)), ('Num', (1, 10), 3)]))]),
('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('ListComp', (2, 2), ('Name', (2, 2), 'i', ('Load',)), [('comprehension', ('Name', (2, 14), 'b', ('Store',)), ('Name', (2, 19), 'c', ('Load',)), [], 1)]))], [], None)]),
('Module', [('Expr', (1, 0), ('NameConstant', (1, 0), None))], None),
('Module', [], 'module docstring'),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (1, 9))], [], None, None)], None),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [], [], None, 'function docstring')], None),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, []), [('Pass', (1, 10))], [], None, None)], None),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, [('Num', (1, 8), 0)]), [('Pass', (1, 12))], [], None, None)], None),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], ('arg', (1, 7), 'args', None), [], [], None, []), [('Pass', (1, 14))], [], None, None)], None),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], ('arg', (1, 8), 'kwargs', None), []), [('Pass', (1, 17))], [], None, None)], None),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None), ('arg', (1, 9), 'b', None), ('arg', (1, 14), 'c', None), ('arg', (1, 22), 'd', None), ('arg', (1, 28), 'e', None)], ('arg', (1, 35), 'args', None), [('arg', (1, 41), 'f', None)], [('Num', (1, 43), 42)], ('arg', (1, 49), 'kwargs', None), [('Num', (1, 11), 1), ('NameConstant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [], [], None, 'doc for f()')], None),
('Module', [('ClassDef', (1, 0), 'C', [], [], [('Pass', (1, 8))], [], None)], None),
('Module', [('ClassDef', (1, 0), 'C', [], [], [], [], 'docstring for class C')], None),
('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], [('Pass', (1, 17))], [], None)], None),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None, None)], None),
('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])], None),
('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))], None),
('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Num', (1, 5), 1))], None),
('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])], None),
('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])], None),
('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])], None),
('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))])], None),
('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))])], None),
('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], []), None)], None),
('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])], None),
('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])], None),
('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)], None),
('Module', [('Import', (1, 0), [('alias', 'sys', None)])], None),
('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)], None),
('Module', [('Global', (1, 0), ['v'])], None),
('Module', [('Expr', (1, 0), ('Num', (1, 0), 1))], None),
('Module', [('Pass', (1, 0))], None),
('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Break', (1, 11))], [])], None),
('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Continue', (1, 11))], [])], None),
('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 4), 'a', ('Store',)), ('Name', (1, 6), 'b', ('Store',))], ('Store',)), ('Name', (1, 11), 'c', ('Load',)), [('Pass', (1, 14))], [])], None),
('Module', [('Expr', (1, 0), ('ListComp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [], 0)]))], None),
('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [], 0)]))], None),
('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 12), [('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 20), 'c', ('Load',)), [], 0)]))], None),
('Module', [('Expr', (1, 0), ('GeneratorExp', (2, 4), ('Tuple', (3, 4), [('Name', (3, 4), 'Aa', ('Load',)), ('Name', (5, 7), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4), [('Name', (8, 4), 'Aa', ('Store',)), ('Name', (10, 4), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10), 'Cc', ('Load',)), [], 0)]))], None),
('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Name', (1, 11), 'w', ('Store',)), ('Name', (1, 16), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22), 'm', ('Store',)), ('Name', (1, 27), 'p', ('Load',)), [('Name', (1, 32), 'g', ('Load',))], 0)]))], None),
('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [], 0)]))], None),
('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))], 0)]))], None),
('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [], 0)]))], None),
('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (3, 1), ('Await', (3, 1), ('Call', (3, 7), ('Name', (3, 7), 'something', ('Load',)), [], [])))], [], None, 'async function')], None),
('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('AsyncFor', (2, 7), ('Name', (2, 11), 'e', ('Store',)), ('Name', (2, 16), 'i', ('Load',)), [('Expr', (2, 19), ('Num', (2, 19), 1))], [('Expr', (3, 7), ('Num', (3, 7), 2))])], [], None, None)], None),
('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('AsyncWith', (2, 7), [('withitem', ('Name', (2, 12), 'a', ('Load',)), ('Name', (2, 17), 'b', ('Store',)))], [('Expr', (2, 20), ('Num', (2, 20), 1))])], [], None, None)], None),
('Module', [('Expr', (1, 0), ('Dict', (1, 0), [None, ('Num', (1, 10), 2)], [('Dict', (1, 3), [('Num', (1, 4), 1)], [('Num', (1, 6), 2)]), ('Num', (1, 12), 3)]))], None),
('Module', [('Expr', (1, 0), ('Set', (1, 0), [('Starred', (1, 1), ('Set', (1, 2), [('Num', (1, 3), 1), ('Num', (1, 6), 2)]), ('Load',)), ('Num', (1, 10), 3)]))], None),
('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('ListComp', (2, 2), ('Name', (2, 2), 'i', ('Load',)), [('comprehension', ('Name', (2, 14), 'b', ('Store',)), ('Name', (2, 19), 'c', ('Load',)), [], 1)]))], [], None, None)], None),
]
single_results = [
('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),

2
Lib/test/test_opcodes.py
View File

@ -27,7 +27,7 @@ class OpcodeTest(unittest.TestCase):
with open(ann_module.__file__) as f:
txt = f.read()
co = compile(txt, ann_module.__file__, 'exec')
self.assertEqual(co.co_firstlineno, 6)
self.assertEqual(co.co_firstlineno, 8)
except OSError:
pass

4
Misc/NEWS
View File

@ -19,6 +19,10 @@ Core and Builtins
- bpo-29347: Fixed possibly dereferencing undefined pointers
when creating weakref objects.
- bpo-29463: Add ``docstring`` field to Module, ClassDef, FunctionDef,
and AsyncFunctionDef ast nodes. docstring is not first stmt in their body
anymore. It affects ``co_firstlineno`` and ``co_lnotab`` of code object
for module and class.
- bpo-29438: Fixed use-after-free problem in key sharing dict.

11
Parser/Python.asdl
View File

@ -6,7 +6,7 @@
module Python
{
mod = Module(stmt* body)
mod = Module(stmt* body, string? docstring)
| Interactive(stmt* body)
| Expression(expr body)
@ -14,15 +14,18 @@ module Python
| Suite(stmt* body)
stmt = FunctionDef(identifier name, arguments args,
stmt* body, expr* decorator_list, expr? returns)
stmt* body, expr* decorator_list, expr? returns,
string? docstring)
| AsyncFunctionDef(identifier name, arguments args,
stmt* body, expr* decorator_list, expr? returns)
stmt* body, expr* decorator_list, expr? returns,
string? docstring)
| ClassDef(identifier name,
expr* bases,
keyword* keywords,
stmt* body,
expr* decorator_list)
expr* decorator_list,
string? docstring)
| Return(expr? value)
| Delete(expr* targets)

107
Python/Python-ast.c
View File

@ -10,8 +10,10 @@ static PyTypeObject *mod_type;
static PyObject* ast2obj_mod(void*);
static PyTypeObject *Module_type;
_Py_IDENTIFIER(body);
_Py_IDENTIFIER(docstring);
static char *Module_fields[]={
"body",
"docstring",
};
static PyTypeObject *Interactive_type;
static char *Interactive_fields[]={
@ -44,6 +46,7 @@ static char *FunctionDef_fields[]={
"body",
"decorator_list",
"returns",
"docstring",
};
static PyTypeObject *AsyncFunctionDef_type;
static char *AsyncFunctionDef_fields[]={
@ -52,6 +55,7 @@ static char *AsyncFunctionDef_fields[]={
"body",
"decorator_list",
"returns",
"docstring",
};
static PyTypeObject *ClassDef_type;
_Py_IDENTIFIER(bases);
@ -62,6 +66,7 @@ static char *ClassDef_fields[]={
"keywords",
"body",
"decorator_list",
"docstring",
};
static PyTypeObject *Return_type;
_Py_IDENTIFIER(value);
@ -861,7 +866,7 @@ static int init_types(void)
mod_type = make_type("mod", &AST_type, NULL, 0);
if (!mod_type) return 0;
if (!add_attributes(mod_type, NULL, 0)) return 0;
Module_type = make_type("Module", mod_type, Module_fields, 1);
Module_type = make_type("Module", mod_type, Module_fields, 2);
if (!Module_type) return 0;
Interactive_type = make_type("Interactive", mod_type, Interactive_fields,
1);
@ -874,12 +879,12 @@ static int init_types(void)
if (!stmt_type) return 0;
if (!add_attributes(stmt_type, stmt_attributes, 2)) return 0;
FunctionDef_type = make_type("FunctionDef", stmt_type, FunctionDef_fields,
5);
6);
if (!FunctionDef_type) return 0;
AsyncFunctionDef_type = make_type("AsyncFunctionDef", stmt_type,
AsyncFunctionDef_fields, 5);
AsyncFunctionDef_fields, 6);
if (!AsyncFunctionDef_type) return 0;
ClassDef_type = make_type("ClassDef", stmt_type, ClassDef_fields, 5);
ClassDef_type = make_type("ClassDef", stmt_type, ClassDef_fields, 6);
if (!ClassDef_type) return 0;
Return_type = make_type("Return", stmt_type, Return_fields, 1);
if (!Return_type) return 0;
@ -1206,7 +1211,7 @@ static int obj2ast_alias(PyObject* obj, alias_ty* out, PyArena* arena);
static int obj2ast_withitem(PyObject* obj, withitem_ty* out, PyArena* arena);
mod_ty
Module(asdl_seq * body, PyArena *arena)
Module(asdl_seq * body, string docstring, PyArena *arena)
{
mod_ty p;
p = (mod_ty)PyArena_Malloc(arena, sizeof(*p));
@ -1214,6 +1219,7 @@ Module(asdl_seq * body, PyArena *arena)
return NULL;
p->kind = Module_kind;
p->v.Module.body = body;
p->v.Module.docstring = docstring;
return p;
}
@ -1260,8 +1266,8 @@ Suite(asdl_seq * body, PyArena *arena)
stmt_ty
FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
decorator_list, expr_ty returns, int lineno, int col_offset,
PyArena *arena)
decorator_list, expr_ty returns, string docstring, int lineno, int
col_offset, PyArena *arena)
{
stmt_ty p;
if (!name) {
@ -1283,6 +1289,7 @@ FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
p->v.FunctionDef.body = body;
p->v.FunctionDef.decorator_list = decorator_list;
p->v.FunctionDef.returns = returns;
p->v.FunctionDef.docstring = docstring;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
@ -1290,8 +1297,8 @@ FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
stmt_ty
AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq
* decorator_list, expr_ty returns, int lineno, int col_offset,
PyArena *arena)
* decorator_list, expr_ty returns, string docstring, int
lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
if (!name) {
@ -1313,6 +1320,7 @@ AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq
p->v.AsyncFunctionDef.body = body;
p->v.AsyncFunctionDef.decorator_list = decorator_list;
p->v.AsyncFunctionDef.returns = returns;
p->v.AsyncFunctionDef.docstring = docstring;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
@ -1320,8 +1328,8 @@ AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq
stmt_ty
ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, asdl_seq *
body, asdl_seq * decorator_list, int lineno, int col_offset, PyArena
*arena)
body, asdl_seq * decorator_list, string docstring, int lineno, int
col_offset, PyArena *arena)
{
stmt_ty p;
if (!name) {
@ -1338,6 +1346,7 @@ ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, asdl_seq *
p->v.ClassDef.keywords = keywords;
p->v.ClassDef.body = body;
p->v.ClassDef.decorator_list = decorator_list;
p->v.ClassDef.docstring = docstring;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
@ -2601,6 +2610,11 @@ ast2obj_mod(void* _o)
if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_string(o->v.Module.docstring);
if (!value) goto failed;
if (_PyObject_SetAttrId(result, &PyId_docstring, value) == -1)
goto failed;
Py_DECREF(value);
break;
case Interactive_kind:
result = PyType_GenericNew(Interactive_type, NULL, NULL);
@ -2675,6 +2689,11 @@ ast2obj_stmt(void* _o)
if (_PyObject_SetAttrId(result, &PyId_returns, value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_string(o->v.FunctionDef.docstring);
if (!value) goto failed;
if (_PyObject_SetAttrId(result, &PyId_docstring, value) == -1)
goto failed;
Py_DECREF(value);
break;
case AsyncFunctionDef_kind:
result = PyType_GenericNew(AsyncFunctionDef_type, NULL, NULL);
@ -2705,6 +2724,11 @@ ast2obj_stmt(void* _o)
if (_PyObject_SetAttrId(result, &PyId_returns, value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_string(o->v.AsyncFunctionDef.docstring);
if (!value) goto failed;
if (_PyObject_SetAttrId(result, &PyId_docstring, value) == -1)
goto failed;
Py_DECREF(value);
break;
case ClassDef_kind:
result = PyType_GenericNew(ClassDef_type, NULL, NULL);
@ -2734,6 +2758,11 @@ ast2obj_stmt(void* _o)
if (_PyObject_SetAttrId(result, &PyId_decorator_list, value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_string(o->v.ClassDef.docstring);
if (!value) goto failed;
if (_PyObject_SetAttrId(result, &PyId_docstring, value) == -1)
goto failed;
Py_DECREF(value);
break;
case Return_kind:
result = PyType_GenericNew(Return_type, NULL, NULL);
@ -3974,6 +4003,7 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
}
if (isinstance) {
asdl_seq* body;
string docstring;
if (_PyObject_HasAttrId(obj, &PyId_body)) {
int res;
@ -4003,7 +4033,17 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Module");
return 1;
}
*out = Module(body, arena);
if (exists_not_none(obj, &PyId_docstring)) {
int res;
tmp = _PyObject_GetAttrId(obj, &PyId_docstring);
if (tmp == NULL) goto failed;
res = obj2ast_string(tmp, &docstring, arena);
if (res != 0) goto failed;
Py_CLEAR(tmp);
} else {
docstring = NULL;
}
*out = Module(body, docstring, arena);
if (*out == NULL) goto failed;
return 0;
}
@ -4159,6 +4199,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
asdl_seq* body;
asdl_seq* decorator_list;
expr_ty returns;
string docstring;
if (_PyObject_HasAttrId(obj, &PyId_name)) {
int res;
@ -4248,8 +4289,18 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
} else {
returns = NULL;
}
*out = FunctionDef(name, args, body, decorator_list, returns, lineno,
col_offset, arena);
if (exists_not_none(obj, &PyId_docstring)) {
int res;
tmp = _PyObject_GetAttrId(obj, &PyId_docstring);
if (tmp == NULL) goto failed;
res = obj2ast_string(tmp, &docstring, arena);
if (res != 0) goto failed;
Py_CLEAR(tmp);
} else {
docstring = NULL;
}
*out = FunctionDef(name, args, body, decorator_list, returns,
docstring, lineno, col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@ -4263,6 +4314,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
asdl_seq* body;
asdl_seq* decorator_list;
expr_ty returns;
string docstring;
if (_PyObject_HasAttrId(obj, &PyId_name)) {
int res;
@ -4352,8 +4404,18 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
} else {
returns = NULL;
}
if (exists_not_none(obj, &PyId_docstring)) {
int res;
tmp = _PyObject_GetAttrId(obj, &PyId_docstring);
if (tmp == NULL) goto failed;
res = obj2ast_string(tmp, &docstring, arena);
if (res != 0) goto failed;
Py_CLEAR(tmp);
} else {
docstring = NULL;
}
*out = AsyncFunctionDef(name, args, body, decorator_list, returns,
lineno, col_offset, arena);
docstring, lineno, col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@ -4367,6 +4429,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
asdl_seq* keywords;
asdl_seq* body;
asdl_seq* decorator_list;
string docstring;
if (_PyObject_HasAttrId(obj, &PyId_name)) {
int res;
@ -4491,8 +4554,18 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
PyErr_SetString(PyExc_TypeError, "required field \"decorator_list\" missing from ClassDef");
return 1;
}
*out = ClassDef(name, bases, keywords, body, decorator_list, lineno,
col_offset, arena);
if (exists_not_none(obj, &PyId_docstring)) {
int res;
tmp = _PyObject_GetAttrId(obj, &PyId_docstring);
if (tmp == NULL) goto failed;
res = obj2ast_string(tmp, &docstring, arena);
if (res != 0) goto failed;
Py_CLEAR(tmp);
} else {
docstring = NULL;
}
*out = ClassDef(name, bases, keywords, body, decorator_list, docstring,
lineno, col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}

89
Python/ast.c
View File

@ -366,9 +366,12 @@ validate_assignlist(asdl_seq *targets, expr_context_ty ctx)
}
static int
validate_body(asdl_seq *body, const char *owner)
validate_body(asdl_seq *body, const char *owner, int allowempty)
{
return validate_nonempty_seq(body, "body", owner) && validate_stmts(body);
if (!allowempty && !validate_nonempty_seq(body, "body", owner)) {
return 0;
}
return validate_stmts(body);
}
static int
@ -377,13 +380,15 @@ validate_stmt(stmt_ty stmt)
int i;
switch (stmt->kind) {
case FunctionDef_kind:
return validate_body(stmt->v.FunctionDef.body, "FunctionDef") &&
return validate_body(stmt->v.FunctionDef.body, "FunctionDef",
stmt->v.FunctionDef.docstring != NULL) &&
validate_arguments(stmt->v.FunctionDef.args) &&
validate_exprs(stmt->v.FunctionDef.decorator_list, Load, 0) &&
(!stmt->v.FunctionDef.returns ||
validate_expr(stmt->v.FunctionDef.returns, Load));
case ClassDef_kind:
return validate_body(stmt->v.ClassDef.body, "ClassDef") &&
return validate_body(stmt->v.ClassDef.body, "ClassDef",
stmt->v.ClassDef.docstring != NULL) &&
validate_exprs(stmt->v.ClassDef.bases, Load, 0) &&
validate_keywords(stmt->v.ClassDef.keywords) &&
validate_exprs(stmt->v.ClassDef.decorator_list, Load, 0);
@ -411,20 +416,20 @@ validate_stmt(stmt_ty stmt)
case For_kind:
return validate_expr(stmt->v.For.target, Store) &&
validate_expr(stmt->v.For.iter, Load) &&
validate_body(stmt->v.For.body, "For") &&
validate_body(stmt->v.For.body, "For", 0) &&
validate_stmts(stmt->v.For.orelse);
case AsyncFor_kind:
return validate_expr(stmt->v.AsyncFor.target, Store) &&
validate_expr(stmt->v.AsyncFor.iter, Load) &&
validate_body(stmt->v.AsyncFor.body, "AsyncFor") &&
validate_body(stmt->v.AsyncFor.body, "AsyncFor", 0) &&
validate_stmts(stmt->v.AsyncFor.orelse);
case While_kind:
return validate_expr(stmt->v.While.test, Load) &&
validate_body(stmt->v.While.body, "While") &&
validate_body(stmt->v.While.body, "While", 0) &&
validate_stmts(stmt->v.While.orelse);
case If_kind:
return validate_expr(stmt->v.If.test, Load) &&
validate_body(stmt->v.If.body, "If") &&
validate_body(stmt->v.If.body, "If", 0) &&
validate_stmts(stmt->v.If.orelse);
case With_kind:
if (!validate_nonempty_seq(stmt->v.With.items, "items", "With"))
@ -435,7 +440,7 @@ validate_stmt(stmt_ty stmt)
(item->optional_vars && !validate_expr(item->optional_vars, Store)))
return 0;
}
return validate_body(stmt->v.With.body, "With");
return validate_body(stmt->v.With.body, "With", 0);
case AsyncWith_kind:
if (!validate_nonempty_seq(stmt->v.AsyncWith.items, "items", "AsyncWith"))
return 0;
@ -445,7 +450,7 @@ validate_stmt(stmt_ty stmt)
(item->optional_vars && !validate_expr(item->optional_vars, Store)))
return 0;
}
return validate_body(stmt->v.AsyncWith.body, "AsyncWith");
return validate_body(stmt->v.AsyncWith.body, "AsyncWith", 0);
case Raise_kind:
if (stmt->v.Raise.exc) {
return validate_expr(stmt->v.Raise.exc, Load) &&
@ -457,7 +462,7 @@ validate_stmt(stmt_ty stmt)
}
return 1;
case Try_kind:
if (!validate_body(stmt->v.Try.body, "Try"))
if (!validate_body(stmt->v.Try.body, "Try", 0))
return 0;
if (!asdl_seq_LEN(stmt->v.Try.handlers) &&
!asdl_seq_LEN(stmt->v.Try.finalbody)) {
@ -473,7 +478,7 @@ validate_stmt(stmt_ty stmt)
excepthandler_ty handler = asdl_seq_GET(stmt->v.Try.handlers, i);
if ((handler->v.ExceptHandler.type &&
!validate_expr(handler->v.ExceptHandler.type, Load)) ||
!validate_body(handler->v.ExceptHandler.body, "ExceptHandler"))
!validate_body(handler->v.ExceptHandler.body, "ExceptHandler", 0))
return 0;
}
return (!asdl_seq_LEN(stmt->v.Try.finalbody) ||
@ -498,7 +503,8 @@ validate_stmt(stmt_ty stmt)
case Expr_kind:
return validate_expr(stmt->v.Expr.value, Load);
case AsyncFunctionDef_kind:
return validate_body(stmt->v.AsyncFunctionDef.body, "AsyncFunctionDef") &&
return validate_body(stmt->v.AsyncFunctionDef.body, "AsyncFunctionDef",
stmt->v.AsyncFunctionDef.docstring != NULL) &&
validate_arguments(stmt->v.AsyncFunctionDef.args) &&
validate_exprs(stmt->v.AsyncFunctionDef.decorator_list, Load, 0) &&
(!stmt->v.AsyncFunctionDef.returns ||
@ -594,7 +600,9 @@ struct compiling {
static asdl_seq *seq_for_testlist(struct compiling *, const node *);
static expr_ty ast_for_expr(struct compiling *, const node *);
static stmt_ty ast_for_stmt(struct compiling *, const node *);
static asdl_seq *ast_for_suite(struct compiling *, const node *);
static asdl_seq *ast_for_body(struct compiling *c, const node *n,
string *docstring);
static string docstring_from_stmts(asdl_seq *stmts);
static asdl_seq *ast_for_exprlist(struct compiling *, const node *,
expr_context_ty);
static expr_ty ast_for_testlist(struct compiling *, const node *);
@ -806,7 +814,7 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags,
}
}
}
res = Module(stmts, arena);
res = Module(stmts, docstring_from_stmts(stmts), arena);
break;
case eval_input: {
expr_ty testlist_ast;
@ -1593,6 +1601,7 @@ ast_for_funcdef_impl(struct compiling *c, const node *n,
arguments_ty args;
asdl_seq *body;
expr_ty returns = NULL;
string docstring;
int name_i = 1;
REQ(n, funcdef);
@ -1611,17 +1620,17 @@ ast_for_funcdef_impl(struct compiling *c, const node *n,
return NULL;
name_i += 2;
}
body = ast_for_suite(c, CHILD(n, name_i + 3));
body = ast_for_body(c, CHILD(n, name_i + 3), &docstring);
if (!body)
return NULL;
if (is_async)
return AsyncFunctionDef(name, args, body, decorator_seq, returns,
LINENO(n),
docstring, LINENO(n),
n->n_col_offset, c->c_arena);
else
return FunctionDef(name, args, body, decorator_seq, returns,
LINENO(n),
docstring, LINENO(n),
n->n_col_offset, c->c_arena);
}
@ -3514,6 +3523,32 @@ ast_for_suite(struct compiling *c, const node *n)
return seq;
}
static string
docstring_from_stmts(asdl_seq *stmts)
{
if (stmts && stmts->size) {
stmt_ty s = (stmt_ty)asdl_seq_GET(stmts, 0);
/* If first statement is a literal string, it's the doc string. */
if (s->kind == Expr_kind && s->v.Expr.value->kind == Str_kind) {
string doc = s->v.Expr.value->v.Str.s;
/* not very efficient, but simple */
memmove(&asdl_seq_GET(stmts, 0), &asdl_seq_GET(stmts, 1),
(stmts->size - 1) * sizeof(void*));
stmts->size--;
return doc;
}
}
return NULL;
}
static asdl_seq *
ast_for_body(struct compiling *c, const node *n, string *docstring)
{
asdl_seq *stmts = ast_for_suite(c, n);
*docstring = docstring_from_stmts(stmts);
return stmts;
}
static stmt_ty
ast_for_if_stmt(struct compiling *c, const node *n)
{
@ -3898,12 +3933,13 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
/* classdef: 'class' NAME ['(' arglist ')'] ':' suite */
PyObject *classname;
asdl_seq *s;
string docstring;
expr_ty call;
REQ(n, classdef);
if (NCH(n) == 4) { /* class NAME ':' suite */
s = ast_for_suite(c, CHILD(n, 3));
s = ast_for_body(c, CHILD(n, 3), &docstring);
if (!s)
return NULL;
classname = NEW_IDENTIFIER(CHILD(n, 1));
@ -3911,12 +3947,12 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
return NULL;
if (forbidden_name(c, classname, CHILD(n, 3), 0))
return NULL;
return ClassDef(classname, NULL, NULL, s, decorator_seq, LINENO(n),
n->n_col_offset, c->c_arena);
return ClassDef(classname, NULL, NULL, s, decorator_seq, docstring,
LINENO(n), n->n_col_offset, c->c_arena);
}
if (TYPE(CHILD(n, 3)) == RPAR) { /* class NAME '(' ')' ':' suite */
s = ast_for_suite(c, CHILD(n,5));
s = ast_for_body(c, CHILD(n, 5), &docstring);
if (!s)
return NULL;
classname = NEW_IDENTIFIER(CHILD(n, 1));
@ -3924,8 +3960,8 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
return NULL;
if (forbidden_name(c, classname, CHILD(n, 3), 0))
return NULL;
return ClassDef(classname, NULL, NULL, s, decorator_seq, LINENO(n),
n->n_col_offset, c->c_arena);
return ClassDef(classname, NULL, NULL, s, decorator_seq, docstring,
LINENO(n), n->n_col_offset, c->c_arena);
}
/* class NAME '(' arglist ')' ':' suite */
@ -3941,7 +3977,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
if (!call)
return NULL;
}
s = ast_for_suite(c, CHILD(n, 6));
s = ast_for_body(c, CHILD(n, 6), &docstring);
if (!s)
return NULL;
classname = NEW_IDENTIFIER(CHILD(n, 1));
@ -3951,7 +3987,8 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
return NULL;
return ClassDef(classname, call->v.Call.args, call->v.Call.keywords, s,
decorator_seq, LINENO(n), n->n_col_offset, c->c_arena);
decorator_seq, docstring, LINENO(n), n->n_col_offset,
c->c_arena);
}
static stmt_ty

64
Python/compile.c
View File

@ -1324,18 +1324,6 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute)
} \
}
static int
compiler_isdocstring(stmt_ty s)
{
if (s->kind != Expr_kind)
return 0;
if (s->v.Expr.value->kind == Str_kind)
return 1;
if (s->v.Expr.value->kind == Constant_kind)
return PyUnicode_CheckExact(s->v.Expr.value->v.Constant.value);
return 0;
}
static int
is_const(expr_ty e)
{
@ -1435,36 +1423,27 @@ find_ann(asdl_seq *stmts)
and for annotations. */
static int
compiler_body(struct compiler *c, asdl_seq *stmts)
compiler_body(struct compiler *c, asdl_seq *stmts, string docstring)
{
int i = 0;
stmt_ty st;
/* Set current line number to the line number of first statement.
This way line number for SETUP_ANNOTATIONS will always
coincide with the line number of first "real" statement in module.
If body is empy, then lineno will be set later in assemble. */
if (c->u->u_scope_type == COMPILER_SCOPE_MODULE &&
!c->u->u_lineno && asdl_seq_LEN(stmts)) {
st = (stmt_ty)asdl_seq_GET(stmts, 0);
stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0);
c->u->u_lineno = st->lineno;
}
/* Every annotated class and module should have __annotations__. */
if (find_ann(stmts)) {
ADDOP(c, SETUP_ANNOTATIONS);
}
if (!asdl_seq_LEN(stmts))
return 1;
st = (stmt_ty)asdl_seq_GET(stmts, 0);
if (compiler_isdocstring(st) && c->c_optimize < 2) {
/* don't generate docstrings if -OO */
i = 1;
VISIT(c, expr, st->v.Expr.value);
if (!compiler_nameop(c, __doc__, Store))
return 0;
/* if not -OO mode, set docstring */
if (c->c_optimize < 2 && docstring) {
ADDOP_O(c, LOAD_CONST, docstring, consts);
ADDOP_NAME(c, STORE_NAME, __doc__, names);
}
for (; i < asdl_seq_LEN(stmts); i++)
VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i));
VISIT_SEQ(c, stmt, stmts);
return 1;
}
@ -1484,7 +1463,7 @@ compiler_mod(struct compiler *c, mod_ty mod)
return NULL;
switch (mod->kind) {
case Module_kind:
if (!compiler_body(c, mod->v.Module.body)) {
if (!compiler_body(c, mod->v.Module.body, mod->v.Module.docstring)) {
compiler_exit_scope(c);
return 0;
}
@ -1812,15 +1791,13 @@ static int
compiler_function(struct compiler *c, stmt_ty s, int is_async)
{
PyCodeObject *co;
PyObject *qualname, *first_const = Py_None;
PyObject *qualname, *docstring = Py_None;
arguments_ty args;
expr_ty returns;
identifier name;
asdl_seq* decos;
asdl_seq *body;
stmt_ty st;
Py_ssize_t i, n, funcflags;
int docstring;
Py_ssize_t i, funcflags;
int annotations;
int scope_type;
@ -1866,27 +1843,18 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
return 0;
}
st = (stmt_ty)asdl_seq_GET(body, 0);
docstring = compiler_isdocstring(st);
if (docstring && c->c_optimize < 2) {
if (st->v.Expr.value->kind == Constant_kind)
first_const = st->v.Expr.value->v.Constant.value;
else
first_const = st->v.Expr.value->v.Str.s;
}
if (compiler_add_o(c, c->u->u_consts, first_const) < 0) {
/* if not -OO mode, add docstring */
if (c->c_optimize < 2 && s->v.FunctionDef.docstring)
docstring = s->v.FunctionDef.docstring;
if (compiler_add_o(c, c->u->u_consts, docstring) < 0) {
compiler_exit_scope(c);
return 0;
}
c->u->u_argcount = asdl_seq_LEN(args->args);
c->u->u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs);
n = asdl_seq_LEN(body);
/* if there was a docstring, we need to skip the first statement */
for (i = docstring; i < n; i++) {
st = (stmt_ty)asdl_seq_GET(body, i);
VISIT_IN_SCOPE(c, stmt, st);
}
VISIT_SEQ_IN_SCOPE(c, stmt, body);
co = assemble(c, 1);
qualname = c->u->u_qualname;
Py_INCREF(qualname);
@ -1967,7 +1935,7 @@ compiler_class(struct compiler *c, stmt_ty s)
}
Py_DECREF(str);
/* compile the body proper */
if (!compiler_body(c, s->v.ClassDef.body)) {
if (!compiler_body(c, s->v.ClassDef.body, s->v.ClassDef.docstring)) {
compiler_exit_scope(c);
return 0;
}

12
Python/future.c
View File

@ -61,7 +61,6 @@ static int
future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename)
{
int i, done = 0, prev_line = 0;
stmt_ty first;
if (!(mod->kind == Module_kind || mod->kind == Interactive_kind))
return 1;
@ -77,16 +76,7 @@ future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename)
but is preceded by a regular import.
*/
i = 0;
first = (stmt_ty)asdl_seq_GET(mod->v.Module.body, i);
if (first->kind == Expr_kind
&& (first->v.Expr.value->kind == Str_kind
|| (first->v.Expr.value->kind == Constant_kind
&& PyUnicode_CheckExact(first->v.Expr.value->v.Constant.value))))
i++;
for (; i < asdl_seq_LEN(mod->v.Module.body); i++) {
for (i = 0; i < asdl_seq_LEN(mod->v.Module.body); i++) {
stmt_ty s = (stmt_ty)asdl_seq_GET(mod->v.Module.body, i);
if (done && s->lineno > prev_line)

3194
Python/importlib.h
View File

File diff suppressed because it is too large Load Diff

2497
Python/importlib_external.h
View File

File diff suppressed because it is too large Load Diff

6
Tools/parser/unparse.py
View File

@ -71,6 +71,8 @@ class Unparser:
########################################################
def _Module(self, tree):
if tree.docstring is not None:
self.fill(repr(tree.docstring))
for stmt in tree.body:
self.dispatch(stmt)
@ -235,6 +237,8 @@ class Unparser:
self.write(")")
self.enter()
if t.docstring is not None:
self.fill(repr(t.docstring))
self.dispatch(t.body)
self.leave()
@ -257,6 +261,8 @@ class Unparser:
self.write(" -> ")
self.dispatch(t.returns)
self.enter()
if t.docstring is not None:
self.fill(repr(t.docstring))
self.dispatch(t.body)
self.leave()