bpo-34876: Change the lineno of the AST for decorated function and class. (GH-9731)

It was overridden by the lineno of the first decorator. Now it is
the lineno of 'def' or 'class'.

Lib/test/test_ast.py

@@ -124,6 +124,12 @@ exec_tests = [
     "{*{1, 2}, 3}",
     # Asynchronous comprehensions
     "async def f():\n [i async for b in c]",
+    # Decorated FunctionDef
+    "@deco1\n@deco2()\ndef f(): pass",
+    # Decorated AsyncFunctionDef
+    "@deco1\n@deco2()\nasync def f(): pass",
+    # Decorated ClassDef
+    "@deco1\n@deco2()\nclass C: pass",
 ]
 
 # These are compiled through "single"
@@ -203,13 +209,16 @@ class AST_Tests(unittest.TestCase):
             return
         if isinstance(ast_node, (ast.expr, ast.stmt, ast.excepthandler)):
             node_pos = (ast_node.lineno, ast_node.col_offset)
-            self.assertTrue(node_pos >= parent_pos)
+            self.assertGreaterEqual(node_pos, parent_pos)
             parent_pos = (ast_node.lineno, ast_node.col_offset)
         for name in ast_node._fields:
             value = getattr(ast_node, name)
             if isinstance(value, list):
+                first_pos = parent_pos
+                if value and name == 'decorator_list':
+                    first_pos = (value[0].lineno, value[0].col_offset)
                 for child in value:
-                    self._assertTrueorder(child, parent_pos)
+                    self._assertTrueorder(child, first_pos)
             elif value is not None:
                 self._assertTrueorder(value, parent_pos)
 
@@ -1289,6 +1298,9 @@ exec_results = [
 ('Module', [('Expr', (1, 0), ('Dict', (1, 0), [None, ('Constant', (1, 10), 2)], [('Dict', (1, 3), [('Constant', (1, 4), 1)], [('Constant', (1, 6), 2)]), ('Constant', (1, 12), 3)]))]),
 ('Module', [('Expr', (1, 0), ('Set', (1, 0), [('Starred', (1, 1), ('Set', (1, 2), [('Constant', (1, 3), 1), ('Constant', (1, 6), 2)]), ('Load',)), ('Constant', (1, 10), 3)]))]),
 ('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('ListComp', (2, 2), ('Name', (2, 2), 'i', ('Load',)), [('comprehension', ('Name', (2, 14), 'b', ('Store',)), ('Name', (2, 19), 'c', ('Load',)), [], 1)]))], [], None)]),
+('Module', [('FunctionDef', (3, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (3, 9))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])], None)]),
+('Module', [('AsyncFunctionDef', (3, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (3, 15))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])], None)]),
+('Module', [('ClassDef', (3, 0), 'C', [], [], [('Pass', (3, 9))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])])]),
 ]
 single_results = [
 ('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Constant', (1, 0), 1), ('Add',), ('Constant', (1, 2), 2)))]),

Lib/test/test_trace.py

@@ -75,6 +75,19 @@ def traced_caller_list_comprehension():
     mylist = [traced_doubler(i) for i in range(k)]
     return mylist
 
+def traced_decorated_function():
+    def decorator1(f):
+        return f
+    def decorator_fabric():
+        def decorator2(f):
+            return f
+        return decorator2
+    @decorator1
+    @decorator_fabric()
+    def func():
+        pass
+    func()
+
 
 class TracedClass(object):
     def __init__(self, x):
@@ -172,6 +185,24 @@ class TestLineCounts(unittest.TestCase):
         }
         self.assertEqual(self.tracer.results().counts, expected)
 
+    def test_traced_decorated_function(self):
+        self.tracer.runfunc(traced_decorated_function)
+
+        firstlineno = get_firstlineno(traced_decorated_function)
+        expected = {
+            (self.my_py_filename, firstlineno + 1): 1,
+            (self.my_py_filename, firstlineno + 2): 1,
+            (self.my_py_filename, firstlineno + 3): 1,
+            (self.my_py_filename, firstlineno + 4): 1,
+            (self.my_py_filename, firstlineno + 5): 1,
+            (self.my_py_filename, firstlineno + 6): 1,
+            (self.my_py_filename, firstlineno + 7): 1,
+            (self.my_py_filename, firstlineno + 8): 1,
+            (self.my_py_filename, firstlineno + 9): 1,
+            (self.my_py_filename, firstlineno + 10): 1,
+            (self.my_py_filename, firstlineno + 11): 1,
+        }
+        self.assertEqual(self.tracer.results().counts, expected)
 
     def test_linear_methods(self):
         # XXX todo: later add 'static_method_linear' and 'class_method_linear'
@@ -189,6 +220,7 @@ class TestLineCounts(unittest.TestCase):
             }
             self.assertEqual(tracer.results().counts, expected)
 
+
 class TestRunExecCounts(unittest.TestCase):
     """A simple sanity test of line-counting, via runctx (exec)"""
     def setUp(self):
@@ -263,6 +295,18 @@ class TestFuncs(unittest.TestCase):
         }
         self.assertEqual(self.tracer.results().calledfuncs, expected)
 
+    def test_traced_decorated_function(self):
+        self.tracer.runfunc(traced_decorated_function)
+
+        expected = {
+            self.filemod + ('traced_decorated_function',): 1,
+            self.filemod + ('decorator_fabric',): 1,
+            self.filemod + ('decorator2',): 1,
+            self.filemod + ('decorator1',): 1,
+            self.filemod + ('func',): 1,
+        }
+        self.assertEqual(self.tracer.results().calledfuncs, expected)
+
 
 class TestCallers(unittest.TestCase):
     """White-box testing of callers tracing"""

Misc/NEWS.d/next/Core and Builtins/2018-10-06-14-02-51.bpo-34876.oBKBA4.rst

@@ -0,0 +1,6 @@
+The *lineno* and *col_offset* attributes of the AST for decorated function
+and class refer now to the position of the corresponding ``def``, ``async
+def`` and ``class`` instead of the position of the first decorator. This
+leads to more correct line reporting in tracing. This is the only case when
+the position of child AST nodes can preceed the position of the parent AST
+node.

Python/ast.c

@@ -1659,12 +1659,6 @@ ast_for_decorated(struct compiling *c, const node *n)
     } else if (TYPE(CHILD(n, 1)) == async_funcdef) {
       thing = ast_for_async_funcdef(c, CHILD(n, 1), decorator_seq);
     }
-    /* we count the decorators in when talking about the class' or
-     * function's line number */
-    if (thing) {
-        thing->lineno = LINENO(n);
-        thing->col_offset = n->n_col_offset;
-    }
     return thing;
 }
 

Python/compile.c

@@ -1950,6 +1950,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
     Py_ssize_t i, funcflags;
     int annotations;
     int scope_type;
+    int firstlineno;
 
     if (is_async) {
         assert(s->kind == AsyncFunctionDef_kind);
@@ -1976,6 +1977,11 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
     if (!compiler_decorators(c, decos))
         return 0;
 
+    firstlineno = s->lineno;
+    if (asdl_seq_LEN(decos)) {
+        firstlineno = ((expr_ty)asdl_seq_GET(decos, 0))->lineno;
+    }
+
     funcflags = compiler_default_arguments(c, args);
     if (funcflags == -1) {
         return 0;
@@ -1989,7 +1995,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
         funcflags |= 0x04;
     }
 
-    if (!compiler_enter_scope(c, name, scope_type, (void *)s, s->lineno)) {
+    if (!compiler_enter_scope(c, name, scope_type, (void *)s, firstlineno)) {
         return 0;
     }
 
@@ -2032,12 +2038,17 @@ compiler_class(struct compiler *c, stmt_ty s)
 {
     PyCodeObject *co;
     PyObject *str;
-    int i;
+    int i, firstlineno;
     asdl_seq* decos = s->v.ClassDef.decorator_list;
 
     if (!compiler_decorators(c, decos))
         return 0;
 
+    firstlineno = s->lineno;
+    if (asdl_seq_LEN(decos)) {
+        firstlineno = ((expr_ty)asdl_seq_GET(decos, 0))->lineno;
+    }
+
     /* ultimately generate code for:
          <name> = __build_class__(<func>, <name>, *<bases>, **<keywords>)
        where:
@@ -2052,7 +2063,7 @@ compiler_class(struct compiler *c, stmt_ty s)
 
     /* 1. compile the class body into a code object */
     if (!compiler_enter_scope(c, s->v.ClassDef.name,
-                              COMPILER_SCOPE_CLASS, (void *)s, s->lineno))
+                              COMPILER_SCOPE_CLASS, (void *)s, firstlineno))
         return 0;
     /* this block represents what we do in the new scope */
     {