bpo-42116: Fix inspect.getsource handling of trailing comments (GH-23630)

https://bugzilla.redhat.com/show_bug.cgi?id=1866884 is fixed in gdb
10.1 (failed to reproduce on gdb-10.1-1.fc34.aarch64).

Lib/dis.py

@@ -384,7 +384,7 @@ def _disassemble_bytes(code, lasti=-1, varnames=None, names=None,
                        constants=None, cells=None, linestarts=None,
                        *, file=None, line_offset=0):
     # Omit the line number column entirely if we have no line number info
-    show_lineno = linestarts is not None
+    show_lineno = bool(linestarts)
     if show_lineno:
         maxlineno = max(linestarts.values()) + line_offset
         if maxlineno >= 1000:

Lib/inspect.py

@@ -930,6 +930,7 @@ class BlockFinder:
         self.indecorator = False
         self.decoratorhasargs = False
         self.last = 1
+        self.body_col0 = None
 
     def tokeneater(self, type, token, srowcol, erowcol, line):
         if not self.started and not self.indecorator:
@@ -961,6 +962,8 @@ class BlockFinder:
         elif self.passline:
             pass
         elif type == tokenize.INDENT:
+            if self.body_col0 is None and self.started:
+                self.body_col0 = erowcol[1]
             self.indent = self.indent + 1
             self.passline = True
         elif type == tokenize.DEDENT:
@@ -970,6 +973,10 @@ class BlockFinder:
             #  not e.g. for "if: else:" or "try: finally:" blocks)
             if self.indent <= 0:
                 raise EndOfBlock
+        elif type == tokenize.COMMENT:
+            if self.body_col0 is not None and srowcol[1] >= self.body_col0:
+                # Include comments if indented at least as much as the block
+                self.last = srowcol[0]
         elif self.indent == 0 and type not in (tokenize.COMMENT, tokenize.NL):
             # any other token on the same indentation level end the previous
             # block as well, except the pseudo-tokens COMMENT and NL.

Lib/test/inspect_fodder.py

@@ -91,3 +91,25 @@ class Callable:
 
 custom_method = Callable().as_method_of(42)
 del Callable
+
+# line 95
+class WhichComments:
+  # line 97
+    # before f
+    def f(self):
+      # line 100
+        # start f
+        return 1
+        # line 103
+        # end f
+       # line 105
+    # after f
+
+    # before asyncf - line 108
+    async def asyncf(self):
+        # start asyncf
+        return 2
+        # end asyncf
+       # after asyncf - line 113
+    # end of WhichComments - line 114
+  # after WhichComments - line 115

Lib/test/test_compile.py

@@ -809,6 +809,24 @@ if 1:
                 func(save_caller_frame)
                 self.assertEqual(frame.f_lineno-frame.f_code.co_firstlineno, lastline)
 
+    def test_lineno_after_no_code(self):
+        def no_code1():
+            "doc string"
+
+        def no_code2():
+            a: int
+
+        for func in (no_code1, no_code2):
+            with self.subTest(func=func):
+                code = func.__code__
+                lines = list(code.co_lines())
+                self.assertEqual(len(lines), 1)
+                start, end, line = lines[0]
+                self.assertEqual(start, 0)
+                self.assertEqual(end, len(code.co_code))
+                self.assertEqual(line, code.co_firstlineno)
+
+
     def test_big_dict_literal(self):
         # The compiler has a flushing point in "compiler_dict" that calls compiles
         # a portion of the dictionary literal when the loop that iterates over the items

Lib/test/test_dis.py

@@ -166,6 +166,20 @@ dis_bug1333982 = """\
        bug1333982.__code__.co_firstlineno + 2,
        bug1333982.__code__.co_firstlineno + 1)
 
+
+def bug42562():
+    pass
+
+
+# Set line number for 'pass' to None
+bug42562.__code__ = bug42562.__code__.replace(co_linetable=b'\x04\x80\xff\x80')
+
+
+dis_bug42562 = """\
+          0 LOAD_CONST               0 (None)
+          2 RETURN_VALUE
+"""
+
 _BIG_LINENO_FORMAT = """\
 %3d           0 LOAD_GLOBAL              0 (spam)
               2 POP_TOP
@@ -520,6 +534,9 @@ class DisTests(unittest.TestCase):
 
         self.do_disassembly_test(bug1333982, dis_bug1333982)
 
+    def test_bug_42562(self):
+        self.do_disassembly_test(bug42562, dis_bug42562)
+
     def test_big_linenos(self):
         def func(count):
             namespace = {}

Lib/test/test_gdb.py

@@ -51,11 +51,6 @@ if gdb_major_version < 7:
                             "embedding. Saw %s.%s:\n%s"
                             % (gdb_major_version, gdb_minor_version,
                                gdb_version))
-if (gdb_major_version, gdb_minor_version) >= (9, 2):
-    # gdb 9.2 on Fedora Rawhide is not reliable, see:
-    # * https://bugs.python.org/issue41473
-    # * https://bugzilla.redhat.com/show_bug.cgi?id=1866884
-    raise unittest.SkipTest("https://bugzilla.redhat.com/show_bug.cgi?id=1866884")
 
 if not sysconfig.is_python_build():
     raise unittest.SkipTest("test_gdb only works on source builds at the moment.")

Lib/test/test_inspect.py

@@ -390,6 +390,7 @@ class TestRetrievingSourceCode(GetSourceBase):
                           ('ParrotDroppings', mod.ParrotDroppings),
                           ('StupidGit', mod.StupidGit),
                           ('Tit', mod.MalodorousPervert),
+                          ('WhichComments', mod.WhichComments),
                          ])
         tree = inspect.getclasstree([cls[1] for cls in classes])
         self.assertEqual(tree,
@@ -403,7 +404,8 @@ class TestRetrievingSourceCode(GetSourceBase):
                             [(mod.FesteringGob, (mod.MalodorousPervert,
                                                     mod.ParrotDroppings))
                              ]
-                            ]
+                            ],
+                            (mod.WhichComments, (object,),)
                            ]
                           ])
         tree = inspect.getclasstree([cls[1] for cls in classes], True)
@@ -415,7 +417,8 @@ class TestRetrievingSourceCode(GetSourceBase):
                             [(mod.FesteringGob, (mod.MalodorousPervert,
                                                     mod.ParrotDroppings))
                              ]
-                            ]
+                            ],
+                            (mod.WhichComments, (object,),)
                            ]
                           ])
 
@@ -646,6 +649,18 @@ class TestOneliners(GetSourceBase):
         # as argument to another function.
         self.assertSourceEqual(mod2.anonymous, 55, 55)
 
+class TestBlockComments(GetSourceBase):
+    fodderModule = mod
+
+    def test_toplevel_class(self):
+        self.assertSourceEqual(mod.WhichComments, 96, 114)
+
+    def test_class_method(self):
+        self.assertSourceEqual(mod.WhichComments.f, 99, 104)
+
+    def test_class_async_method(self):
+        self.assertSourceEqual(mod.WhichComments.asyncf, 109, 112)
+
 class TestBuggyCases(GetSourceBase):
     fodderModule = mod2
 
@@ -4014,8 +4029,8 @@ def foo():
 
 def test_main():
     run_unittest(
-        TestDecorators, TestRetrievingSourceCode, TestOneliners, TestBuggyCases,
-        TestInterpreterStack, TestClassesAndFunctions, TestPredicates,
+        TestDecorators, TestRetrievingSourceCode, TestOneliners, TestBlockComments,
+        TestBuggyCases, TestInterpreterStack, TestClassesAndFunctions, TestPredicates,
         TestGetcallargsFunctions, TestGetcallargsMethods,
         TestGetcallargsUnboundMethods, TestGetattrStatic, TestGetGeneratorState,
         TestNoEOL, TestSignatureObject, TestSignatureBind, TestParameterObject,

Misc/NEWS.d/next/Library/2020-12-03-15-42-32.bpo-42116.yIwroP.rst

@@ -0,0 +1 @@
+Fix handling of trailing comments by :func:`inspect.getsource`.

Misc/NEWS.d/next/Library/2020-12-03-22-42-03.bpo-42562.2hPmhi.rst

@@ -0,0 +1,2 @@
+Fix issue when dis failed to parse function that has no line numbers. Patch
+provided by Yurii Karabas.

Misc/NEWS.d/next/Tests/2020-12-04-11-47-09.bpo-41473.W_updK.rst

@@ -0,0 +1,3 @@
+Reenable test_gdb on gdb 9.2 and newer:
+https://bugzilla.redhat.com/show_bug.cgi?id=1866884 bug is fixed in gdb
+10.1.

Python/compile.c

@@ -6514,6 +6514,7 @@ is_exit_without_lineno(basicblock *b) {
 static int
 ensure_exits_have_lineno(struct compiler *c)
 {
+    basicblock *entry = NULL;
     /* Copy all exit blocks without line number that are targets of a jump.
      */
     for (basicblock *b = c->u->u_blocks; b != NULL; b = b->b_list) {
@@ -6535,6 +6536,11 @@ ensure_exits_have_lineno(struct compiler *c)
                 b->b_instr[b->b_iused-1].i_target = new_target;
             }
         }
+        entry = b;
+    }
+    assert(entry != NULL);
+    if (is_exit_without_lineno(entry)) {
+        entry->b_instr[0].i_lineno = c->u->u_firstlineno;
     }
     /* Any remaining reachable exit blocks without line number can only be reached by
      * fall through, and thus can only have a single predecessor */

Python/importlib.h

@@ -1243,7 +1243,7 @@ const unsigned char _Py_M__importlib_bootstrap[] = {
     114,32,109,111,100,117,108,101,32,99,114,101,97,116,105,111,
     110,46,78,114,10,0,0,0,114,161,0,0,0,114,10,0,
     0,0,114,10,0,0,0,114,11,0,0,0,114,150,0,0,
-    0,66,3,0,0,115,4,0,0,0,4,128,255,128,122,28,
+    0,66,3,0,0,115,4,0,0,0,4,0,255,128,122,28,
     70,114,111,122,101,110,73,109,112,111,114,116,101,114,46,99,
     114,101,97,116,101,95,109,111,100,117,108,101,99,1,0,0,
     0,0,0,0,0,0,0,0,0,3,0,0,0,4,0,0,

Python/importlib_external.h

@@ -1153,7 +1153,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,169,2,114,123,0,0,0,114,191,0,0,0,114,7,0,
     0,0,114,7,0,0,0,114,8,0,0,0,218,13,99,114,
     101,97,116,101,95,109,111,100,117,108,101,55,3,0,0,115,
-    4,0,0,0,4,128,255,128,122,27,95,76,111,97,100,101,
+    4,0,0,0,4,0,255,128,122,27,95,76,111,97,100,101,
     114,66,97,115,105,99,115,46,99,114,101,97,116,101,95,109,
     111,100,117,108,101,99,2,0,0,0,0,0,0,0,0,0,
     0,0,3,0,0,0,5,0,0,0,67,0,0,0,115,56,
@@ -1292,7 +1292,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     32,32,32,78,114,7,0,0,0,41,3,114,123,0,0,0,
     114,52,0,0,0,114,37,0,0,0,114,7,0,0,0,114,
     7,0,0,0,114,8,0,0,0,114,232,0,0,0,105,3,
-    0,0,115,4,0,0,0,4,128,255,128,122,21,83,111,117,
+    0,0,115,4,0,0,0,4,0,255,128,122,21,83,111,117,
     114,99,101,76,111,97,100,101,114,46,115,101,116,95,100,97,
     116,97,99,2,0,0,0,0,0,0,0,0,0,0,0,5,
     0,0,0,10,0,0,0,67,0,0,0,115,70,0,0,0,
@@ -2019,7 +2019,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,115,4,0,0,0,100,1,83,0,114,217,0,0,0,114,
     7,0,0,0,114,218,0,0,0,114,7,0,0,0,114,7,
     0,0,0,114,8,0,0,0,114,219,0,0,0,217,4,0,
-    0,115,4,0,0,0,4,128,255,128,122,30,95,78,97,109,
+    0,115,4,0,0,0,4,0,255,128,122,30,95,78,97,109,
     101,115,112,97,99,101,76,111,97,100,101,114,46,99,114,101,
     97,116,101,95,109,111,100,117,108,101,99,2,0,0,0,0,
     0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,67,