Issue #10992: make tests pass when run under coverage.

Various tests fail when run under coverage. A primary culprit is refcount tests
which fail as the counts are thrown off by the coverage code. A new decorator
-- test.support.refcount_test -- is used to decorate tests which test refcounts
and to skip them when running under coverage. Other tests simply fail because
of changes in the system (e.g., __local__ suddenly appearing).

Thanks to Kristian Vlaardingerbroek for helping to diagnose the test failures.

Lib/ctypes/test/test_memfunctions.py

@@ -1,4 +1,5 @@
 import sys
+from test import support
 import unittest
 from ctypes import *
 
@@ -49,6 +50,7 @@ class MemFunctionsTest(unittest.TestCase):
         self.assertEqual(cast(a, POINTER(c_byte))[:7:7],
                              [97])
 
+    @support.refcount_test
     def test_string_at(self):
         s = string_at(b"foo bar")
         # XXX The following may be wrong, depending on how Python

Lib/ctypes/test/test_python_api.py

@@ -1,5 +1,6 @@
 from ctypes import *
 import unittest, sys
+from test import support
 from ctypes.test import is_resource_enabled
 
 ################################################################
@@ -25,6 +26,7 @@ class PythonAPITestCase(unittest.TestCase):
 
         self.assertEqual(PyBytes_FromStringAndSize(b"abcdefghi", 3), b"abc")
 
+    @support.refcount_test
     def test_PyString_FromString(self):
         pythonapi.PyBytes_FromString.restype = py_object
         pythonapi.PyBytes_FromString.argtypes = (c_char_p,)
@@ -56,6 +58,7 @@ class PythonAPITestCase(unittest.TestCase):
             del res
             self.assertEqual(grc(42), ref42)
 
+    @support.refcount_test
     def test_PyObj_FromPtr(self):
         s = "abc def ghi jkl"
         ref = grc(s)

Lib/ctypes/test/test_refcounts.py

@@ -1,4 +1,5 @@
 import unittest
+from test import support
 import ctypes
 import gc
 
@@ -10,6 +11,7 @@ dll = ctypes.CDLL(_ctypes_test.__file__)
 
 class RefcountTestCase(unittest.TestCase):
 
+    @support.refcount_test
     def test_1(self):
         from sys import getrefcount as grc
 
@@ -34,6 +36,7 @@ class RefcountTestCase(unittest.TestCase):
         self.assertEqual(grc(callback), 2)
 
 
+    @support.refcount_test
     def test_refcount(self):
         from sys import getrefcount as grc
         def func(*args):

Lib/ctypes/test/test_stringptr.py

@@ -1,4 +1,5 @@
 import unittest
+from test import support
 from ctypes import *
 
 import _ctypes_test
@@ -7,6 +8,7 @@ lib = CDLL(_ctypes_test.__file__)
 
 class StringPtrTestCase(unittest.TestCase):
 
+    @support.refcount_test
     def test__POINTER_c_char(self):
         class X(Structure):
             _fields_ = [("str", POINTER(c_char))]

Lib/test/support.py

@@ -1124,6 +1124,17 @@ def no_tracing(func):
         return wrapper
 
 
+def refcount_test(test):
+    """Decorator for tests which involve reference counting.
+
+    To start, the decorator does not run the test if is not run by CPython.
+    After that, any trace function is unset during the test to prevent
+    unexpected refcounts caused by the trace function.
+
+    """
+    return no_tracing(cpython_only(test))
+
+
 def _run_suite(suite):
     """Run tests from a unittest.TestSuite-derived class."""
     if verbose:

Lib/test/test_descr.py

@@ -4243,6 +4243,8 @@ class DictProxyTests(unittest.TestCase):
                 pass
         self.C = C
 
+    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+                        'trace function introduces __local__')
     def test_iter_keys(self):
         # Testing dict-proxy keys...
         it = self.C.__dict__.keys()
@@ -4252,6 +4254,8 @@ class DictProxyTests(unittest.TestCase):
         self.assertEqual(keys, ['__dict__', '__doc__', '__module__',
             '__weakref__', 'meth'])
 
+    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+                        'trace function introduces __local__')
     def test_iter_values(self):
         # Testing dict-proxy values...
         it = self.C.__dict__.values()
@@ -4259,6 +4263,8 @@ class DictProxyTests(unittest.TestCase):
         values = list(it)
         self.assertEqual(len(values), 5)
 
+    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+                        'trace function introduces __local__')
     def test_iter_items(self):
         # Testing dict-proxy iteritems...
         it = self.C.__dict__.items()

Lib/test/test_gc.py

@@ -1,5 +1,6 @@
 import unittest
-from test.support import verbose, run_unittest, strip_python_stderr
+from test.support import (verbose, refcount_test, run_unittest,
+                            strip_python_stderr)
 import sys
 import gc
 import weakref
@@ -175,6 +176,7 @@ class GCTests(unittest.TestCase):
         del d
         self.assertEqual(gc.collect(), 2)
 
+    @refcount_test
     def test_frame(self):
         def f():
             frame = sys._getframe()
@@ -242,6 +244,7 @@ class GCTests(unittest.TestCase):
     # For example:
     # - disposed tuples are not freed, but reused
     # - the call to assertEqual somehow avoids building its args tuple
+    @refcount_test
     def test_get_count(self):
         # Avoid future allocation of method object
         assertEqual = self._baseAssertEqual
@@ -252,6 +255,7 @@ class GCTests(unittest.TestCase):
         # the dict, and the tuple returned by get_count()
         assertEqual(gc.get_count(), (2, 0, 0))
 
+    @refcount_test
     def test_collect_generations(self):
         # Avoid future allocation of method object
         assertEqual = self.assertEqual

Lib/test/test_genexps.py

@@ -257,11 +257,15 @@ Verify that genexps are weakly referencable
 
 """
 
+import sys
 
-__test__ = {'doctests' : doctests}
+# Trace function can throw off the tuple reuse test.
+if hasattr(sys, 'gettrace') and sys.gettrace():
+    __test__ = {}
+else:
+    __test__ = {'doctests' : doctests}
 
 def test_main(verbose=None):
-    import sys
     from test import support
     from test import test_genexps
     support.run_doctest(test_genexps, verbose)

Lib/test/test_metaclass.py

@@ -246,7 +246,13 @@ Test failures in looking up the __prepare__ method work.
 
 """
 
-__test__ = {'doctests' : doctests}
+import sys
+
+# Trace function introduces __locals__ which causes various tests to fail.
+if hasattr(sys, 'gettrace') and sys.gettrace():
+    __test__ = {}
+else:
+    __test__ = {'doctests' : doctests}
 
 def test_main(verbose=False):
     from test import support

Lib/test/test_pydoc.py

@@ -248,6 +248,8 @@ class PydocDocTest(unittest.TestCase):
 
     @unittest.skipIf(sys.flags.optimize >= 2,
                      "Docstrings are omitted with -O2 and above")
+    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+                     'trace function introduces __locals__ unexpectedly')
     def test_html_doc(self):
         result, doc_loc = get_pydoc_html(pydoc_mod)
         mod_file = inspect.getabsfile(pydoc_mod)
@@ -263,6 +265,8 @@ class PydocDocTest(unittest.TestCase):
 
     @unittest.skipIf(sys.flags.optimize >= 2,
                      "Docstrings are omitted with -O2 and above")
+    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+                     'trace function introduces __locals__ unexpectedly')
     def test_text_doc(self):
         result, doc_loc = get_pydoc_text(pydoc_mod)
         expected_text = expected_text_pattern % \
@@ -340,6 +344,8 @@ class PydocDocTest(unittest.TestCase):
 
     @unittest.skipIf(sys.flags.optimize >= 2,
                      'Docstrings are omitted with -O2 and above')
+    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+                     'trace function introduces __locals__ unexpectedly')
     def test_help_output_redirect(self):
         # issue 940286, if output is set in Helper, then all output from
         # Helper.help should be redirected

Lib/test/test_sys.py

@@ -303,6 +303,7 @@ class SysModuleTest(unittest.TestCase):
             self.assertEqual(sys.getdlopenflags(), oldflags+1)
             sys.setdlopenflags(oldflags)
 
+    @test.support.refcount_test
     def test_refcount(self):
         # n here must be a global in order for this test to pass while
         # tracing with a python function.  Tracing calls PyFrame_FastToLocals

Lib/test/test_trace.py

@@ -245,6 +245,8 @@ class TestFuncs(unittest.TestCase):
         }
         self.assertEqual(self.tracer.results().calledfuncs, expected)
 
+    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+                     'pre-existing trace function throws off measurements')
     def test_inst_method_calling(self):
         obj = TracedClass(20)
         self.tracer.runfunc(obj.inst_method_calling, 1)
@@ -264,6 +266,8 @@ class TestCallers(unittest.TestCase):
         self.tracer = Trace(count=0, trace=0, countcallers=1)
         self.filemod = my_file_and_modname()
 
+    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+                     'pre-existing trace function throws off measurements')
     def test_loop_caller_importing(self):
         self.tracer.runfunc(traced_func_importing_caller, 1)
 

Misc/NEWS

@@ -54,6 +54,8 @@ Build
 Tests
 -----
 
+- Issue #10992: Make tests pass under coverage.
+
 - Issue #10826: Prevent sporadic failure in test_subprocess on Solaris due
   to open door files.