Close #19746: expose unittest discovery errors on TestLoader.errors

This makes it possible to examine the errors from unittest discovery
without executing the test suite - important when the test suite may
be very large, or when enumerating the test ids from a test suite.

Doc/library/unittest.rst

@@ -1552,6 +1552,20 @@ Loading and running tests
    :data:`unittest.defaultTestLoader`.  Using a subclass or instance, however,
    allows customization of some configurable properties.
 
+   :class:`TestLoader` objects have the following attributes:
+
+
+   .. attribute:: errors
+
+      A list of the non-fatal errors encountered while loading tests. Not reset
+      by the loader at any point. Fatal errors are signalled by the relevant
+      a method raising an exception to the caller. Non-fatal errors are also
+      indicated by a synthetic test that will raise the original error when
+      run.
+
+      .. versionadded:: 3.5
+
+
    :class:`TestLoader` objects have the following methods:
 
 

Lib/unittest/loader.py

@@ -21,19 +21,22 @@ VALID_MODULE_NAME = re.compile(r'[_a-z]\w*\.py$', re.IGNORECASE)
 
 
 def _make_failed_import_test(name, suiteClass):
-    message = 'Failed to import test module: %s\n%s' % (name, traceback.format_exc())
+    message = 'Failed to import test module: %s\n%s' % (
+        name, traceback.format_exc())
     return _make_failed_test('ModuleImportFailure', name, ImportError(message),
-                             suiteClass)
+                             suiteClass, message)
 
 def _make_failed_load_tests(name, exception, suiteClass):
-    return _make_failed_test('LoadTestsFailure', name, exception, suiteClass)
+    message = 'Failed to call load_tests:\n%s' % (traceback.format_exc(),)
+    return _make_failed_test(
+        'LoadTestsFailure', name, exception, suiteClass, message)
 
-def _make_failed_test(classname, methodname, exception, suiteClass):
+def _make_failed_test(classname, methodname, exception, suiteClass, message):
     def testFailure(self):
         raise exception
     attrs = {methodname: testFailure}
     TestClass = type(classname, (case.TestCase,), attrs)
-    return suiteClass((TestClass(methodname),))
+    return suiteClass((TestClass(methodname),)), message
 
 def _make_skipped_test(methodname, exception, suiteClass):
     @case.skip(str(exception))
@@ -59,6 +62,10 @@ class TestLoader(object):
     suiteClass = suite.TestSuite
     _top_level_dir = None
 
+    def __init__(self):
+        super(TestLoader, self).__init__()
+        self.errors = []
+
     def loadTestsFromTestCase(self, testCaseClass):
         """Return a suite of all tests cases contained in testCaseClass"""
         if issubclass(testCaseClass, suite.TestSuite):
@@ -107,8 +114,10 @@ class TestLoader(object):
             try:
                 return load_tests(self, tests, pattern)
             except Exception as e:
-                return _make_failed_load_tests(module.__name__, e,
+                error_case, error_message = _make_failed_load_tests(
-                                               self.suiteClass)
+                    module.__name__, e, self.suiteClass)
+                self.errors.append(error_message)
+                return error_case
         return tests
 
     def loadTestsFromName(self, name, module=None):
@@ -336,7 +345,10 @@ class TestLoader(object):
                 except case.SkipTest as e:
                     yield _make_skipped_test(name, e, self.suiteClass)
                 except:
-                    yield _make_failed_import_test(name, self.suiteClass)
+                    error_case, error_message = \
+                        _make_failed_import_test(name, self.suiteClass)
+                    self.errors.append(error_message)
+                    yield error_case
                 else:
                     mod_file = os.path.abspath(getattr(module, '__file__', full_path))
                     realpath = _jython_aware_splitext(os.path.realpath(mod_file))
@@ -362,7 +374,10 @@ class TestLoader(object):
                 except case.SkipTest as e:
                     yield _make_skipped_test(name, e, self.suiteClass)
                 except:
-                    yield _make_failed_import_test(name, self.suiteClass)
+                    error_case, error_message = \
+                        _make_failed_import_test(name, self.suiteClass)
+                    self.errors.append(error_message)
+                    yield error_case
                 else:
                     load_tests = getattr(package, 'load_tests', None)
                     tests = self.loadTestsFromModule(package, pattern=pattern)

Lib/unittest/test/test_discovery.py

@@ -399,6 +399,13 @@ class TestDiscovery(unittest.TestCase):
         suite = loader.discover('.')
         self.assertIn(os.getcwd(), sys.path)
         self.assertEqual(suite.countTestCases(), 1)
+        # Errors loading the suite are also captured for introspection.
+        self.assertNotEqual([], loader.errors)
+        self.assertEqual(1, len(loader.errors))
+        error = loader.errors[0]
+        self.assertTrue(
+            'Failed to import test module: test_this_does_not_exist' in error,
+            'missing error string in %r' % error)
         test = list(list(suite)[0])[0] # extract test from suite
 
         with self.assertRaises(ImportError):
@@ -418,6 +425,13 @@ class TestDiscovery(unittest.TestCase):
 
         self.assertIn(abspath('/foo'), sys.path)
         self.assertEqual(suite.countTestCases(), 1)
+        # Errors loading the suite are also captured for introspection.
+        self.assertNotEqual([], loader.errors)
+        self.assertEqual(1, len(loader.errors))
+        error = loader.errors[0]
+        self.assertTrue(
+            'Failed to import test module: my_package' in error,
+            'missing error string in %r' % error)
         test = list(list(suite)[0])[0] # extract test from suite
         with self.assertRaises(ImportError):
             test.my_package()

Lib/unittest/test/test_loader.py

@@ -24,6 +24,13 @@ def warningregistry(func):
 
 class Test_TestLoader(unittest.TestCase):
 
+    ### Basic object tests
+    ################################################################
+
+    def test___init__(self):
+        loader = unittest.TestLoader()
+        self.assertEqual([], loader.errors)
+
     ### Tests for TestLoader.loadTestsFromTestCase
     ################################################################
 
@@ -336,6 +343,13 @@ class Test_TestLoader(unittest.TestCase):
         suite = loader.loadTestsFromModule(m)
         self.assertIsInstance(suite, unittest.TestSuite)
         self.assertEqual(suite.countTestCases(), 1)
+        # Errors loading the suite are also captured for introspection.
+        self.assertNotEqual([], loader.errors)
+        self.assertEqual(1, len(loader.errors))
+        error = loader.errors[0]
+        self.assertTrue(
+            'Failed to call load_tests:' in error,
+            'missing error string in %r' % error)
         test = list(suite)[0]
 
         self.assertRaisesRegex(TypeError, "some failure", test.m)

Misc/NEWS

@@ -186,6 +186,10 @@ Library
 - Issue #9351: Defaults set with set_defaults on an argparse subparser
   are no longer ignored when also set on the parent parser.
 
+- Issue #19746: Make it possible to examine the errors from unittest
+  discovery without executing the test suite. The new `errors` attribute
+  on TestLoader exposes these non-fatal errors encountered during discovery.
+
 - Issue #21991: Make email.headerregistry's header 'params' attributes
   be read-only (MappingProxyType).  Previously the dictionary was modifiable
   but a new one was created on each access of the attribute.