Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Close #7559: ImportError when loading a test now shown as ImportError. 

Previously the ImportError was only shown if the top level containing
package failed to import, with other ImportErrors showing up as
AttributeError - hiding the real cause. As part of this,
`TestLoader.loadTestsFromNames` now captures errors to self.errors.
This commit is contained in:
Robert Collins 2014-10-30 08:27:27 +13:00
parent c134584020
commit 659dd625b4
4 changed files with 191 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Doc/library/unittest.rst
View File

@ -1629,6 +1629,12 @@ Loading and running tests
The method optionally resolves *name* relative to the given *module*. The method optionally resolves *name* relative to the given *module*.
.. versionchanged:: 3.5
If an :exc:`ImportError` or :exc:`AttributeError` occurs while traversing
*name* then a synthetic test that raises that error when run will be
returned. These errors are included in the errors accumulated by
self.errors.
.. method:: loadTestsFromNames(names, module=None) .. method:: loadTestsFromNames(names, module=None)

35
Lib/unittest/loader.py
View File

@ -130,20 +130,47 @@ class TestLoader(object):
The method optionally resolves the names relative to a given module. The method optionally resolves the names relative to a given module.
""" """
parts = name.split('.') parts = name.split('.')
error_case, error_message = None, None
if module is None: if module is None:
parts_copy = parts[:] parts_copy = parts[:]
while parts_copy: while parts_copy:
try: try:
module = __import__('.'.join(parts_copy)) module_name = '.'.join(parts_copy)
module = __import__(module_name)
break break
except ImportError: except ImportError:
del parts_copy[-1] next_attribute = parts_copy.pop()
# Last error so we can give it to the user if needed.
error_case, error_message = _make_failed_import_test(
next_attribute, self.suiteClass)
if not parts_copy: if not parts_copy:
raise # Even the top level import failed: report that error.
self.errors.append(error_message)
return error_case
parts = parts[1:] parts = parts[1:]
obj = module obj = module
for part in parts: for part in parts:
parent, obj = obj, getattr(obj, part) try:
parent, obj = obj, getattr(obj, part)
except AttributeError as e:
# We can't traverse some part of the name.
if (getattr(obj, '__path__', None) is not None
and error_case is not None):
# This is a package (no __path__ per importlib docs), and we
# encountered an error importing something. We cannot tell
# the difference between package.WrongNameTestClass and
# package.wrong_module_name so we just report the
# ImportError - it is more informative.
self.errors.append(error_message)
return error_case
else:
# Otherwise, we signal that an AttributeError has occurred.
error_case, error_message = _make_failed_test(
'AttributeError', part, e, self.suiteClass,
'Failed to access attribute:\n%s' % (
traceback.format_exc(),))
self.errors.append(error_message)
return error_case
if isinstance(obj, types.ModuleType): if isinstance(obj, types.ModuleType):
return self.loadTestsFromModule(obj) return self.loadTestsFromModule(obj)

251
Lib/unittest/test/test_loader.py
View File

@ -385,15 +385,15 @@ class Test_TestLoader(unittest.TestCase):
def test_loadTestsFromName__malformed_name(self): def test_loadTestsFromName__malformed_name(self):
loader = unittest.TestLoader() loader = unittest.TestLoader()
# XXX Should this raise ValueError or ImportError? suite = loader.loadTestsFromName('abc () //')
try: error, test = self.check_deferred_error(loader, suite)
loader.loadTestsFromName('abc () //') expected = "Failed to import test module: abc () //"
except ValueError: expected_regex = "Failed to import test module: abc \(\) //"
pass self.assertIn(
except ImportError: expected, error,
pass 'missing error string in %r' % error)
else: self.assertRaisesRegex(
self.fail("TestLoader.loadTestsFromName failed to raise ValueError") ImportError, expected_regex, getattr(test, 'abc () //'))
# "The specifier name is a ``dotted name'' that may resolve ... to a # "The specifier name is a ``dotted name'' that may resolve ... to a
# module" # module"
@ -402,28 +402,47 @@ class Test_TestLoader(unittest.TestCase):
def test_loadTestsFromName__unknown_module_name(self): def test_loadTestsFromName__unknown_module_name(self):
loader = unittest.TestLoader() loader = unittest.TestLoader()
try: suite = loader.loadTestsFromName('sdasfasfasdf')
loader.loadTestsFromName('sdasfasfasdf') expected = "No module named 'sdasfasfasdf'"
except ImportError as e: error, test = self.check_deferred_error(loader, suite)
self.assertEqual(str(e), "No module named 'sdasfasfasdf'") self.assertIn(
else: expected, error,
self.fail("TestLoader.loadTestsFromName failed to raise ImportError") 'missing error string in %r' % error)
self.assertRaisesRegex(ImportError, expected, test.sdasfasfasdf)
# "The specifier name is a ``dotted name'' that may resolve either to # "The specifier name is a ``dotted name'' that may resolve either to
# a module, a test case class, a TestSuite instance, a test method # a module, a test case class, a TestSuite instance, a test method
# within a test case class, or a callable object which returns a # within a test case class, or a callable object which returns a
# TestCase or TestSuite instance." # TestCase or TestSuite instance."
# #
# What happens when the module is found, but the attribute can't? # What happens when the module is found, but the attribute isn't?
def test_loadTestsFromName__unknown_attr_name(self): def test_loadTestsFromName__unknown_attr_name_on_module(self):
loader = unittest.TestLoader() loader = unittest.TestLoader()
try: suite = loader.loadTestsFromName('unittest.loader.sdasfasfasdf')
loader.loadTestsFromName('unittest.sdasfasfasdf') expected = "module 'unittest.loader' has no attribute 'sdasfasfasdf'"
except AttributeError as e: error, test = self.check_deferred_error(loader, suite)
self.assertEqual(str(e), "module 'unittest' has no attribute 'sdasfasfasdf'") self.assertIn(
else: expected, error,
self.fail("TestLoader.loadTestsFromName failed to raise AttributeError") 'missing error string in %r' % error)
self.assertRaisesRegex(AttributeError, expected, test.sdasfasfasdf)
# "The specifier name is a ``dotted name'' that may resolve either to
# a module, a test case class, a TestSuite instance, a test method
# within a test case class, or a callable object which returns a
# TestCase or TestSuite instance."
#
# What happens when the module is found, but the attribute isn't?
def test_loadTestsFromName__unknown_attr_name_on_package(self):
loader = unittest.TestLoader()
suite = loader.loadTestsFromName('unittest.sdasfasfasdf')
expected = "No module named 'unittest.sdasfasfasdf'"
error, test = self.check_deferred_error(loader, suite)
self.assertIn(
expected, error,
'missing error string in %r' % error)
self.assertRaisesRegex(ImportError, expected, test.sdasfasfasdf)
# "The specifier name is a ``dotted name'' that may resolve either to # "The specifier name is a ``dotted name'' that may resolve either to
# a module, a test case class, a TestSuite instance, a test method # a module, a test case class, a TestSuite instance, a test method
@ -435,12 +454,13 @@ class Test_TestLoader(unittest.TestCase):
def test_loadTestsFromName__relative_unknown_name(self): def test_loadTestsFromName__relative_unknown_name(self):
loader = unittest.TestLoader() loader = unittest.TestLoader()
try: suite = loader.loadTestsFromName('sdasfasfasdf', unittest)
loader.loadTestsFromName('sdasfasfasdf', unittest) expected = "module 'unittest' has no attribute 'sdasfasfasdf'"
except AttributeError as e: error, test = self.check_deferred_error(loader, suite)
self.assertEqual(str(e), "module 'unittest' has no attribute 'sdasfasfasdf'") self.assertIn(
else: expected, error,
self.fail("TestLoader.loadTestsFromName failed to raise AttributeError") 'missing error string in %r' % error)
self.assertRaisesRegex(AttributeError, expected, test.sdasfasfasdf)
# "The specifier name is a ``dotted name'' that may resolve either to # "The specifier name is a ``dotted name'' that may resolve either to
# a module, a test case class, a TestSuite instance, a test method # a module, a test case class, a TestSuite instance, a test method
@ -456,12 +476,13 @@ class Test_TestLoader(unittest.TestCase):
def test_loadTestsFromName__relative_empty_name(self): def test_loadTestsFromName__relative_empty_name(self):
loader = unittest.TestLoader() loader = unittest.TestLoader()
try: suite = loader.loadTestsFromName('', unittest)
loader.loadTestsFromName('', unittest) error, test = self.check_deferred_error(loader, suite)
except AttributeError as e: expected = "has no attribute ''"
pass self.assertIn(
else: expected, error,
self.fail("Failed to raise AttributeError") 'missing error string in %r' % error)
self.assertRaisesRegex(AttributeError, expected, getattr(test, ''))
# "The specifier name is a ``dotted name'' that may resolve either to # "The specifier name is a ``dotted name'' that may resolve either to
# a module, a test case class, a TestSuite instance, a test method # a module, a test case class, a TestSuite instance, a test method
@ -476,14 +497,15 @@ class Test_TestLoader(unittest.TestCase):
loader = unittest.TestLoader() loader = unittest.TestLoader()
# XXX Should this raise AttributeError or ValueError? # XXX Should this raise AttributeError or ValueError?
try: suite = loader.loadTestsFromName('abc () //', unittest)
loader.loadTestsFromName('abc () //', unittest) error, test = self.check_deferred_error(loader, suite)
except ValueError: expected = "module 'unittest' has no attribute 'abc () //'"
pass expected_regex = "module 'unittest' has no attribute 'abc \(\) //'"
except AttributeError: self.assertIn(
pass expected, error,
else: 'missing error string in %r' % error)
self.fail("TestLoader.loadTestsFromName failed to raise ValueError") self.assertRaisesRegex(
AttributeError, expected_regex, getattr(test, 'abc () //'))
# "The method optionally resolves name relative to the given module" # "The method optionally resolves name relative to the given module"
# #
@ -589,12 +611,13 @@ class Test_TestLoader(unittest.TestCase):
m.testcase_1 = MyTestCase m.testcase_1 = MyTestCase
loader = unittest.TestLoader() loader = unittest.TestLoader()
try: suite = loader.loadTestsFromName('testcase_1.testfoo', m)
loader.loadTestsFromName('testcase_1.testfoo', m) expected = "type object 'MyTestCase' has no attribute 'testfoo'"
except AttributeError as e: error, test = self.check_deferred_error(loader, suite)
self.assertEqual(str(e), "type object 'MyTestCase' has no attribute 'testfoo'") self.assertIn(
else: expected, error,
self.fail("Failed to raise AttributeError") 'missing error string in %r' % error)
self.assertRaisesRegex(AttributeError, expected, test.testfoo)
# "The specifier name is a ``dotted name'' that may resolve ... to # "The specifier name is a ``dotted name'' that may resolve ... to
# ... a callable object which returns a ... TestSuite instance" # ... a callable object which returns a ... TestSuite instance"
@ -712,6 +735,23 @@ class Test_TestLoader(unittest.TestCase):
### Tests for TestLoader.loadTestsFromNames() ### Tests for TestLoader.loadTestsFromNames()
################################################################ ################################################################
def check_deferred_error(self, loader, suite):
"""Helper function for checking that errors in loading are reported.
:param loader: A loader with some errors.
:param suite: A suite that should have a late bound error.
:return: The first error message from the loader and the test object
from the suite.
"""
self.assertIsInstance(suite, unittest.TestSuite)
self.assertEqual(suite.countTestCases(), 1)
# Errors loading the suite are also captured for introspection.
self.assertNotEqual([], loader.errors)
self.assertEqual(1, len(loader.errors))
error = loader.errors[0]
test = list(suite)[0]
return error, test
# "Similar to loadTestsFromName(), but takes a sequence of names rather # "Similar to loadTestsFromName(), but takes a sequence of names rather
# than a single name." # than a single name."
# #
@ -764,14 +804,15 @@ class Test_TestLoader(unittest.TestCase):
loader = unittest.TestLoader() loader = unittest.TestLoader()
# XXX Should this raise ValueError or ImportError? # XXX Should this raise ValueError or ImportError?
try: suite = loader.loadTestsFromNames(['abc () //'])
loader.loadTestsFromNames(['abc () //']) error, test = self.check_deferred_error(loader, list(suite)[0])
except ValueError: expected = "Failed to import test module: abc () //"
pass expected_regex = "Failed to import test module: abc \(\) //"
except ImportError: self.assertIn(
pass expected, error,
else: 'missing error string in %r' % error)
self.fail("TestLoader.loadTestsFromNames failed to raise ValueError") self.assertRaisesRegex(
ImportError, expected_regex, getattr(test, 'abc () //'))
# "The specifier name is a ``dotted name'' that may resolve either to # "The specifier name is a ``dotted name'' that may resolve either to
# a module, a test case class, a TestSuite instance, a test method # a module, a test case class, a TestSuite instance, a test method
@ -782,12 +823,13 @@ class Test_TestLoader(unittest.TestCase):
def test_loadTestsFromNames__unknown_module_name(self): def test_loadTestsFromNames__unknown_module_name(self):
loader = unittest.TestLoader() loader = unittest.TestLoader()
try: suite = loader.loadTestsFromNames(['sdasfasfasdf'])
loader.loadTestsFromNames(['sdasfasfasdf']) error, test = self.check_deferred_error(loader, list(suite)[0])
except ImportError as e: expected = "Failed to import test module: sdasfasfasdf"
self.assertEqual(str(e), "No module named 'sdasfasfasdf'") self.assertIn(
else: expected, error,
self.fail("TestLoader.loadTestsFromNames failed to raise ImportError") 'missing error string in %r' % error)
self.assertRaisesRegex(ImportError, expected, test.sdasfasfasdf)
# "The specifier name is a ``dotted name'' that may resolve either to # "The specifier name is a ``dotted name'' that may resolve either to
# a module, a test case class, a TestSuite instance, a test method # a module, a test case class, a TestSuite instance, a test method
@ -798,12 +840,14 @@ class Test_TestLoader(unittest.TestCase):
def test_loadTestsFromNames__unknown_attr_name(self): def test_loadTestsFromNames__unknown_attr_name(self):
loader = unittest.TestLoader() loader = unittest.TestLoader()
try: suite = loader.loadTestsFromNames(
loader.loadTestsFromNames(['unittest.sdasfasfasdf', 'unittest']) ['unittest.loader.sdasfasfasdf', 'unittest'])
except AttributeError as e: error, test = self.check_deferred_error(loader, list(suite)[0])
self.assertEqual(str(e), "module 'unittest' has no attribute 'sdasfasfasdf'") expected = "module 'unittest.loader' has no attribute 'sdasfasfasdf'"
else: self.assertIn(
self.fail("TestLoader.loadTestsFromNames failed to raise AttributeError") expected, error,
'missing error string in %r' % error)
self.assertRaisesRegex(AttributeError, expected, test.sdasfasfasdf)
# "The specifier name is a ``dotted name'' that may resolve either to # "The specifier name is a ``dotted name'' that may resolve either to
# a module, a test case class, a TestSuite instance, a test method # a module, a test case class, a TestSuite instance, a test method
@ -817,12 +861,13 @@ class Test_TestLoader(unittest.TestCase):
def test_loadTestsFromNames__unknown_name_relative_1(self): def test_loadTestsFromNames__unknown_name_relative_1(self):
loader = unittest.TestLoader() loader = unittest.TestLoader()
try: suite = loader.loadTestsFromNames(['sdasfasfasdf'], unittest)
loader.loadTestsFromNames(['sdasfasfasdf'], unittest) error, test = self.check_deferred_error(loader, list(suite)[0])
except AttributeError as e: expected = "module 'unittest' has no attribute 'sdasfasfasdf'"
self.assertEqual(str(e), "module 'unittest' has no attribute 'sdasfasfasdf'") self.assertIn(
else: expected, error,
self.fail("TestLoader.loadTestsFromName failed to raise AttributeError") 'missing error string in %r' % error)
self.assertRaisesRegex(AttributeError, expected, test.sdasfasfasdf)
# "The specifier name is a ``dotted name'' that may resolve either to # "The specifier name is a ``dotted name'' that may resolve either to
# a module, a test case class, a TestSuite instance, a test method # a module, a test case class, a TestSuite instance, a test method
@ -836,12 +881,13 @@ class Test_TestLoader(unittest.TestCase):
def test_loadTestsFromNames__unknown_name_relative_2(self): def test_loadTestsFromNames__unknown_name_relative_2(self):
loader = unittest.TestLoader() loader = unittest.TestLoader()
try: suite = loader.loadTestsFromNames(['TestCase', 'sdasfasfasdf'], unittest)
loader.loadTestsFromNames(['TestCase', 'sdasfasfasdf'], unittest) error, test = self.check_deferred_error(loader, list(suite)[1])
except AttributeError as e: expected = "module 'unittest' has no attribute 'sdasfasfasdf'"
self.assertEqual(str(e), "module 'unittest' has no attribute 'sdasfasfasdf'") self.assertIn(
else: expected, error,
self.fail("TestLoader.loadTestsFromName failed to raise AttributeError") 'missing error string in %r' % error)
self.assertRaisesRegex(AttributeError, expected, test.sdasfasfasdf)
# "The specifier name is a ``dotted name'' that may resolve either to # "The specifier name is a ``dotted name'' that may resolve either to
# a module, a test case class, a TestSuite instance, a test method # a module, a test case class, a TestSuite instance, a test method
@ -857,12 +903,13 @@ class Test_TestLoader(unittest.TestCase):
def test_loadTestsFromNames__relative_empty_name(self): def test_loadTestsFromNames__relative_empty_name(self):
loader = unittest.TestLoader() loader = unittest.TestLoader()
try: suite = loader.loadTestsFromNames([''], unittest)
loader.loadTestsFromNames([''], unittest) error, test = self.check_deferred_error(loader, list(suite)[0])
except AttributeError: expected = "has no attribute ''"
pass self.assertIn(
else: expected, error,
self.fail("Failed to raise ValueError") 'missing error string in %r' % error)
self.assertRaisesRegex(AttributeError, expected, getattr(test, ''))
# "The specifier name is a ``dotted name'' that may resolve either to # "The specifier name is a ``dotted name'' that may resolve either to
# a module, a test case class, a TestSuite instance, a test method # a module, a test case class, a TestSuite instance, a test method
@ -876,14 +923,15 @@ class Test_TestLoader(unittest.TestCase):
loader = unittest.TestLoader() loader = unittest.TestLoader()
# XXX Should this raise AttributeError or ValueError? # XXX Should this raise AttributeError or ValueError?
try: suite = loader.loadTestsFromNames(['abc () //'], unittest)
loader.loadTestsFromNames(['abc () //'], unittest) error, test = self.check_deferred_error(loader, list(suite)[0])
except AttributeError: expected = "module 'unittest' has no attribute 'abc () //'"
pass expected_regex = "module 'unittest' has no attribute 'abc \(\) //'"
except ValueError: self.assertIn(
pass expected, error,
else: 'missing error string in %r' % error)
self.fail("TestLoader.loadTestsFromNames failed to raise ValueError") self.assertRaisesRegex(
AttributeError, expected_regex, getattr(test, 'abc () //'))
# "The method optionally resolves name relative to the given module" # "The method optionally resolves name relative to the given module"
# #
@ -1001,12 +1049,13 @@ class Test_TestLoader(unittest.TestCase):
m.testcase_1 = MyTestCase m.testcase_1 = MyTestCase
loader = unittest.TestLoader() loader = unittest.TestLoader()
try: suite = loader.loadTestsFromNames(['testcase_1.testfoo'], m)
loader.loadTestsFromNames(['testcase_1.testfoo'], m) error, test = self.check_deferred_error(loader, list(suite)[0])
except AttributeError as e: expected = "type object 'MyTestCase' has no attribute 'testfoo'"
self.assertEqual(str(e), "type object 'MyTestCase' has no attribute 'testfoo'") self.assertIn(
else: expected, error,
self.fail("Failed to raise AttributeError") 'missing error string in %r' % error)
self.assertRaisesRegex(AttributeError, expected, test.testfoo)
# "The specifier name is a ``dotted name'' that may resolve ... to # "The specifier name is a ``dotted name'' that may resolve ... to
# ... a callable object which returns a ... TestSuite instance" # ... a callable object which returns a ... TestSuite instance"

4
Misc/NEWS
View File

@ -196,6 +196,10 @@ Library
- Issue #9351: Defaults set with set_defaults on an argparse subparser - Issue #9351: Defaults set with set_defaults on an argparse subparser
are no longer ignored when also set on the parent parser. are no longer ignored when also set on the parent parser.
- Issue #7559: unittest test loading ImportErrors are reported as import errors
with their import exception rather than as attribute errors after the import
has already failed.
- Issue #19746: Make it possible to examine the errors from unittest - Issue #19746: Make it possible to examine the errors from unittest
discovery without executing the test suite. The new `errors` attribute discovery without executing the test suite. The new `errors` attribute
on TestLoader exposes these non-fatal errors encountered during discovery. on TestLoader exposes these non-fatal errors encountered during discovery.