From 08d9e597c8ef5a2b26375ac954fdf224f5d82c3c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 30 Aug 2021 19:25:59 +0300 Subject: [PATCH] bpo-43913: Fix bugs in cleaning up classes and modules in unittest. (GH-28006) * Functions registered with addModuleCleanup() were not called unless the user defines tearDownModule() in their test module. * Functions registered with addClassCleanup() were not called if tearDownClass is set to None. * Buffering in TestResult did not work with functions registered with addClassCleanup() and addModuleCleanup(). * Errors in functions registered with addClassCleanup() and addModuleCleanup() were not handled correctly in buffered and debug modes. * Errors in setUpModule() and functions registered with addModuleCleanup() were reported in wrong order. * And several lesser bugs. --- Lib/unittest/suite.py | 140 +++--- Lib/unittest/test/test_result.py | 432 +++++++++++++++++- Lib/unittest/test/test_runner.py | 209 ++++++++- .../2021-08-27-23-40-51.bpo-43913.Uo1Gt5.rst | 8 + 4 files changed, 719 insertions(+), 70 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-08-27-23-40-51.bpo-43913.Uo1Gt5.rst diff --git a/Lib/unittest/suite.py b/Lib/unittest/suite.py index 41993f9cf69..6f45b6fe5f6 100644 --- a/Lib/unittest/suite.py +++ b/Lib/unittest/suite.py @@ -149,6 +149,7 @@ class TestSuite(BaseTestSuite): if getattr(currentClass, "__unittest_skip__", False): return + failed = False try: currentClass._classSetupFailed = False except TypeError: @@ -157,27 +158,32 @@ class TestSuite(BaseTestSuite): pass setUpClass = getattr(currentClass, 'setUpClass', None) + doClassCleanups = getattr(currentClass, 'doClassCleanups', None) if setUpClass is not None: _call_if_exists(result, '_setupStdout') try: - setUpClass() - except Exception as e: - if isinstance(result, _DebugResult): - raise - currentClass._classSetupFailed = True - className = util.strclass(currentClass) - self._createClassOrModuleLevelException(result, e, - 'setUpClass', - className) + try: + setUpClass() + except Exception as e: + if isinstance(result, _DebugResult): + raise + failed = True + try: + currentClass._classSetupFailed = True + except TypeError: + pass + className = util.strclass(currentClass) + self._createClassOrModuleLevelException(result, e, + 'setUpClass', + className) + if failed and doClassCleanups is not None: + doClassCleanups() + for exc_info in currentClass.tearDown_exceptions: + self._createClassOrModuleLevelException( + result, exc_info[1], 'setUpClass', className, + info=exc_info) finally: _call_if_exists(result, '_restoreStdout') - if currentClass._classSetupFailed is True: - currentClass.doClassCleanups() - if len(currentClass.tearDown_exceptions) > 0: - for exc in currentClass.tearDown_exceptions: - self._createClassOrModuleLevelException( - result, exc[1], 'setUpClass', className, - info=exc) def _get_previous_module(self, result): previousModule = None @@ -205,20 +211,22 @@ class TestSuite(BaseTestSuite): if setUpModule is not None: _call_if_exists(result, '_setupStdout') try: - setUpModule() - except Exception as e: try: - case.doModuleCleanups() - except Exception as exc: - self._createClassOrModuleLevelException(result, exc, + setUpModule() + except Exception as e: + if isinstance(result, _DebugResult): + raise + result._moduleSetUpFailed = True + self._createClassOrModuleLevelException(result, e, 'setUpModule', currentModule) - if isinstance(result, _DebugResult): - raise - result._moduleSetUpFailed = True - self._createClassOrModuleLevelException(result, e, - 'setUpModule', - currentModule) + if result._moduleSetUpFailed: + try: + case.doModuleCleanups() + except Exception as e: + self._createClassOrModuleLevelException(result, e, + 'setUpModule', + currentModule) finally: _call_if_exists(result, '_restoreStdout') @@ -251,30 +259,33 @@ class TestSuite(BaseTestSuite): except KeyError: return - tearDownModule = getattr(module, 'tearDownModule', None) - if tearDownModule is not None: - _call_if_exists(result, '_setupStdout') + _call_if_exists(result, '_setupStdout') + try: + tearDownModule = getattr(module, 'tearDownModule', None) + if tearDownModule is not None: + try: + tearDownModule() + except Exception as e: + if isinstance(result, _DebugResult): + raise + self._createClassOrModuleLevelException(result, e, + 'tearDownModule', + previousModule) try: - tearDownModule() + case.doModuleCleanups() except Exception as e: if isinstance(result, _DebugResult): raise self._createClassOrModuleLevelException(result, e, 'tearDownModule', previousModule) - finally: - _call_if_exists(result, '_restoreStdout') - try: - case.doModuleCleanups() - except Exception as e: - self._createClassOrModuleLevelException(result, e, - 'tearDownModule', - previousModule) + finally: + _call_if_exists(result, '_restoreStdout') def _tearDownPreviousClass(self, test, result): previousClass = getattr(result, '_previousTestClass', None) currentClass = test.__class__ - if currentClass == previousClass: + if currentClass == previousClass or previousClass is None: return if getattr(previousClass, '_classSetupFailed', False): return @@ -284,27 +295,34 @@ class TestSuite(BaseTestSuite): return tearDownClass = getattr(previousClass, 'tearDownClass', None) - if tearDownClass is not None: - _call_if_exists(result, '_setupStdout') - try: - tearDownClass() - except Exception as e: - if isinstance(result, _DebugResult): - raise - className = util.strclass(previousClass) - self._createClassOrModuleLevelException(result, e, - 'tearDownClass', - className) - finally: - _call_if_exists(result, '_restoreStdout') - previousClass.doClassCleanups() - if len(previousClass.tearDown_exceptions) > 0: - for exc in previousClass.tearDown_exceptions: - className = util.strclass(previousClass) - self._createClassOrModuleLevelException(result, exc[1], - 'tearDownClass', - className, - info=exc) + doClassCleanups = getattr(previousClass, 'doClassCleanups', None) + if tearDownClass is None and doClassCleanups is None: + return + + _call_if_exists(result, '_setupStdout') + try: + if tearDownClass is not None: + try: + tearDownClass() + except Exception as e: + if isinstance(result, _DebugResult): + raise + className = util.strclass(previousClass) + self._createClassOrModuleLevelException(result, e, + 'tearDownClass', + className) + if doClassCleanups is not None: + doClassCleanups() + for exc_info in previousClass.tearDown_exceptions: + if isinstance(result, _DebugResult): + raise exc_info[1] + className = util.strclass(previousClass) + self._createClassOrModuleLevelException(result, exc_info[1], + 'tearDownClass', + className, + info=exc_info) + finally: + _call_if_exists(result, '_restoreStdout') class _ErrorHolder(object): diff --git a/Lib/unittest/test/test_result.py b/Lib/unittest/test/test_result.py index a4af67bd8d5..d6efc7ef066 100644 --- a/Lib/unittest/test/test_result.py +++ b/Lib/unittest/test/test_result.py @@ -2,10 +2,11 @@ import io import sys import textwrap -from test.support import warnings_helper +from test.support import warnings_helper, captured_stdout, captured_stderr import traceback import unittest +from unittest.util import strclass class MockTraceback(object): @@ -22,6 +23,16 @@ def restore_traceback(): unittest.result.traceback = traceback +def bad_cleanup1(): + print('do cleanup1') + raise TypeError('bad cleanup1') + + +def bad_cleanup2(): + print('do cleanup2') + raise ValueError('bad cleanup2') + + class Test_TestResult(unittest.TestCase): # Note: there are not separate tests for TestResult.wasSuccessful(), # TestResult.errors, TestResult.failures, TestResult.testsRun or @@ -633,36 +644,320 @@ class TestOutputBuffering(unittest.TestCase): self.assertEqual(result._original_stderr.getvalue(), expectedErrMessage) self.assertMultiLineEqual(message, expectedFullMessage) + def testBufferSetUp(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + def setUp(self): + print('set up') + 1/0 + def test_foo(self): + pass + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\nset up\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 1) + description = f'test_foo ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(str(test_case), description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferTearDown(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + def tearDown(self): + print('tear down') + 1/0 + def test_foo(self): + pass + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\ntear down\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 1) + description = f'test_foo ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(str(test_case), description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferDoCleanups(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + def setUp(self): + print('set up') + self.addCleanup(bad_cleanup1) + self.addCleanup(bad_cleanup2) + def test_foo(self): + pass + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\nset up\ndo cleanup2\ndo cleanup1\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 2) + description = f'test_foo ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(str(test_case), description) + self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertEqual(str(test_case), description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferSetUp_DoCleanups(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + def setUp(self): + print('set up') + self.addCleanup(bad_cleanup1) + self.addCleanup(bad_cleanup2) + 1/0 + def test_foo(self): + pass + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\nset up\ndo cleanup2\ndo cleanup1\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 3) + description = f'test_foo ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(str(test_case), description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertEqual(str(test_case), description) + self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[2] + self.assertEqual(str(test_case), description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferTearDown_DoCleanups(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + def setUp(self): + print('set up') + self.addCleanup(bad_cleanup1) + self.addCleanup(bad_cleanup2) + def tearDown(self): + print('tear down') + 1/0 + def test_foo(self): + pass + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\nset up\ntear down\ndo cleanup2\ndo cleanup1\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 3) + description = f'test_foo ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(str(test_case), description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertEqual(str(test_case), description) + self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[2] + self.assertEqual(str(test_case), description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + def testBufferSetupClass(self): - result = unittest.TestResult() + with captured_stdout() as stdout: + result = unittest.TestResult() result.buffer = True class Foo(unittest.TestCase): @classmethod def setUpClass(cls): + print('set up class') 1/0 def test_foo(self): pass suite = unittest.TestSuite([Foo('test_foo')]) suite(result) + expected_out = '\nStdout:\nset up class\n' + self.assertEqual(stdout.getvalue(), expected_out) self.assertEqual(len(result.errors), 1) + description = f'setUpClass ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertIn(expected_out, formatted_exc) def testBufferTearDownClass(self): - result = unittest.TestResult() + with captured_stdout() as stdout: + result = unittest.TestResult() result.buffer = True class Foo(unittest.TestCase): @classmethod def tearDownClass(cls): + print('tear down class') 1/0 def test_foo(self): pass suite = unittest.TestSuite([Foo('test_foo')]) suite(result) + expected_out = '\nStdout:\ntear down class\n' + self.assertEqual(stdout.getvalue(), expected_out) self.assertEqual(len(result.errors), 1) + description = f'tearDownClass ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferDoClassCleanups(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + @classmethod + def setUpClass(cls): + print('set up class') + cls.addClassCleanup(bad_cleanup1) + cls.addClassCleanup(bad_cleanup2) + @classmethod + def tearDownClass(cls): + print('tear down class') + def test_foo(self): + pass + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\ntear down class\ndo cleanup2\ndo cleanup1\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 2) + description = f'tearDownClass ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertEqual(test_case.description, description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferSetupClass_DoClassCleanups(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + @classmethod + def setUpClass(cls): + print('set up class') + cls.addClassCleanup(bad_cleanup1) + cls.addClassCleanup(bad_cleanup2) + 1/0 + def test_foo(self): + pass + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\nset up class\ndo cleanup2\ndo cleanup1\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 3) + description = f'setUpClass ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn('\nStdout:\nset up class\n', formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertEqual(test_case.description, description) + self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[2] + self.assertEqual(test_case.description, description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferTearDownClass_DoClassCleanups(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + @classmethod + def setUpClass(cls): + print('set up class') + cls.addClassCleanup(bad_cleanup1) + cls.addClassCleanup(bad_cleanup2) + @classmethod + def tearDownClass(cls): + print('tear down class') + 1/0 + def test_foo(self): + pass + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\ntear down class\ndo cleanup2\ndo cleanup1\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 3) + description = f'tearDownClass ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn('\nStdout:\ntear down class\n', formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertEqual(test_case.description, description) + self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[2] + self.assertEqual(test_case.description, description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) def testBufferSetUpModule(self): - result = unittest.TestResult() + with captured_stdout() as stdout: + result = unittest.TestResult() result.buffer = True class Foo(unittest.TestCase): @@ -671,6 +966,7 @@ class TestOutputBuffering(unittest.TestCase): class Module(object): @staticmethod def setUpModule(): + print('set up module') 1/0 Foo.__module__ = 'Module' @@ -678,10 +974,18 @@ class TestOutputBuffering(unittest.TestCase): self.addCleanup(sys.modules.pop, 'Module') suite = unittest.TestSuite([Foo('test_foo')]) suite(result) + expected_out = '\nStdout:\nset up module\n' + self.assertEqual(stdout.getvalue(), expected_out) self.assertEqual(len(result.errors), 1) + description = 'setUpModule (Module)' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertIn(expected_out, formatted_exc) def testBufferTearDownModule(self): - result = unittest.TestResult() + with captured_stdout() as stdout: + result = unittest.TestResult() result.buffer = True class Foo(unittest.TestCase): @@ -690,6 +994,7 @@ class TestOutputBuffering(unittest.TestCase): class Module(object): @staticmethod def tearDownModule(): + print('tear down module') 1/0 Foo.__module__ = 'Module' @@ -697,7 +1002,124 @@ class TestOutputBuffering(unittest.TestCase): self.addCleanup(sys.modules.pop, 'Module') suite = unittest.TestSuite([Foo('test_foo')]) suite(result) + expected_out = '\nStdout:\ntear down module\n' + self.assertEqual(stdout.getvalue(), expected_out) self.assertEqual(len(result.errors), 1) + description = 'tearDownModule (Module)' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferDoModuleCleanups(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + def test_foo(self): + pass + class Module(object): + @staticmethod + def setUpModule(): + print('set up module') + unittest.addModuleCleanup(bad_cleanup1) + unittest.addModuleCleanup(bad_cleanup2) + + Foo.__module__ = 'Module' + sys.modules['Module'] = Module + self.addCleanup(sys.modules.pop, 'Module') + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\ndo cleanup2\ndo cleanup1\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 1) + description = 'tearDownModule (Module)' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferSetUpModule_DoModuleCleanups(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + def test_foo(self): + pass + class Module(object): + @staticmethod + def setUpModule(): + print('set up module') + unittest.addModuleCleanup(bad_cleanup1) + unittest.addModuleCleanup(bad_cleanup2) + 1/0 + + Foo.__module__ = 'Module' + sys.modules['Module'] = Module + self.addCleanup(sys.modules.pop, 'Module') + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\nset up module\ndo cleanup2\ndo cleanup1\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 2) + description = 'setUpModule (Module)' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn('\nStdout:\nset up module\n', formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertIn(expected_out, formatted_exc) + self.assertEqual(test_case.description, description) + self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferTearDownModule_DoModuleCleanups(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + def test_foo(self): + pass + class Module(object): + @staticmethod + def setUpModule(): + print('set up module') + unittest.addModuleCleanup(bad_cleanup1) + unittest.addModuleCleanup(bad_cleanup2) + @staticmethod + def tearDownModule(): + print('tear down module') + 1/0 + + Foo.__module__ = 'Module' + sys.modules['Module'] = Module + self.addCleanup(sys.modules.pop, 'Module') + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\ntear down module\ndo cleanup2\ndo cleanup1\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 2) + description = 'tearDownModule (Module)' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn('\nStdout:\ntear down module\n', formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertEqual(test_case.description, description) + self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) if __name__ == '__main__': diff --git a/Lib/unittest/test/test_runner.py b/Lib/unittest/test/test_runner.py index dd9a1b6d9ae..453e6c3d11c 100644 --- a/Lib/unittest/test/test_runner.py +++ b/Lib/unittest/test/test_runner.py @@ -222,14 +222,42 @@ class TestClassCleanup(unittest.TestCase): self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass', 'cleanup_good']) - def test_debug_executes_classCleanUp(self): + def test_run_class_cleanUp_without_tearDownClass(self): ordering = [] + blowUp = True class TestableTest(unittest.TestCase): @classmethod def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering) + if blowUp: + raise Exception() + def testNothing(self): + ordering.append('test') + @classmethod + @property + def tearDownClass(cls): + raise AttributeError + + runTests(TestableTest) + self.assertEqual(ordering, ['setUpClass', 'cleanup_good']) + + ordering = [] + blowUp = False + runTests(TestableTest) + self.assertEqual(ordering, + ['setUpClass', 'test', 'cleanup_good']) + + def test_debug_executes_classCleanUp(self): + ordering = [] + blowUp = False + + class TestableTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + ordering.append('setUpClass') + cls.addClassCleanup(cleanup, ordering, blowUp=blowUp) def testNothing(self): ordering.append('test') @classmethod @@ -241,6 +269,48 @@ class TestClassCleanup(unittest.TestCase): self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass', 'cleanup_good']) + ordering = [] + blowUp = True + suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) + with self.assertRaises(Exception) as cm: + suite.debug() + self.assertEqual(str(cm.exception), 'CleanUpExc') + self.assertEqual(ordering, + ['setUpClass', 'test', 'tearDownClass', 'cleanup_exc']) + + def test_debug_executes_classCleanUp_when_teardown_exception(self): + ordering = [] + blowUp = False + + class TestableTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + ordering.append('setUpClass') + cls.addClassCleanup(cleanup, ordering, blowUp=blowUp) + def testNothing(self): + ordering.append('test') + @classmethod + def tearDownClass(cls): + raise Exception('TearDownClassExc') + + suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) + with self.assertRaises(Exception) as cm: + suite.debug() + self.assertEqual(str(cm.exception), 'TearDownClassExc') + self.assertEqual(ordering, ['setUpClass', 'test']) + self.assertTrue(TestableTest._class_cleanups) + TestableTest._class_cleanups.clear() + + ordering = [] + blowUp = True + suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) + with self.assertRaises(Exception) as cm: + suite.debug() + self.assertEqual(str(cm.exception), 'TearDownClassExc') + self.assertEqual(ordering, ['setUpClass', 'test']) + self.assertTrue(TestableTest._class_cleanups) + TestableTest._class_cleanups.clear() + def test_doClassCleanups_with_errors_addClassCleanUp(self): class TestableTest(unittest.TestCase): def testNothing(self): @@ -332,6 +402,7 @@ class TestClassCleanup(unittest.TestCase): self.assertEqual(ordering, ['setUpClass', 'setUp', 'test', 'tearDownClass', 'cleanup_exc']) + ordering = [] class_blow_up = True method_blow_up = False @@ -355,6 +426,26 @@ class TestClassCleanup(unittest.TestCase): ['setUpClass', 'setUp', 'tearDownClass', 'cleanup_exc']) + def test_with_errors_in_tearDownClass(self): + ordering = [] + class TestableTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + ordering.append('setUpClass') + cls.addClassCleanup(cleanup, ordering) + def testNothing(self): + ordering.append('test') + @classmethod + def tearDownClass(cls): + ordering.append('tearDownClass') + raise Exception('TearDownExc') + + result = runTests(TestableTest) + self.assertEqual(result.errors[0][1].splitlines()[-1], + 'Exception: TearDownExc') + self.assertEqual(ordering, + ['setUpClass', 'test', 'tearDownClass', 'cleanup_good']) + class TestModuleCleanUp(unittest.TestCase): def test_add_and_do_ModuleCleanup(self): @@ -532,13 +623,69 @@ class TestModuleCleanUp(unittest.TestCase): 'tearDownModule2', 'cleanup_good']) self.assertEqual(unittest.case._module_cleanups, []) - def test_debug_module_executes_cleanUp(self): + def test_run_module_cleanUp_without_teardown(self): ordering = [] class Module(object): @staticmethod def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering) + + class TestableTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + ordering.append('setUpClass') + def testNothing(self): + ordering.append('test') + @classmethod + def tearDownClass(cls): + ordering.append('tearDownClass') + + TestableTest.__module__ = 'Module' + sys.modules['Module'] = Module + runTests(TestableTest) + self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', + 'tearDownClass', 'cleanup_good']) + self.assertEqual(unittest.case._module_cleanups, []) + + def test_run_module_cleanUp_when_teardown_exception(self): + ordering = [] + class Module(object): + @staticmethod + def setUpModule(): + ordering.append('setUpModule') + unittest.addModuleCleanup(cleanup, ordering) + @staticmethod + def tearDownModule(): + raise Exception('CleanUpExc') + + class TestableTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + ordering.append('setUpClass') + def testNothing(self): + ordering.append('test') + @classmethod + def tearDownClass(cls): + ordering.append('tearDownClass') + + TestableTest.__module__ = 'Module' + sys.modules['Module'] = Module + result = runTests(TestableTest) + self.assertEqual(result.errors[0][1].splitlines()[-1], + 'Exception: CleanUpExc') + self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', + 'tearDownClass', 'cleanup_good']) + self.assertEqual(unittest.case._module_cleanups, []) + + def test_debug_module_executes_cleanUp(self): + ordering = [] + blowUp = False + class Module(object): + @staticmethod + def setUpModule(): + ordering.append('setUpModule') + unittest.addModuleCleanup(cleanup, ordering, blowUp=blowUp) @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -562,6 +709,60 @@ class TestModuleCleanUp(unittest.TestCase): 'tearDownModule', 'cleanup_good']) self.assertEqual(unittest.case._module_cleanups, []) + ordering = [] + blowUp = True + suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) + with self.assertRaises(Exception) as cm: + suite.debug() + self.assertEqual(str(cm.exception), 'CleanUpExc') + self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', + 'tearDownClass', 'tearDownModule', 'cleanup_exc']) + self.assertEqual(unittest.case._module_cleanups, []) + + def test_debug_module_cleanUp_when_teardown_exception(self): + ordering = [] + blowUp = False + class Module(object): + @staticmethod + def setUpModule(): + ordering.append('setUpModule') + unittest.addModuleCleanup(cleanup, ordering, blowUp=blowUp) + @staticmethod + def tearDownModule(): + raise Exception('TearDownModuleExc') + + class TestableTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + ordering.append('setUpClass') + def testNothing(self): + ordering.append('test') + @classmethod + def tearDownClass(cls): + ordering.append('tearDownClass') + + TestableTest.__module__ = 'Module' + sys.modules['Module'] = Module + suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) + with self.assertRaises(Exception) as cm: + suite.debug() + self.assertEqual(str(cm.exception), 'TearDownModuleExc') + self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', + 'tearDownClass']) + self.assertTrue(unittest.case._module_cleanups) + unittest.case._module_cleanups.clear() + + ordering = [] + blowUp = True + suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) + with self.assertRaises(Exception) as cm: + suite.debug() + self.assertEqual(str(cm.exception), 'TearDownModuleExc') + self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', + 'tearDownClass']) + self.assertTrue(unittest.case._module_cleanups) + unittest.case._module_cleanups.clear() + def test_addClassCleanup_arg_errors(self): cleanups = [] def cleanup(*args, **kwargs): @@ -717,9 +918,9 @@ class TestModuleCleanUp(unittest.TestCase): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') - self.assertEqual(result.errors[1][1].splitlines()[-1], 'Exception: ModuleExc') + self.assertEqual(result.errors[1][1].splitlines()[-1], + 'Exception: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'cleanup_exc']) ordering = [] diff --git a/Misc/NEWS.d/next/Library/2021-08-27-23-40-51.bpo-43913.Uo1Gt5.rst b/Misc/NEWS.d/next/Library/2021-08-27-23-40-51.bpo-43913.Uo1Gt5.rst new file mode 100644 index 00000000000..cf3d5ee0e45 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-27-23-40-51.bpo-43913.Uo1Gt5.rst @@ -0,0 +1,8 @@ +Fix bugs in cleaning up classes and modules in :mod:`unittest`: + +* Functions registered with :func:`~unittest.addModuleCleanup` were not called unless the user defines ``tearDownModule()`` in their test module. +* Functions registered with :meth:`~unittest.TestCase.addClassCleanup` were not called if ``tearDownClass`` is set to ``None``. +* Buffering in :class:`~unittest.TestResult` did not work with functions registered with ``addClassCleanup()`` and ``addModuleCleanup()``. +* Errors in functions registered with ``addClassCleanup()`` and ``addModuleCleanup()`` were not handled correctly in buffered and debug modes. +* Errors in ``setUpModule()`` and functions registered with ``addModuleCleanup()`` were reported in wrong order. +* And several lesser bugs.