bpo-43913: Fix bugs in cleaning up classes and modules in unittest. (GH-28006)

* Functions registered with addModuleCleanup() were not called unless
  the user defines tearDownModule() in their test module.
* Functions registered with addClassCleanup() were not called if
  tearDownClass is set to None.
* Buffering in TestResult did not work with functions registered
  with addClassCleanup() and addModuleCleanup().
* Errors in functions registered with addClassCleanup() and
  addModuleCleanup() were not handled correctly in buffered and
  debug modes.
* Errors in setUpModule() and functions registered with
  addModuleCleanup() were reported in wrong order.
* And several lesser bugs.
This commit is contained in:
Serhiy Storchaka 2021-08-30 19:25:59 +03:00 committed by GitHub
parent 7e246a3a7b
commit 08d9e597c8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 719 additions and 70 deletions

View file

@ -2,10 +2,11 @@ import io
import sys
import textwrap
from test.support import warnings_helper
from test.support import warnings_helper, captured_stdout, captured_stderr
import traceback
import unittest
from unittest.util import strclass
class MockTraceback(object):
@ -22,6 +23,16 @@ def restore_traceback():
unittest.result.traceback = traceback
def bad_cleanup1():
print('do cleanup1')
raise TypeError('bad cleanup1')
def bad_cleanup2():
print('do cleanup2')
raise ValueError('bad cleanup2')
class Test_TestResult(unittest.TestCase):
# Note: there are not separate tests for TestResult.wasSuccessful(),
# TestResult.errors, TestResult.failures, TestResult.testsRun or
@ -633,36 +644,320 @@ class TestOutputBuffering(unittest.TestCase):
self.assertEqual(result._original_stderr.getvalue(), expectedErrMessage)
self.assertMultiLineEqual(message, expectedFullMessage)
def testBufferSetUp(self):
with captured_stdout() as stdout:
result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
def setUp(self):
print('set up')
1/0
def test_foo(self):
pass
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
expected_out = '\nStdout:\nset up\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 1)
description = f'test_foo ({strclass(Foo)})'
test_case, formatted_exc = result.errors[0]
self.assertEqual(str(test_case), description)
self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
self.assertIn(expected_out, formatted_exc)
def testBufferTearDown(self):
with captured_stdout() as stdout:
result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
def tearDown(self):
print('tear down')
1/0
def test_foo(self):
pass
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
expected_out = '\nStdout:\ntear down\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 1)
description = f'test_foo ({strclass(Foo)})'
test_case, formatted_exc = result.errors[0]
self.assertEqual(str(test_case), description)
self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
self.assertIn(expected_out, formatted_exc)
def testBufferDoCleanups(self):
with captured_stdout() as stdout:
result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
def setUp(self):
print('set up')
self.addCleanup(bad_cleanup1)
self.addCleanup(bad_cleanup2)
def test_foo(self):
pass
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
expected_out = '\nStdout:\nset up\ndo cleanup2\ndo cleanup1\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 2)
description = f'test_foo ({strclass(Foo)})'
test_case, formatted_exc = result.errors[0]
self.assertEqual(str(test_case), description)
self.assertIn('ValueError: bad cleanup2', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
self.assertIn(expected_out, formatted_exc)
test_case, formatted_exc = result.errors[1]
self.assertEqual(str(test_case), description)
self.assertIn('TypeError: bad cleanup1', formatted_exc)
self.assertNotIn('ValueError', formatted_exc)
self.assertIn(expected_out, formatted_exc)
def testBufferSetUp_DoCleanups(self):
with captured_stdout() as stdout:
result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
def setUp(self):
print('set up')
self.addCleanup(bad_cleanup1)
self.addCleanup(bad_cleanup2)
1/0
def test_foo(self):
pass
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
expected_out = '\nStdout:\nset up\ndo cleanup2\ndo cleanup1\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 3)
description = f'test_foo ({strclass(Foo)})'
test_case, formatted_exc = result.errors[0]
self.assertEqual(str(test_case), description)
self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
self.assertNotIn('ValueError', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
self.assertIn(expected_out, formatted_exc)
test_case, formatted_exc = result.errors[1]
self.assertEqual(str(test_case), description)
self.assertIn('ValueError: bad cleanup2', formatted_exc)
self.assertNotIn('ZeroDivisionError', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
self.assertIn(expected_out, formatted_exc)
test_case, formatted_exc = result.errors[2]
self.assertEqual(str(test_case), description)
self.assertIn('TypeError: bad cleanup1', formatted_exc)
self.assertNotIn('ZeroDivisionError', formatted_exc)
self.assertNotIn('ValueError', formatted_exc)
self.assertIn(expected_out, formatted_exc)
def testBufferTearDown_DoCleanups(self):
with captured_stdout() as stdout:
result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
def setUp(self):
print('set up')
self.addCleanup(bad_cleanup1)
self.addCleanup(bad_cleanup2)
def tearDown(self):
print('tear down')
1/0
def test_foo(self):
pass
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
expected_out = '\nStdout:\nset up\ntear down\ndo cleanup2\ndo cleanup1\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 3)
description = f'test_foo ({strclass(Foo)})'
test_case, formatted_exc = result.errors[0]
self.assertEqual(str(test_case), description)
self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
self.assertNotIn('ValueError', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
self.assertIn(expected_out, formatted_exc)
test_case, formatted_exc = result.errors[1]
self.assertEqual(str(test_case), description)
self.assertIn('ValueError: bad cleanup2', formatted_exc)
self.assertNotIn('ZeroDivisionError', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
self.assertIn(expected_out, formatted_exc)
test_case, formatted_exc = result.errors[2]
self.assertEqual(str(test_case), description)
self.assertIn('TypeError: bad cleanup1', formatted_exc)
self.assertNotIn('ZeroDivisionError', formatted_exc)
self.assertNotIn('ValueError', formatted_exc)
self.assertIn(expected_out, formatted_exc)
def testBufferSetupClass(self):
result = unittest.TestResult()
with captured_stdout() as stdout:
result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
@classmethod
def setUpClass(cls):
print('set up class')
1/0
def test_foo(self):
pass
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
expected_out = '\nStdout:\nset up class\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 1)
description = f'setUpClass ({strclass(Foo)})'
test_case, formatted_exc = result.errors[0]
self.assertEqual(test_case.description, description)
self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
self.assertIn(expected_out, formatted_exc)
def testBufferTearDownClass(self):
result = unittest.TestResult()
with captured_stdout() as stdout:
result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
@classmethod
def tearDownClass(cls):
print('tear down class')
1/0
def test_foo(self):
pass
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
expected_out = '\nStdout:\ntear down class\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 1)
description = f'tearDownClass ({strclass(Foo)})'
test_case, formatted_exc = result.errors[0]
self.assertEqual(test_case.description, description)
self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
self.assertIn(expected_out, formatted_exc)
def testBufferDoClassCleanups(self):
with captured_stdout() as stdout:
result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
@classmethod
def setUpClass(cls):
print('set up class')
cls.addClassCleanup(bad_cleanup1)
cls.addClassCleanup(bad_cleanup2)
@classmethod
def tearDownClass(cls):
print('tear down class')
def test_foo(self):
pass
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
expected_out = '\nStdout:\ntear down class\ndo cleanup2\ndo cleanup1\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 2)
description = f'tearDownClass ({strclass(Foo)})'
test_case, formatted_exc = result.errors[0]
self.assertEqual(test_case.description, description)
self.assertIn('ValueError: bad cleanup2', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
self.assertIn(expected_out, formatted_exc)
test_case, formatted_exc = result.errors[1]
self.assertEqual(test_case.description, description)
self.assertIn('TypeError: bad cleanup1', formatted_exc)
self.assertNotIn('ValueError', formatted_exc)
self.assertIn(expected_out, formatted_exc)
def testBufferSetupClass_DoClassCleanups(self):
with captured_stdout() as stdout:
result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
@classmethod
def setUpClass(cls):
print('set up class')
cls.addClassCleanup(bad_cleanup1)
cls.addClassCleanup(bad_cleanup2)
1/0
def test_foo(self):
pass
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
expected_out = '\nStdout:\nset up class\ndo cleanup2\ndo cleanup1\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 3)
description = f'setUpClass ({strclass(Foo)})'
test_case, formatted_exc = result.errors[0]
self.assertEqual(test_case.description, description)
self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
self.assertNotIn('ValueError', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
self.assertIn('\nStdout:\nset up class\n', formatted_exc)
test_case, formatted_exc = result.errors[1]
self.assertEqual(test_case.description, description)
self.assertIn('ValueError: bad cleanup2', formatted_exc)
self.assertNotIn('ZeroDivisionError', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
self.assertIn(expected_out, formatted_exc)
test_case, formatted_exc = result.errors[2]
self.assertEqual(test_case.description, description)
self.assertIn('TypeError: bad cleanup1', formatted_exc)
self.assertNotIn('ZeroDivisionError', formatted_exc)
self.assertNotIn('ValueError', formatted_exc)
self.assertIn(expected_out, formatted_exc)
def testBufferTearDownClass_DoClassCleanups(self):
with captured_stdout() as stdout:
result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
@classmethod
def setUpClass(cls):
print('set up class')
cls.addClassCleanup(bad_cleanup1)
cls.addClassCleanup(bad_cleanup2)
@classmethod
def tearDownClass(cls):
print('tear down class')
1/0
def test_foo(self):
pass
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
expected_out = '\nStdout:\ntear down class\ndo cleanup2\ndo cleanup1\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 3)
description = f'tearDownClass ({strclass(Foo)})'
test_case, formatted_exc = result.errors[0]
self.assertEqual(test_case.description, description)
self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
self.assertNotIn('ValueError', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
self.assertIn('\nStdout:\ntear down class\n', formatted_exc)
test_case, formatted_exc = result.errors[1]
self.assertEqual(test_case.description, description)
self.assertIn('ValueError: bad cleanup2', formatted_exc)
self.assertNotIn('ZeroDivisionError', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
self.assertIn(expected_out, formatted_exc)
test_case, formatted_exc = result.errors[2]
self.assertEqual(test_case.description, description)
self.assertIn('TypeError: bad cleanup1', formatted_exc)
self.assertNotIn('ZeroDivisionError', formatted_exc)
self.assertNotIn('ValueError', formatted_exc)
self.assertIn(expected_out, formatted_exc)
def testBufferSetUpModule(self):
result = unittest.TestResult()
with captured_stdout() as stdout:
result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
@ -671,6 +966,7 @@ class TestOutputBuffering(unittest.TestCase):
class Module(object):
@staticmethod
def setUpModule():
print('set up module')
1/0
Foo.__module__ = 'Module'
@ -678,10 +974,18 @@ class TestOutputBuffering(unittest.TestCase):
self.addCleanup(sys.modules.pop, 'Module')
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
expected_out = '\nStdout:\nset up module\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 1)
description = 'setUpModule (Module)'
test_case, formatted_exc = result.errors[0]
self.assertEqual(test_case.description, description)
self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
self.assertIn(expected_out, formatted_exc)
def testBufferTearDownModule(self):
result = unittest.TestResult()
with captured_stdout() as stdout:
result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
@ -690,6 +994,7 @@ class TestOutputBuffering(unittest.TestCase):
class Module(object):
@staticmethod
def tearDownModule():
print('tear down module')
1/0
Foo.__module__ = 'Module'
@ -697,7 +1002,124 @@ class TestOutputBuffering(unittest.TestCase):
self.addCleanup(sys.modules.pop, 'Module')
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
expected_out = '\nStdout:\ntear down module\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 1)
description = 'tearDownModule (Module)'
test_case, formatted_exc = result.errors[0]
self.assertEqual(test_case.description, description)
self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
self.assertIn(expected_out, formatted_exc)
def testBufferDoModuleCleanups(self):
with captured_stdout() as stdout:
result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
def test_foo(self):
pass
class Module(object):
@staticmethod
def setUpModule():
print('set up module')
unittest.addModuleCleanup(bad_cleanup1)
unittest.addModuleCleanup(bad_cleanup2)
Foo.__module__ = 'Module'
sys.modules['Module'] = Module
self.addCleanup(sys.modules.pop, 'Module')
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
expected_out = '\nStdout:\ndo cleanup2\ndo cleanup1\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 1)
description = 'tearDownModule (Module)'
test_case, formatted_exc = result.errors[0]
self.assertEqual(test_case.description, description)
self.assertIn('ValueError: bad cleanup2', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
self.assertIn(expected_out, formatted_exc)
def testBufferSetUpModule_DoModuleCleanups(self):
with captured_stdout() as stdout:
result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
def test_foo(self):
pass
class Module(object):
@staticmethod
def setUpModule():
print('set up module')
unittest.addModuleCleanup(bad_cleanup1)
unittest.addModuleCleanup(bad_cleanup2)
1/0
Foo.__module__ = 'Module'
sys.modules['Module'] = Module
self.addCleanup(sys.modules.pop, 'Module')
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
expected_out = '\nStdout:\nset up module\ndo cleanup2\ndo cleanup1\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 2)
description = 'setUpModule (Module)'
test_case, formatted_exc = result.errors[0]
self.assertEqual(test_case.description, description)
self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
self.assertNotIn('ValueError', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
self.assertIn('\nStdout:\nset up module\n', formatted_exc)
test_case, formatted_exc = result.errors[1]
self.assertIn(expected_out, formatted_exc)
self.assertEqual(test_case.description, description)
self.assertIn('ValueError: bad cleanup2', formatted_exc)
self.assertNotIn('ZeroDivisionError', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
self.assertIn(expected_out, formatted_exc)
def testBufferTearDownModule_DoModuleCleanups(self):
with captured_stdout() as stdout:
result = unittest.TestResult()
result.buffer = True
class Foo(unittest.TestCase):
def test_foo(self):
pass
class Module(object):
@staticmethod
def setUpModule():
print('set up module')
unittest.addModuleCleanup(bad_cleanup1)
unittest.addModuleCleanup(bad_cleanup2)
@staticmethod
def tearDownModule():
print('tear down module')
1/0
Foo.__module__ = 'Module'
sys.modules['Module'] = Module
self.addCleanup(sys.modules.pop, 'Module')
suite = unittest.TestSuite([Foo('test_foo')])
suite(result)
expected_out = '\nStdout:\ntear down module\ndo cleanup2\ndo cleanup1\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 2)
description = 'tearDownModule (Module)'
test_case, formatted_exc = result.errors[0]
self.assertEqual(test_case.description, description)
self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
self.assertNotIn('ValueError', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
self.assertIn('\nStdout:\ntear down module\n', formatted_exc)
test_case, formatted_exc = result.errors[1]
self.assertEqual(test_case.description, description)
self.assertIn('ValueError: bad cleanup2', formatted_exc)
self.assertNotIn('ZeroDivisionError', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
self.assertIn(expected_out, formatted_exc)
if __name__ == '__main__':