Better exception messages for unittest assert methods.

- unittest.assertNotEqual() now uses the inequality operator (!=) instead
  of the equality operator.

- Default assertTrue and assertFalse messages are now useful.

- TestCase has a longMessage attribute. This defaults to False, but if set to True
  useful error messages are shown in addition to explicit messages passed to assert methods.

Issue #5663
This commit is contained in:
Michael Foord 2009-04-02 03:20:38 +00:00
parent c4f90ebea7
commit 345b2fe21e
4 changed files with 297 additions and 71 deletions

View file

@ -1,4 +1,3 @@
:mod:`unittest` --- Unit testing framework :mod:`unittest` --- Unit testing framework
========================================== ==========================================
@ -885,6 +884,25 @@ Test cases
fair" with the framework. The initial value of this attribute is fair" with the framework. The initial value of this attribute is
:exc:`AssertionError`. :exc:`AssertionError`.
.. attribute:: longMessage
If set to True then any explicit failure message you pass in to the
assert methods will be appended to the end of the normal failure message.
The normal messages contain useful information about the objects involved,
for example the message from assertEqual shows you the repr of the two
unequal objects. Setting this attribute to True allows you to have a
custom error message in addition to the normal one.
This attribute defaults to False, meaning that a custom message passed
to an assert method will silence the normal message.
The class setting can be overridden in individual tests by assigning an
instance attribute to True or False before calling the assert methods.
.. versionadded:: 2.7
Testing frameworks can use the following methods to collect information on Testing frameworks can use the following methods to collect information on
the test: the test:

View file

@ -54,6 +54,8 @@ class LoggingResult(unittest.TestResult):
class TestEquality(object): class TestEquality(object):
"""Used as a mixin for TestCase"""
# Check for a valid __eq__ implementation # Check for a valid __eq__ implementation
def test_eq(self): def test_eq(self):
for obj_1, obj_2 in self.eq_pairs: for obj_1, obj_2 in self.eq_pairs:
@ -67,6 +69,8 @@ class TestEquality(object):
self.failIfEqual(obj_2, obj_1) self.failIfEqual(obj_2, obj_1)
class TestHashing(object): class TestHashing(object):
"""Used as a mixin for TestCase"""
# Check for a valid __hash__ implementation # Check for a valid __hash__ implementation
def test_hash(self): def test_hash(self):
for obj_1, obj_2 in self.eq_pairs: for obj_1, obj_2 in self.eq_pairs:
@ -2835,6 +2839,172 @@ class Test_Assertions(TestCase):
self.fail("assertRaises() didn't let exception pass through") self.fail("assertRaises() didn't let exception pass through")
class TestLongMessage(TestCase):
"""Test that the individual asserts honour longMessage.
This actually tests all the message behaviour for
asserts that use longMessage."""
def setUp(self):
class TestableTestFalse(TestCase):
longMessage = False
failureException = self.failureException
def testTest(self):
pass
class TestableTestTrue(TestCase):
longMessage = True
failureException = self.failureException
def testTest(self):
pass
self.testableTrue = TestableTestTrue('testTest')
self.testableFalse = TestableTestFalse('testTest')
def testDefault(self):
self.assertFalse(TestCase.longMessage)
def test_formatMsg(self):
self.assertEquals(self.testableFalse._formatMessage(None, "foo"), "foo")
self.assertEquals(self.testableFalse._formatMessage("foo", "bar"), "foo")
self.assertEquals(self.testableTrue._formatMessage(None, "foo"), "foo")
self.assertEquals(self.testableTrue._formatMessage("foo", "bar"), "bar : foo")
def assertMessages(self, methodName, args, errors):
def getMethod(i):
useTestableFalse = i < 2
if useTestableFalse:
test = self.testableFalse
else:
test = self.testableTrue
return getattr(test, methodName)
for i, expected_regexp in enumerate(errors):
testMethod = getMethod(i)
kwargs = {}
withMsg = i % 2
if withMsg:
kwargs = {"msg": "oops"}
with self.assertRaisesRegexp(self.failureException,
expected_regexp=expected_regexp):
testMethod(*args, **kwargs)
def testAssertTrue(self):
self.assertMessages('assertTrue', (False,),
["^False is not True$", "^oops$", "^False is not True$",
"^False is not True : oops$"])
def testAssertFalse(self):
self.assertMessages('assertFalse', (True,),
["^True is not False$", "^oops$", "^True is not False$",
"^True is not False : oops$"])
def testNotEqual(self):
self.assertMessages('assertNotEqual', (1, 1),
["^1 == 1$", "^oops$", "^1 == 1$",
"^1 == 1 : oops$"])
def testAlmostEqual(self):
self.assertMessages('assertAlmostEqual', (1, 2),
["^1 != 2 within 7 places$", "^oops$",
"^1 != 2 within 7 places$", "^1 != 2 within 7 places : oops$"])
def testNotAlmostEqual(self):
self.assertMessages('assertNotAlmostEqual', (1, 1),
["^1 == 1 within 7 places$", "^oops$",
"^1 == 1 within 7 places$", "^1 == 1 within 7 places : oops$"])
def test_baseAssertEqual(self):
self.assertMessages('_baseAssertEqual', (1, 2),
["^1 != 2$", "^oops$", "^1 != 2$", "^1 != 2 : oops$"])
def testAssertSequenceEqual(self):
# Error messages are multiline so not testing on full message
# assertTupleEqual and assertListEqual delegate to this method
self.assertMessages('assertSequenceEqual', ([], [None]),
["\+ \[None\]$", "^oops$", r"\+ \[None\]$",
r"\+ \[None\] : oops$"])
def testAssertSetEqual(self):
self.assertMessages('assertSetEqual', (set(), set([None])),
["None$", "^oops$", "None$",
"None : oops$"])
def testAssertIn(self):
self.assertMessages('assertIn', (None, []),
['^None not found in \[\]$', "^oops$",
'^None not found in \[\]$',
'^None not found in \[\] : oops$'])
def testAssertNotIn(self):
self.assertMessages('assertNotIn', (None, [None]),
['^None unexpectedly found in \[None\]$', "^oops$",
'^None unexpectedly found in \[None\]$',
'^None unexpectedly found in \[None\] : oops$'])
def testAssertDictEqual(self):
self.assertMessages('assertDictEqual', ({}, {'key': 'value'}),
[r"\+ \{'key': 'value'\}$", "^oops$",
"\+ \{'key': 'value'\}$",
"\+ \{'key': 'value'\} : oops$"])
def testAssertDictContainsSubset(self):
self.assertMessages('assertDictContainsSubset', ({'key': 'value'}, {}),
["^Missing: 'key'$", "^oops$",
"^Missing: 'key'$",
"^Missing: 'key' : oops$"])
def testAssertSameElements(self):
self.assertMessages('assertSameElements', ([], [None]),
[r"\[None\]$", "^oops$",
r"\[None\]$",
r"\[None\] : oops$"])
def testAssertMultiLineEqual(self):
self.assertMessages('assertMultiLineEqual', ("", "foo"),
[r"\+ foo$", "^oops$",
r"\+ foo$",
r"\+ foo : oops$"])
def testAssertLess(self):
self.assertMessages('assertLess', (2, 1),
["^2 not less than 1$", "^oops$",
"^2 not less than 1$", "^2 not less than 1 : oops$"])
def testAssertLessEqual(self):
self.assertMessages('assertLessEqual', (2, 1),
["^2 not less than or equal to 1$", "^oops$",
"^2 not less than or equal to 1$",
"^2 not less than or equal to 1 : oops$"])
def testAssertGreater(self):
self.assertMessages('assertGreater', (1, 2),
["^1 not greater than 2$", "^oops$",
"^1 not greater than 2$",
"^1 not greater than 2 : oops$"])
def testAssertGreaterEqual(self):
self.assertMessages('assertGreaterEqual', (1, 2),
["^1 not greater than or equal to 2$", "^oops$",
"^1 not greater than or equal to 2$",
"^1 not greater than or equal to 2 : oops$"])
def testAssertIsNone(self):
self.assertMessages('assertIsNone', ('not None',),
["^'not None' is not None$", "^oops$",
"^'not None' is not None$",
"^'not None' is not None : oops$"])
def testAssertIsNotNone(self):
self.assertMessages('assertIsNotNone', (None,),
["^unexpectedly None$", "^oops$",
"^unexpectedly None$",
"^unexpectedly None : oops$"])
###################################################################### ######################################################################
## Main ## Main
###################################################################### ######################################################################
@ -2842,7 +3012,7 @@ class Test_Assertions(TestCase):
def test_main(): def test_main():
test_support.run_unittest(Test_TestCase, Test_TestLoader, test_support.run_unittest(Test_TestCase, Test_TestLoader,
Test_TestSuite, Test_TestResult, Test_FunctionTestCase, Test_TestSuite, Test_TestResult, Test_FunctionTestCase,
Test_TestSkipping, Test_Assertions) Test_TestSkipping, Test_Assertions, TestLongMessage)
if __name__ == "__main__": if __name__ == "__main__":
test_main() test_main()

View file

@ -275,7 +275,7 @@ class _AssertRaisesContext(object):
raise self.failureException( raise self.failureException(
"{0} not raised".format(exc_name)) "{0} not raised".format(exc_name))
if not issubclass(exc_type, self.expected): if not issubclass(exc_type, self.expected):
# let unexpexted exceptions pass through # let unexpected exceptions pass through
return False return False
if self.expected_regex is None: if self.expected_regex is None:
return True return True
@ -318,6 +318,13 @@ class TestCase(object):
failureException = AssertionError failureException = AssertionError
# This attribute determines whether long messages (including repr of
# objects used in assert methods) will be printed on failure in *addition*
# to any explicit message passed.
longMessage = False
def __init__(self, methodName='runTest'): def __init__(self, methodName='runTest'):
"""Create an instance of the class that will use the named test """Create an instance of the class that will use the named test
method when executed. Raises a ValueError if the instance does method when executed. Raises a ValueError if the instance does
@ -471,13 +478,32 @@ class TestCase(object):
def assertFalse(self, expr, msg=None): def assertFalse(self, expr, msg=None):
"Fail the test if the expression is true." "Fail the test if the expression is true."
if expr: if expr:
msg = self._formatMessage(msg, "%r is not False" % expr)
raise self.failureException(msg) raise self.failureException(msg)
def assertTrue(self, expr, msg=None): def assertTrue(self, expr, msg=None):
"""Fail the test unless the expression is true.""" """Fail the test unless the expression is true."""
if not expr: if not expr:
msg = self._formatMessage(msg, "%r is not True" % expr)
raise self.failureException(msg) raise self.failureException(msg)
def _formatMessage(self, msg, standardMsg):
"""Honour the longMessage attribute when generating failure messages.
If longMessage is False this means:
* Use only an explicit message if it is provided
* Otherwise use the standard message for the assert
If longMessage is True:
* Use the standard message
* If an explicit message is provided, plus ' : ' and the explicit message
"""
if not self.longMessage:
return msg or standardMsg
if msg is None:
return standardMsg
return standardMsg + ' : ' + msg
def assertRaises(self, excClass, callableObj=None, *args, **kwargs): def assertRaises(self, excClass, callableObj=None, *args, **kwargs):
"""Fail unless an exception of class excClass is thrown """Fail unless an exception of class excClass is thrown
by callableObj when invoked with arguments args and keyword by callableObj when invoked with arguments args and keyword
@ -523,7 +549,9 @@ class TestCase(object):
def _baseAssertEqual(self, first, second, msg=None): def _baseAssertEqual(self, first, second, msg=None):
"""The default assertEqual implementation, not type specific.""" """The default assertEqual implementation, not type specific."""
if not first == second: if not first == second:
raise self.failureException(msg or '%r != %r' % (first, second)) standardMsg = '%r != %r' % (first, second)
msg = self._formatMessage(msg, standardMsg)
raise self.failureException(msg)
def assertEqual(self, first, second, msg=None): def assertEqual(self, first, second, msg=None):
"""Fail if the two objects are unequal as determined by the '==' """Fail if the two objects are unequal as determined by the '=='
@ -536,8 +564,9 @@ class TestCase(object):
"""Fail if the two objects are equal as determined by the '==' """Fail if the two objects are equal as determined by the '=='
operator. operator.
""" """
if first == second: if not first != second:
raise self.failureException(msg or '%r == %r' % (first, second)) msg = self._formatMessage(msg, '%r == %r' % (first, second))
raise self.failureException(msg)
def assertAlmostEqual(self, first, second, places=7, msg=None): def assertAlmostEqual(self, first, second, places=7, msg=None):
"""Fail if the two objects are unequal as determined by their """Fail if the two objects are unequal as determined by their
@ -548,8 +577,9 @@ class TestCase(object):
as significant digits (measured from the most signficant digit). as significant digits (measured from the most signficant digit).
""" """
if round(abs(second-first), places) != 0: if round(abs(second-first), places) != 0:
raise self.failureException( standardMsg = '%r != %r within %r places' % (first, second, places)
msg or '%r != %r within %r places' % (first, second, places)) msg = self._formatMessage(msg, standardMsg)
raise self.failureException(msg)
def assertNotAlmostEqual(self, first, second, places=7, msg=None): def assertNotAlmostEqual(self, first, second, places=7, msg=None):
"""Fail if the two objects are equal as determined by their """Fail if the two objects are equal as determined by their
@ -560,8 +590,9 @@ class TestCase(object):
as significant digits (measured from the most signficant digit). as significant digits (measured from the most signficant digit).
""" """
if round(abs(second-first), places) == 0: if round(abs(second-first), places) == 0:
raise self.failureException( standardMsg = '%r == %r within %r places' % (first, second, places)
msg or '%r == %r within %r places' % (first, second, places)) msg = self._formatMessage(msg, standardMsg)
raise self.failureException(msg)
# Synonyms for assertion methods # Synonyms for assertion methods
@ -680,10 +711,10 @@ class TestCase(object):
except (TypeError, IndexError, NotImplementedError): except (TypeError, IndexError, NotImplementedError):
differing += ('Unable to index element %d ' differing += ('Unable to index element %d '
'of second %s\n' % (len1, seq_type_name)) 'of second %s\n' % (len1, seq_type_name))
if not msg: standardMsg = differing + '\n'.join(difflib.ndiff(pprint.pformat(seq1).splitlines(),
msg = '\n'.join(difflib.ndiff(pprint.pformat(seq1).splitlines(), pprint.pformat(seq2).splitlines()))
pprint.pformat(seq2).splitlines())) msg = self._formatMessage(msg, standardMsg)
self.fail(differing + msg) self.fail(msg)
def assertListEqual(self, list1, list2, msg=None): def assertListEqual(self, list1, list2, msg=None):
"""A list-specific equality assertion. """A list-specific equality assertion.
@ -739,9 +770,6 @@ class TestCase(object):
if not (difference1 or difference2): if not (difference1 or difference2):
return return
if msg is not None:
self.fail(msg)
lines = [] lines = []
if difference1: if difference1:
lines.append('Items in the first set but not the second:') lines.append('Items in the first set but not the second:')
@ -751,28 +779,31 @@ class TestCase(object):
lines.append('Items in the second set but not the first:') lines.append('Items in the second set but not the first:')
for item in difference2: for item in difference2:
lines.append(repr(item)) lines.append(repr(item))
self.fail('\n'.join(lines))
def assertIn(self, a, b, msg=None): standardMsg = '\n'.join(lines)
"""Just like self.assert_(a in b), but with a nicer default message.""" self.fail(self._formatMessage(msg, standardMsg))
if msg is None:
msg = '"%s" not found in "%s"' % (a, b)
self.assert_(a in b, msg)
def assertNotIn(self, a, b, msg=None): def assertIn(self, member, container, msg=None):
"""Just like self.assert_(a not in b), but with a nicer default message.""" """Just like self.assertTrue(a in b), but with a nicer default message."""
if msg is None: if member not in container:
msg = '"%s" unexpectedly found in "%s"' % (a, b) standardMsg = '%r not found in %r' % (member, container)
self.assert_(a not in b, msg) self.fail(self._formatMessage(msg, standardMsg))
def assertNotIn(self, member, container, msg=None):
"""Just like self.assertTrue(a not in b), but with a nicer default message."""
if member in container:
standardMsg = '%r unexpectedly found in %r' % (member, container)
self.fail(self._formatMessage(msg, standardMsg))
def assertDictEqual(self, d1, d2, msg=None): def assertDictEqual(self, d1, d2, msg=None):
self.assert_(isinstance(d1, dict), 'First argument is not a dictionary') self.assert_(isinstance(d1, dict), 'First argument is not a dictionary')
self.assert_(isinstance(d2, dict), 'Second argument is not a dictionary') self.assert_(isinstance(d2, dict), 'Second argument is not a dictionary')
if d1 != d2: if d1 != d2:
self.fail(msg or ('\n' + '\n'.join(difflib.ndiff( standardMsg = ('\n' + '\n'.join(difflib.ndiff(
pprint.pformat(d1).splitlines(), pprint.pformat(d1).splitlines(),
pprint.pformat(d2).splitlines())))) pprint.pformat(d2).splitlines())))
self.fail(self._formatMessage(msg, standardMsg))
def assertDictContainsSubset(self, expected, actual, msg=None): def assertDictContainsSubset(self, expected, actual, msg=None):
"""Checks whether actual is a superset of expected.""" """Checks whether actual is a superset of expected."""
@ -782,23 +813,20 @@ class TestCase(object):
if key not in actual: if key not in actual:
missing.append(key) missing.append(key)
elif value != actual[key]: elif value != actual[key]:
mismatched.append('%s, expected: %s, actual: %s' % (key, value, mismatched.append('%s, expected: %s, actual: %s' % (key, value, actual[key]))
actual[key]))
if not (missing or mismatched): if not (missing or mismatched):
return return
missing_msg = mismatched_msg = '' standardMsg = ''
if missing: if missing:
missing_msg = 'Missing: %s' % ','.join(missing) standardMsg = 'Missing: %r' % ','.join(missing)
if mismatched: if mismatched:
mismatched_msg = 'Mismatched values: %s' % ','.join(mismatched) if standardMsg:
standardMsg += '; '
standardMsg += 'Mismatched values: %s' % ','.join(mismatched)
if msg: self.fail(self._formatMessage(msg, standardMsg))
msg = '%s: %s; %s' % (msg, missing_msg, mismatched_msg)
else:
msg = '%s; %s' % (missing_msg, mismatched_msg)
self.fail(msg)
def assertSameElements(self, expected_seq, actual_seq, msg=None): def assertSameElements(self, expected_seq, actual_seq, msg=None):
"""An unordered sequence specific comparison. """An unordered sequence specific comparison.
@ -823,57 +851,59 @@ class TestCase(object):
missing, unexpected = _SortedListDifference(expected, actual) missing, unexpected = _SortedListDifference(expected, actual)
errors = [] errors = []
if missing: if missing:
errors.append('Expected, but missing:\n %r\n' % missing) errors.append('Expected, but missing:\n %r' % missing)
if unexpected: if unexpected:
errors.append('Unexpected, but present:\n %r\n' % unexpected) errors.append('Unexpected, but present:\n %r' % unexpected)
if errors: if errors:
self.fail(msg or ''.join(errors)) standardMsg = '\n'.join(errors)
self.fail(self._formatMessage(msg, standardMsg))
def assertMultiLineEqual(self, first, second, msg=None): def assertMultiLineEqual(self, first, second, msg=None):
"""Assert that two multi-line strings are equal.""" """Assert that two multi-line strings are equal."""
self.assert_(isinstance(first, types.StringTypes), ( self.assert_(isinstance(first, basestring), (
'First argument is not a string')) 'First argument is not a string'))
self.assert_(isinstance(second, types.StringTypes), ( self.assert_(isinstance(second, basestring), (
'Second argument is not a string')) 'Second argument is not a string'))
if first != second: if first != second:
raise self.failureException( standardMsg = '\n' + ''.join(difflib.ndiff(first.splitlines(True), second.splitlines(True)))
msg or '\n' + ''.join(difflib.ndiff(first.splitlines(True), self.fail(self._formatMessage(msg, standardMsg))
second.splitlines(True))))
def assertLess(self, a, b, msg=None): def assertLess(self, a, b, msg=None):
"""Just like self.assert_(a < b), but with a nicer default message.""" """Just like self.assertTrue(a < b), but with a nicer default message."""
if msg is None: if not a < b:
msg = '"%r" unexpectedly not less than "%r"' % (a, b) standardMsg = '%r not less than %r' % (a, b)
self.assert_(a < b, msg) self.fail(self._formatMessage(msg, standardMsg))
def assertLessEqual(self, a, b, msg=None): def assertLessEqual(self, a, b, msg=None):
"""Just like self.assert_(a <= b), but with a nicer default message.""" """Just like self.assertTrue(a <= b), but with a nicer default message."""
if msg is None: if not a <= b:
msg = '"%r" unexpectedly not less than or equal to "%r"' % (a, b) standardMsg = '%r not less than or equal to %r' % (a, b)
self.assert_(a <= b, msg) self.fail(self._formatMessage(msg, standardMsg))
def assertGreater(self, a, b, msg=None): def assertGreater(self, a, b, msg=None):
"""Just like self.assert_(a > b), but with a nicer default message.""" """Just like self.assertTrue(a > b), but with a nicer default message."""
if msg is None: if not a > b:
msg = '"%r" unexpectedly not greater than "%r"' % (a, b) standardMsg = '%r not greater than %r' % (a, b)
self.assert_(a > b, msg) self.fail(self._formatMessage(msg, standardMsg))
def assertGreaterEqual(self, a, b, msg=None): def assertGreaterEqual(self, a, b, msg=None):
"""Just like self.assert_(a >= b), but with a nicer default message.""" """Just like self.assertTrue(a >= b), but with a nicer default message."""
if msg is None: if not a >= b:
msg = '"%r" unexpectedly not greater than or equal to "%r"' % (a, b) standardMsg = '%r not greater than or equal to %r' % (a, b)
self.assert_(a >= b, msg) self.fail(self._formatMessage(msg, standardMsg))
def assertIsNone(self, obj, msg=None): def assertIsNone(self, obj, msg=None):
"""Same as self.assert_(obj is None), with a nicer default message.""" """Same as self.assertTrue(obj is None), with a nicer default message."""
if msg is None: if obj is not None:
msg = '"%s" unexpectedly not None' % obj standardMsg = '%r is not None' % obj
self.assert_(obj is None, msg) self.fail(self._formatMessage(msg, standardMsg))
def assertIsNotNone(self, obj, msg='unexpectedly None'): def assertIsNotNone(self, obj, msg=None):
"""Included for symmetry with assertIsNone.""" """Included for symmetry with assertIsNone."""
self.assert_(obj is not None, msg) if obj is None:
standardMsg = 'unexpectedly None'
self.fail(self._formatMessage(msg, standardMsg))
def assertRaisesRegexp(self, expected_exception, expected_regexp, def assertRaisesRegexp(self, expected_exception, expected_regexp,
callable_obj=None, *args, **kwargs): callable_obj=None, *args, **kwargs):

View file

@ -202,6 +202,14 @@ Core and Builtins
Library Library
------- -------
- unittest.assertNotEqual() now uses the inequality operator (!=) instead
of the equality operator.
- Issue #5663: better failure messages for unittest asserts. Default assertTrue
and assertFalse messages are now useful. TestCase has a longMessage attribute.
This defaults to False, but if set to True useful error messages are shown in
addition to explicit messages passed to assert methods.
- Issue #3110: Add additional protect around SEM_VALUE_MAX for multiprocessing - Issue #3110: Add additional protect around SEM_VALUE_MAX for multiprocessing
- In Pdb, prevent the reassignment of __builtin__._ by sys.displayhook on - In Pdb, prevent the reassignment of __builtin__._ by sys.displayhook on