diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -7,6 +7,12 @@ import pprint import re import warnings import collections +import contextlib + +try: + import threading +except ImportError: + import dummy_threading as threading from . import result from .util import (strclass, safe_repr, _count_diff_all_purpose, @@ -42,16 +48,61 @@ class _UnexpectedSuccess(Exception): The test was supposed to fail, but it didn't! """ +class _ShouldStop(Exception): + """ + The test should stop. + """ + class _Outcome(object): - def __init__(self): + def __init__(self, result=None): + self.result = result self.success = True - self.skipped = None - self.unexpectedSuccess = None - self.expectedFailure = None + self.skipped = [] + self.unexpectedSuccesses = [] + self.expectedFailures = [] self.errors = [] self.failures = [] + @contextlib.contextmanager + def testPartExecutor(self, test_case, isTest=False): + try: + yield + except KeyboardInterrupt: + raise + except SkipTest as e: + self.success = False + self.skipped.append((test_case, str(e))) + except _ShouldStop: + pass + except _UnexpectedSuccess: + exc_info = sys.exc_info() + self.success = False + if isTest: + self.unexpectedSuccesses.append((test_case, exc_info)) + else: + self.errors.append((test_case, exc_info)) + except _ExpectedFailure: + self.success = False + exc_info = sys.exc_info() + if isTest: + self.expectedFailures.append((test_case, exc_info)) + else: + self.errors.append((test_case, exc_info)) + except test_case.failureException: + self.success = False + # If the test is expecting a failure, we really want to + # stop now and register the expected failure. + if getattr(_context, 'expecting_failure', 0): + raise + self.failures.append((test_case, sys.exc_info())) + except: + self.success = False + # See above. + if getattr(_context, 'expecting_failure', 0): + raise + self.errors.append((test_case, sys.exc_info())) + def _id(obj): return obj @@ -88,14 +139,23 @@ def skipUnless(condition, reason): return skip(reason) return _id +# This unfortunate hack workarounds the fact that expectedFailure() doesn't +# communicate with the rest of the test machinery. +_context = threading.local() def expectedFailure(func): @functools.wraps(func) def wrapper(*args, **kwargs): try: + _context.expecting_failure += 1 + except AttributeError: + _context.expecting_failure = 1 + try: func(*args, **kwargs) except Exception: raise _ExpectedFailure(sys.exc_info()) + finally: + _context.expecting_failure -= 1 raise _UnexpectedSuccess return wrapper @@ -271,7 +331,7 @@ class TestCase(object): not have a method with the specified name. """ self._testMethodName = methodName - self._outcomeForDoCleanups = None + self._outcome = None self._testMethodDoc = 'No test' try: testMethod = getattr(self, methodName) @@ -284,6 +344,7 @@ class TestCase(object): else: self._testMethodDoc = testMethod.__doc__ self._cleanups = [] + self._subtest = None # Map types to custom assertEqual functions that will compare # instances of said type in more detail to generate a more useful @@ -371,44 +432,32 @@ class TestCase(object): return "<%s testMethod=%s>" % \ (strclass(self.__class__), self._testMethodName) - def _addSkip(self, result, reason): + def _addSkip(self, result, test_case, reason): addSkip = getattr(result, 'addSkip', None) if addSkip is not None: - addSkip(self, reason) + addSkip(test_case, reason) else: warnings.warn("TestResult has no addSkip method, skips not reported", RuntimeWarning, 2) - result.addSuccess(self) + result.addSuccess(test_case) - def _executeTestPart(self, function, outcome, isTest=False): + @contextlib.contextmanager + def subTest(self, **params): + parent = self._subtest + if parent is None: + params_map = collections.ChainMap(params) + else: + params_map = parent.params.new_child(params) + self._subtest = _SubTest(self, params_map, parent) try: - function() - except KeyboardInterrupt: - raise - except SkipTest as e: - outcome.success = False - outcome.skipped = str(e) - except _UnexpectedSuccess: - exc_info = sys.exc_info() - outcome.success = False - if isTest: - outcome.unexpectedSuccess = exc_info - else: - outcome.errors.append(exc_info) - except _ExpectedFailure: - outcome.success = False - exc_info = sys.exc_info() - if isTest: - outcome.expectedFailure = exc_info - else: - outcome.errors.append(exc_info) - except self.failureException: - outcome.success = False - outcome.failures.append(sys.exc_info()) - exc_info = sys.exc_info() - except: - outcome.success = False - outcome.errors.append(sys.exc_info()) + with self._outcome.testPartExecutor(self._subtest, isTest=True): + yield + if not self._outcome.success: + result = self._outcome.result + if result is not None and result.failfast: + raise _ShouldStop + finally: + self._subtest = parent def run(self, result=None): orig_result = result @@ -427,46 +476,49 @@ class TestCase(object): try: skip_why = (getattr(self.__class__, '__unittest_skip_why__', '') or getattr(testMethod, '__unittest_skip_why__', '')) - self._addSkip(result, skip_why) + self._addSkip(result, self, skip_why) finally: result.stopTest(self) return try: - outcome = _Outcome() - self._outcomeForDoCleanups = outcome + outcome = _Outcome(result) + self._outcome = outcome - self._executeTestPart(self.setUp, outcome) + with outcome.testPartExecutor(self): + self.setUp() if outcome.success: - self._executeTestPart(testMethod, outcome, isTest=True) - self._executeTestPart(self.tearDown, outcome) + with outcome.testPartExecutor(self, isTest=True): + testMethod() + with outcome.testPartExecutor(self): + self.tearDown() self.doCleanups() if outcome.success: result.addSuccess(self) else: - if outcome.skipped is not None: - self._addSkip(result, outcome.skipped) - for exc_info in outcome.errors: - result.addError(self, exc_info) - for exc_info in outcome.failures: - result.addFailure(self, exc_info) - if outcome.unexpectedSuccess is not None: + for test, reason in outcome.skipped: + self._addSkip(result, test, reason) + for test, exc_info in outcome.errors: + result.addError(test, exc_info) + for test, exc_info in outcome.failures: + result.addFailure(test, exc_info) + for test, exc_info in outcome.unexpectedSuccesses: addUnexpectedSuccess = getattr(result, 'addUnexpectedSuccess', None) if addUnexpectedSuccess is not None: - addUnexpectedSuccess(self) + addUnexpectedSuccess(test) else: warnings.warn("TestResult has no addUnexpectedSuccess method, reporting as failures", RuntimeWarning) - result.addFailure(self, outcome.unexpectedSuccess) + result.addFailure(test, exc_info) - if outcome.expectedFailure is not None: + for test, exc_info in outcome.expectedFailures: addExpectedFailure = getattr(result, 'addExpectedFailure', None) if addExpectedFailure is not None: - addExpectedFailure(self, outcome.expectedFailure) + addExpectedFailure(test, exc_info) else: warnings.warn("TestResult has no addExpectedFailure method, reporting as passes", RuntimeWarning) - result.addSuccess(self) + result.addSuccess(test) return result finally: result.stopTest(self) @@ -478,11 +530,11 @@ class TestCase(object): def doCleanups(self): """Execute all cleanup functions. Normally called for you after tearDown.""" - outcome = self._outcomeForDoCleanups or _Outcome() + outcome = self._outcome or _Outcome() while self._cleanups: function, args, kwargs = self._cleanups.pop() - part = lambda: function(*args, **kwargs) - self._executeTestPart(part, outcome) + with outcome.testPartExecutor(self): + function(*args, **kwargs) # return this for backwards compatibility # even though we no longer us it internally @@ -1213,3 +1265,35 @@ class FunctionTestCase(TestCase): return self._description doc = self._testFunc.__doc__ return doc and doc.split("\n")[0].strip() or None + + +class _SubTest(TestCase): + + def __init__(self, test_case, params, parent=None): + super().__init__() + self.test_case = test_case + self.params = params + self.failureException = test_case.failureException + + def runTest(self): + raise NotImplementedError("subtests cannot be run directly") + + def _paramsDescription(self): + if self.params: + return ', '.join( + "{}={!r}".format(k, v) + for (k, v) in sorted(self.params.items())) + else: + return '' + + def id(self): + return "{} ({})".format(self.test_case.id(), self._paramsDescription()) + + def shortDescription(self): + """Returns a one-line description of the subtest, or None if no + description has been provided. + """ + return self.test_case.shortDescription() + + def __str__(self): + return "{} ({})".format(self.test_case, self._paramsDescription()) diff --git a/Lib/unittest/test/test_case.py b/Lib/unittest/test/test_case.py --- a/Lib/unittest/test/test_case.py +++ b/Lib/unittest/test/test_case.py @@ -297,6 +297,55 @@ class Test_TestCase(unittest.TestCase, T Foo('test').run() + def test_run_call_order__subtests(self): + events = [] + result = LoggingResult(events) + + class Foo(Test.LoggingTestCase): + def test(self): + super(Foo, self).test() + for i in range(1, 4): + with self.subTest(i=i): + if i == 1: + self.fail('failure') + for j in range(0, 5): + with self.subTest(j=j): + if i * j == 6: + raise RuntimeError('raised by Foo.test') + self.fail('failure') + + # Order is the following: + # i=1 => failure + # i=2, j=3 => error + # i=3, j=2 => error + # toplevel => failure + # However, errors and failures get accumulated at the end of the + # run() method, regardless of order. + expected = ['startTest', 'setUp', 'test', 'tearDown', + 'addError', 'addError', 'addFailure', 'addFailure', + 'stopTest'] + Foo(events).run(result) + self.assertEqual(events, expected) + + def test_run_call_order__subtests_failfast(self): + events = [] + result = LoggingResult(events) + result.failfast = True + + class Foo(Test.LoggingTestCase): + def test(self): + super(Foo, self).test() + with self.subTest(i=1): + self.fail('failure') + with self.subTest(i=2): + self.fail('failure') + self.fail('failure') + + expected = ['startTest', 'setUp', 'test', 'tearDown', + 'addFailure', 'stopTest'] + Foo(events).run(result) + self.assertEqual(events, expected) + # "This class attribute gives the exception raised by the test() method. # If a test framework needs to use a specialized exception, possibly to # carry additional information, it must subclass this exception in diff --git a/Lib/unittest/test/test_result.py b/Lib/unittest/test/test_result.py --- a/Lib/unittest/test/test_result.py +++ b/Lib/unittest/test/test_result.py @@ -234,6 +234,31 @@ class Test_TestResult(unittest.TestCase) 'testGetDescriptionWithoutDocstring (' + __name__ + '.Test_TestResult)') + def testGetSubTestDescriptionWithoutDocstring(self): + with self.subTest(foo=1, bar=2): + result = unittest.TextTestResult(None, True, 1) + self.assertEqual( + result.getDescription(self._subtest), + 'testGetSubTestDescriptionWithoutDocstring (' + __name__ + + '.Test_TestResult) (bar=2, foo=1)') + + def testGetSubTestDescriptionWithoutDocstringAndParams(self): + with self.subTest(): + result = unittest.TextTestResult(None, True, 1) + self.assertEqual( + result.getDescription(self._subtest), + 'testGetSubTestDescriptionWithoutDocstringAndParams ' + '(' + __name__ + '.Test_TestResult) ()') + + def testGetNestedSubTestDescriptionWithoutDocstring(self): + with self.subTest(foo=1): + with self.subTest(bar=2): + result = unittest.TextTestResult(None, True, 1) + self.assertEqual( + result.getDescription(self._subtest), + 'testGetNestedSubTestDescriptionWithoutDocstring ' + '(' + __name__ + '.Test_TestResult) (bar=2, foo=1)') + @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def testGetDescriptionWithOneLineDocstring(self): @@ -247,6 +272,18 @@ class Test_TestResult(unittest.TestCase) @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") + def testGetSubTestDescriptionWithOneLineDocstring(self): + """Tests getDescription() for a method with a docstring.""" + result = unittest.TextTestResult(None, True, 1) + with self.subTest(foo=1, bar=2): + self.assertEqual( + result.getDescription(self._subtest), + ('testGetSubTestDescriptionWithOneLineDocstring ' + '(' + __name__ + '.Test_TestResult) (bar=2, foo=1)\n' + 'Tests getDescription() for a method with a docstring.')) + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") def testGetDescriptionWithMultiLineDocstring(self): """Tests getDescription() for a method with a longer docstring. The second line of the docstring. @@ -259,6 +296,21 @@ class Test_TestResult(unittest.TestCase) 'Tests getDescription() for a method with a longer ' 'docstring.')) + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def testGetSubTestDescriptionWithMultiLineDocstring(self): + """Tests getDescription() for a method with a longer docstring. + The second line of the docstring. + """ + result = unittest.TextTestResult(None, True, 1) + with self.subTest(foo=1, bar=2): + self.assertEqual( + result.getDescription(self._subtest), + ('testGetSubTestDescriptionWithMultiLineDocstring ' + '(' + __name__ + '.Test_TestResult) (bar=2, foo=1)\n' + 'Tests getDescription() for a method with a longer ' + 'docstring.')) + def testStackFrameTrimming(self): class Frame(object): class tb_frame(object): diff --git a/Lib/unittest/test/test_runner.py b/Lib/unittest/test/test_runner.py --- a/Lib/unittest/test/test_runner.py +++ b/Lib/unittest/test/test_runner.py @@ -5,6 +5,7 @@ import pickle import subprocess import unittest +from unittest.case import _Outcome from .support import LoggingResult, ResultWithNoStartTestRunStopTestRun @@ -42,12 +43,8 @@ class TestCleanUp(unittest.TestCase): def testNothing(self): pass - class MockOutcome(object): - success = True - errors = [] - test = TestableTest('testNothing') - test._outcomeForDoCleanups = MockOutcome + outcome = test._outcome = _Outcome() exc1 = Exception('foo') exc2 = Exception('bar') @@ -61,9 +58,10 @@ class TestCleanUp(unittest.TestCase): test.addCleanup(cleanup2) self.assertFalse(test.doCleanups()) - self.assertFalse(MockOutcome.success) + self.assertFalse(outcome.success) - (Type1, instance1, _), (Type2, instance2, _) = reversed(MockOutcome.errors) + ((_, (Type1, instance1, _)), + (_, (Type2, instance2, _))) = reversed(outcome.errors) self.assertEqual((Type1, instance1), (Exception, exc1)) self.assertEqual((Type2, instance2), (Exception, exc2)) diff --git a/Lib/unittest/test/test_skipping.py b/Lib/unittest/test/test_skipping.py --- a/Lib/unittest/test/test_skipping.py +++ b/Lib/unittest/test/test_skipping.py @@ -29,6 +29,31 @@ class Test_TestSkipping(unittest.TestCas self.assertEqual(result.skipped, [(test, "testing")]) self.assertEqual(result.testsRun, 1) + def test_skipping_subtests(self): + class Foo(unittest.TestCase): + def test_skip_me(self): + with self.subTest(a=1): + with self.subTest(b=2): + self.skipTest("skip 1") + self.skipTest("skip 2") + self.skipTest("skip 3") + events = [] + result = LoggingResult(events) + test = Foo("test_skip_me") + test.run(result) + self.assertEqual(events, ['startTest', 'addSkip', 'addSkip', + 'addSkip', 'stopTest']) + self.assertEqual(len(result.skipped), 3) + subtest, msg = result.skipped[0] + self.assertEqual(msg, "skip 1") + self.assertIsInstance(subtest, unittest.TestCase) + self.assertIsNot(subtest, test) + subtest, msg = result.skipped[1] + self.assertEqual(msg, "skip 2") + self.assertIsInstance(subtest, unittest.TestCase) + self.assertIsNot(subtest, test) + self.assertEqual(result.skipped[2], (test, "skip 3")) + def test_skipping_decorators(self): op_table = ((unittest.skipUnless, False, True), (unittest.skipIf, True, False)) @@ -95,6 +120,30 @@ class Test_TestSkipping(unittest.TestCas self.assertEqual(result.expectedFailures[0][0], test) self.assertTrue(result.wasSuccessful()) + def test_expected_failure_subtests(self): + # A failure in any subtest counts as the expected failure of the + # whole test. + class Foo(unittest.TestCase): + @unittest.expectedFailure + def test_die(self): + with self.subTest(): + # This one succeeds + pass + with self.subTest(): + self.fail("help me!") + with self.subTest(): + # This one doesn't get executed + self.fail("shouldn't come here") + events = [] + result = LoggingResult(events) + test = Foo("test_die") + test.run(result) + self.assertEqual(events, + ['startTest', 'addExpectedFailure', 'stopTest']) + self.assertEqual(len(result.expectedFailures), 1) + self.assertIs(result.expectedFailures[0][0], test) + self.assertTrue(result.wasSuccessful()) + def test_unexpected_success(self): class Foo(unittest.TestCase): @unittest.expectedFailure @@ -110,6 +159,28 @@ class Test_TestSkipping(unittest.TestCas self.assertEqual(result.unexpectedSuccesses, [test]) self.assertTrue(result.wasSuccessful()) + def test_unexpected_success_subtests(self): + # Success in all subtests counts as the unexpected success of + # the whole test. + class Foo(unittest.TestCase): + @unittest.expectedFailure + def test_die(self): + with self.subTest(): + # This one succeeds + pass + with self.subTest(): + # So does this one + pass + events = [] + result = LoggingResult(events) + test = Foo("test_die") + test.run(result) + self.assertEqual(events, + ['startTest', 'addUnexpectedSuccess', 'stopTest']) + self.assertFalse(result.failures) + self.assertEqual(result.unexpectedSuccesses, [test]) + self.assertTrue(result.wasSuccessful()) + def test_skip_doesnt_run_setup(self): class Foo(unittest.TestCase): wasSetUp = False