diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -7,6 +7,7 @@ import pprint import re import warnings import collections +import contextlib from . import result from .util import (strclass, safe_repr, _count_diff_all_purpose, @@ -42,9 +43,15 @@ class _UnexpectedSuccess(Exception): The test was supposed to fail, but it didn't! """ +class _ShouldStop(Exception): + """ + The test should stop. + """ + class _Outcome(object): - def __init__(self): + def __init__(self, result=None): + self.result = result self.success = True self.skipped = None self.unexpectedSuccess = None @@ -52,6 +59,39 @@ class _Outcome(object): self.errors = [] self.failures = [] + @contextlib.contextmanager + def testPartExecutor(self, test_case, isTest=False): + try: + yield + except KeyboardInterrupt: + raise + except SkipTest as e: + self.success = False + self.skipped = str(e) + except _ShouldStop: + pass + except _UnexpectedSuccess: + exc_info = sys.exc_info() + self.success = False + if isTest: + self.unexpectedSuccess = test_case, exc_info + else: + self.errors.append((test_case, exc_info)) + except _ExpectedFailure: + self.success = False + exc_info = sys.exc_info() + if isTest: + self.expectedFailure = test_case, exc_info + else: + self.errors.append((test_case, exc_info)) + except test_case.failureException: + self.success = False + self.failures.append((test_case, sys.exc_info())) + exc_info = sys.exc_info() + except: + self.success = False + self.errors.append((test_case, sys.exc_info())) + def _id(obj): return obj @@ -217,6 +257,32 @@ class _AssertWarnsContext(_AssertRaisesB self._raiseFailure("{} not triggered".format(exc_name)) +class _SubTest(object): + + def __init__(self, test_case, params, parent=None): + self.test_case = test_case + self.params = params + self.parent = parent + self.failureException = test_case.failureException + + def _paramsDescription(self): + if self.params: + return ', '.join( + "{}={!r}".format(k, v) + for (k, v) in sorted(self.params.items())) + else: + return '' + + def shortDescription(self): + """Returns a one-line description of the subtest, or None if no + description has been provided. + """ + return self.test_case.shortDescription() + + def __str__(self): + return "{} ({})".format(self.test_case, self._paramsDescription()) + + class TestCase(object): """A class whose instances are single test cases. @@ -271,7 +337,7 @@ class TestCase(object): not have a method with the specified name. """ self._testMethodName = methodName - self._outcomeForDoCleanups = None + self._outcome = None self._testMethodDoc = 'No test' try: testMethod = getattr(self, methodName) @@ -284,6 +350,7 @@ class TestCase(object): else: self._testMethodDoc = testMethod.__doc__ self._cleanups = [] + self._subtest = None # Map types to custom assertEqual functions that will compare # instances of said type in more detail to generate a more useful @@ -380,35 +447,23 @@ class TestCase(object): RuntimeWarning, 2) result.addSuccess(self) - def _executeTestPart(self, function, outcome, isTest=False): + @contextlib.contextmanager + def subTest(self, **params): + parent = self._subtest + if parent is None: + params_map = collections.ChainMap(params) + else: + params_map = parent.params.new_child(params) + self._subtest = _SubTest(self, params_map, parent) try: - function() - except KeyboardInterrupt: - raise - except SkipTest as e: - outcome.success = False - outcome.skipped = str(e) - except _UnexpectedSuccess: - exc_info = sys.exc_info() - outcome.success = False - if isTest: - outcome.unexpectedSuccess = exc_info - else: - outcome.errors.append(exc_info) - except _ExpectedFailure: - outcome.success = False - exc_info = sys.exc_info() - if isTest: - outcome.expectedFailure = exc_info - else: - outcome.errors.append(exc_info) - except self.failureException: - outcome.success = False - outcome.failures.append(sys.exc_info()) - exc_info = sys.exc_info() - except: - outcome.success = False - outcome.errors.append(sys.exc_info()) + with self._outcome.testPartExecutor(self._subtest, isTest=True): + yield + if not self._outcome.success: + result = self._outcome.result + if result is not None and result.failfast: + raise _ShouldStop + finally: + self._subtest = parent def run(self, result=None): orig_result = result @@ -432,13 +487,16 @@ class TestCase(object): result.stopTest(self) return try: - outcome = _Outcome() - self._outcomeForDoCleanups = outcome + outcome = _Outcome(result) + self._outcome = outcome - self._executeTestPart(self.setUp, outcome) + with outcome.testPartExecutor(self): + self.setUp() if outcome.success: - self._executeTestPart(testMethod, outcome, isTest=True) - self._executeTestPart(self.tearDown, outcome) + with outcome.testPartExecutor(self, isTest=True): + testMethod() + with outcome.testPartExecutor(self): + self.tearDown() self.doCleanups() if outcome.success: @@ -446,27 +504,29 @@ class TestCase(object): else: if outcome.skipped is not None: self._addSkip(result, outcome.skipped) - for exc_info in outcome.errors: - result.addError(self, exc_info) - for exc_info in outcome.failures: - result.addFailure(self, exc_info) + for test, exc_info in outcome.errors: + result.addError(test, exc_info) + for test, exc_info in outcome.failures: + result.addFailure(test, exc_info) if outcome.unexpectedSuccess is not None: + test, exc_info = outcome.unexpectedSuccess addUnexpectedSuccess = getattr(result, 'addUnexpectedSuccess', None) if addUnexpectedSuccess is not None: - addUnexpectedSuccess(self) + addUnexpectedSuccess(test) else: warnings.warn("TestResult has no addUnexpectedSuccess method, reporting as failures", RuntimeWarning) - result.addFailure(self, outcome.unexpectedSuccess) + result.addFailure(test, exc_info) if outcome.expectedFailure is not None: + test, exc_info = outcome.expectedFailure addExpectedFailure = getattr(result, 'addExpectedFailure', None) if addExpectedFailure is not None: - addExpectedFailure(self, outcome.expectedFailure) + addExpectedFailure(test, exc_info) else: warnings.warn("TestResult has no addExpectedFailure method, reporting as passes", RuntimeWarning) - result.addSuccess(self) + result.addSuccess(test) return result finally: result.stopTest(self) @@ -478,11 +538,11 @@ class TestCase(object): def doCleanups(self): """Execute all cleanup functions. Normally called for you after tearDown.""" - outcome = self._outcomeForDoCleanups or _Outcome() + outcome = self._outcome or _Outcome() while self._cleanups: function, args, kwargs = self._cleanups.pop() - part = lambda: function(*args, **kwargs) - self._executeTestPart(part, outcome) + with outcome.testPartExecutor(self): + function(*args, **kwargs) # return this for backwards compatibility # even though we no longer us it internally diff --git a/Lib/unittest/test/test_case.py b/Lib/unittest/test/test_case.py --- a/Lib/unittest/test/test_case.py +++ b/Lib/unittest/test/test_case.py @@ -297,6 +297,55 @@ class Test_TestCase(unittest.TestCase, T Foo('test').run() + def test_run_call_order__subtests(self): + events = [] + result = LoggingResult(events) + + class Foo(Test.LoggingTestCase): + def test(self): + super(Foo, self).test() + for i in range(1, 4): + with self.subTest(i=i): + if i == 1: + self.fail('failure') + for j in range(0, 5): + with self.subTest(j=j): + if i * j == 6: + raise RuntimeError('raised by Foo.test') + self.fail('failure') + + # Order is the following: + # i=1 => failure + # i=2, j=3 => error + # i=3, j=2 => error + # toplevel => failure + # However, errors and failures get accumulated at the end of the + # run() method, regardless of order. + expected = ['startTest', 'setUp', 'test', 'tearDown', + 'addError', 'addError', 'addFailure', 'addFailure', + 'stopTest'] + Foo(events).run(result) + self.assertEqual(events, expected) + + def test_run_call_order__subtests_failfast(self): + events = [] + result = LoggingResult(events) + result.failfast = True + + class Foo(Test.LoggingTestCase): + def test(self): + super(Foo, self).test() + with self.subTest(i=1): + self.fail('failure') + with self.subTest(i=2): + self.fail('failure') + self.fail('failure') + + expected = ['startTest', 'setUp', 'test', 'tearDown', + 'addFailure', 'stopTest'] + Foo(events).run(result) + self.assertEqual(events, expected) + # "This class attribute gives the exception raised by the test() method. # If a test framework needs to use a specialized exception, possibly to # carry additional information, it must subclass this exception in diff --git a/Lib/unittest/test/test_result.py b/Lib/unittest/test/test_result.py --- a/Lib/unittest/test/test_result.py +++ b/Lib/unittest/test/test_result.py @@ -234,6 +234,31 @@ class Test_TestResult(unittest.TestCase) 'testGetDescriptionWithoutDocstring (' + __name__ + '.Test_TestResult)') + def testGetSubTestDescriptionWithoutDocstring(self): + with self.subTest(foo=1, bar=2): + result = unittest.TextTestResult(None, True, 1) + self.assertEqual( + result.getDescription(self._subtest), + 'testGetSubTestDescriptionWithoutDocstring (' + __name__ + + '.Test_TestResult) (bar=2, foo=1)') + + def testGetSubTestDescriptionWithoutDocstringAndParams(self): + with self.subTest(): + result = unittest.TextTestResult(None, True, 1) + self.assertEqual( + result.getDescription(self._subtest), + 'testGetSubTestDescriptionWithoutDocstringAndParams ' + '(' + __name__ + '.Test_TestResult) ()') + + def testGetNestedSubTestDescriptionWithoutDocstring(self): + with self.subTest(foo=1): + with self.subTest(bar=2): + result = unittest.TextTestResult(None, True, 1) + self.assertEqual( + result.getDescription(self._subtest), + 'testGetNestedSubTestDescriptionWithoutDocstring ' + '(' + __name__ + '.Test_TestResult) (bar=2, foo=1)') + @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def testGetDescriptionWithOneLineDocstring(self): @@ -247,6 +272,18 @@ class Test_TestResult(unittest.TestCase) @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") + def testGetSubTestDescriptionWithOneLineDocstring(self): + """Tests getDescription() for a method with a docstring.""" + result = unittest.TextTestResult(None, True, 1) + with self.subTest(foo=1, bar=2): + self.assertEqual( + result.getDescription(self._subtest), + ('testGetSubTestDescriptionWithOneLineDocstring ' + '(' + __name__ + '.Test_TestResult) (bar=2, foo=1)\n' + 'Tests getDescription() for a method with a docstring.')) + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") def testGetDescriptionWithMultiLineDocstring(self): """Tests getDescription() for a method with a longer docstring. The second line of the docstring. @@ -259,6 +296,21 @@ class Test_TestResult(unittest.TestCase) 'Tests getDescription() for a method with a longer ' 'docstring.')) + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def testGetSubTestDescriptionWithMultiLineDocstring(self): + """Tests getDescription() for a method with a longer docstring. + The second line of the docstring. + """ + result = unittest.TextTestResult(None, True, 1) + with self.subTest(foo=1, bar=2): + self.assertEqual( + result.getDescription(self._subtest), + ('testGetSubTestDescriptionWithMultiLineDocstring ' + '(' + __name__ + '.Test_TestResult) (bar=2, foo=1)\n' + 'Tests getDescription() for a method with a longer ' + 'docstring.')) + def testStackFrameTrimming(self): class Frame(object): class tb_frame(object): diff --git a/Lib/unittest/test/test_runner.py b/Lib/unittest/test/test_runner.py --- a/Lib/unittest/test/test_runner.py +++ b/Lib/unittest/test/test_runner.py @@ -5,6 +5,7 @@ import pickle import subprocess import unittest +from unittest.case import _Outcome from .support import LoggingResult, ResultWithNoStartTestRunStopTestRun @@ -42,12 +43,8 @@ class TestCleanUp(unittest.TestCase): def testNothing(self): pass - class MockOutcome(object): - success = True - errors = [] - test = TestableTest('testNothing') - test._outcomeForDoCleanups = MockOutcome + outcome = test._outcome = _Outcome() exc1 = Exception('foo') exc2 = Exception('bar') @@ -61,9 +58,10 @@ class TestCleanUp(unittest.TestCase): test.addCleanup(cleanup2) self.assertFalse(test.doCleanups()) - self.assertFalse(MockOutcome.success) + self.assertFalse(outcome.success) - (Type1, instance1, _), (Type2, instance2, _) = reversed(MockOutcome.errors) + ((_, (Type1, instance1, _)), + (_, (Type2, instance2, _))) = reversed(outcome.errors) self.assertEqual((Type1, instance1), (Exception, exc1)) self.assertEqual((Type2, instance2), (Exception, exc2))