diff -r 3eca51115205 Doc/library/unittest.rst --- a/Doc/library/unittest.rst Tue Sep 09 09:48:02 2014 +1200 +++ b/Doc/library/unittest.rst Tue Sep 09 11:19:37 2014 +1200 @@ -1627,6 +1627,14 @@ The method optionally resolves *name* relative to the given *module*. + .. versionchanged:: 3.5 + If an :exc:`ImportError` or :exc:`AttributeError` occurs while traversing + *name* then synthetic test that raises that error when run will be + returned. Additionally as a non-fatal error the error will also be + accumulated to self.errors for introspection. These details are only + relevant to the authors of frameworks depending on / using the Python + :mod:`unittest` package. + .. method:: loadTestsFromNames(names, module=None) diff -r 3eca51115205 Lib/unittest/loader.py --- a/Lib/unittest/loader.py Tue Sep 09 09:48:02 2014 +1200 +++ b/Lib/unittest/loader.py Tue Sep 09 11:19:37 2014 +1200 @@ -127,20 +127,53 @@ The method optionally resolves the names relative to a given module. """ parts = name.split('.') + error_case, error_message = None, None if module is None: parts_copy = parts[:] + # Walk the dotted path from deepest to shallowest. + # If we had an Import error, and the next name out + # Issue 7559: we swallow ImportError exceptions + # making debugging hard. + # We've found a module when the import succeeds. If the import + # does not succeed at all, thats easy. If it succeeds but a child + # did not then we should report the child import failure, but only + # if the parent is a package. If the child was actually meant to be + # an attribute we should report it as an AttributeError. + # Last error so we can give it to the user if needed. while parts_copy: try: - module = __import__('.'.join(parts_copy)) + module_name = '.'.join(parts_copy) + module = __import__(module_name) break except ImportError: - del parts_copy[-1] + next_attribute = parts_copy.pop() + error_case, error_message = _make_failed_import_test( + next_attribute, self.suiteClass) if not parts_copy: - raise + # Nothing succeeded + self.errors.append(error_message) + return error_case parts = parts[1:] obj = module for part in parts: - parent, obj = obj, getattr(obj, part) + try: + parent, obj = obj, getattr(obj, part) + except AttributeError as e: + if (getattr(obj, '__path__', None) is not None + and error_case is not None): + # This is a package (no __path__ per importlib docs), and we + # encountered an error importing something. We cannot tell + # the difference between package.WrongNameTestClass and + # package.wrong_module_name so we just report the + # ImportError - it is more informative. + self.errors.append(error_message) + return error_case + error_case, error_message = _make_failed_test( + 'AttributeError', part, e, self.suiteClass, + 'Failed to access attribute:\n%s' % (traceback.format_exc(),)) + self.errors.append(error_message) + return error_case + if isinstance(obj, types.ModuleType): return self.loadTestsFromModule(obj) diff -r 3eca51115205 Lib/unittest/test/test_loader.py --- a/Lib/unittest/test/test_loader.py Tue Sep 09 09:48:02 2014 +1200 +++ b/Lib/unittest/test/test_loader.py Tue Sep 09 11:19:37 2014 +1200 @@ -381,14 +381,15 @@ loader = unittest.TestLoader() # XXX Should this raise ValueError or ImportError? - try: - loader.loadTestsFromName('abc () //') - except ValueError: - pass - except ImportError: - pass - else: - self.fail("TestLoader.loadTestsFromName failed to raise ValueError") + suite = loader.loadTestsFromName('abc () //') + error, test = self.check_deferred_error(loader, suite) + expected = "Failed to import test module: abc () //" + expected_regex = "Failed to import test module: abc \(\) //" + self.assertTrue( + expected in error, + 'missing error string in %r' % error) + self.assertRaisesRegex( + ImportError, expected_regex, getattr(test, 'abc () //')) # "The specifier name is a ``dotted name'' that may resolve ... to a # module" @@ -397,12 +398,13 @@ def test_loadTestsFromName__unknown_module_name(self): loader = unittest.TestLoader() - try: - loader.loadTestsFromName('sdasfasfasdf') - except ImportError as e: - self.assertEqual(str(e), "No module named 'sdasfasfasdf'") - else: - self.fail("TestLoader.loadTestsFromName failed to raise ImportError") + suite = loader.loadTestsFromName('sdasfasfasdf') + expected = "No module named 'sdasfasfasdf'" + error, test = self.check_deferred_error(loader, suite) + self.assertTrue( + expected in error, + 'missing error string in %r' % error) + self.assertRaisesRegex(ImportError, expected, test.sdasfasfasdf) # "The specifier name is a ``dotted name'' that may resolve either to # a module, a test case class, a TestSuite instance, a test method @@ -410,15 +412,33 @@ # TestCase or TestSuite instance." # # What happens when the module is found, but the attribute can't? - def test_loadTestsFromName__unknown_attr_name(self): + def test_loadTestsFromName__unknown_attr_name_on_module(self): loader = unittest.TestLoader() - try: - loader.loadTestsFromName('unittest.sdasfasfasdf') - except AttributeError as e: - self.assertEqual(str(e), "module 'unittest' has no attribute 'sdasfasfasdf'") - else: - self.fail("TestLoader.loadTestsFromName failed to raise AttributeError") + suite = loader.loadTestsFromName('unittest.loader.sdasfasfasdf') + expected = "module 'unittest.loader' has no attribute 'sdasfasfasdf'" + error, test = self.check_deferred_error(loader, suite) + self.assertTrue( + expected in error, + 'missing error string in %r' % error) + self.assertRaisesRegex(AttributeError, expected, test.sdasfasfasdf) + + # "The specifier name is a ``dotted name'' that may resolve either to + # a module, a test case class, a TestSuite instance, a test method + # within a test case class, or a callable object which returns a + # TestCase or TestSuite instance." + # + # What happens when the module is found, but the attribute can't? + def test_loadTestsFromName__unknown_attr_name_on_package(self): + loader = unittest.TestLoader() + + suite = loader.loadTestsFromName('unittest.sdasfasfasdf') + expected = "No module named 'unittest.sdasfasfasdf'" + error, test = self.check_deferred_error(loader, suite) + self.assertTrue( + expected in error, + 'missing error string in %r' % error) + self.assertRaisesRegex(ImportError, expected, test.sdasfasfasdf) # "The specifier name is a ``dotted name'' that may resolve either to # a module, a test case class, a TestSuite instance, a test method @@ -430,12 +450,13 @@ def test_loadTestsFromName__relative_unknown_name(self): loader = unittest.TestLoader() - try: - loader.loadTestsFromName('sdasfasfasdf', unittest) - except AttributeError as e: - self.assertEqual(str(e), "module 'unittest' has no attribute 'sdasfasfasdf'") - else: - self.fail("TestLoader.loadTestsFromName failed to raise AttributeError") + suite = loader.loadTestsFromName('sdasfasfasdf', unittest) + expected = "module 'unittest' has no attribute 'sdasfasfasdf'" + error, test = self.check_deferred_error(loader, suite) + self.assertTrue( + expected in error, + 'missing error string in %r' % error) + self.assertRaisesRegex(AttributeError, expected, test.sdasfasfasdf) # "The specifier name is a ``dotted name'' that may resolve either to # a module, a test case class, a TestSuite instance, a test method @@ -451,12 +472,13 @@ def test_loadTestsFromName__relative_empty_name(self): loader = unittest.TestLoader() - try: - loader.loadTestsFromName('', unittest) - except AttributeError as e: - pass - else: - self.fail("Failed to raise AttributeError") + suite = loader.loadTestsFromName('', unittest) + error, test = self.check_deferred_error(loader, suite) + expected = "has no attribute ''" + self.assertTrue( + expected in error, + 'missing error string in %r' % error) + self.assertRaisesRegex(AttributeError, expected, getattr(test, '')) # "The specifier name is a ``dotted name'' that may resolve either to # a module, a test case class, a TestSuite instance, a test method @@ -471,14 +493,15 @@ loader = unittest.TestLoader() # XXX Should this raise AttributeError or ValueError? - try: - loader.loadTestsFromName('abc () //', unittest) - except ValueError: - pass - except AttributeError: - pass - else: - self.fail("TestLoader.loadTestsFromName failed to raise ValueError") + suite = loader.loadTestsFromName('abc () //', unittest) + error, test = self.check_deferred_error(loader, suite) + expected = "module 'unittest' has no attribute 'abc () //'" + expected_regex = "module 'unittest' has no attribute 'abc \(\) //'" + self.assertTrue( + expected in error, + 'missing error string in %r' % error) + self.assertRaisesRegex( + AttributeError, expected_regex, getattr(test, 'abc () //')) # "The method optionally resolves name relative to the given module" # @@ -584,12 +607,13 @@ m.testcase_1 = MyTestCase loader = unittest.TestLoader() - try: - loader.loadTestsFromName('testcase_1.testfoo', m) - except AttributeError as e: - self.assertEqual(str(e), "type object 'MyTestCase' has no attribute 'testfoo'") - else: - self.fail("Failed to raise AttributeError") + suite = loader.loadTestsFromName('testcase_1.testfoo', m) + expected = "type object 'MyTestCase' has no attribute 'testfoo'" + error, test = self.check_deferred_error(loader, suite) + self.assertTrue( + expected in error, + 'missing error string in %r' % error) + self.assertRaisesRegex(AttributeError, expected, test.testfoo) # "The specifier name is a ``dotted name'' that may resolve ... to # ... a callable object which returns a ... TestSuite instance" @@ -707,6 +731,23 @@ ### Tests for TestLoader.loadTestsFromNames() ################################################################ + def check_deferred_error(self, loader, suite): + """Helper function for checking that errors in loading are reported. + + :param loader: A loader with some errors. + :param suite: A suite that should have a late bound error. + :return: The first error message from the loader and the test object + from the suite. + """ + self.assertIsInstance(suite, unittest.TestSuite) + self.assertEqual(suite.countTestCases(), 1) + # Errors loading the suite are also captured for introspection. + self.assertNotEqual([], loader.errors) + self.assertEqual(1, len(loader.errors)) + error = loader.errors[0] + test = list(suite)[0] + return error, test + # "Similar to loadTestsFromName(), but takes a sequence of names rather # than a single name." # @@ -759,14 +800,15 @@ loader = unittest.TestLoader() # XXX Should this raise ValueError or ImportError? - try: - loader.loadTestsFromNames(['abc () //']) - except ValueError: - pass - except ImportError: - pass - else: - self.fail("TestLoader.loadTestsFromNames failed to raise ValueError") + suite = loader.loadTestsFromNames(['abc () //']) + error, test = self.check_deferred_error(loader, list(suite)[0]) + expected = "Failed to import test module: abc () //" + expected_regex = "Failed to import test module: abc \(\) //" + self.assertTrue( + expected in error, + 'missing error string in %r' % error) + self.assertRaisesRegex( + ImportError, expected_regex, getattr(test, 'abc () //')) # "The specifier name is a ``dotted name'' that may resolve either to # a module, a test case class, a TestSuite instance, a test method @@ -777,12 +819,13 @@ def test_loadTestsFromNames__unknown_module_name(self): loader = unittest.TestLoader() - try: - loader.loadTestsFromNames(['sdasfasfasdf']) - except ImportError as e: - self.assertEqual(str(e), "No module named 'sdasfasfasdf'") - else: - self.fail("TestLoader.loadTestsFromNames failed to raise ImportError") + suite = loader.loadTestsFromNames(['sdasfasfasdf']) + error, test = self.check_deferred_error(loader, list(suite)[0]) + expected = "Failed to import test module: sdasfasfasdf" + self.assertTrue( + expected in error, + 'missing error string in %r' % error) + self.assertRaisesRegex(ImportError, expected, test.sdasfasfasdf) # "The specifier name is a ``dotted name'' that may resolve either to # a module, a test case class, a TestSuite instance, a test method @@ -793,12 +836,14 @@ def test_loadTestsFromNames__unknown_attr_name(self): loader = unittest.TestLoader() - try: - loader.loadTestsFromNames(['unittest.sdasfasfasdf', 'unittest']) - except AttributeError as e: - self.assertEqual(str(e), "module 'unittest' has no attribute 'sdasfasfasdf'") - else: - self.fail("TestLoader.loadTestsFromNames failed to raise AttributeError") + suite = loader.loadTestsFromNames( + ['unittest.loader.sdasfasfasdf', 'unittest']) + error, test = self.check_deferred_error(loader, list(suite)[0]) + expected = "module 'unittest.loader' has no attribute 'sdasfasfasdf'" + self.assertTrue( + expected in error, + 'missing error string in %r' % error) + self.assertRaisesRegex(AttributeError, expected, test.sdasfasfasdf) # "The specifier name is a ``dotted name'' that may resolve either to # a module, a test case class, a TestSuite instance, a test method @@ -812,12 +857,13 @@ def test_loadTestsFromNames__unknown_name_relative_1(self): loader = unittest.TestLoader() - try: - loader.loadTestsFromNames(['sdasfasfasdf'], unittest) - except AttributeError as e: - self.assertEqual(str(e), "module 'unittest' has no attribute 'sdasfasfasdf'") - else: - self.fail("TestLoader.loadTestsFromName failed to raise AttributeError") + suite = loader.loadTestsFromNames(['sdasfasfasdf'], unittest) + error, test = self.check_deferred_error(loader, list(suite)[0]) + expected = "module 'unittest' has no attribute 'sdasfasfasdf'" + self.assertTrue( + expected in error, + 'missing error string in %r' % error) + self.assertRaisesRegex(AttributeError, expected, test.sdasfasfasdf) # "The specifier name is a ``dotted name'' that may resolve either to # a module, a test case class, a TestSuite instance, a test method @@ -831,12 +877,13 @@ def test_loadTestsFromNames__unknown_name_relative_2(self): loader = unittest.TestLoader() - try: - loader.loadTestsFromNames(['TestCase', 'sdasfasfasdf'], unittest) - except AttributeError as e: - self.assertEqual(str(e), "module 'unittest' has no attribute 'sdasfasfasdf'") - else: - self.fail("TestLoader.loadTestsFromName failed to raise AttributeError") + suite = loader.loadTestsFromNames(['TestCase', 'sdasfasfasdf'], unittest) + error, test = self.check_deferred_error(loader, list(suite)[1]) + expected = "module 'unittest' has no attribute 'sdasfasfasdf'" + self.assertTrue( + expected in error, + 'missing error string in %r' % error) + self.assertRaisesRegex(AttributeError, expected, test.sdasfasfasdf) # "The specifier name is a ``dotted name'' that may resolve either to # a module, a test case class, a TestSuite instance, a test method @@ -852,12 +899,13 @@ def test_loadTestsFromNames__relative_empty_name(self): loader = unittest.TestLoader() - try: - loader.loadTestsFromNames([''], unittest) - except AttributeError: - pass - else: - self.fail("Failed to raise ValueError") + suite = loader.loadTestsFromNames([''], unittest) + error, test = self.check_deferred_error(loader, list(suite)[0]) + expected = "has no attribute ''" + self.assertTrue( + expected in error, + 'missing error string in %r' % error) + self.assertRaisesRegex(AttributeError, expected, getattr(test, '')) # "The specifier name is a ``dotted name'' that may resolve either to # a module, a test case class, a TestSuite instance, a test method @@ -871,14 +919,15 @@ loader = unittest.TestLoader() # XXX Should this raise AttributeError or ValueError? - try: - loader.loadTestsFromNames(['abc () //'], unittest) - except AttributeError: - pass - except ValueError: - pass - else: - self.fail("TestLoader.loadTestsFromNames failed to raise ValueError") + suite = loader.loadTestsFromNames(['abc () //'], unittest) + error, test = self.check_deferred_error(loader, list(suite)[0]) + expected = "module 'unittest' has no attribute 'abc () //'" + expected_regex = "module 'unittest' has no attribute 'abc \(\) //'" + self.assertTrue( + expected in error, + 'missing error string in %r' % error) + self.assertRaisesRegex( + AttributeError, expected_regex, getattr(test, 'abc () //')) # "The method optionally resolves name relative to the given module" # @@ -996,12 +1045,13 @@ m.testcase_1 = MyTestCase loader = unittest.TestLoader() - try: - loader.loadTestsFromNames(['testcase_1.testfoo'], m) - except AttributeError as e: - self.assertEqual(str(e), "type object 'MyTestCase' has no attribute 'testfoo'") - else: - self.fail("Failed to raise AttributeError") + suite = loader.loadTestsFromNames(['testcase_1.testfoo'], m) + error, test = self.check_deferred_error(loader, list(suite)[0]) + expected = "type object 'MyTestCase' has no attribute 'testfoo'" + self.assertTrue( + expected in error, + 'missing error string in %r' % error) + self.assertRaisesRegex(AttributeError, expected, test.testfoo) # "The specifier name is a ``dotted name'' that may resolve ... to # ... a callable object which returns a ... TestSuite instance"