diff -r 30530f6c24bb Doc/library/unittest.rst --- a/Doc/library/unittest.rst Mon Mar 18 10:10:08 2013 -0700 +++ b/Doc/library/unittest.rst Mon Mar 18 15:26:52 2013 -0700 @@ -1512,6 +1512,10 @@ *start_dir* can be a dotted module name as well as a directory. + Paths are sorted before being imported to ensure execution order for a + given test suite is the same even if the underlying file system's ordering + is not dependent on file name like in ext3/4. + .. versionadded:: 3.2 .. versionchanged:: 3.4 diff -r 30530f6c24bb Lib/unittest/loader.py --- a/Lib/unittest/loader.py Mon Mar 18 10:10:08 2013 -0700 +++ b/Lib/unittest/loader.py Mon Mar 18 15:26:52 2013 -0700 @@ -177,6 +177,9 @@ The pattern is deliberately not stored as a loader attribute so that packages can continue discovery themselves. top_level_dir is stored so load_tests does not need to pass this argument in to loader.discover(). + + Paths are sorted before being imported to ensure reproducible execution + order even on filesystems with non-alphabetical ordering like ext3/4. """ set_implicit_top = False if top_level_dir is None and self._top_level_dir is not None: @@ -253,7 +256,7 @@ def _find_tests(self, start_dir, pattern): """Used by discovery. Yields test suites it loads.""" - paths = os.listdir(start_dir) + paths = sorted(os.listdir(start_dir)) for path in paths: full_path = os.path.join(start_dir, path) diff -r 30530f6c24bb Lib/unittest/test/test_discovery.py --- a/Lib/unittest/test/test_discovery.py Mon Mar 18 10:10:08 2013 -0700 +++ b/Lib/unittest/test/test_discovery.py Mon Mar 18 15:26:52 2013 -0700 @@ -46,9 +46,9 @@ def restore_isdir(): os.path.isdir = original_isdir - path_lists = [['test1.py', 'test2.py', 'not_a_test.py', 'test_dir', + path_lists = [['test2.py', 'test1.py', 'not_a_test.py', 'test_dir', 'test.foo', 'test-not-a-module.py', 'another_dir'], - ['test3.py', 'test4.py', ]] + ['test4.py', 'test3.py', ]] os.listdir = lambda path: path_lists.pop(0) self.addCleanup(restore_listdir) @@ -70,6 +70,8 @@ loader._top_level_dir = top_level suite = list(loader._find_tests(top_level, 'test*.py')) + # The test suites found should be sorted alphabetically for reliable + # execution order. expected = [name + ' module tests' for name in ('test1', 'test2')] expected.extend([('test_dir.%s' % name) + ' module tests' for name in @@ -132,6 +134,7 @@ # and directly from the test_directory2 package self.assertEqual(suite, ['load_tests', 'test_directory2' + ' module tests']) + # The test modules paths should be sorted for reliable execution order self.assertEqual(Module.paths, ['test_directory', 'test_directory2']) # load_tests should have been called once with loader, tests and pattern