diff -r d9a881b0d6ca Lib/doctest.py --- a/Lib/doctest.py Tue Jul 24 21:23:53 2012 +0200 +++ b/Lib/doctest.py Tue Jul 24 22:03:33 2012 -0400 @@ -209,6 +209,7 @@ raise TypeError("Expected a module, string, or None") def _load_testfile(filename, package, module_relative, encoding): + file_contents = None if module_relative: package = _normalize_module(package, 3) filename = _module_relative_path(package, filename) @@ -217,10 +218,19 @@ file_contents = package.__loader__.get_data(filename) file_contents = file_contents.decode(encoding) # get_data() opens files as 'rb', so one must do the equivalent - # conversion as universal newlines would do. - return file_contents.replace(os.linesep, '\n'), filename - with open(filename, encoding=encoding) as f: - return f.read(), filename + # conversion as universal newlines would do -- see below + if file_contents is None: + # Universal newline support is the default for open + with open(filename, encoding=encoding) as f: + file_contents = f.read() + # But we still have to check to see if universal newlines are enabled + if hasattr(f, 'newlines'): + return file_contents, filename + # If we get here, we need to do the universal newline conversion ourselves; + # we have two possible cases and we need to do them in the right order + for linesep in ('\r\n', '\r'): + file_contents = file_contents.replace(linesep, '\n') + return file_contents, filename def _indent(s, indent=4): """ diff -r d9a881b0d6ca Lib/test/doctest_testfile.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/test/doctest_testfile.py Tue Jul 24 22:03:33 2012 -0400 @@ -0,0 +1,141 @@ +""" +Test script to exercise all code paths of +_load_testfile in doctest.py + +To be called from test_doctest.py +""" + +import sys +import os +import doctest +import tempfile +from test import test_importhooks + +testpkgname = 'doctest_testfile_test' +hookpkgname = 'hooktestpackage' + +# This ensures we have our own unique tmp directory for testing +rootdir = tempfile.mkdtemp() + +testpkgdir = os.path.join(rootdir, testpkgname) +txtfile = "doctest_testfile.txt" +initpypath = os.path.join(testpkgdir, "__init__.py") +initpycpath = os.path.join(testpkgdir, "__init__.pyc") +pycachepath = os.path.join(testpkgdir, "__pycache__") +txtpath = os.path.join(testpkgdir, txtfile) + +# XXX (pdonis) Note: to set up this test, we are creating a +# fake package in a temporary directory and writing a txt file +# into it to be used as a doctest. We do this because the txt +# file needs to have intentionally mismatched newlines, and if +# we stored the file permanently in the Python distribution, +# there could be issues with making sure it is reproduced +# correctly across repository checkouts/commits, etc. Since the +# exact byte by byte content of the file is critical for this +# test, it's safer to just generate the temporary txt file so +# we can control its exact contents. + +class TestFixture: + def __init__(self): + # Create the testing directory + os.mkdir(testpkgdir, 0o700) + # Write txt file with intentionally mismatched newlines + txtfile_contents = b''.join([ + b'>>> print("Unix newlines OK.")\n', + b'Unix newlines OK.\n', + b'>>> print("Windows newlines OK.")\r\n', + b'Windows newlines OK.\r\n', + b'>>> print("Mac newlines OK.")\r', + b'Mac newlines OK.\r' + ]) + with open(txtpath, mode='wb') as f: + f.write(txtfile_contents) + # Make the testing directory importable as a package + # (must contain __init__.py and its parent must be on sys.path) + with open(initpypath, mode='w'): + pass # creates a 0-byte file + self.path = sys.path[:] + sys.path.append(rootdir) + + def cleanup(self): + sys.path[:] = self.path + os.remove(txtpath) + os.remove(initpypath) + # Don't forget to remove the compiled bytecode created during the test + if os.path.isfile(initpycpath): + os.remove(initpycpath) + elif os.path.isdir(pycachepath): + filenames = os.listdir(pycachepath) + for filename in filenames: + os.remove(os.path.join(pycachepath, filename)) + os.rmdir(pycachepath) + os.rmdir(testpkgdir) + os.rmdir(rootdir) + +# Helper classes for the package loader test, code cribbed +# shamelessly from test_importhooks +class PackageLoaderTestImporter(test_importhooks.PathImporter): + def __init__(self, path=''): + self.path = path + def get_data(self, path): + with open(os.path.join(self.path, path), mode='rb') as f: + return f.read() + +class InstallHook: + def __init__(self, pkgname): + self.pkgname = pkgname + self.meta_path = sys.meta_path[:] + self.path_hooks = sys.path_hooks[:] + sys.path_importer_cache.clear() + self.modules_before = sys.modules.copy() + self.importer = PackageLoaderTestImporter(path=testpkgdir) + sys.meta_path.append(self.importer) + + def remove(self): + sys.meta_path[:] = self.meta_path + sys.path_hooks[:] = self.path_hooks + sys.path_importer_cache.clear() + sys.modules.clear() + sys.modules.update(self.modules_before) + del self.importer + +def run(): + # Set up the test environment + fixture = TestFixture() + + # If there are any exceptions during testing, we want to + # make sure we clean up + try: + # XXX (pdonis) Technically we should include a test + # with no package argument to doctest.testfile, but: + # (1) The code path is the same for no package as for + # a package with no __loader__, so we aren't giving + # up any coverage as long as that remains true; + # (2) With no package, we can't control what directory + # doctest.testfile will look in for the file (at + # least, I can't see how to do it without major + # hackery to fool doctest's internal routines), + # so we can't depend on writing a temporary txt + # file to use as our doctest (see comments above + # for why we want to do the test that way). + + # First try a package with no __loader___ + doctest.testfile(txtfile, package=testpkgname) + doctest.master = None + + # Now try a package with a __loader___.get_data method + hook = InstallHook(hookpkgname) + doctest.testfile(txtfile, package=hookpkgname) + hook.remove() + del hook + doctest.master = None + + # Finally, try loading with module_relative=False and + # the full path to the txt file + doctest.testfile(txtpath, module_relative=False) + + finally: + fixture.cleanup() + +if __name__ == '__main__': + run() diff -r d9a881b0d6ca Lib/test/test_doctest.py --- a/Lib/test/test_doctest.py Tue Jul 24 21:23:53 2012 +0200 +++ b/Lib/test/test_doctest.py Tue Jul 24 22:03:33 2012 -0400 @@ -2482,6 +2482,13 @@ TestResults(failed=0, attempted=2) >>> doctest.master = None # Reset master. >>> sys.argv = save_argv + +We use a helper script to test for correct newline handling in +the _load_testfile function: + + >>> from test import doctest_testfile + >>> doctest_testfile.run() + >>> doctest.master = None # Reset master. """ def test_testmod(): r""" diff -r d9a881b0d6ca Misc/ACKS --- a/Misc/ACKS Tue Jul 24 21:23:53 2012 +0200 +++ b/Misc/ACKS Tue Jul 24 22:03:33 2012 -0400 @@ -262,6 +262,7 @@ Daniel Dittmar Josip Djolonga Jaromir Dolecek +Peter Donis Ismail Donmez Marcos Donolo Dima Dorfman