diff -r 0259c2c555fb Lib/shutil.py --- a/Lib/shutil.py Fri Dec 04 14:52:07 2015 -0800 +++ b/Lib/shutil.py Sat Dec 05 22:36:56 2015 +0100 @@ -87,6 +87,40 @@ return (os.path.normcase(os.path.abspath(src)) == os.path.normcase(os.path.abspath(dst))) +if hasattr(os, 'sendfile'): + # errnos sendfile can set if not supported on the system + _sendfile_err_codes = {code for code, name in errno.errorcode.items() \ + if name in ('EINVAL', 'ENOSYS', 'ENOTSUP', + 'EBADF', 'ENOTSOCK', 'EOPNOTSUPP')} + + def _copyfile_sendfile(fsrc, fdst): + """Copy data from fsrc to fdst using sendfile, return True if success. """ + status = False + max_bcount = 2 ** 31 - 1 + bcount = max_bcount + offset = 0 + fdstno = fdst.fileno() + fsrcno = fsrc.fileno() + + try: + while bcount > 0: + bcount = os.sendfile(fdstno, fsrcno, offset, max_bcount) + offset += bcount + status = True + except OSError as e: + if e.errno in _sendfile_err_codes: + # sendfile is not supported or does not support classic + # files (only sockets) + pass + else: + raise + + return status + +else: + def _copyfile_sendfile(*args, **kwargs): + return False + def copyfile(src, dst, *, follow_symlinks=True): """Copy data from src to dst. @@ -111,8 +145,10 @@ if not follow_symlinks and os.path.islink(src): os.symlink(os.readlink(src), dst) else: - with open(src, 'rb') as fsrc: - with open(dst, 'wb') as fdst: + with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst: + # Try to use os.sendfile if available for performance + if not _copyfile_sendfile(fsrc, fdst): + # sendfile is not available or failed, fallback to copyfileobj copyfileobj(fsrc, fdst) return dst diff -r 0259c2c555fb Lib/test/test_shutil.py --- a/Lib/test/test_shutil.py Fri Dec 04 14:52:07 2015 -0800 +++ b/Lib/test/test_shutil.py Sat Dec 05 22:36:56 2015 +0100 @@ -604,6 +604,39 @@ shutil.copyfile(link, dst) self.assertFalse(os.path.islink(dst)) + @unittest.skipUnless(support.is_resource_enabled('largefile'), + 'test requires >4GB file support') + def test_copyfile_hugefile(self): + tmp_dir = self.mkdtemp() + src_path = os.path.join(tmp_dir, "src") + # Create a > 4GB file with non trivial content + chunk_size = 2 ** 14 + with open(src_path, "wb") as src_file: + for offset in range(0, 2 ** 32 + chunk_size + 1, chunk_size): + # Generate and write a unique data chunk + chunk = str(offset).encode("ascii") + chunk = chunk + b"\x00" * (chunk_size - len(chunk)) + src_file.write(chunk) + dst_path = os.path.join(tmp_dir, "dst") + try: + shutil.copyfile(src_path, dst_path) + except OSError as e: + if e.errno != getattr(errno, 'ENOSPC', -1): + raise + self.skipTest("not enough space left") + except OverflowError: + self.skipTest("no large file support, or not enough space left") + self.assertTrue(os.path.isfile(dst_path)) + # Compare files + with open(src_path, "rb") as src_file, \ + open(dst_path, "rb") as dst_file: + while True: + src_chunk = src_file.read(chunk_size) + dst_chunk = dst_file.read(chunk_size) + self.assertEqual(src_chunk, dst_chunk) + if not src_chunk: + break + def test_rmtree_uses_safe_fd_version_if_available(self): _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <= os.supports_dir_fd and