diff -r 0e8743a6924e Lib/shutil.py --- a/Lib/shutil.py Sat Oct 31 12:23:03 2015 +0000 +++ b/Lib/shutil.py Sat Oct 31 17:10:08 2015 +0100 @@ -113,7 +113,44 @@ else: with open(src, 'rb') as fsrc: with open(dst, 'wb') as fdst: - copyfileobj(fsrc, fdst) + done = False + if hasattr(os, 'sendfile'): + # Prefer os.sendfile if available for performance + max_bcount = 2 ** 31 - 1 + bcount = max_bcount + offset = 0 + fdstno = fdst.fileno() + fsrcno = fsrc.fileno() + err_codes = set() + for err_name in ('EINVAL', 'ENOSYS', 'ENOTSUP', 'EBADF', + 'ENOTSOCK', 'EOPNOTSUPP'): + try: + err_code = getattr(errno, err_name) + except AttributeError: + pass + else: + err_codes.add(err_code) + try: + while bcount > 0: + bcount = os.sendfile(fdstno, + fsrcno, + offset, + max_bcount) + offset += bcount + except OSError as e: + if e.errno in err_codes: + # sendfile is not supported or does not support + # classic files (only sockets) + pass + else: + raise + else: + # Copy with os.sendfile succeeded + done = True + if not done: + # sendfile is not available or failed, fallback + # to copyfileobj + copyfileobj(fsrc, fdst) return dst def copymode(src, dst, *, follow_symlinks=True): diff -r 0e8743a6924e Lib/test/test_shutil.py --- a/Lib/test/test_shutil.py Sat Oct 31 12:23:03 2015 +0000 +++ b/Lib/test/test_shutil.py Sat Oct 31 17:10:08 2015 +0100 @@ -606,6 +606,39 @@ shutil.copyfile(link, dst) self.assertFalse(os.path.islink(dst)) + @unittest.skipUnless(support.is_resource_enabled('largefile'), + 'test requires >4GB file support') + def test_copyfile_hugefile(self): + tmp_dir = self.mkdtemp() + src_path = os.path.join(tmp_dir, "src") + # Create a > 4GB file with non trivial content + chunk_size = 2 ** 14 + with open(src_path, "wb") as src_file: + for offset in range(0, 2 ** 32 + chunk_size + 1, chunk_size): + # Generate and write a unique data chunk + chunk = str(offset).encode("ascii") + chunk = chunk + b"\x00" * (chunk_size - len(chunk)) + src_file.write(chunk) + dst_path = os.path.join(tmp_dir, "dst") + try: + shutil.copyfile(src_path, dst_path) + except OSError as e: + if e.errno != getattr(errno, 'ENOSPC', -1): + raise + self.skipTest("not enough space left") + except OverflowError: + self.skipTest("no large file support, or not enough space left") + self.assertTrue(os.path.isfile(dst_path)) + # Compare files + with open(src_path, "rb") as src_file, \ + open(dst_path, "rb") as dst_file: + while True: + src_chunk = src_file.read(chunk_size) + dst_chunk = dst_file.read(chunk_size) + self.assertEqual(src_chunk, dst_chunk) + if not src_chunk: + break + def test_rmtree_uses_safe_fd_version_if_available(self): _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <= os.supports_dir_fd and