diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index dcb2a16cff..177ba019c6 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -207,6 +207,18 @@ Directory and files operations :func:`copytree`\'s *ignore* argument, ignoring files and directories that match one of the glob-style *patterns* provided. See the example below. +.. function:: reflink(src, dst, fallback=None) + + Perform a lightweight copy of a file, where the data blocks are copied + only when modified. + This is also known as CoW (Copy on Write), instantaneous copy, or reflink, + and it's the same as "cp --reflink $src $dst" on Linux. + If the filesystem does not support CoW this function will raise + :exc:`ReflinkNotSupportedError`. + + Availability: Linux, macOS + + .. versionadded:: 3.9 .. function:: copytree(src, dst, symlinks=False, ignore=None, \ copy_function=copy2, ignore_dangling_symlinks=False, \ diff --git a/Lib/shutil.py b/Lib/shutil.py index 6486cd6e5d..e195c3d7c9 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -48,10 +48,16 @@ if os.name == 'posix': import posix elif _WINDOWS: import nt +try: + import fcntl +except ImportError: + fcntl = None COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 _USE_CP_SENDFILE = hasattr(os, "sendfile") and sys.platform.startswith("linux") _HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS +_HAS_FICLONE = fcntl and hasattr(fcntl, 'FICLONE') +_HAS_CLONEFILE = posix and hasattr(posix, '_clonefile') __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2", "copytree", "move", "rmtree", "Error", "SpecialFileError", @@ -83,6 +89,11 @@ class RegistryError(Exception): """Raised when a registry operation with the archiving and unpacking registries fails""" +class ReflinkNotSupportedError(Error): + """Raised by reflink() in case the file-system does not support + reflinks.""" + + class _GiveupOnFastCopy(Exception): """Raised as a signal to fallback on using raw read()/write() file copy when fast-copy functions fail to do so. @@ -200,6 +211,48 @@ def copyfileobj(fsrc, fdst, length=0): break fdst_write(buf) + +if _HAS_FICLONE or _HAS_CLONEFILE: + + def reflink(src, dst): + """Perform a lightweight copy of two files, where the data + blocks are copied only when modified. + + This is also known as CoW (Copy on Write), instantaneous copy + or reflink, and it's the same as "cp --reflink $src $dst" + on Linux. If the filesystem does not support CoW this function + will raise ReflinkNotSupportedError. + + Availability: Linux, macOS + """ + if _samefile(src, dst): + raise SameFileError( + "{!r} and {!r} are the same file".format(src, dst)) + + # Linux + if _HAS_FICLONE: + with open(src, "rb") as fsrc, open(dst, "wb") as fdst: + try: + fcntl.ioctl(fdst.fileno(), fcntl.FICLONE, fsrc.fileno()) + return dst + except EnvironmentError as err: + if err.errno in (errno.EBADF, errno.EOPNOTSUPP, + errno.ETXTBSY, errno.EXDEV): + raise ReflinkNotSupportedError + raise + # macOS + else: + try: + posix._clonefile(src, dst, posix._CLONE_NOOWNERCOPY) + return dst + except OSError as err: + if err.errno in (errno.ENOTSUP, errno.EXDEV): + raise ReflinkNotSupportedError + raise + + __all__.append('reflink') + + def _samefile(src, dst): # Macintosh, Unix. if isinstance(src, os.DirEntry) and hasattr(os.path, 'samestat'): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 208718bb12..8e73072af7 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -2419,6 +2419,36 @@ class TermsizeTests(unittest.TestCase): self.assertEqual(size.lines, 40) +@unittest.skipIf(not hasattr(shutil, "reflink"), "not supported") +class TestReflink(unittest.TestCase): + FILESIZE = 2 * 1024 * 1024 + + @classmethod + def setUpClass(cls): + write_test_file(TESTFN, cls.FILESIZE) + + @classmethod + def tearDownClass(cls): + support.unlink(TESTFN) + support.unlink(TESTFN2) + + def tearDown(self): + support.unlink(TESTFN2) + + def assert_files_eq(self, src, dst): + with open(src, 'rb') as fsrc: + with open(dst, 'rb') as fdst: + self.assertEqual(fsrc.read(), fdst.read()) + + def test_reflink(self): + try: + shutil.reflink(TESTFN, TESTFN2) + except shutil.ReflinkNotSupportedError: + self.skipTest('reflink not supported') + else: + self.assert_files_eq(TESTFN, TESTFN2) + + class PublicAPITests(unittest.TestCase): """Ensures that the correct values are exposed in the public API.""" @@ -2434,6 +2464,8 @@ class PublicAPITests(unittest.TestCase): 'get_terminal_size', 'SameFileError'] if hasattr(os, 'statvfs') or os.name == 'nt': target_api.append('disk_usage') + if hasattr(shutil, 'reflink'): + target_api.append('reflink') self.assertEqual(set(shutil.__all__), set(target_api)) diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 22cb94761d..cf775dde39 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -5047,6 +5047,64 @@ exit: #endif /* defined(__APPLE__) */ + +#if defined(__APPLE__) +#ifdef HAVE_CLONEFILE +PyDoc_STRVAR(os__clonefile__doc__, +"_clonefile($module, /, src, dst, flags)\n" +"--\n" +"\n" +"Create a reflink (macOS only)."); + +#define OS__CLONEFILE_METHODDEF \ + {"_clonefile", (PyCFunction)(void(*)(void))os__clonefile, METH_FASTCALL|METH_KEYWORDS, os__clonefile__doc__}, + +static PyObject * +os__clonefile_impl(PyObject *module, path_t *src, path_t *dst, int flags); + +static PyObject * +os__clonefile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"src", "dst", "flags", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "_clonefile", 0}; + PyObject *argsbuf[3]; + path_t src = PATH_T_INITIALIZE("_clonefile", "src", 0, 0); + path_t dst = PATH_T_INITIALIZE("_clonefile", "dst", 0, 0); + int flags; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + if (!path_converter(args[0], &src)) { + goto exit; + } + if (!path_converter(args[1], &dst)) { + goto exit; + } + if (PyFloat_Check(args[2])) { + PyErr_SetString(PyExc_TypeError, + "integer argument expected, got float" ); + goto exit; + } + flags = _PyLong_AsInt(args[2]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = os__clonefile_impl(module, &src, &dst, flags); + +exit: + /* Cleanup for src */ + path_cleanup(&src); + /* Cleanup for dst */ + path_cleanup(&dst); + + return return_value; +} +#endif /* defined(__APPLE__) */ +#endif /* defined(HAVE_CLONEFILE) */ + PyDoc_STRVAR(os_fstat__doc__, "fstat($module, /, fd)\n" "--\n" @@ -8741,4 +8799,4 @@ exit: #ifndef OS__REMOVE_DLL_DIRECTORY_METHODDEF #define OS__REMOVE_DLL_DIRECTORY_METHODDEF #endif /* !defined(OS__REMOVE_DLL_DIRECTORY_METHODDEF) */ -/*[clinic end generated code: output=b3ae8afd275ea5cd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=281b2b0bd3622707 input=a9049054013a1b77]*/ diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 0fbf7876c3..d011e53abd 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -15,6 +15,12 @@ #include #endif +#if defined __linux__ +#if !defined FICLONE +#define FICLONE _IOW(0x94, 9, int) +#endif +#endif + /*[clinic input] module fcntl [clinic start generated code]*/ @@ -628,6 +634,9 @@ all_ins(PyObject* m) if (PyModule_AddIntMacro(m, F_SEAL_SHRINK)) return -1; if (PyModule_AddIntMacro(m, F_SEAL_GROW)) return -1; if (PyModule_AddIntMacro(m, F_SEAL_WRITE)) return -1; +#endif +#ifdef FICLONE + if (PyModule_AddIntMacro(m, FICLONE)) return -1; #endif return 0; } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 8f6cffffcd..5b4b227460 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -20,6 +20,11 @@ # pragma weak statvfs # pragma weak fstatvfs +#if defined(MAC_OS_X_VERSION_10_12) +#include +#define HAVE_CLONEFILE +#endif + #endif /* __APPLE__ */ #define PY_SSIZE_T_CLEAN @@ -9109,6 +9114,32 @@ os__fcopyfile_impl(PyObject *module, int infd, int outfd, int flags) #endif +#ifdef HAVE_CLONEFILE +/*[clinic input] +os._clonefile + + src: path_t + dst: path_t + flags: int + +Create a reflink (macOS only). +[clinic start generated code]*/ + +static PyObject * +os__clonefile_impl(PyObject *module, path_t *src, path_t *dst, int flags) +/*[clinic end generated code: output=330df208859c712e input=08d9953339138fa4]*/ +{ + int ret; + + Py_BEGIN_ALLOW_THREADS + ret = clonefile(src, dst, flags); + Py_END_ALLOW_THREADS + if (ret < 0) + return PyErr_SetFromErrno(PyExc_OSError); + Py_RETURN_NONE; +} +#endif + /*[clinic input] os.fstat @@ -13516,6 +13547,9 @@ static PyMethodDef posix_methods[] = { OS_TIMES_METHODDEF OS__EXIT_METHODDEF OS__FCOPYFILE_METHODDEF +#ifdef HAVE_CLONEFILE + OS__CLONEFILE_METHODDEF +#endif OS_EXECV_METHODDEF OS_EXECVE_METHODDEF OS_SPAWNV_METHODDEF @@ -14164,6 +14198,10 @@ all_ins(PyObject *m) #if defined(__APPLE__) if (PyModule_AddIntConstant(m, "_COPYFILE_DATA", COPYFILE_DATA)) return -1; +#ifdef HAVE_CLONEFILE + if (PyModule_AddIntConstant(m, "_CLONE_NOFOLLOW", CLONE_NOFOLLOW)) return -1; + if (PyModule_AddIntConstant(m, "_CLONE_NOOWNERCOPY", CLONE_NOOWNERCOPY)) return -1; +#endif #endif #ifdef MS_WINDOWS