# HG changeset patch # Parent 671894ae19a24ac66312408e9258c3ca8b865574 diff -r 671894ae19a2 Lib/share.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/share.py Tue Jul 03 20:02:52 2012 +0100 @@ -0,0 +1,133 @@ +''' +Replacements for os.open and io.open which allow renaming and unlinking +of the file before the file is closed. + +On Windows io.open() and os.open() (without the os.O_TEMPORARY flag) +invoke the Win32 function CreateFile() using the share mode +FILE_SHARE_READ | FILE_SHARE_WRITE. This means that the file can be +opened multiple times with the same functions for reading and writing +at the same time. However, until all the file descriptors are closed, +the file cannot be moved or unlinked. + +These replacements use the share mode FILE_SHARE_READ | +FILE_SHARE_WRITE | FILE_SHARE_DELETE, allowing the file to be moved or +unlinked while file decriptors for the file are still open, without +the file descriptors becoming invalid. This allows Unix-like +behaviour. + +Note that if a file is currently opened with full share mode, then it +must be reopened with full share mode. Also a temporary file created +using the tempfile module can only be reopened on Windows using full +share mode. + +On non-Windows platforms os.open and io.open are used unchanged. +''' + +import sys + +__all__ = ['open', 'os_open'] + +if sys.platform != 'win32': + from os import open as os_open + from _io import open + +else: + import os + import msvcrt + import warnings + import _winapi + import _io + + CREATE_NEW = 1 + CREATE_ALWAYS = 2 + OPEN_EXISTING = 3 + OPEN_ALWAYS = 4 + TRUNCATE_EXISTING = 5 + FILE_SHARE_READ = 0x00000001 + FILE_SHARE_WRITE = 0x00000002 + FILE_SHARE_DELETE = 0x00000004 + FILE_SHARE_VALID_FLAGS = 0x00000007 + FILE_ATTRIBUTE_READONLY = 0x00000001 + FILE_ATTRIBUTE_NORMAL = 0x00000080 + FILE_ATTRIBUTE_TEMPORARY = 0x00000100 + FILE_FLAG_DELETE_ON_CLOSE = 0x04000000 + FILE_FLAG_SEQUENTIAL_SCAN = 0x08000000 + FILE_FLAG_RANDOM_ACCESS = 0x10000000 + GENERIC_READ = 0x80000000 + GENERIC_WRITE = 0x40000000 + DELETE = 0x00010000 + NULL = 0 + + _ACCESS_MASK = os.O_RDONLY | os.O_WRONLY | os.O_RDWR + _ACCESS_MAP = {os.O_RDONLY : GENERIC_READ, + os.O_WRONLY : GENERIC_WRITE, + os.O_RDWR : GENERIC_READ | GENERIC_WRITE} + + _CREATE_MASK = os.O_CREAT | os.O_EXCL | os.O_TRUNC + _CREATE_MAP = {0 : OPEN_EXISTING, + os.O_EXCL : OPEN_EXISTING, + os.O_CREAT : OPEN_ALWAYS, + os.O_CREAT | os.O_EXCL : CREATE_NEW, + os.O_CREAT | os.O_TRUNC | os.O_EXCL : CREATE_NEW, + os.O_TRUNC : TRUNCATE_EXISTING, + os.O_TRUNC | os.O_EXCL : TRUNCATE_EXISTING, + os.O_CREAT | os.O_TRUNC : CREATE_ALWAYS} + + + def os_open(file, flags, mode=0o777, + *, share_flags=FILE_SHARE_VALID_FLAGS): + ''' + Replacement for os.open() allowing moving or unlinking before closing + ''' + if not isinstance(flags, int) and mode >= 0: + raise ValueError('bad flags: %r' % flags) + + if not isinstance(mode, int) and mode >= 0: + raise ValueError('bad mode: %r' % mode) + + if share_flags & ~FILE_SHARE_VALID_FLAGS: + raise ValueError('bad share_flags: %r' % share_flags) + + if isinstance(file, bytes): + warnings.warn('The Windows bytes API has been deprecated', + DeprecationWarning) + file = os.fsdecode(file) + + access_flags = _ACCESS_MAP[flags & _ACCESS_MASK] + create_flags = _CREATE_MAP[flags & _CREATE_MASK] + attrib_flags = FILE_ATTRIBUTE_NORMAL + + if flags & os.O_CREAT and mode & ~0o444 == 0: + attrib_flags = FILE_ATTRIBUTE_READONLY + + if flags & os.O_TEMPORARY: + share_flags |= FILE_SHARE_DELETE + attrib_flags |= FILE_FLAG_DELETE_ON_CLOSE + access_flags |= DELETE + + if flags & os.O_SHORT_LIVED: + attrib_flags |= FILE_ATTRIBUTE_TEMPORARY + + if flags & os.O_SEQUENTIAL: + attrib_flags |= FILE_FLAG_SEQUENTIAL_SCAN + + if flags & os.O_RANDOM: + attrib_flags |= FILE_FLAG_RANDOM_ACCESS + + h = _winapi.CreateFile(file, access_flags, share_flags, NULL, + create_flags, attrib_flags, NULL) + return msvcrt.open_osfhandle(h, flags | os.O_NOINHERIT) + + + def open(file, mode='r', buffering=-1, encoding=None, + errors=None, newline=None, closefd=True, opener=None, + *, share_flags=FILE_SHARE_VALID_FLAGS): + ''' + Replacement for io.open() allowing moving or unlinking before closing + ''' + if opener is None: + def opener(file, flags, mode=0o777): + return os_open(file, flags, mode, share_flags=share_flags) + return _io.open(file, mode=mode, buffering=buffering, + encoding=encoding, errors=errors, newline=newline, + closefd=closefd, opener=opener) diff -r 671894ae19a2 Lib/test/test_share.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/test/test_share.py Tue Jul 03 20:02:52 2012 +0100 @@ -0,0 +1,186 @@ +# +# To run the full regression test suite with os.open, +# builtins.open, io.open replaced you can do +# +# python -m test.test_share --regrtest +# +# To print traces of calls to the monkey patched functions to stderr do +# +# python -m test.test_share --regrtest --trace +# +# (This breaks a couple of unittests which check sys.stderr.) The +# usual arguments and options for test/regrtest.py are recognized, so +# +# python -m test.test_share --regrtest -v -n test_sys +# +# is equivalent to +# +# python -m test -v -n test_sys +# + +import unittest +import share +import os +import io +import builtins +import sys +import tempfile + +from test import support, regrtest + + +try: + O_BINARY = os.O_BINARY +except AttributeError: + O_BINARY = 0 + + +def readall(fd): + buf = [] + while True: + s = os.read(fd, 256) + if not s: + return b''.join(buf) + buf.append(s) + + +def writeall(fd, s): + while s: + n = os.write(fd, s) + s = s[n:] + + +class TestShare(unittest.TestCase): + + def test_open(self): + self._test_open(support.TESTFN, support.TESTFN + '_new') + + def test_open_unicode(self): + self._test_open(support.TESTFN_UNICODE, + support.TESTFN_UNICODE + '_new') + + def _test_open(self, fname1, fname2): + DATA1 = 'hffjkdfd' + DATA2 = 'dfdgshlfsd' + DATA3 = 'ghfyfkufkg' + + with share.open(fname1, 'w+') as f: + f.write(DATA1) + f.flush() + + os.rename(fname1, fname2) + self.assertFalse(os.path.exists(fname1)) + self.assertTrue(os.path.exists(fname2)) + + f.write(DATA2) + f.flush() + + f.seek(0) + self.assertEqual(f.read(), DATA1 + DATA2) + + with share.open(fname2, 'a+') as f: + os.unlink(fname2) + self.assertFalse(os.path.exists(fname2)) + + f.write(DATA3) + f.flush() + + f.seek(0) + self.assertEqual(f.read(), DATA1 + DATA2 + DATA3) + + def test_os_open(self): + self._test_os_open(support.TESTFN, support.TESTFN + '_new') + + def test_os_open_unicode(self): + self._test_os_open(support.TESTFN_UNICODE, + support.TESTFN_UNICODE + '_new') + + def _test_os_open(self, fname1, fname2): + DATA1 = b'sadfdsaf' + DATA2 = b'ashkahlfjdsafd' + DATA3 = b'689698769697' + + fd = share.os_open(fname1, os.O_CREAT | os.O_TRUNC | os.O_RDWR | + O_BINARY) + try: + writeall(fd, DATA1) + + os.rename(fname1, fname2) + self.assertFalse(os.path.exists(fname1)) + self.assertTrue(os.path.exists(fname2)) + + writeall(fd, DATA2) + + os.lseek(fd, 0, 0) + self.assertEqual(readall(fd), DATA1 + DATA2) + finally: + os.close(fd) + + fd = share.os_open(fname2, os.O_RDWR | os.O_APPEND | O_BINARY) + try: + os.unlink(fname2) + self.assertFalse(os.path.exists(fname2)) + + writeall(fd, DATA3) + + os.lseek(fd, 0, 0) + self.assertEqual(readall(fd), DATA1 + DATA2 + DATA3) + finally: + os.close(fd) + + def test_reopen_tempfile(self): + DATA = b'hlkhdalfhladgfljw' + fd = None + try: + with tempfile.NamedTemporaryFile() as f: + f.write(DATA) + f.flush() + + with share.open(f.name, 'rb') as f2: + self.assertEqual(f2.read(), DATA) + + fd = share.os_open(f.name, os.O_RDONLY | O_BINARY) + + self.assertFalse(os.path.exists(f.name)) + self.assertEqual(readall(fd), DATA) + finally: + if fd is not None: + os.close(fd) + + +class FunctionWrapper: + def __init__(self, func, trace): + self._func = func + self._trace = trace + self.__name__ = func.__name__ + + def __call__(self, file, *args, **kwds): + if self._trace: + tmp = [str((file,) + args)[1:-1].rstrip(',')] + tmp.extend('%s=%r' % it for it in kwds.items()) + print('%s(%s)' % (self.__name__, ', '.join(tmp)), file=sys.stderr) + try: + return self._func(file, *args, **kwds) + except OSError as e: + e.filename = file + raise e + + +if __name__ == '__main__': + if '--regrtest' in sys.argv: + sys.argv.remove('--regrtest') + if '--trace' in sys.argv: + sys.argv.remove('--trace') + trace = True + else: + trace = False + print('MONKEY PATCHING os.open, builtins.open, io.open') + os.open = FunctionWrapper(share.os_open, trace) + builtins.open = io.open = FunctionWrapper(share.open, trace) + TEMPDIR, TESTCWD = regrtest._make_temp_dir_for_build(regrtest.TEMPDIR) + regrtest.TEMPDIR = TEMPDIR + regrtest.TESTCWD = TESTCWD + with support.temp_cwd(TESTCWD, quiet=True): + regrtest.main() + else: + unittest.main() diff -r 671894ae19a2 Modules/_winapi.c --- a/Modules/_winapi.c Mon Jul 02 13:29:57 2012 -0700 +++ b/Modules/_winapi.c Tue Jul 03 20:02:52 2012 +0100 @@ -349,7 +349,7 @@ static PyObject * winapi_CreateFile(PyObject *self, PyObject *args) { - LPCTSTR lpFileName; + LPWSTR lpFileName; DWORD dwDesiredAccess; DWORD dwShareMode; LPSECURITY_ATTRIBUTES lpSecurityAttributes; @@ -358,7 +358,7 @@ HANDLE hTemplateFile; HANDLE handle; - if (!PyArg_ParseTuple(args, "s" F_DWORD F_DWORD F_POINTER + if (!PyArg_ParseTuple(args, "u" F_DWORD F_DWORD F_POINTER F_DWORD F_DWORD F_HANDLE, &lpFileName, &dwDesiredAccess, &dwShareMode, &lpSecurityAttributes, &dwCreationDisposition, @@ -366,10 +366,10 @@ return NULL; Py_BEGIN_ALLOW_THREADS - handle = CreateFile(lpFileName, dwDesiredAccess, - dwShareMode, lpSecurityAttributes, - dwCreationDisposition, - dwFlagsAndAttributes, hTemplateFile); + handle = CreateFileW(lpFileName, dwDesiredAccess, + dwShareMode, lpSecurityAttributes, + dwCreationDisposition, + dwFlagsAndAttributes, hTemplateFile); Py_END_ALLOW_THREADS if (handle == INVALID_HANDLE_VALUE)