diff -r 32a4e7b337c9 Doc/library/zipfile.rst --- a/Doc/library/zipfile.rst Fri Jan 15 02:37:21 2016 +0000 +++ b/Doc/library/zipfile.rst Fri Jan 29 12:00:47 2016 +0000 @@ -208,15 +208,15 @@ .. index:: single: universal newlines; zipfile.ZipFile.open method -.. method:: ZipFile.open(name, mode='r', pwd=None) +.. method:: ZipFile.open(name, mode='r', pwd=None, force_zip64=False) - Extract a member from the archive as a file-like object (ZipExtFile). *name* + Access a member of the archive as a file-like object. *name* is the name of the file in the archive, or a :class:`ZipInfo` object. The *mode* parameter, if included, must be one of the following: ``'r'`` (the - default), ``'U'``, or ``'rU'``. Choosing ``'U'`` or ``'rU'`` will enable - :term:`universal newlines` support in the read-only object. *pwd* is the - password used for encrypted files. Calling :meth:`.open` on a closed - ZipFile will raise a :exc:`RuntimeError`. + default), ``'U'``, ``'rU'`` or ``'w'``. Choosing ``'U'`` or ``'rU'`` will + enable :term:`universal newlines` support in the read-only object. *pwd* is + the password used to decrypt encrypted zip files. Calling :meth:`.open` on + a closed ZipFile will raise a :exc:`RuntimeError`. :meth:`~ZipFile.open` is also a context manager and therefore supports the :keyword:`with` statement:: @@ -225,17 +225,23 @@ with myzip.open('eggs.txt') as myfile: print(myfile.read()) - .. note:: + With *mode* ``'r'``, ``'U'`` or ``'rU'``, the file-like object + (``ZipExtFile``) is read-only and provides the following methods: + :meth:`~io.BufferedIOBase.read`, :meth:`~io.IOBase.readline`, + :meth:`~io.IOBase.readlines`, :meth:`__iter__`, + :meth:`~iterator.__next__`. These objects can operate independently of the + ZipFile. - The file-like object is read-only and provides the following methods: - :meth:`~io.BufferedIOBase.read`, :meth:`~io.IOBase.readline`, - :meth:`~io.IOBase.readlines`, :meth:`__iter__`, - :meth:`~iterator.__next__`. + With ``mode='w'``, a writable file handle is returned, which supports the + :meth:`~io.BufferedIOBase.write` method. While a writable file handle is open, + attempting to read or write other files in the zip file will raise a + :exc:`RuntimeError`. - .. note:: - - Objects returned by :meth:`.open` can operate independently of the - ZipFile. + When writing a file, if the file size is not known in advance but may exceed + 2 GiB, pass ``force_zip64=True`` to ensure that it the header format is + capable of supporting large files. If the file size is known in advance, + construct a :class:`ZipInfo` object with :attr:`~ZipInfo.file_size` set, and + use that as the *name* parameter. .. note:: @@ -243,6 +249,10 @@ or a :class:`ZipInfo` object. You will appreciate this when trying to read a ZIP file that contains members with duplicate names. + .. versionchanged:: 3.6 + :meth:`open` can now be used to write files into the archive with the + ``mode='w'`` option. + .. deprecated-removed:: 3.4 3.6 The ``'U'`` or ``'rU'`` mode. Use :class:`io.TextIOWrapper` for reading compressed text files in :term:`universal newlines` mode. diff -r 32a4e7b337c9 Doc/whatsnew/3.6.rst --- a/Doc/whatsnew/3.6.rst Fri Jan 15 02:37:21 2016 +0000 +++ b/Doc/whatsnew/3.6.rst Fri Jan 29 12:00:47 2016 +0000 @@ -139,6 +139,15 @@ ``Request-rate`` extensions. (Contributed by Nikolay Bogoychev in :issue:`16099`.) +zipfile +------- + +:meth:`zipfile.ZipFile.open` can now be used to write data into a zip file, +as well as for extracting data. + +:class:`zipfile.ZipInfo` has a new classmethod :meth:`~zipfile.ZipInfo.from_file` +to construct it from a filesystem file, and a new read-only property +:attr:`~zipfile.ZipInfo.isdir`. Optimizations ============= diff -r 32a4e7b337c9 Lib/test/test_zipfile.py --- a/Lib/test/test_zipfile.py Fri Jan 15 02:37:21 2016 +0000 +++ b/Lib/test/test_zipfile.py Fri Jan 29 12:00:47 2016 +0000 @@ -61,6 +61,9 @@ zipfp.write(TESTFN, "another.name") zipfp.write(TESTFN, TESTFN) zipfp.writestr("strfile", self.data) + with zipfp.open('written-open-w', mode='w') as f: + for line in self.line_gen: + f.write(line) def zip_test(self, f, compression): self.make_test_archive(f, compression) @@ -76,7 +79,7 @@ zipfp.printdir(file=fp) directory = fp.getvalue() lines = directory.splitlines() - self.assertEqual(len(lines), 4) # Number of files + header + self.assertEqual(len(lines), 5) # Number of files + header self.assertIn('File Name', lines[0]) self.assertIn('Modified', lines[0]) @@ -90,23 +93,25 @@ # Check the namelist names = zipfp.namelist() - self.assertEqual(len(names), 3) + self.assertEqual(len(names), 4) self.assertIn(TESTFN, names) self.assertIn("another.name", names) self.assertIn("strfile", names) + self.assertIn("written-open-w", names) # Check infolist infos = zipfp.infolist() names = [i.filename for i in infos] - self.assertEqual(len(names), 3) + self.assertEqual(len(names), 4) self.assertIn(TESTFN, names) self.assertIn("another.name", names) self.assertIn("strfile", names) + self.assertIn("written-open-w", names) for i in infos: self.assertEqual(i.file_size, len(self.data)) # check getinfo - for nm in (TESTFN, "another.name", "strfile"): + for nm in (TESTFN, "another.name", "strfile", "written-open-w"): info = zipfp.getinfo(nm) self.assertEqual(info.filename, nm) self.assertEqual(info.file_size, len(self.data)) @@ -372,14 +377,17 @@ test_low_compression = None def zip_test_writestr_permissions(self, f, compression): - # Make sure that writestr creates files with mode 0600, - # when it is passed a name rather than a ZipInfo instance. + # Make sure that writestr and open(... mode='w') create files with mode 0600, + # when they are passed a name rather than a ZipInfo instance. self.make_test_archive(f, compression) with zipfile.ZipFile(f, "r") as zipfp: zinfo = zipfp.getinfo('strfile') self.assertEqual(zinfo.external_attr, 0o600 << 16) + zinfo2 = zipfp.getinfo('written-open-w') + self.assertEqual(zinfo2.external_attr, 0o600 << 16) + def test_writestr_permissions(self): for f in get_files(self): self.zip_test_writestr_permissions(f, zipfile.ZIP_STORED) @@ -1428,6 +1436,31 @@ # testzip returns the name of the first corrupt file, or None self.assertIsNone(zipf.testzip()) + def test_open_conflicting_handles(self): + # It's only possible to open one writable file handle at a time + with zipfile.ZipFile(TESTFN, "w", zipfile.ZIP_STORED) as zipf: + w1 = zipf.open('foo', mode='w') + with self.assertRaises(RuntimeError): + zipf.open('bar', mode='w') + with self.assertRaises(RuntimeError): + zipf.writestr('baz', 'abcde') + w1.close() + w2 = zipf.open('bar', mode='w') + w2.close() + + with zipfile.ZipFile(TESTFN, "w", zipfile.ZIP_STORED) as zipf: + zipf.writestr('foo.txt', "It's fun to charter an accountant!") + + # It's not possible to open write and read handles simultaneously + with zipfile.ZipFile(TESTFN, "a", zipfile.ZIP_STORED) as zipf: + r1 = zipf.open('foo.txt', mode='r') + with self.assertRaises(RuntimeError): + zipf.open('blah', mode='w') + r1.close() + w1 = zipf.open('blah', mode='w') + with self.assertRaises(RuntimeError): + zipf.open('foo.txt', mode='r') + def tearDown(self): unlink(TESTFN) unlink(TESTFN2) diff -r 32a4e7b337c9 Lib/zipfile.py --- a/Lib/zipfile.py Fri Jan 15 02:37:21 2016 +0000 +++ b/Lib/zipfile.py Fri Jan 29 12:00:47 2016 +0000 @@ -958,6 +958,82 @@ super().close() +class _ZipWriteFile: + def __init__(self, zf, zinfo, zip64): + self.zinfo = zinfo + self.zip64 = zip64 + self.zipfile = zf + self.compressor = _get_compressor(zinfo.compress_type) + self.file_size = 0 + self.compress_size = 0 + self.crc = 0 + + @property + def fileobj(self): + return self.zipfile.fp + + def write(self, data): + nbytes = len(data) + self.file_size += nbytes + self.crc = crc32(data, self.crc) + if self.compressor: + data = self.compressor.compress(data) + self.compress_size += len(data) + with self.zipfile._lock: + self.fileobj.write(data) + return nbytes + + def writelines(self, lines): + for l in lines: + self.write(l) + + def close(self): + # Flush any data from the compressor, and update header info + if self.compressor: + buf = self.compressor.flush() + self.compress_size += len(buf) + with self.zipfile._lock: + self.fileobj.write(buf) + self.zinfo.compress_size = self.compress_size + else: + self.zinfo.compress_size = self.file_size + self.zinfo.CRC = self.crc + self.zinfo.file_size = self.file_size + + # Write updated header info + if self.zinfo.flag_bits & 0x08: + # Write CRC and file sizes after the file data + fmt = ' ZIP64_LIMIT: + raise RuntimeError('File size unexpectedly exceeded ZIP64 limit') + if self.compress_size > ZIP64_LIMIT: + raise RuntimeError('Compressed size unexpectedly exceeded ZIP64 limit') + # Seek backwards and write file header (which will now include + # correct CRC and file sizes) + with self.zipfile._lock: + self.zipfile.start_dir = self.fileobj.tell() # Preserve current position in file + self.fileobj.seek(self.zinfo.header_offset) + self.fileobj.write(self.zinfo.FileHeader(self.zip64)) + self.fileobj.seek(self.zipfile.start_dir) + + self.zipfile._fpclose(self.fileobj) + + self.zipfile._open_writer = False + + # Context manager protocol: with zf.open(spam, 'w') as f: ... + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + class ZipFile: """ Class with methods to open, read, write, close, list zip files. @@ -1020,6 +1096,7 @@ self._fileRefCnt = 1 self._lock = threading.RLock() self._seekable = True + self._open_writer = False try: if mode == 'r': @@ -1232,28 +1309,104 @@ with self.open(name, "r", pwd) as fp: return fp.read() - def open(self, name, mode="r", pwd=None): - """Return file-like object for 'name'.""" - if mode not in ("r", "U", "rU"): - raise RuntimeError('open() requires mode "r", "U", or "rU"') + def _open_to_write(self, zinfo, force_zip64=False): + if force_zip64 and not self._allowZip64: + raise ValueError( + "force_zip64 is True, but allowZip64 was False when opening " + "the zip file." + ) + if self._fileRefCnt > 1: + raise RuntimeError("Can't write to the zip file while there is " + "another open handle on it (read or write). " + "Close the first handle before opening another.") + + # Sizes and CRC are overwritten with correct data after processing the file + if not hasattr(zinfo, 'file_size'): + zinfo.file_size = 0 + zinfo.compress_size = 0 + zinfo.CRC = 0 + + zinfo.flag_bits = 0x00 + if zinfo.compress_type == ZIP_LZMA: + # Compressed data includes an end-of-stream (EOS) marker + zinfo.flag_bits |= 0x02 + if not self._seekable: + zinfo.flag_bits |= 0x08 + + zinfo.external_attr = 0o600 << 16 # permissions: ?rw------- + + # Compressed size can be larger than uncompressed size + zip64 = self._allowZip64 and \ + (force_zip64 or zinfo.file_size * 1.05 > ZIP64_LIMIT) + + with self._lock: + self._writecheck(zinfo) + self._didModify = True + + if self._seekable: + self.fp.seek(self.start_dir) + zinfo.header_offset = self.fp.tell() + + self.fp.write(zinfo.FileHeader(zip64)) + + # Add file to our caches + self.filelist.append(zinfo) + self.NameToInfo[zinfo.filename] = zinfo + + self._fileRefCnt += 1 + self._open_writer = True + return _ZipWriteFile(self, zinfo, zip64) + + def open(self, name, mode="r", pwd=None, force_zip64=False): + """Return file-like object for 'name'. + + name is a string for the file name within the zip file, or a ZipInfo + object. + + mode should be 'r' to read a file already in the zip file, or 'w' to + write to a file newly added to the archive. + + pwd is the password to decrypt files (on read only) + + When writing, if the file size is not known in advance but may exceed + 2 GiB, pass force_zip64 to use the ZIP64 format, which can handle large + files. If the size is known in advance, it is best to pass a ZipInfo + instance for name, with zinfo.file_size set. + """ + if mode not in {"r", "w", "U", "rU"}: + raise RuntimeError('open() requires mode "r", "w", "U", or "rU"') if 'U' in mode: import warnings warnings.warn("'U' mode is deprecated", DeprecationWarning, 2) if pwd and not isinstance(pwd, bytes): raise TypeError("pwd: expected bytes, got %s" % type(pwd)) + if pwd and (mode == "w"): + raise ValueError("pwd is only supported for reading files") if not self.fp: raise RuntimeError( - "Attempt to read ZIP archive that was already closed") + "Attempt to use ZIP archive that was already closed") # Make sure we have an info object if isinstance(name, ZipInfo): # 'name' is already an info object zinfo = name + elif mode == 'w': + zinfo = ZipInfo(name) + zinfo.compress_type = self.compression else: # Get info object for name zinfo = self.getinfo(name) + if mode == 'w': + return self._open_to_write(zinfo, force_zip64=force_zip64) + + if self._open_writer: + raise RuntimeError("Can't open a reading file handle while there " + "an open writing handle on this zip file. " + "Close the writing handle before trying to read.") + + # Open for reading: self._fileRefCnt += 1 zef_file = _SharedFile(self.fp, zinfo.header_offset, self._fpclose, self._lock) try: @@ -1429,6 +1582,10 @@ if not self.fp: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") + if self._open_writer: + raise RuntimeError( + "Can't write to ZIP archive while an open writing handle exists." + ) st = os.stat(filename) isdir = stat.S_ISDIR(st.st_mode) @@ -1453,18 +1610,19 @@ zinfo.file_size = st.st_size zinfo.flag_bits = 0x00 - with self._lock: - if self._seekable: - self.fp.seek(self.start_dir) - zinfo.header_offset = self.fp.tell() # Start of header bytes - if zinfo.compress_type == ZIP_LZMA: - # Compressed data includes an end-of-stream (EOS) marker - zinfo.flag_bits |= 0x02 - self._writecheck(zinfo) - self._didModify = True + if isdir: + with self._lock: + if self._seekable: + self.fp.seek(self.start_dir) + zinfo.header_offset = self.fp.tell() # Start of header bytes + if zinfo.compress_type == ZIP_LZMA: + # Compressed data includes an end-of-stream (EOS) marker + zinfo.flag_bits |= 0x02 - if isdir: + self._writecheck(zinfo) + self._didModify = True + zinfo.file_size = 0 zinfo.compress_size = 0 zinfo.CRC = 0 @@ -1473,59 +1631,9 @@ self.NameToInfo[zinfo.filename] = zinfo self.fp.write(zinfo.FileHeader(False)) self.start_dir = self.fp.tell() - return - - cmpr = _get_compressor(zinfo.compress_type) - if not self._seekable: - zinfo.flag_bits |= 0x08 - with open(filename, "rb") as fp: - # Must overwrite CRC and sizes with correct data later - zinfo.CRC = CRC = 0 - zinfo.compress_size = compress_size = 0 - # Compressed size can be larger than uncompressed size - zip64 = self._allowZip64 and \ - zinfo.file_size * 1.05 > ZIP64_LIMIT - self.fp.write(zinfo.FileHeader(zip64)) - file_size = 0 - while 1: - buf = fp.read(1024 * 8) - if not buf: - break - file_size = file_size + len(buf) - CRC = crc32(buf, CRC) - if cmpr: - buf = cmpr.compress(buf) - compress_size = compress_size + len(buf) - self.fp.write(buf) - if cmpr: - buf = cmpr.flush() - compress_size = compress_size + len(buf) - self.fp.write(buf) - zinfo.compress_size = compress_size - else: - zinfo.compress_size = file_size - zinfo.CRC = CRC - zinfo.file_size = file_size - if zinfo.flag_bits & 0x08: - # Write CRC and file sizes after the file data - fmt = ' ZIP64_LIMIT: - raise RuntimeError('File size has increased during compressing') - if compress_size > ZIP64_LIMIT: - raise RuntimeError('Compressed size larger than uncompressed size') - # Seek backwards and write file header (which will now include - # correct CRC and file sizes) - self.start_dir = self.fp.tell() # Preserve current position in file - self.fp.seek(zinfo.header_offset) - self.fp.write(zinfo.FileHeader(zip64)) - self.fp.seek(self.start_dir) - self.filelist.append(zinfo) - self.NameToInfo[zinfo.filename] = zinfo + else: + with open(filename, "rb") as src, self.open(zinfo, 'w') as dest: + shutil.copyfileobj(src, dest, 1024*8) def writestr(self, zinfo_or_arcname, data, compress_type=None): """Write a file into the archive. The contents is 'data', which @@ -1550,6 +1658,10 @@ if not self.fp: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") + if self._open_writer: + raise RuntimeError( + "Can't write to ZIP archive while an open writing handle exists." + ) zinfo.file_size = len(data) # Uncompressed size with self._lock: