diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -13,12 +13,12 @@ -------------- The :mod:`tarfile` module makes it possible to read and write tar -archives, including those using gzip or bz2 compression. +archives, including those using gzip, bz2 or lzma compression. (:file:`.zip` files can be read and written using the :mod:`zipfile` module.) Some facts and figures: -* reads and writes :mod:`gzip` and :mod:`bz2` compressed archives. +* reads and writes :mod:`gzip`, :mod:`bz2` and :mod:`lzma` compressed archives. * read/write support for the POSIX.1-1988 (ustar) format. @@ -55,6 +55,8 @@ +------------------+---------------------------------------------+ | ``'r:bz2'`` | Open for reading with bzip2 compression. | +------------------+---------------------------------------------+ + | ``'r:xz'`` | Open for reading with lzma compression. | + +------------------+---------------------------------------------+ | ``'a' or 'a:'`` | Open for appending with no compression. The | | | file is created if it does not exist. | +------------------+---------------------------------------------+ @@ -64,11 +66,13 @@ +------------------+---------------------------------------------+ | ``'w:bz2'`` | Open for bzip2 compressed writing. | +------------------+---------------------------------------------+ + | ``'w:xz'`` | Open for lzma compressed writing. | + +------------------+---------------------------------------------+ - Note that ``'a:gz'`` or ``'a:bz2'`` is not possible. If *mode* is not suitable - to open a certain (compressed) file for reading, :exc:`ReadError` is raised. Use - *mode* ``'r'`` to avoid this. If a compression method is not supported, - :exc:`CompressionError` is raised. + Note that ``'a:gz'``, ``'a:bz2'`` or ``'a:xz'`` is not possible. If *mode* + is not suitable to open a certain (compressed) file for reading, + :exc:`ReadError` is raised. Use *mode* ``'r'`` to avoid this. If a + compression method is not supported, :exc:`CompressionError` is raised. If *fileobj* is specified, it is used as an alternative to a :term:`file object` opened in binary mode for *name*. It is supposed to be at position 0. @@ -99,12 +103,18 @@ | ``'r|bz2'`` | Open a bzip2 compressed *stream* for | | | reading. | +-------------+--------------------------------------------+ + | ``'r|xz'`` | Open a lzma compressed *stream* for | + | | reading. | + +-------------+--------------------------------------------+ | ``'w|'`` | Open an uncompressed *stream* for writing. | +-------------+--------------------------------------------+ - | ``'w|gz'`` | Open an gzip compressed *stream* for | + | ``'w|gz'`` | Open a gzip compressed *stream* for | | | writing. | +-------------+--------------------------------------------+ - | ``'w|bz2'`` | Open an bzip2 compressed *stream* for | + | ``'w|bz2'`` | Open a bzip2 compressed *stream* for | + | | writing. | + +-------------+--------------------------------------------+ + | ``'w|xz'`` | Open an lzma compressed *stream* for | | | writing. | +-------------+--------------------------------------------+ diff --git a/Lib/tarfile.py b/Lib/tarfile.py --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -421,7 +421,7 @@ else: self._init_write_gz() - if comptype == "bz2": + elif comptype == "bz2": try: import bz2 except ImportError: @@ -431,6 +431,18 @@ self.cmp = bz2.BZ2Decompressor() else: self.cmp = bz2.BZ2Compressor() + + elif comptype == "xz": + try: + import lzma + except ImportError: + raise CompressionError("lzma module is not available") + if mode == "r": + self.dbuf = b"" + self.cmp = lzma.LZMADecompressor() + else: + self.cmp = lzma.LZMACompressor() + except: if not self._extfileobj: self.fileobj.close() @@ -582,7 +594,7 @@ break try: buf = self.cmp.decompress(buf) - except IOError: + except: # FIXME must catch zlib.error(gzip), IOError(bz2) and LZMAError(lzma) here. raise ReadError("invalid compressed data") self.dbuf += buf c += len(buf) @@ -620,10 +632,12 @@ return self.buf def getcomptype(self): - if self.buf.startswith(b"\037\213\010"): + if self.buf.startswith(b"\x1f\x8b\x08"): return "gz" - if self.buf.startswith(b"BZh91"): + elif self.buf.startswith(b"BZh91"): return "bz2" + elif self.buf.startswith(b"\x5d\x00\x00\x80") or self.buf.startswith(b"\xfd7zXZ"): + return "xz" return "tar" def close(self): @@ -1718,9 +1732,11 @@ 'r|' open an uncompressed stream of tar blocks for reading 'r|gz' open a gzip compressed stream of tar blocks 'r|bz2' open a bzip2 compressed stream of tar blocks + 'r|xz' open an lzma compressed stream of tar blocks 'w|' open an uncompressed stream for writing 'w|gz' open a gzip compressed stream for writing 'w|bz2' open a bzip2 compressed stream for writing + 'w|xz' open an lzma compressed stream for writing """ if not name and not fileobj: @@ -1840,11 +1856,39 @@ t._extfileobj = False return t + @classmethod + def lzmaopen(cls, name, mode="r", fileobj=None, preset=9, **kwargs): + """Open lzma compressed tar archive name for reading or writing. + Appending is not allowed. + """ + if len(mode) > 1 or mode not in "rw": + raise ValueError("mode must be 'r' or 'w'.") + + try: + import lzma + except ImportError: + raise CompressionError("lzma module is not available") + + if mode == "r": + # LZMAFile complains about a preset argument in read mode. + preset = None + + fileobj = lzma.LZMAFile(name, mode, preset=preset, fileobj=fileobj) + + try: + t = cls.taropen(name, mode, fileobj, **kwargs) + except lzma.LZMAError: + fileobj.close() + raise ReadError("not an lzma file") + t._extfileobj = False + return t + # All *open() methods are registered here. OPEN_METH = { "tar": "taropen", # uncompressed tar "gz": "gzopen", # gzip compressed tar - "bz2": "bz2open" # bzip2 compressed tar + "bz2": "bz2open", # bzip2 compressed tar + "xz": "lzmaopen" # lzma compressed tar } #-------------------------------------------------------------------------- diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -21,6 +21,10 @@ import bz2 except ImportError: bz2 = None +try: + import lzma +except ImportError: + lzma = None def md5sum(data): return md5(data).hexdigest() @@ -29,6 +33,7 @@ tarname = support.findfile("testtar.tar") gzipname = os.path.join(TEMPDIR, "testtar.tar.gz") bz2name = os.path.join(TEMPDIR, "testtar.tar.bz2") +xzname = os.path.join(TEMPDIR, "testtar.tar.xz") tmpname = os.path.join(TEMPDIR, "tmp.tar") md5_regtype = "65f477c818ad9e15f7feab0c6d37742f" @@ -201,6 +206,8 @@ _open = gzip.GzipFile elif self.mode.endswith(":bz2"): _open = bz2.BZ2File + elif self.mode.endswith(":xz"): + _open = lzma.LZMAFile else: _open = open @@ -262,6 +269,8 @@ _open = gzip.GzipFile elif self.mode.endswith(":bz2"): _open = bz2.BZ2File + elif self.mode.endswith(":xz"): + _open = lzma.LZMAFile else: _open = open fobj = _open(self.tarname, "rb") @@ -523,6 +532,18 @@ testfunc(bz2name, "r|*") testfunc(bz2name, "r|bz2") + if lzma: + self.assertRaises(tarfile.ReadError, tarfile.open, tarname, mode="r:xz") + self.assertRaises(tarfile.ReadError, tarfile.open, tarname, mode="r|xz") + self.assertRaises(tarfile.ReadError, tarfile.open, xzname, mode="r:") + self.assertRaises(tarfile.ReadError, tarfile.open, xzname, mode="r|") + + testfunc(xzname, "r") + testfunc(xzname, "r:*") + testfunc(xzname, "r:xz") + testfunc(xzname, "r|*") + testfunc(xzname, "r|xz") + def test_detect_file(self): self._test_modes(self._testfunc_file) @@ -1076,6 +1097,9 @@ data = dec.decompress(data) self.assertTrue(len(dec.unused_data) == 0, "found trailing data") + elif self.mode.endswith("xz"): + with lzma.LZMAFile(tmpname) as fobj: + data = fobj.read() else: with open(tmpname, "rb") as fobj: data = fobj.read() @@ -1490,6 +1514,12 @@ self._create_testtar("w:bz2") self.assertRaises(tarfile.ReadError, tarfile.open, tmpname, "a") + def test_append_lzma(self): + if lzma is None: + return + self._create_testtar("w:xz") + self.assertRaises(tarfile.ReadError, tarfile.open, tmpname, "a") + # Append mode is supposed to fail if the tarfile to append to # does not end with a zero block. def _test_error(self, data): @@ -1746,6 +1776,21 @@ self._test_partial_input("r:bz2") +class LzmaMiscReadTest(MiscReadTest): + tarname = xzname + mode = "r:xz" +class LzmaUstarReadTest(UstarReadTest): + tarname = xzname + mode = "r:xz" +class LzmaStreamReadTest(StreamReadTest): + tarname = xzname + mode = "r|xz" +class LzmaWriteTest(WriteTest): + mode = "w:xz" +class LzmaStreamWriteTest(StreamWriteTest): + mode = "w|xz" + + def test_main(): support.unlink(TEMPDIR) os.makedirs(TEMPDIR) @@ -1811,6 +1856,23 @@ Bz2PartialReadTest, ] + if lzma: + # Create testtar.tar.bz2 and add bz2-specific tests. + support.unlink(xzname) + tar = lzma.LZMAFile(xzname, "w") # FIXME lzma doesn't support "wb" + try: + tar.write(data) + finally: + tar.close() + + tests += [ + LzmaMiscReadTest, + LzmaUstarReadTest, + LzmaStreamReadTest, + LzmaWriteTest, + LzmaStreamWriteTest, + ] + try: support.run_unittest(*tests) finally: