# HG changeset patch # Parent aa304ad32292265819d411ba965539a9fddac1b7 Issue #5784: Expand documentation and tests for zlib wbits parameter diff -r aa304ad32292 Doc/library/zlib.rst --- a/Doc/library/zlib.rst Thu Nov 26 23:49:42 2015 +0200 +++ b/Doc/library/zlib.rst Sun Nov 29 05:38:20 2015 +0000 @@ -71,9 +71,22 @@ *method* is the compression algorithm. Currently, the only supported value is ``DEFLATED``. - *wbits* is the base two logarithm of the size of the window buffer. This - should be an integer from ``8`` to ``15``. Higher values give better - compression, but use more memory. + The *wbits* argument controls the size of the history buffer (or the + "window size") used when compressing data, and whether a header and + trailer is included in the output. It can take several ranges of values: + + * +9 to +15: The base-two logarithm of the window size, which + therefore ranges between 512 and 32768. Larger values produce + better compression at the expense of greater memory usage. The + resulting output will include a zlib-specific header and trailer. + + * −9 to −15: Uses the absolute value of *wbits* as the + window size logarithm, while producing a raw output stream with no + header or trailing checksum. + + * +25 to +31 = 16 + (9 to 15): Uses the low 4 bits of the value as the + window size logarithm, while including a basic :program:`gzip` header + and trailing checksum in the output. The *memLevel* argument controls the amount of memory used for the internal compression state. Valid values range from ``1`` to ``9``. @@ -119,20 +132,39 @@ .. function:: decompress(data[, wbits[, bufsize]]) Decompresses the bytes in *data*, returning a bytes object containing the - uncompressed data. The *wbits* parameter controls the size of the window - buffer, and is discussed further below. + uncompressed data. The *wbits* parameter depends on + the format of *data*, and is discussed further below. If *bufsize* is given, it is used as the initial size of the output buffer. Raises the :exc:`error` exception if any error occurs. - The absolute value of *wbits* is the base two logarithm of the size of the - history buffer (the "window size") used when compressing data. Its absolute - value should be between 8 and 15 for the most recent versions of the zlib - library, larger values resulting in better compression at the expense of greater - memory usage. When decompressing a stream, *wbits* must not be smaller + .. _decompress-wbits: + + The *wbits* parameter controls the size of the history buffer + (or "window size"), and what header and trailer format is expected. + It is similar to the parameter for :func:`compressobj`, but accepts + more ranges of values: + + * +8 to +15: The base-two logarithm of the window size. The input + must include a zlib header and trailer. + + * 0: Automatically determine the window size from the zlib header. + + * −8 to −15: Uses the absolute value of *wbits* as the window size + logarithm. The input must be a raw stream with no header or trailer. + + * +24 to +31 = 16 + (8 to 15): Uses the low 4 bits of the value as + the window size logarithm. The input must include a gzip header and + trailer. + + * +40 to +47 = 32 + (8 to 15): Uses the low 4 bits of the value as + the window size logarithm, and automatically accept either + the zlib or gzip format. + + When decompressing a stream, the window size must not be smaller than the size originally used to compress the stream; using a too-small - value will result in an exception. The default value is therefore the - highest value, 15. When *wbits* is negative, the standard - :program:`gzip` header is suppressed. + value may result in an :exc:`error` exception. The default *wbits* value + is 15, which corresponds to the largest window size and requires a zlib + header and trailer to be included. *bufsize* is the initial size of the buffer used to hold decompressed data. If more space is required, the buffer size will be increased as needed, so you @@ -145,7 +177,9 @@ Returns a decompression object, to be used for decompressing data streams that won't fit into memory at once. - The *wbits* parameter controls the size of the window buffer. + The *wbits* parameter controls the size of the history buffer (or the + "window size"), and what header and trailer format is expected. It has + the same meaning as `described for decompress() <#decompress-wbits>`__. The *zdict* parameter specifies a predefined compression dictionary. If provided, this must be the same dictionary as was used by the compressor that @@ -285,4 +319,3 @@ http://www.zlib.net/manual.html The zlib manual explains the semantics and usage of the library's many functions. - diff -r aa304ad32292 Lib/test/test_zlib.py --- a/Lib/test/test_zlib.py Thu Nov 26 23:49:42 2015 +0200 +++ b/Lib/test/test_zlib.py Sun Nov 29 05:38:20 2015 +0000 @@ -636,6 +636,49 @@ finally: data = None + def test_wbits(self): + co = zlib.compressobj(level=1, wbits=15) + zlib15 = co.compress(HAMLET_SCENE) + co.flush() + self.assertEqual(zlib.decompress(zlib15, 15), HAMLET_SCENE) + self.assertEqual(zlib.decompress(zlib15, 0), HAMLET_SCENE) + self.assertEqual(zlib.decompress(zlib15, 32 + 15), HAMLET_SCENE) + with self.assertRaisesRegex(zlib.error, 'invalid window size'): + zlib.decompress(zlib15, 14) + dco = zlib.decompressobj(wbits=32 + 15) + self.assertEqual(dco.decompress(zlib15), HAMLET_SCENE) + dco = zlib.decompressobj(wbits=14) + with self.assertRaisesRegex(zlib.error, 'invalid window size'): + dco.decompress(zlib15) + + co = zlib.compressobj(level=1, wbits=9) + zlib9 = co.compress(HAMLET_SCENE) + co.flush() + self.assertEqual(zlib.decompress(zlib9, 9), HAMLET_SCENE) + self.assertEqual(zlib.decompress(zlib9, 15), HAMLET_SCENE) + self.assertEqual(zlib.decompress(zlib9, 0), HAMLET_SCENE) + self.assertEqual(zlib.decompress(zlib9, 32 + 9), HAMLET_SCENE) + dco = zlib.decompressobj(wbits=32 + 9) + self.assertEqual(dco.decompress(zlib9), HAMLET_SCENE) + + co = zlib.compressobj(level=1, wbits=-15) + deflate15 = co.compress(HAMLET_SCENE) + co.flush() + self.assertEqual(zlib.decompress(deflate15, -15), HAMLET_SCENE) + dco = zlib.decompressobj(wbits=-15) + self.assertEqual(dco.decompress(deflate15), HAMLET_SCENE) + + co = zlib.compressobj(level=1, wbits=-9) + deflate9 = co.compress(HAMLET_SCENE) + co.flush() + self.assertEqual(zlib.decompress(deflate9, -9), HAMLET_SCENE) + self.assertEqual(zlib.decompress(deflate9, -15), HAMLET_SCENE) + dco = zlib.decompressobj(wbits=-9) + self.assertEqual(dco.decompress(deflate9), HAMLET_SCENE) + + co = zlib.compressobj(level=1, wbits=16 + 15) + gzip = co.compress(HAMLET_SCENE) + co.flush() + self.assertEqual(zlib.decompress(gzip, 16 + 15), HAMLET_SCENE) + self.assertEqual(zlib.decompress(gzip, 32 + 15), HAMLET_SCENE) + dco = zlib.decompressobj(32 + 15) + self.assertEqual(dco.decompress(gzip), HAMLET_SCENE) + def genblock(seed, length, step=1024, generator=random): """length-byte stream of random data from a seed (in step-byte blocks).""" diff -r aa304ad32292 Modules/clinic/zlibmodule.c.h --- a/Modules/clinic/zlibmodule.c.h Thu Nov 26 23:49:42 2015 +0200 +++ b/Modules/clinic/zlibmodule.c.h Sun Nov 29 05:38:20 2015 +0000 @@ -49,7 +49,7 @@ " data\n" " Compressed data.\n" " wbits\n" -" The window buffer size.\n" +" The window buffer size and container format.\n" " bufsize\n" " The initial output buffer size."); @@ -95,7 +95,10 @@ " method\n" " The compression algorithm. If given, this must be DEFLATED.\n" " wbits\n" -" The base two logarithm of the window size (range: 8..15).\n" +" +9 to +15: The base-two logarithm of the window size. Include a zlib\n" +" container.\n" +" -9 to -15: Generate a raw stream.\n" +" +25 to +31: Include a gzip container.\n" " memLevel\n" " Controls the amount of memory used for internal compression state.\n" " Valid values range from 1 to 9. Higher values result in higher memory\n" @@ -146,7 +149,7 @@ "Return a decompressor object.\n" "\n" " wbits\n" -" The window buffer size.\n" +" The window buffer size and container format.\n" " zdict\n" " The predefined compression dictionary. This must be the same\n" " dictionary as used by the compressor that produced the input data."); @@ -450,4 +453,4 @@ return return_value; } -/*[clinic end generated code: output=bc9473721ca7c962 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=775025f51b9c8fc1 input=a9049054013a1b77]*/ diff -r aa304ad32292 Modules/zlibmodule.c --- a/Modules/zlibmodule.c Thu Nov 26 23:49:42 2015 +0200 +++ b/Modules/zlibmodule.c Sun Nov 29 05:38:20 2015 +0000 @@ -272,7 +272,7 @@ data: Py_buffer Compressed data. wbits: int(c_default="MAX_WBITS") = MAX_WBITS - The window buffer size. + The window buffer size and container format. bufsize: uint(c_default="DEF_BUF_SIZE") = DEF_BUF_SIZE The initial output buffer size. / @@ -282,7 +282,7 @@ static PyObject * zlib_decompress_impl(PyModuleDef *module, Py_buffer *data, int wbits, unsigned int bufsize) -/*[clinic end generated code: output=9e5464e72df9cb5f input=0f4b9abb7103f50e]*/ +/*[clinic end generated code: output=9e5464e72df9cb5f input=2ddf158b2ffe8be3]*/ { PyObject *result_str = NULL; Byte *input; @@ -395,7 +395,10 @@ method: int(c_default="DEFLATED") = DEFLATED The compression algorithm. If given, this must be DEFLATED. wbits: int(c_default="MAX_WBITS") = MAX_WBITS - The base two logarithm of the window size (range: 8..15). + +9 to +15: The base-two logarithm of the window size. Include a zlib + container. + -9 to -15: Generate a raw stream. + +25 to +31: Include a gzip container. memLevel: int(c_default="DEF_MEM_LEVEL") = DEF_MEM_LEVEL Controls the amount of memory used for internal compression state. Valid values range from 1 to 9. Higher values result in higher memory @@ -412,7 +415,7 @@ static PyObject * zlib_compressobj_impl(PyModuleDef *module, int level, int method, int wbits, int memLevel, int strategy, Py_buffer *zdict) -/*[clinic end generated code: output=89e5a6c1449caa9e input=b034847f8821f6af]*/ +/*[clinic end generated code: output=89e5a6c1449caa9e input=caa7962a46284539]*/ { compobject *self = NULL; int err; @@ -473,7 +476,7 @@ zlib.decompressobj wbits: int(c_default="MAX_WBITS") = MAX_WBITS - The window buffer size. + The window buffer size and container format. zdict: object(c_default="NULL") = b'' The predefined compression dictionary. This must be the same dictionary as used by the compressor that produced the input data. @@ -483,7 +486,7 @@ static PyObject * zlib_decompressobj_impl(PyModuleDef *module, int wbits, PyObject *zdict) -/*[clinic end generated code: output=8ccd583fbd631798 input=67f05145a6920127]*/ +/*[clinic end generated code: output=8ccd583fbd631798 input=d3832b8511fc977b]*/ { int err; compobject *self; @@ -1334,7 +1337,7 @@ "decompress(string,[wbits],[bufsize]) -- Decompresses a compressed string.\n" "decompressobj([wbits[, zdict]]]) -- Return a decompressor object.\n" "\n" -"'wbits' is window buffer size.\n" +"'wbits' is window buffer size and container format.\n" "Compressor objects support compress() and flush() methods; decompressor\n" "objects support decompress() and flush().");