diff -r 801567d6302c Doc/library/base64.rst --- a/Doc/library/base64.rst Thu May 23 20:25:09 2013 +1000 +++ b/Doc/library/base64.rst Thu May 23 21:29:00 2013 +1000 @@ -27,6 +27,10 @@ ASCII-only Unicode strings are now accepted by the decoding functions of the modern interface. +.. versionchanged:: 3.4 + Any :term:`bytes-like object`\ s are now accepted by all + encoding and decoding functions in this module. + The modern interface provides: .. function:: b64encode(s, altchars=None) diff -r 801567d6302c Doc/library/codecs.rst --- a/Doc/library/codecs.rst Thu May 23 20:25:09 2013 +1000 +++ b/Doc/library/codecs.rst Thu May 23 21:29:00 2013 +1000 @@ -1204,36 +1204,41 @@ .. tabularcolumns:: |l|L|L| -+----------------------+---------------------------+------------------------------+ -| Codec | Purpose | Encoder/decoder | -+======================+===========================+==============================+ -| base64_codec [#b64]_ | Convert operand to MIME | :meth:`base64.b64encode`, | -| | base64 (the result always | :meth:`base64.b64decode` | -| | includes a trailing | | -| | ``'\n'``) | | -+----------------------+---------------------------+------------------------------+ -| bz2_codec | Compress the operand | :meth:`bz2.compress`, | -| | using bz2 | :meth:`bz2.decompress` | -+----------------------+---------------------------+------------------------------+ -| hex_codec | Convert operand to | :meth:`base64.b16encode`, | -| | hexadecimal | :meth:`base64.b16decode` | -| | representation, with two | | -| | digits per byte | | -+----------------------+---------------------------+------------------------------+ -| quopri_codec | Convert operand to MIME | :meth:`quopri.encodestring`, | -| | quoted printable | :meth:`quopri.decodestring` | -+----------------------+---------------------------+------------------------------+ -| uu_codec | Convert the operand using | :meth:`uu.encode`, | -| | uuencode | :meth:`uu.decode` | -+----------------------+---------------------------+------------------------------+ -| zlib_codec | Compress the operand | :meth:`zlib.compress`, | -| | using gzip | :meth:`zlib.decompress` | -+----------------------+---------------------------+------------------------------+ ++----------------------+------------------------------+------------------------------+ +| Codec | Purpose | Encoder / decoder | ++======================+==============================+==============================+ +| base64_codec [#b64]_ | Convert operand to MIME | :meth:`base64.b64encode` / | +| | base64 (the result always | :meth:`base64.b64decode` | +| | includes a trailing | | +| | ``'\n'``) | | +| | | | +| | .. versionchanged:: 3.4 | | +| | accepts any | | +| | :term:`bytes-like object` | | +| | as input for encoding and | | +| | decoding | | ++----------------------+------------------------------+------------------------------+ +| bz2_codec | Compress the operand | :meth:`bz2.compress` / | +| | using bz2 | :meth:`bz2.decompress` | ++----------------------+------------------------------+------------------------------+ +| hex_codec | Convert operand to | :meth:`base64.b16encode` / | +| | hexadecimal | :meth:`base64.b16decode` | +| | representation, with two | | +| | digits per byte | | ++----------------------+------------------------------+------------------------------+ +| quopri_codec | Convert operand to MIME | :meth:`quopri.encodestring` /| +| | quoted printable | :meth:`quopri.decodestring` | ++----------------------+------------------------------+------------------------------+ +| uu_codec | Convert the operand using | :meth:`uu.encode` / | +| | uuencode | :meth:`uu.decode` | ++----------------------+------------------------------+------------------------------+ +| zlib_codec | Compress the operand | :meth:`zlib.compress` / | +| | using gzip | :meth:`zlib.decompress` | ++----------------------+------------------------------+------------------------------+ -.. [#b64] Rather than accepting any :term:`bytes-like object`, - ``'base64_codec'`` accepts only :class:`bytes` and :class:`bytearray` for - encoding and only :class:`bytes`, :class:`bytearray`, and ASCII-only - instances of :class:`str` for decoding +.. [#b64] In addition to :term:`bytes-like objects `, + ``'base64_codec'`` also accepts ASCII-only instances of :class:`str` for + decoding The following codecs provide :class:`str` to :class:`str` mappings. diff -r 801567d6302c Lib/base64.py --- a/Lib/base64.py Thu May 23 20:25:09 2013 +1000 +++ b/Lib/base64.py Thu May 23 21:29:00 2013 +1000 @@ -35,11 +35,13 @@ return s.encode('ascii') except UnicodeEncodeError: raise ValueError('string argument should contain only ASCII characters') - elif isinstance(s, bytes_types): + if isinstance(s, bytes_types): return s - else: - raise TypeError("argument should be bytes or ASCII string, not %s" % s.__class__.__name__) - + try: + return memoryview(s).tobytes() + except TypeError: + raise TypeError("argument should be a bytes-like object or ASCII " + "string, not %r" % s.__class__.__name__) from None # Base64 encoding/decoding uses binascii @@ -54,14 +56,9 @@ The encoded byte string is returned. """ - if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) # Strip off the trailing newline encoded = binascii.b2a_base64(s)[:-1] if altchars is not None: - if not isinstance(altchars, bytes_types): - raise TypeError("expected bytes, not %s" - % altchars.__class__.__name__) assert len(altchars) == 2, repr(altchars) return encoded.translate(bytes.maketrans(b'+/', altchars)) return encoded @@ -149,7 +146,7 @@ s is the byte string to encode. The encoded byte string is returned. """ if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) + s = memoryview(s).tobytes() leftover = len(s) % 5 # Pad the last quantum with zero bits if necessary if leftover: @@ -250,8 +247,6 @@ s is the byte string to encode. The encoded byte string is returned. """ - if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) return binascii.hexlify(s).upper() @@ -306,12 +301,17 @@ s = binascii.a2b_base64(line) output.write(s) +def _input_type_check(s): + try: + memoryview(s) + except TypeError as err: + msg = "expected bytes-like object, not %s" % s.__class__.__name__ + raise TypeError(msg) from err def encodebytes(s): """Encode a bytestring into a bytestring containing multiple lines of base-64 data.""" - if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) + _input_type_check(s) pieces = [] for i in range(0, len(s), MAXBINSIZE): chunk = s[i : i + MAXBINSIZE] @@ -328,8 +328,7 @@ def decodebytes(s): """Decode a bytestring of base-64 data into a bytestring.""" - if not isinstance(s, bytes_types): - raise TypeError("expected bytes, not %s" % s.__class__.__name__) + _input_type_check(s) return binascii.a2b_base64(s) def decodestring(s): diff -r 801567d6302c Lib/test/test_base64.py --- a/Lib/test/test_base64.py Thu May 23 20:25:09 2013 +1000 +++ b/Lib/test/test_base64.py Thu May 23 21:29:00 2013 +1000 @@ -5,6 +5,7 @@ import os import sys import subprocess +from array import array @@ -24,7 +25,10 @@ b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==\n") # Non-bytes eq(base64.encodebytes(bytearray(b'abc')), b'YWJj\n') + eq(base64.encodebytes(memoryview(b'abc')), b'YWJj\n') + eq(base64.encodebytes(array('B', b'abc')), b'YWJj\n') self.assertRaises(TypeError, base64.encodebytes, "") + self.assertRaises(TypeError, base64.encodebytes, []) def test_decodebytes(self): eq = self.assertEqual @@ -41,7 +45,10 @@ eq(base64.decodebytes(b''), b'') # Non-bytes eq(base64.decodebytes(bytearray(b'YWJj\n')), b'abc') + eq(base64.decodebytes(memoryview(b'YWJj\n')), b'abc') + eq(base64.decodebytes(array('B', b'YWJj\n')), b'abc') self.assertRaises(TypeError, base64.decodebytes, "") + self.assertRaises(TypeError, base64.decodebytes, []) def test_encode(self): eq = self.assertEqual @@ -92,11 +99,18 @@ eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=b'*$'), b'01a*b$cd') # Non-bytes eq(base64.b64encode(bytearray(b'abcd')), b'YWJjZA==') + eq(base64.b64encode(memoryview(b'abcd')), b'YWJjZA==') + eq(base64.b64encode(array('B', b'abcd')), b'YWJjZA==') eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=bytearray(b'*$')), b'01a*b$cd') + eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=memoryview(b'*$')), + b'01a*b$cd') + eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=array('B', b'*$')), + b'01a*b$cd') + self.assertRaises(TypeError, base64.b64encode, []) # Check if passing a str object raises an error self.assertRaises(TypeError, base64.b64encode, "") - self.assertRaises(TypeError, base64.b64encode, b"", altchars="") + self.assertRaises(TypeError, base64.b64encode, b"", altchars="*$") # Test standard alphabet eq(base64.standard_b64encode(b"www.python.org"), b"d3d3LnB5dGhvbi5vcmc=") eq(base64.standard_b64encode(b"a"), b"YQ==") @@ -111,12 +125,21 @@ b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==") # Non-bytes eq(base64.standard_b64encode(bytearray(b'abcd')), b'YWJjZA==') + eq(base64.standard_b64encode(memoryview(b'abcd')), b'YWJjZA==') + eq(base64.standard_b64encode(array('B', b'abcd')), b'YWJjZA==') + self.assertRaises(TypeError, base64.standard_b64encode, []) # Check if passing a str object raises an error self.assertRaises(TypeError, base64.standard_b64encode, "") # Test with 'URL safe' alternative characters eq(base64.urlsafe_b64encode(b'\xd3V\xbeo\xf7\x1d'), b'01a-b_cd') # Non-bytes - eq(base64.urlsafe_b64encode(bytearray(b'\xd3V\xbeo\xf7\x1d')), b'01a-b_cd') + eq(base64.urlsafe_b64encode(bytearray(b'\xd3V\xbeo\xf7\x1d')), + b'01a-b_cd') + eq(base64.urlsafe_b64encode(memoryview(b'\xd3V\xbeo\xf7\x1d')), + b'01a-b_cd') + eq(base64.urlsafe_b64encode(array('B', b'\xd3V\xbeo\xf7\x1d')), + b'01a-b_cd') + self.assertRaises(TypeError, base64.urlsafe_b64encode, []) # Check if passing a str object raises an error self.assertRaises(TypeError, base64.urlsafe_b64encode, "") @@ -142,6 +165,9 @@ eq(base64.b64decode(data.decode('ascii')), res) # Non-bytes eq(base64.b64decode(bytearray(b"YWJj")), b"abc") + eq(base64.b64decode(memoryview(b"YWJj")), b"abc") + eq(base64.b64decode(array('B', b"YWJj")), b"abc") + self.assertRaises(TypeError, base64.b64decode, []) # Test with arbitrary alternative characters tests_altchars = {(b'01a*b$cd', b'*$'): b'\xd3V\xbeo\xf7\x1d', @@ -161,6 +187,9 @@ eq(base64.standard_b64decode(data.decode('ascii')), res) # Non-bytes eq(base64.standard_b64decode(bytearray(b"YWJj")), b"abc") + eq(base64.standard_b64decode(memoryview(b"YWJj")), b"abc") + eq(base64.standard_b64decode(array('B', b"YWJj")), b"abc") + self.assertRaises(TypeError, base64.standard_b64decode, []) # Test with 'URL safe' alternative characters tests_urlsafe = {b'01a-b_cd': b'\xd3V\xbeo\xf7\x1d', @@ -170,7 +199,13 @@ eq(base64.urlsafe_b64decode(data), res) eq(base64.urlsafe_b64decode(data.decode('ascii')), res) # Non-bytes - eq(base64.urlsafe_b64decode(bytearray(b'01a-b_cd')), b'\xd3V\xbeo\xf7\x1d') + eq(base64.urlsafe_b64decode(bytearray(b'01a-b_cd')), + b'\xd3V\xbeo\xf7\x1d') + eq(base64.urlsafe_b64decode(memoryview(b'01a-b_cd')), + b'\xd3V\xbeo\xf7\x1d') + eq(base64.urlsafe_b64decode(array('B', b'01a-b_cd')), + b'\xd3V\xbeo\xf7\x1d') + self.assertRaises(TypeError, base64.urlsafe_b64decode, []) def test_b64decode_padding_error(self): self.assertRaises(binascii.Error, base64.b64decode, b'abc') @@ -206,7 +241,10 @@ eq(base64.b32encode(b'abcde'), b'MFRGGZDF') # Non-bytes eq(base64.b32encode(bytearray(b'abcd')), b'MFRGGZA=') + eq(base64.b32encode(memoryview(b'abcd')), b'MFRGGZA=') + eq(base64.b32encode(array('B', b'abcd')), b'MFRGGZA=') self.assertRaises(TypeError, base64.b32encode, "") + self.assertRaises(TypeError, base64.b32encode, []) def test_b32decode(self): eq = self.assertEqual @@ -223,6 +261,9 @@ eq(base64.b32decode(data.decode('ascii')), res) # Non-bytes eq(base64.b32decode(bytearray(b'MFRGG===')), b'abc') + eq(base64.b32decode(memoryview(b'MFRGG===')), b'abc') + eq(base64.b32decode(array('B', b'MFRGG===')), b'abc') + self.assertRaises(TypeError, base64.b32decode, []) def test_b32decode_casefold(self): eq = self.assertEqual @@ -276,7 +317,10 @@ eq(base64.b16encode(b'\x00'), b'00') # Non-bytes eq(base64.b16encode(bytearray(b'\x01\x02\xab\xcd\xef')), b'0102ABCDEF') + eq(base64.b16encode(memoryview(b'\x01\x02\xab\xcd\xef')), b'0102ABCDEF') + eq(base64.b16encode(array('B', b'\x01\x02\xab\xcd\xef')), b'0102ABCDEF') self.assertRaises(TypeError, base64.b16encode, "") + self.assertRaises(TypeError, base64.b16encode, []) def test_b16decode(self): eq = self.assertEqual @@ -292,6 +336,15 @@ eq(base64.b16decode('0102abcdef', True), b'\x01\x02\xab\xcd\xef') # Non-bytes eq(base64.b16decode(bytearray(b"0102ABCDEF")), b'\x01\x02\xab\xcd\xef') + eq(base64.b16decode(memoryview(b"0102ABCDEF")), b'\x01\x02\xab\xcd\xef') + eq(base64.b16decode(array('B', b"0102ABCDEF")), b'\x01\x02\xab\xcd\xef') + eq(base64.b16decode(bytearray(b"0102abcdef"), True), + b'\x01\x02\xab\xcd\xef') + eq(base64.b16decode(memoryview(b"0102abcdef"), True), + b'\x01\x02\xab\xcd\xef') + eq(base64.b16decode(array('B', b"0102abcdef"), True), + b'\x01\x02\xab\xcd\xef') + self.assertRaises(TypeError, base64.b16decode, []) def test_decode_nonascii_str(self): decode_funcs = (base64.b64decode, diff -r 801567d6302c Lib/test/test_codecs.py --- a/Lib/test/test_codecs.py Thu May 23 20:25:09 2013 +1000 +++ b/Lib/test/test_codecs.py Thu May 23 21:29:00 2013 +1000 @@ -2285,6 +2285,24 @@ sout = reader.readline() self.assertEqual(sout, b"\x80") + def test_buffer_api_usage(self): + # We check all the transform codecs accept memoryview input + # for encoding and decoding + # and also that they roundtrip correctly + original = b"12345\x80" + for encoding in bytes_transform_encodings: + data = original + view = memoryview(data) + data = codecs.encode(data, encoding) + view_encoded = codecs.encode(view, encoding) + self.assertEqual(view_encoded, data) + view = memoryview(data) + data = codecs.decode(data, encoding) + self.assertEqual(data, original) + view_decoded = codecs.decode(view, encoding) + self.assertEqual(view_decoded, data) + + @unittest.skipUnless(sys.platform == 'win32', 'code pages are specific to Windows')