diff -r 0eeb39fc8ff5 Lib/codecs.py --- a/Lib/codecs.py Tue Dec 01 19:59:53 2015 +1100 +++ b/Lib/codecs.py Wed Dec 02 21:57:34 2015 +0200 @@ -335,6 +335,13 @@ class BufferedIncrementalDecoder(Increme # ignore additional state info self.buffer = state[0] + +def _getattr_proxy(self, name, _getattr=getattr): + # Public attribute lookups are delegated to the underlying stream + if name[:1] == '_': + raise AttributeError(name) + return _getattr(self.__dict__['stream'], name) + # # The StreamWriter and StreamReader class provide generic working # interfaces which can be used to implement new encoding submodules @@ -400,19 +407,15 @@ class StreamWriter(Codec): if whence == 0 and offset == 0: self.reset() - def __getattr__(self, name, - getattr=getattr): - - """ Inherit all other methods from the underlying stream. - """ - return getattr(self.stream, name) - def __enter__(self): return self def __exit__(self, type, value, tb): self.stream.close() + # Inherit all other public methods from the underlying stream. + __getattr__ = _getattr_proxy + ### class StreamReader(Codec): @@ -440,8 +443,7 @@ class StreamReader(Codec): self.stream = stream self.errors = errors self.bytebuffer = b"" - self._empty_charbuffer = self.charbuffertype() - self.charbuffer = self._empty_charbuffer + self.charbuffer = self.charbuffertype() self.linebuffer = None def decode(self, input, errors='strict'): @@ -476,7 +478,7 @@ class StreamReader(Codec): """ # If we have lines cached, first merge them back into characters if self.linebuffer: - self.charbuffer = self._empty_charbuffer.join(self.linebuffer) + self.charbuffer = self.charbuffer[:0].join(self.linebuffer) self.linebuffer = None # read until we get the required number of characters (if available) @@ -518,7 +520,7 @@ class StreamReader(Codec): if chars < 0: # Return everything we've got result = self.charbuffer - self.charbuffer = self._empty_charbuffer + self.charbuffer = self.charbuffer[:0] else: # Return the first chars characters result = self.charbuffer[:chars] @@ -549,7 +551,7 @@ class StreamReader(Codec): return line readsize = size or 72 - line = self._empty_charbuffer + line = self.charbuffer[:0] # If size is given, we call read() only once while True: data = self.read(readsize, firstline=True) @@ -573,7 +575,7 @@ class StreamReader(Codec): # cache the remaining lines lines[-1] += self.charbuffer self.linebuffer = lines - self.charbuffer = None + self.charbuffer = self.charbuffer[:0] else: # only one remaining line, put it back into charbuffer self.charbuffer = lines[0] + self.charbuffer @@ -584,7 +586,7 @@ class StreamReader(Codec): line0withoutend = lines[0].splitlines(keepends=False)[0] if line0withend != line0withoutend: # We really have a line end # Put the rest back together and keep it until the next call - self.charbuffer = self._empty_charbuffer.join(lines[1:]) + \ + self.charbuffer = self.charbuffer[:0].join(lines[1:]) + \ self.charbuffer if keepends: line = line0withend @@ -625,7 +627,7 @@ class StreamReader(Codec): """ self.bytebuffer = b"" - self.charbuffer = self._empty_charbuffer + self.charbuffer = self.charbuffer[:0] self.linebuffer = None def seek(self, offset, whence=0): @@ -647,19 +649,15 @@ class StreamReader(Codec): def __iter__(self): return self - def __getattr__(self, name, - getattr=getattr): - - """ Inherit all other methods from the underlying stream. - """ - return getattr(self.stream, name) - def __enter__(self): return self def __exit__(self, type, value, tb): self.stream.close() + # Inherit all other public methods from the underlying stream. + __getattr__ = _getattr_proxy + ### class StreamReaderWriter: @@ -732,13 +730,6 @@ class StreamReaderWriter: if whence == 0 and offset == 0: self.writer.reset() - def __getattr__(self, name, - getattr=getattr): - - """ Inherit all other methods from the underlying stream. - """ - return getattr(self.stream, name) - # these are needed to make "with codecs.open(...)" work properly def __enter__(self): @@ -747,6 +738,9 @@ class StreamReaderWriter: def __exit__(self, type, value, tb): self.stream.close() + # Inherit all other public methods from the underlying stream. + __getattr__ = _getattr_proxy + ### class StreamRecoder: @@ -844,19 +838,15 @@ class StreamRecoder: self.reader.reset() self.writer.reset() - def __getattr__(self, name, - getattr=getattr): - - """ Inherit all other methods from the underlying stream. - """ - return getattr(self.stream, name) - def __enter__(self): return self def __exit__(self, type, value, tb): self.stream.close() + # Inherit all other public methods from the underlying stream. + __getattr__ = _getattr_proxy + ### Shortcuts def open(filename, mode='r', encoding=None, errors='strict', buffering=1): diff -r 0eeb39fc8ff5 Lib/test/test_codecs.py --- a/Lib/test/test_codecs.py Tue Dec 01 19:59:53 2015 +1100 +++ b/Lib/test/test_codecs.py Wed Dec 02 21:57:34 2015 +0200 @@ -1,7 +1,9 @@ import codecs import contextlib +import copy import io import locale +import pickle import sys import unittest import warnings @@ -1832,6 +1834,78 @@ class StreamReaderTest(unittest.TestCase f = self.reader(self.stream) self.assertEqual(f.readlines(), ['\ud55c\n', '\uae00']) + def test_deepcopy(self): + f = self.reader(Queue(b'\xed\x95\x9c\n\xea\xb8\x80')) + f_copy = copy.deepcopy(f) + self.assertEqual(f.readlines(), ['\ud55c\n', '\uae00']) + self.assertEqual(f_copy.readlines(), ['\ud55c\n', '\uae00']) + + def test_pickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f = self.reader(Queue(b'\xed\x95\x9c\n\xea\xb8\x80')) + f_copy = pickle.loads(pickle.dumps(f, proto)) + self.assertEqual(f.readlines(), ['\ud55c\n', '\uae00']) + self.assertEqual(f_copy.readlines(), ['\ud55c\n', '\uae00']) + + +class StreamWriterTest(unittest.TestCase): + + def setUp(self): + self.writer = codecs.getwriter('utf-8') + + def test_deepcopy(self): + f = self.writer(Queue(b'')) + f.write('\xe9') + f2 = copy.deepcopy(f) + self.assertEqual(f.stream.read(), b'\xc3\xa9') + self.assertEqual(f2.stream.read(), b'\xc3\xa9') + f.write('\U0001f418') + f2.write('\u20ac') + self.assertEqual(f.stream.read(), b'\xf0\x9f\x90\x98') + self.assertEqual(f2.stream.read(), b'\xe2\x82\xac') + + def test_pickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f = self.writer(Queue(b'')) + f.write('\xe9') + f2 = pickle.loads(pickle.dumps(f, proto)) + self.assertEqual(f.stream.read(), b'\xc3\xa9') + self.assertEqual(f2.stream.read(), b'\xc3\xa9') + f.write('\U0001f418') + f2.write('\u20ac') + self.assertEqual(f.stream.read(), b'\xf0\x9f\x90\x98') + self.assertEqual(f2.stream.read(), b'\xe2\x82\xac') + + +class StreamReaderWriterTest(unittest.TestCase): + + def setUp(self): + self.reader = codecs.getreader('latin1') + self.writer = codecs.getwriter('utf-8') + + def test_deepcopy(self): + f = codecs.StreamReaderWriter(Queue(b''), self.reader, self.writer) + f.write('\xe9') + f2 = copy.deepcopy(f) + self.assertEqual(f.read(1), '\xc3') + self.assertEqual(f2.read(1), '\xc3') + f.write('\U0001f418') + f2.write('\u20ac') + self.assertEqual(f.read(), '\xa9\xf0\x9f\x90\x98') + self.assertEqual(f2.read(), '\xa9\xe2\x82\xac') + + def test_pickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f = codecs.StreamReaderWriter(Queue(b''), self.reader, self.writer) + f.write('\xe9') + f2 = pickle.loads(pickle.dumps(f, proto)) + self.assertEqual(f.read(1), '\xc3') + self.assertEqual(f2.read(1), '\xc3') + f.write('\U0001f418') + f2.write('\u20ac') + self.assertEqual(f.read(), '\xa9\xf0\x9f\x90\x98') + self.assertEqual(f2.read(), '\xa9\xe2\x82\xac') + class EncodedFileTest(unittest.TestCase):