Index: Lib/io.py =================================================================== --- Lib/io.py (revision 58902) +++ Lib/io.py (working copy) @@ -1075,7 +1075,8 @@ self._pending = "" self._snapshot = None self._seekable = self._telling = self.buffer.seekable() - + self._nl_straddle = False + @property def encoding(self): return self._encoding @@ -1136,16 +1137,35 @@ decoder = self._decoder = make_decoder() # XXX: errors return decoder + def _adjust_chunk(self, readahead, pending): + if self._readtranslate: + if self._nl_straddle and pending and pending[0] == "\n": + pending = pending[1:] + # should not adjust readahead as otherwise, it can become nul and + # terminate read loop in self.read() incorrectly. + # readahead = readahead[1:] + self._nl_straddle = False + if pending and pending[-1] == "\r": + self._nl_straddle = True + else: + self._nl_straddle = False + + pending = self._replacenl(pending) + + return readahead, pending + def _read_chunk(self): if self._decoder is None: raise ValueError("no decoder") if not self._telling: readahead = self.buffer.read1(self._CHUNK_SIZE) pending = self._decoder.decode(readahead, not readahead) - return readahead, pending + return self._adjust_chunk(readahead, pending) + decoder_buffer, decoder_state = self._decoder.getstate() readahead = self.buffer.read1(self._CHUNK_SIZE) pending = self._decoder.decode(readahead, not readahead) + readahead, pending = self._adjust_chunk(readahead, pending) self._snapshot = (decoder_state, decoder_buffer + readahead, pending) return readahead, pending @@ -1244,6 +1264,10 @@ res = self._pending if n < 0: res += decoder.decode(self.buffer.read(), True) + if self._readtranslate: + if self._nl_straddle and res and res[0] == "\n": + res = res[1:] + self._nl_straddle = False self._pending = "" self._snapshot = None return self._replacenl(res) @@ -1253,8 +1277,9 @@ res += pending if not readahead: break + # res = self._replacenl(res) self._pending = res[n:] - return self._replacenl(res[:n]) + return res[:n] def __next__(self): self._telling = False