Index: Lib/io.py =================================================================== --- Lib/io.py (revision 68796) +++ Lib/io.py (working copy) @@ -1047,11 +1047,42 @@ self._flush_unlocked() except BlockingIOError as e: # We can't accept anything else. - # XXX Why not just let the exception pass through? + # Reraise this with 0 in the written field as none of the + # data passed to this call has been written. raise BlockingIOError(e.errno, e.strerror, 0) before = len(self._write_buf) - self._write_buf.extend(b) - written = len(self._write_buf) - before + bytes_to_consume = self.max_buffer_size - before + # b is an iterable of ints, it won't always support len(). + if hasattr(b, '__len__') and len(b) > bytes_to_consume: + try: + chunk = memoryview(b)[:bytes_to_consume] + except TypeError: + # No buffer API? Make intermediate slice copies instead. + chunk = b[:bytes_to_consume] + # Loop over the data, flushing it to the underlying raw IO + # stream in self.max_buffer_size chunks. + written = 0 + self._write_buf.extend(chunk) + while chunk and len(self._write_buf) > self.buffer_size: + try: + self._flush_unlocked() + except BlockingIOError as e: + written += e.characters_written + raise BlockingIOError(e.errno, e.strerror, written) + written += len(chunk) + assert not self._write_buf, "_write_buf should be empty" + if isinstance(chunk, memoryview): + chunk = memoryview(b)[written: + written + self.max_buffer_size] + else: + chunk = b[written:written + self.max_buffer_size] + self._write_buf.extend(chunk) + else: + # This could go beyond self.max_buffer_size as we don't know + # the length of b. The alternative of iterating over it one + # byte at a time in python would be slow. + self._write_buf.extend(b) + written = len(self._write_buf) - before if len(self._write_buf) > self.buffer_size: try: self._flush_unlocked() Index: Lib/test/test_io.py =================================================================== --- Lib/test/test_io.py (revision 68796) +++ Lib/test/test_io.py (working copy) @@ -479,6 +479,33 @@ self.assertEquals(b"abcdefghijkl", writer._write_stack[0]) + def testWriteMaxBufferSize(self): + writer = MockRawIO() + bufio = io.BufferedWriter(writer, buffer_size=8, max_buffer_size=13) + + bufio.write(b"abcdefghij") + bufio.write(b"klmnopqrstuvwxyz") + bufio.write(b"0123456789"*3) + + expected_writes = [b"abcdefghij", b"klmnopqrstuvw", b"xyz0123456789", + b"0123456789012"] + self.assertEquals(expected_writes, writer._write_stack) + bufio.close() + self.assertEquals(b"3456789", writer._write_stack[4]) + + def testWriteNoLengthIterable(self): + writer = MockRawIO() + bufio = io.BufferedWriter(writer, buffer_size=8) + + def Generator(value): + for x in value: + yield x + iterable = Generator(b'ABCDEFGHIJKLMNOPQRSTUVWXYZ') + self.assertRaises(TypeError, len, iterable) + bufio.write(iterable) + + self.assertEquals(b'ABCDEFGHIJKLMNOPQRSTUVWXYZ', writer._write_stack[0]) + def testWriteNonBlocking(self): raw = MockNonBlockWriterIO((9, 2, 22, -6, 10, 12, 12)) bufio = io.BufferedWriter(raw, 8, 16)