# HG changeset patch # Parent 4cb316fe6bf2bfc3da3c87434c48a6f5e799b9b5 #13322: Non-blocking reads may return None * Document existing behaviour of C “io” module buffered read(), readinto/1() returning None * Remove lies about short reads for non-file streams * Fix _pyio return values to match C “io” module returning None * Fix BufferedIOBase.readinto/1() implementation when read/1() returns None * Change buffered read1() and peek() to return None diff -r 4cb316fe6bf2 Doc/library/io.rst --- a/Doc/library/io.rst Tue Feb 03 01:50:31 2015 +0200 +++ b/Doc/library/io.rst Tue Feb 03 07:03:59 2015 +0000 @@ -386,7 +386,9 @@ .. method:: readall() Read and return all the bytes from the stream until EOF, using multiple - calls to the stream if necessary. + calls to the stream if necessary. In non-blocking mode, all bytes are + read until the stream would block, and if no bytes are available and + EOF has not been signalled, ``None`` is returned. .. method:: readinto(b) @@ -414,10 +416,10 @@ input as requested or to consume all given output, at the expense of making perhaps more than one system call. - In addition, those methods can raise :exc:`BlockingIOError` if the - underlying raw stream is in non-blocking mode and cannot take or give - enough data; unlike their :class:`RawIOBase` counterparts, they will - never return ``None``. + In addition, :meth:`write` raises :exc:`BlockingIOError` if the + underlying raw stream is in non-blocking mode and cannot take + enough data; unlike its :meth:`RawIOBase.write` counterpart, its + return value is always the total number of bytes. Besides, the :meth:`read` method does not have a default implementation that defers to :meth:`readinto`. @@ -453,15 +455,12 @@ Read and return up to *size* bytes. If the argument is omitted, ``None``, or negative, data is read and returned until EOF is reached. An empty :class:`bytes` object is returned if the stream is already at EOF. + If the argument is positive, multiple raw reads may be issued to + satisfy the byte count (unless EOF is reached first). - If the argument is positive, and the underlying raw stream is not - interactive, multiple raw reads may be issued to satisfy the byte count - (unless EOF is reached first). But for interactive raw streams, at most - one raw read will be issued, and a short result does not imply that EOF is - imminent. - - A :exc:`BlockingIOError` is raised if the underlying raw stream is in - non blocking-mode, and has no data available at the moment. + In non-blocking mode, a short result may also be returned if not enough + data is available, and ``None`` is returned if no data is available and + EOF has not been signalled. .. method:: read1(size=-1) @@ -471,25 +470,30 @@ implementing your own buffering on top of a :class:`BufferedIOBase` object. + .. versionchanged:: 3.5 + ``None`` is now returned if no non-blocking data is available. + .. method:: readinto(b) Read up to ``len(b)`` bytes into bytearray *b* and return the number of bytes read. Like :meth:`read`, multiple reads may be issued to the underlying raw - stream, unless the latter is interactive. + stream. - A :exc:`BlockingIOError` is raised if the underlying raw stream is in non - blocking-mode, and has no data available at the moment. + In non-blocking mode, the number of bytes read may be less than + requested if not enough data is available, and ``None`` is returned if + no data is available and EOF has not been signalled. .. method:: readinto1(b) - Read up to ``len(b)`` bytes into bytearray *b*, ,using at most one call to + Read up to ``len(b)`` bytes into bytearray *b*, using at most one call to the underlying raw stream's :meth:`~RawIOBase.read` (or :meth:`~RawIOBase.readinto`) method. Return the number of bytes read. - A :exc:`BlockingIOError` is raised if the underlying raw stream is in non - blocking-mode, and has no data available at the moment. + ``None`` is returned if the underlying raw stream is in non-blocking + mode, has no data available at the moment, and EOF has not been + signalled. .. versionadded:: 3.5 @@ -638,6 +642,9 @@ single read on the raw stream is done to satisfy the call. The number of bytes returned may be less or more than requested. + .. versionchanged:: 3.5 + ``None`` is now returned if no non-blocking data is available. + .. method:: read([size]) Read and return *size* bytes, or if *size* is not given or negative, until diff -r 4cb316fe6bf2 Lib/_pyio.py --- a/Lib/_pyio.py Tue Feb 03 01:50:31 2015 +0200 +++ b/Lib/_pyio.py Tue Feb 03 07:03:59 2015 +0000 @@ -576,7 +576,7 @@ return bytes(b) def readall(self): - """Read until EOF, using multiple read() call.""" + """Read until EOF, using multiple read() calls.""" res = bytearray() while True: data = self.read(DEFAULT_BUFFER_SIZE) @@ -614,13 +614,11 @@ """Base class for buffered IO objects. The main difference with RawIOBase is that the read() method - supports omitting the size argument, and does not have a default - implementation that defers to readinto(). + does not have a default implementation that defers to readinto(). - In addition, read(), readinto() and write() may raise - BlockingIOError if the underlying raw stream is in non-blocking - mode and not ready; unlike their raw counterparts, they will never - return None. + In addition, write() may raise BlockingIOError if the underlying raw + stream is in non-blocking mode and is not ready; unlike its raw + counterpart, its return value is always the total number of bytes. A typical implementation should not inherit from a RawIOBase implementation, but wrap one. @@ -632,16 +630,12 @@ If the argument is omitted, None, or negative, reads and returns all data until EOF. - If the argument is positive, and the underlying raw stream is - not 'interactive', multiple raw reads may be issued to satisfy - the byte count (unless EOF is reached first). But for - interactive raw streams (XXX and for pipes?), at most one raw - read will be issued, and a short result does not imply that - EOF is imminent. + If the argument is positive, multiple raw reads may be issued to + satisfy the byte count (unless EOF is reached first). Returns an empty bytes array on EOF. - Raises BlockingIOError if the underlying raw stream has no + Returns None if an underlying non-blocking raw stream has no data at the moment. """ self._unsupported("read") @@ -656,11 +650,11 @@ """Read up to len(b) bytes into bytearray b. Like read(), this may issue multiple reads to the underlying raw - stream, unless the latter is 'interactive'. + stream. Returns an int representing the number of bytes read (0 for EOF). - Raises BlockingIOError if the underlying raw stream has no + Returns None if an underlying non-blocking raw stream has no data at the moment. """ @@ -671,7 +665,7 @@ Returns an int representing the number of bytes read (0 for EOF). - Raises BlockingIOError if the underlying raw stream has no + Returns None if an underlying non-blocking raw stream has no data at the moment. """ @@ -686,6 +680,8 @@ data = self.read1(len(b)) else: data = self.read(len(b)) + if data is None: + return None n = len(data) b[:n] = data @@ -1037,7 +1033,7 @@ break avail += len(chunk) chunks.append(chunk) - # n is more then avail only when an EOF occurred or when + # n is more than avail only when an EOF occurred or when # read() would have blocked. n = min(n, avail) out = b"".join(chunks) @@ -1069,6 +1065,8 @@ if current: self._read_buf = self._read_buf[self._read_pos:] + current self._read_pos = 0 + elif current is None and have <= 0: + return None return self._read_buf[self._read_pos:] def read1(self, size): @@ -1080,7 +1078,9 @@ if size == 0: return b"" with self._read_lock: - self._peek_unlocked(1) + if (self._peek_unlocked(1) is None and + self._read_pos == len(self._read_buf)): + return None return self._read_unlocked( min(size, len(self._read_buf) - self._read_pos)) @@ -1103,6 +1103,7 @@ buf = buf.cast('B') written = 0 + read_result = 0 # None if raw read returned None with self._read_lock: while written < len(buf): @@ -1119,21 +1120,24 @@ # If remaining space in callers buffer is larger than # internal buffer, read directly into callers buffer if len(buf) - written > self.buffer_size: - n = self.raw.readinto(buf[written:]) - if not n: - break # eof - written += n + read_result = self.raw.readinto(buf[written:]) + if not read_result: + break # eof or would block + written += read_result # Otherwise refill internal buffer - unless we're # in read1 mode and already got some data elif not (read1 and written): - if not self._peek_unlocked(1): - break # eof + read_result = self._peek_unlocked(1) + if not read_result: + break # eof or would block # In readinto1 mode, return as soon as we have some data if read1 and written: break + if not written and read_result is None: + return None return written def tell(self): diff -r 4cb316fe6bf2 Lib/test/test_io.py --- a/Lib/test/test_io.py Tue Feb 03 01:50:31 2015 +0200 +++ b/Lib/test/test_io.py Tue Feb 03 07:03:59 2015 +0000 @@ -663,6 +663,29 @@ support.gc_collect() self.assertEqual(recorded, []) + def test_buffered_mixin(self): + # Test the implementations provided by BufferedIOBase + reads = ( + # (data, length, buffer) + (b"123", 3, b"123"), + (b"12", 2, b"12#"), + (b"", 0, b"###"), + (None, None, b"###"), + ) + for suffix in ("", "1"): + readinto = "readinto" + suffix + for data, length, expected_buffer in reads: + with self.subTest(readinto, length=length): + file = self.BufferedIOBase() + def read_method(size): + self.assertEqual(size, 3) + return data + setattr(file, "read" + suffix, read_method) + buffer = bytearray(b"###") + readinto_method = getattr(file, readinto) + self.assertEqual(readinto_method(buffer), length) + self.assertEqual(buffer, expected_buffer) + class CIOTest(IOTest): @@ -1035,15 +1058,33 @@ self.assertEqual(rawio.read_history, raw_read_sizes) def test_read_non_blocking(self): - # Inject some None's in there to simulate EWOULDBLOCK - rawio = self.MockRawIO((b"abc", b"d", None, b"efg", None, None, None)) - bufio = self.tp(rawio) - self.assertEqual(b"abcd", bufio.read(6)) - self.assertEqual(b"e", bufio.read(1)) - self.assertEqual(b"fg", bufio.read()) - self.assertEqual(b"", bufio.peek(1)) - self.assertIsNone(bufio.read()) - self.assertEqual(b"", bufio.read()) + def read_via_readinto(bufio, size): + buffer = bytearray(size) + size = bufio.readinto(buffer) + if size is None: + return None + return buffer[:size] + for method in (self.tp.read, read_via_readinto): + with self.subTest(method): + # Inject some None's in there to simulate EWOULDBLOCK + rawio = self.MockRawIO((b"abc", b"d", None, b"efg", + None, # First read() + None, # peek(1) + None, # read(10) + None, # Second read() + )) + bufio = self.tp(rawio) + self.assertEqual(b"abcd", method(bufio, 6)) + self.assertEqual(b"e", method(bufio, 1)) + self.assertEqual(b"fg", bufio.read()) + self.assertIsNone(bufio.peek(1)) + self.assertIsNone(method(bufio, 10)) + self.assertIsNone(bufio.read()) + self.assertEqual(b"", bufio.read()) + + bufio = self.tp(self.MockRawIO((None,))) + self.assertIsNone(bufio.read1(30)) + self.assertEqual(b"", bufio.read1(30)) rawio = self.MockRawIO((b"a", None, None)) self.assertEqual(b"a", rawio.readall()) diff -r 4cb316fe6bf2 Modules/_io/bufferedio.c --- a/Modules/_io/bufferedio.c Tue Feb 03 01:50:31 2015 +0200 +++ b/Modules/_io/bufferedio.c Tue Feb 03 07:03:59 2015 +0000 @@ -35,13 +35,11 @@ "Base class for buffered IO objects.\n" "\n" "The main difference with RawIOBase is that the read() method\n" - "supports omitting the size argument, and does not have a default\n" - "implementation that defers to readinto().\n" + "does not have a default implementation that defers to readinto().\n" "\n" - "In addition, read(), readinto() and write() may raise\n" - "BlockingIOError if the underlying raw stream is in non-blocking\n" - "mode and not ready; unlike their raw counterparts, they will never\n" - "return None.\n" + "In addition, write() may raise BlockingIOError if the underlying raw\n" + "stream is in non-blocking mode and not ready; unlike its raw\n" + "counterpart, its return value is always the total number of bytes.\n" "\n" "A typical implementation should not inherit from a RawIOBase\n" "implementation, but wrap one.\n" @@ -65,6 +63,10 @@ "n", buf.len); if (data == NULL) goto error; + if (data == Py_None) { + PyBuffer_Release(&buf); + return data; + } if (!PyBytes_Check(data)) { Py_DECREF(data); @@ -132,17 +134,13 @@ "If the argument is omitted, None, or negative, reads and\n" "returns all data until EOF.\n" "\n" - "If the argument is positive, and the underlying raw stream is\n" - "not 'interactive', multiple raw reads may be issued to satisfy\n" - "the byte count (unless EOF is reached first). But for\n" - "interactive raw streams (as well as sockets and pipes), at most\n" - "one raw read will be issued, and a short result does not imply\n" - "that EOF is imminent.\n" + "If the argument is positive, multiple raw reads may be issued to\n" + "satisfy the byte count (unless EOF is reached first).\n" "\n" "Returns an empty bytes object on EOF.\n" "\n" - "Returns None if the underlying raw stream was open in non-blocking\n" - "mode and no data is available at the moment.\n"); + "Returns None if the underlying raw stream is in non-blocking mode and\n" + "no data is available at the moment.\n"); static PyObject * bufferediobase_read(PyObject *self, PyObject *args) @@ -983,12 +981,16 @@ _bufferedreader_reset_buf(self); r = _bufferedreader_raw_read(self, PyBytes_AS_STRING(res), n); LEAVE_BUFFERED(self) - if (r == -1) { + if (r < 0) { Py_DECREF(res); - return NULL; + if (r == -2) { /* No non-blocking data */ + Py_INCREF(Py_None); + return Py_None; + } + else { + return NULL; + } } - if (r == -2) - r = 0; if (n > r) _PyBytes_Resize(&res, r); return res; @@ -1758,11 +1760,13 @@ /* Fill the buffer from the raw stream, and copy it to the result. */ _bufferedreader_reset_buf(self); r = _bufferedreader_fill_buffer(self); - if (r == -1) + if (r == -1) /* Exception raised */ return NULL; - if (r == -2) - r = 0; self->pos = 0; + if (r == -2) { /* No non-blocking data */ + Py_INCREF(Py_None); + return Py_None; + } return PyBytes_FromStringAndSize(self->buffer, r); } diff -r 4cb316fe6bf2 Modules/_io/iobase.c --- a/Modules/_io/iobase.c Tue Feb 03 01:50:31 2015 +0200 +++ b/Modules/_io/iobase.c Tue Feb 03 07:03:59 2015 +0000 @@ -837,7 +837,7 @@ PyDoc_STRVAR(rawiobase_readall_doc, - "Read until EOF, using multiple read() call."); + "Read until EOF, using multiple read() calls."); static PyObject * rawiobase_readall(PyObject *self, PyObject *args)