Index: Lib/_pyio.py =================================================================== --- Lib/_pyio.py (revision 73434) +++ Lib/_pyio.py (working copy) @@ -894,7 +894,7 @@ with self._read_lock: return self._read_unlocked(n) - def _read_unlocked(self, n=None): + def _read_unlocked(self, n=None, raw_read_once=False): nodata_val = b"" empty_values = (b"", None) buf = self._read_buf @@ -913,6 +913,8 @@ break current_size += len(chunk) chunks.append(chunk) + if raw_read_once: + break return b"".join(chunks) or nodata_val # The number of bytes to read is specified, return at most n bytes. @@ -932,6 +934,10 @@ break avail += len(chunk) chunks.append(chunk) + if raw_read_once: + # if raw_read_once is set we can only + # do one raw_read() + break # n is more then avail only when an EOF occurred or when # read() would have blocked. n = min(n, avail) @@ -951,15 +957,18 @@ return self._peek_unlocked(n) def _peek_unlocked(self, n=0): - want = min(n, self.buffer_size) - have = len(self._read_buf) - self._read_pos - if have < want or have <= 0: - to_read = self.buffer_size - have - current = self.raw.read(to_read) - if current: - self._read_buf = self._read_buf[self._read_pos:] + current - self._read_pos = 0 - return self._read_buf[self._read_pos:] + if n<=0: + return b"" + else: + want = min(n, self.buffer_size) + have = len(self._read_buf) - self._read_pos + if have < want or have <= 0: + to_read = want - have + current = self.raw.read(to_read) + if current: + self._read_buf = self._read_buf[self._read_pos:] + current + self._read_pos = 0 + return self._read_buf[self._read_pos:self._read_pos+want] def read1(self, n): """Reads up to n bytes, with at most one read() system call.""" @@ -970,9 +979,7 @@ if n == 0: return b"" with self._read_lock: - self._peek_unlocked(1) - return self._read_unlocked( - min(n, len(self._read_buf) - self._read_pos)) + return self._read_unlocked(n, True) def tell(self): return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos Index: Lib/test/test_io.py =================================================================== --- Lib/test/test_io.py (revision 73434) +++ Lib/test/test_io.py (working copy) @@ -659,9 +659,7 @@ self.assertEquals(b"a", bufio.read(1)) self.assertEquals(b"b", bufio.read1(1)) self.assertEquals(rawio._reads, 1) - self.assertEquals(b"c", bufio.read1(100)) - self.assertEquals(rawio._reads, 1) - self.assertEquals(b"d", bufio.read1(100)) + self.assertEquals(b"cd", bufio.read1(100)) self.assertEquals(rawio._reads, 2) self.assertEquals(b"efg", bufio.read1(100)) self.assertEquals(rawio._reads, 3) @@ -669,6 +667,13 @@ # Invalid args self.assertRaises(ValueError, bufio.read1, -1) + def test_peek_limits(self): + bufsize = 100 + rawio = self.MockRawIO([b"ab" * bufsize]) + bufio = self.tp(rawio, buffer_size=bufsize) + self.assertEquals(len(bufio.peek(bufsize+1)), bufsize) + self.assertEquals(b"", bufio.peek(0)) + def test_readinto(self): rawio = self.MockRawIO((b"abc", b"d", b"efg")) bufio = self.tp(rawio) @@ -1273,12 +1278,10 @@ self.check_flush_and_read(_readinto) def test_flush_and_peek(self): - def _peek(bufio, n=-1): - # This relies on the fact that the buffer can contain the whole - # raw stream, otherwise peek() can return less. + def _peek(bufio, n=3): + # The default value for n is 3 because of the + # way check_flush_and_read() works. b = bufio.peek(n) - if n != -1: - b = b[:n] bufio.seek(len(b), 1) return b self.check_flush_and_read(_peek) Index: Modules/_io/bufferedio.c =================================================================== --- Modules/_io/bufferedio.c (revision 73434) +++ Modules/_io/bufferedio.c (working copy) @@ -686,7 +686,7 @@ static PyObject * buffered_peek(buffered *self, PyObject *args) { - Py_ssize_t n = 0; + Py_ssize_t n = 1; PyObject *res = NULL; CHECK_INITIALIZED(self) @@ -750,7 +750,7 @@ buffered_read1(buffered *self, PyObject *args) { Py_ssize_t n, have, r; - PyObject *res = NULL; + PyObject *res = NULL, *extra = NULL; CHECK_INITIALIZED(self) if (!PyArg_ParseTuple(args, "n:read1", &n)) { @@ -784,13 +784,30 @@ have = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); if (have > 0) { - if (n > have) - n = have; - res = PyBytes_FromStringAndSize(self->buffer + self->pos, n); - if (res == NULL) + if (n > have) { + res = PyBytes_FromStringAndSize(self->buffer + self->pos, have); + n -= have; + if (res == NULL) + goto end; + _bufferedreader_reset_buf(self); + r = _bufferedreader_fill_buffer(self); + self->pos = 0; + if (n > r) + n = r; + extra = PyBytes_FromStringAndSize(self->buffer, n); + if (extra == NULL) + goto end; + PyBytes_ConcatAndDel(&res, extra); + self->pos = n; goto end; - self->pos += n; - goto end; + } + else { + res = PyBytes_FromStringAndSize(self->buffer + self->pos, n); + if (res == NULL) + goto end; + self->pos += n; + goto end; + } } /* Fill the buffer from the raw stream, and copy it to the result. */ @@ -1421,31 +1438,59 @@ return NULL; } +/* this function moves the buffer to the position 0 */ +void +_bufferedreader_move_buffer_back(buffered * self) +{ + Py_ssize_t len; + + if (VALID_READ_BUFFER(self)) { + /* when we get here, there's nothing left to write */ + len = Py_SAFE_DOWNCAST(self->read_end - self->pos, Py_off_t, Py_ssize_t); + Py_MEMCPY(self->buffer, self->buffer + self->pos, len); + + if (VALID_WRITE_BUFFER(self)) { + self->write_pos -= self->pos; + self->write_end -= self->pos; + } + self->read_end = len; + self->pos = 0; + } + + +} + static PyObject * _bufferedreader_peek_unlocked(buffered *self, Py_ssize_t n) { Py_ssize_t have, r; - have = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); - /* Constraints: - 1. we don't want to advance the file position. - 2. we don't want to lose block alignment, so we can't shift the buffer - to make some place. - Therefore, we either return `have` bytes (if > 0), or a full buffer. - */ - if (have > 0) { - return PyBytes_FromStringAndSize(self->buffer + self->pos, have); + if(n <= 0){ /* an empty string would be the answer */ + return PyBytes_FromString(""); } - - /* Fill the buffer from the raw stream, and copy it to the result. */ - _bufferedreader_reset_buf(self); - r = _bufferedreader_fill_buffer(self); - if (r == -1) - return NULL; - if (r == -2) - r = 0; - self->pos = 0; - return PyBytes_FromStringAndSize(self->buffer, r); + else { + have = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); + if (have > 0) { + if (have >= n) + return PyBytes_FromStringAndSize(self->buffer + self->pos, n); + else { + _bufferedreader_move_buffer_back(self); + _bufferedreader_fill_buffer(self); + have = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); + return PyBytes_FromStringAndSize(self->buffer + self->pos, (havepos = 0; + return PyBytes_FromStringAndSize(self->buffer, (r