diff -r d76fb24d79c3 Lib/test/test_StringIO.py --- a/Lib/test/test_StringIO.py Sun Feb 03 19:25:11 2013 -0500 +++ b/Lib/test/test_StringIO.py Mon Feb 04 09:28:19 2013 +0200 @@ -5,6 +5,7 @@ import cStringIO import types import array +import sys from test import test_support @@ -105,6 +106,21 @@ self._fp.close() self.assertRaises(ValueError, self._fp.getvalue) + # In worst case cStringIO requires 2 + 1 + 1/2 + 1/2**2 + ... = 4 + # bytes per input character. + @test_support.bigmemtest(test_support._2G, memuse=4) + def test_writes_large(self, size): + s = 'x' * 2**26 # 64 MiB + f = self.MODULE.StringIO() + n = size + while n > len(s): + f.write(s) + n -= len(s) + s = None + f.write('x' * n) + self.assertEqual(len(f.getvalue()), size) + + class TestStringIO(TestGenericStringIO): MODULE = StringIO diff -r d76fb24d79c3 Modules/cStringIO.c --- a/Modules/cStringIO.c Sun Feb 03 19:25:11 2013 -0500 +++ b/Modules/cStringIO.c Mon Feb 04 09:28:19 2013 +0200 @@ -170,10 +170,15 @@ n = l; if (n < 0) n=0; } + if (n > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "length too large"); + return -1; + } *output=((IOobject*)self)->buf + ((IOobject*)self)->pos; ((IOobject*)self)->pos += n; - return n; + return (int)n; } static PyObject * @@ -192,26 +197,33 @@ static int IO_creadline(PyObject *self, char **output) { - char *n, *s; - Py_ssize_t l; + char *n, *start, *end; + Py_ssize_t len; if (!IO__opencheck(IOOOBJECT(self))) return -1; - for (n = ((IOobject*)self)->buf + ((IOobject*)self)->pos, - s = ((IOobject*)self)->buf + ((IOobject*)self)->string_size; - n < s && *n != '\n'; n++); + n = start = ((IOobject*)self)->buf + ((IOobject*)self)->pos; + end = ((IOobject*)self)->buf + ((IOobject*)self)->string_size; + while (n < end && *n != '\n') + n++; - if (n < s) n++; + if (n < end) n++; - *output=((IOobject*)self)->buf + ((IOobject*)self)->pos; - l = n - ((IOobject*)self)->buf - ((IOobject*)self)->pos; + len = n - start; + if (len > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "length too large"); + return -1; + } - assert(IOOOBJECT(self)->pos <= PY_SSIZE_T_MAX - l); + *output=start; + + assert(IOOOBJECT(self)->pos <= PY_SSIZE_T_MAX - len); assert(IOOOBJECT(self)->pos >= 0); assert(IOOOBJECT(self)->string_size >= 0); - ((IOobject*)self)->pos += l; - return (int)l; + ((IOobject*)self)->pos += len; + return (int)len; } static PyObject * @@ -239,9 +251,9 @@ int n; char *output; PyObject *result, *line; - int hint = 0, length = 0; + Py_ssize_t hint = 0, length = 0; - if (!PyArg_ParseTuple(args, "|i:readlines", &hint)) return NULL; + if (!PyArg_ParseTuple(args, "|n:readlines", &hint)) return NULL; result = PyList_New(0); if (!result) @@ -377,31 +389,41 @@ static int -O_cwrite(PyObject *self, const char *c, Py_ssize_t l) { - Py_ssize_t newl; +O_cwrite(PyObject *self, const char *c, Py_ssize_t len) { + Py_ssize_t newpos; Oobject *oself; char *newbuf; if (!IO__opencheck(IOOOBJECT(self))) return -1; oself = (Oobject *)self; - newl = oself->pos+l; - if (newl >= oself->buf_size) { - oself->buf_size *= 2; - if (oself->buf_size <= newl) { - assert(newl + 1 < INT_MAX); - oself->buf_size = (int)(newl+1); + if (len > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "length too large"); + return -1; + } + assert(len >= 0); + if (oself->pos >= PY_SSIZE_T_MAX - len) { + PyErr_SetString(PyExc_OverflowError, + "new position too large"); + return -1; + } + newpos = oself->pos + len; + if (newpos >= oself->buf_size) { + size_t newsize = oself->buf_size; + newsize *= 2; + if (newsize <= (size_t)newpos || newsize > PY_SSIZE_T_MAX) { + assert(newpos < PY_SSIZE_T_MAX - 1); + newsize = newpos + 1; } - newbuf = (char*)realloc(oself->buf, oself->buf_size); + newbuf = (char*)realloc(oself->buf, newsize); if (!newbuf) { PyErr_SetString(PyExc_MemoryError,"out of memory"); - free(oself->buf); - oself->buf = 0; - oself->buf_size = oself->pos = 0; return -1; - } + } + oself->buf_size = (Py_ssize_t)newsize; oself->buf = newbuf; - } + } if (oself->string_size < oself->pos) { /* In case of overseek, pad with null bytes the buffer region between @@ -416,16 +438,15 @@ (oself->pos - oself->string_size) * sizeof(char)); } - memcpy(oself->buf+oself->pos,c,l); + memcpy(oself->buf + oself->pos, c, len); - assert(oself->pos + l < INT_MAX); - oself->pos += (int)l; + oself->pos = newpos; if (oself->string_size < oself->pos) { oself->string_size = oself->pos; } - return (int)l; + return (int)len; } static PyObject * @@ -561,7 +582,7 @@ }; static PyObject * -newOobject(int size) { +newOobject(int size) { Oobject *self; self = PyObject_New(Oobject, &Otype);