Under very low memory condition, PyErr_NoMemory() or PyErr_NormalizeException() enters an unlimited loop when the free list of MemoryError becomes empty. I propose to add a MemoryError read-only singleton to fix this corner case. Attributes cannot be modified, new attributes cannot be added. Attributes values: * __cause__ = None * __context__ = None * __suppress_context__ = False * __traceback__ = None * args = () PyException_SetTraceback(), PyException_SetCause() and PyException_SetContext() do nothing when calling on the MemoryError singleton. A MemoryError can be raised on an integer overflow, when computing the size of a memory block. In this case, you may still have available free memory and so you may be able to build a traceback object, chain exceptions, etc. If you consider that attributes must be modifiable in this case, we can keep the MemoryError unchanged and keep the free list, but add a new special "ReadOnlyMemoryError" type which has exactly one instance, preallocated at startup. Read-only attributes are required to not keep references on objects when the MemoryError (singleton) is "destroyed": see for example test_memory_error_cleanup() of test_exceptions. Python has already a PyExc_RecursionErrorInst singleton, which is used on recursion error. It is just a preallocated RuntimeError error, attributes are modifiable. Does this exception keep a traceback when it is no more used ("destroyed")? Maybe using read-only attributes is overkill? Replacing the free list with a singleton is enough? diff -r 6a3e09cd96f3 Objects/exceptions.c --- a/Objects/exceptions.c Fri Nov 29 12:19:53 2013 +0200 +++ b/Objects/exceptions.c Fri Nov 29 17:57:32 2013 +0100 @@ -23,6 +23,9 @@ PyObject *PyExc_VMSError = NULL; /* The dict map from errno codes to OSError subclasses */ static PyObject *errnomap = NULL; +/* Preallocated MemoryError singleton */ +static PyObject *MemoryError_singleton = NULL; + /* NOTE: If the exception class hierarchy changes, don't forget to update * Lib/test/exception_hierarchy.txt @@ -321,6 +324,9 @@ PyException_GetTraceback(PyObject *self) int PyException_SetTraceback(PyObject *self, PyObject *tb) { + /* MemoryError singleton is read-only */ + if (self == MemoryError_singleton) + return 0; return BaseException_set_tb((PyBaseExceptionObject *)self, tb); } @@ -334,9 +340,16 @@ PyException_GetCause(PyObject *self) { /* Steals a reference to cause */ void PyException_SetCause(PyObject *self, PyObject *cause) { - PyObject *old_cause = ((PyBaseExceptionObject *)self)->cause; - ((PyBaseExceptionObject *)self)->cause = cause; - ((PyBaseExceptionObject *)self)->suppress_context = 1; + PyBaseExceptionObject *exc = (PyBaseExceptionObject *)self; + PyObject *old_cause; + + /* MemoryError singleton is read-only */ + if (self == MemoryError_singleton) + return; + + old_cause = exc->cause; + exc->cause = cause; + exc->suppress_context = 1; Py_XDECREF(old_cause); } @@ -350,8 +363,15 @@ PyException_GetContext(PyObject *self) { /* Steals a reference to context */ void PyException_SetContext(PyObject *self, PyObject *context) { - PyObject *old_context = ((PyBaseExceptionObject *)self)->context; - ((PyBaseExceptionObject *)self)->context = context; + PyBaseExceptionObject *exc = (PyBaseExceptionObject *)self; + PyObject *old_context; + + /* MemoryError singleton is read-only */ + if (self == MemoryError_singleton) + return; + + old_context = exc->context; + exc->context = context; Py_XDECREF(old_context); } @@ -2122,87 +2142,106 @@ SimpleExtendsException(PyExc_Exception, * MemoryError extends Exception */ -#define MEMERRORS_SAVE 16 -static PyBaseExceptionObject *memerrors_freelist = NULL; -static int memerrors_numfree = 0; - static PyObject * MemoryError_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyBaseExceptionObject *self; + PyObject *obj; if (type != (PyTypeObject *) PyExc_MemoryError) return BaseException_new(type, args, kwds); - if (memerrors_freelist == NULL) - return BaseException_new(type, args, kwds); - /* Fetch object from freelist and revive it */ - self = memerrors_freelist; - self->args = PyTuple_New(0); - /* This shouldn't happen since the empty tuple is persistent */ - if (self->args == NULL) + if (MemoryError_singleton == NULL) { + obj = BaseException_new(type, args, kwds); + if (obj != NULL) { + Py_INCREF(obj); + MemoryError_singleton = obj; + } + return obj; + } + Py_INCREF(MemoryError_singleton); + return MemoryError_singleton; +} + +static PyObject* +MemoryError_get_dict(PyObject *obj, void *context) +{ + PyObject *dict, *proxy; + + dict = PyObject_GenericGetDict(obj, context); + if (dict == NULL) return NULL; - memerrors_freelist = (PyBaseExceptionObject *) self->dict; - memerrors_numfree--; - self->dict = NULL; - _Py_NewReference((PyObject *)self); - _PyObject_GC_TRACK(self); - return (PyObject *)self; + + proxy = PyDictProxy_New(dict); + Py_DECREF(dict); + return proxy; } -static void -MemoryError_dealloc(PyBaseExceptionObject *self) +int +MemoryError_GenericSetAttr(PyObject *obj, PyObject *name, PyObject *value) { - _PyObject_GC_UNTRACK(self); - BaseException_clear(self); - if (memerrors_numfree >= MEMERRORS_SAVE) - Py_TYPE(self)->tp_free((PyObject *)self); - else { - self->dict = (PyObject *) memerrors_freelist; - memerrors_freelist = self; - memerrors_numfree++; - } + PyErr_SetString(PyExc_AttributeError, + "MemoryError object cannot be modified"); + return -1; } -static void -preallocate_memerrors(void) -{ - /* We create enough MemoryErrors and then decref them, which will fill - up the freelist. */ - int i; - PyObject *errors[MEMERRORS_SAVE]; - for (i = 0; i < MEMERRORS_SAVE; i++) { - errors[i] = MemoryError_new((PyTypeObject *) PyExc_MemoryError, - NULL, NULL); - if (!errors[i]) - Py_FatalError("Could not preallocate MemoryError object"); - } - for (i = 0; i < MEMERRORS_SAVE; i++) { - Py_DECREF(errors[i]); - } -} - -static void -free_preallocated_memerrors(void) -{ - while (memerrors_freelist != NULL) { - PyObject *self = (PyObject *) memerrors_freelist; - memerrors_freelist = (PyBaseExceptionObject *) memerrors_freelist->dict; - Py_TYPE(self)->tp_free((PyObject *)self); - } -} +static struct PyMemberDef MemoryError_members[] = { + {"__suppress_context__", T_BOOL, + offsetof(PyBaseExceptionObject, suppress_context), + READONLY}, + {NULL} +}; + +static PyGetSetDef MemoryError_getset[] = { + {"__dict__", MemoryError_get_dict, NULL}, + {"args", (getter)BaseException_get_args, NULL}, + {"__traceback__", (getter)BaseException_get_tb, NULL}, + {"__context__", (getter)BaseException_get_context, + NULL, PyDoc_STR("exception context")}, + {"__cause__", (getter)BaseException_get_cause, + NULL, PyDoc_STR("exception cause")}, + {NULL}, +}; static PyTypeObject _PyExc_MemoryError = { PyVarObject_HEAD_INIT(NULL, 0) "MemoryError", sizeof(PyBaseExceptionObject), - 0, (destructor)MemoryError_dealloc, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + MemoryError_GenericSetAttr, /* tp_setattro */ + 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - PyDoc_STR("Out of memory."), (traverseproc)BaseException_traverse, - (inquiry)BaseException_clear, 0, 0, 0, 0, 0, 0, 0, &_PyExc_Exception, - 0, 0, 0, offsetof(PyBaseExceptionObject, dict), - (initproc)BaseException_init, 0, MemoryError_new + PyDoc_STR("Out of memory."), + (traverseproc)BaseException_traverse, + (inquiry)BaseException_clear, + 0, + 0, + 0, + 0, + 0, + MemoryError_members, /* tp_members */ + MemoryError_getset, /* tp_getset */ + &_PyExc_Exception, + 0, + 0, + 0, + offsetof(PyBaseExceptionObject, + dict), + (initproc)BaseException_init, + 0, + MemoryError_new }; PyObject *PyExc_MemoryError = (PyObject *) &_PyExc_MemoryError; @@ -2387,7 +2426,7 @@ PyObject *PyExc_RecursionErrorInst = NUL void _PyExc_Init(PyObject *bltinmod) { - PyObject *bdict; + PyObject *bdict, *v; PRE_INIT(BaseException) PRE_INIT(Exception) @@ -2558,7 +2597,11 @@ void POST_INIT(TimeoutError); ADD_ERRNO(TimeoutError, ETIMEDOUT); - preallocate_memerrors(); + /* allocate */ + v = MemoryError_new((PyTypeObject *) PyExc_MemoryError, NULL, NULL); + Py_XDECREF(v); + if (MemoryError_singleton == NULL) + Py_FatalError("Cannot allocate MemoryError singleton"); if (!PyExc_RecursionErrorInst) { PyExc_RecursionErrorInst = BaseException_new(&_PyExc_RuntimeError, NULL, NULL); @@ -2590,7 +2633,7 @@ void _PyExc_Fini(void) { Py_CLEAR(PyExc_RecursionErrorInst); - free_preallocated_memerrors(); + Py_CLEAR(MemoryError_singleton); Py_CLEAR(errnomap); }