From e5373154bccaa9e6ccd2359e88584cedd3dfc570 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Sat, 14 Feb 2009 18:57:07 -0600 Subject: [PATCH] update GC shutdown patch --- Lib/test/test_sys.py | 2 +- Modules/gcmodule.c | 22 ++++++------ Objects/moduleobject.c | 6 +++- Python/import.c | 92 +++++++++++++++++++++++++---------------------- Python/pythonrun.c | 4 +- 5 files changed, 68 insertions(+), 58 deletions(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index bd819c6..09e868c 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -600,7 +600,7 @@ class SizeofTest(unittest.TestCase): check(32768L*32768L-1, size(vh + 'H') + self.H) check(32768L*32768L, size(vh + 'H') + 2*self.H) # module - check(unittest, size(h + 'P')) + check(unittest, size(h + 'PP')) # None check(None, size(h + '')) # object diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 4d71591..42927ce 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -55,7 +55,7 @@ static int enabled = 1; /* automatic collection enabled? */ static int collecting = 0; /* list of uncollectable objects */ -static PyObject *garbage = NULL; +PyObject *_PyGC_garbage = NULL; /* Python string to use if unhandled exception occurs */ static PyObject *gc_str = NULL; @@ -722,16 +722,16 @@ handle_finalizers(PyGC_Head *finalizers, PyGC_Head *old) { PyGC_Head *gc = finalizers->gc.gc_next; - if (garbage == NULL) { - garbage = PyList_New(0); - if (garbage == NULL) + if (_PyGC_garbage == NULL) { + _PyGC_garbage = PyList_New(0); + if (_PyGC_garbage == NULL) Py_FatalError("gc couldn't create gc.garbage list"); } for (; gc != finalizers; gc = gc->gc.gc_next) { PyObject *op = FROM_GC(gc); if ((debug & DEBUG_SAVEALL) || has_finalizer(op)) { - if (PyList_Append(garbage, op) < 0) + if (PyList_Append(_PyGC_garbage, op) < 0) return -1; } } @@ -755,7 +755,7 @@ delete_garbage(PyGC_Head *collectable, PyGC_Head *old) assert(IS_TENTATIVELY_UNREACHABLE(op)); if (debug & DEBUG_SAVEALL) { - PyList_Append(garbage, op); + PyList_Append(_PyGC_garbage, op); } else { if ((clear = Py_TYPE(op)->tp_clear) != NULL) { @@ -1313,13 +1313,13 @@ initgc(void) if (m == NULL) return; - if (garbage == NULL) { - garbage = PyList_New(0); - if (garbage == NULL) + if (_PyGC_garbage == NULL) { + _PyGC_garbage = PyList_New(0); + if (_PyGC_garbage == NULL) return; } - Py_INCREF(garbage); - if (PyModule_AddObject(m, "garbage", garbage) < 0) + Py_INCREF(_PyGC_garbage); + if (PyModule_AddObject(m, "garbage", _PyGC_garbage) < 0) return; /* Importing can't be done in collect() because collect() diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index d1aa771..0c7cd46 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -7,6 +7,7 @@ typedef struct { PyObject_HEAD PyObject *md_dict; + PyObject *md_weakreflist; } PyModuleObject; static PyMemberDef module_members[] = { @@ -24,6 +25,7 @@ PyModule_New(const char *name) return NULL; nameobj = PyString_FromString(name); m->md_dict = PyDict_New(); + m->md_weakreflist = NULL; if (m->md_dict == NULL || nameobj == NULL) goto fail; if (PyDict_SetItemString(m->md_dict, "__name__", nameobj) != 0) @@ -174,6 +176,8 @@ static void module_dealloc(PyModuleObject *m) { PyObject_GC_UnTrack(m); + if (m->md_weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *) m); if (m->md_dict != NULL) { _PyModule_Clear((PyObject *)m); Py_DECREF(m->md_dict); @@ -242,7 +246,7 @@ PyTypeObject PyModule_Type = { (traverseproc)module_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ + offsetof(PyModuleObject, md_weakreflist), /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ 0, /* tp_methods */ diff --git a/Python/import.c b/Python/import.c index b00986c..056b2dd 100644 --- a/Python/import.c +++ b/Python/import.c @@ -84,6 +84,9 @@ static long pyc_magic = MAGIC; /* See _PyImport_FixupExtension() below */ static PyObject *extensions = NULL; +/* Defined in gcmodule.c */ +extern PyObject *_PyGC_garbage; + /* This table is defined in config.c: */ extern struct _inittab _PyImport_Inittab[]; @@ -393,7 +396,7 @@ static char* sys_files[] = { void PyImport_Cleanup(void) { - Py_ssize_t pos, ndone; + Py_ssize_t pos; char *name; PyObject *key, *value, *dict; PyInterpreterState *interp = PyThreadState_GET()->interp; @@ -435,15 +438,6 @@ PyImport_Cleanup(void) } } - /* First, delete __main__ */ - value = PyDict_GetItemString(modules, "__main__"); - if (value != NULL && PyModule_Check(value)) { - if (Py_VerboseFlag) - PySys_WriteStderr("# cleanup __main__\n"); - _PyModule_Clear(value); - PyDict_SetItemString(modules, "__main__", Py_None); - } - /* The special treatment of __builtin__ here is because even when it's not referenced as a module, its dictionary is referenced by almost every module's __builtins__. Since @@ -452,37 +446,14 @@ PyImport_Cleanup(void) module last. Likewise, we don't delete sys until the very end because it is implicitly referenced (e.g. by print). - Also note that we 'delete' modules by replacing their entry - in the modules dict with None, rather than really deleting - them; this avoids a rehash of the modules dictionary and - also marks them as "non existent" so they won't be - re-imported. */ - - /* Next, repeatedly delete modules with a reference count of - one (skipping __builtin__ and sys) and delete them */ - do { - ndone = 0; - pos = 0; - while (PyDict_Next(modules, &pos, &key, &value)) { - if (value->ob_refcnt != 1) - continue; - if (PyString_Check(key) && PyModule_Check(value)) { - name = PyString_AS_STRING(key); - if (strcmp(name, "__builtin__") == 0) - continue; - if (strcmp(name, "sys") == 0) - continue; - if (Py_VerboseFlag) - PySys_WriteStderr( - "# cleanup[1] %s\n", name); - _PyModule_Clear(value); - PyDict_SetItem(modules, key, Py_None); - ndone++; - } - } - } while (ndone > 0); + We delete modules from the modules dict. Most of them will + be deallocated, clearing their globals in the process. - /* Next, delete all modules (still skipping __builtin__ and sys) */ + Also note that we 'delete' modules by replacing their entry + in the modules dict with a (proxy) weak reference object, + rather than really deleting them; this avoids a rehash of + the modules dictionary and is also a convenient way to + keep track of which modules the GC could not release. */ pos = 0; while (PyDict_Next(modules, &pos, &key, &value)) { if (PyString_Check(key) && PyModule_Check(value)) { @@ -492,12 +463,47 @@ PyImport_Cleanup(void) if (strcmp(name, "sys") == 0) continue; if (Py_VerboseFlag) - PySys_WriteStderr("# cleanup[2] %s\n", name); - _PyModule_Clear(value); - PyDict_SetItem(modules, key, Py_None); + PySys_WriteStderr("# unlink %s\n", name); + value = PyWeakref_NewProxy(value, NULL); + if (value == NULL) { + PyErr_Clear(); + value = Py_None; + Py_INCREF(value); + } + PyDict_SetItem(modules, key, value); + Py_DECREF(value); + } + } + + /* Run the GC. This will free e.g. cycles of modules that + import each other. */ + PyGC_Collect(); + + /* Some uncleared modules could remain. This occurs if there + are objects with __del__ methods in a cycle of modules. */ + pos = 0; + while (PyDict_Next(modules, &pos, &key, &value)) { + if (PyString_Check(key) && PyWeakref_Check(value) && + PyModule_Check(PyWeakref_GET_OBJECT(value))) { + if (Py_VerboseFlag) { + name = PyString_AS_STRING(key); + PySys_WriteStderr("# cleanup %s\n", name); + } + _PyModule_Clear(PyWeakref_GET_OBJECT(value)); } } + /* The objects with __del__ methods that were previously added to + gc.garbage could now be freeable, e.g. if they were part of + a cycle involving one of the now-cleared module globals. + Objects that genuinely belong to gc.garbage will just get + re-inserted into the list by the following PyGC_Collect(). */ + if (_PyGC_garbage != NULL) { + (void) PyList_SetSlice(_PyGC_garbage, 0, + PyList_Size(_PyGC_garbage), 0); + PyGC_Collect(); + } + /* Next, delete sys and __builtin__ (in that order) */ value = PyDict_GetItemString(modules, "sys"); if (value != NULL && PyModule_Check(value)) { diff --git a/Python/pythonrun.c b/Python/pythonrun.c index f2cd819..0eb0862 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -433,8 +433,8 @@ Py_Finalize(void) /* Destroy all modules */ PyImport_Cleanup(); - /* Collect final garbage. This disposes of cycles created by - * new-style class definitions, for example. + /* Collect final garbage. This disposes of cycles dependent on + * the __builtin__ and sys modules. * XXX This is disabled because it caused too many problems. If * XXX a __del__ or weakref callback triggers here, Python code has * XXX a hard time running, because even the sys module has been -- 1.5.6.5