diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -29,11 +29,29 @@ Notes: """ -import builtins -import __main__ +import sys __all__ = ["Completer"] +# HACK: the completer function is almost immortal as long as it is +# registered by the readline module (which will only be finalized at +# the very end of the interpreter lifetime). We therefore avoid keeping +# any references to modules such as __main__ or builtins in the rlcompleter +# globals, lest they become immortal too. + +def _module_dict(name): + if name in sys.modules: + return sys.modules[name].__dict__ + else: + return {} + +def _builtins_dict(): + return _module_dict('builtins') + +def _main_dict(): + return _module_dict('__main__') + + class Completer: def __init__(self, namespace = None): """Create a new completer for the command line. @@ -70,7 +88,7 @@ class Completer: """ if self.use_main_ns: - self.namespace = __main__.__dict__ + self.namespace = _main_dict() if state == 0: if "." in text: @@ -100,7 +118,7 @@ class Completer: for word in keyword.kwlist: if word[:n] == text: matches.append(word) - for nspace in [builtins.__dict__, self.namespace]: + for nspace in [_builtins_dict(), self.namespace]: for word, val in nspace.items(): if word[:n] == text and word != "__builtins__": matches.append(self._callable_postfix(val, word)) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -810,7 +810,7 @@ class SizeofTest(unittest.TestCase): # memoryview check(memoryview(b''), size('Pnin 2P2n2i5P 3cPn')) # module - check(unittest, size('PnP')) + check(unittest, size('PnPPP')) # None check(None, size('')) # NotImplementedType diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -11,6 +11,8 @@ typedef struct { PyObject *md_dict; struct PyModuleDef *md_def; void *md_state; + PyObject *md_weaklist; + PyObject *md_name; /* for logging purposes after md_dict is cleared */ } PyModuleObject; static PyMemberDef module_members[] = { @@ -27,7 +29,8 @@ static PyTypeObject moduledef_type = { static int -module_init_dict(PyObject *md_dict, PyObject *name, PyObject *doc) +module_init_dict(PyModuleObject *mod, PyObject *md_dict, + PyObject *name, PyObject *doc) { if (md_dict == NULL) return -1; @@ -42,6 +45,10 @@ module_init_dict(PyObject *md_dict, PyOb return -1; if (PyDict_SetItemString(md_dict, "__loader__", Py_None) != 0) return -1; + if (PyUnicode_CheckExact(name)) { + Py_INCREF(name); + mod->md_name = name; + } return 0; } @@ -56,8 +63,10 @@ PyModule_NewObject(PyObject *name) return NULL; m->md_def = NULL; m->md_state = NULL; + m->md_weaklist = NULL; + m->md_name = NULL; m->md_dict = PyDict_New(); - if (module_init_dict(m->md_dict, name, NULL) != 0) + if (module_init_dict(m, m->md_dict, name, NULL) != 0) goto fail; PyObject_GC_Track(m); return (PyObject *)m; @@ -362,7 +371,7 @@ module_init(PyModuleObject *m, PyObject return -1; m->md_dict = dict; } - if (module_init_dict(dict, name, doc) < 0) + if (module_init_dict(m, dict, name, doc) < 0) return -1; return 0; } @@ -371,12 +380,15 @@ static void module_dealloc(PyModuleObject *m) { PyObject_GC_UnTrack(m); + if (Py_VerboseFlag && m->md_name) { + PySys_FormatStderr("# destroy %S\n", m->md_name); + } + if (m->md_weaklist != NULL) + PyObject_ClearWeakRefs((PyObject *) m); if (m->md_def && m->md_def->m_free) m->md_def->m_free(m); - if (m->md_dict != NULL) { - _PyModule_Clear((PyObject *)m); - Py_DECREF(m->md_dict); - } + Py_XDECREF(m->md_dict); + Py_XDECREF(m->md_name); if (m->md_state != NULL) PyMem_FREE(m->md_state); Py_TYPE(m)->tp_free((PyObject *)m); @@ -522,7 +534,7 @@ PyTypeObject PyModule_Type = { (traverseproc)module_traverse, /* tp_traverse */ (inquiry)module_clear, /* tp_clear */ 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ + offsetof(PyModuleObject, md_weaklist), /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ module_methods, /* tp_methods */ diff --git a/Python/import.c b/Python/import.c --- a/Python/import.c +++ b/Python/import.c @@ -277,6 +277,7 @@ static char* sys_deletes[] = { "path", "argv", "ps1", "ps2", "last_type", "last_value", "last_traceback", "path_hooks", "path_importer_cache", "meta_path", + "__interactivehook__", /* misc stuff */ "flags", "float_info", NULL @@ -323,6 +324,8 @@ PyImport_Cleanup(void) PyObject *key, *value, *dict; PyInterpreterState *interp = PyThreadState_GET()->interp; PyObject *modules = interp->modules; + PyObject *builtins = interp->builtins; + PyObject *weaklist = NULL; if (modules == NULL) return; /* Already done */ @@ -333,6 +336,8 @@ PyImport_Cleanup(void) deleted *last* of all, they would come too late in the normal destruction order. Sigh. */ + /* XXX Perhaps these precautions are obsolete. Who knows? */ + value = PyDict_GetItemString(modules, "builtins"); if (value != NULL && PyModule_Check(value)) { dict = PyModule_GetDict(value); @@ -360,31 +365,39 @@ PyImport_Cleanup(void) } } + /* We prepare a list which will receive (name, weakref) tuples of + modules when they are removed from sys.modules. The name is used + for diagnosis messages (in verbose mode), while the weakref helps + detect those modules which have been held alive. */ + weaklist = PyList_New(0); + +#define STORE_MODULE_WEAKREF(mod) \ + if (weaklist != NULL) { \ + PyObject *name = PyModule_GetNameObject(mod); \ + PyObject *wr = PyWeakref_NewRef(mod, NULL); \ + if (name && wr) { \ + PyObject *tup = PyTuple_Pack(2, name, wr); \ + PyList_Append(weaklist, tup); \ + Py_XDECREF(tup); \ + } \ + Py_XDECREF(name); \ + Py_XDECREF(wr); \ + if (PyErr_Occurred()) \ + PyErr_Clear(); \ + } + /* First, delete __main__ */ value = PyDict_GetItemString(modules, "__main__"); if (value != NULL && PyModule_Check(value)) { if (Py_VerboseFlag) - PySys_WriteStderr("# cleanup __main__\n"); - _PyModule_Clear(value); + PySys_WriteStderr("# removing __main__\n"); + STORE_MODULE_WEAKREF(value); PyDict_SetItemString(modules, "__main__", Py_None); } - /* The special treatment of "builtins" here is because even - when it's not referenced as a module, its dictionary is - referenced by almost every module's __builtins__. Since - deleting a module clears its dictionary (even if there are - references left to it), we need to delete the "builtins" - module last. Likewise, we don't delete sys until the very - end because it is implicitly referenced (e.g. by print). - - Also note that we 'delete' modules by replacing their entry - in the modules dict with None, rather than really deleting - them; this avoids a rehash of the modules dictionary and - also marks them as "non existent" so they won't be - re-imported. */ - /* Next, repeatedly delete modules with a reference count of - one (skipping builtins and sys) and delete them */ + one (skipping essential modules such as those necessary + for standard I/O) and delete them */ do { ndone = 0; pos = 0; @@ -396,23 +409,23 @@ PyImport_Cleanup(void) continue; if (Py_VerboseFlag) PySys_FormatStderr( - "# cleanup[1] %U\n", key); - _PyModule_Clear(value); + "# cleanup[1] removing %U\n", key, value); + STORE_MODULE_WEAKREF(value); PyDict_SetItem(modules, key, Py_None); ndone++; } } } while (ndone > 0); - /* Next, delete all modules (still skipping builtins and sys) */ + /* Next, delete all modules (still skipping essential ones). */ pos = 0; while (PyDict_Next(modules, &pos, &key, &value)) { if (PyUnicode_Check(key) && PyModule_Check(value)) { if (is_essential_module(key)) continue; if (Py_VerboseFlag) - PySys_FormatStderr("# cleanup[2] %U\n", key); - _PyModule_Clear(value); + PySys_FormatStderr("# cleanup[2] removing %U\n", key, value); + STORE_MODULE_WEAKREF(value); PyDict_SetItem(modules, key, Py_None); } } @@ -425,22 +438,56 @@ PyImport_Cleanup(void) machinery. */ _PyGC_DumpShutdownStats(); - /* Next, delete all remaining modules */ + /* Next, delete all remaining modules including essential ones. */ pos = 0; while (PyDict_Next(modules, &pos, &key, &value)) { if (PyUnicode_Check(key) && PyModule_Check(value)) { if (Py_VerboseFlag) - PySys_FormatStderr("# cleanup[3] %U\n", key); - _PyModule_Clear(value); + PySys_FormatStderr("# cleanup[3] removing %U\n", key, value); + STORE_MODULE_WEAKREF(value); PyDict_SetItem(modules, key, Py_None); } } - /* Finally, clear and delete the modules directory */ + /* Clear more references to modules and various other objects */ PyDict_Clear(modules); + builtins = interp->builtins; + interp->builtins = PyDict_New(); + Py_DECREF(builtins); _PyGC_CollectNoFail(); + + /* Now, if there are any modules left alive, clear their globals to + minimize potential leaks. All extension modules actually end + up here, since they are kept alive in the interpreter state. */ + if (weaklist != NULL) { + Py_ssize_t i, n; + n = PyList_GET_SIZE(weaklist); + for (i = 0; i < n; i++) { + PyObject *tup = PyList_GET_ITEM(weaklist, i); + PyObject *mod = PyWeakref_GET_OBJECT(PyTuple_GET_ITEM(tup, 1)); + if (mod == Py_None) + continue; + Py_INCREF(mod); + assert(PyModule_Check(mod)); + if (Py_VerboseFlag) + PySys_FormatStderr("# cleanup[4] wiping %U\n", + PyTuple_GET_ITEM(tup, 0), mod); + _PyModule_Clear(mod); + Py_DECREF(mod); + } + Py_DECREF(weaklist); + } + + /* Clear and delete the modules directory. Actual modules will + be there only if imported during the execution of some + destructor during the steps above. */ interp->modules = NULL; Py_DECREF(modules); + + _PyGC_CollectNoFail(); + /* NOTE: A couple of modules will still be alive in memory, such as _io. */ + +#undef STORE_MODULE_WEAKREF }