diff -r 125e111f2e02 Python/import.c --- a/Python/import.c Wed Jul 14 13:52:38 2010 +0200 +++ b/Python/import.c Wed Jul 14 16:38:15 2010 +0200 @@ -136,6 +136,240 @@ static const struct filedescr _PyImport_ {0, 0} }; +/* Locking primitives to prevent parallel imports of the same module + in different threads to return with a partially loaded module. + These calls are serialized by the global interpreter lock. */ + +#ifdef WITH_THREAD +#include "pythread.h" +#endif + +struct importlock { +#ifdef WITH_THREAD + PyThread_type_lock lock; + long thread_id; +#endif + int level; +}; + +static int +init_import_lock(struct importlock *lock) +{ +#ifdef WITH_THREAD + lock->lock = PyThread_allocate_lock(); + if (lock->lock == NULL) + return 0; /* Nothing much we can do. */ + lock->thread_id = -1; +#endif + lock->level = 0; + return 1; +} + +static void +acquire_import_lock(struct importlock *lock) +{ +#ifdef WITH_THREAD + long me = PyThread_get_thread_ident(); + if (lock->thread_id == me) { + lock->level++; + return; + } + if (lock->thread_id != -1 || !PyThread_acquire_lock(lock->lock, 0)) + { + PyThreadState *tstate = PyEval_SaveThread(); + PyThread_acquire_lock(lock->lock, 1); + PyEval_RestoreThread(tstate); + } + assert(lock->level == 0); + lock->thread_id = me; + lock->level = 1; +#else + lock->level++; + return; +#endif +} + +/* 1 on success, 0 on failure (unacquired lock) */ + +static int +release_import_lock(struct importlock *lock) +{ +#ifdef WITH_THREAD + long me = PyThread_get_thread_ident(); + if (me == -1 || lock->lock == NULL) + return 0; /* Too bad */ + if (lock->thread_id != me) + return 0; + lock->level--; + assert(lock->level >= 0); + if (lock->level == 0) { + lock->thread_id = -1; + PyThread_release_lock(lock->lock); + } + return 1; +#else + if (lock->level <= 0) + return 0; /* Too bad */ + lock->level--; + return 1; +#endif +} + +/* Module-specific import locks */ + +static PyObject *module_locks = NULL; + +static int +init_module_locks(void) +{ + if (module_locks) { + Py_CLEAR(module_locks); + } + module_locks = PyDict_New(); + return module_locks != NULL; +} + +#undef MAX +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#define IMPORTLOCK_ALIGN ((size_t) MAX(sizeof(PyThread_type_lock), sizeof(long))) +/* XXX this assumes that IMPORTLOCK_ALIGN is a power of 2 but this isn't + guaranteed by the above */ +#define IMPORTLOCK_FROM_BUF(b) \ + (struct importlock *) (((size_t) b + IMPORTLOCK_ALIGN - 1) & ~IMPORTLOCK_ALIGN) + +/* + Returns the lock structure for the given module name, or NULL if not + found. + (note: NULL with an exception means another error occurred). +*/ +static struct importlock * +_get_module_lock(const char *fullname) +{ + PyObject *k, *v; + if (module_locks == NULL) + return NULL; + k = PyBytes_FromString(fullname); + if (k == NULL) + return NULL; + v = PyDict_GetItemWithError(module_locks, k); + Py_DECREF(k); + if (v == NULL) + return NULL; + assert(PyByteArray_CheckExact(v)); + return IMPORTLOCK_FROM_BUF(PyByteArray_AS_STRING(v)); +} + +/* NULL if an exception occurred. */ +static struct importlock * +_create_module_lock(const char *fullname) +{ + PyObject *k, *v; + struct importlock *lock; + int r; + if (module_locks == NULL && !init_module_locks()) + return NULL; + k = PyBytes_FromString(fullname); + if (k == NULL) + return NULL; + v = PyByteArray_FromStringAndSize(NULL, sizeof(*lock) + IMPORTLOCK_ALIGN - 1); + if (v == NULL) { + Py_DECREF(k); + return NULL; + } + lock = IMPORTLOCK_FROM_BUF(PyByteArray_AS_STRING(v)); + if (!init_import_lock(lock)) { + PyErr_Format(PyExc_RuntimeError, + "Couldn't initialize import lock for '%.200s'", + fullname); + Py_DECREF(k); + Py_DECREF(v); + return NULL; + } + r = PyDict_SetItem(module_locks, k, v); + Py_DECREF(k); + Py_DECREF(v); + if (r < 0) + return NULL; + return lock; +} + +/* 1 on success, -1 on exception */ + +static int +acquire_module_lock(const char *fullname) +{ + struct importlock *lock; +// fprintf(stderr, "Acquiring module lock for '%s'\n", fullname); + _PyImport_AcquireLock(); + lock = _get_module_lock(fullname); + if (!lock && !PyErr_Occurred()) + lock = _create_module_lock(fullname); + _PyImport_ReleaseLock(); + if (!lock) + return -1; + acquire_import_lock(lock); + return 1; +} + +/* 1 on success, 0 on failure (unacquired lock), -1 on exception */ + +static int +release_module_lock(const char *fullname) +{ + struct importlock *lock; + int r; +// fprintf(stderr, "Releasing module lock for '%s'\n", fullname); + lock = _get_module_lock(fullname); + if (!lock) { + if (!PyErr_Occurred()) + return 0; + return -1; + } + r = release_import_lock(lock); + return r; +} + + +/* XXX we could define them for non-threaded builds too */ +#ifdef WITH_THREAD + +static struct importlock global_import_lock; + +void +_PyImport_AcquireLock(void) +{ + struct importlock *lock = &global_import_lock; + /* This function can be called before _PyImport_Init(), therefore + we have to init the import lock ourselves */ + if (lock->lock == NULL) { + if (init_import_lock(lock) <= 0) + Py_FatalError("Failed to initialized the import lock"); + } + acquire_import_lock(lock); +} + +int +_PyImport_ReleaseLock(void) +{ + struct importlock *lock = &global_import_lock; + return release_import_lock(lock) > 0 ? 1 : -1; +} + +/* This function is called from PyOS_AfterFork to ensure that newly + created child processes do not share locks with the parent. + We now acquire the import lock around fork() calls but on some platforms + (Solaris 9 and earlier? see isue7242) that still left us with problems. */ + +void +_PyImport_ReInitLock(void) +{ + init_import_lock(&global_import_lock); + init_module_locks(); +} + +#endif + + /* Initialize things */ @@ -146,6 +380,12 @@ _PyImport_Init(void) struct filedescr *filetab; int countD = 0; int countS = 0; + struct importlock *lock = &global_import_lock; + + if (lock->lock == NULL && lock->thread_id == 0) { + if (init_import_lock(&global_import_lock) <= 0) + Py_FatalError("Failed to initialized the import lock"); + } /* prepare _PyImport_Filetab: copy entries from _PyImport_DynLoadFiletab and _PyImport_StandardFiletab. @@ -257,81 +497,13 @@ _PyImport_Fini(void) _PyImport_Filetab = NULL; } - -/* Locking primitives to prevent parallel imports of the same module - in different threads to return with a partially loaded module. - These calls are serialized by the global interpreter lock. */ - -#ifdef WITH_THREAD - -#include "pythread.h" - -static PyThread_type_lock import_lock = 0; -static long import_lock_thread = -1; -static int import_lock_level = 0; - -void -_PyImport_AcquireLock(void) -{ - long me = PyThread_get_thread_ident(); - if (me == -1) - return; /* Too bad */ - if (import_lock == NULL) { - import_lock = PyThread_allocate_lock(); - if (import_lock == NULL) - return; /* Nothing much we can do. */ - } - if (import_lock_thread == me) { - import_lock_level++; - return; - } - if (import_lock_thread != -1 || !PyThread_acquire_lock(import_lock, 0)) - { - PyThreadState *tstate = PyEval_SaveThread(); - PyThread_acquire_lock(import_lock, 1); - PyEval_RestoreThread(tstate); - } - import_lock_thread = me; - import_lock_level = 1; -} - -int -_PyImport_ReleaseLock(void) -{ - long me = PyThread_get_thread_ident(); - if (me == -1 || import_lock == NULL) - return 0; /* Too bad */ - if (import_lock_thread != me) - return -1; - import_lock_level--; - if (import_lock_level == 0) { - import_lock_thread = -1; - PyThread_release_lock(import_lock); - } - return 1; -} - -/* This function is called from PyOS_AfterFork to ensure that newly - created child processes do not share locks with the parent. - We now acquire the import lock around fork() calls but on some platforms - (Solaris 9 and earlier? see isue7242) that still left us with problems. */ - -void -_PyImport_ReInitLock(void) -{ - if (import_lock != NULL) - import_lock = PyThread_allocate_lock(); - import_lock_thread = -1; - import_lock_level = 0; -} - -#endif +/* Methods of the `imp` module */ static PyObject * imp_lock_held(PyObject *self, PyObject *noargs) { #ifdef WITH_THREAD - return PyBool_FromLong(import_lock_thread != -1); + return PyBool_FromLong(global_import_lock.thread_id != -1); #else return PyBool_FromLong(0); #endif @@ -2093,7 +2265,11 @@ load_module(char *name, FILE *fp, char * "import hook without loader"); return NULL; } + /* For backwards compatibility with non-thread safe import hooks. + XXX perhaps we should allow hooks to choose this */ + _PyImport_AcquireLock(); m = PyObject_CallMethod(loader, "load_module", "s", name); + _PyImport_ReleaseLock(); break; } @@ -2309,6 +2485,9 @@ PyImport_ImportModule(const char *name) * ImportError instead of blocking. * * Returns the module object with incremented ref count. + * + * XXX this function should be modified in regard of the new module-specific + * locks. */ PyObject * PyImport_ImportModuleNoBlock(const char *name) @@ -2316,6 +2495,7 @@ PyImport_ImportModuleNoBlock(const char PyObject *result; PyObject *modules; long me; + struct importlock *lock = &global_import_lock; /* Try to get the module from sys.modules[name] */ modules = PyImport_GetModuleDict(); @@ -2335,7 +2515,7 @@ PyImport_ImportModuleNoBlock(const char * me might be -1 but I ignore the error here, the lock function * takes care of the problem */ me = PyThread_get_thread_ident(); - if (import_lock_thread == -1 || import_lock_thread == me) { + if (lock->thread_id == -1 || lock->thread_id == me) { /* no thread or me is holding the lock */ return PyImport_ImportModule(name); } @@ -2436,14 +2616,7 @@ PyImport_ImportModuleLevel(char *name, P PyObject *fromlist, int level) { PyObject *result; - _PyImport_AcquireLock(); result = import_module_level(name, globals, locals, fromlist, level); - if (_PyImport_ReleaseLock() < 0) { - Py_XDECREF(result); - PyErr_SetString(PyExc_RuntimeError, - "not holding the import lock"); - return NULL; - } return result; } @@ -2826,6 +2999,7 @@ import_submodule(PyObject *mod, char *su else: mod.__name__ + "." + subname == fullname */ + acquire_module_lock(fullname); if ((m = PyDict_GetItemString(modules, fullname)) != NULL) { Py_INCREF(m); } @@ -2841,8 +3015,9 @@ import_submodule(PyObject *mod, char *su path = PyObject_GetAttrString(mod, "__path__"); if (path == NULL) { PyErr_Clear(); + m = Py_None; Py_INCREF(Py_None); - return Py_None; + goto bail; } } @@ -2852,10 +3027,11 @@ import_submodule(PyObject *mod, char *su Py_XDECREF(path); if (fdp == NULL) { if (!PyErr_ExceptionMatches(PyExc_ImportError)) - return NULL; + goto bail; PyErr_Clear(); + m = Py_None; Py_INCREF(Py_None); - return Py_None; + goto bail; } m = load_module(fullname, fp, buf, fdp->type, loader); Py_XDECREF(loader); @@ -2867,6 +3043,8 @@ import_submodule(PyObject *mod, char *su } } +bail: + release_module_lock(fullname); return m; } @@ -2923,18 +3101,15 @@ PyImport_ReloadModule(PyObject *m) else { PyObject *parentname, *parent; parentname = PyUnicode_FromStringAndSize(name, (subname-name)); - if (parentname == NULL) { - imp_modules_reloading_clear(); - return NULL; - } + if (parentname == NULL) + goto error; parent = PyDict_GetItem(modules, parentname); if (parent == NULL) { PyErr_Format(PyExc_ImportError, "reload(): parent %U not in sys.modules", parentname); Py_DECREF(parentname); - imp_modules_reloading_clear(); - return NULL; + goto error; } Py_DECREF(parentname); subname++; @@ -2946,14 +3121,14 @@ PyImport_ReloadModule(PyObject *m) fdp = find_module(name, subname, path, buf, MAXPATHLEN+1, &fp, &loader); Py_XDECREF(path); - if (fdp == NULL) { - Py_XDECREF(loader); - imp_modules_reloading_clear(); - return NULL; - } - + if (fdp == NULL) + goto error; + + if (acquire_module_lock(name) < 0) + goto error; newm = load_module(name, fp, buf, fdp->type, loader); - Py_XDECREF(loader); + if (release_module_lock(name) < 0) + goto error; if (fp) fclose(fp); @@ -2965,8 +3140,14 @@ PyImport_ReloadModule(PyObject *m) */ PyDict_SetItemString(modules, name, m); } + Py_XDECREF(loader); imp_modules_reloading_clear(); return newm; + +error: + Py_XDECREF(loader); + imp_modules_reloading_clear(); + return NULL; }