Index: Include/pystate.h =================================================================== RCS file: /cvsroot/python/python/dist/src/Include/pystate.h,v retrieving revision 2.23 diff -u -r2.23 pystate.h --- Include/pystate.h 19 Mar 2003 00:35:35 -0000 2.23 +++ Include/pystate.h 19 Apr 2003 02:31:23 -0000 @@ -72,6 +72,7 @@ PyObject *dict; int tick_counter; + int gilstate_counter; /* XXX signal handlers should also be here */ @@ -103,6 +104,51 @@ #else #define PyThreadState_GET() (_PyThreadState_Current) #endif + +typedef + enum {PyGILState_LOCKED, PyGILState_UNLOCKED} + PyGILState_STATE; + +/* Ensure that the current thread is ready to call the Python + C API, regardless of the current state of Python, or of its + thread lock. This may be called as many times as desired + by a thread so long as each call is matched with a call to + PyGILState_Release(). In general, other thread-state APIs may + be used between _Ensure() and _Release() calls, so long as the + thread-state is restored to its previous state before the Release(). + For example, normal use of the Py_BEGIN_ALLOW_THREADS/ + Py_END_ALLOW_THREADS macros are acceptable. + + The return value is an opaque "handle" to the thread state when + PyGILState_Acquire() was called, and must be passed to + PyGILState_Release() to ensure Python is left in the same state. Even + though recursive calls are allowed, these handles can *not* be shared - + each unique call to PyGILState_Ensure must save the handle for its + call to PyGILState_Release. + + When the function returns, the current thread will hold the GIL. + + Failure is a fatal error. +*/ +PyAPI_FUNC(PyGILState_STATE) PyGILState_Ensure(void); + +/* Release any resources previously acquired. After this call, Python's + state will be the same as it was prior to the corresponding + PyGILState_Acquire call (but generally this state will be unknown to + the caller, hence the use of the GILState API.) + + Every call to PyGILState_Ensure must be matched by a call to + PyGILState_Release on the same thread. +*/ +PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE); + +/* Helper/diagnostic function - get the current thread state for + this thread. May return NULL if no GILState API has been used + on the current thread. Note the main thread always has such a + thread-state, even if no auto-thread-state call has been made + on the main thread. +*/ +PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); /* Routines for advanced debuggers, requested by David Beazley. Don't use unless you know what you are doing! */ Index: Include/pythread.h =================================================================== RCS file: /cvsroot/python/python/dist/src/Include/pythread.h,v retrieving revision 2.20 diff -u -r2.20 pythread.h --- Include/pythread.h 12 Aug 2002 07:21:57 -0000 2.20 +++ Include/pythread.h 19 Apr 2003 02:31:23 -0000 @@ -30,10 +30,12 @@ PyAPI_FUNC(void) PyThread__PyThread_exit_prog(int); #endif +/* Thread Local Storage (TLS) API */ PyAPI_FUNC(int) PyThread_create_key(void); PyAPI_FUNC(void) PyThread_delete_key(int); PyAPI_FUNC(int) PyThread_set_key_value(int, void *); PyAPI_FUNC(void *) PyThread_get_key_value(int); +PyAPI_FUNC(void) PyThread_delete_key_value(int key); #ifdef __cplusplus } Index: Lib/test/test_capi.py =================================================================== RCS file: /cvsroot/python/python/dist/src/Lib/test/test_capi.py,v retrieving revision 1.4 diff -u -r1.4 test_capi.py --- Lib/test/test_capi.py 23 Jul 2002 19:03:45 -0000 1.4 +++ Lib/test/test_capi.py 19 Apr 2003 02:31:24 -0000 @@ -14,3 +14,32 @@ test() except _testcapi.error: raise test_support.TestFailed, sys.exc_info()[1] + +# some extra thread-state tests driven via _testcapi +def TestThreadState(): + import thread + import time + + if test_support.verbose: + print "auto-thread-state" + + idents = [] + + def callback(): + idents.append(thread.get_ident()) + + _testcapi._test_thread_state(callback) + time.sleep(1) + # Check our main thread is in the list exactly 3 times. + if idents.count(thread.get_ident()) != 3: + raise test_support.TestFailed, \ + "Couldn't find main thread correctly in the list" + +try: + _testcapi._test_thread_state + have_thread_state = True +except AttributeError: + have_thread_state = False + +if have_thread_state: + TestThreadState() Index: Modules/_testcapimodule.c =================================================================== RCS file: /cvsroot/python/python/dist/src/Modules/_testcapimodule.c,v retrieving revision 1.21 diff -u -r1.21 _testcapimodule.c --- Modules/_testcapimodule.c 17 Apr 2003 18:55:25 -0000 1.21 +++ Modules/_testcapimodule.c 19 Apr 2003 02:31:25 -0000 @@ -7,6 +7,10 @@ #include "Python.h" +#ifdef WITH_THREAD +#include "pythread.h" +#endif /* WITH_THREAD */ + static PyObject *TestError; /* set to exception object in init */ /* Raise TestError with test_name + ": " + msg, and return NULL. */ @@ -535,6 +539,46 @@ return NULL; } +#ifdef WITH_THREAD + +void _make_call(void *callable) +{ + PyObject *rc; + PyGILState_STATE s = PyGILState_Ensure(); + rc = PyObject_CallFunction(callable, ""); + Py_XDECREF(rc); + PyGILState_Release(s); +} + +static PyObject * +test_thread_state(PyObject *self, PyObject *args) +{ + PyObject *fn; + if (!PyArg_ParseTuple(args, "O:test_thread_state", &fn)) + return NULL; + /* Ensure Python is setup for threading */ + PyEval_InitThreads(); + /* Start a new thread for our callback. */ + PyThread_start_new_thread( _make_call, fn); + /* Make the callback with the thread lock held by this thread */ + _make_call(fn); + /* Do it all again, but this time with the thread-lock released */ + Py_BEGIN_ALLOW_THREADS + _make_call(fn); + Py_END_ALLOW_THREADS + /* And once more with and without a thread + XXX - should use a lock and work out exactly what we are trying + to test + */ + Py_BEGIN_ALLOW_THREADS + PyThread_start_new_thread( _make_call, fn); + _make_call(fn); + Py_END_ALLOW_THREADS + Py_INCREF(Py_None); + return Py_None; +} +#endif + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, {"test_config", (PyCFunction)test_config, METH_NOARGS}, @@ -553,6 +597,9 @@ #endif #ifdef Py_USING_UNICODE {"test_u_code", (PyCFunction)test_u_code, METH_NOARGS}, +#endif +#ifdef WITH_THREAD + {"_test_thread_state", (PyCFunction)test_thread_state, METH_VARARGS}, #endif {NULL, NULL} /* sentinel */ }; Index: Modules/posixmodule.c =================================================================== RCS file: /cvsroot/python/python/dist/src/Modules/posixmodule.c,v retrieving revision 2.294 diff -u -r2.294 posixmodule.c --- Modules/posixmodule.c 29 Mar 2003 10:04:54 -0000 2.294 +++ Modules/posixmodule.c 19 Apr 2003 02:31:28 -0000 @@ -4354,22 +4354,11 @@ * exit code as the result of the close() operation. This permits the * files to be closed in any order - it is always the close() of the * final handle that will return the exit code. + * + * NOTE: This function is currently called with the GIL released. + * hence we use the GILState API to manage our state. */ - /* RED_FLAG 31-Aug-2000 Tim - * This is always called (today!) between a pair of - * Py_BEGIN_ALLOW_THREADS/ Py_END_ALLOW_THREADS - * macros. So the thread running this has no valid thread state, as - * far as Python is concerned. However, this calls some Python API - * functions that cannot be called safely without a valid thread - * state, in particular PyDict_GetItem. - * As a temporary hack (although it may last for years ...), we - * *rely* on not having a valid thread state in this function, in - * order to create our own "from scratch". - * This will deadlock if _PyPclose is ever called by a thread - * holding the global lock. - */ - static int _PyPclose(FILE *file) { int result; @@ -4378,40 +4367,16 @@ PyObject *procObj, *hProcessObj, *intObj, *fileObj; long file_count; #ifdef WITH_THREAD - PyInterpreterState* pInterpreterState; - PyThreadState* pThreadState; + PyGILState_STATE state; #endif /* Close the file handle first, to ensure it can't block the * child from exiting if it's the last handle. */ result = fclose(file); - #ifdef WITH_THREAD - /* Bootstrap a valid thread state into existence. */ - pInterpreterState = PyInterpreterState_New(); - if (!pInterpreterState) { - /* Well, we're hosed now! We don't have a thread - * state, so can't call a nice error routine, or raise - * an exception. Just die. - */ - Py_FatalError("unable to allocate interpreter state " - "when closing popen object"); - return -1; /* unreachable */ - } - pThreadState = PyThreadState_New(pInterpreterState); - if (!pThreadState) { - Py_FatalError("unable to allocate thread state " - "when closing popen object"); - return -1; /* unreachable */ - } - /* Grab the global lock. Note that this will deadlock if the - * current thread already has the lock! (see RED_FLAG comments - * before this function) - */ - PyEval_RestoreThread(pThreadState); + state = PyGILState_Ensure(); #endif - if (_PyPopenProcs) { if ((fileObj = PyLong_FromVoidPtr(file)) != NULL && (procObj = PyDict_GetItem(_PyPopenProcs, @@ -4470,17 +4435,8 @@ } /* if _PyPopenProcs */ #ifdef WITH_THREAD - /* Tear down the thread & interpreter states. - * Note that interpreter state clear & delete functions automatically - * call the thread clear & delete functions, and indeed insist on - * doing that themselves. The lock must be held during the clear, but - * need not be held during the delete. - */ - PyInterpreterState_Clear(pInterpreterState); - PyEval_ReleaseThread(pThreadState); - PyInterpreterState_Delete(pInterpreterState); + PyGILState_Release(state); #endif - return result; } Index: Python/ceval.c =================================================================== RCS file: /cvsroot/python/python/dist/src/Python/ceval.c,v retrieving revision 2.358 diff -u -r2.358 ceval.c --- Python/ceval.c 9 Apr 2003 19:06:17 -0000 2.358 +++ Python/ceval.c 19 Apr 2003 02:31:30 -0000 @@ -321,6 +321,8 @@ { if (tstate == NULL) Py_FatalError("PyEval_AcquireThread: NULL new thread state"); + /* Check someone has called PyEval_InitThreads() to create the lock */ + assert(interpreter_lock); PyThread_acquire_lock(interpreter_lock, 1); if (PyThreadState_Swap(tstate) != NULL) Py_FatalError( Index: Python/pystate.c =================================================================== RCS file: /cvsroot/python/python/dist/src/Python/pystate.c,v retrieving revision 2.25 diff -u -r2.25 pystate.c --- Python/pystate.c 15 Apr 2003 15:12:39 -0000 2.25 +++ Python/pystate.c 19 Apr 2003 02:31:31 -0000 @@ -142,6 +142,7 @@ tstate->tracing = 0; tstate->use_tracing = 0; tstate->tick_counter = 0; + tstate->gilstate_counter = 0; tstate->dict = NULL; @@ -259,7 +260,17 @@ PyThreadState *old = _PyThreadState_Current; _PyThreadState_Current = new; - + /* It should not be possible for more than one thread state + to be used for a thread. Check this the best we can in debug + builds. + */ +#if defined(Py_DEBUG) + if (new) { + PyThreadState *check = PyGILState_GetThisThreadState(); + if (check && check != new) + Py_FatalError("Invalid thread state for this thread"); + } +#endif return old; } @@ -308,3 +319,131 @@ PyThreadState_Next(PyThreadState *tstate) { return tstate->next; } + +/* Python "auto thread state" API. */ +#ifdef WITH_THREAD + +/* Keep this as a static, as it is not reliable! It can only + ever be compared to the state for the *current* thread. + * If not equal, then it doesn't matter that the actual + value may change immediately after comparison, as it can't + possibly change to the current thread's state. + * If equal, then the current thread holds the lock, so the value can't + change until we yield the lock. +*/ +static int +PyThreadState_IsCurrent(PyThreadState *tstate) +{ + /* Must be the tstate for this thread */ + assert(PyGILState_GetThisThreadState()==tstate); + /* On Windows at least, simple reads and writes to 32 bit values + are atomic. + */ + return tstate == _PyThreadState_Current; +} + +/* The single PyInterpreterState used by this process' + GILState implementation +*/ +static PyInterpreterState *autoInterpreterState = NULL; +static int autoTLSkey = 0; + +/* Internal initialization/finalization functions called by + Py_Initialize/Py_Finalize +*/ +void _PyGILState_Init(PyInterpreterState *i, PyThreadState *t) +{ + assert(i && t); /* must init with a valid states */ + autoTLSkey = PyThread_create_key(); + autoInterpreterState = i; + /* Now stash the thread state for this thread in TLS */ + PyThread_set_key_value(autoTLSkey, (void *)t); + assert(t->gilstate_counter==0); /* must be a new thread state */ + t->gilstate_counter = 1; +} + +void _PyGILState_Fini(void) +{ + PyThread_delete_key(autoTLSkey); + autoTLSkey = 0; + autoInterpreterState = NULL;; +} + +/* The public functions */ +PyThreadState *PyGILState_GetThisThreadState(void) +{ + if (autoInterpreterState==NULL || autoTLSkey==0) + return NULL; + return (PyThreadState *) PyThread_get_key_value(autoTLSkey); +} + +PyGILState_STATE PyGILState_Ensure(void) +{ + int current; + PyThreadState *tcur; + /* Note that we do not auto-init Python here - apart from + potential races with 2 threads auto-initializing, pep-311 + spells out other issues. Embedders are expected to have + called Py_Initialize() and usually PyEval_InitThreads(). + */ + assert(autoInterpreterState); /* Py_Initialize() hasn't been called! */ + tcur = PyThread_get_key_value(autoTLSkey); + if (tcur==NULL) { + /* Create a new thread state for this thread */ + tcur = PyThreadState_New(autoInterpreterState); + if (tcur==NULL) + Py_FatalError("Couldn't create thread-state for new thread"); + PyThread_set_key_value(autoTLSkey, (void *)tcur); + current = 0; /* new thread state is never current */ + } else + current = PyThreadState_IsCurrent(tcur); + if (!current) + PyEval_RestoreThread(tcur); + /* Update our counter in the thread-state - no need for locks: + - tcur will remain valid as we hold the GIL. + - the counter is safe as we are the only thread "allowed" + to modify this value + */ + tcur->gilstate_counter++; + return current ? PyGILState_LOCKED : PyGILState_UNLOCKED; +} + +void PyGILState_Release(PyGILState_STATE oldstate) +{ + PyThreadState *tcur = PyThread_get_key_value(autoTLSkey); + if (tcur==NULL) + Py_FatalError("auto-releasing thread-state, " + "but no thread-state for this thread"); + /* We must hold the GIL and have our thread state current */ + /* XXX - remove the check - the assert should be fine, + but while this is very new (April 2003), the extra check + by release-only users can't hurt. + */ + if (!PyThreadState_IsCurrent(tcur)) + Py_FatalError("This thread state must be current when releasing"); + assert (PyThreadState_IsCurrent(tcur)); + tcur->gilstate_counter -= 1; + assert (tcur->gilstate_counter >= 0); /* illegal counter value */ + + /* If we are about to destroy this thread-state, we must + clear it while the lock is held, as destructors may run + */ + if (tcur->gilstate_counter==0) { + /* can't have been locked when we created it */ + assert(oldstate==PyGILState_UNLOCKED); + PyThreadState_Clear(tcur); + } + + /* Release the lock if necessary */ + if (oldstate==PyGILState_UNLOCKED) + PyEval_ReleaseThread(tcur); + + /* Now complete destruction of the thread if necessary */ + if (tcur->gilstate_counter==0) { + /* Delete this thread from our TLS */ + PyThread_delete_key_value(autoTLSkey); + /* Delete the thread-state */ + PyThreadState_Delete(tcur); + } +} +#endif /* WITH_THREAD */ Index: Python/pythonrun.c =================================================================== RCS file: /cvsroot/python/python/dist/src/Python/pythonrun.c,v retrieving revision 2.190 diff -u -r2.190 pythonrun.c --- Python/pythonrun.c 17 Apr 2003 19:52:29 -0000 2.190 +++ Python/pythonrun.c 19 Apr 2003 02:31:38 -0000 @@ -50,6 +50,11 @@ extern void _PyUnicode_Init(void); extern void _PyUnicode_Fini(void); +#ifdef WITH_THREAD +extern void _PyGILState_Init(PyInterpreterState *, PyThreadState *); +extern void _PyGILState_Fini(void); +#endif /* WITH_THREAD */ + int Py_DebugFlag; /* Needed by parser.c */ int Py_VerboseFlag; /* Needed by import.c */ int Py_InteractiveFlag; /* Needed by Py_FdIsInteractive() below */ @@ -180,6 +185,11 @@ if (!Py_NoSiteFlag) initsite(); /* Module site */ + /* auto-thread-state API, if available */ +#ifdef WITH_THREAD + _PyGILState_Init(interp, tstate); +#endif /* WITH_THREAD */ + PyModule_WarningsModule = PyImport_ImportModule("warnings"); #if defined(Py_USING_UNICODE) && defined(HAVE_LANGINFO_H) && defined(CODESET) @@ -243,6 +253,11 @@ */ call_sys_exitfunc(); initialized = 0; + + /* Cleanup auto-thread-state */ +#ifdef WITH_THREAD + _PyGILState_Fini(); +#endif /* WITH_THREAD */ /* Get current thread state and interpreter pointer */ tstate = PyThreadState_Get(); Index: Python/thread.c =================================================================== RCS file: /cvsroot/python/python/dist/src/Python/thread.c,v retrieving revision 2.45 diff -u -r2.45 thread.c --- Python/thread.c 19 Jul 2002 06:55:41 -0000 2.45 +++ Python/thread.c 19 Apr 2003 02:31:38 -0000 @@ -137,3 +137,111 @@ #include "thread_foobar.h" #endif */ + +#ifndef Py_HAVE_NATIVE_TLS +/* If the platform has not supplied a platform specific + TLS implementation, provide our own. + + This code stolen from "thread_sgi.h", where it was the only + implementation of an existing Python TLS API. +*/ +/* + * Per-thread data ("key") support. + */ + +struct key { + struct key *next; + long id; + int key; + void *value; +}; + +static struct key *keyhead = NULL; +static int nkeys = 0; +static PyThread_type_lock keymutex = NULL; + +static struct key *find_key(int key, void *value) +{ + struct key *p; + long id = PyThread_get_thread_ident(); + for (p = keyhead; p != NULL; p = p->next) { + if (p->id == id && p->key == key) + return p; + } + if (value == NULL) + return NULL; + p = (struct key *)malloc(sizeof(struct key)); + if (p != NULL) { + p->id = id; + p->key = key; + p->value = value; + PyThread_acquire_lock(keymutex, 1); + p->next = keyhead; + keyhead = p; + PyThread_release_lock(keymutex); + } + return p; +} + +int PyThread_create_key(void) +{ + if (keymutex == NULL) + keymutex = PyThread_allocate_lock(); + return ++nkeys; +} + +void PyThread_delete_key(int key) +{ + struct key *p, **q; + PyThread_acquire_lock(keymutex, 1); + q = &keyhead; + while ((p = *q) != NULL) { + if (p->key == key) { + *q = p->next; + free((void *)p); + /* NB This does *not* free p->value! */ + } + else + q = &p->next; + } + PyThread_release_lock(keymutex); +} + +int PyThread_set_key_value(int key, void *value) +{ + struct key *p = find_key(key, value); + if (p == NULL) + return -1; + else + return 0; +} + +void *PyThread_get_key_value(int key) +{ + struct key *p = find_key(key, NULL); + if (p == NULL) + return NULL; + else + return p->value; +} + +void PyThread_delete_key_value(int key) +{ + long id = PyThread_get_thread_ident(); + struct key *p, **q; + PyThread_acquire_lock(keymutex, 1); + q = &keyhead; + while ((p = *q) != NULL) { + if (p->key == key && p->id == id) { + *q = p->next; + free((void *)p); + /* NB This does *not* free p->value! */ + break; + } + else + q = &p->next; + } + PyThread_release_lock(keymutex); +} + +#endif /* Py_HAVE_NATIVE_TLS */ Index: Python/thread_sgi.h =================================================================== RCS file: /cvsroot/python/python/dist/src/Python/thread_sgi.h,v retrieving revision 2.16 diff -u -r2.16 thread_sgi.h --- Python/thread_sgi.h 19 Jan 2002 22:02:55 -0000 2.16 +++ Python/thread_sgi.h 19 Apr 2003 02:31:39 -0000 @@ -377,83 +377,3 @@ if (usunsetlock((ulock_t) lock) < 0) perror("usunsetlock"); } - -/* - * Per-thread data ("key") support. - */ - -struct key { - struct key *next; - long id; - int key; - void *value; -}; - -static struct key *keyhead = NULL; -static int nkeys = 0; -static PyThread_type_lock keymutex = NULL; - -static struct key *find_key(int key, void *value) -{ - struct key *p; - long id = PyThread_get_thread_ident(); - for (p = keyhead; p != NULL; p = p->next) { - if (p->id == id && p->key == key) - return p; - } - if (value == NULL) - return NULL; - p = (struct key *)malloc(sizeof(struct key)); - if (p != NULL) { - p->id = id; - p->key = key; - p->value = value; - PyThread_acquire_lock(keymutex, 1); - p->next = keyhead; - keyhead = p; - PyThread_release_lock(keymutex); - } - return p; -} - -int PyThread_create_key(void) -{ - if (keymutex == NULL) - keymutex = PyThread_allocate_lock(); - return ++nkeys; -} - -void PyThread_delete_key(int key) -{ - struct key *p, **q; - PyThread_acquire_lock(keymutex, 1); - q = &keyhead; - while ((p = *q) != NULL) { - if (p->key == key) { - *q = p->next; - free((void *)p); - /* NB This does *not* free p->value! */ - } - else - q = &p->next; - } - PyThread_release_lock(keymutex); -} - -int PyThread_set_key_value(int key, void *value) -{ - struct key *p = find_key(key, value); - if (p == NULL) - return -1; - else - return 0; -} - -void *PyThread_get_key_value(int key) -{ - struct key *p = find_key(key, NULL); - if (p == NULL) - return NULL; - else - return p->value; -}