diff --git a/Include/ceval.h b/Include/ceval.h --- a/Include/ceval.h +++ b/Include/ceval.h @@ -69,10 +69,6 @@ PyAPI_FUNC(PyObject *) PyEval_EvalFrame(struct _frame *); PyAPI_FUNC(PyObject *) PyEval_EvalFrameEx(struct _frame *f, int exc); -/* this used to be handled on a per-thread basis - now just two globals */ -PyAPI_DATA(volatile int) _Py_Ticker; -PyAPI_DATA(int) _Py_CheckInterval; - /* Interface for threads. A module that plans to do a blocking system call (or something else @@ -131,6 +127,9 @@ PyAPI_FUNC(void) PyEval_ReleaseThread(PyThreadState *tstate); PyAPI_FUNC(void) PyEval_ReInitThreads(void); +PyAPI_FUNC(void) _PyEval_SetSwitchInterval(unsigned long microseconds); +PyAPI_FUNC(unsigned long) _PyEval_GetSwitchInterval(void); + #define Py_BEGIN_ALLOW_THREADS { \ PyThreadState *_save; \ _save = PyEval_SaveThread(); @@ -149,6 +148,7 @@ #endif /* !WITH_THREAD */ PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *); +PyAPI_FUNC(void) _PyEval_SignalAsyncExc(void); #ifdef __cplusplus diff --git a/Include/pystate.h b/Include/pystate.h --- a/Include/pystate.h +++ b/Include/pystate.h @@ -82,6 +82,8 @@ PyObject *dict; /* Stores per-thread state */ + /* XXX doesn't mean anything anymore (the comment below is obsolete) + => deprecate or remove? */ /* tick_counter is incremented whenever the check_interval ticker * reaches zero. The purpose is to give a useful measure of the number * of interpreted bytecode instructions in a given thread. This diff --git a/Include/sysmodule.h b/Include/sysmodule.h --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -19,7 +19,6 @@ Py_GCC_ATTRIBUTE((format(printf, 1, 2))); PyAPI_DATA(PyObject *) _PySys_TraceFunc, *_PySys_ProfileFunc; -PyAPI_DATA(int) _PySys_CheckInterval; PyAPI_FUNC(void) PySys_ResetWarnOptions(void); PyAPI_FUNC(void) PySys_AddWarnOption(char *); diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -193,6 +193,21 @@ sys.setcheckinterval(n) self.assertEquals(sys.getcheckinterval(), n) + def test_switchinterval(self): + self.assertRaises(TypeError, sys.setswitchinterval) + self.assertRaises(TypeError, sys.setswitchinterval, "a") + self.assertRaises(ValueError, sys.setswitchinterval, -1.0) + self.assertRaises(ValueError, sys.setswitchinterval, 0.0) + orig = sys.getswitchinterval() + # sanity check + self.assertTrue(orig < 0.5, orig) + try: + for n in 0.00001, 0.05, 3.0, orig: + sys.setswitchinterval(n) + self.assertAlmostEquals(sys.getswitchinterval(), n) + finally: + sys.setswitchinterval(orig) + def test_recursionlimit(self): self.assertRaises(TypeError, sys.getrecursionlimit, 42) oldlimit = sys.getrecursionlimit() diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py --- a/Lib/unittest/runner.py +++ b/Lib/unittest/runner.py @@ -8,8 +8,14 @@ class _WritelnDecorator(object): """Used to decorate file-like objects with a handy 'writeln' method""" - def __init__(self,stream): + def __new__(cls, stream): + self = object.__new__(cls) self.stream = stream + return self + + # XXX for test_multiprocessing (!) + def __getnewargs__(self): + return self.stream, def __getattr__(self, attr): if attr in ('stream', '__getstate__'): diff --git a/Objects/longobject.c b/Objects/longobject.c --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -34,10 +34,7 @@ #define MIN(x, y) ((x) > (y) ? (y) : (x)) #define SIGCHECK(PyTryBlock) \ - if (--_Py_Ticker < 0) { \ - _Py_Ticker = _Py_CheckInterval; \ - if (PyErr_CheckSignals()) PyTryBlock \ - } + if (PyErr_CheckSignals()) PyTryBlock \ /* Normalize (remove leading zeros from) a long int object. Doesn't attempt to free the storage--in most cases, due to the nature diff --git a/Python/ceval.c b/Python/ceval.c --- a/Python/ceval.c +++ b/Python/ceval.c @@ -225,6 +225,28 @@ #endif +#define COMPUTE_EVAL_BREAKER() \ + (eval_breaker = gil_drop_request | pendingcalls_to_do | pending_async_exc) + +#define SET_GIL_DROP_REQUEST() \ + do { gil_drop_request = 1; eval_breaker = 1; } while (0) + +#define RESET_GIL_DROP_REQUEST() \ + do { gil_drop_request = 0; COMPUTE_EVAL_BREAKER(); } while (0) + +#define SIGNAL_PENDING_CALLS() \ + do { pendingcalls_to_do = 1; eval_breaker = 1; } while (0) + +#define UNSIGNAL_PENDING_CALLS() \ + do { pendingcalls_to_do = 0; COMPUTE_EVAL_BREAKER(); } while (0) + +#define SIGNAL_ASYNC_EXC() \ + do { pending_async_exc = 1; eval_breaker = 1; } while (0) + +#define UNSIGNAL_ASYNC_EXC() \ + do { pending_async_exc = 0; COMPUTE_EVAL_BREAKER(); } while (0) + + #ifdef WITH_THREAD #ifdef HAVE_ERRNO_H @@ -232,36 +254,55 @@ #endif #include "pythread.h" -static PyThread_type_lock interpreter_lock = 0; /* This is the GIL */ static PyThread_type_lock pending_lock = 0; /* for pending calls */ static long main_thread = 0; +/* This single variable consolidates all requests to break out of the fast path + in the eval loop. */ +static volatile int eval_breaker = 0; +/* Request for droppping the GIL */ +static volatile int gil_drop_request = 0; +/* Request for running pending calls */ +static volatile int pendingcalls_to_do = 0; +/* Request for looking at the `async_exc` field of the current thread state */ +static volatile int pending_async_exc = 0; + +#include "ceval_gil.h" int PyEval_ThreadsInitialized(void) { - return interpreter_lock != 0; + return gil_created(); } void PyEval_InitThreads(void) { - if (interpreter_lock) + if (gil_created()) return; - interpreter_lock = PyThread_allocate_lock(); - PyThread_acquire_lock(interpreter_lock, 1); + create_gil(); + take_gil(PyThreadState_GET()); main_thread = PyThread_get_thread_ident(); + if (!pending_lock) + pending_lock = PyThread_allocate_lock(); } void PyEval_AcquireLock(void) { - PyThread_acquire_lock(interpreter_lock, 1); + PyThreadState *tstate = PyThreadState_GET(); + if (tstate == NULL) + Py_FatalError("PyEval_AcquireLock: current thread state is NULL"); + take_gil(tstate); } void PyEval_ReleaseLock(void) { - PyThread_release_lock(interpreter_lock); + /* This function must succeed when the current thread state is NULL. + We therefore avoid PyThreadState_GET() which dumps a fatal error + in debug mode. + */ + drop_gil(_PyThreadState_Current); } void @@ -270,8 +311,8 @@ if (tstate == NULL) Py_FatalError("PyEval_AcquireThread: NULL new thread state"); /* Check someone has called PyEval_InitThreads() to create the lock */ - assert(interpreter_lock); - PyThread_acquire_lock(interpreter_lock, 1); + assert(gil_created()); + take_gil(tstate); if (PyThreadState_Swap(tstate) != NULL) Py_FatalError( "PyEval_AcquireThread: non-NULL old thread state"); @@ -284,7 +325,7 @@ Py_FatalError("PyEval_ReleaseThread: NULL thread state"); if (PyThreadState_Swap(NULL) != tstate) Py_FatalError("PyEval_ReleaseThread: wrong thread state"); - PyThread_release_lock(interpreter_lock); + drop_gil(tstate); } /* This function is called from PyOS_AfterFork to ensure that newly @@ -296,17 +337,17 @@ PyEval_ReInitThreads(void) { PyObject *threading, *result; - PyThreadState *tstate; - - if (!interpreter_lock) + PyThreadState *tstate = PyThreadState_GET(); + + if (!gil_created()) return; /*XXX Can't use PyThread_free_lock here because it does too much error-checking. Doing this cleanly would require adding a new function to each thread_*.h. Instead, just create a new lock and waste a little bit of memory */ - interpreter_lock = PyThread_allocate_lock(); + recreate_gil(); pending_lock = PyThread_allocate_lock(); - PyThread_acquire_lock(interpreter_lock, 1); + take_gil(tstate); main_thread = PyThread_get_thread_ident(); /* Update the threading module with the new state. @@ -326,7 +367,21 @@ Py_DECREF(result); Py_DECREF(threading); } -#endif + +#else +static int eval_breaker = 0; +static int gil_drop_request = 0; +static int pending_async_exc = 0; +#endif /* WITH_THREAD */ + +/* This function is used to signal that async exceptions are waiting to be + raised, therefore it is also useful in non-threaded builds. */ + +void +_PyEval_SignalAsyncExc(void) +{ + SIGNAL_ASYNC_EXC(); +} /* Functions save_thread and restore_thread are always defined so dynamically loaded modules needn't be compiled separately for use @@ -339,8 +394,8 @@ if (tstate == NULL) Py_FatalError("PyEval_SaveThread: NULL tstate"); #ifdef WITH_THREAD - if (interpreter_lock) - PyThread_release_lock(interpreter_lock); + if (gil_created()) + drop_gil(tstate); #endif return tstate; } @@ -351,9 +406,9 @@ if (tstate == NULL) Py_FatalError("PyEval_RestoreThread: NULL tstate"); #ifdef WITH_THREAD - if (interpreter_lock) { + if (gil_created()) { int err = errno; - PyThread_acquire_lock(interpreter_lock, 1); + take_gil(tstate); errno = err; } #endif @@ -399,7 +454,6 @@ } pendingcalls[NPENDINGCALLS]; static int pendingfirst = 0; static int pendinglast = 0; -static volatile int pendingcalls_to_do = 1; /* trigger initialization of lock */ static char pendingbusy = 0; int @@ -438,8 +492,7 @@ pendinglast = j; } /* signal main loop */ - _Py_Ticker = 0; - pendingcalls_to_do = 1; + SIGNAL_PENDING_CALLS(); if (lock != NULL) PyThread_release_lock(lock); return result; @@ -481,7 +534,10 @@ arg = pendingcalls[j].arg; pendingfirst = (j + 1) % NPENDINGCALLS; } - pendingcalls_to_do = pendingfirst != pendinglast; + if (pendingfirst != pendinglast) + SIGNAL_PENDING_CALLS(); + else + UNSIGNAL_PENDING_CALLS(); PyThread_release_lock(pending_lock); /* having released the lock, perform the callback */ if (func == NULL) @@ -547,8 +603,7 @@ pendingcalls[i].arg = arg; pendinglast = j; - _Py_Ticker = 0; - pendingcalls_to_do = 1; /* Signal main loop */ + SIGNAL_PENDING_CALLS(); busy = 0; /* XXX End critical section */ return 0; @@ -561,7 +616,7 @@ if (busy) return 0; busy = 1; - pendingcalls_to_do = 0; + UNSIGNAL_PENDING_CALLS(); for (;;) { int i; int (*func)(void *); @@ -574,7 +629,7 @@ pendingfirst = (i + 1) % NPENDINGCALLS; if (func(arg) < 0) { busy = 0; - pendingcalls_to_do = 1; /* We're not done yet */ + SIGNAL_PENDING_CALLS(); /* We're not done yet */ return -1; } } @@ -655,10 +710,7 @@ fast_next_opcode*/ static int _Py_TracingPossible = 0; -/* for manipulating the thread switch and periodic "stuff" - used to be - per thread, now just a pair o' globals */ -int _Py_CheckInterval = 100; -volatile int _Py_Ticker = 0; /* so that we hit a "tick" first thing */ + PyObject * PyEval_EvalCode(PyCodeObject *co, PyObject *globals, PyObject *locals) @@ -983,13 +1035,12 @@ async I/O handler); see Py_AddPendingCall() and Py_MakePendingCalls() above. */ - if (--_Py_Ticker < 0) { + if (eval_breaker) { if (*next_instr == SETUP_FINALLY) { /* Make the last opcode before a try: finally: block uninterruptable. */ goto fast_next_opcode; } - _Py_Ticker = _Py_CheckInterval; tstate->tick_counter++; #ifdef WITH_TSC ticked = 1; @@ -999,39 +1050,31 @@ why = WHY_EXCEPTION; goto on_error; } - if (pendingcalls_to_do) - /* MakePendingCalls() didn't succeed. - Force early re-execution of this - "periodic" code, possibly after - a thread switch */ - _Py_Ticker = 0; } + if (gil_drop_request) { #ifdef WITH_THREAD - if (interpreter_lock) { /* Give another thread a chance */ - if (PyThreadState_Swap(NULL) != tstate) Py_FatalError("ceval: tstate mix-up"); - PyThread_release_lock(interpreter_lock); - + drop_gil(tstate); + /* Other threads may run now */ - - PyThread_acquire_lock(interpreter_lock, 1); + + take_gil(tstate); if (PyThreadState_Swap(tstate) != NULL) Py_FatalError("ceval: orphan tstate"); - - /* Check for thread interrupts */ - - if (tstate->async_exc != NULL) { - x = tstate->async_exc; - tstate->async_exc = NULL; - PyErr_SetNone(x); - Py_DECREF(x); - why = WHY_EXCEPTION; - goto on_error; - } +#endif } -#endif + /* Check for asynchronous exceptions. */ + if (tstate->async_exc != NULL) { + x = tstate->async_exc; + tstate->async_exc = NULL; + UNSIGNAL_ASYNC_EXC(); + PyErr_SetNone(x); + Py_DECREF(x); + why = WHY_EXCEPTION; + goto on_error; + } } fast_next_opcode: diff --git a/Python/pystate.c b/Python/pystate.c --- a/Python/pystate.c +++ b/Python/pystate.c @@ -395,6 +395,7 @@ p->async_exc = exc; HEAD_UNLOCK(); Py_XDECREF(old_exc); + _PyEval_SignalAsyncExc(); return 1; } } diff --git a/Python/sysmodule.c b/Python/sysmodule.c --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -461,10 +461,18 @@ See the profiler chapter in the library manual." ); +/* TODO: deprecate */ +static int _check_interval = 100; + static PyObject * sys_setcheckinterval(PyObject *self, PyObject *args) { - if (!PyArg_ParseTuple(args, "i:setcheckinterval", &_Py_CheckInterval)) + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "sys.getcheckinterval() and sys.setcheckinterval() " + "are deprecated. Use sys.setswitchinterval() " + "instead.", 1) < 0) + return NULL; + if (!PyArg_ParseTuple(args, "i:setcheckinterval", &_check_interval)) return NULL; Py_INCREF(Py_None); return Py_None; @@ -480,13 +488,59 @@ static PyObject * sys_getcheckinterval(PyObject *self, PyObject *args) { - return PyInt_FromLong(_Py_CheckInterval); + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "sys.getcheckinterval() and sys.setcheckinterval() " + "are deprecated. Use sys.getswitchinterval() " + "instead.", 1) < 0) + return NULL; + return PyLong_FromLong(_check_interval); } PyDoc_STRVAR(getcheckinterval_doc, "getcheckinterval() -> current check interval; see setcheckinterval()." ); +#ifdef WITH_THREAD +static PyObject * +sys_setswitchinterval(PyObject *self, PyObject *args) +{ + double d; + if (!PyArg_ParseTuple(args, "d:setswitchinterval", &d)) + return NULL; + if (d <= 0.0) { + PyErr_SetString(PyExc_ValueError, + "switch interval must be strictly positive"); + return NULL; + } + _PyEval_SetSwitchInterval((unsigned long) (1e6 * d)); + Py_INCREF(Py_None); + return Py_None; +} + +PyDoc_STRVAR(setswitchinterval_doc, +"setswitchinterval(n)\n\ +\n\ +Set the ideal thread switching delay inside the Python interpreter\n\ +The actual frequency of switching threads can be lower if the\n\ +interpreter executes long sequences of uninterruptible code\n\ +(this is implementation-specific and workload-dependent).\n\ +\n\ +The parameter must represent the desired switching delay in seconds\n\ +A typical value is 0.005 (5 milliseconds)." +); + +static PyObject * +sys_getswitchinterval(PyObject *self, PyObject *args) +{ + return PyFloat_FromDouble(1e-6 * _PyEval_GetSwitchInterval()); +} + +PyDoc_STRVAR(getswitchinterval_doc, +"getswitchinterval() -> current thread switch interval; see setswitchinterval()." +); + +#endif /* WITH_THREAD */ + #ifdef WITH_TSC static PyObject * sys_settscdump(PyObject *self, PyObject *args) @@ -917,6 +971,12 @@ setcheckinterval_doc}, {"getcheckinterval", sys_getcheckinterval, METH_NOARGS, getcheckinterval_doc}, +#ifdef WITH_THREAD + {"setswitchinterval", sys_setswitchinterval, METH_VARARGS, + setswitchinterval_doc}, + {"getswitchinterval", sys_getswitchinterval, METH_NOARGS, + getswitchinterval_doc}, +#endif #ifdef HAVE_DLOPEN {"setdlopenflags", sys_setdlopenflags, METH_VARARGS, setdlopenflags_doc},