changeset:   100525:109fdcc140fd
tag:         tip
user:        Victor Stinner <victor.stinner@gmail.com>
date:        Mon Mar 14 15:40:14 2016 +0100
files:       Include/pystate.h Objects/obmalloc.c Parser/pgenmain.c Python/fileutils.c Python/pylifecycle.c Python/pystate.c
description:
Issue #26558: add more checks on the GIL

* PyGILState_Check() now returns 1 (success) before the creation of the GIL and
  after the destruction of the GIL
* Add a flag to disable PyGILState_Check(). Disable PyGILState_Check() when
  Py_NewInterpreter() is called
* Add assert(PyGILState_Check()) to:

  - _Py_dup()
  - _Py_fstat()
  - _Py_read()
  - _Py_write()
  - PyObject_Malloc()
  - PyObject_Calloc()
  - PyObject_Realloc()
  - PyObject_Free()


diff -r aa280432e9c7 -r 109fdcc140fd Include/pystate.h
--- a/Include/pystate.h	Mon Mar 14 12:04:26 2016 +0100
+++ b/Include/pystate.h	Mon Mar 14 15:40:14 2016 +0100
@@ -247,6 +247,10 @@ PyAPI_FUNC(PyThreadState *) PyGILState_G
  * currently holds the GIL, 0 otherwise
  */
 #ifndef Py_LIMITED_API
+/* Issue #26558: Flag to disable PyGILState_Check().
+   If set, PyGILState_Check() always return 1. */
+PyAPI_DATA(int) _PyGILState_check_enabled;
+
 PyAPI_FUNC(int) PyGILState_Check(void);
 #endif
 
diff -r aa280432e9c7 -r 109fdcc140fd Objects/obmalloc.c
--- a/Objects/obmalloc.c	Mon Mar 14 12:04:26 2016 +0100
+++ b/Objects/obmalloc.c	Mon Mar 14 15:40:14 2016 +0100
@@ -467,6 +467,10 @@ char *
 void *
 PyObject_Malloc(size_t size)
 {
+#ifdef WITH_THREAD
+    assert(PyGILState_Check());
+#endif
+
     /* see PyMem_RawMalloc() */
     if (size > (size_t)PY_SSIZE_T_MAX)
         return NULL;
@@ -476,6 +480,10 @@ PyObject_Malloc(size_t size)
 void *
 PyObject_Calloc(size_t nelem, size_t elsize)
 {
+#ifdef WITH_THREAD
+    assert(PyGILState_Check());
+#endif
+
     /* see PyMem_RawMalloc() */
     if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
         return NULL;
@@ -485,6 +493,10 @@ PyObject_Calloc(size_t nelem, size_t els
 void *
 PyObject_Realloc(void *ptr, size_t new_size)
 {
+#ifdef WITH_THREAD
+    assert(PyGILState_Check());
+#endif
+
     /* see PyMem_RawMalloc() */
     if (new_size > (size_t)PY_SSIZE_T_MAX)
         return NULL;
@@ -494,6 +506,10 @@ PyObject_Realloc(void *ptr, size_t new_s
 void
 PyObject_Free(void *ptr)
 {
+#ifdef WITH_THREAD
+    assert(PyGILState_Check());
+#endif
+
     _PyObject.free(_PyObject.ctx, ptr);
 }
 
diff -r aa280432e9c7 -r 109fdcc140fd Parser/pgenmain.c
--- a/Parser/pgenmain.c	Mon Mar 14 12:04:26 2016 +0100
+++ b/Parser/pgenmain.c	Mon Mar 14 15:40:14 2016 +0100
@@ -37,6 +37,14 @@ Py_Exit(int sts)
     exit(sts);
 }
 
+#ifdef WITH_THREAD
+/* Needed by obmalloc.c */
+int PyGILState_Check(void)
+{
+    return 1;
+}
+#endif
+
 int
 main(int argc, char **argv)
 {
diff -r aa280432e9c7 -r 109fdcc140fd Python/fileutils.c
--- a/Python/fileutils.c	Mon Mar 14 12:04:26 2016 +0100
+++ b/Python/fileutils.c	Mon Mar 14 15:40:14 2016 +0100
@@ -683,6 +683,10 @@ int
 {
     int res;
 
+#ifdef WITH_THREAD
+    assert(PyGILState_Check());
+#endif
+
     Py_BEGIN_ALLOW_THREADS
     res = _Py_fstat_noraise(fd, status);
     Py_END_ALLOW_THREADS
@@ -1164,6 +1168,10 @@ Py_ssize_t
     int err;
     int async_err = 0;
 
+#ifdef WITH_THREAD
+    assert(PyGILState_Check());
+#endif
+
     /* _Py_read() must not be called with an exception set, otherwise the
      * caller may think that read() was interrupted by a signal and the signal
      * handler raised an exception. */
@@ -1319,6 +1327,10 @@ static Py_ssize_t
 Py_ssize_t
 _Py_write(int fd, const void *buf, size_t count)
 {
+#ifdef WITH_THREAD
+    assert(PyGILState_Check());
+#endif
+
     /* _Py_write() must not be called with an exception set, otherwise the
      * caller may think that write() was interrupted by a signal and the signal
      * handler raised an exception. */
@@ -1468,6 +1480,10 @@ int
     DWORD ftype;
 #endif
 
+#ifdef WITH_THREAD
+    assert(PyGILState_Check());
+#endif
+
     if (!_PyVerify_fd(fd)) {
         PyErr_SetFromErrno(PyExc_OSError);
         return -1;
diff -r aa280432e9c7 -r 109fdcc140fd Python/pylifecycle.c
--- a/Python/pylifecycle.c	Mon Mar 14 12:04:26 2016 +0100
+++ b/Python/pylifecycle.c	Mon Mar 14 15:40:14 2016 +0100
@@ -326,6 +326,9 @@ void
         Py_FatalError("Py_Initialize: can't make first thread");
     (void) PyThreadState_Swap(tstate);
 
+    /* Python GIL is ready: we can start to check the GIL */
+    _PyGILState_check_enabled = 1;
+
 #ifdef WITH_THREAD
     /* We can't call _PyEval_FiniThreads() in Py_FinalizeEx because
        destroying the GIL might fail when it is being referenced from
@@ -692,6 +695,7 @@ Py_FinalizeEx(void)
 
     /* Delete current thread. After this, many C API calls become crashy. */
     PyThreadState_Swap(NULL);
+
     PyInterpreterState_Delete(interp);
 
 #ifdef Py_TRACE_REFS
@@ -743,6 +747,10 @@ Py_NewInterpreter(void)
     if (!initialized)
         Py_FatalError("Py_NewInterpreter: call Py_Initialize first");
 
+    /* Issue #26558: Disable PyGILState_Check() when multiple interpreters
+       are used. */
+    _PyGILState_check_enabled = 0;
+
     interp = PyInterpreterState_New();
     if (interp == NULL)
         return NULL;
diff -r aa280432e9c7 -r 109fdcc140fd Python/pystate.c
--- a/Python/pystate.c	Mon Mar 14 12:04:26 2016 +0100
+++ b/Python/pystate.c	Mon Mar 14 15:40:14 2016 +0100
@@ -34,6 +34,8 @@ to avoid the expense of doing their own 
 extern "C" {
 #endif
 
+int _PyGILState_check_enabled = 1;
+
 #ifdef WITH_THREAD
 #include "pythread.h"
 static PyThread_type_lock head_mutex = NULL; /* Protects interp->tstate_head */
@@ -45,7 +47,7 @@ static PyThread_type_lock head_mutex = N
    GILState implementation
 */
 static PyInterpreterState *autoInterpreterState = NULL;
-static int autoTLSkey = 0;
+static int autoTLSkey = -1;
 #else
 #define HEAD_INIT() /* Nothing */
 #define HEAD_LOCK() /* Nothing */
@@ -449,10 +451,10 @@ PyThreadState_DeleteCurrent()
     if (tstate == NULL)
         Py_FatalError(
             "PyThreadState_DeleteCurrent: no current tstate");
-    SET_TSTATE(NULL);
+    tstate_delete_common(tstate);
     if (autoInterpreterState && PyThread_get_key_value(autoTLSkey) == tstate)
         PyThread_delete_key_value(autoTLSkey);
-    tstate_delete_common(tstate);
+    SET_TSTATE(NULL);
     PyEval_ReleaseLock();
 }
 #endif /* WITH_THREAD */
@@ -716,6 +718,7 @@ void
 _PyGILState_Fini(void)
 {
     PyThread_delete_key(autoTLSkey);
+    autoTLSkey = -1;
     autoInterpreterState = NULL;
 }
 
@@ -784,8 +787,19 @@ PyGILState_GetThisThreadState(void)
 int
 PyGILState_Check(void)
 {
-    PyThreadState *tstate = GET_TSTATE();
-    return tstate && (tstate == PyGILState_GetThisThreadState());
+    PyThreadState *tstate;
+
+    if (!_PyGILState_check_enabled)
+        return 1;
+
+    if (autoTLSkey == -1)
+        return 1;
+
+    tstate = GET_TSTATE();
+    if (tstate == NULL)
+        return 0;
+
+    return (tstate == PyGILState_GetThisThreadState());
 }
 
 PyGILState_STATE