diff -r 20bd2bdc0a2a Include/fileobject.h --- a/Include/fileobject.h Thu Mar 27 14:34:59 2008 +0100 +++ b/Include/fileobject.h Fri Mar 28 19:07:30 2008 +0100 @@ -9,10 +9,10 @@ extern "C" { typedef struct { PyObject_HEAD - FILE *f_fp; + FILE *volatile f_fp; PyObject *f_name; PyObject *f_mode; - int (*f_close)(FILE *); + int (*volatile f_close)(FILE *); int f_softspace; /* Flag used by 'print' command */ int f_binary; /* Flag which indicates whether the file is open in binary (1) or text (0) mode */ diff -r 20bd2bdc0a2a Lib/test/test_file.py --- a/Lib/test/test_file.py Thu Mar 27 14:34:59 2008 +0100 +++ b/Lib/test/test_file.py Fri Mar 28 19:07:30 2008 +0100 @@ -1,6 +1,9 @@ import sys import sys import os import unittest +import tempfile +import time +import threading from array import array from weakref import proxy @@ -339,11 +342,74 @@ class FileSubclassTests(unittest.TestCas self.failUnless(f.subclass_closed) +class FileThreadingTests(unittest.TestCase): + + def setUp(self): + self.to_remove = set() + self.f = None + + def tearDown(self): + for filename in self.to_remove: + try: + os.remove(filename) + except (IOError, OSError): + pass + if self.f: + try: + self.f.close() + except (IOError, OSError, ValueError): + pass + + def _create_file(self): + filename = tempfile.mktemp() + self.f = open(filename, "w") + self.to_remove.add(filename) + + def _run_workers(self, func, nb_instances=2, duration=0.5): + self.do_continue = True + threads = [] + try: + for i in range(nb_instances): + t = threading.Thread(target=func) + t.start() + threads.append(t) + time.sleep(duration) + finally: + self.do_continue = False + for t in threads: + t.join(1.0) + + def test_close_open(self): + def worker(): + while self.do_continue: + try: + self.f.close() + self._create_file() + except (IOError, OSError, ValueError): + pass + self._create_file() + self._run_workers(worker, 20) + + def test_close_open_seek(self): + def worker(): + while self.do_continue: + try: + self.f.close() + self._create_file() + self.f.seek(0,0) + except (IOError, OSError, ValueError): + pass + self._create_file() + # XXX Increasing the number of workers produces crashes here + self._run_workers(worker, 2) + + def test_main(): # Historically, these tests have been sloppy about removing TESTFN. # So get rid of it no matter what. try: - run_unittest(AutoFileTests, OtherFileTests, FileSubclassTests) + run_unittest(AutoFileTests, OtherFileTests, FileSubclassTests, + FileThreadingTests) finally: if os.path.exists(TESTFN): os.unlink(TESTFN) diff -r 20bd2bdc0a2a Objects/fileobject.c --- a/Objects/fileobject.c Thu Mar 27 14:34:59 2008 +0100 +++ b/Objects/fileobject.c Fri Mar 28 19:07:30 2008 +0100 @@ -47,6 +47,21 @@ #define NEWLINE_CR 1 /* \r newline seen */ #define NEWLINE_LF 2 /* \n newline seen */ #define NEWLINE_CRLF 4 /* \r\n newline seen */ + +/* + These macros allow fetching the FILE pointer to a local variable before + releasing the GIL, so that it isn't modified in our back. + See #815646 +*/ + +#define GET_FILE_BEGIN_ALLOW_THREADS(fobj) \ + { \ + FILE *volatile local_fp = fobj->f_fp; \ + Py_BEGIN_ALLOW_THREADS + +#define GET_FILE_END_ALLOW_THREADS(fobj) \ + Py_END_ALLOW_THREADS \ + } #ifdef __cplusplus extern "C" { @@ -271,6 +286,26 @@ cleanup: return (PyObject *)f; } +static int +close_the_file(PyFileObject *f) +{ + int sts = 0; + int (*volatile local_close)(FILE *); + if (f->f_fp != NULL) { + local_close = f->f_close; + /* NULL out the f_close before releasing the GIL. */ + f->f_close = NULL; + if (local_close != NULL) { + GET_FILE_BEGIN_ALLOW_THREADS(f) + errno = 0; + sts = (*local_close)(local_fp); + GET_FILE_END_ALLOW_THREADS(f) + } + f->f_fp = NULL; + } + return sts; +} + PyObject * PyFile_FromFile(FILE *fp, char *name, char *mode, int (*close)(FILE *)) { @@ -389,13 +424,9 @@ file_dealloc(PyFileObject *f) int sts = 0; if (f->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) f); - if (f->f_fp != NULL && f->f_close != NULL) { - Py_BEGIN_ALLOW_THREADS - sts = (*f->f_close)(f->f_fp); - Py_END_ALLOW_THREADS - if (sts == EOF) - PySys_WriteStderr("close failed: [Errno %d] %s\n", errno, strerror(errno)); - } + sts = close_the_file(f); + if (sts == EOF) + PySys_WriteStderr("close failed: [Errno %d] %s\n", errno, strerror(errno)); PyMem_Free(f->f_setbuf); Py_XDECREF(f->f_name); Py_XDECREF(f->f_mode); @@ -432,16 +463,7 @@ static PyObject * static PyObject * file_close(PyFileObject *f) { - int sts = 0; - if (f->f_fp != NULL) { - if (f->f_close != NULL) { - Py_BEGIN_ALLOW_THREADS - errno = 0; - sts = (*f->f_close)(f->f_fp); - Py_END_ALLOW_THREADS - } - f->f_fp = NULL; - } + int sts = close_the_file(f); PyMem_Free(f->f_setbuf); f->f_setbuf = NULL; if (sts == EOF) @@ -566,14 +588,15 @@ file_seek(PyFileObject *f, PyObject *arg if (PyErr_Occurred()) return NULL; - Py_BEGIN_ALLOW_THREADS + GET_FILE_BEGIN_ALLOW_THREADS(f) errno = 0; - ret = _portable_fseek(f->f_fp, offset, whence); - Py_END_ALLOW_THREADS + ret = _portable_fseek(local_fp, offset, whence); + GET_FILE_END_ALLOW_THREADS(f) if (ret != 0) { PyErr_SetFromErrno(PyExc_IOError); - clearerr(f->f_fp); + if (f->f_fp) + clearerr(f->f_fp); return NULL; } f->f_skipnextlf = 0;