diff -r 416dcc95b56d Include/pyerrors.h --- a/Include/pyerrors.h Thu Oct 14 19:03:51 2010 +0200 +++ b/Include/pyerrors.h Thu Oct 14 21:41:42 2010 +0200 @@ -170,6 +170,7 @@ PyAPI_DATA(PyObject *) PyExc_FutureWarni PyAPI_DATA(PyObject *) PyExc_ImportWarning; PyAPI_DATA(PyObject *) PyExc_UnicodeWarning; PyAPI_DATA(PyObject *) PyExc_BytesWarning; +PyAPI_DATA(PyObject *) PyExc_ResourceWarning; /* Convenience functions */ diff -r 416dcc95b56d Lib/base64.py --- a/Lib/base64.py Thu Oct 14 19:03:51 2010 +0200 +++ b/Lib/base64.py Thu Oct 14 21:41:42 2010 +0200 @@ -383,7 +383,8 @@ def main(): if o == '-u': func = decode if o == '-t': test(); return if args and args[0] != '-': - func(open(args[0], 'rb'), sys.stdout.buffer) + with open(args[0], 'rb') as f: + func(f, sys.stdout.buffer) else: func(sys.stdin.buffer, sys.stdout.buffer) diff -r 416dcc95b56d Lib/mimetypes.py --- a/Lib/mimetypes.py Thu Oct 14 19:03:51 2010 +0200 +++ b/Lib/mimetypes.py Thu Oct 14 21:41:42 2010 +0200 @@ -199,9 +199,8 @@ class MimeTypes: list of standard types, else to the list of non-standard types. """ - fp = open(filename) - self.readfp(fp, strict) - fp.close() + with open(filename) as fp: + self.readfp(fp, strict) def readfp(self, fp, strict=True): """ @@ -348,7 +347,7 @@ def init(files=None): files = knownfiles for file in files: if os.path.isfile(file): - db.readfp(open(file)) + db.read(file) encodings_map = db.encodings_map suffix_map = db.suffix_map types_map = db.types_map[True] diff -r 416dcc95b56d Lib/socket.py --- a/Lib/socket.py Thu Oct 14 19:03:51 2010 +0200 +++ b/Lib/socket.py Thu Oct 14 21:41:42 2010 +0200 @@ -108,7 +108,7 @@ class socket(_socket.socket): if s.startswith(" + >>> with open(SIMPLE_XMLFILE) as f: + ... data = f.read() + >>> parser = ET.XMLParser() >>> parser.version # doctest: +ELLIPSIS 'Expat ...' - >>> parser.feed(open(SIMPLE_XMLFILE).read()) + >>> parser.feed(data) >>> print(serialize(parser.close())) text @@ -614,7 +617,7 @@ def parsefile(): >>> parser = ET.XMLTreeBuilder() # 1.2 compatibility - >>> parser.feed(open(SIMPLE_XMLFILE).read()) + >>> parser.feed(data) >>> print(serialize(parser.close())) text @@ -624,7 +627,7 @@ def parsefile(): >>> target = ET.TreeBuilder() >>> parser = ET.XMLParser(target=target) - >>> parser.feed(open(SIMPLE_XMLFILE).read()) + >>> parser.feed(data) >>> print(serialize(parser.close())) text @@ -727,7 +730,8 @@ def iterparse(): end-ns None >>> events = ("start", "end", "bogus") - >>> context = iterparse(SIMPLE_XMLFILE, events) + >>> with open(SIMPLE_XMLFILE, "rb") as f: + ... iterparse(f, events) Traceback (most recent call last): ValueError: unknown event 'bogus' @@ -779,6 +783,8 @@ def custom_builder(): """ Test parser w. custom builder. + >>> with open(SIMPLE_XMLFILE) as f: + ... data = f.read() >>> class Builder: ... def start(self, tag, attrib): ... print("start", tag) @@ -788,7 +794,7 @@ def custom_builder(): ... pass >>> builder = Builder() >>> parser = ET.XMLParser(target=builder) - >>> parser.feed(open(SIMPLE_XMLFILE, "r").read()) + >>> parser.feed(data) start root start element end element @@ -798,6 +804,8 @@ def custom_builder(): end empty-element end root + >>> with open(SIMPLE_NS_XMLFILE) as f: + ... data = f.read() >>> class Builder: ... def start(self, tag, attrib): ... print("start", tag) @@ -811,7 +819,7 @@ def custom_builder(): ... print("comment", repr(data)) >>> builder = Builder() >>> parser = ET.XMLParser(target=builder) - >>> parser.feed(open(SIMPLE_NS_XMLFILE, "r").read()) + >>> parser.feed(data) pi pi 'data' comment ' comment ' start {namespace}root @@ -829,7 +837,8 @@ def getchildren(): """ Test Element.getchildren() - >>> tree = ET.parse(open(SIMPLE_XMLFILE, "rb")) + >>> with open(SIMPLE_XMLFILE, "rb") as f: + ... tree = ET.parse(f) >>> for elem in tree.getroot().iter(): ... summarize_list(elem.getchildren()) ['element', 'element', 'empty-element'] diff -r 416dcc95b56d Lib/warnings.py --- a/Lib/warnings.py Thu Oct 14 19:03:51 2010 +0200 +++ b/Lib/warnings.py Thu Oct 14 21:41:42 2010 +0200 @@ -383,4 +383,10 @@ if not _warnings_defaults: else: bytes_action = "ignore" simplefilter(bytes_action, category=BytesWarning, append=1) + # Resource consumption warnings are enabled by default in pydebug mode + if hasattr(sys, 'gettotalrefcount'): + resource_action = "always" + else: + resource_action = "ignore" + simplefilter(resource_action, category=ResourceWarning, append=1) del _warnings_defaults diff -r 416dcc95b56d Lib/xml/etree/ElementTree.py --- a/Lib/xml/etree/ElementTree.py Thu Oct 14 19:03:51 2010 +0200 +++ b/Lib/xml/etree/ElementTree.py Thu Oct 14 21:41:42 2010 +0200 @@ -662,17 +662,23 @@ class ElementTree: # @exception ParseError If the parser fails to parse the document. def parse(self, source, parser=None): + close_source = False if not hasattr(source, "read"): source = open(source, "rb") - if not parser: - parser = XMLParser(target=TreeBuilder()) - while 1: - data = source.read(65536) - if not data: - break - parser.feed(data) - self._root = parser.close() - return self._root + close_source = True + try: + if not parser: + parser = XMLParser(target=TreeBuilder()) + while 1: + data = source.read(65536) + if not data: + break + parser.feed(data) + self._root = parser.close() + return self._root + finally: + if close_source: + source.close() ## # Creates a tree iterator for the root element. The iterator loops @@ -1226,16 +1232,19 @@ def parse(source, parser=None): # @return A (event, elem) iterator. def iterparse(source, events=None, parser=None): + close_source = False if not hasattr(source, "read"): source = open(source, "rb") + close_source = True if not parser: parser = XMLParser(target=TreeBuilder()) - return _IterParseIterator(source, events, parser) + return _IterParseIterator(source, events, parser, close_source) class _IterParseIterator: - def __init__(self, source, events, parser): + def __init__(self, source, events, parser, close_source=False): self._file = source + self._close_file = close_source self._events = [] self._index = 0 self.root = self._root = None @@ -1282,6 +1291,8 @@ class _IterParseIterator: except IndexError: if self._parser is None: self.root = self._root + if self._close_file: + self._file.close() raise StopIteration # load event buffer del self._events[:] diff -r 416dcc95b56d Modules/_elementtree.c --- a/Modules/_elementtree.c Thu Oct 14 19:03:51 2010 +0200 +++ b/Modules/_elementtree.c Thu Oct 14 21:41:42 2010 +0200 @@ -2946,19 +2946,25 @@ PyInit__elementtree(void) "class ElementTree(ET.ElementTree):\n" /* public */ " def parse(self, source, parser=None):\n" + " close_source = False\n" " if not hasattr(source, 'read'):\n" " source = open(source, 'rb')\n" - " if parser is not None:\n" - " while 1:\n" - " data = source.read(65536)\n" - " if not data:\n" - " break\n" - " parser.feed(data)\n" - " self._root = parser.close()\n" - " else:\n" - " parser = cElementTree.XMLParser()\n" - " self._root = parser._parse(source)\n" - " return self._root\n" + " close_source = True\n" + " try:\n" + " if parser is not None:\n" + " while 1:\n" + " data = source.read(65536)\n" + " if not data:\n" + " break\n" + " parser.feed(data)\n" + " self._root = parser.close()\n" + " else:\n" + " parser = cElementTree.XMLParser()\n" + " self._root = parser._parse(source)\n" + " return self._root\n" + " finally:\n" + " if close_source:\n" + " source.close()\n" "cElementTree.ElementTree = ElementTree\n" "def iter(node, tag=None):\n" /* helper */ @@ -2988,8 +2994,10 @@ PyInit__elementtree(void) "class iterparse:\n" " root = None\n" " def __init__(self, file, events=None):\n" + " self._close_file = False\n" " if not hasattr(file, 'read'):\n" " file = open(file, 'rb')\n" + " self._close_file = True\n" " self._file = file\n" " self._events = []\n" " self._index = 0\n" @@ -3004,6 +3012,8 @@ PyInit__elementtree(void) " except IndexError:\n" " if self._parser is None:\n" " self.root = self._root\n" + " if self._close_file:\n" + " self._file.close()\n" " raise StopIteration\n" " # load event buffer\n" " del self._events[:]\n" diff -r 416dcc95b56d Modules/_io/bufferedio.c --- a/Modules/_io/bufferedio.c Thu Oct 14 19:03:51 2010 +0200 +++ b/Modules/_io/bufferedio.c Thu Oct 14 21:41:42 2010 +0200 @@ -197,6 +197,7 @@ typedef struct { int detached; int readable; int writable; + int deallocating; /* True if this is a vanilla Buffered object (rather than a user derived class) *and* the raw stream is a vanilla FileIO object. */ @@ -342,6 +343,7 @@ typedef struct { static void buffered_dealloc(buffered *self) { + self->deallocating = 1; if (self->ok && _PyIOBase_finalize((PyObject *) self) < 0) return; _PyObject_GC_UNTRACK(self); @@ -382,6 +384,23 @@ buffered_clear(buffered *self) return 0; } +/* Because this can call arbitrary code, it shouldn't be called when + the refcount is 0 (that is, not directly from tp_dealloc unless + the refcount has been temporarily re-incremented). */ +PyObject * +buffered_dealloc_warn(buffered *self, PyObject *source) +{ + if (self->ok && self->raw) { + PyObject *r; + r = PyObject_CallMethod(self->raw, "_dealloc_warn", "O", source); + if (r) + Py_DECREF(r); + else + PyErr_Clear(); + } + Py_RETURN_NONE; +} + /* * _BufferedIOMixin methods * This is not a class, just a collection of methods that will be reused @@ -435,6 +454,14 @@ buffered_close(buffered *self, PyObject Py_INCREF(res); goto end; } + + if (self->deallocating) { + PyObject *r = buffered_dealloc_warn(self, (PyObject *) self); + if (r) + Py_DECREF(r); + else + PyErr_Clear(); + } /* flush() will most probably re-take the lock, so drop it first */ LEAVE_BUFFERED(self) res = PyObject_CallMethodObjArgs((PyObject *)self, _PyIO_str_flush, NULL); @@ -1461,6 +1488,7 @@ static PyMethodDef bufferedreader_method {"writable", (PyCFunction)buffered_writable, METH_NOARGS}, {"fileno", (PyCFunction)buffered_fileno, METH_NOARGS}, {"isatty", (PyCFunction)buffered_isatty, METH_NOARGS}, + {"_dealloc_warn", (PyCFunction)buffered_dealloc_warn, METH_O}, {"read", (PyCFunction)buffered_read, METH_VARARGS}, {"peek", (PyCFunction)buffered_peek, METH_VARARGS}, @@ -1843,6 +1871,7 @@ static PyMethodDef bufferedwriter_method {"writable", (PyCFunction)buffered_writable, METH_NOARGS}, {"fileno", (PyCFunction)buffered_fileno, METH_NOARGS}, {"isatty", (PyCFunction)buffered_isatty, METH_NOARGS}, + {"_dealloc_warn", (PyCFunction)buffered_dealloc_warn, METH_O}, {"write", (PyCFunction)bufferedwriter_write, METH_VARARGS}, {"truncate", (PyCFunction)buffered_truncate, METH_VARARGS}, @@ -2227,6 +2256,7 @@ static PyMethodDef bufferedrandom_method {"writable", (PyCFunction)buffered_writable, METH_NOARGS}, {"fileno", (PyCFunction)buffered_fileno, METH_NOARGS}, {"isatty", (PyCFunction)buffered_isatty, METH_NOARGS}, + {"_dealloc_warn", (PyCFunction)buffered_dealloc_warn, METH_O}, {"flush", (PyCFunction)buffered_flush, METH_NOARGS}, @@ -2296,4 +2326,3 @@ PyTypeObject PyBufferedRandom_Type = { 0, /* tp_alloc */ PyType_GenericNew, /* tp_new */ }; - diff -r 416dcc95b56d Modules/_io/fileio.c --- a/Modules/_io/fileio.c Thu Oct 14 19:03:51 2010 +0200 +++ b/Modules/_io/fileio.c Thu Oct 14 21:41:42 2010 +0200 @@ -2,6 +2,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "structmember.h" #ifdef HAVE_SYS_TYPES_H #include #endif @@ -55,6 +56,7 @@ typedef struct { unsigned int writable : 1; signed int seekable : 2; /* -1 means unknown */ unsigned int closefd : 1; + unsigned int deallocating: 1; PyObject *weakreflist; PyObject *dict; } fileio; @@ -69,6 +71,26 @@ _PyFileIO_closed(PyObject *self) return ((fileio *)self)->fd < 0; } +/* Because this can call arbitrary code, it shouldn't be called when + the refcount is 0 (that is, not directly from tp_dealloc unless + the refcount has been temporarily re-incremented). */ +static PyObject * +fileio_dealloc_warn(fileio *self, PyObject *source) +{ + if (self->fd >= 0 && self->closefd) { + PyObject *exc, *val, *tb; + PyErr_Fetch(&exc, &val, &tb); + if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, + "unclosed file %R", source)) { + /* Spurious errors can appear at shutdown */ + if (PyErr_ExceptionMatches(PyExc_Warning)) + PyErr_WriteUnraisable((PyObject *) self); + } + PyErr_Restore(exc, val, tb); + } + Py_RETURN_NONE; +} + static PyObject * portable_lseek(int fd, PyObject *posobj, int whence); @@ -110,6 +132,13 @@ fileio_close(fileio *self) self->fd = -1; Py_RETURN_NONE; } + if (self->deallocating) { + PyObject *r = fileio_dealloc_warn(self, (PyObject *) self); + if (r) + Py_DECREF(r); + else + PyErr_Clear(); + } errno = internal_close(self); if (errno < 0) return NULL; @@ -398,6 +427,7 @@ fileio_clear(fileio *self) static void fileio_dealloc(fileio *self) { + self->deallocating = 1; if (_PyIOBase_finalize((PyObject *) self) < 0) return; _PyObject_GC_UNTRACK(self); @@ -1007,6 +1037,7 @@ static PyMethodDef fileio_methods[] = { {"writable", (PyCFunction)fileio_writable, METH_NOARGS, writable_doc}, {"fileno", (PyCFunction)fileio_fileno, METH_NOARGS, fileno_doc}, {"isatty", (PyCFunction)fileio_isatty, METH_NOARGS, isatty_doc}, + {"_dealloc_warn", (PyCFunction)fileio_dealloc_warn, METH_O, NULL}, {NULL, NULL} /* sentinel */ }; diff -r 416dcc95b56d Modules/_io/textio.c --- a/Modules/_io/textio.c Thu Oct 14 19:03:51 2010 +0200 +++ b/Modules/_io/textio.c Thu Oct 14 21:41:42 2010 +0200 @@ -658,6 +658,7 @@ typedef struct char writetranslate; char seekable; char telling; + char deallocating; /* Specialized encoding func (see below) */ encodefunc_t encodefunc; /* Whether or not it's the start of the stream */ @@ -1094,6 +1095,7 @@ _textiowrapper_clear(textio *self) static void textiowrapper_dealloc(textio *self) { + self->deallocating = 1; if (_textiowrapper_clear(self) < 0) return; _PyObject_GC_UNTRACK(self); @@ -2410,6 +2412,13 @@ textiowrapper_close(textio *self, PyObje Py_RETURN_NONE; /* stream already closed */ } else { + if (self->deallocating) { + res = PyObject_CallMethod(self->buffer, "_dealloc_warn", "O", self); + if (res) + Py_DECREF(res); + else + PyErr_Clear(); + } res = PyObject_CallMethod((PyObject *)self, "flush", NULL); if (res == NULL) { return NULL; diff -r 416dcc95b56d Modules/socketmodule.c --- a/Modules/socketmodule.c Thu Oct 14 19:03:51 2010 +0200 +++ b/Modules/socketmodule.c Thu Oct 14 21:41:42 2010 +0200 @@ -2931,8 +2931,20 @@ static PyMemberDef sock_memberlist[] = { static void sock_dealloc(PySocketSockObject *s) { - if (s->sock_fd != -1) + if (s->sock_fd != -1) { + PyObject *exc, *val, *tb; + Py_ssize_t old_refcount = Py_REFCNT(s); + Py_INCREF(s); + PyErr_Fetch(&exc, &val, &tb); + if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, + "unclosed %R", s)) + /* Spurious errors can appear at shutdown */ + if (PyErr_ExceptionMatches(PyExc_Warning)) + PyErr_WriteUnraisable((PyObject *) s); + PyErr_Restore(exc, val, tb); (void) SOCKETCLOSE(s->sock_fd); + Py_REFCNT(s) = old_refcount; + } Py_TYPE(s)->tp_free((PyObject *)s); } diff -r 416dcc95b56d Objects/exceptions.c --- a/Objects/exceptions.c Thu Oct 14 19:03:51 2010 +0200 +++ b/Objects/exceptions.c Thu Oct 14 21:41:42 2010 +0200 @@ -1831,6 +1831,13 @@ SimpleExtendsException(PyExc_Warning, Ru /* + * ResourceWarning extends Warning + */ +SimpleExtendsException(PyExc_Warning, ResourceWarning, + "Base class for warnings about resource usage."); + + +/* * FutureWarning extends Warning */ SimpleExtendsException(PyExc_Warning, FutureWarning, @@ -1935,6 +1942,7 @@ _PyExc_Init(void) PRE_INIT(PendingDeprecationWarning) PRE_INIT(SyntaxWarning) PRE_INIT(RuntimeWarning) + PRE_INIT(ResourceWarning) PRE_INIT(FutureWarning) PRE_INIT(ImportWarning) PRE_INIT(UnicodeWarning) @@ -1997,6 +2005,7 @@ _PyExc_Init(void) POST_INIT(PendingDeprecationWarning) POST_INIT(SyntaxWarning) POST_INIT(RuntimeWarning) + POST_INIT(ResourceWarning) POST_INIT(FutureWarning) POST_INIT(ImportWarning) POST_INIT(UnicodeWarning) diff -r 416dcc95b56d Python/_warnings.c --- a/Python/_warnings.c Thu Oct 14 19:03:51 2010 +0200 +++ b/Python/_warnings.c Thu Oct 14 21:41:42 2010 +0200 @@ -835,6 +835,7 @@ create_filter(PyObject *category, const static PyObject *ignore_str = NULL; static PyObject *error_str = NULL; static PyObject *default_str = NULL; + static PyObject *always_str = NULL; PyObject *action_obj = NULL; PyObject *lineno, *result; @@ -862,6 +863,14 @@ create_filter(PyObject *category, const } action_obj = default_str; } + else if (!strcmp(action, "always")) { + if (always_str == NULL) { + always_str = PyUnicode_InternFromString("always"); + if (always_str == NULL) + return NULL; + } + action_obj = always_str; + } else { Py_FatalError("unknown action"); } @@ -879,10 +888,10 @@ static PyObject * init_filters(void) { /* Don't silence DeprecationWarning if -3 was used. */ - PyObject *filters = PyList_New(4); + PyObject *filters = PyList_New(5); unsigned int pos = 0; /* Post-incremented in each use. */ unsigned int x; - const char *bytes_action; + const char *bytes_action, *resource_action; if (filters == NULL) return NULL; @@ -901,6 +910,13 @@ init_filters(void) bytes_action = "ignore"; PyList_SET_ITEM(filters, pos++, create_filter(PyExc_BytesWarning, bytes_action)); +#ifdef Py_DEBUG + resource_action = "always"; +#else + resource_action = "ignore"; +#endif + PyList_SET_ITEM(filters, pos++, create_filter(PyExc_ResourceWarning, + resource_action)); for (x = 0; x < pos; x += 1) { if (PyList_GET_ITEM(filters, x) == NULL) { diff -r 416dcc95b56d Python/traceback.c --- a/Python/traceback.c Thu Oct 14 19:03:51 2010 +0200 +++ b/Python/traceback.c Thu Oct 14 21:41:42 2010 +0200 @@ -208,6 +208,7 @@ _Py_DisplaySourceLine(PyObject *f, PyObj PyObject *binary; PyObject *fob = NULL; PyObject *lineobj = NULL; + PyObject *res; char buf[MAXPATHLEN+1]; Py_UNICODE *u, *p; Py_ssize_t len; @@ -253,6 +254,11 @@ _Py_DisplaySourceLine(PyObject *f, PyObj break; } } + res = PyObject_CallMethod(fob, "close", ""); + if (res) + Py_DECREF(res); + else + PyErr_Clear(); Py_DECREF(fob); if (!lineobj || !PyUnicode_Check(lineobj)) { Py_XDECREF(lineobj);