diff -r 5206b9dbf1ac Doc/library/weakref.rst --- a/Doc/library/weakref.rst Thu Aug 16 07:27:47 2012 +0300 +++ b/Doc/library/weakref.rst Fri Aug 17 14:05:45 2012 +0100 @@ -48,10 +48,15 @@ but keeps weak references to its elements, just like a :class:`WeakKeyDictionary` does. -Most programs should find that using one of these weak container types is all -they need -- it's not usually necessary to create your own weak references -directly. The low-level machinery used by the weak dictionary implementations -is exposed by the :mod:`weakref` module for the benefit of advanced uses. +:class:`finalize` provides a straight forward way to register a +cleanup function to be called when an object is garbage collected. +This is simpler to use than setting up a callback function on a raw +weak reference. + +Most programs should find that using one of these weak container types +or :class:`finalize` is all they need -- it's not usually necessary to +create your own weak references directly. The low-level machinery is +exposed by the :mod:`weakref` module for the benefit of advanced uses. .. note:: @@ -114,6 +119,15 @@ This is a subclassable type rather than a factory function. + .. note:: + + Like :meth:`__del__` methods, weak reference callbacks can be + called during interpreter shutdown when module globals have been + overwritten with :const:`None`. This can make writing robust + weak reference callbacks a challenge. Callbacks registered + using :class:`finalize` do not have to worry about this issue + because they will not be run after module teardown has begun. + .. function:: proxy(object[, callback]) @@ -195,6 +209,64 @@ discarded when no strong reference to it exists any more. +.. class:: finalize(obj, func, *args, **kwds) + + Return a callable finalizer object which will be called when *obj* + is garbage collected. When the finalizer is called, if it is alive + then it is marked as dead and the result of evaluating ``func(*arg, + **kwds)`` is returned. If it is dead then :const:`None` is + returned instead. + + It is important to ensure that *func*, *args* and *kwds* do not own + any references to *obj*, either directly or indirectly, since + otherwise *obj* will never be garbage collected. In particular, + *func* must not be a bound method of *obj*. + + Exceptions raised by finalizer callbacks during garbage collection + will be shown on the standard error output, but cannot be + propagated. They are handled in the same way as exceptions raised + from an object's :meth:`__del__` method or a weak reference's + callback. + + When the program exits, each remaining live finalizer is called + unless its :attr:`atexit` attribute has been set to false. They + are called in reverse order of creation. + + A finalizer will never invoke its callback during the later part of + the interpreter shutdown when module globals are liable to have + been replaced by :const:`None`. + + .. method:: __call__() + + If *self* is alive then mark it as dead and return the result of + calling ``func(*args, **kwds)``. If *self* is dead then return + :const:`None`. + + .. method:: detach() + + If *self* is alive then mark it as dead and return the tuple + ``(obj, func, args, kwds)``. If *self* is dead then return + :const:`None`. + + .. method:: peek() + + If *self* is alive then return the tuple ``(obj, func, args, + kwds)``. If *self* is dead then return :const:`None`. + + .. attribute:: alive + + Property which is true if the finalizer is alive, false otherwise. + + .. attribute:: atexit + + A writable boolean property which by default is true. When the + program exits, it calls all remaining live finalizers for which + :attr:`.atexit` is true. They are called in reverse order of + creation. + + .. versionadded:: 3.4 + + .. data:: ReferenceType The type object for weak references objects. @@ -329,3 +401,97 @@ def id2obj(oid): return _id2obj_dict[oid] + +.. _finalize-examples: + +Finalizer Objects +----------------- + +Often one uses :class:`finalize` to register a callback without +bothering to keep the returned finalizer object. For instance + + >>> import weakref + >>> class Object: + ... pass + ... + >>> kenny = Object() + >>> weakref.finalize(kenny, print, "You killed Kenny!") #doctest:+ELLIPSIS + + >>> del kenny + You killed Kenny! + +The finalizer can be called directly as well. However the finalizer +will invoke the callback at most once. + + >>> def callback(x, y, z): + ... print("CALLBACK") + ... return x + y + z + ... + >>> obj = Object() + >>> f = weakref.finalize(obj, callback, 1, 2, z=3) + >>> assert f.alive + >>> assert f() == 6 + CALLBACK + >>> assert not f.alive + >>> f() # callback not called because finalizer dead + >>> del obj # callback not called because finalizer dead + +You can unregister a finalizer using its :meth:`~finalize.detach` +method. This kills the finalizer and returns the arguments passed to +the constructor when it was created. + + >>> obj = Object() + >>> f = weakref.finalize(obj, callback, 1, 2, z=3) + >>> f.detach() #doctest:+ELLIPSIS + (<__main__.Object object ...>, , (1, 2), {'z': 3}) + >>> newobj, func, args, kwds = _ + >>> assert not f.alive + >>> assert newobj is obj + >>> assert func(*args, **kwds) == 6 + CALLBACK + +Unless you set the :attr:`~finalize.atexit` attribute to +:const:`False`, a finalizer will be called when the program exit if it +is still alive. For instance + + >>> obj = Object() + >>> weakref.finalize(obj, print, "obj dead or exiting") #doctest:+ELLIPSIS + + >>> exit() #doctest:+SKIP + obj dead or exiting + + +Using a Finalizer in a Class +---------------------------- + +Suppose we want to create a class representing file descriptors which +automatically closes the descriptor when the instance is garbage +collected. (Compare the `PyHandle +`_ +class used by `PyWin32 `_.) +Although one could use a :meth:`__del__` method, using a finalizer +this can be done as follows:: + + class FileDescriptor: + def __init__(self, fd): + self._finalizer = weakref.finalize(self, os.close, fd) + self._fd = fd + + def close(self): + self._finalizer() + + def detach(self): + if not self._finalizer.detach(): + raise IOError("already closed or detached") + return self._fd + + def fileno(self): + if not self._finalizer.alive: + raise IOError("already closed or detached") + return self._fd + + __index__ = __int__ = fileno + + @property + def closed(self): + return not self._finalizer.alive diff -r 5206b9dbf1ac Lib/test/test_weakref.py --- a/Lib/test/test_weakref.py Thu Aug 16 07:27:47 2012 +0300 +++ b/Lib/test/test_weakref.py Fri Aug 17 14:05:45 2012 +0100 @@ -7,11 +7,15 @@ import contextlib import copy -from test import support +from test import support, script_helper # Used in ReferencesTestCase.test_ref_created_during_del() . ref_from_del = None +# Used by FinalizeTestCase as a global that may be replaced by None +# when the interpreter shuts down. +_global_var = 'foobar' + class C: def method(self): pass @@ -1304,6 +1308,151 @@ def _reference(self): return self.__ref.copy() + +class FinalizeTestCase(unittest.TestCase): + + class A: + pass + + def _collect_if_necessary(self): + # we create no ref-cycles so in CPython no gc should be needed + if sys.implementation.name != 'cpython': + support.gc_collect() + + def test_finalize(self): + def add(x,y,z): + res.append(x + y + z) + return x + y + z + + a = self.A() + + res = [] + f = weakref.finalize(a, add, 67, 43, z=89) + self.assertEqual(f.alive, True) + self.assertEqual(f.peek(), (a, add, (67,43), {'z':89})) + self.assertEqual(f(), 199) + self.assertEqual(f(), None) + self.assertEqual(f(), None) + self.assertEqual(f.peek(), None) + self.assertEqual(f.detach(), None) + self.assertEqual(f.alive, False) + self.assertEqual(res, [199]) + + res = [] + f = weakref.finalize(a, add, 67, 43, 89) + self.assertEqual(f.peek(), (a, add, (67,43,89), {})) + self.assertEqual(f.detach(), (a, add, (67,43,89), {})) + self.assertEqual(f(), None) + self.assertEqual(f(), None) + self.assertEqual(f.peek(), None) + self.assertEqual(f.detach(), None) + self.assertEqual(f.alive, False) + self.assertEqual(res, []) + + res = [] + f = weakref.finalize(a, add, x=67, y=43, z=89) + del a + self._collect_if_necessary() + self.assertEqual(f(), None) + self.assertEqual(f(), None) + self.assertEqual(f.peek(), None) + self.assertEqual(f.detach(), None) + self.assertEqual(f.alive, False) + self.assertEqual(res, [199]) + + def test_order(self): + a = self.A() + res = [] + + f1 = weakref.finalize(a, res.append, 'f1') + f2 = weakref.finalize(a, res.append, 'f2') + f3 = weakref.finalize(a, res.append, 'f3') + f4 = weakref.finalize(a, res.append, 'f4') + f5 = weakref.finalize(a, res.append, 'f5') + + # make sure finalizers can keep themselves alive + del f1, f4 + + self.assertTrue(f2.alive) + self.assertTrue(f3.alive) + self.assertTrue(f5.alive) + + self.assertTrue(f5.detach()) + self.assertFalse(f5.alive) + + f5() # nothing because previously unregistered + res.append('A') + f3() # => res.append('f3') + self.assertFalse(f3.alive) + res.append('B') + f3() # nothing because previously called + res.append('C') + del a + self._collect_if_necessary() + # => res.append('f4') + # => res.append('f2') + # => res.append('f1') + self.assertFalse(f2.alive) + res.append('D') + f2() # nothing because previously called by gc + + expected = ['A', 'f3', 'B', 'C', 'f4', 'f2', 'f1', 'D'] + self.assertEqual(res, expected) + + def test_all_freed(self): + # we want a weakrefable subclass of weakref.finalize + class MyFinalizer(weakref.finalize): + pass + + a = self.A() + res = [] + def callback(): + res.append(123) + f = MyFinalizer(a, callback) + + wr_callback = weakref.ref(callback) + wr_f = weakref.ref(f) + del callback, f + + self.assertIsNotNone(wr_callback()) + self.assertIsNotNone(wr_f()) + + del a + self._collect_if_necessary() + + self.assertIsNone(wr_callback()) + self.assertIsNone(wr_f()) + self.assertEqual(res, [123]) + + @classmethod + def run_in_child(cls): + def error(): + # Create an atexit finalizer from inside a finalizer called + # at exit. This should be the next to be run. + g1 = weakref.finalize(cls, print, 'g1') + print('f3 error') + 1/0 + + # cls should stay alive till atexit callbacks run + f1 = weakref.finalize(cls, print, 'f1', _global_var) + f2 = weakref.finalize(cls, print, 'f2', _global_var) + f3 = weakref.finalize(cls, error) + f4 = weakref.finalize(cls, print, 'f4', _global_var) + + assert f1.atexit == True + f2.atexit = False + assert f3.atexit == True + assert f4.atexit == True + + def test_atexit(self): + prog = ('from test.test_weakref import FinalizeTestCase;'+ + 'FinalizeTestCase.run_in_child()') + rc, out, err = script_helper.assert_python_ok('-c', prog) + out = out.decode('ascii').splitlines() + self.assertEqual(out, ['f4 foobar', 'f3 error', 'g1', 'f1 foobar']) + self.assertTrue(b'ZeroDivisionError' in err) + + libreftest = """ Doctest for examples in the library reference: weakref.rst >>> import weakref @@ -1396,6 +1545,7 @@ WeakValueDictionaryTestCase, WeakKeyDictionaryTestCase, SubclassableWeakrefTestCase, + FinalizeTestCase, ) support.run_doctest(sys.modules[__name__]) diff -r 5206b9dbf1ac Lib/weakref.py --- a/Lib/weakref.py Thu Aug 16 07:27:47 2012 +0300 +++ b/Lib/weakref.py Fri Aug 17 14:05:45 2012 +0100 @@ -21,13 +21,16 @@ from _weakrefset import WeakSet, _IterationGuard import collections # Import after _weakref to avoid circular import. +import sys +import atexit +import itertools ProxyTypes = (ProxyType, CallableProxyType) __all__ = ["ref", "proxy", "getweakrefcount", "getweakrefs", "WeakKeyDictionary", "ReferenceType", "ProxyType", "CallableProxyType", "ProxyTypes", "WeakValueDictionary", - "WeakSet"] + "WeakSet", "finalize"] class WeakValueDictionary(collections.MutableMapping): @@ -383,3 +386,125 @@ d[ref(key, self._remove)] = value if len(kwargs): self.update(kwargs) + + +class finalize: + """Class for finalization of weakrefable objects + + finalize(obj, func, *args, **kwds) returns a callable finalizer + object which will be called when obj is garbage collected. The + first time the finalizer is called it evaluates func(*arg, **kwds) + and returns the result. After this the finalizer is dead, and + calling it just returns None. + + When the program exits any remaining finalizers for which the + atexit attribute is true will be run in reverse order of creation. + By default atexit is true. + """ + + # Finalizer objects don't have any state of their own. They are + # just used as keys to lookup Info objects in the registry. This + # ensures that they cannot be part of a ref-cycle. + + __slots__ = () + _registry = {} + _shutdown = False + _counter = itertools.count() + + class Info: + __slots__ = ("wr", "func", "args", "kwds", "atexit", "count") + + def __init__(self, obj, func, *args, **kwds): + info = self.Info() + info.wr = ref(obj, self) + info.func = func + info.args = args + info.kwds = kwds or None + info.atexit = True + info.count = next(self._counter) + self._registry[self] = info + finalize._dirty = True + + def __call__(self, _=None): + """If alive then mark as dead and return func(*args, **kwds)""" + info = self._registry.pop(self, None) + if info and not self._shutdown: + return info.func(*info.args, **(info.kwds or {})) + + def detach(self): + """If alive then mark as dead and return (obj, func, args, kwds)""" + info = self._registry.get(self) + obj = info and info.wr() + if obj is not None and self._registry.pop(self, None): + return (obj, info.func, info.args, info.kwds or {}) + + def peek(self): + """If alive then return (obj, func, args, kwds)""" + info = self._registry.get(self) + obj = info and info.wr() + if obj is not None: + return (obj, info.func, info.args, info.kwds or {}) + + @property + def alive(self): + """Whether finalizer is alive""" + return self in self._registry + + @property + def atexit(self): + """Whether finalizer should be called at exit""" + info = self._registry.get(self) + return bool(info) and info.atexit + + @atexit.setter + def atexit(self, value): + info = self._registry.get(self) + if info: + info.atexit = bool(value) + + def __repr__(self): + info = self._registry.get(self) + obj = info and info.wr() + if obj is None: + return '<%s object at %#x; dead>' % (type(self).__name__, id(self)) + else: + return '<%s object at %#x; for %r at %#x>' % \ + (type(self).__name__, id(self), type(obj).__name__, id(obj)) + + @classmethod + def _select_for_exit(cls): + # Return live finalizers marked for exit, oldest first + L = [(f,i) for (f,i) in cls._registry.items() if i.atexit] + L.sort(key=lambda item:item[1].count) + return [f for (f,i) in L] + + @classmethod + def _exitfunc(cls): + # At shutdown invoke finalizers for which atexit is true + gc = None + try: + if cls._registry: + import gc + gc.disable() + finalize._dirty = True + while True: + if finalize._dirty: + pending = cls._select_for_exit() + finalize._dirty = False + if not pending: + break + f = pending.pop() + try: + # since gc is disabled, the following is the + # only line in this function which might + # trigger creation of a new finalizer + f() + except: + sys.excepthook(*sys.exc_info()) + assert f not in cls._registry + finally: + # prevent any more finalizers from executing during shutdown + finalize._shutdown = True + gc and gc.enable() + +atexit.register(finalize._exitfunc)