# HG changeset patch # Parent 58f1477e8d26836a55eea562b938f5b3388514e4 Patch to make pickle aware of __qualname__ This makes unbound instance methods and static methods potentially picklable, along with nested classes. diff -r 58f1477e8d26 Lib/functools.py --- a/Lib/functools.py Thu Dec 01 20:12:35 2011 +0000 +++ b/Lib/functools.py Fri Dec 02 15:30:40 2011 +0000 @@ -21,7 +21,8 @@ # update_wrapper() and wraps() are tools to help write # wrapper functions that can handle naive introspection -WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__doc__', '__annotations__') +WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__doc__', '__annotations__', + '__qualname__') WRAPPER_UPDATES = ('__dict__',) def update_wrapper(wrapper, wrapped, diff -r 58f1477e8d26 Lib/pickle.py --- a/Lib/pickle.py Thu Dec 01 20:12:35 2011 +0000 +++ b/Lib/pickle.py Fri Dec 02 15:30:40 2011 +0000 @@ -663,6 +663,7 @@ def save_global(self, obj, name=None, pack=struct.pack): write = self.write memo = self.memo + namearg = name if name is None: name = obj.__name__ @@ -674,16 +675,29 @@ try: __import__(module, level=0) mod = sys.modules[module] - klass = getattr(mod, name) + try: + klass = getattr(mod, name) + if klass is not obj: + raise AttributeError + except AttributeError: + if namearg is not None: + raise AttributeError + klass = mod + for part in obj.__qualname__.split("."): + klass = getattr(klass, part) + if klass is not obj: + raise AttributeError + name = obj.__qualname__ except (ImportError, KeyError, AttributeError): - raise PicklingError( - "Can't pickle %r: it's not found as %s.%s" % - (obj, module, name)) - else: - if klass is not obj: + qualname = getattr(obj, "__qualname__", None) + if qualname is None or qualname == name: raise PicklingError( - "Can't pickle %r: it's not the same object as %s.%s" % + "Can't pickle %r: it's not found as %s.%s" % (obj, module, name)) + else: + raise PicklingError( + "Can't pickle %r: it's not found as %s.%s or %s.%s" % + (obj, module, name, module, qualname)) if self.proto >= 2: code = _extension_registry.get((module, name)) @@ -1108,7 +1122,12 @@ module = _compat_pickle.IMPORT_MAPPING[module] __import__(module, level=0) mod = sys.modules[module] - klass = getattr(mod, name) + try: + klass = getattr(mod, name) + except AttributeError: + klass = mod + for part in name.split("."): + klass = getattr(klass, part) return klass def load_reduce(self): diff -r 58f1477e8d26 Lib/test/pickletester.py --- a/Lib/test/pickletester.py Thu Dec 01 20:12:35 2011 +0000 +++ b/Lib/test/pickletester.py Fri Dec 02 15:30:40 2011 +0000 @@ -5,6 +5,7 @@ import sys import copyreg import weakref +import functools from http.cookies import SimpleCookie from test.support import ( @@ -135,6 +136,76 @@ result.reduce_args = (name, bases) return result +# Before __qualname__ was introduced, functions or classes at module +# scope would be picklable as long as __name__ was correct. We want +# that to remain true even if __qualname__ is something nonsensical +# like '_make_counter..counter'. + +def _make_counter(): + i = 0 + def counter(): + nonlocal i + i += 1 + return i + return counter + +counter = _make_counter() + +def my_decorator(f): + @functools.wraps(f) + def wrapper(*args, **kwds): + print('Calling decorated function') + return f(*args, **kwds) + return wrapper + +@my_decorator +def decorated_function(): + pass + +some_global = None + +class Outer: + class Inner: + class InnerInner: + pass + + def instance_method(self): + pass + + @staticmethod + def static_method(): + pass + + # Outer.Inner.class_method is a bound instance method where + # Outer.Inner is the instance. Bound instance methods are not + # currently picklable, so we can't pickle class_method. + @classmethod + def class_method(cls): + pass + + # Check functools.wraps() fixes __qualname__ + @my_decorator + def decorated_method(self): + pass + + # Check not confused by variable of the same name at module scope + def some_global(self): + pass + +# List of objects with a non-trivial __qualname__ which we will test +# for picklability. + +qualname_objects = [ + counter, + decorated_function, + Outer.Inner, + Outer.Inner.InnerInner, + Outer.Inner.instance_method, + Outer.Inner.static_method, + Outer.Inner.decorated_method, + Outer.Inner.some_global, +] + # DATA0 .. DATA2 are the pickles we expect under the various protocols, for # the object returned by create_data(). @@ -1183,6 +1254,19 @@ dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.' self.assertRaises(ValueError, self.loads, dumped) + def test_qualname(self): + for proto in protocols: + x = Outer.Inner() + s = self.dumps(x) + y = self.loads(s) + self.assertIsInstance(y, Outer.Inner) + + for obj in qualname_objects: + x = obj + s = self.dumps(x) + y = self.loads(s) + self.assertIs(x, y) + class BigmemPickleTests(unittest.TestCase): diff -r 58f1477e8d26 Modules/_pickle.c --- a/Modules/_pickle.c Thu Dec 01 20:12:35 2011 +0000 +++ b/Modules/_pickle.c Fri Dec 02 15:30:40 2011 +0000 @@ -2522,11 +2522,60 @@ return status; } +/* Replacement for PyObject_GetAttr(obj, qualname) when qualname is a + qualified name with embedded period(s). */ +static PyObject * +GetQualAttr(PyObject *obj, PyObject *qualname) +{ + static PyObject *period = NULL; + PyObject *parts; + PyObject *tmp; + Py_ssize_t i, length; + + if (period == NULL) { + period = PyUnicode_InternFromString("."); + if (period == NULL) + return NULL; + } + + if (obj == NULL || qualname == NULL) + return NULL; + + Py_INCREF(obj); + parts = PyUnicode_Split(qualname, period, -1); + if (parts == NULL) + goto error; + + if (!PyList_CheckExact(parts)) { + PyErr_SetString(PyExc_AssertionError, + "Expected split() method to return list"); + goto error; + } + + length = PyList_GET_SIZE(parts); + for (i = 0 ; i < length ; i++) { + tmp = PyObject_GetAttr(obj, PyList_GET_ITEM(parts, i)); + if (tmp == NULL) + goto error; + Py_DECREF(obj); + obj = tmp; + } + + Py_DECREF(parts); + return obj; +error: + Py_XDECREF(parts); + Py_XDECREF(obj); + return NULL; +} + static int save_global(PicklerObject *self, PyObject *obj, PyObject *name) { static PyObject *name_str = NULL; + static PyObject *qualname_str = NULL; PyObject *global_name = NULL; + PyObject *qual_name = NULL; PyObject *module_name = NULL; PyObject *module = NULL; PyObject *cls; @@ -2538,6 +2587,9 @@ name_str = PyUnicode_InternFromString("__name__"); if (name_str == NULL) goto error; + qualname_str = PyUnicode_InternFromString("__qualname__"); + if (qualname_str == NULL) + goto error; } if (name) { @@ -2569,20 +2621,37 @@ obj, module_name); goto error; } + /* try using __name__ first */ cls = PyObject_GetAttr(module, global_name); - if (cls == NULL) { + if (cls == obj) + goto found_cls; + /* now try using __qualname__ instead */ + PyErr_Clear(); + Py_XDECREF(cls); + qual_name = PyObject_GetAttr(obj, qualname_str); + if (qual_name != NULL) { + cls = GetQualAttr(module, qual_name); + if (cls == obj) { + /* __qualname__ worked, so set global_name to it */ + Py_DECREF(global_name); + global_name = qual_name; + Py_INCREF(global_name); + goto found_cls; + } + Py_XDECREF(cls); + } + if (qual_name != NULL && PyUnicode_Compare(qual_name, global_name) != 0) PyErr_Format(PicklingError, - "Can't pickle %R: attribute lookup %S.%S failed", + "Can't pickle %R: it's not found as %S.%S or " + "%S.%S", obj, module_name, global_name, + module_name, qual_name); + else + PyErr_Format(PicklingError, + "Can't pickle %R: it's not found as %S.%S", obj, module_name, global_name); - goto error; - } - if (cls != obj) { - Py_DECREF(cls); - PyErr_Format(PicklingError, - "Can't pickle %R: it's not the same object as %S.%S", - obj, module_name, global_name); - goto error; - } + goto error; + +found_cls: Py_DECREF(cls); if (self->proto >= 2) { @@ -2772,6 +2841,7 @@ } Py_XDECREF(module_name); Py_XDECREF(global_name); + Py_XDECREF(qual_name); Py_XDECREF(module); return status; @@ -5452,6 +5522,10 @@ } else { global = PyObject_GetAttr(module, global_name); + if (global == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + global = GetQualAttr(module, global_name); + } } return global; }