diff -r ce67f965a37d Lib/pickle.py --- a/Lib/pickle.py Sun Mar 08 09:42:40 2015 -0400 +++ b/Lib/pickle.py Sun Mar 08 22:40:39 2015 +0200 @@ -258,24 +258,20 @@ class _Unframer: # Tools used for pickling. -def _getattribute(obj, name, allow_qualname=False): - dotted_path = name.split(".") - if not allow_qualname and len(dotted_path) > 1: - raise AttributeError("Can't get qualified attribute {!r} on {!r}; " + - "use protocols >= 4 to enable support" - .format(name, obj)) - for subpath in dotted_path: +def _getattribute(obj, name): + for subpath in name.split('.'): if subpath == '': raise AttributeError("Can't get local attribute {!r} on {!r}" .format(name, obj)) try: + parent = obj obj = getattr(obj, subpath) except AttributeError: raise AttributeError("Can't get attribute {!r} on {!r}" .format(name, obj)) - return obj + return obj, parent -def whichmodule(obj, name, allow_qualname=False): +def whichmodule(obj, name): """Find the module an object belong to.""" module_name = getattr(obj, '__module__', None) if module_name is not None: @@ -286,7 +282,7 @@ def whichmodule(obj, name, allow_qualnam if module_name == '__main__' or module is None: continue try: - if _getattribute(module, name, allow_qualname) is obj: + if _getattribute(module, name)[0] is obj: return module_name except AttributeError: pass @@ -899,16 +895,16 @@ class _Pickler: write = self.write memo = self.memo - if name is None and self.proto >= 4: + if name is None: name = getattr(obj, '__qualname__', None) if name is None: name = obj.__name__ - module_name = whichmodule(obj, name, allow_qualname=self.proto >= 4) + module_name = whichmodule(obj, name) try: __import__(module_name, level=0) module = sys.modules[module_name] - obj2 = _getattribute(module, name, allow_qualname=self.proto >= 4) + obj2, parent = _getattribute(module, name) except (ImportError, KeyError, AttributeError): raise PicklingError( "Can't pickle %r: it's not found as %s.%s" % @@ -930,11 +926,16 @@ class _Pickler: else: write(EXT4 + pack("= 3. if self.proto >= 4: self.save(module_name) self.save(name) write(STACK_GLOBAL) + elif parent is not module: + self.save_reduce(getattr, (parent, lastname)) elif self.proto >= 3: write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + bytes(name, "utf-8") + b'\n') @@ -1373,8 +1374,10 @@ class _Unpickler: if module in _compat_pickle.IMPORT_MAPPING: module = _compat_pickle.IMPORT_MAPPING[module] __import__(module, level=0) - return _getattribute(sys.modules[module], name, - allow_qualname=self.proto >= 4) + if self.proto >= 4: + return _getattribute(sys.modules[module], name)[0] + else: + return getattr(sys.modules[module], name) def load_reduce(self): stack = self.stack diff -r ce67f965a37d Lib/test/pickletester.py --- a/Lib/test/pickletester.py Sun Mar 08 09:42:40 2015 -0400 +++ b/Lib/test/pickletester.py Sun Mar 08 22:40:39 2015 +0200 @@ -1600,13 +1600,24 @@ class AbstractPickleTests(unittest.TestC class B: class C: pass - - for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): for obj in [Nested.A, Nested.A.B, Nested.A.B.C]: with self.subTest(proto=proto, obj=obj): unpickled = self.loads(self.dumps(obj, proto)) self.assertIs(obj, unpickled) + def test_recursive_nested_names(self): + global Recursive + class Recursive: + pass + Recursive.mod = sys.modules[Recursive.__module__] + Recursive.__qualname__ = 'Recursive.mod.Recursive' + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + unpickled = self.loads(self.dumps(Recursive, proto)) + self.assertIs(unpickled, Recursive) + del Recursive.mod # break reference loop + def test_py_methods(self): global PyMethodsTest class PyMethodsTest: @@ -1645,7 +1656,7 @@ class AbstractPickleTests(unittest.TestC (PyMethodsTest.biscuits, PyMethodsTest), (PyMethodsTest.Nested.pie, PyMethodsTest.Nested) ) - for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): for method in py_methods: with self.subTest(proto=proto, method=method): unpickled = self.loads(self.dumps(method, proto)) @@ -1685,7 +1696,7 @@ class AbstractPickleTests(unittest.TestC (Subclass.Nested("sweet").count, ("e",)), (Subclass.Nested.count, (Subclass.Nested("sweet"), "e")), ) - for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): for method, args in c_methods: with self.subTest(proto=proto, method=method): unpickled = self.loads(self.dumps(method, proto)) diff -r ce67f965a37d Modules/_pickle.c --- a/Modules/_pickle.c Sun Mar 08 09:42:40 2015 -0400 +++ b/Modules/_pickle.c Sun Mar 08 22:40:39 2015 +0200 @@ -152,6 +152,8 @@ typedef struct { /* codecs.encode, used for saving bytes in older protocols */ PyObject *codecs_encode; + /* builtins.getattr, used for saving nested names with protocol < 4 */ + PyObject *getattr; } PickleState; /* Forward declaration of the _pickle module definition. */ @@ -188,16 +190,26 @@ static void Py_CLEAR(st->name_mapping_3to2); Py_CLEAR(st->import_mapping_3to2); Py_CLEAR(st->codecs_encode); + Py_CLEAR(st->getattr); } /* Initialize the given pickle module state. */ static int _Pickle_InitState(PickleState *st) { + PyObject *builtins; PyObject *copyreg = NULL; PyObject *compat_pickle = NULL; PyObject *codecs = NULL; + builtins = PyEval_GetBuiltins(); + if (builtins == NULL) + goto error; + st->getattr = PyDict_GetItemString(builtins, "getattr"); + if (st->getattr == NULL) + goto error; + Py_INCREF(st->getattr); + copyreg = PyImport_ImportModule("copyreg"); if (!copyreg) goto error; @@ -1535,7 +1547,7 @@ memo_put(PicklerObject *self, PyObject * } static PyObject * -get_dotted_path(PyObject *obj, PyObject *name, int allow_qualname) { +get_dotted_path(PyObject *obj, PyObject *name) { _Py_static_string(PyId_dot, "."); _Py_static_string(PyId_locals, ""); PyObject *dotted_path; @@ -1546,20 +1558,6 @@ get_dotted_path(PyObject *obj, PyObject return NULL; n = PyList_GET_SIZE(dotted_path); assert(n >= 1); - if (!allow_qualname && n > 1) { - if (obj == NULL) - PyErr_Format(PyExc_AttributeError, - "Can't pickle qualified object %R; " - "use protocols >= 4 to enable support", - name); - else - PyErr_Format(PyExc_AttributeError, - "Can't pickle qualified attribute %R on %R; " - "use protocols >= 4 to enable support", - name, obj); - Py_DECREF(dotted_path); - return NULL; - } for (i = 0; i < n; i++) { PyObject *subpath = PyList_GET_ITEM(dotted_path, i); PyObject *result = PyUnicode_RichCompare( @@ -1582,22 +1580,28 @@ get_dotted_path(PyObject *obj, PyObject } static PyObject * -get_deep_attribute(PyObject *obj, PyObject *names) +get_deep_attribute(PyObject *obj, PyObject *names, PyObject **pparent) { Py_ssize_t i, n; + PyObject *parent = NULL; assert(PyList_CheckExact(names)); Py_INCREF(obj); n = PyList_GET_SIZE(names); for (i = 0; i < n; i++) { PyObject *name = PyList_GET_ITEM(names, i); - PyObject *tmp; - tmp = PyObject_GetAttr(obj, name); - Py_DECREF(obj); - if (tmp == NULL) + Py_XDECREF(parent); + parent = obj; + obj = PyObject_GetAttr(parent, name); + if (obj == NULL) { + Py_DECREF(parent); return NULL; - obj = tmp; - } + } + } + if (pparent != NULL) + *pparent = parent; + else + Py_XDECREF(parent); return obj; } @@ -1617,18 +1621,22 @@ getattribute(PyObject *obj, PyObject *na { PyObject *dotted_path, *attr; - dotted_path = get_dotted_path(obj, name, allow_qualname); - if (dotted_path == NULL) - return NULL; - attr = get_deep_attribute(obj, dotted_path); - Py_DECREF(dotted_path); + if (allow_qualname) { + dotted_path = get_dotted_path(obj, name); + if (dotted_path == NULL) + return NULL; + attr = get_deep_attribute(obj, dotted_path, NULL); + Py_DECREF(dotted_path); + } + else + attr = PyObject_GetAttr(obj, name); if (attr == NULL) reformat_attribute_error(obj, name); return attr; } static PyObject * -whichmodule(PyObject *global, PyObject *global_name, int allow_qualname) +whichmodule(PyObject *global, PyObject *dotted_path) { PyObject *module_name; PyObject *modules_dict; @@ -1637,7 +1645,6 @@ whichmodule(PyObject *global, PyObject * _Py_IDENTIFIER(__module__); _Py_IDENTIFIER(modules); _Py_IDENTIFIER(__main__); - PyObject *dotted_path; module_name = _PyObject_GetAttrId(global, &PyId___module__); @@ -1663,10 +1670,6 @@ whichmodule(PyObject *global, PyObject * return NULL; } - dotted_path = get_dotted_path(NULL, global_name, allow_qualname); - if (dotted_path == NULL) - return NULL; - i = 0; while (PyDict_Next(modules_dict, &i, &module_name, &module)) { PyObject *candidate; @@ -1676,19 +1679,16 @@ whichmodule(PyObject *global, PyObject * if (module == Py_None) continue; - candidate = get_deep_attribute(module, dotted_path); + candidate = get_deep_attribute(module, dotted_path, NULL); if (candidate == NULL) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - Py_DECREF(dotted_path); + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) return NULL; - } PyErr_Clear(); continue; } if (candidate == global) { Py_INCREF(module_name); - Py_DECREF(dotted_path); Py_DECREF(candidate); return module_name; } @@ -1698,7 +1698,6 @@ whichmodule(PyObject *global, PyObject * /* If no module is found, use __main__. */ module_name = _PyUnicode_FromId(&PyId___main__); Py_INCREF(module_name); - Py_DECREF(dotted_path); return module_name; } @@ -3104,6 +3103,9 @@ save_global(PicklerObject *self, PyObjec PyObject *global_name = NULL; PyObject *module_name = NULL; PyObject *module = NULL; + PyObject *parent = NULL; + PyObject *dotted_path = NULL; + PyObject *lastname = NULL; PyObject *cls; PickleState *st = _Pickle_GetGlobalState(); int status = 0; @@ -3117,13 +3119,11 @@ save_global(PicklerObject *self, PyObjec global_name = name; } else { - if (self->proto >= 4) { - global_name = _PyObject_GetAttrId(obj, &PyId___qualname__); - if (global_name == NULL) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) - goto error; - PyErr_Clear(); - } + global_name = _PyObject_GetAttrId(obj, &PyId___qualname__); + if (global_name == NULL) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) + goto error; + PyErr_Clear(); } if (global_name == NULL) { global_name = _PyObject_GetAttrId(obj, &PyId___name__); @@ -3132,7 +3132,10 @@ save_global(PicklerObject *self, PyObjec } } - module_name = whichmodule(obj, global_name, self->proto >= 4); + dotted_path = get_dotted_path(module, global_name); + if (dotted_path == NULL) + goto error; + module_name = whichmodule(obj, dotted_path); if (module_name == NULL) goto error; @@ -3151,7 +3154,10 @@ save_global(PicklerObject *self, PyObjec obj, module_name); goto error; } - cls = getattribute(module, global_name, self->proto >= 4); + lastname = PyList_GET_ITEM(dotted_path, PyList_GET_SIZE(dotted_path)-1); + Py_INCREF(lastname); + cls = get_deep_attribute(module, dotted_path, &parent); + Py_CLEAR(dotted_path); if (cls == NULL) { PyErr_Format(st->PicklingError, "Can't pickle %R: attribute lookup %S on %S failed", @@ -3238,6 +3244,11 @@ save_global(PicklerObject *self, PyObjec } else { gen_global: + if (parent == module) { + Py_INCREF(lastname); + Py_DECREF(global_name); + global_name = lastname; + } if (self->proto >= 4) { const char stack_global_op = STACK_GLOBAL; @@ -3249,6 +3260,15 @@ save_global(PicklerObject *self, PyObjec if (_Pickler_Write(self, &stack_global_op, 1) < 0) goto error; } + else if (parent != module) { + PickleState *st = _Pickle_GetGlobalState(); + PyObject *reduce_value = Py_BuildValue("(O(OO))", + st->getattr, parent, lastname); + status = save_reduce(self, reduce_value, NULL); + Py_DECREF(reduce_value); + if (status < 0) + goto error; + } else { /* Generate a normal global opcode if we are using a pickle protocol < 4, or if the object is not registered in the @@ -3327,6 +3347,9 @@ save_global(PicklerObject *self, PyObjec Py_XDECREF(module_name); Py_XDECREF(global_name); Py_XDECREF(module); + Py_XDECREF(parent); + Py_XDECREF(dotted_path); + Py_XDECREF(lastname); return status; } @@ -7148,6 +7171,7 @@ pickle_traverse(PyObject *m, visitproc v Py_VISIT(st->name_mapping_3to2); Py_VISIT(st->import_mapping_3to2); Py_VISIT(st->codecs_encode); + Py_VISIT(st->getattr); return 0; }