diff -r d84c1b42a8f3 Doc/reference/datamodel.rst --- a/Doc/reference/datamodel.rst Sat Dec 03 15:57:00 2016 -0800 +++ b/Doc/reference/datamodel.rst Sun Dec 04 22:38:04 2016 +1000 @@ -1796,6 +1796,27 @@ lexical scoping, while the class or instance that was used to make the current call is identified based on the first argument passed to the method. +.. impl-detail:: + + In CPython 3.6 and later, the ``__class__`` cell is passed to the metaclass + as a ``__classcell__`` entry in the class namespace. If present, this must + be propagated up to the ``type.__new__`` call in order for the class to be + initialised correctly. + + Failing to do so will result in a :exc:`DeprecationWarning` in Python 3.6, + and a :exc:`RuntimeWarning` in the future. + +When using the default metaclass :class:`type`, or any metaclass that ultimately +calls ``type.__new__``, the following additional customisation steps are +invoked after creating the class object: + +* first, ``type.__new__`` collects all of the descriptors in the class + namespace that define a ``__set_name__`` method +* second, all of these ``__set_name__`` methods are called with the class + being defined and the assigned name of that particular descriptor +* finally, the ``__init_subclass__`` hook is called on the immediate parent of + the new class in its method resolution order + After the class object is created, it is passed to the class decorators included in the class definition (if any) and the resulting object is bound in the local namespace as the defined class. diff -r d84c1b42a8f3 Doc/whatsnew/3.6.rst --- a/Doc/whatsnew/3.6.rst Sat Dec 03 15:57:00 2016 -0800 +++ b/Doc/whatsnew/3.6.rst Sun Dec 04 22:38:04 2016 +1000 @@ -351,6 +351,10 @@ class Plugin2(PluginBase): pass +In order to allow zero-argument :func:`super` calls to work correctly from +``__init_subclass__`` implementations, custom metaclasses must ensure that +the new ``__classcell__`` namespace entry is propagated to ``type.__new__``. + .. seealso:: :pep:`487` -- Simpler customization of class creation @@ -2235,6 +2239,11 @@ on a ZipFile created with mode ``'r'`` will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised in those scenarios. +* when custom metaclasses are combined with zero-argument :func:`super` or + direct references from methods to the implicit ``__class__`` closure + variable, the implicit ``__classcell__`` namespace entry must now be passed + up to ``type.__new__`` for initialisation. Failing to do so will result in + a :exc:`DeprecationWarning` in 3.6 and a :exc:`RuntimeWarning` in the future. Changes in the C API -------------------- diff -r d84c1b42a8f3 Lib/importlib/_bootstrap_external.py --- a/Lib/importlib/_bootstrap_external.py Sat Dec 03 15:57:00 2016 -0800 +++ b/Lib/importlib/_bootstrap_external.py Sun Dec 04 22:38:04 2016 +1000 @@ -239,6 +239,7 @@ # Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL) # Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722) # Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257) +# Python 3.6rc1 3379 (more thorough __class__ validation #23722) # # MAGIC must change whenever the bytecode emitted by the compiler may no # longer be understood by older implementations of the eval loop (usually @@ -247,7 +248,7 @@ # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3378).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3379).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' diff -r d84c1b42a8f3 Lib/test/test_super.py --- a/Lib/test/test_super.py Sat Dec 03 15:57:00 2016 -0800 +++ b/Lib/test/test_super.py Sun Dec 04 22:38:04 2016 +1000 @@ -1,7 +1,9 @@ -"""Unit tests for new super() implementation.""" +"""Unit tests for zero-argument super() & related machinery.""" import sys import unittest +import warnings +from test.support import check_warnings class A: @@ -144,6 +146,8 @@ self.assertIs(X.f(), X) def test___class___new(self): + # See issue #23722 + # Ensure zero-arg super() works as soon as type.__new__() is completed test_class = None class Meta(type): @@ -161,6 +165,7 @@ self.assertIs(test_class, A) def test___class___delayed(self): + # See issue #23722 test_namespace = None class Meta(type): @@ -169,10 +174,14 @@ test_namespace = namespace return None - class A(metaclass=Meta): - @staticmethod - def f(): - return __class__ + # This case shouldn't trigger the __classcell__ deprecation warning + with check_warnings() as w: + warnings.simplefilter("always", DeprecationWarning) + class A(metaclass=Meta): + @staticmethod + def f(): + return __class__ + self.assertEqual(w.warnings, []) self.assertIs(A, None) @@ -180,6 +189,7 @@ self.assertIs(B.f(), B) def test___class___mro(self): + # See issue #23722 test_class = None class Meta(type): @@ -195,34 +205,105 @@ self.assertIs(test_class, A) - def test___classcell___deleted(self): + def test___classcell___expected_behaviour(self): + # See issue #23722 class Meta(type): def __new__(cls, name, bases, namespace): - del namespace['__classcell__'] + nonlocal namespace_snapshot + namespace_snapshot = namespace.copy() return super().__new__(cls, name, bases, namespace) - class A(metaclass=Meta): - @staticmethod - def f(): - __class__ + # __classcell__ is injected into the class namespace by the compiler + # when at least one method needs it, and should be omitted otherwise + namespace_snapshot = None + class WithoutClassRef(metaclass=Meta): + pass + self.assertNotIn("__classcell__", namespace_snapshot) - with self.assertRaises(NameError): - A.f() + # With zero-arg super() or an explicit __class__ reference, + # __classcell__ is the exact cell reference to be populated by + # type.__new__ + namespace_snapshot = None + class WithClassRef(metaclass=Meta): + def f(self): + return __class__ - def test___classcell___reset(self): + class_cell = namespace_snapshot["__classcell__"] + method_closure = WithClassRef.f.__closure__ + self.assertEqual(len(method_closure), 1) + self.assertIs(class_cell, method_closure[0]) + # Ensure the cell reference *doesn't* get turned into an attribute + with self.assertRaises(AttributeError): + WithClassRef.__classcell__ + + def test___classcell___missing(self): + # See issue #23722 + # Some metaclasses may not pass the original namespace to type.__new__ + # We test that case here by forcibly deleting __classcell__ class Meta(type): def __new__(cls, name, bases, namespace): - namespace['__classcell__'] = 0 + namespace.pop('__classcell__', None) return super().__new__(cls, name, bases, namespace) - class A(metaclass=Meta): - @staticmethod - def f(): - __class__ + # The default case should continue to work without any warnings + with check_warnings() as w: + warnings.simplefilter("always", DeprecationWarning) + class WithoutClassRef(metaclass=Meta): + pass + self.assertEqual(w.warnings, []) - with self.assertRaises(NameError): - A.f() - self.assertEqual(A.__classcell__, 0) + # With zero-arg super() or an explicit __class__ reference, we expect + # __build_class__ to emit a DeprecationWarning complaining that + # __class__ was not set, and asking if __classcell__ was propagated + # to type.__new__. + # In Python 3.7, that warning will become a RuntimeError. + expected_warning = ( + '__class__ not set.*__classcell__ propagated', + DeprecationWarning + ) + with check_warnings(expected_warning): + warnings.simplefilter("always", DeprecationWarning) + class WithClassRef(metaclass=Meta): + def f(self): + return __class__ + # Check __class__ still gets set despite the warning + self.assertIs(WithClassRef().f(), WithClassRef) + + # Check the warning is turned into an error as expected + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + with self.assertRaises(DeprecationWarning): + class WithClassRef(metaclass=Meta): + def f(self): + return __class__ + + def test___classcell___overwrite(self): + # See issue #23722 + # Overwriting __classcell__ with nonsense is explicitly prohibited + class Meta(type): + def __new__(cls, name, bases, namespace, cell): + namespace['__classcell__'] = cell + return super().__new__(cls, name, bases, namespace) + + for bad_cell in (None, 0, "", object()): + with self.subTest(bad_cell=bad_cell): + with self.assertRaises(TypeError): + class A(metaclass=Meta, cell=bad_cell): + pass + + def test___classcell___wrong_cell(self): + # See issue #23722 + # Pointing the cell reference at the wrong class is also prohibited + class Meta(type): + def __new__(cls, name, bases, namespace): + cls = super().__new__(cls, name, bases, namespace) + B = type("B", (), namespace) + return cls + + with self.assertRaises(TypeError): + class A(metaclass=Meta): + def f(self): + return __class__ def test_obscure_super_errors(self): def f(): diff -r d84c1b42a8f3 Objects/typeobject.c --- a/Objects/typeobject.c Sat Dec 03 15:57:00 2016 -0800 +++ b/Objects/typeobject.c Sun Dec 04 22:38:04 2016 +1000 @@ -2687,9 +2687,16 @@ else type->tp_free = PyObject_Del; - /* store type in class' cell */ + /* store type in class' cell if one is supplied */ cell = _PyDict_GetItemId(dict, &PyId___classcell__); - if (cell != NULL && PyCell_Check(cell)) { + if (cell != NULL) { + /* At least one method requires a reference to its defining class */ + if (!PyCell_Check(cell)) { + PyErr_Format(PyExc_TypeError, + "__classcell__ must be a nonlocal cell, not %.200R", + Py_TYPE(cell)); + goto error; + } PyCell_Set(cell, (PyObject *) type); _PyDict_DelItemId(dict, &PyId___classcell__); PyErr_Clear(); diff -r d84c1b42a8f3 Python/bltinmodule.c --- a/Python/bltinmodule.c Sat Dec 03 15:57:00 2016 -0800 +++ b/Python/bltinmodule.c Sun Dec 04 22:38:04 2016 +1000 @@ -54,8 +54,8 @@ static PyObject * builtin___build_class__(PyObject *self, PyObject *args, PyObject *kwds) { - PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns, *none; - PyObject *cls = NULL; + PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns; + PyObject *cls = NULL, *cell = NULL; Py_ssize_t nargs; int isclass = 0; /* initialize to prevent gcc warning */ @@ -167,14 +167,44 @@ Py_DECREF(bases); return NULL; } - none = PyEval_EvalCodeEx(PyFunction_GET_CODE(func), PyFunction_GET_GLOBALS(func), ns, + cell = PyEval_EvalCodeEx(PyFunction_GET_CODE(func), PyFunction_GET_GLOBALS(func), ns, NULL, 0, NULL, 0, NULL, 0, NULL, PyFunction_GET_CLOSURE(func)); - if (none != NULL) { + if (cell != NULL) { PyObject *margs[3] = {name, bases, ns}; cls = _PyObject_FastCallDict(meta, margs, 3, mkw); - Py_DECREF(none); + if (cls != NULL && PyType_Check(cls) && PyCell_Check(cell)) { + PyObject *cell_cls = PyCell_GET(cell); + if (cell_cls != cls) { + /* TODO: In 3.7, the deprecation warning becomes a RuntimeError + * At that point, cell_error won't be needed. + */ + int cell_error; + if (cell_cls == NULL) { + const char *msg = + "__class__ not set defining %.200R as %.200R. " + "Was __classcell__ propagated to type.__new__?"; + cell_error = PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, msg, name, cls); + } else { + const char *msg = + "__class__ set to %.200R defining %.200R as %.200R."; + PyErr_Format(PyExc_TypeError, msg, cell_cls, name, cls); + cell_error = 1; + } + if (cell_error) { + Py_DECREF(cls); + cls = NULL; + goto error; + } else { + /* Fill in the cell, since type.__new__ didn't do it */ + PyCell_Set(cell, cls); + } + } + } } +error: + Py_XDECREF(cell); Py_DECREF(ns); Py_DECREF(meta); Py_XDECREF(mkw); diff -r d84c1b42a8f3 Python/compile.c --- a/Python/compile.c Sat Dec 03 15:57:00 2016 -0800 +++ b/Python/compile.c Sun Dec 04 22:38:04 2016 +1000 @@ -1967,8 +1967,9 @@ compiler_exit_scope(c); return 0; } + /* Return __classcell__ if it is referenced, otherwise return None */ if (c->u->u_ste->ste_needs_class_closure) { - /* store __classcell__ into class namespace */ + /* Store __classcell__ into class namespace & return it */ str = PyUnicode_InternFromString("__class__"); if (str == NULL) { compiler_exit_scope(c); @@ -1983,6 +1984,7 @@ assert(i == 0); ADDOP_I(c, LOAD_CLOSURE, i); + ADDOP(c, DUP_TOP); str = PyUnicode_InternFromString("__classcell__"); if (!str || !compiler_nameop(c, str, Store)) { Py_XDECREF(str); @@ -1992,9 +1994,11 @@ Py_DECREF(str); } else { - /* This happens when nobody references the cell. */ + /* No methods referenced __class__, so just return None */ assert(PyDict_Size(c->u->u_cellvars) == 0); + ADDOP_O(c, LOAD_CONST, Py_None, consts); } + ADDOP_IN_SCOPE(c, RETURN_VALUE); /* create the code object */ co = assemble(c, 1); }