diff -r d84c1b42a8f3 Lib/importlib/_bootstrap_external.py --- a/Lib/importlib/_bootstrap_external.py Sat Dec 03 15:57:00 2016 -0800 +++ b/Lib/importlib/_bootstrap_external.py Sun Dec 04 16:41:58 2016 +1000 @@ -239,6 +239,7 @@ # Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL) # Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722) # Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257) +# Python 3.6rc1 3379 (more thorough __class__ validation #23722) # # MAGIC must change whenever the bytecode emitted by the compiler may no # longer be understood by older implementations of the eval loop (usually @@ -247,7 +248,7 @@ # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3378).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3379).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' diff -r d84c1b42a8f3 Lib/test/test_super.py --- a/Lib/test/test_super.py Sat Dec 03 15:57:00 2016 -0800 +++ b/Lib/test/test_super.py Sun Dec 04 16:41:58 2016 +1000 @@ -1,7 +1,10 @@ -"""Unit tests for new super() implementation.""" +"""Unit tests for zero-argument super() & related machinery.""" +import inspect import sys import unittest +import warnings +from test.support import check_warnings class A: @@ -144,6 +147,8 @@ self.assertIs(X.f(), X) def test___class___new(self): + # See issue #23722 + # Ensure zero-arg super() works as soon as type.__new__() is completed test_class = None class Meta(type): @@ -161,6 +166,7 @@ self.assertIs(test_class, A) def test___class___delayed(self): + # See issue #23722 test_namespace = None class Meta(type): @@ -180,6 +186,7 @@ self.assertIs(B.f(), B) def test___class___mro(self): + # See issue #23722 test_class = None class Meta(type): @@ -195,34 +202,102 @@ self.assertIs(test_class, A) - def test___classcell___deleted(self): + def test___classcell___expected_behaviour(self): + # See issue #23722 + namespace_snapshot = None class Meta(type): def __new__(cls, name, bases, namespace): - del namespace['__classcell__'] + nonlocal namespace_snapshot + namespace_snapshot = namespace.copy() return super().__new__(cls, name, bases, namespace) - class A(metaclass=Meta): - @staticmethod - def f(): - __class__ + # __classcell__ is injected into the class namespace by the compiler + # when at least one method needs it, and should be omitted otherwise + class WithoutClassRef(metaclass=Meta): + pass + self.assertNotIn("__classcell__", namespace_snapshot) - with self.assertRaises(NameError): - A.f() + # With zero-arg super() or an explicit __class__ reference, + # __classcell__ is the exact cell reference to be populated by + # type.__new__ + class WithClassRef(metaclass=Meta): + def f(self): + return __class__ - def test___classcell___reset(self): + class_cell = namespace_snapshot["__classcell__"] + method_closure = WithClassRef.f.__closure__ + self.assertEqual(len(method_closure), 1) + self.assertIs(class_cell, method_closure[0]) + # Ensure the cell reference *doesn't* get turned into an attribute + with self.assertRaises(AttributeError): + WithClassRef.__classcell__ + + def test___classcell___missing(self): + # See issue #23722 + # Some metaclasses may not pass the original namespace to type.__new__ + # We test that case here by forcibly deleting __classcell__ class Meta(type): def __new__(cls, name, bases, namespace): - namespace['__classcell__'] = 0 + namespace.pop('__classcell__', None) return super().__new__(cls, name, bases, namespace) - class A(metaclass=Meta): - @staticmethod - def f(): - __class__ + # The default case should continue to work without any warnings + with check_warnings() as w: + class WithoutClassRef(metaclass=Meta): + pass + self.assertEqual(len(w.warnings), 0) - with self.assertRaises(NameError): - A.f() - self.assertEqual(A.__classcell__, 0) + # With zero-arg super() or an explicit __class__ reference, we expect + # __build_class__ to emit a DeprecationWarning complaining that + # __class__ was not set, and asking if __classcell__ was propagated + # to type.__new__. + # In Python 3.7, that warning will become a RuntimeError. + expected_warning = ( + '__class__ not set.*__classcell__ propagated', + DeprecationWarning + ) + with check_warnings(expected_warning): + class WithClassRef(metaclass=Meta): + def f(self): + return __class__ + # Check __class__ still gets set despite the warning + self.assertIs(WithClassRef().f(), WithClassRef) + + # Check the warning is turned into an error as expected + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + with self.assertRaises(DeprecationWarning): + class WithClassRef(metaclass=Meta): + def f(self): + return __class__ + + def test___classcell___overwrite(self): + # See issue #23722 + # Overwriting __classcell__ with nonsense is explicitly prohibited + class Meta(type): + def __new__(cls, name, bases, namespace, cell): + namespace['__classcell__'] = cell + return super().__new__(cls, name, bases, namespace) + + for bad_cell in (None, 0, "", object()): + with self.subTest(bad_cell=bad_cell): + with self.assertRaises(TypeError): + class A(metaclass=Meta, cell=bad_cell): + pass + + def test___classcell___wrong_cell(self): + # See issue #23722 + # Pointing the cell reference at the wrong class is also prohibited + class Meta(type): + def __new__(cls, name, bases, namespace): + cls = super().__new__(cls, name, bases, namespace) + B = type("B", (), namespace) + return cls + + with self.assertRaises(TypeError): + class A(metaclass=Meta): + def f(self): + return __class__ def test_obscure_super_errors(self): def f(): diff -r d84c1b42a8f3 Objects/typeobject.c --- a/Objects/typeobject.c Sat Dec 03 15:57:00 2016 -0800 +++ b/Objects/typeobject.c Sun Dec 04 16:41:58 2016 +1000 @@ -2687,9 +2687,16 @@ else type->tp_free = PyObject_Del; - /* store type in class' cell */ + /* store type in class' cell if one is supplied */ cell = _PyDict_GetItemId(dict, &PyId___classcell__); - if (cell != NULL && PyCell_Check(cell)) { + if (cell != NULL) { + /* At least one method requires a reference to its defining class */ + if (!PyCell_Check(cell)) { + PyErr_Format(PyExc_TypeError, + "__classcell__ must be a nonlocal cell, not %.200R", + Py_TYPE(cell)); + goto error; + } PyCell_Set(cell, (PyObject *) type); _PyDict_DelItemId(dict, &PyId___classcell__); PyErr_Clear(); diff -r d84c1b42a8f3 Python/bltinmodule.c --- a/Python/bltinmodule.c Sat Dec 03 15:57:00 2016 -0800 +++ b/Python/bltinmodule.c Sun Dec 04 16:41:58 2016 +1000 @@ -54,8 +54,8 @@ static PyObject * builtin___build_class__(PyObject *self, PyObject *args, PyObject *kwds) { - PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns, *none; - PyObject *cls = NULL; + PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns; + PyObject *cls = NULL, *cell = NULL; Py_ssize_t nargs; int isclass = 0; /* initialize to prevent gcc warning */ @@ -167,14 +167,40 @@ Py_DECREF(bases); return NULL; } - none = PyEval_EvalCodeEx(PyFunction_GET_CODE(func), PyFunction_GET_GLOBALS(func), ns, + cell = PyEval_EvalCodeEx(PyFunction_GET_CODE(func), PyFunction_GET_GLOBALS(func), ns, NULL, 0, NULL, 0, NULL, 0, NULL, PyFunction_GET_CLOSURE(func)); - if (none != NULL) { + if (cell != NULL) { PyObject *margs[3] = {name, bases, ns}; cls = _PyObject_FastCallDict(meta, margs, 3, mkw); - Py_DECREF(none); + if (cls != NULL && PyCell_Check(cell)) { + PyObject *cell_cls = PyCell_GET(cell); + if (cell_cls != cls) { + int cell_error = 0; + /*TODO: In 3.7, deprecation warning becomes a RuntimeError */ + if (cell_cls == NULL) { + char *msg = "__class__ not set for %.200R. " + "Was __classcell__ propagated to type.__new__?"; + cell_error = PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, msg, cls); + } else { + char *msg = "__class__ set to %.200R for %.200R. "; + PyErr_Format(PyExc_TypeError, msg, cell_cls, cls); + cell_error = 1; + } + if (cell_error) { + Py_DECREF(cls); + cls = NULL; + goto error; + } else { + /* Fill in the cell, since type.__new__ didn't do it */ + PyCell_SET(cell, cls); + } + } + } } +error: + Py_XDECREF(cell); Py_DECREF(ns); Py_DECREF(meta); Py_XDECREF(mkw); diff -r d84c1b42a8f3 Python/compile.c --- a/Python/compile.c Sat Dec 03 15:57:00 2016 -0800 +++ b/Python/compile.c Sun Dec 04 16:41:58 2016 +1000 @@ -1967,8 +1967,9 @@ compiler_exit_scope(c); return 0; } + /* Return __classcell__ if it is referenced, otherwise return None */ if (c->u->u_ste->ste_needs_class_closure) { - /* store __classcell__ into class namespace */ + /* Store __classcell__ into class namespace & return it */ str = PyUnicode_InternFromString("__class__"); if (str == NULL) { compiler_exit_scope(c); @@ -1983,6 +1984,7 @@ assert(i == 0); ADDOP_I(c, LOAD_CLOSURE, i); + ADDOP(c, DUP_TOP); str = PyUnicode_InternFromString("__classcell__"); if (!str || !compiler_nameop(c, str, Store)) { Py_XDECREF(str); @@ -1992,9 +1994,11 @@ Py_DECREF(str); } else { - /* This happens when nobody references the cell. */ + /* No methods referenced __class__, so just return None */ assert(PyDict_Size(c->u->u_cellvars) == 0); + ADDOP_O(c, LOAD_CONST, Py_None, consts); } + ADDOP_IN_SCOPE(c, RETURN_VALUE); /* create the code object */ co = assemble(c, 1); }