diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -396,13 +396,14 @@ 3210 (added size modulo 2**32 to the pyc header) Python 3.3a1 3220 (changed PEP 380 implementation) Python 3.3a4 3230 (revert changes to implicit __class__ closure) + Python 3.4.0a0: 3240 (class statement creates two scopes) MAGIC must change whenever the bytecode emitted by the compiler may no longer be understood by older implementations of the eval loop (usually due to the addition of new opcodes). """ -_RAW_MAGIC_NUMBER = 3230 | ord('\r') << 16 | ord('\n') << 24 +_RAW_MAGIC_NUMBER = 3240 | ord('\r') << 16 | ord('\n') << 24 _MAGIC_BYTES = bytes(_RAW_MAGIC_NUMBER >> n & 0xff for n in range(0, 25, 8)) _PYCACHE = '__pycache__' diff --git a/Lib/test/test_metaclass.py b/Lib/test/test_metaclass.py --- a/Lib/test/test_metaclass.py +++ b/Lib/test/test_metaclass.py @@ -138,7 +138,7 @@ ... Traceback (most recent call last): [...] - TypeError: __build_class__() got multiple values for keyword argument 'metaclass' + TypeError: C() got multiple values for keyword argument 'metaclass' >>> Use a __prepare__ method that returns an instrumented dict. diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -41,6 +41,16 @@ class G(A): pass +class H(A): + # See issue #12370 + def f(self): + return super().f() + def g(self): + return FOO + __class__ = 413 + +FOO = 'FOO' + class TestSuper(unittest.TestCase): @@ -81,7 +91,6 @@ self.assertEqual(E().f(), 'AE') - @unittest.expectedFailure def test___class___set(self): # See issue #12370 class X(A): @@ -92,6 +101,13 @@ self.assertEqual(x.f(), 'A') self.assertEqual(x.__class__, 413) + def test___class___set_module_level(self): + # See issue #12370 + x = H() + self.assertEqual(x.f(), 'A') + self.assertEqual(x.g(), 'FOO') + self.assertEqual(x.__class__, 413) + def test___class___instancemethod(self): # See issue #14857 class X: @@ -115,6 +131,147 @@ return __class__ self.assertIs(X.f(), X) + def test___class___behavior(self): + # test that the changes in class scopes + # doesn't modify existing behavior + # (see issue #12370) + def func1(): + class X(A): + def f(self): + return super().f() + def g(self): + return __class__ + def h(self): + return FOO + return X + X = func1() + self.assertEqual(X().f(), 'A') + self.assertIs(X().g(), X) + self.assertEqual(X().h(), 'FOO') + + def func2(): + a = 1 + class X(A): + def f(self): + return super().f(), a + def g(self): + return __class__, b + b = 2 + return X + X = func2() + self.assertEqual(X().f(), ('A', 1)) + cls, b = X().g() + self.assertIs(cls, X) + self.assertEqual(b, 2) + + z = 42 + def func3(): + nonlocal z + class X(A): + def f(self): + return super().f(), z + def g(self): + return __class__, z + z = 99 + z = 0 + return X + X = func3() + self.assertEqual(X().f(), ('A', 0)) + cls, z_ = X().g() + self.assertIs(cls, X) + self.assertEqual(z_, 0) + self.assertEqual(z, 0) + + z = 42 + def func4(): + nonlocal z + class Outer: + class X(A): + def f(self): + return super().f(), z + def g(self): + return __class__, z + z = 99 + nonlocal z + z = 0 + return Outer.X + X = func4() + self.assertEqual(X().f(), ('A', 0)) + cls, z_ = X().g() + self.assertIs(cls, X) + self.assertEqual(z_, 0) + self.assertEqual(z, 0) + + class X: + __class__ = 45 + self.assertEqual(X().__class__, 45) + + def test_class_outer_scope_1(self): + # See issue #12370 + z = 42 + def func1(): + nonlocal z + class X(A): + def f(self): + return super().f(), z + def g(self): + return __class__, z + z = 99 + __class__ = 55 + z = 0 + return X + X = func1() + self.assertEqual(X().f(), ('A', 0)) + cls, z_ = X().g() + self.assertIs(cls, X) + self.assertEqual(z_, 0) + self.assertEqual(z, 0) + self.assertEqual(X().__class__, 55) + + def test_class_outer_scope_2(self): + # See issue #12370 + z = 42 + def func4(): + nonlocal z + class Outer: + class X(A): + def f(self): + return super().f(), z + def g(self): + return __class__, z + def h(self): + return FOO + z = 99 + __class__ = 12 + __class__ = 24 + nonlocal z + z = 0 + self.assertEqual(Outer().__class__, 24) + return Outer.X + X = func4() + self.assertEqual(X().f(), ('A', 0)) + self.assertEqual(X().h(), 'FOO') + cls, z_ = X().g() + self.assertIs(cls, X) + self.assertEqual(z_, 0) + self.assertEqual(z, 0) + self.assertEqual(X().__class__, 12) + + def test_class_outer_scope_3(self): + # See issue #12370 + __class__ = 45 + class X: + def f(self): + return __class__ + nonlocal __class__ + __class__ = 99 # writes into the special scope + # but after the class is created, __class__ is + # overwritten again with the correct value + self.assertEqual(__class__, 45) + self.assertIs(X.__class__, type) + self.assertIs(X().__class__, X) + self.assertIs(X().f(), X) + def test_obscure_super_errors(self): def f(): super() diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -44,7 +44,9 @@ top = symtable.symtable(TEST_CODE, "?", "exec") # These correspond to scopes in TEST_CODE - Mine = find_block(top, "Mine") + # A class has two scopes nested in each other, we + # get the inner scope (the scope of the class body): + Mine = find_block(find_block(top, "Mine"), "Mine") a_method = find_block(Mine, "a_method") spam = find_block(top, "spam") internal = find_block(spam, "internal") @@ -65,7 +67,9 @@ def test_nested(self): self.assertFalse(self.top.is_nested()) - self.assertFalse(self.Mine.is_nested()) + # The class body scope is nested in the outer scope + # of the class (which contains __class__): + self.assertTrue(self.Mine.is_nested()) self.assertFalse(self.spam.is_nested()) self.assertTrue(self.internal.is_nested()) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -38,7 +38,7 @@ static PyObject * builtin___build_class__(PyObject *self, PyObject *args, PyObject *kwds) { - PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns, *cell; + PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns, *none; PyObject *cls = NULL; Py_ssize_t nargs; int isclass; @@ -155,17 +155,15 @@ Py_DECREF(bases); return NULL; } - cell = PyObject_CallFunctionObjArgs(func, ns, NULL); - if (cell != NULL) { + none = PyObject_CallFunctionObjArgs(func, ns, NULL); + if (none != NULL) { PyObject *margs; margs = PyTuple_Pack(3, name, bases, ns); if (margs != NULL) { cls = PyEval_CallObjectWithKeywords(meta, margs, mkw); Py_DECREF(margs); } - if (cls != NULL && PyCell_Check(cell)) - PyCell_Set(cell, cls); - Py_DECREF(cell); + Py_DECREF(none); } Py_DECREF(ns); Py_DECREF(meta); diff --git a/Python/compile.c b/Python/compile.c --- a/Python/compile.c +++ b/Python/compile.c @@ -95,6 +95,8 @@ COMPILER_SCOPE_CLASS, COMPILER_SCOPE_FUNCTION, COMPILER_SCOPE_COMPREHENSION, + /* the outer scope of a class (it contains __class__): */ + COMPILER_SCOPE_CLASS_OUTER, }; /* The following items change on entry and exit of code blocks. @@ -628,7 +630,9 @@ capsule = PyList_GET_ITEM(c->c_stack, i); u = (struct compiler_unit *)PyCapsule_GetPointer(capsule, COMPILER_CAPSULE_NAME_COMPILER_UNIT); assert(u); - if (u->u_scope_type == COMPILER_SCOPE_MODULE) + if ((u->u_scope_type == COMPILER_SCOPE_MODULE) || + /* we also skip the outer scope of a class: */ + (u->u_scope_type == COMPILER_SCOPE_CLASS_OUTER)) continue; if (PyList_Append(seq, u->u_name)) goto _error; @@ -1636,19 +1640,35 @@ if (!compiler_decorators(c, decos)) return 0; - /* ultimately generate code for: - = __build_class__(, , *, **) - where: - is a function/closure created from the class body; - it has a single argument (__locals__) where the dict - (or MutableSequence) representing the locals is passed - is the class name - is the positional arguments and *varargs argument - is the keyword arguments and **kwds argument + /* From a class statement like this: + class X(A, B, c=C, d=D): + ... # class body + + we will generate code like this: + def outerX(*__args__, **__kw__): + innerX = __build_class__(, 'X', *__args__, **__kw__) + __class__ = innerX + return innerX + X = outerX(A, B, c=C, d=D) + + where is a function/closure created from the class body; + it has a single argument (__locals__) where the dict + (or MutableSequence) representing the locals is passed. + This borrows from compiler_call. */ - /* 1. compile the class body into a code object */ + /* Create the outer scope of the class, which will contain __class__: */ + if (!compiler_enter_scope(c, s->v.ClassDef.name, + COMPILER_SCOPE_CLASS_OUTER, + /* Note: we use the body AST node as key, + because the class stmt node will be + used by the inner scope.*/ + (void *)s->v.ClassDef.body, + s->lineno)) + return 0; + + /* Create the inner scope of the class (the scope of the class body): */ if (!compiler_enter_scope(c, s->v.ClassDef.name, COMPILER_SCOPE_CLASS, (void *)s, s->lineno)) return 0; @@ -1700,57 +1720,108 @@ compiler_exit_scope(c); return 0; } - /* return the (empty) __class__ cell */ - str = PyUnicode_InternFromString("__class__"); - if (str == NULL) { + + /* Return None */ + ADDOP_O(c, LOAD_CONST, Py_None, consts); + ADDOP_IN_SCOPE(c, RETURN_VALUE); + + /* create the code object of the class body */ + co = assemble(c, 1); + } + /* leave the inner scope of the class */ + compiler_exit_scope(c); + if (co == NULL) + return 0; + + /* load the '__build_class__' function */ + ADDOP(c, LOAD_BUILD_CLASS); + + /* Load the arguments to __build_class__: */ + + /* 1. a function (or closure) made from the + code object of the class body: */ + compiler_make_closure(c, co, 0, NULL); + Py_DECREF(co); + + /* 2. the class name: */ + ADDOP_O(c, LOAD_CONST, s->v.ClassDef.name, consts); + + /* 3. __args__: */ + str = PyUnicode_InternFromString("__args__"); + if (!str || !compiler_nameop(c, str, Load)) { + Py_XDECREF(str); + compiler_exit_scope(c); + return 0; + } + Py_DECREF(str); + + /* 4. __kw__: */ + str = PyUnicode_InternFromString("__kw__"); + if (!str || !compiler_nameop(c, str, Load)) { + Py_XDECREF(str); + compiler_exit_scope(c); + return 0; + } + Py_DECREF(str); + + /* call __build_class__: */ + ADDOP_I(c, CALL_FUNCTION_VAR_KW, 2); + + str = PyUnicode_InternFromString("__class__"); + if (str == NULL) { + compiler_exit_scope(c); + return 0; + } + + i = compiler_lookup_arg(c->u->u_cellvars, str); + if (i == -1) { + /* Nobody references __class__, so we don't have to store it. */ + PyErr_Clear(); + } + else { + /* Duplicate the class object on the stack, because + we'll both store and return it: */ + ADDOP(c, DUP_TOP); + + /* Store the class object into __class__: */ + if (!compiler_nameop(c, str, Store)) { + Py_DECREF(str); compiler_exit_scope(c); return 0; } - i = compiler_lookup_arg(c->u->u_cellvars, str); - Py_DECREF(str); - if (i == -1) { - /* This happens when nobody references the cell */ - PyErr_Clear(); - /* Return None */ - ADDOP_O(c, LOAD_CONST, Py_None, consts); - } - else { - /* Return the cell where to store __class__ */ - ADDOP_I(c, LOAD_CLOSURE, i); - } - ADDOP_IN_SCOPE(c, RETURN_VALUE); - /* create the code object */ - co = assemble(c, 1); } - /* leave the new scope */ + Py_DECREF(str); + + /* Return the class object: */ + ADDOP_IN_SCOPE(c, RETURN_VALUE); + + /* Create the outer code object of the class: */ + co = assemble(c, 1); + + /* Leave the outer scope of the class: */ compiler_exit_scope(c); if (co == NULL) return 0; - /* 2. load the 'build_class' function */ - ADDOP(c, LOAD_BUILD_CLASS); - - /* 3. load a function (or closure) made from the code object */ + /* Create the closure from the outer code object: */ compiler_make_closure(c, co, 0, NULL); Py_DECREF(co); - /* 4. load class name */ - ADDOP_O(c, LOAD_CONST, s->v.ClassDef.name, consts); - - /* 5. generate the rest of the code for the call */ - if (!compiler_call_helper(c, 2, + /* Call it with the args of the class definition: */ + if (!compiler_call_helper(c, 0, s->v.ClassDef.bases, s->v.ClassDef.keywords, s->v.ClassDef.starargs, - s->v.ClassDef.kwargs)) + s->v.ClassDef.kwargs)) { return 0; - - /* 6. apply decorators */ + } + + /* apply decorators */ for (i = 0; i < asdl_seq_LEN(decos); i++) { ADDOP_I(c, CALL_FUNCTION, 1); } - /* 7. store into */ + /* store into the class name */ if (!compiler_nameop(c, s->v.ClassDef.name, Store)) return 0; return 1; diff --git a/Python/symtable.c b/Python/symtable.c --- a/Python/symtable.c +++ b/Python/symtable.c @@ -191,7 +191,7 @@ static identifier top = NULL, lambda = NULL, genexpr = NULL, listcomp = NULL, setcomp = NULL, dictcomp = NULL, - __class__ = NULL, __locals__ = NULL; + __class__ = NULL, __locals__ = NULL, __args__ = NULL, __kw__ = NULL; #define GET_IDENTIFIER(VAR) \ ((VAR) ? (VAR) : ((VAR) = PyUnicode_InternFromString(# VAR))) @@ -781,14 +781,6 @@ goto error; Py_DECREF(temp); } - else { - /* Special-case __class__ */ - if (!GET_IDENTIFIER(__class__)) - goto error; - assert(PySet_Contains(local, __class__) == 1); - if (PySet_Add(newbound, __class__) < 0) - goto error; - } /* Recursively call analyze_child_block() on each child block. @@ -821,9 +813,6 @@ if (ste->ste_type == FunctionBlock && !analyze_cells(scopes, newfree, NULL)) goto error; - else if (ste->ste_type == ClassBlock && !analyze_cells(scopes, newfree, - "__class__")) - goto error; /* Records the results of the analysis in the symbol table entry */ if (!update_symbols(ste->ste_symbols, scopes, bound, newfree, ste->ste_type == ClassBlock)) @@ -1176,22 +1165,53 @@ VISIT(st, expr, s->v.ClassDef.kwargs); if (s->v.ClassDef.decorator_list) VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list); + + /* The outer scope of the class (it contains __class__). + Note: it's a FunctionBlock, because we need to see __class__ in the method bodies. */ + if (!symtable_enter_block(st, s->v.ClassDef.name, FunctionBlock, + /* Note: we use the body AST node as key, + because the class stmt node will be + used by the inner scope.*/ + (void *)s->v.ClassDef.body, + s->lineno, s->col_offset)) + VISIT_QUIT(st, 0); + + /* The outer scope contains the definition of __class__ + and has two args (*__args__, **__kw__): */ + if (!GET_IDENTIFIER(__class__) || + !symtable_add_def(st, __class__, DEF_LOCAL) || + !GET_IDENTIFIER(__args__) || + !symtable_add_def(st, __args__, DEF_PARAM | USE) || + !GET_IDENTIFIER(__kw__) || + !symtable_add_def(st, __kw__, DEF_PARAM | USE)) { + symtable_exit_block(st, s->v.ClassDef.body); + VISIT_QUIT(st, 0); + } + + /* We have *__args__ and **__kw__: */ + st->st_cur->ste_varargs = 1; + st->st_cur->ste_varkeywords = 1; + + /* Inner scope of the class (the scope of the class body): */ if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock, (void *)s, s->lineno, s->col_offset)) VISIT_QUIT(st, 0); - if (!GET_IDENTIFIER(__class__) || - !symtable_add_def(st, __class__, DEF_LOCAL) || - !GET_IDENTIFIER(__locals__) || + if (!GET_IDENTIFIER(__locals__) || !symtable_add_def(st, __locals__, DEF_PARAM)) { symtable_exit_block(st, s); + symtable_exit_block(st, s->v.ClassDef.body); VISIT_QUIT(st, 0); } tmp = st->st_private; st->st_private = s->v.ClassDef.name; VISIT_SEQ(st, stmt, s->v.ClassDef.body); st->st_private = tmp; + /* Exit the inner scope: */ if (!symtable_exit_block(st, s)) VISIT_QUIT(st, 0); + /* Exit the outer scope: */ + if (!symtable_exit_block(st, s->v.ClassDef.body)) + VISIT_QUIT(st, 0); break; } case Return_kind: