diff -r 87dcd948dc74 Include/ceval.h --- a/Include/ceval.h Wed Feb 29 16:21:25 2012 -0800 +++ b/Include/ceval.h Thu Mar 08 20:48:47 2012 +0000 @@ -37,6 +37,7 @@ flag was set, else return 0. */ #ifndef Py_LIMITED_API PyAPI_FUNC(int) PyEval_MergeCompilerFlags(PyCompilerFlags *cf); +PyObject *_PyEval_YieldFromValue(struct _frame *); #endif PyAPI_FUNC(int) Py_AddPendingCall(int (*func)(void *), void *arg); diff -r 87dcd948dc74 Include/frameobject.h --- a/Include/frameobject.h Wed Feb 29 16:21:25 2012 -0800 +++ b/Include/frameobject.h Thu Mar 08 20:48:47 2012 +0000 @@ -27,7 +27,6 @@ to the current stack top. */ PyObject **f_stacktop; PyObject *f_trace; /* Trace function */ - PyObject *f_yieldfrom; /* Iterator being delegated to by yield from */ /* In a generator, we need to be able to swap between the exception state inside the generator and the exception state of the calling diff -r 87dcd948dc74 Include/genobject.h --- a/Include/genobject.h Wed Feb 29 16:21:25 2012 -0800 +++ b/Include/genobject.h Thu Mar 08 20:48:47 2012 +0000 @@ -35,6 +35,7 @@ PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *); PyAPI_FUNC(int) PyGen_NeedsFinalizing(PyGenObject *); PyAPI_FUNC(int) PyGen_FetchStopIterationValue(PyObject **); +PyObject *_PyGen_Send(PyGenObject *, PyObject *); #ifdef __cplusplus } diff -r 87dcd948dc74 Lib/test/test_pep380.py --- a/Lib/test/test_pep380.py Wed Feb 29 16:21:25 2012 -0800 +++ b/Lib/test/test_pep380.py Thu Mar 08 20:48:47 2012 +0000 @@ -10,7 +10,7 @@ import unittest import io import sys -import traceback +import inspect import parser from test.support import captured_stderr @@ -847,6 +847,99 @@ yield from () self.assertRaises(StopIteration, next, g()) + def test_delegating_generators_claim_to_be_running(self): + # Check with basic iteration + def one(): + yield 0 + yield from two() + yield 3 + def two(): + yield 1 + try: + yield from g1 + except ValueError: + pass + yield 2 + g1 = one() + self.assertEqual(list(g1), [0, 1, 2, 3]) + # Check with send + g1 = one() + res = [next(g1)] + try: + while True: + res.append(g1.send(42)) + except StopIteration: + pass + self.assertEqual(res, [0, 1, 2, 3]) + # Check with throw + class MyErr(Exception): + pass + def one(): + try: + yield 0 + except MyErr: + pass + yield from two() + try: + yield 3 + except MyErr: + pass + def two(): + try: + yield 1 + except MyErr: + pass + try: + yield from g1 + except ValueError: + pass + try: + yield 2 + except MyErr: + pass + g1 = one() + res = [next(g1)] + try: + while True: + res.append(g1.throw(MyErr)) + except StopIteration: + pass + # Check with close + class MyIt(object): + def __iter__(self): + return self + def __next__(self): + return 42 + def close(self_): + self.assertTrue(g1.gi_running) + self.assertRaises(ValueError, next, g1) + def one(): + yield from MyIt() + g1 = one() + next(g1) + g1.close() + + def test_delegator_is_visible_to_debugger(self): + def call_stack(): + return [f[3] for f in inspect.stack()] + + def gen(): + yield call_stack() + yield call_stack() + yield call_stack() + + def spam(g): + yield from g + + def eggs(g): + yield from g + + for stack in spam(gen()): + self.assertTrue('spam' in stack) + + for stack in spam(eggs(gen())): + self.assertTrue('spam' in stack and 'eggs' in stack) + def test_main(): from test import support diff -r 87dcd948dc74 Objects/frameobject.c --- a/Objects/frameobject.c Wed Feb 29 16:21:25 2012 -0800 +++ b/Objects/frameobject.c Thu Mar 08 20:48:47 2012 +0000 @@ -20,7 +20,6 @@ {"f_builtins", T_OBJECT, OFF(f_builtins), READONLY}, {"f_globals", T_OBJECT, OFF(f_globals), READONLY}, {"f_lasti", T_INT, OFF(f_lasti), READONLY}, {NULL} /* Sentinel */ }; @@ -445,7 +444,6 @@ Py_CLEAR(f->f_exc_type); Py_CLEAR(f->f_exc_value); Py_CLEAR(f->f_exc_traceback); - Py_CLEAR(f->f_yieldfrom); co = f->f_code; if (co->co_zombieframe == NULL) @@ -477,7 +475,6 @@ Py_VISIT(f->f_exc_type); Py_VISIT(f->f_exc_value); Py_VISIT(f->f_exc_traceback); - Py_VISIT(f->f_yieldfrom); /* locals */ slots = f->f_code->co_nlocals + PyTuple_GET_SIZE(f->f_code->co_cellvars) + PyTuple_GET_SIZE(f->f_code->co_freevars); @@ -511,7 +508,6 @@ Py_CLEAR(f->f_exc_value); Py_CLEAR(f->f_exc_traceback); Py_CLEAR(f->f_trace); - Py_CLEAR(f->f_yieldfrom); /* locals */ slots = f->f_code->co_nlocals + PyTuple_GET_SIZE(f->f_code->co_cellvars) + PyTuple_GET_SIZE(f->f_code->co_freevars); @@ -715,7 +711,6 @@ f->f_lasti = -1; f->f_lineno = code->co_firstlineno; f->f_iblock = 0; - f->f_yieldfrom = NULL; _PyObject_GC_TRACK(f); return f; diff -r 87dcd948dc74 Objects/genobject.c --- a/Objects/genobject.c Wed Feb 29 16:21:25 2012 -0800 +++ b/Objects/genobject.c Thu Mar 08 20:48:47 2012 +0000 @@ -6,7 +6,6 @@ #include "opcode.h" static PyObject *gen_close(PyGenObject *gen, PyObject *args); -static void gen_undelegate(PyGenObject *gen); static int gen_traverse(PyGenObject *gen, visitproc visit, void *arg) @@ -131,41 +130,10 @@ "send(arg) -> send 'arg' into generator,\n\ return next yielded value or raise StopIteration."); -static PyObject * -gen_send(PyGenObject *gen, PyObject *arg) +PyObject * +_PyGen_Send(PyGenObject *gen, PyObject *arg) { - int exc = 0; - PyObject *ret; - PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL; - /* XXX (ncoghlan): Are the incref/decref on arg and yf strictly needed? - * Or would it be valid to rely on borrowed references? - */ - Py_INCREF(arg); - if (yf) { - Py_INCREF(yf); - if (PyGen_CheckExact(yf)) { - ret = gen_send((PyGenObject *)yf, arg); - } else { - if (arg == Py_None) - ret = PyIter_Next(yf); - else - ret = PyObject_CallMethod(yf, "send", "O", arg); - } - if (ret) { - Py_DECREF(yf); - goto done; - } - gen_undelegate(gen); - Py_CLEAR(arg); - if (PyGen_FetchStopIterationValue(&arg) < 0) { - exc = 1; - } - Py_DECREF(yf); - } - ret = gen_send_ex(gen, arg, exc); -done: - Py_XDECREF(arg); - return ret; + return gen_send_ex(gen, arg, 0); } PyDoc_STRVAR(close_doc, @@ -208,13 +176,13 @@ gen_close(PyGenObject *gen, PyObject *args) { PyObject *retval; - PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL; + PyObject *yf = gen->gi_frame ? _PyEval_YieldFromValue(gen->gi_frame) : NULL; int err = 0; if (yf) { - Py_INCREF(yf); + gen->gi_running = 1; err = gen_close_iter(yf); - gen_undelegate(gen); + gen->gi_running = 0; Py_DECREF(yf); } if (err == 0) @@ -309,7 +277,7 @@ PyObject *typ; PyObject *tb = NULL; PyObject *val = NULL; - PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL; + PyObject *yf = gen->gi_frame ? _PyEval_YieldFromValue(gen->gi_frame) : NULL; if (!PyArg_UnpackTuple(args, "throw", 1, 3, &typ, &val, &tb)) return NULL; @@ -317,17 +285,19 @@ if (yf) { PyObject *ret; int err; - Py_INCREF(yf); if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) { + gen->gi_running = 1; err = gen_close_iter(yf); + gen->gi_running = 0; Py_DECREF(yf); - gen_undelegate(gen); if (err < 0) return gen_send_ex(gen, Py_None, 1); goto throw_here; } if (PyGen_CheckExact(yf)) { + gen->gi_running = 1; ret = gen_throw((PyGenObject *)yf, args); + gen->gi_running = 0; } else { PyObject *meth = PyObject_GetAttrString(yf, "throw"); if (meth == NULL) { @@ -337,16 +307,22 @@ } PyErr_Clear(); Py_DECREF(yf); - gen_undelegate(gen); goto throw_here; } + gen->gi_running = 1; ret = PyObject_CallObject(meth, args); + gen->gi_running = 0; Py_DECREF(meth); } Py_DECREF(yf); if (!ret) { PyObject *val; - gen_undelegate(gen); + /* Pop subiterator from stack */ + ret = *(--gen->gi_frame->f_stacktop); + assert(ret == yf); + Py_DECREF(ret); + /* Termination repetition of YIELD_FROM */ + gen->gi_frame->f_lasti++; if (PyGen_FetchStopIterationValue(&val) == 0) { ret = gen_send_ex(gen, val, 0); Py_DECREF(val); @@ -421,38 +397,9 @@ { PyObject *val = NULL; PyObject *ret; - int exc = 0; - PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL; - if (yf) { - Py_INCREF(yf); - /* ceval.c ensures that yf is an iterator */ - ret = Py_TYPE(yf)->tp_iternext(yf); - if (ret) { - Py_DECREF(yf); - return ret; - } - gen_undelegate(gen); - if (PyGen_FetchStopIterationValue(&val) < 0) - exc = 1; - Py_DECREF(yf); - } - ret = gen_send_ex(gen, val, exc); + ret = gen_send_ex(gen, val, 0); Py_XDECREF(val); return ret; -} - -/* - * In certain recursive situations, a generator may lose its frame - * before we get a chance to clear f_yieldfrom, so we use this - * helper function. - */ - -static void -gen_undelegate(PyGenObject *gen) { - if (gen->gi_frame) { - Py_XDECREF(gen->gi_frame->f_yieldfrom); - gen->gi_frame->f_yieldfrom = NULL; - } } /* @@ -524,7 +471,7 @@ }; static PyMethodDef gen_methods[] = { - {"send",(PyCFunction)gen_send, METH_O, send_doc}, + {"send",(PyCFunction)_PyGen_Send, METH_O, send_doc}, {"throw",(PyCFunction)gen_throw, METH_VARARGS, throw_doc}, {"close",(PyCFunction)gen_close, METH_NOARGS, close_doc}, {NULL, NULL} /* Sentinel */ diff -r 87dcd948dc74 Python/ceval.c --- a/Python/ceval.c Wed Feb 29 16:21:25 2012 -0800 +++ b/Python/ceval.c Thu Mar 08 20:48:47 2012 +0000 @@ -1170,6 +1170,8 @@ f->f_lasti to -1 (i.e. the index *before* the first instruction) and YIELD_VALUE doesn't fiddle with f_lasti any more. So this does work. Promise. + YIELD_FROM sets f_lasti to itself, in order to repeated yield + multiple values. When the PREDICT() macros are enabled, some opcode pairs follow in direct succession without updating f->f_lasti. A successful @@ -1830,49 +1832,32 @@ TARGET(YIELD_FROM) u = POP(); - x = PyObject_GetIter(u); - Py_DECREF(u); - if (x == NULL) - break; - /* x is now the iterator, make the first next() call */ - retval = (*Py_TYPE(x)->tp_iternext)(x); + x = TOP(); + /* send u to x */ + if (PyGen_CheckExact(x)) { + retval = _PyGen_Send((PyGenObject *)x, u); + } else { + if (u == Py_None) + retval = PyIter_Next(x); + else + retval = PyObject_CallMethod(x, "send", "O", u); + } + Py_DECREF(u); if (!retval) { - PyObject *et, *ev, *tb; - /* iter may be exhausted */ + PyObject *val; + x = POP(); /* Remove iter from stack */ Py_CLEAR(x); - if (PyErr_Occurred() && - !PyErr_ExceptionMatches(PyExc_StopIteration)) { - /* some other exception */ - break; - } - /* try to get return value from exception */ - PyErr_Fetch(&et, &ev, &tb); - Py_XDECREF(et); - Py_XDECREF(tb); - /* u is return value */ - u = NULL; - if (ev) { - u = PyObject_GetAttrString(ev, "value"); - Py_DECREF(ev); - if (u == NULL) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - /* some other exception */ - break; - } - PyErr_Clear(); - } - } - if (u == NULL) { - u = Py_None; - Py_INCREF(u); - } - PUSH(u); + if ((err = PyGen_FetchStopIterationValue(&val)) < 0) + break; + x = val; + PUSH(x); continue; } - /* x is iterator, retval is value to be yielded */ - f->f_yieldfrom = x; + /* x remains on stack, retval is value to be yielded */ f->f_stacktop = stack_pointer; why = WHY_YIELD; + /* and repeat... */ + f->f_lasti--; goto fast_yield; TARGET(YIELD_VALUE) @@ -4619,6 +4604,20 @@ return res; } +PyObject * +_PyEval_YieldFromValue(PyFrameObject* f) { + PyObject *yf; + unsigned char *first_instr = (unsigned char*) + PyBytes_AS_STRING(f->f_code->co_code); + if (first_instr[f->f_lasti+1] != YIELD_FROM) + return NULL; + yf = f->f_stacktop[-1]; + Py_INCREF(yf); + return yf; +} + #ifdef DYNAMIC_EXECUTION_PROFILE static PyObject * diff -r 87dcd948dc74 Python/compile.c --- a/Python/compile.c Wed Feb 29 16:21:25 2012 -0800 +++ b/Python/compile.c Thu Mar 08 20:48:47 2012 +0000 @@ -840,9 +840,9 @@ case IMPORT_STAR: return -1; case YIELD_VALUE: + return 0; case YIELD_FROM: - return 0; + return -1; case POP_BLOCK: return 0; case POP_EXCEPT: @@ -3323,6 +3323,8 @@ ADDOP_O(c, LOAD_CONST, Py_None, consts); } if (e->kind == YieldFrom_kind) { + ADDOP(c, GET_ITER); + ADDOP_O(c, LOAD_CONST, Py_None, consts); ADDOP(c, YIELD_FROM); } else {