diff -r 52597f888e7a Include/ceval.h --- a/Include/ceval.h Thu Mar 15 18:08:34 2012 +0100 +++ b/Include/ceval.h Thu Mar 15 19:05:01 2012 +0000 @@ -37,6 +37,7 @@ flag was set, else return 0. */ #ifndef Py_LIMITED_API PyAPI_FUNC(int) PyEval_MergeCompilerFlags(PyCompilerFlags *cf); +PyObject *_PyEval_YieldFromValue(struct _frame *); #endif PyAPI_FUNC(int) Py_AddPendingCall(int (*func)(void *), void *arg); diff -r 52597f888e7a Include/frameobject.h --- a/Include/frameobject.h Thu Mar 15 18:08:34 2012 +0100 +++ b/Include/frameobject.h Thu Mar 15 19:05:01 2012 +0000 @@ -27,7 +27,6 @@ to the current stack top. */ PyObject **f_stacktop; PyObject *f_trace; /* Trace function */ - PyObject *f_yieldfrom; /* Iterator being delegated to by yield from */ /* In a generator, we need to be able to swap between the exception state inside the generator and the exception state of the calling diff -r 52597f888e7a Include/genobject.h --- a/Include/genobject.h Thu Mar 15 18:08:34 2012 +0100 +++ b/Include/genobject.h Thu Mar 15 19:05:01 2012 +0000 @@ -19,7 +19,7 @@ /* True if generator is being executed. */ char gi_running; - + /* The code object backing the generator */ PyObject *gi_code; @@ -35,6 +35,7 @@ PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *); PyAPI_FUNC(int) PyGen_NeedsFinalizing(PyGenObject *); PyAPI_FUNC(int) PyGen_FetchStopIterationValue(PyObject **); +PyObject *_PyGen_Send(PyGenObject *, PyObject *); #ifdef __cplusplus } diff -r 52597f888e7a Include/opcode.h --- a/Include/opcode.h Thu Mar 15 18:08:34 2012 +0100 +++ b/Include/opcode.h Thu Mar 15 19:05:01 2012 +0000 @@ -121,9 +121,9 @@ #define MAKE_CLOSURE 134 /* same as MAKE_FUNCTION */ #define LOAD_CLOSURE 135 /* Load free variable from closure */ -#define LOAD_DEREF 136 /* Load and dereference from closure cell */ -#define STORE_DEREF 137 /* Store into cell */ -#define DELETE_DEREF 138 /* Delete closure cell */ +#define LOAD_DEREF 136 /* Load and dereference from closure cell */ +#define STORE_DEREF 137 /* Store into cell */ +#define DELETE_DEREF 138 /* Delete closure cell */ /* The next 3 opcodes must be contiguous and satisfy (CALL_FUNCTION_VAR - CALL_FUNCTION) & 3 == 1 */ diff -r 52597f888e7a Lib/test/test_pep380.py --- a/Lib/test/test_pep380.py Thu Mar 15 18:08:34 2012 +0100 +++ b/Lib/test/test_pep380.py Thu Mar 15 19:05:01 2012 +0000 @@ -10,7 +10,7 @@ import unittest import io import sys -import traceback +import inspect import parser from test.support import captured_stderr @@ -919,6 +919,27 @@ next(g1) g1.close() + def test_delegator_is_visible_to_debugger(self): + def call_stack(): + return [f[3] for f in inspect.stack()] + + def gen(): + yield call_stack() + yield call_stack() + yield call_stack() + + def spam(g): + yield from g + + def eggs(g): + yield from g + + for stack in spam(gen()): + self.assertTrue('spam' in stack) + + for stack in spam(eggs(gen())): + self.assertTrue('spam' in stack and 'eggs' in stack) + def test_main(): from test import support diff -r 52597f888e7a Lib/test/test_sys.py --- a/Lib/test/test_sys.py Thu Mar 15 18:08:34 2012 +0100 +++ b/Lib/test/test_sys.py Thu Mar 15 19:05:01 2012 +0000 @@ -730,7 +730,7 @@ nfrees = len(x.f_code.co_freevars) extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\ ncells + nfrees - 1 - check(x, size(vh + '13P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P')) + check(x, size(vh + '12P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P')) # function def func(): pass check(func, size(h + '12P')) diff -r 52597f888e7a Objects/frameobject.c --- a/Objects/frameobject.c Thu Mar 15 18:08:34 2012 +0100 +++ b/Objects/frameobject.c Thu Mar 15 19:05:01 2012 +0000 @@ -444,7 +444,6 @@ Py_CLEAR(f->f_exc_type); Py_CLEAR(f->f_exc_value); Py_CLEAR(f->f_exc_traceback); - Py_CLEAR(f->f_yieldfrom); co = f->f_code; if (co->co_zombieframe == NULL) @@ -476,7 +475,6 @@ Py_VISIT(f->f_exc_type); Py_VISIT(f->f_exc_value); Py_VISIT(f->f_exc_traceback); - Py_VISIT(f->f_yieldfrom); /* locals */ slots = f->f_code->co_nlocals + PyTuple_GET_SIZE(f->f_code->co_cellvars) + PyTuple_GET_SIZE(f->f_code->co_freevars); @@ -510,7 +508,6 @@ Py_CLEAR(f->f_exc_value); Py_CLEAR(f->f_exc_traceback); Py_CLEAR(f->f_trace); - Py_CLEAR(f->f_yieldfrom); /* locals */ slots = f->f_code->co_nlocals + PyTuple_GET_SIZE(f->f_code->co_cellvars) + PyTuple_GET_SIZE(f->f_code->co_freevars); @@ -714,7 +711,6 @@ f->f_lasti = -1; f->f_lineno = code->co_firstlineno; f->f_iblock = 0; - f->f_yieldfrom = NULL; _PyObject_GC_TRACK(f); return f; diff -r 52597f888e7a Objects/genobject.c --- a/Objects/genobject.c Thu Mar 15 18:08:34 2012 +0100 +++ b/Objects/genobject.c Thu Mar 15 19:05:01 2012 +0000 @@ -6,7 +6,6 @@ #include "opcode.h" static PyObject *gen_close(PyGenObject *gen, PyObject *args); -static void gen_undelegate(PyGenObject *gen); static int gen_traverse(PyGenObject *gen, visitproc visit, void *arg) @@ -41,15 +40,6 @@ PyObject_GC_Del(gen); } -static int -gen_running(PyGenObject *gen) -{ - if (gen->gi_running) { - PyErr_SetString(PyExc_ValueError, "generator already executing"); - return 1; - } - return 0; -} static PyObject * gen_send_ex(PyGenObject *gen, PyObject *arg, int exc) @@ -58,7 +48,11 @@ PyFrameObject *f = gen->gi_frame; PyObject *result; - assert(!gen->gi_running); + if (gen->gi_running) { + PyErr_SetString(PyExc_ValueError, + "generator already executing"); + return NULL; + } if (f==NULL || f->f_stacktop == NULL) { /* Only set exception if called from send() */ if (arg && !exc) @@ -136,45 +130,10 @@ "send(arg) -> send 'arg' into generator,\n\ return next yielded value or raise StopIteration."); -static PyObject * -gen_send(PyGenObject *gen, PyObject *arg) +PyObject * +_PyGen_Send(PyGenObject *gen, PyObject *arg) { - int exc = 0; - PyObject *ret; - PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL; - if (gen_running(gen)) - return NULL; - /* XXX (ncoghlan): Are the incref/decref on arg and yf strictly needed? - * Or would it be valid to rely on borrowed references? - */ - Py_INCREF(arg); - if (yf) { - Py_INCREF(yf); - gen->gi_running = 1; - if (PyGen_CheckExact(yf)) { - ret = gen_send((PyGenObject *)yf, arg); - } else { - if (arg == Py_None) - ret = PyIter_Next(yf); - else - ret = PyObject_CallMethod(yf, "send", "O", arg); - } - gen->gi_running = 0; - if (ret) { - Py_DECREF(yf); - goto done; - } - gen_undelegate(gen); - Py_CLEAR(arg); - if (PyGen_FetchStopIterationValue(&arg) < 0) { - exc = 1; - } - Py_DECREF(yf); - } - ret = gen_send_ex(gen, arg, exc); -done: - Py_XDECREF(arg); - return ret; + return gen_send_ex(gen, arg, 0); } PyDoc_STRVAR(close_doc, @@ -186,19 +145,17 @@ */ static int -gen_close_iter(PyGenObject *gen, PyObject *yf) +gen_close_iter(PyObject *yf) { PyObject *retval = NULL; - int err = 0; - + if (PyGen_CheckExact(yf)) { retval = gen_close((PyGenObject *)yf, NULL); - if (!retval) - err = -1; + if (retval == NULL) { + return -1; + } } else { - PyObject *meth; - gen->gi_running = 1; - meth = PyObject_GetAttrString(yf, "close"); + PyObject *meth = PyObject_GetAttrString(yf, "close"); if (meth == NULL) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_WriteUnraisable(yf); @@ -208,27 +165,24 @@ retval = PyObject_CallFunction(meth, ""); Py_DECREF(meth); if (!retval) - err = -1; + return -1; } - gen->gi_running = 0; } Py_XDECREF(retval); - return err; -} + return 0; +} static PyObject * gen_close(PyGenObject *gen, PyObject *args) { PyObject *retval; - PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL; + PyObject *yf = gen->gi_frame ? _PyEval_YieldFromValue(gen->gi_frame) : NULL; int err = 0; - if (gen_running(gen)) - return NULL; if (yf) { - Py_INCREF(yf); - err = gen_close_iter(gen, yf); - gen_undelegate(gen); + gen->gi_running = 1; + err = gen_close_iter(yf); + gen->gi_running = 0; Py_DECREF(yf); } if (err == 0) @@ -323,29 +277,27 @@ PyObject *typ; PyObject *tb = NULL; PyObject *val = NULL; - PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL; + PyObject *yf = gen->gi_frame ? _PyEval_YieldFromValue(gen->gi_frame) : NULL; if (!PyArg_UnpackTuple(args, "throw", 1, 3, &typ, &val, &tb)) - return NULL; - - if (gen_running(gen)) return NULL; if (yf) { PyObject *ret; int err; - Py_INCREF(yf); if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) { - err = gen_close_iter(gen, yf); + gen->gi_running = 1; + err = gen_close_iter(yf); + gen->gi_running = 0; Py_DECREF(yf); - gen_undelegate(gen); if (err < 0) return gen_send_ex(gen, Py_None, 1); goto throw_here; } - gen->gi_running = 1; if (PyGen_CheckExact(yf)) { + gen->gi_running = 1; ret = gen_throw((PyGenObject *)yf, args); + gen->gi_running = 0; } else { PyObject *meth = PyObject_GetAttrString(yf, "throw"); if (meth == NULL) { @@ -355,18 +307,22 @@ } PyErr_Clear(); Py_DECREF(yf); - gen_undelegate(gen); - gen->gi_running = 0; goto throw_here; } + gen->gi_running = 1; ret = PyObject_CallObject(meth, args); + gen->gi_running = 0; Py_DECREF(meth); } - gen->gi_running = 0; Py_DECREF(yf); if (!ret) { PyObject *val; - gen_undelegate(gen); + /* Pop subiterator from stack */ + ret = *(--gen->gi_frame->f_stacktop); + assert(ret == yf); + Py_DECREF(ret); + /* Termination repetition of YIELD_FROM */ + gen->gi_frame->f_lasti++; if (PyGen_FetchStopIterationValue(&val) == 0) { ret = gen_send_ex(gen, val, 0); Py_DECREF(val); @@ -441,42 +397,9 @@ { PyObject *val = NULL; PyObject *ret; - int exc = 0; - PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL; - if (gen_running(gen)) - return NULL; - if (yf) { - Py_INCREF(yf); - /* ceval.c ensures that yf is an iterator */ - gen->gi_running = 1; - ret = Py_TYPE(yf)->tp_iternext(yf); - gen->gi_running = 0; - if (ret) { - Py_DECREF(yf); - return ret; - } - gen_undelegate(gen); - if (PyGen_FetchStopIterationValue(&val) < 0) - exc = 1; - Py_DECREF(yf); - } - ret = gen_send_ex(gen, val, exc); + ret = gen_send_ex(gen, val, 0); Py_XDECREF(val); return ret; -} - -/* - * In certain recursive situations, a generator may lose its frame - * before we get a chance to clear f_yieldfrom, so we use this - * helper function. - */ - -static void -gen_undelegate(PyGenObject *gen) { - if (gen->gi_frame) { - Py_XDECREF(gen->gi_frame->f_yieldfrom); - gen->gi_frame->f_yieldfrom = NULL; - } } /* @@ -492,7 +415,7 @@ PyGen_FetchStopIterationValue(PyObject **pvalue) { PyObject *et, *ev, *tb; PyObject *value = NULL; - + if (PyErr_ExceptionMatches(PyExc_StopIteration)) { PyErr_Fetch(&et, &ev, &tb); Py_XDECREF(et); @@ -548,7 +471,7 @@ }; static PyMethodDef gen_methods[] = { - {"send",(PyCFunction)gen_send, METH_O, send_doc}, + {"send",(PyCFunction)_PyGen_Send, METH_O, send_doc}, {"throw",(PyCFunction)gen_throw, METH_VARARGS, throw_doc}, {"close",(PyCFunction)gen_close, METH_NOARGS, close_doc}, {NULL, NULL} /* Sentinel */ diff -r 52597f888e7a Python/ceval.c --- a/Python/ceval.c Thu Mar 15 18:08:34 2012 +0100 +++ b/Python/ceval.c Thu Mar 15 19:05:01 2012 +0000 @@ -1170,6 +1170,8 @@ f->f_lasti to -1 (i.e. the index *before* the first instruction) and YIELD_VALUE doesn't fiddle with f_lasti any more. So this does work. Promise. + YIELD_FROM sets f_lasti to itself, in order to repeated yield + multiple values. When the PREDICT() macros are enabled, some opcode pairs follow in direct succession without updating f->f_lasti. A successful @@ -1830,49 +1832,32 @@ TARGET(YIELD_FROM) u = POP(); - x = PyObject_GetIter(u); - Py_DECREF(u); - if (x == NULL) - break; - /* x is now the iterator, make the first next() call */ - retval = (*Py_TYPE(x)->tp_iternext)(x); + x = TOP(); + /* send u to x */ + if (PyGen_CheckExact(x)) { + retval = _PyGen_Send((PyGenObject *)x, u); + } else { + if (u == Py_None) + retval = PyIter_Next(x); + else + retval = PyObject_CallMethod(x, "send", "O", u); + } + Py_DECREF(u); if (!retval) { - PyObject *et, *ev, *tb; - /* iter may be exhausted */ + PyObject *val; + x = POP(); /* Remove iter from stack */ Py_CLEAR(x); - if (PyErr_Occurred() && - !PyErr_ExceptionMatches(PyExc_StopIteration)) { - /* some other exception */ - break; - } - /* try to get return value from exception */ - PyErr_Fetch(&et, &ev, &tb); - Py_XDECREF(et); - Py_XDECREF(tb); - /* u is return value */ - u = NULL; - if (ev) { - u = PyObject_GetAttrString(ev, "value"); - Py_DECREF(ev); - if (u == NULL) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - /* some other exception */ - break; - } - PyErr_Clear(); - } - } - if (u == NULL) { - u = Py_None; - Py_INCREF(u); - } - PUSH(u); + if ((err = PyGen_FetchStopIterationValue(&val)) < 0) + break; + x = val; + PUSH(x); continue; } - /* x is iterator, retval is value to be yielded */ - f->f_yieldfrom = x; + /* x remains on stack, retval is value to be yielded */ f->f_stacktop = stack_pointer; why = WHY_YIELD; + /* and repeat... */ + f->f_lasti--; goto fast_yield; TARGET(YIELD_VALUE) @@ -4619,6 +4604,18 @@ return res; } +PyObject * +_PyEval_YieldFromValue(PyFrameObject* f) { + PyObject *yf; + unsigned char *first_instr = (unsigned char*) + PyBytes_AS_STRING(f->f_code->co_code); + if (first_instr[f->f_lasti+1] != YIELD_FROM) + return NULL; + yf = f->f_stacktop[-1]; + Py_INCREF(yf); + return yf; +} + #ifdef DYNAMIC_EXECUTION_PROFILE static PyObject * diff -r 52597f888e7a Python/compile.c --- a/Python/compile.c Thu Mar 15 18:08:34 2012 +0100 +++ b/Python/compile.c Thu Mar 15 19:05:01 2012 +0000 @@ -840,9 +840,9 @@ case IMPORT_STAR: return -1; case YIELD_VALUE: + return 0; case YIELD_FROM: - return 0; - + return -1; case POP_BLOCK: return 0; case POP_EXCEPT: @@ -3323,6 +3323,8 @@ ADDOP_O(c, LOAD_CONST, Py_None, consts); } if (e->kind == YieldFrom_kind) { + ADDOP(c, GET_ITER); + ADDOP_O(c, LOAD_CONST, Py_None, consts); ADDOP(c, YIELD_FROM); } else {