diff -r 8e538ed41766 Include/Python-ast.h --- a/Include/Python-ast.h Fri Jan 13 22:12:37 2012 +0100 +++ b/Include/Python-ast.h Sat Jan 14 01:15:29 2012 -0500 @@ -175,20 +175,20 @@ struct _stmt { } v; int lineno; int col_offset; }; enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4, IfExp_kind=5, Dict_kind=6, Set_kind=7, ListComp_kind=8, SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11, - Yield_kind=12, Compare_kind=13, Call_kind=14, Num_kind=15, - Str_kind=16, Bytes_kind=17, Ellipsis_kind=18, - Attribute_kind=19, Subscript_kind=20, Starred_kind=21, - Name_kind=22, List_kind=23, Tuple_kind=24}; + Yield_kind=12, YieldFrom_kind=13, Compare_kind=14, + Call_kind=15, Num_kind=16, Str_kind=17, Bytes_kind=18, + Ellipsis_kind=19, Attribute_kind=20, Subscript_kind=21, + Starred_kind=22, Name_kind=23, List_kind=24, Tuple_kind=25}; struct _expr { enum _expr_kind kind; union { struct { boolop_ty op; asdl_seq *values; } BoolOp; @@ -240,21 +240,24 @@ struct _expr { } DictComp; struct { expr_ty elt; asdl_seq *generators; } GeneratorExp; struct { - int is_from; expr_ty value; } Yield; struct { + expr_ty value; + } YieldFrom; + + struct { expr_ty left; asdl_int_seq *ops; asdl_seq *comparators; } Compare; struct { expr_ty func; asdl_seq *args; @@ -483,19 +486,21 @@ expr_ty _Py_ListComp(expr_ty elt, asdl_s expr_ty _Py_SetComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena *arena); #define DictComp(a0, a1, a2, a3, a4, a5) _Py_DictComp(a0, a1, a2, a3, a4, a5) expr_ty _Py_DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int lineno, int col_offset, PyArena *arena); #define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4) expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena *arena); -#define Yield(a0, a1, a2, a3, a4) _Py_Yield(a0, a1, a2, a3, a4) -expr_ty _Py_Yield(int is_from, expr_ty value, int lineno, int col_offset, - PyArena *arena); +#define Yield(a0, a1, a2, a3) _Py_Yield(a0, a1, a2, a3) +expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, PyArena *arena); +#define YieldFrom(a0, a1, a2, a3) _Py_YieldFrom(a0, a1, a2, a3) +expr_ty _Py_YieldFrom(expr_ty value, int lineno, int col_offset, PyArena + *arena); #define Compare(a0, a1, a2, a3, a4, a5) _Py_Compare(a0, a1, a2, a3, a4, a5) expr_ty _Py_Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators, int lineno, int col_offset, PyArena *arena); #define Call(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Call(a0, a1, a2, a3, a4, a5, a6, a7) expr_ty _Py_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty starargs, expr_ty kwargs, int lineno, int col_offset, PyArena *arena); #define Num(a0, a1, a2, a3) _Py_Num(a0, a1, a2, a3) diff -r 8e538ed41766 Lib/test/test_ast.py --- a/Lib/test/test_ast.py Fri Jan 13 22:12:37 2012 +0100 +++ b/Lib/test/test_ast.py Sat Jan 14 01:15:29 2012 -0500 @@ -808,18 +808,18 @@ class ASTValidatorTests(unittest.TestCas self.expr(c, "must have Load context") def factory(comps): k = ast.Name("x", ast.Load()) v = ast.Name("y", ast.Load()) return ast.DictComp(k, v, comps) self._check_comprehension(factory) def test_yield(self): - self.expr(ast.Yield(0, ast.Name("x", ast.Store())), "must have Load") - self.expr(ast.Yield(1, ast.Name("x", ast.Store())), "must have Load") + self.expr(ast.Yield(ast.Name("x", ast.Store())), "must have Load") + self.expr(ast.YieldFrom(ast.Name("x", ast.Store())), "must have Load") def test_compare(self): left = ast.Name("x", ast.Load()) comp = ast.Compare(left, [ast.In()], []) self.expr(comp, "no comparators") comp = ast.Compare(left, [ast.In()], [ast.Num(4), ast.Num(5)]) self.expr(comp, "different number of comparators and operands") comp = ast.Compare(ast.Num("blah"), [ast.In()], [left]) diff -r 8e538ed41766 Parser/Python.asdl --- a/Parser/Python.asdl Fri Jan 13 22:12:37 2012 +0100 +++ b/Parser/Python.asdl Sat Jan 14 01:15:29 2012 -0500 @@ -54,17 +54,18 @@ module Python | IfExp(expr test, expr body, expr orelse) | Dict(expr* keys, expr* values) | Set(expr* elts) | ListComp(expr elt, comprehension* generators) | SetComp(expr elt, comprehension* generators) | DictComp(expr key, expr value, comprehension* generators) | GeneratorExp(expr elt, comprehension* generators) -- the grammar constrains where yield expressions can occur - | Yield(int is_from, expr? value) + | Yield(expr? value) + | YieldFrom(expr? value) -- need sequences for compare to distinguish between -- x < 4 < 3 and (x < 4) < 3 | Compare(expr left, cmpop* ops, expr* comparators) | Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs) | Num(object n) -- a number as a PyObject. | Str(string s) -- need to specify raw, unicode, etc? | Bytes(bytes s) diff -r 8e538ed41766 Python/Python-ast.c --- a/Python/Python-ast.c Fri Jan 13 22:12:37 2012 +0100 +++ b/Python/Python-ast.c Sat Jan 14 01:15:29 2012 -0500 @@ -226,19 +226,21 @@ static char *DictComp_fields[]={ "generators", }; static PyTypeObject *GeneratorExp_type; static char *GeneratorExp_fields[]={ "elt", "generators", }; static PyTypeObject *Yield_type; -_Py_IDENTIFIER(is_from); static char *Yield_fields[]={ - "is_from", + "value", +}; +static PyTypeObject *YieldFrom_type; +static char *YieldFrom_fields[]={ "value", }; static PyTypeObject *Compare_type; _Py_IDENTIFIER(ops); _Py_IDENTIFIER(comparators); static char *Compare_fields[]={ "left", "ops", @@ -807,18 +809,20 @@ static int init_types(void) if (!ListComp_type) return 0; SetComp_type = make_type("SetComp", expr_type, SetComp_fields, 2); if (!SetComp_type) return 0; DictComp_type = make_type("DictComp", expr_type, DictComp_fields, 3); if (!DictComp_type) return 0; GeneratorExp_type = make_type("GeneratorExp", expr_type, GeneratorExp_fields, 2); if (!GeneratorExp_type) return 0; - Yield_type = make_type("Yield", expr_type, Yield_fields, 2); + Yield_type = make_type("Yield", expr_type, Yield_fields, 1); if (!Yield_type) return 0; + YieldFrom_type = make_type("YieldFrom", expr_type, YieldFrom_fields, 1); + if (!YieldFrom_type) return 0; Compare_type = make_type("Compare", expr_type, Compare_fields, 3); if (!Compare_type) return 0; Call_type = make_type("Call", expr_type, Call_fields, 5); if (!Call_type) return 0; Num_type = make_type("Num", expr_type, Num_fields, 1); if (!Num_type) return 0; Str_type = make_type("Str", expr_type, Str_fields, 1); if (!Str_type) return 0; @@ -1744,31 +1748,44 @@ GeneratorExp(expr_ty elt, asdl_seq * gen p->v.GeneratorExp.elt = elt; p->v.GeneratorExp.generators = generators; p->lineno = lineno; p->col_offset = col_offset; return p; } expr_ty -Yield(int is_from, expr_ty value, int lineno, int col_offset, PyArena *arena) +Yield(expr_ty value, int lineno, int col_offset, PyArena *arena) { expr_ty p; p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; p->kind = Yield_kind; - p->v.Yield.is_from = is_from; p->v.Yield.value = value; p->lineno = lineno; p->col_offset = col_offset; return p; } expr_ty +YieldFrom(expr_ty value, int lineno, int col_offset, PyArena *arena) +{ + expr_ty p; + p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = YieldFrom_kind; + p->v.YieldFrom.value = value; + p->lineno = lineno; + p->col_offset = col_offset; + return p; +} + +expr_ty Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators, int lineno, int col_offset, PyArena *arena) { expr_ty p; if (!left) { PyErr_SetString(PyExc_ValueError, "field left is required for Compare"); return NULL; @@ -2793,22 +2810,26 @@ ast2obj_expr(void* _o) if (!value) goto failed; if (_PyObject_SetAttrId(result, &PyId_generators, value) == -1) goto failed; Py_DECREF(value); break; case Yield_kind: result = PyType_GenericNew(Yield_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_int(o->v.Yield.is_from); + value = ast2obj_expr(o->v.Yield.value); if (!value) goto failed; - if (_PyObject_SetAttrId(result, &PyId_is_from, value) == -1) + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.Yield.value); + break; + case YieldFrom_kind: + result = PyType_GenericNew(YieldFrom_type, NULL, NULL); + if (!result) goto failed; + value = ast2obj_expr(o->v.YieldFrom.value); if (!value) goto failed; if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) goto failed; Py_DECREF(value); break; case Compare_kind: result = PyType_GenericNew(Compare_type, NULL, NULL); if (!result) goto failed; @@ -5340,43 +5361,52 @@ obj2ast_expr(PyObject* obj, expr_ty* out if (*out == NULL) goto failed; return 0; } isinstance = PyObject_IsInstance(obj, (PyObject*)Yield_type); if (isinstance == -1) { return 1; } if (isinstance) { - int is_from; expr_ty value; - if (_PyObject_HasAttrId(obj, &PyId_is_from)) { - int res; - tmp = _PyObject_GetAttrId(obj, &PyId_is_from); - if (tmp == NULL) goto failed; - res = obj2ast_int(tmp, &is_from, arena); - if (res != 0) goto failed; - Py_XDECREF(tmp); - tmp = NULL; - } else { - PyErr_SetString(PyExc_TypeError, "required field \"is_from\" missing from Yield"); - return 1; - } if (_PyObject_HasAttrId(obj, &PyId_value)) { int res; tmp = _PyObject_GetAttrId(obj, &PyId_value); if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; Py_XDECREF(tmp); tmp = NULL; } else { value = NULL; } - *out = Yield(is_from, value, lineno, col_offset, arena); + *out = Yield(value, lineno, col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } + isinstance = PyObject_IsInstance(obj, (PyObject*)YieldFrom_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + expr_ty value; + + if (_PyObject_HasAttrId(obj, &PyId_value)) { + int res; + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &value, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + value = NULL; + } + *out = YieldFrom(value, lineno, col_offset, arena); if (*out == NULL) goto failed; return 0; } isinstance = PyObject_IsInstance(obj, (PyObject*)Compare_type); if (isinstance == -1) { return 1; } if (isinstance) { @@ -6923,16 +6953,18 @@ PyInit__ast(void) if (PyDict_SetItemString(d, "SetComp", (PyObject*)SetComp_type) < 0) return NULL; if (PyDict_SetItemString(d, "DictComp", (PyObject*)DictComp_type) < 0) return NULL; if (PyDict_SetItemString(d, "GeneratorExp", (PyObject*)GeneratorExp_type) < 0) return NULL; if (PyDict_SetItemString(d, "Yield", (PyObject*)Yield_type) < 0) return NULL; + if (PyDict_SetItemString(d, "YieldFrom", (PyObject*)YieldFrom_type) < + 0) return NULL; if (PyDict_SetItemString(d, "Compare", (PyObject*)Compare_type) < 0) return NULL; if (PyDict_SetItemString(d, "Call", (PyObject*)Call_type) < 0) return NULL; if (PyDict_SetItemString(d, "Num", (PyObject*)Num_type) < 0) return NULL; if (PyDict_SetItemString(d, "Str", (PyObject*)Str_type) < 0) return NULL; diff -r 8e538ed41766 Python/ast.c --- a/Python/ast.c Fri Jan 13 22:12:37 2012 +0100 +++ b/Python/ast.c Sat Jan 14 01:15:29 2012 -0500 @@ -218,16 +218,19 @@ validate_expr(expr_ty exp, expr_context_ COMP(GeneratorExp) #undef COMP case DictComp_kind: return validate_comprehension(exp->v.DictComp.generators) && validate_expr(exp->v.DictComp.key, Load) && validate_expr(exp->v.DictComp.value, Load); case Yield_kind: return !exp->v.Yield.value || validate_expr(exp->v.Yield.value, Load); + case YieldFrom_kind: + return !exp->v.YieldFrom.value || + validate_expr(exp->v.YieldFrom.value, Load); case Compare_kind: if (!asdl_seq_LEN(exp->v.Compare.comparators)) { PyErr_SetString(PyExc_ValueError, "Compare with no comparators"); return 0; } if (asdl_seq_LEN(exp->v.Compare.comparators) != asdl_seq_LEN(exp->v.Compare.ops)) { PyErr_SetString(PyExc_ValueError, "Compare has a different number " @@ -937,16 +940,17 @@ set_context(struct compiling *c, expr_ty case BinOp_kind: case UnaryOp_kind: expr_name = "operator"; break; case GeneratorExp_kind: expr_name = "generator expression"; break; case Yield_kind: + case YieldFrom_kind: expr_name = "yield expression"; break; case ListComp_kind: expr_name = "list comprehension"; break; case SetComp_kind: expr_name = "set comprehension"; break; @@ -2381,17 +2385,19 @@ ast_for_expr(struct compiling *c, const is_from = 1; exp = ast_for_expr(c, en); } else exp = ast_for_testlist(c, en); if (!exp) return NULL; } - return Yield(is_from, exp, LINENO(n), n->n_col_offset, c->c_arena); + if (is_from) + return YieldFrom(exp, LINENO(n), n->n_col_offset, c->c_arena); + return Yield(exp, LINENO(n), n->n_col_offset, c->c_arena); } case factor: if (NCH(n) == 1) { n = CHILD(n, 0); goto loop; } return ast_for_factor(c, n); case power: diff -r 8e538ed41766 Python/compile.c --- a/Python/compile.c Fri Jan 13 22:12:37 2012 +0100 +++ b/Python/compile.c Sat Jan 14 01:15:29 2012 -0500 @@ -3306,31 +3306,35 @@ compiler_visit_expr(struct compiler *c, return compiler_genexp(c, e); case ListComp_kind: return compiler_listcomp(c, e); case SetComp_kind: return compiler_setcomp(c, e); case DictComp_kind: return compiler_dictcomp(c, e); case Yield_kind: + case YieldFrom_kind: { + expr_ty value; if (c->u->u_ste->ste_type != FunctionBlock) return compiler_error(c, "'yield' outside function"); - if (e->v.Yield.value) { - VISIT(c, expr, e->v.Yield.value); + value = (e->kind == YieldFrom_kind) ? e->v.YieldFrom.value : e->v.Yield.value; + if (value) { + VISIT(c, expr, value); } else { ADDOP_O(c, LOAD_CONST, Py_None, consts); } - if (e->v.Yield.is_from) { + if (e->kind == YieldFrom_kind) { ADDOP(c, YIELD_FROM); } else { ADDOP(c, YIELD_VALUE); } break; + } case Compare_kind: return compiler_compare(c, e); case Call_kind: return compiler_call(c, e); case Num_kind: ADDOP_O(c, LOAD_CONST, e->v.Num.n, consts); break; case Str_kind: diff -r 8e538ed41766 Python/symtable.c --- a/Python/symtable.c Fri Jan 13 22:12:37 2012 +0100 +++ b/Python/symtable.c Sat Jan 14 01:15:29 2012 -0500 @@ -1325,20 +1325,24 @@ symtable_visit_expr(struct symtable *st, if (!symtable_visit_setcomp(st, e)) return 0; break; case DictComp_kind: if (!symtable_visit_dictcomp(st, e)) return 0; break; case Yield_kind: - if (e->v.Yield.value) - VISIT(st, expr, e->v.Yield.value); + case YieldFrom_kind: { + expr_ty value; + value = (e->kind == YieldFrom_kind) ? e->v.YieldFrom.value : e->v.Yield.value; + if (value) + VISIT(st, expr, value); st->st_cur->ste_generator = 1; break; + } case Compare_kind: VISIT(st, expr, e->v.Compare.left); VISIT_SEQ(st, expr, e->v.Compare.comparators); break; case Call_kind: VISIT(st, expr, e->v.Call.func); VISIT_SEQ(st, expr, e->v.Call.args); VISIT_SEQ(st, keyword, e->v.Call.keywords);