diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -55,6 +55,22 @@ ============= +PEP 380: Syntax for Delegating to a Subgenerator +================================================ + +PEP 380 adds the ``yield from`` expression, allowing a generator to delegate +part of its operations to another generator. This allows a section of code +containing 'yield' to be factored out and placed in another generator. +Additionally, the subgenerator is allowed to return with a value, and the +value is made available to the delegating generator. + +While designed primarily for use in delegating to a subgenerator, the ``yield +from`` expression actually allows delegation to arbitrary subiterators. + +(Implementation by Greg Ewing, integrated into 3.3 by Renaud Blanch, Ryan +Kelly and Nick Coghlan) + + Other Language Changes ====================== diff --git a/Grammar/Grammar b/Grammar/Grammar --- a/Grammar/Grammar +++ b/Grammar/Grammar @@ -119,7 +119,7 @@ |'**' test) # The reason that keywords are test nodes instead of NAME is that using NAME # results in an ambiguity. ast.c makes sure it's a NAME. -argument: test [comp_for] | test '=' test # Really [keyword '='] test +argument: (test | yield_from) [comp_for] | test '=' (test | yield_from) # Really [keyword '='] test comp_iter: comp_for | comp_if comp_for: 'for' exprlist 'in' or_test [comp_iter] comp_if: 'if' test_nocond [comp_iter] @@ -127,4 +127,6 @@ # not used in grammar, but may appear in "node" passed from Parser to Compiler encoding_decl: NAME -yield_expr: 'yield' [testlist] +yield_expr: 'yield' [yield_arg] +yield_arg: 'from' test | testlist +yield_from: 'yield' 'from' test diff --git a/Include/Python-ast.h b/Include/Python-ast.h --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -245,6 +245,7 @@ } GeneratorExp; struct { + int is_from; expr_ty value; } Yield; @@ -487,8 +488,9 @@ #define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4) expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena *arena); -#define Yield(a0, a1, a2, a3) _Py_Yield(a0, a1, a2, a3) -expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, PyArena *arena); +#define Yield(a0, a1, a2, a3, a4) _Py_Yield(a0, a1, a2, a3, a4) +expr_ty _Py_Yield(int is_from, expr_ty value, int lineno, int col_offset, + PyArena *arena); #define Compare(a0, a1, a2, a3, a4, a5) _Py_Compare(a0, a1, a2, a3, a4, a5) expr_ty _Py_Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators, int lineno, int col_offset, PyArena *arena); diff --git a/Include/frameobject.h b/Include/frameobject.h --- a/Include/frameobject.h +++ b/Include/frameobject.h @@ -27,6 +27,7 @@ to the current stack top. */ PyObject **f_stacktop; PyObject *f_trace; /* Trace function */ + PyObject *f_yieldfrom; /* Iterator being delegated to by yield from */ /* In a generator, we need to be able to swap between the exception state inside the generator and the exception state of the calling diff --git a/Include/genobject.h b/Include/genobject.h --- a/Include/genobject.h +++ b/Include/genobject.h @@ -34,6 +34,7 @@ PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *); PyAPI_FUNC(int) PyGen_NeedsFinalizing(PyGenObject *); +PyAPI_FUNC(int) PyGen_FetchStopIterationValue(PyObject **); #ifdef __cplusplus } diff --git a/Include/graminit.h b/Include/graminit.h --- a/Include/graminit.h +++ b/Include/graminit.h @@ -81,3 +81,5 @@ #define comp_if 334 #define encoding_decl 335 #define yield_expr 336 +#define yield_arg 337 +#define yield_from 338 diff --git a/Include/opcode.h b/Include/opcode.h --- a/Include/opcode.h +++ b/Include/opcode.h @@ -52,6 +52,7 @@ #define STORE_LOCALS 69 #define PRINT_EXPR 70 #define LOAD_BUILD_CLASS 71 +#define YIELD_FROM 72 #define INPLACE_LSHIFT 75 #define INPLACE_RSHIFT 76 diff --git a/Include/pyerrors.h b/Include/pyerrors.h --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -47,6 +47,11 @@ PyObject *filename; } PyEnvironmentErrorObject; +typedef struct { + PyException_HEAD + PyObject *value; +} PyStopIterationObject; + #ifdef MS_WINDOWS typedef struct { PyException_HEAD @@ -356,6 +361,8 @@ const char *reason /* UTF-8 encoded string */ ); +/* create a StopIteration exception with the given value */ +PyAPI_FUNC(PyObject *) PyStopIteration_New(PyObject *); /* These APIs aren't really part of the error implementation, but often needed to format error messages; the native C lib APIs are diff --git a/Lib/opcode.py b/Lib/opcode.py --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -87,6 +87,7 @@ def_op('PRINT_EXPR', 70) def_op('LOAD_BUILD_CLASS', 71) +def_op('YIELD_FROM', 72) def_op('INPLACE_LSHIFT', 75) def_op('INPLACE_RSHIFT', 76) diff --git a/Lib/test/bytecode_helper.py b/Lib/test/bytecode_helper.py --- a/Lib/test/bytecode_helper.py +++ b/Lib/test/bytecode_helper.py @@ -14,6 +14,11 @@ dis.dis(co, file=s) return s.getvalue() + def assertBytecodeMatches(self, x, expected, attrs=("opname", "argval")): + for opinfo, values in zip(dis.get_opinfo(x), expected): + for attr, expected_value in zip(attrs, values): + self.assertEqual(getattr(opinfo, attr), expected_value) + def assertBytecodeExactlyMatches(self, x, expected, *, line_offset=0): """Throws AssertionError if any discrepancy is found in bytecode""" actual = dis.get_opinfo(x) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -802,7 +802,8 @@ self._check_comprehension(factory) def test_yield(self): - self.expr(ast.Yield(ast.Name("x", ast.Store())), "must have Load") + self.expr(ast.Yield(0, ast.Name("x", ast.Store())), "must have Load") + self.expr(ast.Yield(1, ast.Name("x", ast.Store())), "must have Load") def test_compare(self): left = ast.Name("x", ast.Load()) diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -728,29 +728,6 @@ syntax_tests = """ ->>> def f(): -... return 22 -... yield 1 -Traceback (most recent call last): - .. -SyntaxError: 'return' with argument inside generator - ->>> def f(): -... yield 1 -... return 22 -Traceback (most recent call last): - .. -SyntaxError: 'return' with argument inside generator - -"return None" is not the same as "return" in a generator: - ->>> def f(): -... yield 1 -... return None -Traceback (most recent call last): - .. -SyntaxError: 'return' with argument inside generator - These are fine: >>> def f(): @@ -866,20 +843,6 @@ >>> type(f()) - ->>> def f(): -... if 0: -... lambda x: x # shouldn't trigger here -... return # or here -... def f(i): -... return 2*i # or here -... if 0: -... return 3 # but *this* sucks (line 8) -... if 0: -... yield 2 # because it's a generator (line 10) -Traceback (most recent call last): -SyntaxError: 'return' with argument inside generator - This one caused a crash (see SF bug 567538): >>> def f(): @@ -1566,11 +1529,6 @@ ... SyntaxError: 'yield' outside function ->>> def f(): return lambda x=(yield): 1 -Traceback (most recent call last): - ... -SyntaxError: 'return' with argument inside generator - >>> def f(): x = yield = y Traceback (most recent call last): ... diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -459,6 +459,7 @@ def test_yield(self): check_syntax_error(self, "class foo:yield 1") + check_syntax_error(self, "yield from") def test_raise(self): # 'raise' test [',' test] diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -50,6 +50,10 @@ self.check_suite("def f(): (yield 1)*2") self.check_suite("def f(): return; yield 1") self.check_suite("def f(): yield 1; return") + self.check_suite("def f(): yield from 1") + self.check_suite("def f(): x = yield from 1") + self.check_suite("def f(): f(yield from 1)") + self.check_suite("def f(): yield 1; return 1") self.check_suite("def f():\n" " for x in range(30):\n" " yield x\n") diff --git a/Lib/test/test_pep380.py b/Lib/test/test_pep380.py new file mode 100644 --- /dev/null +++ b/Lib/test/test_pep380.py @@ -0,0 +1,895 @@ +# -*- coding: utf-8 -*- + +""" +Test suite for PEP 380 implementation + +adapted from original tests written by Greg Ewing +see +""" + +import unittest +import io +import sys +import traceback +import parser + +from test.bytecode_helper import BytecodeTestCase + + +class PEP380Test(unittest.TestCase): + pass + + +class TestPEP380Parsing(unittest.TestCase): + """ + Test grammar. + """ + + def assertCodeParsesAsExpected(self,code,expected): + st1 = parser.suite(code) + tup1 = st1.totuple() + self.assertEqual(tup1,expected) + st2 = parser.sequence2st(tup1) + tup2 = st2.totuple() + self.assertEqual(tup1,tup2) + + def test_parser_module(self): + """ + Test parser module + """ + self.assertCodeParsesAsExpected("yield from 1", +(257, (268, (269, (270, (276, (280, (336, (1, 'yield'), (337, (1, 'from'), (302, (306, (307, (308, (309, (312, (313, (314, (315, (316, (317, (318, (319, (320, (2, '1')))))))))))))))))))), (4, ''))), (4, ''), (0, '')) + ) + self.assertCodeParsesAsExpected("f(yield from 1)", +(257, (268, (269, (270, (271, (272, (302, (306, (307, (308, (309, (312, (313, (314, (315, (316, (317, (318, (319, (320, (1, 'f')), (322, (7, '('), (330, (331, (338, (1, 'yield'), (1, 'from'), (302, (306, (307, (308, (309, (312, (313, (314, (315, (316, (317, (318, (319, (320, (2, '1')))))))))))))))))), (8, ')')))))))))))))))))), (4, ''))), (4, ''), (0, '')) + ) + + +class TestPEP380Bytecode(BytecodeTestCase): + """ + Test code generation + """ + + def test_ordinary_yield(self): + def g(): + yield + yield 42 + self.assertBytecodeMatches(g, ( + ('LOAD_CONST', None), + ('YIELD_VALUE', None), + ('POP_TOP', None), + ('LOAD_CONST', 42), + ('YIELD_VALUE', None), + ('POP_TOP', None), + ('LOAD_CONST', None), + ('RETURN_VALUE', None), + )) + + def test_bare_yield_from(self): + def g(): + yield from x + self.assertBytecodeMatches(g, ( + ('LOAD_GLOBAL', 'x'), + ('YIELD_FROM', None), + ('POP_TOP', None), + ('LOAD_CONST', None), + ('RETURN_VALUE', None), + )) + + def test_yield_from_assigned_to_local(self): + def g(): + x = yield from x + self.assertBytecodeMatches(g, ( + ('LOAD_FAST', 'x'), + ('YIELD_FROM', None), + ('STORE_FAST', 'x'), + ('LOAD_CONST', None), + ('RETURN_VALUE', None), + )) + + def test_parsing_yield_from_as_function_argument(self): + def g(): + f(yield from x) + self.assertBytecodeMatches(g, ( + ('LOAD_GLOBAL', 'f'), + ('LOAD_GLOBAL', 'x'), + ('YIELD_FROM', None), + ('CALL_FUNCTION', 1), + ('POP_TOP', None), + ('LOAD_CONST', None), + ('RETURN_VALUE', None), + )) + + +class TestPEP380Operation(unittest.TestCase): + """ + Test semantics. + """ + + def test_delegation_of_initial_next_to_subgenerator(self): + """ + Test delegation of initial next() call to subgenerator + """ + trace = [] + def g1(): + trace.append("Starting g1") + yield from g2() + trace.append("Finishing g1") + def g2(): + trace.append("Starting g2") + yield 42 + trace.append("Finishing g2") + for x in g1(): + trace.append("Yielded %s" % (x,)) + self.assertEqual(trace,[ + "Starting g1", + "Starting g2", + "Yielded 42", + "Finishing g2", + "Finishing g1", + ]) + + def test_raising_exception_in_initial_next_call(self): + """ + Test raising exception in initial next() call + """ + trace = [] + def g1(): + try: + trace.append("Starting g1") + yield from g2() + finally: + trace.append("Finishing g1") + def g2(): + try: + trace.append("Starting g2") + raise ValueError("spanish inquisition occurred") + finally: + trace.append("Finishing g2") + try: + for x in g1(): + trace.append("Yielded %s" % (x,)) + except ValueError as e: + self.assertEqual(e.args[0], "spanish inquisition occurred") + else: + self.fail("subgenerator failed to raise ValueError") + self.assertEqual(trace,[ + "Starting g1", + "Starting g2", + "Finishing g2", + "Finishing g1", + ]) + + def test_delegation_of_next_call_to_subgenerator(self): + """ + Test delegation of next() call to subgenerator + """ + trace = [] + def g1(): + trace.append("Starting g1") + yield "g1 ham" + yield from g2() + yield "g1 eggs" + trace.append("Finishing g1") + def g2(): + trace.append("Starting g2") + yield "g2 spam" + yield "g2 more spam" + trace.append("Finishing g2") + for x in g1(): + trace.append("Yielded %s" % (x,)) + self.assertEqual(trace,[ + "Starting g1", + "Yielded g1 ham", + "Starting g2", + "Yielded g2 spam", + "Yielded g2 more spam", + "Finishing g2", + "Yielded g1 eggs", + "Finishing g1", + ]) + + def test_raising_exception_in_delegated_next_call(self): + """ + Test raising exception in delegated next() call + """ + trace = [] + def g1(): + try: + trace.append("Starting g1") + yield "g1 ham" + yield from g2() + yield "g1 eggs" + finally: + trace.append("Finishing g1") + def g2(): + try: + trace.append("Starting g2") + yield "g2 spam" + raise ValueError("hovercraft is full of eels") + yield "g2 more spam" + finally: + trace.append("Finishing g2") + try: + for x in g1(): + trace.append("Yielded %s" % (x,)) + except ValueError as e: + self.assertEqual(e.args[0], "hovercraft is full of eels") + else: + self.fail("subgenerator failed to raise ValueError") + self.assertEqual(trace,[ + "Starting g1", + "Yielded g1 ham", + "Starting g2", + "Yielded g2 spam", + "Finishing g2", + "Finishing g1", + ]) + + def test_delegation_of_send(self): + """ + Test delegation of send() + """ + trace = [] + def g1(): + trace.append("Starting g1") + x = yield "g1 ham" + trace.append("g1 received %s" % (x,)) + yield from g2() + x = yield "g1 eggs" + trace.append("g1 received %s" % (x,)) + trace.append("Finishing g1") + def g2(): + trace.append("Starting g2") + x = yield "g2 spam" + trace.append("g2 received %s" % (x,)) + x = yield "g2 more spam" + trace.append("g2 received %s" % (x,)) + trace.append("Finishing g2") + g = g1() + y = next(g) + x = 1 + try: + while 1: + y = g.send(x) + trace.append("Yielded %s" % (y,)) + x += 1 + except StopIteration: + pass + self.assertEqual(trace,[ + "Starting g1", + "g1 received 1", + "Starting g2", + "Yielded g2 spam", + "g2 received 2", + "Yielded g2 more spam", + "g2 received 3", + "Finishing g2", + "Yielded g1 eggs", + "g1 received 4", + "Finishing g1", + ]) + + def test_handling_exception_while_delegating_send(self): + """ + Test handling exception while delegating 'send' + """ + trace = [] + def g1(): + trace.append("Starting g1") + x = yield "g1 ham" + trace.append("g1 received %s" % (x,)) + yield from g2() + x = yield "g1 eggs" + trace.append("g1 received %s" % (x,)) + trace.append("Finishing g1") + def g2(): + trace.append("Starting g2") + x = yield "g2 spam" + trace.append("g2 received %s" % (x,)) + raise ValueError("hovercraft is full of eels") + x = yield "g2 more spam" + trace.append("g2 received %s" % (x,)) + trace.append("Finishing g2") + def run(): + g = g1() + y = next(g) + x = 1 + try: + while 1: + y = g.send(x) + trace.append("Yielded %s" % (y,)) + x += 1 + except StopIteration: + trace.append("StopIteration") + self.assertRaises(ValueError,run) + self.assertEqual(trace,[ + "Starting g1", + "g1 received 1", + "Starting g2", + "Yielded g2 spam", + "g2 received 2", + ]) + + def test_delegating_close(self): + """ + Test delegating 'close' + """ + trace = [] + def g1(): + try: + trace.append("Starting g1") + yield "g1 ham" + yield from g2() + yield "g1 eggs" + finally: + trace.append("Finishing g1") + def g2(): + try: + trace.append("Starting g2") + yield "g2 spam" + yield "g2 more spam" + finally: + trace.append("Finishing g2") + g = g1() + for i in range(2): + x = next(g) + trace.append("Yielded %s" % (x,)) + g.close() + self.assertEqual(trace,[ + "Starting g1", + "Yielded g1 ham", + "Starting g2", + "Yielded g2 spam", + "Finishing g2", + "Finishing g1" + ]) + + def test_handing_exception_while_delegating_close(self): + """ + Test handling exception while delegating 'close' + """ + trace = [] + def g1(): + try: + trace.append("Starting g1") + yield "g1 ham" + yield from g2() + yield "g1 eggs" + finally: + trace.append("Finishing g1") + def g2(): + try: + trace.append("Starting g2") + yield "g2 spam" + yield "g2 more spam" + finally: + trace.append("Finishing g2") + raise ValueError("nybbles have exploded with delight") + try: + g = g1() + for i in range(2): + x = next(g) + trace.append("Yielded %s" % (x,)) + g.close() + except ValueError as e: + self.assertEqual(e.args[0], "nybbles have exploded with delight") + self.assertIsInstance(e.__context__, GeneratorExit) + else: + self.fail("subgenerator failed to raise ValueError") + self.assertEqual(trace,[ + "Starting g1", + "Yielded g1 ham", + "Starting g2", + "Yielded g2 spam", + "Finishing g2", + "Finishing g1", + ]) + + def test_delegating_throw(self): + """ + Test delegating 'throw' + """ + trace = [] + def g1(): + try: + trace.append("Starting g1") + yield "g1 ham" + yield from g2() + yield "g1 eggs" + finally: + trace.append("Finishing g1") + def g2(): + try: + trace.append("Starting g2") + yield "g2 spam" + yield "g2 more spam" + finally: + trace.append("Finishing g2") + try: + g = g1() + for i in range(2): + x = next(g) + trace.append("Yielded %s" % (x,)) + e = ValueError("tomato ejected") + g.throw(e) + except ValueError as e: + self.assertEqual(e.args[0], "tomato ejected") + else: + self.fail("subgenerator failed to raise ValueError") + self.assertEqual(trace,[ + "Starting g1", + "Yielded g1 ham", + "Starting g2", + "Yielded g2 spam", + "Finishing g2", + "Finishing g1", + ]) + + def test_value_attribute_of_StopIteration_exception(self): + """ + Test 'value' attribute of StopIteration exception + """ + trace = [] + def pex(e): + trace.append("%s: %s" % (e.__class__.__name__, e)) + trace.append("value = %s" % (e.value,)) + e = StopIteration() + pex(e) + e = StopIteration("spam") + pex(e) + e.value = "eggs" + pex(e) + self.assertEqual(trace,[ + "StopIteration: ", + "value = None", + "StopIteration: spam", + "value = spam", + "StopIteration: spam", + "value = eggs", + ]) + + + def test_generator_return_value(self): + """ + Test generator return value + """ + trace = [] + def g1(): + trace.append("Starting g1") + yield "g1 ham" + ret = yield from g2() + trace.append("g2 returned %s" % (ret,)) + ret = yield from g2(42) + trace.append("g2 returned %s" % (ret,)) + yield "g1 eggs" + trace.append("Finishing g1") + def g2(v = None): + trace.append("Starting g2") + yield "g2 spam" + yield "g2 more spam" + trace.append("Finishing g2") + if v: + return v + for x in g1(): + trace.append("Yielded %s" % (x,)) + self.assertEqual(trace,[ + "Starting g1", + "Yielded g1 ham", + "Starting g2", + "Yielded g2 spam", + "Yielded g2 more spam", + "Finishing g2", + "g2 returned None", + "Starting g2", + "Yielded g2 spam", + "Yielded g2 more spam", + "Finishing g2", + "g2 returned 42", + "Yielded g1 eggs", + "Finishing g1", + ]) + + def test_delegation_of_next_to_non_generator(self): + """ + Test delegation of next() to non-generator + """ + trace = [] + def g(): + yield from range(3) + for x in g(): + trace.append("Yielded %s" % (x,)) + self.assertEqual(trace,[ + "Yielded 0", + "Yielded 1", + "Yielded 2", + ]) + + + def test_conversion_of_sendNone_to_next(self): + """ + Test conversion of send(None) to next() + """ + trace = [] + def g(): + yield from range(3) + gi = g() + for x in range(3): + y = gi.send(None) + trace.append("Yielded: %s" % (y,)) + self.assertEqual(trace,[ + "Yielded: 0", + "Yielded: 1", + "Yielded: 2", + ]) + + def test_delegation_of_close_to_non_generator(self): + """ + Test delegation of close() to non-generator + """ + trace = [] + def g(): + try: + trace.append("starting g") + yield from range(3) + trace.append("g should not be here") + finally: + trace.append("finishing g") + gi = g() + next(gi) + gi.close() + self.assertEqual(trace,[ + "starting g", + "finishing g", + ]) + + def test_delegating_throw_to_non_generator(self): + """ + Test delegating 'throw' to non-generator + """ + trace = [] + def g(): + try: + trace.append("Starting g") + yield from range(10) + finally: + trace.append("Finishing g") + try: + gi = g() + for i in range(5): + x = next(gi) + trace.append("Yielded %s" % (x,)) + e = ValueError("tomato ejected") + gi.throw(e) + except ValueError as e: + self.assertEqual(e.args[0],"tomato ejected") + else: + self.fail("subgenerator failed to raise ValueError") + self.assertEqual(trace,[ + "Starting g", + "Yielded 0", + "Yielded 1", + "Yielded 2", + "Yielded 3", + "Yielded 4", + "Finishing g", + ]) + + def test_attempting_to_send_to_non_generator(self): + """ + Test attempting to send to non-generator + """ + trace = [] + def g(): + try: + trace.append("starting g") + yield from range(3) + trace.append("g should not be here") + finally: + trace.append("finishing g") + try: + gi = g() + next(gi) + for x in range(3): + y = gi.send(42) + trace.append("Should not have yielded:", y) + except AttributeError as e: + self.assertEqual(e.args[0],"send") + else: + self.fail("was able to send into non-generator") + self.assertEqual(trace,[ + "starting g", + "finishing g", + ]) + + def test_exception_in_initial_next_call(self): + """ + Test exception in initial next() call + """ + trace = [] + def g1(): + trace.append("g1 about to yield from g2") + yield from g2() + trace.append("g1 should not be here") + def g2(): + yield 1/0 + def run(): + gi = g1() + next(gi) + self.assertRaises(ZeroDivisionError,run) + self.assertEqual(trace,[ + "g1 about to yield from g2" + ]) + + def test_attempted_yield_from_loop(self): + """ + Test attempted yield-from loop + """ + trace = [] + def g1(): + trace.append("g1: starting") + yield "y1" + trace.append("g1: about to yield from g2") + yield from g2() + trace.append("g1 should not be here") + + def g2(): + trace.append("g2: starting") + yield "y2" + trace.append("g2: about to yield from g1") + yield from gi + trace.append("g2 should not be here") + try: + gi = g1() + for y in gi: + trace.append("Yielded: %s" % (y,)) + except ValueError as e: + self.assertEqual(e.args[0],"generator already executing") + else: + self.fail("subgenerator didn't raise ValueError") + self.assertEqual(trace,[ + "g1: starting", + "Yielded: y1", + "g1: about to yield from g2", + "g2: starting", + "Yielded: y2", + "g2: about to yield from g1", + ]) + + def test_returning_value_from_delegated_throw(self): + """ + Test returning value from delegated 'throw' + """ + trace = [] + def g1(): + try: + trace.append("Starting g1") + yield "g1 ham" + yield from g2() + yield "g1 eggs" + finally: + trace.append("Finishing g1") + def g2(): + try: + trace.append("Starting g2") + yield "g2 spam" + yield "g2 more spam" + except LunchError: + trace.append("Caught LunchError in g2") + yield "g2 lunch saved" + yield "g2 yet more spam" + class LunchError(Exception): + pass + g = g1() + for i in range(2): + x = next(g) + trace.append("Yielded %s" % (x,)) + e = LunchError("tomato ejected") + g.throw(e) + for x in g: + trace.append("Yielded %s" % (x,)) + self.assertEqual(trace,[ + "Starting g1", + "Yielded g1 ham", + "Starting g2", + "Yielded g2 spam", + "Caught LunchError in g2", + "Yielded g2 yet more spam", + "Yielded g1 eggs", + "Finishing g1", + ]) + + def test_next_and_return_with_value(self): + """ + Test next and return with value + """ + trace = [] + def f(r): + gi = g(r) + next(gi) + try: + trace.append("f resuming g") + next(gi) + trace.append("f SHOULD NOT BE HERE") + except StopIteration as e: + trace.append("f caught %s" % (repr(e),)) + def g(r): + trace.append("g starting") + yield + trace.append("g returning %s" % (r,)) + return r + f(None) + f(42) + self.assertEqual(trace,[ + "g starting", + "f resuming g", + "g returning None", + "f caught StopIteration()", + "g starting", + "f resuming g", + "g returning 42", + "f caught StopIteration(42,)", + ]) + + def test_send_and_return_with_value(self): + """ + Test send and return with value + """ + trace = [] + def f(r): + gi = g(r) + next(gi) + try: + trace.append("f sending spam to g") + gi.send("spam") + trace.append("f SHOULD NOT BE HERE") + except StopIteration as e: + trace.append("f caught %r" % (e,)) + def g(r): + trace.append("g starting") + x = yield + trace.append("g received %s" % (x,)) + trace.append("g returning %s" % (r,)) + return r + f(None) + f(42) + self.assertEqual(trace,[ + "g starting", + "f sending spam to g", + "g received spam", + "g returning None", + "f caught StopIteration()", + "g starting", + "f sending spam to g", + "g received spam", + "g returning 42", + "f caught StopIteration(42,)", + ]) + + def test_catching_exception_from_subgen_and_returning(self): + """ + Test catching an exception thrown into a + subgenerator and returning a value + """ + trace = [] + def inner(): + try: + yield 1 + except ValueError: + trace.append("inner caught ValueError") + return 2 + + def outer(): + v = yield from inner() + trace.append("inner returned %r to outer" % v) + yield v + g = outer() + trace.append(next(g)) + trace.append(g.throw(ValueError)) + self.assertEqual(trace,[ + 1, + "inner caught ValueError", + "inner returned 2 to outer", + 2, + ]) + + def test_throwing_GeneratorExit_into_subgen_that_returns(self): + """ + Test throwing GeneratorExit into a subgenerator that + catches it and returns normally. + """ + trace = [] + def f(): + try: + trace.append("Enter f") + yield + trace.append("Exit f") + except GeneratorExit: + return + def g(): + trace.append("Enter g") + yield from f() + trace.append("Exit g") + try: + gi = g() + next(gi) + gi.throw(GeneratorExit) + except GeneratorExit: + pass + else: + self.fail("subgenerator failed to raise GeneratorExit") + self.assertEqual(trace,[ + "Enter g", + "Enter f", + ]) + + def test_throwing_GeneratorExit_into_subgenerator_that_yields(self): + """ + Test throwing GeneratorExit into a subgenerator that + catches it and yields. + """ + trace = [] + def f(): + try: + trace.append("Enter f") + yield + trace.append("Exit f") + except GeneratorExit: + yield + def g(): + trace.append("Enter g") + yield from f() + trace.append("Exit g") + try: + gi = g() + next(gi) + gi.throw(GeneratorExit) + except RuntimeError as e: + self.assertEqual(e.args[0], "generator ignored GeneratorExit") + else: + self.fail("subgenerator failed to raise GeneratorExit") + self.assertEqual(trace,[ + "Enter g", + "Enter f", + ]) + + def test_throwing_GeneratorExit_into_subgen_that_raises(self): + """ + Test throwing GeneratorExit into a subgenerator that + catches it and raises a different exception. + """ + trace = [] + def f(): + try: + trace.append("Enter f") + yield + trace.append("Exit f") + except GeneratorExit: + raise ValueError("Vorpal bunny encountered") + def g(): + trace.append("Enter g") + yield from f() + trace.append("Exit g") + try: + gi = g() + next(gi) + gi.throw(GeneratorExit) + except ValueError as e: + self.assertEqual(e.args[0], "Vorpal bunny encountered") + self.assertIsInstance(e.__context__, GeneratorExit) + else: + self.fail("subgenerator failed to raise ValueError") + self.assertEqual(trace,[ + "Enter g", + "Enter f", + ]) + + +def test_main(): + from test import support + test_classes = [TestPEP380Parsing, TestPEP380Bytecode, TestPEP380Operation] + support.run_unittest(*test_classes) + + +if __name__ == '__main__': + test_main() diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -726,7 +726,7 @@ nfrees = len(x.f_code.co_freevars) extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\ ncells + nfrees - 1 - check(x, size(vh + '12P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P')) + check(x, size(vh + '13P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P')) # function def func(): pass check(func, size(h + '11P')) diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -94,6 +94,7 @@ Philippe Biondi Stuart Bishop Roy Bixler +Renaud Blanch Mike Bland Martin Bless Pablo Bleyer diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Core and Builtins ----------------- +- PEP 380, Issue #11682: Add "yield from " to support easy delegation to + subgenerators (initial patch by Greg Ewing, integration into 3.3 by + Renaud Blanch, Ryan Kelly and Nick Coghlan) + - Issue #12802: the Windows error ERROR_DIRECTORY (numbered 267) is now mapped to POSIX errno ENOTDIR (previously EINVAL). diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -976,6 +976,7 @@ VALIDATER(testlist_comp); VALIDATER(yield_expr); VALIDATER(or_test); VALIDATER(test_nocond); VALIDATER(lambdef_nocond); +VALIDATER(yield_arg); VALIDATER(yield_from); #undef VALIDATER @@ -1636,22 +1637,61 @@ } -/* yield_expr: 'yield' [testlist] +/* yield_expr: 'yield' [yield_arg] */ static int validate_yield_expr(node *tree) { int nch = NCH(tree); - int res = (validate_ntype(tree, yield_expr) - && ((nch == 1) || (nch == 2)) - && validate_name(CHILD(tree, 0), "yield")); - - if (res && (nch == 2)) - res = validate_testlist(CHILD(tree, 1)); - - return (res); + if (nch < 1 || nch > 2) + return 0; + if (!validate_ntype(tree, yield_expr)) + return 0; + if (!validate_name(CHILD(tree, 0), "yield")) + return 0; + if (nch == 2) { + if (!validate_yield_arg(CHILD(tree, 1))) + return 0; + } + return 1; } +/* yield_arg: 'from' test | testlist + */ +static int +validate_yield_arg(node *tree) +{ + int nch = NCH(tree); + if (!validate_ntype(tree, yield_arg)) + return 0; + if (nch == 1) { + if (!validate_testlist(CHILD(tree, nch - 1))) + return 0; + } + else if (nch == 2) { + if (!validate_name(CHILD(tree, 0), "from")) + return 0; + if (!validate_test(CHILD(tree, 1))) + return 0; + } + else + return 0; + return 1; +} + +/* yield_from: 'yield' 'from' test + */ +static int +validate_yield_from(node *tree) +{ + int nch = NCH(tree); + int res = (validate_ntype(tree, yield_from) + && (nch == 3) + && validate_name(CHILD(tree, 0), "yield") + && validate_name(CHILD(tree, 1), "from") + && validate_test(CHILD(tree, 2))); + return res; +} /* yield_stmt: yield_expr */ @@ -2665,9 +2705,13 @@ { int nch = NCH(tree); int res = (validate_ntype(tree, argument) - && ((nch == 1) || (nch == 2) || (nch == 3)) - && validate_test(CHILD(tree, 0))); - + && ((nch == 1) || (nch == 2) || (nch == 3))); + if (res) { + if (TYPE(CHILD(tree, 0)) == yield_from) + res = (nch == 1) && validate_yield_from(CHILD(tree, 0)); + else + res = validate_test(CHILD(tree, 0)); + } if (res && (nch == 2)) res = validate_comp_for(CHILD(tree, 1)); else if (res && (nch == 3)) diff --git a/Objects/exceptions.c b/Objects/exceptions.c --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -471,8 +471,65 @@ /* * StopIteration extends Exception */ -SimpleExtendsException(PyExc_Exception, StopIteration, - "Signal the end from iterator.__next__()."); + +//SimpleExtendsException(PyExc_Exception, StopIteration, +// "Signal the end from iterator.__next__()."); + +static PyMemberDef StopIteration_members[] = { + {"value", T_OBJECT, offsetof(PyStopIterationObject, value), 0, + PyDoc_STR("generator return value")}, + {NULL} /* Sentinel */ +}; + +static int +StopIteration_init(PyStopIterationObject *self, PyObject *args, PyObject *kwds) +{ + Py_ssize_t size = PyTuple_GET_SIZE(args); + PyObject *value; + + if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) + return -1; + Py_CLEAR(self->value); + if (size > 0) + value = PyTuple_GET_ITEM(args, 0); + else + value = Py_None; + Py_INCREF(value); + self->value = value; + return 0; +} + +static int +StopIteration_clear(PyStopIterationObject *self) +{ + Py_CLEAR(self->value); + return BaseException_clear((PyBaseExceptionObject *)self); +} + +static void +StopIteration_dealloc(PyStopIterationObject *self) +{ + _PyObject_GC_UNTRACK(self); + StopIteration_clear(self); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static int +StopIteration_traverse(PyStopIterationObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->value); + return BaseException_traverse((PyBaseExceptionObject *)self, visit, arg); +} + +PyObject * +PyStopIteration_New(PyObject *value) +{ + return PyObject_CallFunctionObjArgs(PyExc_StopIteration, value, NULL); +} + +ComplexExtendsException(PyExc_Exception, StopIteration, StopIteration, + StopIteration_dealloc, 0, StopIteration_members, 0, + "Signal the end from iterator.__next__()."); /* diff --git a/Objects/frameobject.c b/Objects/frameobject.c --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -15,11 +15,12 @@ #define OFF(x) offsetof(PyFrameObject, x) static PyMemberDef frame_memberlist[] = { - {"f_back", T_OBJECT, OFF(f_back), READONLY}, - {"f_code", T_OBJECT, OFF(f_code), READONLY}, - {"f_builtins", T_OBJECT, OFF(f_builtins),READONLY}, - {"f_globals", T_OBJECT, OFF(f_globals), READONLY}, - {"f_lasti", T_INT, OFF(f_lasti), READONLY}, + {"f_back", T_OBJECT, OFF(f_back), READONLY}, + {"f_code", T_OBJECT, OFF(f_code), READONLY}, + {"f_builtins", T_OBJECT, OFF(f_builtins), READONLY}, + {"f_globals", T_OBJECT, OFF(f_globals), READONLY}, + {"f_lasti", T_INT, OFF(f_lasti), READONLY}, + {"f_yieldfrom", T_OBJECT, OFF(f_yieldfrom), READONLY}, {NULL} /* Sentinel */ }; @@ -444,6 +445,7 @@ Py_CLEAR(f->f_exc_type); Py_CLEAR(f->f_exc_value); Py_CLEAR(f->f_exc_traceback); + Py_CLEAR(f->f_yieldfrom); co = f->f_code; if (co->co_zombieframe == NULL) @@ -475,6 +477,7 @@ Py_VISIT(f->f_exc_type); Py_VISIT(f->f_exc_value); Py_VISIT(f->f_exc_traceback); + Py_VISIT(f->f_yieldfrom); /* locals */ slots = f->f_code->co_nlocals + PyTuple_GET_SIZE(f->f_code->co_cellvars) + PyTuple_GET_SIZE(f->f_code->co_freevars); @@ -508,6 +511,7 @@ Py_CLEAR(f->f_exc_value); Py_CLEAR(f->f_exc_traceback); Py_CLEAR(f->f_trace); + Py_CLEAR(f->f_yieldfrom); /* locals */ slots = f->f_code->co_nlocals + PyTuple_GET_SIZE(f->f_code->co_cellvars) + PyTuple_GET_SIZE(f->f_code->co_freevars); @@ -711,6 +715,7 @@ f->f_lasti = -1; f->f_lineno = code->co_firstlineno; f->f_iblock = 0; + f->f_yieldfrom = NULL; _PyObject_GC_TRACK(f); return f; diff --git a/Objects/genobject.c b/Objects/genobject.c --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -5,6 +5,9 @@ #include "structmember.h" #include "opcode.h" +static PyObject *gen_close(PyGenObject *gen, PyObject *args); +static void gen_undelegate(PyGenObject *); + static int gen_traverse(PyGenObject *gen, visitproc visit, void *arg) { @@ -90,12 +93,15 @@ /* If the generator just returned (as opposed to yielding), signal * that the generator is exhausted. */ - if (result == Py_None && f->f_stacktop == NULL) { + if (result && f->f_stacktop == NULL) { + if (result == Py_None) + PyErr_SetNone(PyExc_StopIteration); + else { + PyObject *e = PyStopIteration_New(result); + PyErr_SetObject(PyExc_StopIteration, e); + } Py_DECREF(result); result = NULL; - /* Set exception if not called by gen_iternext() */ - if (arg) - PyErr_SetNone(PyExc_StopIteration); } if (!result || f->f_stacktop == NULL) { @@ -125,17 +131,82 @@ static PyObject * gen_send(PyGenObject *gen, PyObject *arg) { - return gen_send_ex(gen, arg, 0); + int exc = 0; + PyObject *ret; + PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL; + Py_INCREF(arg); + if (yf) { + Py_INCREF(yf); + if (PyGen_CheckExact(yf)) + ret = gen_send((PyGenObject *)yf, arg); + else { + if (arg == Py_None) + ret = PyIter_Next(yf); + else + ret = PyObject_CallMethod(yf, "send", "O", arg); + } + if (ret) { + Py_DECREF(yf); + goto done; + } + gen_undelegate(gen); + Py_DECREF(arg); + arg = NULL; + if (PyGen_FetchStopIterationValue(&arg) < 0) + exc = 1; + Py_DECREF(yf); + } + ret = gen_send_ex(gen, arg, exc); +done: + Py_XDECREF(arg); + return ret; } PyDoc_STRVAR(close_doc, "close(arg) -> raise GeneratorExit inside generator."); +/* + * This helper function is used by gen_close and gen_throw to + * close a subiterator being delegated to by yield-from. + */ + +static int +gen_close_iter(PyObject *yf) { + PyObject *retval = NULL; + + if (PyGen_CheckExact(yf)) { + retval = gen_close((PyGenObject *)yf, NULL); + if (!retval) + return -1; + } + else { + PyObject *meth = PyObject_GetAttrString(yf, "close"); + if (meth) { + retval = PyObject_CallFunction(meth, ""); + Py_DECREF(meth); + if (!retval) + return -1; + } + } + Py_XDECREF(retval); + return 0; +} + static PyObject * gen_close(PyGenObject *gen, PyObject *args) { PyObject *retval; - PyErr_SetNone(PyExc_GeneratorExit); + PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL; + int err = 0; + + if (yf) { + Py_INCREF(yf); + err = gen_close_iter(yf); + gen_undelegate(gen); + Py_DECREF(yf); + } + if (err == 0) + PyErr_SetNone(PyExc_GeneratorExit); retval = gen_send_ex(gen, Py_None, 1); if (retval) { Py_DECREF(retval); @@ -226,10 +297,51 @@ PyObject *typ; PyObject *tb = NULL; PyObject *val = NULL; + PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL; if (!PyArg_UnpackTuple(args, "throw", 1, 3, &typ, &val, &tb)) return NULL; + if (yf) { + PyObject *ret; + int err; + Py_INCREF(yf); + if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) { + err = gen_close_iter(yf); + Py_DECREF(yf); + gen_undelegate(gen); + if (err < 0) + return gen_send_ex(gen, Py_None, 1); + goto throw_here; + } + if (PyGen_CheckExact(yf)) + ret = gen_throw((PyGenObject *)yf, args); + else { + PyObject *meth = PyObject_GetAttrString(yf, "throw"); + if (!meth) { + PyErr_Clear(); + Py_DECREF(yf); + gen_undelegate(gen); + goto throw_here; + } + ret = PyObject_CallObject(meth, args); + Py_DECREF(meth); + } + Py_DECREF(yf); + if (!ret) { + PyObject *val; + gen_undelegate(gen); + if (PyGen_FetchStopIterationValue(&val) == 0) { + ret = gen_send_ex(gen, val, 0); + Py_DECREF(val); + } + else + ret = gen_send_ex(gen, Py_None, 1); + } + return ret; + } + +throw_here: /* First, check the traceback argument, replacing None with NULL. */ if (tb == Py_None) @@ -287,9 +399,74 @@ static PyObject * gen_iternext(PyGenObject *gen) { - return gen_send_ex(gen, NULL, 0); + PyObject *val = NULL; + PyObject *ret; + int exc = 0; + PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL; + if (yf) { + Py_INCREF(yf); + /* ceval.c ensures that yf is an iterator */ + ret = yf->ob_type->tp_iternext(yf); + if (ret) { + Py_DECREF(yf); + return ret; + } + gen_undelegate(gen); + if (PyGen_FetchStopIterationValue(&val) < 0) + exc = 1; + Py_DECREF(yf); + } + ret = gen_send_ex(gen, val, exc); + Py_XDECREF(val); + return ret; } +/* + * In certain recursive situations, a generator may lose its frame + * before we get a chance to clear f_yieldfrom, so we use this + * helper function. + */ + +static void +gen_undelegate(PyGenObject *gen) { + if (gen->gi_frame) { + Py_XDECREF(gen->gi_frame->f_yieldfrom); + gen->gi_frame->f_yieldfrom = NULL; + } +} + +/* + * If StopIteration exception is set, fetches its 'value' + * attribute if any, otherwise sets pvalue to None. + * + * Returns 0 if no exception or StopIteration is set. + * If any other exception is set, returns -1 and leaves + * pvalue unchanged. + */ + +int +PyGen_FetchStopIterationValue(PyObject **pvalue) { + PyObject *et, *ev, *tb; + PyObject *value = NULL; + + if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + PyErr_Fetch(&et, &ev, &tb); + Py_XDECREF(et); Py_XDECREF(tb); + if (ev) { + value = ((PyStopIterationObject *)ev)->value; + Py_XINCREF(value); + Py_DECREF(ev); + } + } + else if (PyErr_Occurred()) + return -1; + if (!value) { + value = Py_None; + Py_INCREF(value); + } + *pvalue = value; + return 0; +} static PyObject * gen_repr(PyGenObject *gen) diff --git a/Parser/Python.asdl b/Parser/Python.asdl --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -59,7 +59,7 @@ | DictComp(expr key, expr value, comprehension* generators) | GeneratorExp(expr elt, comprehension* generators) -- the grammar constrains where yield expressions can occur - | Yield(expr? value) + | Yield(int is_from, expr? value) -- need sequences for compare to distinguish between -- x < 4 < 3 and (x < 4) < 3 | Compare(expr left, cmpop* ops, expr* comparators) diff --git a/Python/Python-ast.c b/Python/Python-ast.c --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -196,6 +196,7 @@ }; static PyTypeObject *Yield_type; static char *Yield_fields[]={ + "is_from", "value", }; static PyTypeObject *Compare_type; @@ -748,7 +749,7 @@ GeneratorExp_type = make_type("GeneratorExp", expr_type, GeneratorExp_fields, 2); if (!GeneratorExp_type) return 0; - Yield_type = make_type("Yield", expr_type, Yield_fields, 1); + Yield_type = make_type("Yield", expr_type, Yield_fields, 2); if (!Yield_type) return 0; Compare_type = make_type("Compare", expr_type, Compare_fields, 3); if (!Compare_type) return 0; @@ -1685,13 +1686,14 @@ } expr_ty -Yield(expr_ty value, int lineno, int col_offset, PyArena *arena) +Yield(int is_from, expr_ty value, int lineno, int col_offset, PyArena *arena) { expr_ty p; p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; p->kind = Yield_kind; + p->v.Yield.is_from = is_from; p->v.Yield.value = value; p->lineno = lineno; p->col_offset = col_offset; @@ -2733,6 +2735,11 @@ case Yield_kind: result = PyType_GenericNew(Yield_type, NULL, NULL); if (!result) goto failed; + value = ast2obj_int(o->v.Yield.is_from); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "is_from", value) == -1) + goto failed; + Py_DECREF(value); value = ast2obj_expr(o->v.Yield.value); if (!value) goto failed; if (PyObject_SetAttrString(result, "value", value) == -1) @@ -5275,8 +5282,21 @@ return 1; } if (isinstance) { + int is_from; expr_ty value; + if (PyObject_HasAttrString(obj, "is_from")) { + int res; + tmp = PyObject_GetAttrString(obj, "is_from"); + if (tmp == NULL) goto failed; + res = obj2ast_int(tmp, &is_from, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"is_from\" missing from Yield"); + return 1; + } if (PyObject_HasAttrString(obj, "value")) { int res; tmp = PyObject_GetAttrString(obj, "value"); @@ -5288,7 +5308,7 @@ } else { value = NULL; } - *out = Yield(value, lineno, col_offset, arena); + *out = Yield(is_from, value, lineno, col_offset, arena); if (*out == NULL) goto failed; return 0; } diff --git a/Python/ast.c b/Python/ast.c --- a/Python/ast.c +++ b/Python/ast.c @@ -2373,13 +2373,28 @@ } return ast_for_binop(c, n); case yield_expr: { + node *an = NULL; + node *en = NULL; + int is_from = 0; expr_ty exp = NULL; - if (NCH(n) == 2) { - exp = ast_for_testlist(c, CHILD(n, 1)); + if (NCH(n) > 1) + an = CHILD(n, 1); /* yield_arg */ + if (an) { + en = CHILD(an, NCH(an) - 1); + if (NCH(an) == 2) { + is_from = 1; + exp = ast_for_expr(c, en); + } + else + exp = ast_for_testlist(c, en); if (!exp) return NULL; } - return Yield(exp, LINENO(n), n->n_col_offset, c->c_arena); + return Yield(is_from, exp, LINENO(n), n->n_col_offset, c->c_arena); + } + case yield_from: { + expr_ty exp = ast_for_expr(c, CHILD(n, 2)); + return Yield(1, exp, LINENO(n), n->n_col_offset, c->c_arena); } case factor: if (NCH(n) == 1) { @@ -2403,7 +2418,7 @@ /* arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test) - argument: [test '='] test [comp_for] # Really [keyword '='] test + argument: [test '='] (test | yield_from) [comp_for] # Really [keyword '='] test */ int i, nargs, nkeywords, ngens; @@ -2697,7 +2712,7 @@ continue_stmt: 'continue' return_stmt: 'return' [testlist] yield_stmt: yield_expr - yield_expr: 'yield' testlist + yield_expr: 'yield' testlist | 'yield' 'from' test raise_stmt: 'raise' [test [',' test [',' test]]] */ node *ch; diff --git a/Python/ceval.c b/Python/ceval.c --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1827,8 +1827,47 @@ why = WHY_RETURN; goto fast_block_end; + TARGET(YIELD_FROM) + u = POP(); + x = PyObject_GetIter(u); + Py_DECREF(u); + if (!x) + break; + /* x is now the iterator, make the first next() call */ + retval = (*x->ob_type->tp_iternext)(x); + if (!retval) { + /* iter may be exhausted */ + Py_DECREF(x); + if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + /* try to get return value from exception */ + PyObject *et, *ev, *tb; + PyErr_Fetch(&et, &ev, &tb); + Py_XDECREF(et); Py_XDECREF(tb); + u = NULL; + if (ev) { + u = PyObject_GetAttrString(ev, "value"); + Py_DECREF(ev); + if (!u) PyErr_Clear(); + } + if (!u) { + u = Py_None; + Py_INCREF(u); + } + /* u is return value */ + PUSH(u); + continue; + } + /* some other exception */ + x = NULL; + break; + } + /* x is iterator, retval is value to be yielded */ + f->f_yieldfrom = x; + goto yield_retval; + TARGET(YIELD_VALUE) retval = POP(); + yield_retval: f->f_stacktop = stack_pointer; why = WHY_YIELD; goto fast_yield; diff --git a/Python/compile.c b/Python/compile.c --- a/Python/compile.c +++ b/Python/compile.c @@ -763,6 +763,7 @@ case IMPORT_STAR: return -1; case YIELD_VALUE: + case YIELD_FROM: return 0; case POP_BLOCK: @@ -3202,7 +3203,12 @@ else { ADDOP_O(c, LOAD_CONST, Py_None, consts); } - ADDOP(c, YIELD_VALUE); + if (e->v.Yield.is_from) { + ADDOP(c, YIELD_FROM); + } + else { + ADDOP(c, YIELD_VALUE); + } break; case Compare_kind: return compiler_compare(c, e); diff --git a/Python/graminit.c b/Python/graminit.c --- a/Python/graminit.c +++ b/Python/graminit.c @@ -1700,29 +1700,36 @@ {1, arcs_74_6}, {2, arcs_74_7}, }; -static arc arcs_75_0[1] = { +static arc arcs_75_0[2] = { {24, 1}, + {164, 2}, }; static arc arcs_75_1[3] = { - {158, 2}, - {29, 3}, + {158, 3}, + {29, 4}, {0, 1}, }; -static arc arcs_75_2[1] = { +static arc arcs_75_2[2] = { + {158, 3}, {0, 2}, }; static arc arcs_75_3[1] = { - {24, 2}, + {0, 3}, }; -static state states_75[4] = { - {1, arcs_75_0}, +static arc arcs_75_4[2] = { + {24, 3}, + {164, 3}, +}; +static state states_75[5] = { + {2, arcs_75_0}, {3, arcs_75_1}, - {1, arcs_75_2}, + {2, arcs_75_2}, {1, arcs_75_3}, + {2, arcs_75_4}, }; static arc arcs_76_0[2] = { {158, 1}, - {165, 1}, + {166, 1}, }; static arc arcs_76_1[1] = { {0, 1}, @@ -1744,7 +1751,7 @@ {107, 4}, }; static arc arcs_77_4[2] = { - {164, 5}, + {165, 5}, {0, 4}, }; static arc arcs_77_5[1] = { @@ -1765,7 +1772,7 @@ {109, 2}, }; static arc arcs_78_2[2] = { - {164, 3}, + {165, 3}, {0, 2}, }; static arc arcs_78_3[1] = { @@ -1788,10 +1795,10 @@ {1, arcs_79_1}, }; static arc arcs_80_0[1] = { - {167, 1}, + {168, 1}, }; static arc arcs_80_1[2] = { - {9, 2}, + {169, 2}, {0, 1}, }; static arc arcs_80_2[1] = { @@ -1802,171 +1809,208 @@ {2, arcs_80_1}, {1, arcs_80_2}, }; -static dfa dfas[81] = { +static arc arcs_81_0[2] = { + {73, 1}, + {9, 2}, +}; +static arc arcs_81_1[1] = { + {24, 2}, +}; +static arc arcs_81_2[1] = { + {0, 2}, +}; +static state states_81[3] = { + {2, arcs_81_0}, + {1, arcs_81_1}, + {1, arcs_81_2}, +}; +static arc arcs_82_0[1] = { + {168, 1}, +}; +static arc arcs_82_1[1] = { + {73, 2}, +}; +static arc arcs_82_2[1] = { + {24, 3}, +}; +static arc arcs_82_3[1] = { + {0, 3}, +}; +static state states_82[4] = { + {1, arcs_82_0}, + {1, arcs_82_1}, + {1, arcs_82_2}, + {1, arcs_82_3}, +}; +static dfa dfas[83] = { {256, "single_input", 0, 3, states_0, - "\004\050\060\200\000\000\000\240\340\223\160\220\045\200\020\000\000\206\120\076\204"}, + "\004\050\060\200\000\000\000\240\340\223\160\220\045\200\020\000\000\206\120\076\004\001"}, {257, "file_input", 0, 2, states_1, - "\204\050\060\200\000\000\000\240\340\223\160\220\045\200\020\000\000\206\120\076\204"}, + "\204\050\060\200\000\000\000\240\340\223\160\220\045\200\020\000\000\206\120\076\004\001"}, {258, "eval_input", 0, 3, states_2, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {259, "decorator", 0, 7, states_3, - "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {260, "decorators", 0, 2, states_4, - "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {261, "decorated", 0, 3, states_5, - "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {262, "funcdef", 0, 8, states_6, - "\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {263, "parameters", 0, 4, states_7, - "\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {264, "typedargslist", 0, 18, states_8, - "\000\000\040\200\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\200\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {265, "tfpdef", 0, 4, states_9, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {266, "varargslist", 0, 18, states_10, - "\000\000\040\200\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\200\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {267, "vfpdef", 0, 2, states_11, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {268, "stmt", 0, 2, states_12, - "\000\050\060\200\000\000\000\240\340\223\160\220\045\200\020\000\000\206\120\076\204"}, + "\000\050\060\200\000\000\000\240\340\223\160\220\045\200\020\000\000\206\120\076\004\001"}, {269, "simple_stmt", 0, 4, states_13, - "\000\040\040\200\000\000\000\240\340\223\160\000\000\200\020\000\000\206\120\076\200"}, + "\000\040\040\200\000\000\000\240\340\223\160\000\000\200\020\000\000\206\120\076\000\001"}, {270, "small_stmt", 0, 2, states_14, - "\000\040\040\200\000\000\000\240\340\223\160\000\000\200\020\000\000\206\120\076\200"}, + "\000\040\040\200\000\000\000\240\340\223\160\000\000\200\020\000\000\206\120\076\000\001"}, {271, "expr_stmt", 0, 6, states_15, - "\000\040\040\200\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\200\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {272, "testlist_star_expr", 0, 3, states_16, - "\000\040\040\200\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\200\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {273, "augassign", 0, 2, states_17, - "\000\000\000\000\000\000\376\037\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\376\037\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {274, "del_stmt", 0, 3, states_18, - "\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {275, "pass_stmt", 0, 2, states_19, - "\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {276, "flow_stmt", 0, 2, states_20, - "\000\000\000\000\000\000\000\000\340\001\000\000\000\000\000\000\000\000\000\000\200"}, + "\000\000\000\000\000\000\000\000\340\001\000\000\000\000\000\000\000\000\000\000\000\001"}, {277, "break_stmt", 0, 2, states_21, - "\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {278, "continue_stmt", 0, 2, states_22, - "\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {279, "return_stmt", 0, 3, states_23, - "\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {280, "yield_stmt", 0, 2, states_24, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\200"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001"}, {281, "raise_stmt", 0, 5, states_25, - "\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000"}, {282, "import_stmt", 0, 2, states_26, - "\000\000\000\000\000\000\000\000\000\022\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\022\000\000\000\000\000\000\000\000\000\000\000\000"}, {283, "import_name", 0, 3, states_27, - "\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000"}, {284, "import_from", 0, 8, states_28, - "\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000"}, {285, "import_as_name", 0, 4, states_29, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {286, "dotted_as_name", 0, 4, states_30, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {287, "import_as_names", 0, 3, states_31, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {288, "dotted_as_names", 0, 2, states_32, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {289, "dotted_name", 0, 2, states_33, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {290, "global_stmt", 0, 3, states_34, - "\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000"}, {291, "nonlocal_stmt", 0, 3, states_35, - "\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000"}, {292, "assert_stmt", 0, 5, states_36, - "\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000"}, {293, "compound_stmt", 0, 2, states_37, - "\000\010\020\000\000\000\000\000\000\000\000\220\045\000\000\000\000\000\000\000\004"}, + "\000\010\020\000\000\000\000\000\000\000\000\220\045\000\000\000\000\000\000\000\004\000"}, {294, "if_stmt", 0, 8, states_38, - "\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"}, {295, "while_stmt", 0, 8, states_39, - "\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000"}, {296, "for_stmt", 0, 10, states_40, - "\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"}, {297, "try_stmt", 0, 13, states_41, - "\000\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000"}, {298, "with_stmt", 0, 5, states_42, - "\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000"}, {299, "with_item", 0, 4, states_43, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {300, "except_clause", 0, 5, states_44, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000"}, {301, "suite", 0, 5, states_45, - "\004\040\040\200\000\000\000\240\340\223\160\000\000\200\020\000\000\206\120\076\200"}, + "\004\040\040\200\000\000\000\240\340\223\160\000\000\200\020\000\000\206\120\076\000\001"}, {302, "test", 0, 6, states_46, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {303, "test_nocond", 0, 2, states_47, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {304, "lambdef", 0, 5, states_48, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000"}, {305, "lambdef_nocond", 0, 5, states_49, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000"}, {306, "or_test", 0, 2, states_50, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\020\000\000\206\120\076\000\000"}, {307, "and_test", 0, 2, states_51, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\020\000\000\206\120\076\000\000"}, {308, "not_test", 0, 3, states_52, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\020\000\000\206\120\076\000\000"}, {309, "comparison", 0, 2, states_53, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000\000"}, {310, "comp_op", 0, 4, states_54, - "\000\000\000\000\000\000\000\000\000\000\000\000\002\000\220\177\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\002\000\220\177\000\000\000\000\000\000"}, {311, "star_expr", 0, 3, states_55, - "\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {312, "expr", 0, 2, states_56, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000\000"}, {313, "xor_expr", 0, 2, states_57, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000\000"}, {314, "and_expr", 0, 2, states_58, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000\000"}, {315, "shift_expr", 0, 2, states_59, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000\000"}, {316, "arith_expr", 0, 2, states_60, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000\000"}, {317, "term", 0, 2, states_61, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000\000"}, {318, "factor", 0, 3, states_62, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000\000"}, {319, "power", 0, 4, states_63, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\120\076\000\000"}, {320, "atom", 0, 9, states_64, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\120\076\000\000"}, {321, "testlist_comp", 0, 5, states_65, - "\000\040\040\200\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\200\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {322, "trailer", 0, 7, states_66, - "\000\040\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\020\000\000"}, + "\000\040\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\020\000\000\000"}, {323, "subscriptlist", 0, 3, states_67, - "\000\040\040\002\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\002\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {324, "subscript", 0, 5, states_68, - "\000\040\040\002\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\002\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {325, "sliceop", 0, 3, states_69, - "\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {326, "exprlist", 0, 3, states_70, - "\000\040\040\200\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000"}, + "\000\040\040\200\000\000\000\000\000\200\000\000\000\000\000\000\000\206\120\076\000\000"}, {327, "testlist", 0, 3, states_71, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {328, "dictorsetmaker", 0, 11, states_72, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\000"}, {329, "classdef", 0, 8, states_73, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\004"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\004\000"}, {330, "arglist", 0, 8, states_74, - "\000\040\040\200\001\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, - {331, "argument", 0, 4, states_75, - "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000"}, + "\000\040\040\200\001\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\001"}, + {331, "argument", 0, 5, states_75, + "\000\040\040\000\000\000\000\000\000\200\000\000\000\200\020\000\000\206\120\076\000\001"}, {332, "comp_iter", 0, 2, states_76, - "\000\000\000\000\000\000\000\000\000\000\000\020\001\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\020\001\000\000\000\000\000\000\000\000\000"}, {333, "comp_for", 0, 6, states_77, - "\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"}, {334, "comp_if", 0, 4, states_78, - "\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"}, {335, "encoding_decl", 0, 2, states_79, - "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {336, "yield_expr", 0, 3, states_80, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\200"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001"}, + {337, "yield_arg", 0, 3, states_81, + "\000\040\040\000\000\000\000\000\000\202\000\000\000\200\020\000\000\206\120\076\000\000"}, + {338, "yield_from", 0, 4, states_82, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001"}, }; -static label labels[168] = { +static label labels[170] = { {0, "EMPTY"}, {256, 0}, {4, 0}, @@ -2131,14 +2175,16 @@ {325, 0}, {1, "class"}, {331, 0}, + {338, 0}, {332, 0}, {334, 0}, {335, 0}, {1, "yield"}, + {337, 0}, }; grammar _PyParser_Grammar = { - 81, + 83, dfas, - {168, labels}, + {170, labels}, 256 }; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -71,7 +71,7 @@ &&TARGET_STORE_LOCALS, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, - &&_unknown_opcode, + &&TARGET_YIELD_FROM, &&_unknown_opcode, &&_unknown_opcode, &&TARGET_INPLACE_LSHIFT, diff --git a/Python/symtable.c b/Python/symtable.c --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1133,14 +1133,6 @@ if (s->v.Return.value) { VISIT(st, expr, s->v.Return.value); st->st_cur->ste_returns_value = 1; - if (st->st_cur->ste_generator) { - PyErr_SetString(PyExc_SyntaxError, - RETURN_VAL_IN_GENERATOR); - PyErr_SyntaxLocationEx(st->st_filename, - s->lineno, - s->col_offset); - return 0; - } } break; case Delete_kind: @@ -1342,13 +1334,6 @@ if (e->v.Yield.value) VISIT(st, expr, e->v.Yield.value); st->st_cur->ste_generator = 1; - if (st->st_cur->ste_returns_value) { - PyErr_SetString(PyExc_SyntaxError, - RETURN_VAL_IN_GENERATOR); - PyErr_SyntaxLocationEx(st->st_filename, - e->lineno, e->col_offset); - return 0; - } break; case Compare_kind: VISIT(st, expr, e->v.Compare.left);