diff -r 47b4dbd451f5 Include/ceval.h --- a/Include/ceval.h Wed Sep 07 09:31:52 2016 -0700 +++ b/Include/ceval.h Wed Sep 07 10:28:40 2016 -0700 @@ -25,6 +25,10 @@ PyAPI_FUNC(void) PyEval_SetTrace(Py_tracefunc, PyObject *); PyAPI_FUNC(void) _PyEval_SetCoroutineWrapper(PyObject *); PyAPI_FUNC(PyObject *) _PyEval_GetCoroutineWrapper(void); +PyAPI_FUNC(void) _PyEval_SetAsyncGenFirstiter(PyObject *); +PyAPI_FUNC(PyObject *) _PyEval_GetAsyncGenFirstiter(void); +PyAPI_FUNC(void) _PyEval_SetAsyncGenFinalizer(PyObject *); +PyAPI_FUNC(PyObject *) _PyEval_GetAsyncGenFinalizer(void); #endif struct _frame; /* Avoid including frameobject.h */ diff -r 47b4dbd451f5 Include/code.h --- a/Include/code.h Wed Sep 07 09:31:52 2016 -0700 +++ b/Include/code.h Wed Sep 07 10:28:40 2016 -0700 @@ -55,6 +55,7 @@ ``async def`` keywords) */ #define CO_COROUTINE 0x0080 #define CO_ITERABLE_COROUTINE 0x0100 +#define CO_ASYNC_GENERATOR 0x0200 /* These are no longer used. */ #if 0 diff -r 47b4dbd451f5 Include/genobject.h --- a/Include/genobject.h Wed Sep 07 09:31:52 2016 -0700 +++ b/Include/genobject.h Wed Sep 07 10:28:40 2016 -0700 @@ -61,6 +61,30 @@ PyObject *_PyCoro_GetAwaitableIter(PyObject *o); PyAPI_FUNC(PyObject *) PyCoro_New(struct _frame *, PyObject *name, PyObject *qualname); + +/* Asynchronous Generators */ + +typedef struct { + _PyGenObject_HEAD(ag) + PyObject *ag_finalizer; + int ag_hooks_inited; + int ag_closed; +} PyAsyncGenObject; + +PyAPI_DATA(PyTypeObject) PyAsyncGen_Type; +PyAPI_DATA(PyTypeObject) _PyAsyncGenASend_Type; +PyAPI_DATA(PyTypeObject) _PyAsyncGenWrappedValue_Type; +PyAPI_DATA(PyTypeObject) _PyAsyncGenAThrow_Type; + +PyAPI_FUNC(PyObject *) PyAsyncGen_New(struct _frame *, + PyObject *name, PyObject *qualname); + +#define PyAsyncGen_CheckExact(op) (Py_TYPE(op) == &PyAsyncGen_Type) + +PyObject *_PyAsyncGenWrapValue(PyObject *); + +int PyAsyncGen_ClearFreeLists(void); + #endif #undef _PyGenObject_HEAD diff -r 47b4dbd451f5 Include/pylifecycle.h --- a/Include/pylifecycle.h Wed Sep 07 09:31:52 2016 -0700 +++ b/Include/pylifecycle.h Wed Sep 07 10:28:40 2016 -0700 @@ -107,6 +107,7 @@ PyAPI_FUNC(void) PySlice_Fini(void); PyAPI_FUNC(void) _PyType_Fini(void); PyAPI_FUNC(void) _PyRandom_Fini(void); +PyAPI_FUNC(void) PyAsyncGen_Fini(void); PyAPI_DATA(PyThreadState *) _Py_Finalizing; #endif diff -r 47b4dbd451f5 Include/pystate.h --- a/Include/pystate.h Wed Sep 07 09:31:52 2016 -0700 +++ b/Include/pystate.h Wed Sep 07 10:28:40 2016 -0700 @@ -141,6 +141,9 @@ PyObject *coroutine_wrapper; int in_coroutine_wrapper; + PyObject *async_gen_firstiter; + PyObject *async_gen_finalizer; + /* XXX signal handlers should also be here */ } PyThreadState; diff -r 47b4dbd451f5 Include/symtable.h --- a/Include/symtable.h Wed Sep 07 09:31:52 2016 -0700 +++ b/Include/symtable.h Wed Sep 07 10:28:40 2016 -0700 @@ -48,6 +48,7 @@ unsigned ste_child_free : 1; /* true if a child block has free vars, including free refs to globals */ unsigned ste_generator : 1; /* true if namespace is a generator */ + unsigned ste_coroutine : 1; /* true if namespace is a coroutine */ unsigned ste_varargs : 1; /* true if block has varargs */ unsigned ste_varkeywords : 1; /* true if block has varkeywords */ unsigned ste_returns_value : 1; /* true if namespace uses return with diff -r 47b4dbd451f5 Lib/asyncio/base_events.py --- a/Lib/asyncio/base_events.py Wed Sep 07 09:31:52 2016 -0700 +++ b/Lib/asyncio/base_events.py Wed Sep 07 10:28:40 2016 -0700 @@ -12,7 +12,8 @@ conscious design decision, leaving the door open for keyword arguments to modify the meaning of the API call itself. """ - +import gc +gc.disable() import collections import concurrent.futures @@ -28,6 +29,7 @@ import traceback import sys import warnings +import weakref from . import compat from . import coroutines @@ -241,6 +243,8 @@ self._current_handle = None self._task_factory = None self._coroutine_wrapper_set = False + self._asyncgens = weakref.WeakSet() + self._asyncgens_shutdown_called = False def __repr__(self): return ('<%s running=%s closed=%s debug=%s>' @@ -333,6 +337,49 @@ if self._closed: raise RuntimeError('Event loop is closed') + def _asyncgen_finalizer_hook(self, ag): + if ag in self._asyncgens: + self._asyncgens.remove(ag) + + if not self.is_closed(): + self.create_task(ag.aclose()) + + def _asyncgen_firstiter_hook(self, ag): + if self._asyncgens_shutdown_called: + warnings.warn( + "asynchronous generator {!r} was scheduled after " + "loop.shutdown_asyncgens() call".format(ag), + ResourceWarning, source=self) + + self._asyncgens.add(ag) + + @coroutine + def shutdown_asyncgens(self): + """Shutdown all active asynchronous generators. + + This coroutine method should be scheduled before calling loop.close(). + """ + self._asyncgens_shutdown_called = True + + if not len(self._asyncgens): + return + + shutdown_coro = tasks.gather( + *[ag.aclose() for ag in self._asyncgens], + return_exceptions=True, + loop=self) + + self._asyncgens.clear() + + results = yield from shutdown_coro + for result in results: + if isinstance(result, Exception): + self.call_exception_handler({ + 'message': 'an error occurred during asynchronous ' + 'generator closing', + 'exception': result + }) + def run_forever(self): """Run until stop() is called.""" self._check_closed() @@ -340,6 +387,9 @@ raise RuntimeError('Event loop is running.') self._set_coroutine_wrapper(self._debug) self._thread_id = threading.get_ident() + old_agen_hooks = sys.get_asyncgen_hooks() + sys.set_asyncgen_hooks(firstiter=self._asyncgen_firstiter_hook, + finalizer=self._asyncgen_finalizer_hook) try: while True: self._run_once() @@ -349,6 +399,7 @@ self._stopping = False self._thread_id = None self._set_coroutine_wrapper(False) + sys.set_asyncgen_hooks(*old_agen_hooks) def run_until_complete(self, future): """Run until the Future is done. diff -r 47b4dbd451f5 Lib/asyncio/coroutines.py --- a/Lib/asyncio/coroutines.py Wed Sep 07 09:31:52 2016 -0700 +++ b/Lib/asyncio/coroutines.py Wed Sep 07 10:28:40 2016 -0700 @@ -276,7 +276,10 @@ try: coro_code = coro.gi_code except AttributeError: - coro_code = coro.cr_code + try: + coro_code = coro.cr_code + except AttributeError: + return repr(coro) try: coro_frame = coro.gi_frame diff -r 47b4dbd451f5 Lib/asyncio/events.py --- a/Lib/asyncio/events.py Wed Sep 07 09:31:52 2016 -0700 +++ b/Lib/asyncio/events.py Wed Sep 07 10:28:40 2016 -0700 @@ -248,6 +248,13 @@ """ raise NotImplementedError + def shutdown_asyncgens(self): + """Shutdown all active asynchronous generators. + + This coroutine method should be scheduled before calling loop.close(). + """ + raise NotImplementedError + # Methods scheduling callbacks. All these return Handles. def _timer_handle_cancelled(self, handle): diff -r 47b4dbd451f5 Lib/dis.py --- a/Lib/dis.py Wed Sep 07 09:31:52 2016 -0700 +++ b/Lib/dis.py Wed Sep 07 10:28:40 2016 -0700 @@ -87,6 +87,7 @@ 64: "NOFREE", 128: "COROUTINE", 256: "ITERABLE_COROUTINE", + 512: "ASYNC_GENERATOR", } def pretty_flags(flags): diff -r 47b4dbd451f5 Lib/inspect.py --- a/Lib/inspect.py Wed Sep 07 09:31:52 2016 -0700 +++ b/Lib/inspect.py Wed Sep 07 10:28:40 2016 -0700 @@ -186,6 +186,13 @@ return bool((isfunction(object) or ismethod(object)) and object.__code__.co_flags & CO_COROUTINE) +def isasyncgenfunction(object): + return bool((isfunction(object) or ismethod(object)) and + object.__code__.co_flags & CO_ASYNC_GENERATOR) + +def isasyncgen(object): + return isinstance(object, types.AsyncGeneratorType) + def isgenerator(object): """Return true if the object is a generator. diff -r 47b4dbd451f5 Lib/test/test_coroutines.py --- a/Lib/test/test_coroutines.py Wed Sep 07 09:31:52 2016 -0700 +++ b/Lib/test/test_coroutines.py Wed Sep 07 10:28:40 2016 -0700 @@ -88,12 +88,6 @@ with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): import test.badsyntax_async5 - def test_badsyntax_6(self): - with self.assertRaisesRegex( - SyntaxError, "'yield' inside async function"): - - import test.badsyntax_async6 - def test_badsyntax_7(self): with self.assertRaisesRegex( SyntaxError, "'yield from' inside async function"): diff -r 47b4dbd451f5 Lib/test/test_dis.py --- a/Lib/test/test_dis.py Wed Sep 07 09:31:52 2016 -0700 +++ b/Lib/test/test_dis.py Wed Sep 07 10:28:40 2016 -0700 @@ -541,7 +541,7 @@ Kw-only arguments: 0 Number of locals: 2 Stack size: 17 -Flags: OPTIMIZED, NEWLOCALS, GENERATOR, NOFREE, COROUTINE +Flags: OPTIMIZED, NEWLOCALS, NOFREE, COROUTINE Constants: 0: None 1: 1""" diff -r 47b4dbd451f5 Lib/test/test_inspect.py --- a/Lib/test/test_inspect.py Wed Sep 07 09:31:52 2016 -0700 +++ b/Lib/test/test_inspect.py Wed Sep 07 10:28:40 2016 -0700 @@ -65,7 +65,8 @@ inspect.isframe, inspect.isfunction, inspect.ismethod, inspect.ismodule, inspect.istraceback, inspect.isgenerator, inspect.isgeneratorfunction, - inspect.iscoroutine, inspect.iscoroutinefunction]) + inspect.iscoroutine, inspect.iscoroutinefunction, + inspect.isasyncgen, inspect.isasyncgenfunction]) def istest(self, predicate, exp): obj = eval(exp) @@ -73,6 +74,7 @@ for other in self.predicates - set([predicate]): if (predicate == inspect.isgeneratorfunction or \ + predicate == inspect.isasyncgenfunction or \ predicate == inspect.iscoroutinefunction) and \ other == inspect.isfunction: continue @@ -82,6 +84,10 @@ for i in range(2): yield i +async def async_generator_function_example(self): + async for i in range(2): + yield i + async def coroutine_function_example(self): return 'spam' @@ -122,6 +128,10 @@ self.istest(inspect.isdatadescriptor, 'collections.defaultdict.default_factory') self.istest(inspect.isgenerator, '(x for x in range(2))') self.istest(inspect.isgeneratorfunction, 'generator_function_example') + self.istest(inspect.isasyncgen, + 'async_generator_function_example(1)') + self.istest(inspect.isasyncgenfunction, + 'async_generator_function_example') with warnings.catch_warnings(): warnings.simplefilter("ignore") diff -r 47b4dbd451f5 Lib/types.py --- a/Lib/types.py Wed Sep 07 09:31:52 2016 -0700 +++ b/Lib/types.py Wed Sep 07 10:28:40 2016 -0700 @@ -24,6 +24,10 @@ CoroutineType = type(_c) _c.close() # Prevent ResourceWarning +async def _ag(): yield +_ag = _ag() +AsyncGeneratorType = type(_ag) + class _C: _nsType = type(locals()) def _m(self): pass diff -r 47b4dbd451f5 Modules/gcmodule.c --- a/Modules/gcmodule.c Wed Sep 07 09:31:52 2016 -0700 +++ b/Modules/gcmodule.c Wed Sep 07 10:28:40 2016 -0700 @@ -892,6 +892,7 @@ (void)PyList_ClearFreeList(); (void)PyDict_ClearFreeList(); (void)PySet_ClearFreeList(); + (void)PyAsyncGen_ClearFreeLists(); } /* This is the main function. Read this to understand how the diff -r 47b4dbd451f5 Objects/genobject.c --- a/Objects/genobject.c Wed Sep 07 09:31:52 2016 -0700 +++ b/Objects/genobject.c Wed Sep 07 10:28:40 2016 -0700 @@ -5,7 +5,15 @@ #include "structmember.h" #include "opcode.h" -static PyObject *gen_close(PyGenObject *gen, PyObject *args); +static PyObject *gen_close(PyGenObject *, PyObject *); +static PyObject *async_gen_asend_new(PyAsyncGenObject *, PyObject *); +static PyObject *async_gen_athrow_new(PyAsyncGenObject *, PyObject *); + +static char *NON_INIT_CORO_MSG = "can't send non-None value to a " + "just-started coroutine"; + +static char *ASYNC_GEN_IGNORED_EXIT_MSG = + "async generator ignored GeneratorExit"; static int gen_traverse(PyGenObject *gen, visitproc visit, void *arg) @@ -28,6 +36,26 @@ /* Generator isn't paused, so no need to close */ return; + if (PyAsyncGen_CheckExact(self)) { + PyAsyncGenObject *agen = (PyAsyncGenObject*)self; + PyObject *finalizer = agen->ag_finalizer; + if (finalizer && !agen->ag_closed) { + /* Save the current exception, if any. */ + PyErr_Fetch(&error_type, &error_value, &error_traceback); + + res = PyObject_CallFunction(finalizer, "O", self); + + if (res == NULL) { + PyErr_WriteUnraisable(self); + } else { + Py_DECREF(res); + } + /* Restore the saved exception. */ + PyErr_Restore(error_type, error_value, error_traceback); + return; + } + } + /* Save the current exception, if any. */ PyErr_Fetch(&error_type, &error_value, &error_traceback); @@ -74,6 +102,9 @@ return; /* resurrected. :( */ _PyObject_GC_UNTRACK(self); + if (PyAsyncGen_CheckExact(gen)) { + Py_CLEAR(((PyAsyncGenObject*)gen)->ag_finalizer); + } if (gen->gi_frame != NULL) { gen->gi_frame->f_gen = NULL; Py_CLEAR(gen->gi_frame); @@ -84,6 +115,31 @@ PyObject_GC_Del(gen); } +static void +gen_chain_runtime_error(const char *msg) +{ + PyObject *exc, *val, *val2, *tb; + + PyErr_Fetch(&exc, &val, &tb); + PyErr_NormalizeException(&exc, &val, &tb); + if (tb != NULL) { + PyException_SetTraceback(val, tb); + } + + Py_DECREF(exc); + Py_XDECREF(tb); + + PyErr_SetString(PyExc_RuntimeError, msg); + PyErr_Fetch(&exc, &val2, &tb); + PyErr_NormalizeException(&exc, &val2, &tb); + + Py_INCREF(val); + PyException_SetCause(val2, val); + PyException_SetContext(val2, val); + + PyErr_Restore(exc, val2, tb); +} + static PyObject * gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing) { @@ -93,8 +149,11 @@ if (gen->gi_running) { char *msg = "generator already executing"; - if (PyCoro_CheckExact(gen)) + if (PyCoro_CheckExact(gen)) { msg = "coroutine already executing"; + } else if (PyAsyncGen_CheckExact(gen)) { + msg = "async generator already executing"; + } PyErr_SetString(PyExc_ValueError, msg); return NULL; } @@ -109,7 +168,11 @@ } else if (arg && !exc) { /* `gen` is an exhausted generator: only set exception if called from send(). */ - PyErr_SetNone(PyExc_StopIteration); + if (PyAsyncGen_CheckExact(gen)) { + PyErr_SetNone(PyExc_StopAsyncIteration); + } else { + PyErr_SetNone(PyExc_StopIteration); + } } return NULL; } @@ -118,9 +181,12 @@ if (arg && arg != Py_None) { char *msg = "can't send non-None value to a " "just-started generator"; - if (PyCoro_CheckExact(gen)) + if (PyCoro_CheckExact(gen)) { + msg = NON_INIT_CORO_MSG; + } else if (PyAsyncGen_CheckExact(gen)) { msg = "can't send non-None value to a " - "just-started coroutine"; + "just-started async generator"; + } PyErr_SetString(PyExc_TypeError, msg); return NULL; } @@ -152,10 +218,18 @@ if (result && f->f_stacktop == NULL) { if (result == Py_None) { /* Delay exception instantiation if we can */ - PyErr_SetNone(PyExc_StopIteration); + if (PyAsyncGen_CheckExact(gen)) { + PyErr_SetNone(PyExc_StopAsyncIteration); + } else { + PyErr_SetNone(PyExc_StopIteration); + } } else { PyObject *e = PyObject_CallFunctionObjArgs( PyExc_StopIteration, result, NULL); + + /* Async generators cannot return anything but None */ + assert(!PyAsyncGen_CheckExact(gen)); + if (e != NULL) { PyErr_SetObject(PyExc_StopIteration, e); Py_DECREF(e); @@ -167,28 +241,36 @@ /* Check for __future__ generator_stop and conditionally turn * a leaking StopIteration into RuntimeError (with its cause * set appropriately). */ - if (((PyCodeObject *)gen->gi_code)->co_flags & - (CO_FUTURE_GENERATOR_STOP | CO_COROUTINE | CO_ITERABLE_COROUTINE)) - { - PyObject *exc, *val, *val2, *tb; - char *msg = "generator raised StopIteration"; - if (PyCoro_CheckExact(gen)) + + const int strict_gen_flags = CO_FUTURE_GENERATOR_STOP | + CO_COROUTINE | + CO_ITERABLE_COROUTINE | + CO_ASYNC_GENERATOR; + + const int co_flags = ((PyCodeObject *)gen->gi_code)->co_flags; + + if (co_flags & strict_gen_flags) { + /* `gen` is either: + * a generator with CO_FUTURE_GENERATOR_STOP flag; + * a coroutine; + * a generator with CO_ITERABLE_COROUTINE flag + (decorated with types.coroutine decorator); + * an async generator. + */ + const char *msg = "generator raised StopIteration"; + if (PyCoro_CheckExact(gen)) { msg = "coroutine raised StopIteration"; - PyErr_Fetch(&exc, &val, &tb); - PyErr_NormalizeException(&exc, &val, &tb); - if (tb != NULL) - PyException_SetTraceback(val, tb); - Py_DECREF(exc); - Py_XDECREF(tb); - PyErr_SetString(PyExc_RuntimeError, msg); - PyErr_Fetch(&exc, &val2, &tb); - PyErr_NormalizeException(&exc, &val2, &tb); - Py_INCREF(val); - PyException_SetCause(val2, val); - PyException_SetContext(val2, val); - PyErr_Restore(exc, val2, tb); + } else if PyAsyncGen_CheckExact(gen) { + msg = "async generator raised StopIteration"; + } + /* Raise a RuntimeError */ + gen_chain_runtime_error(msg); } else { + /* `gen` is an ordinary generator without + CO_FUTURE_GENERATOR_STOP flag. + */ + PyObject *exc, *val, *tb; /* Pop the exception before issuing a warning. */ @@ -206,6 +288,14 @@ PyErr_Restore(exc, val, tb); } } + } else if (PyAsyncGen_CheckExact(gen) && !result && + PyErr_ExceptionMatches(PyExc_StopAsyncIteration) + ) { + /* code in `gen` raised a StopAsyncIteration error: + raise a RuntimeError. + */ + const char *msg = "async generator raised StopAsyncIteration"; + gen_chain_runtime_error(msg); } if (!result || f->f_stacktop == NULL) { @@ -253,7 +343,7 @@ PyObject *retval = NULL; _Py_IDENTIFIER(close); - if (PyGen_CheckExact(yf)) { + if (PyGen_CheckExact(yf) || PyCoro_CheckExact(yf)) { retval = gen_close((PyGenObject *)yf, NULL); if (retval == NULL) return -1; @@ -311,8 +401,11 @@ retval = gen_send_ex(gen, Py_None, 1, 1); if (retval) { char *msg = "generator ignored GeneratorExit"; - if (PyCoro_CheckExact(gen)) + if (PyCoro_CheckExact(gen)) { msg = "coroutine ignored GeneratorExit"; + } else if (PyAsyncGen_CheckExact(gen)) { + msg = ASYNC_GEN_IGNORED_EXIT_MSG; + } Py_DECREF(retval); PyErr_SetString(PyExc_RuntimeError, msg); return NULL; @@ -332,21 +425,18 @@ return next yielded value or raise StopIteration."); static PyObject * -gen_throw(PyGenObject *gen, PyObject *args) +_gen_throw(PyGenObject *gen, int close_on_genexit, + PyObject *typ, PyObject *val, PyObject *tb) { - PyObject *typ; - PyObject *tb = NULL; - PyObject *val = NULL; PyObject *yf = _PyGen_yf(gen); _Py_IDENTIFIER(throw); - if (!PyArg_UnpackTuple(args, "throw", 1, 3, &typ, &val, &tb)) - return NULL; - if (yf) { PyObject *ret; int err; - if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) { + if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit) && + close_on_genexit + ) { gen->gi_running = 1; err = gen_close_iter(yf); gen->gi_running = 0; @@ -355,9 +445,10 @@ return gen_send_ex(gen, Py_None, 1, 0); goto throw_here; } - if (PyGen_CheckExact(yf)) { + if (PyGen_CheckExact(yf) || PyCoro_CheckExact(yf)) { gen->gi_running = 1; - ret = gen_throw((PyGenObject *)yf, args); + ret = _gen_throw((PyGenObject *)yf, close_on_genexit, + typ, val, tb); gen->gi_running = 0; } else { PyObject *meth = _PyObject_GetAttrId(yf, &PyId_throw); @@ -371,7 +462,7 @@ goto throw_here; } gen->gi_running = 1; - ret = PyObject_CallObject(meth, args); + ret = PyObject_CallFunctionObjArgs(meth, typ, val, tb); gen->gi_running = 0; Py_DECREF(meth); } @@ -454,6 +545,21 @@ static PyObject * +gen_throw(PyGenObject *gen, PyObject *args) +{ + PyObject *typ; + PyObject *tb = NULL; + PyObject *val = NULL; + + if (!PyArg_UnpackTuple(args, "throw", 1, 3, &typ, &val, &tb)) { + return NULL; + } + + return _gen_throw(gen, 1, typ, val, tb); +} + + +static PyObject * gen_iternext(PyGenObject *gen) { return gen_send_ex(gen, NULL, 0, 0); @@ -1085,3 +1191,796 @@ _PyObject_GC_TRACK(aw); return (PyObject *)aw; } + + +/* ========= Asynchronous Generators ========= */ + + +typedef enum { + AWAITABLE_STATE_INIT, /* new awaitable, has not yet been iterated */ + AWAITABLE_STATE_ITER, /* being iterated */ + AWAITABLE_STATE_CLOSED, /* closed */ +} AwaitableState; + + +typedef struct { + PyObject_HEAD + PyAsyncGenObject *aw_gen; + PyObject *aw_sendval; + AwaitableState aw_state; +} PyAsyncGenASend; + + +typedef struct { + PyObject_HEAD + PyAsyncGenObject *ac_gen; + PyObject *ac_args; + AwaitableState ac_state; +} PyAsyncGenAThrow; + + +typedef struct { + PyObject_HEAD + PyObject *val; +} _PyAsyncGenWrappedValue; + + +#ifndef _PyAsyncGen_MAXFREELIST +#define _PyAsyncGen_MAXFREELIST 80 +#endif + +/* Freelists boost performance 6-10%; they also reduce memory + fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend + are short-living objects that are instantiated for every + __anext__ call. +*/ + +static _PyAsyncGenWrappedValue *ag_value_fl[_PyAsyncGen_MAXFREELIST]; +static int ag_value_fl_free = 0; + +static PyAsyncGenASend *ag_asend_fl[_PyAsyncGen_MAXFREELIST]; +static int ag_asend_fl_free = 0; + +#define _PyAsyncGenWrappedValue_CheckExact(o) \ + (Py_TYPE(o) == &_PyAsyncGenWrappedValue_Type) + +#define PyAsyncGenASend_CheckExact(o) \ + (Py_TYPE(o) == &_PyAsyncGenASend_Type) + + +static int +async_gen_traverse(PyAsyncGenObject *gen, visitproc visit, void *arg) +{ + Py_VISIT(gen->ag_finalizer); + return gen_traverse((PyGenObject*)gen, visit, arg); +} + + +static PyObject * +async_gen_repr(PyAsyncGenObject *o) +{ + return PyUnicode_FromFormat("", + o->ag_qualname, o); +} + + +static int +async_gen_init_finalizer(PyAsyncGenObject *o) +{ + PyThreadState *tstate; + PyObject *finalizer; + PyObject *firstiter; + + if (o->ag_hooks_inited) { + return 0; + } + + o->ag_hooks_inited = 1; + + tstate = PyThreadState_GET(); + + finalizer = tstate->async_gen_finalizer; + if (finalizer) { + Py_INCREF(finalizer); + o->ag_finalizer = finalizer; + } + + firstiter = tstate->async_gen_firstiter; + if (firstiter) { + PyObject *res; + + Py_INCREF(firstiter); + res = PyObject_CallFunction(firstiter, "O", o); + Py_DECREF(firstiter); + if (res == NULL) { + return 1; + } + Py_DECREF(res); + } + + return 0; +} + + +static PyObject * +async_gen_anext(PyAsyncGenObject *o) +{ + if (async_gen_init_finalizer(o)) { + return NULL; + } + return async_gen_asend_new(o, NULL); +} + + +static PyObject * +async_gen_asend(PyAsyncGenObject *o, PyObject *arg) +{ + if (async_gen_init_finalizer(o)) { + return NULL; + } + return async_gen_asend_new(o, arg); +} + + +static PyObject * +async_gen_aclose(PyAsyncGenObject *o, PyObject *arg) +{ + if (async_gen_init_finalizer(o)) { + return NULL; + } + return async_gen_athrow_new(o, NULL); +} + +static PyObject * +async_gen_athrow(PyAsyncGenObject *o, PyObject *args) +{ + if (async_gen_init_finalizer(o)) { + return NULL; + } + return async_gen_athrow_new(o, args); +} + + +static PyGetSetDef async_gen_getsetlist[] = { + {"__name__", (getter)gen_get_name, (setter)gen_set_name, + PyDoc_STR("name of the async generator")}, + {"__qualname__", (getter)gen_get_qualname, (setter)gen_set_qualname, + PyDoc_STR("qualified name of the async generator")}, + {"ag_await", (getter)coro_get_cr_await, NULL, + PyDoc_STR("object being awaited on, or None")}, + {NULL} /* Sentinel */ +}; + +static PyMemberDef async_gen_memberlist[] = { + {"ag_frame", T_OBJECT, offsetof(PyAsyncGenObject, ag_frame), READONLY}, + {"ag_running", T_BOOL, offsetof(PyAsyncGenObject, ag_running), READONLY}, + {"ag_code", T_OBJECT, offsetof(PyAsyncGenObject, ag_code), READONLY}, + {NULL} /* Sentinel */ +}; + +PyDoc_STRVAR(async_aclose_doc, +"aclose() -> raise GeneratorExit inside generator."); + +PyDoc_STRVAR(async_asend_doc, +"asend(v) -> send 'v' in generator."); + +PyDoc_STRVAR(async_athrow_doc, +"athrow(typ[,val[,tb]]) -> raise exception in generator."); + +static PyMethodDef async_gen_methods[] = { + {"asend", (PyCFunction)async_gen_asend, METH_O, async_asend_doc}, + {"athrow",(PyCFunction)async_gen_athrow, METH_VARARGS, async_athrow_doc}, + {"aclose", (PyCFunction)async_gen_aclose, METH_NOARGS, async_aclose_doc}, + {NULL, NULL} /* Sentinel */ +}; + + +static PyAsyncMethods async_gen_as_async = { + 0, /* am_await */ + PyObject_SelfIter, /* am_aiter */ + (unaryfunc)async_gen_anext /* am_anext */ +}; + + +PyTypeObject PyAsyncGen_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "async_generator", /* tp_name */ + sizeof(PyAsyncGenObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)gen_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + &async_gen_as_async, /* tp_as_async */ + (reprfunc)async_gen_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)async_gen_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + offsetof(PyAsyncGenObject, ag_weakreflist), /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + async_gen_methods, /* tp_methods */ + async_gen_memberlist, /* tp_members */ + async_gen_getsetlist, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + _PyGen_Finalize, /* tp_finalize */ +}; + + +PyObject * +PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname) +{ + PyAsyncGenObject *o; + o = (PyAsyncGenObject *)gen_new_with_qualname( + &PyAsyncGen_Type, f, name, qualname); + if (o == NULL) { + return NULL; + } + o->ag_finalizer = NULL; + o->ag_closed = 0; + o->ag_hooks_inited = 0; + return (PyObject*)o; +} + + +int +PyAsyncGen_ClearFreeLists(void) +{ + int ret = ag_value_fl_free + ag_asend_fl_free; + + while (ag_value_fl_free) { + _PyAsyncGenWrappedValue *o; + o = ag_value_fl[--ag_value_fl_free]; + assert(_PyAsyncGenWrappedValue_CheckExact(o)); + PyObject_Del(o); + } + + while (ag_asend_fl_free) { + PyAsyncGenASend *o; + o = ag_asend_fl[--ag_asend_fl_free]; + assert(Py_TYPE(o) == &_PyAsyncGenASend_Type); + PyObject_Del(o); + } + + return ret; +} + +void +PyAsyncGen_Fini(void) +{ + PyAsyncGen_ClearFreeLists(); +} + + +static PyObject * +async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result) +{ + if (result == NULL) { + if (!PyErr_Occurred()) { + PyErr_SetNone(PyExc_StopAsyncIteration); + } + + if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration) + || PyErr_ExceptionMatches(PyExc_GeneratorExit) + ) { + gen->ag_closed = 1; + } + + return NULL; + } + + if (_PyAsyncGenWrappedValue_CheckExact(result)) { + /* async yield */ + PyObject *e = PyObject_CallFunctionObjArgs( + PyExc_StopIteration, + ((_PyAsyncGenWrappedValue*)result)->val, + NULL); + Py_DECREF(result); + PyErr_SetObject(PyExc_StopIteration, e); + Py_DECREF(e); + return NULL; + } + + return result; +} + + +/* ---------- Async Generator ASend Awaitable ------------ */ + + +static void +async_gen_asend_dealloc(PyAsyncGenASend *o) +{ + Py_CLEAR(o->aw_gen); + Py_CLEAR(o->aw_sendval); + if (ag_asend_fl_free < _PyAsyncGen_MAXFREELIST) { + assert(PyAsyncGenASend_CheckExact(o)); + ag_asend_fl[ag_asend_fl_free++] = o; + } else { + PyObject_Del(o); + } +} + + +static PyObject * +async_gen_asend_send(PyAsyncGenASend *o, PyObject *arg) +{ + PyObject *result; + + if (o->aw_state == AWAITABLE_STATE_CLOSED) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + + if (o->aw_state == AWAITABLE_STATE_INIT) { + if (arg == NULL || arg == Py_None) { + arg = o->aw_sendval; + } + o->aw_state = AWAITABLE_STATE_ITER; + } + + result = gen_send_ex((PyGenObject*)o->aw_gen, arg, 0, 0); + result = async_gen_unwrap_value(o->aw_gen, result); + + if (result == NULL) { + o->aw_state = AWAITABLE_STATE_CLOSED; + } + + return result; +} + + +static PyObject * +async_gen_asend_iternext(PyAsyncGenASend *o) +{ + return async_gen_asend_send(o, NULL); +} + + +static PyObject * +async_gen_asend_throw(PyAsyncGenASend *o, PyObject *args) +{ + PyObject *result; + + if (o->aw_state == AWAITABLE_STATE_CLOSED) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + + result = gen_throw((PyGenObject*)o->aw_gen, args); + result = async_gen_unwrap_value(o->aw_gen, result); + + if (result == NULL) { + o->aw_state = AWAITABLE_STATE_CLOSED; + } + + return result; +} + + +static PyObject * +async_gen_asend_close(PyAsyncGenASend *o, PyObject *args) +{ + o->aw_state = AWAITABLE_STATE_CLOSED; + Py_RETURN_NONE; +} + + +static PyMethodDef async_gen_asend_methods[] = { + {"send", (PyCFunction)async_gen_asend_send, METH_O, send_doc}, + {"throw", (PyCFunction)async_gen_asend_throw, METH_VARARGS, throw_doc}, + {"close", (PyCFunction)async_gen_asend_close, METH_NOARGS, close_doc}, + {NULL, NULL} /* Sentinel */ +}; + + +static PyAsyncMethods async_gen_asend_as_async = { + PyObject_SelfIter, /* am_await */ + 0, /* am_aiter */ + 0 /* am_anext */ +}; + + +PyTypeObject _PyAsyncGenASend_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "async_generator_asend", /* tp_name */ + sizeof(PyAsyncGenASend), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)async_gen_asend_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + &async_gen_asend_as_async, /* tp_as_async */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)async_gen_asend_iternext, /* tp_iternext */ + async_gen_asend_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + + +static PyObject * +async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval) +{ + PyAsyncGenASend *o; + if (ag_asend_fl_free) { + ag_asend_fl_free--; + o = ag_asend_fl[ag_asend_fl_free]; + _Py_NewReference((PyObject *)o); + } else { + o = PyObject_New(PyAsyncGenASend, &_PyAsyncGenASend_Type); + if (o == NULL) { + return NULL; + } + } + o->aw_gen = gen; + o->aw_state = AWAITABLE_STATE_INIT; + o->aw_sendval = sendval; + Py_XINCREF(sendval); + Py_INCREF(gen); + return (PyObject*)o; +} + + +/* ---------- Async Generator Value Wrapper ------------ */ + + +static void +async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o) +{ + Py_CLEAR(o->val); + if (ag_value_fl_free < _PyAsyncGen_MAXFREELIST) { + assert(_PyAsyncGenWrappedValue_CheckExact(o)); + ag_value_fl[ag_value_fl_free++] = o; + } else { + PyObject_Del(o); + } +} + + +PyTypeObject _PyAsyncGenWrappedValue_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "async_generator_wrapped_value", /* tp_name */ + sizeof(_PyAsyncGenWrappedValue), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)async_gen_wrapped_val_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_as_async */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + + +PyObject * +_PyAsyncGenWrapValue(PyObject *val) +{ + _PyAsyncGenWrappedValue *o; + assert(val); + + if (ag_value_fl_free) { + ag_value_fl_free--; + o = ag_value_fl[ag_value_fl_free]; + assert(_PyAsyncGenWrappedValue_CheckExact(o)); + _Py_NewReference((PyObject*)o); + } else { + o = PyObject_New(_PyAsyncGenWrappedValue, &_PyAsyncGenWrappedValue_Type); + if (o == NULL) { + return NULL; + } + } + o->val = val; + Py_INCREF(val); + return (PyObject*)o; +} + + +/* ---------- Async Generator AThrow awaitable ------------ */ + + +static void +async_gen_athrow_dealloc(PyAsyncGenAThrow *o) +{ + Py_CLEAR(o->ac_gen); + Py_CLEAR(o->ac_args); + PyObject_Del(o); +} + + +static PyObject * +async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) +{ + PyGenObject *gen = (PyGenObject*)o->ac_gen; + PyFrameObject *f = gen->gi_frame; + PyObject *retval; + + if (f == NULL || f->f_stacktop == NULL || + o->ac_state == AWAITABLE_STATE_CLOSED) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + + if (o->ac_state == AWAITABLE_STATE_INIT) { + if (o->ac_gen->ag_closed) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + + if (arg != Py_None) { + PyErr_SetString(PyExc_RuntimeError, NON_INIT_CORO_MSG); + return NULL; + } + + o->ac_state = AWAITABLE_STATE_ITER; + + if (o->ac_args == NULL) { + /* aclose() mode */ + o->ac_gen->ag_closed = 1; + + retval = _gen_throw((PyGenObject *)gen, + 0, /* Do not close generator when + PyExc_GeneratorExit is passed */ + PyExc_GeneratorExit, NULL, NULL); + + if (retval && _PyAsyncGenWrappedValue_CheckExact(retval)) { + Py_DECREF(retval); + goto yield_close; + } + } else { + PyObject *typ; + PyObject *tb = NULL; + PyObject *val = NULL; + + if (!PyArg_UnpackTuple(o->ac_args, "athrow", 1, 3, + &typ, &val, &tb)) { + return NULL; + } + + retval = _gen_throw((PyGenObject *)gen, + 0, /* Do not close generator when + PyExc_GeneratorExit is passed */ + typ, val, tb); + retval = async_gen_unwrap_value(o->ac_gen, retval); + } + if (retval == NULL) { + goto check_error; + } + return retval; + } + + if (o->ac_state == AWAITABLE_STATE_ITER) { + PyObject *retval = gen_send_ex((PyGenObject *)gen, arg, 0, 0); + if (o->ac_args) { + return async_gen_unwrap_value(o->ac_gen, retval); + } else { + /* aclose() mode */ + if (retval && _PyAsyncGenWrappedValue_CheckExact(retval)) { + Py_DECREF(retval); + goto yield_close; + } + if (retval == NULL) { + goto check_error; + } + return retval; + } + } + + return NULL; + +yield_close: + PyErr_SetString( + PyExc_RuntimeError, ASYNC_GEN_IGNORED_EXIT_MSG); + return NULL; + +check_error: + if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration) + || PyErr_ExceptionMatches(PyExc_GeneratorExit) + ) { + o->ac_state = AWAITABLE_STATE_CLOSED; + PyErr_Clear(); /* ignore these errors */ + PyErr_SetNone(PyExc_StopIteration); + } + return NULL; +} + + +static PyObject * +async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *args) +{ + PyObject *retval; + + if (o->ac_state == AWAITABLE_STATE_INIT) { + PyErr_SetString(PyExc_RuntimeError, NON_INIT_CORO_MSG); + return NULL; + } + + if (o->ac_state == AWAITABLE_STATE_CLOSED) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + + retval = gen_throw((PyGenObject*)o->ac_gen, args); + if (o->ac_args) { + return async_gen_unwrap_value(o->ac_gen, retval); + } else { + /* aclose() mode */ + if (retval && _PyAsyncGenWrappedValue_CheckExact(retval)) { + Py_DECREF(retval); + PyErr_SetString(PyExc_RuntimeError, ASYNC_GEN_IGNORED_EXIT_MSG); + return NULL; + } + return retval; + } +} + + +static PyObject * +async_gen_athrow_iternext(PyAsyncGenAThrow *o) +{ + return async_gen_athrow_send(o, Py_None); +} + + +static PyObject * +async_gen_athrow_close(PyAsyncGenAThrow *o, PyObject *args) +{ + o->ac_state = AWAITABLE_STATE_CLOSED; + Py_RETURN_NONE; +} + + +static PyMethodDef async_gen_athrow_methods[] = { + {"send", (PyCFunction)async_gen_athrow_send, METH_O, send_doc}, + {"throw", (PyCFunction)async_gen_athrow_throw, METH_VARARGS, throw_doc}, + {"close", (PyCFunction)async_gen_athrow_close, METH_NOARGS, close_doc}, + {NULL, NULL} /* Sentinel */ +}; + + +static PyAsyncMethods async_gen_athrow_as_async = { + PyObject_SelfIter, /* am_await */ + 0, /* am_aiter */ + 0 /* am_anext */ +}; + + +PyTypeObject _PyAsyncGenAThrow_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "async_generator_athrow", /* tp_name */ + sizeof(PyAsyncGenAThrow), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)async_gen_athrow_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + &async_gen_athrow_as_async, /* tp_as_async */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)async_gen_athrow_iternext, /* tp_iternext */ + async_gen_athrow_methods, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + + +static PyObject * +async_gen_athrow_new(PyAsyncGenObject *gen, PyObject *args) +{ + PyAsyncGenAThrow *o; + o = PyObject_New(PyAsyncGenAThrow, &_PyAsyncGenAThrow_Type); + if (o == NULL) { + return NULL; + } + o->ac_gen = gen; + o->ac_args = args; + o->ac_state = AWAITABLE_STATE_INIT; + Py_INCREF(gen); + Py_XINCREF(args); + return (PyObject*)o; +} diff -r 47b4dbd451f5 Python/ceval.c --- a/Python/ceval.c Wed Sep 07 09:31:52 2016 -0700 +++ b/Python/ceval.c Wed Sep 07 10:28:40 2016 -0700 @@ -1204,7 +1204,7 @@ f->f_stacktop = NULL; /* remains NULL unless yield suspends frame */ f->f_executing = 1; - if (co->co_flags & (CO_GENERATOR | CO_COROUTINE)) { + if (co->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { if (!throwflag && f->f_exc_type != NULL && f->f_exc_type != Py_None) { /* We were in an except handler when we left, restore the exception state which was put aside @@ -2027,36 +2027,45 @@ PyObject *aiter = TOP(); PyTypeObject *type = Py_TYPE(aiter); - if (type->tp_as_async != NULL) - getter = type->tp_as_async->am_anext; - - if (getter != NULL) { - next_iter = (*getter)(aiter); - if (next_iter == NULL) { + if (PyAsyncGen_CheckExact(aiter)) { + awaitable = type->tp_as_async->am_anext(aiter); + if (awaitable == NULL) { goto error; } + } else { + if (type->tp_as_async != NULL){ + getter = type->tp_as_async->am_anext; + } + + if (getter != NULL) { + next_iter = (*getter)(aiter); + if (next_iter == NULL) { + goto error; + } + } + else { + PyErr_Format( + PyExc_TypeError, + "'async for' requires an iterator with " + "__anext__ method, got %.100s", + type->tp_name); + goto error; + } + + awaitable = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable == NULL) { + PyErr_Format( + PyExc_TypeError, + "'async for' received an invalid object " + "from __anext__: %.100s", + Py_TYPE(next_iter)->tp_name); + + Py_DECREF(next_iter); + goto error; + } else { + Py_DECREF(next_iter); + } } - else { - PyErr_Format( - PyExc_TypeError, - "'async for' requires an iterator with " - "__anext__ method, got %.100s", - type->tp_name); - goto error; - } - - awaitable = _PyCoro_GetAwaitableIter(next_iter); - if (awaitable == NULL) { - PyErr_Format( - PyExc_TypeError, - "'async for' received an invalid object " - "from __anext__: %.100s", - Py_TYPE(next_iter)->tp_name); - - Py_DECREF(next_iter); - goto error; - } else - Py_DECREF(next_iter); PUSH(awaitable); PREDICT(LOAD_CONST); @@ -2131,6 +2140,17 @@ TARGET(YIELD_VALUE) { retval = POP(); + + if (co->co_flags & CO_ASYNC_GENERATOR) { + PyObject *w = _PyAsyncGenWrapValue(retval); + Py_DECREF(retval); + if (w == NULL) { + retval = NULL; + goto error; + } + retval = w; + } + f->f_stacktop = stack_pointer; why = WHY_YIELD; goto fast_yield; @@ -3598,7 +3618,7 @@ assert((retval != NULL) ^ (PyErr_Occurred() != NULL)); fast_yield: - if (co->co_flags & (CO_GENERATOR | CO_COROUTINE)) { + if (co->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { /* The purpose of this block is to put aside the generator's exception state and restore that of the calling frame. If the current @@ -4042,8 +4062,8 @@ freevars[PyTuple_GET_SIZE(co->co_cellvars) + i] = o; } - /* Handle generator/coroutine */ - if (co->co_flags & (CO_GENERATOR | CO_COROUTINE)) { + /* Handle generator/coroutine/asyncronous generator */ + if (co->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { PyObject *gen; PyObject *coro_wrapper = tstate->coroutine_wrapper; int is_coro = co->co_flags & CO_COROUTINE; @@ -4068,6 +4088,8 @@ * and return that as the value. */ if (is_coro) { gen = PyCoro_New(f, name, qualname); + } else if (co->co_flags & CO_ASYNC_GENERATOR) { + gen = PyAsyncGen_New(f, name, qualname); } else { gen = PyGen_NewWithQualName(f, name, qualname); } @@ -4546,6 +4568,38 @@ return tstate->coroutine_wrapper; } +void +_PyEval_SetAsyncGenFirstiter(PyObject *firstiter) +{ + PyThreadState *tstate = PyThreadState_GET(); + + Py_XINCREF(firstiter); + Py_XSETREF(tstate->async_gen_firstiter, firstiter); +} + +PyObject * +_PyEval_GetAsyncGenFirstiter(void) +{ + PyThreadState *tstate = PyThreadState_GET(); + return tstate->async_gen_firstiter; +} + +void +_PyEval_SetAsyncGenFinalizer(PyObject *finalizer) +{ + PyThreadState *tstate = PyThreadState_GET(); + + Py_XINCREF(finalizer); + Py_XSETREF(tstate->async_gen_finalizer, finalizer); +} + +PyObject * +_PyEval_GetAsyncGenFinalizer(void) +{ + PyThreadState *tstate = PyThreadState_GET(); + return tstate->async_gen_finalizer; +} + PyObject * PyEval_GetBuiltins(void) { diff -r 47b4dbd451f5 Python/compile.c --- a/Python/compile.c Wed Sep 07 09:31:52 2016 -0700 +++ b/Python/compile.c Wed Sep 07 10:28:40 2016 -0700 @@ -1807,8 +1807,6 @@ return 0; } - if (is_async) - co->co_flags |= CO_COROUTINE; compiler_make_closure(c, co, funcflags, qualname); Py_DECREF(qualname); Py_DECREF(co); @@ -2722,6 +2720,9 @@ if (c->u->u_ste->ste_type != FunctionBlock) return compiler_error(c, "'return' outside function"); if (s->v.Return.value) { + if (c->u->u_ste->ste_coroutine && c->u->u_ste->ste_generator) + return compiler_error( + c, "'return' with value in async generator"); VISIT(c, expr, s->v.Return.value); } else @@ -4034,8 +4035,6 @@ case Yield_kind: if (c->u->u_ste->ste_type != FunctionBlock) return compiler_error(c, "'yield' outside function"); - if (c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION) - return compiler_error(c, "'yield' inside async function"); if (e->v.Yield.value) { VISIT(c, expr, e->v.Yield.value); } @@ -4779,8 +4778,12 @@ flags |= CO_NEWLOCALS | CO_OPTIMIZED; if (ste->ste_nested) flags |= CO_NESTED; - if (ste->ste_generator) + if (ste->ste_generator && !ste->ste_coroutine) flags |= CO_GENERATOR; + if (!ste->ste_generator && ste->ste_coroutine) + flags |= CO_COROUTINE; + if (ste->ste_generator && ste->ste_coroutine) + flags |= CO_ASYNC_GENERATOR; if (ste->ste_varargs) flags |= CO_VARARGS; if (ste->ste_varkeywords) diff -r 47b4dbd451f5 Python/pylifecycle.c --- a/Python/pylifecycle.c Wed Sep 07 09:31:52 2016 -0700 +++ b/Python/pylifecycle.c Wed Sep 07 10:28:40 2016 -0700 @@ -675,6 +675,7 @@ _PyGC_Fini(); _PyRandom_Fini(); _PyArg_Fini(); + PyAsyncGen_Fini(); /* Cleanup Unicode implementation */ _PyUnicode_Fini(); diff -r 47b4dbd451f5 Python/pystate.c --- a/Python/pystate.c Wed Sep 07 09:31:52 2016 -0700 +++ b/Python/pystate.c Wed Sep 07 10:28:40 2016 -0700 @@ -228,6 +228,9 @@ tstate->coroutine_wrapper = NULL; tstate->in_coroutine_wrapper = 0; + tstate->async_gen_firstiter = NULL; + tstate->async_gen_finalizer = NULL; + if (init) _PyThreadState_Init(tstate); @@ -407,6 +410,8 @@ Py_CLEAR(tstate->c_traceobj); Py_CLEAR(tstate->coroutine_wrapper); + Py_CLEAR(tstate->async_gen_firstiter); + Py_CLEAR(tstate->async_gen_finalizer); } diff -r 47b4dbd451f5 Python/symtable.c --- a/Python/symtable.c Wed Sep 07 09:31:52 2016 -0700 +++ b/Python/symtable.c Wed Sep 07 10:28:40 2016 -0700 @@ -63,6 +63,7 @@ ste->ste_nested = 1; ste->ste_child_free = 0; ste->ste_generator = 0; + ste->ste_coroutine = 0; ste->ste_returns_value = 0; ste->ste_needs_class_closure = 0; @@ -378,7 +379,7 @@ PyLong_AsLong(PyTuple_GET_ITEM(data, 2))); return 0; - } + } } PyErr_SetString(PyExc_RuntimeError, "BUG: internal directive bookkeeping broken"); @@ -1341,6 +1342,7 @@ FunctionBlock, (void *)s, s->lineno, s->col_offset)) VISIT_QUIT(st, 0); + st->st_cur->ste_coroutine = 1; VISIT(st, arguments, s->v.AsyncFunctionDef.args); VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body); if (!symtable_exit_block(st, s)) @@ -1436,7 +1438,7 @@ break; case Await_kind: VISIT(st, expr, e->v.Await.value); - st->st_cur->ste_generator = 1; + st->st_cur->ste_coroutine = 1; break; case Compare_kind: VISIT(st, expr, e->v.Compare.left); diff -r 47b4dbd451f5 Python/sysmodule.c --- a/Python/sysmodule.c Wed Sep 07 09:31:52 2016 -0700 +++ b/Python/sysmodule.c Wed Sep 07 10:28:40 2016 -0700 @@ -700,6 +700,113 @@ ); +static PyTypeObject AsyncGenHooksType; + +PyDoc_STRVAR(asyncgen_hooks_doc, +"asyncgen_hooks\n\ +\n\ +A struct sequence providing information about asynhronous\n\ +generators hooks. The attributes are read only."); + +static PyStructSequence_Field asyncgen_hooks_fields[] = { + {"firstiter", "Hook to intercept first iteration"}, + {"finalizer", "Hook to intercept finalization"}, + {0} +}; + +static PyStructSequence_Desc asyncgen_hooks_desc = { + "asyncgen_hooks", /* name */ + asyncgen_hooks_doc, /* doc */ + asyncgen_hooks_fields , /* fields */ + 2 +}; + + +static PyObject * +sys_set_asyncgen_hooks(PyObject *self, PyObject *args, PyObject *kw) +{ + static char *keywords[] = {"firstiter", "finalizer", NULL}; + PyObject *firstiter = NULL; + PyObject *finalizer = NULL; + + if (!PyArg_ParseTupleAndKeywords( + args, kw, "|OO", keywords, + &firstiter, &finalizer)) { + return NULL; + } + + if (finalizer && finalizer != Py_None) { + if (!PyCallable_Check(finalizer)) { + PyErr_Format(PyExc_TypeError, + "callable finalizer expected, got %.50s", + Py_TYPE(finalizer)->tp_name); + return NULL; + } + _PyEval_SetAsyncGenFinalizer(finalizer); + } + else if (finalizer == Py_None) { + _PyEval_SetAsyncGenFinalizer(NULL); + } + + if (firstiter && firstiter != Py_None) { + if (!PyCallable_Check(firstiter)) { + PyErr_Format(PyExc_TypeError, + "callable firstiter expected, got %.50s", + Py_TYPE(firstiter)->tp_name); + return NULL; + } + _PyEval_SetAsyncGenFirstiter(firstiter); + } + else if (firstiter == Py_None) { + _PyEval_SetAsyncGenFirstiter(NULL); + } + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(set_asyncgen_hooks_doc, +"set_asyncgen_hooks(*, firstiter=None, finalizer=None)\n\ +\n\ +Set a finalizer for async generators objects." +); + +static PyObject * +sys_get_asyncgen_hooks(PyObject *self, PyObject *args) +{ + PyObject *res; + PyObject *firstiter = _PyEval_GetAsyncGenFirstiter(); + PyObject *finalizer = _PyEval_GetAsyncGenFinalizer(); + + res = PyStructSequence_New(&AsyncGenHooksType); + if (res == NULL) { + return NULL; + } + + if (firstiter == NULL) { + firstiter = Py_None; + } + + if (finalizer == NULL) { + finalizer = Py_None; + } + + Py_INCREF(firstiter); + PyStructSequence_SET_ITEM(res, 0, firstiter); + + Py_INCREF(finalizer); + PyStructSequence_SET_ITEM(res, 1, finalizer); + + return res; +} + +PyDoc_STRVAR(get_asyncgen_hooks_doc, +"get_asyncgen_hooks()\n\ +\n\ +Return a namedtuple of installed asynchronous generators hooks \ +(firstiter, finalizer)." +); + + static PyTypeObject Hash_InfoType; PyDoc_STRVAR(hash_info_doc, @@ -1276,6 +1383,10 @@ set_coroutine_wrapper_doc}, {"get_coroutine_wrapper", sys_get_coroutine_wrapper, METH_NOARGS, get_coroutine_wrapper_doc}, + {"set_asyncgen_hooks", sys_set_asyncgen_hooks, + METH_VARARGS | METH_KEYWORDS, set_asyncgen_hooks_doc}, + {"get_asyncgen_hooks", sys_get_asyncgen_hooks, METH_NOARGS, + get_asyncgen_hooks_doc}, {NULL, NULL} /* sentinel */ }; @@ -1904,6 +2015,14 @@ SET_SYS_FROM_STRING("thread_info", PyThread_GetInfo()); #endif + /* initialize asyncgen_hooks */ + if (AsyncGenHooksType.tp_name == NULL) { + if (PyStructSequence_InitType2( + &AsyncGenHooksType, &asyncgen_hooks_desc) < 0) { + return NULL; + } + } + #undef SET_SYS_FROM_STRING #undef SET_SYS_FROM_STRING_BORROW if (PyErr_Occurred())