diff -r 2fcfa9badfcd Doc/glossary.rst --- a/Doc/glossary.rst Wed Jun 08 13:32:49 2016 +0000 +++ b/Doc/glossary.rst Wed Jun 08 11:34:43 2016 -0400 @@ -76,13 +76,12 @@ asynchronous iterable An object, that can be used in an :keyword:`async for` statement. - Must return an :term:`awaitable` from its :meth:`__aiter__` method, - which should in turn be resolved in an :term:`asynchronous iterator` - object. Introduced by :pep:`492`. + Must return an :term:`asyncronous iterator` from its + :meth:`__aiter__` method. Introduced by :pep:`492`. asynchronous iterator An object that implements :meth:`__aiter__` and :meth:`__anext__` - methods, that must return :term:`awaitable` objects. + methods. ``__anext__`` must return an :term:`awaitable` object. :keyword:`async for` resolves awaitable returned from asynchronous iterator's :meth:`__anext__` method until it raises :exc:`StopAsyncIteration` exception. Introduced by :pep:`492`. diff -r 2fcfa9badfcd Doc/reference/compound_stmts.rst --- a/Doc/reference/compound_stmts.rst Wed Jun 08 13:32:49 2016 +0000 +++ b/Doc/reference/compound_stmts.rst Wed Jun 08 11:34:43 2016 -0400 @@ -726,7 +726,7 @@ Is semantically equivalent to:: iter = (ITER) - iter = await type(iter).__aiter__(iter) + iter = type(iter).__aiter__(iter) running = True while running: try: diff -r 2fcfa9badfcd Doc/reference/datamodel.rst --- a/Doc/reference/datamodel.rst Wed Jun 08 13:32:49 2016 +0000 +++ b/Doc/reference/datamodel.rst Wed Jun 08 11:34:43 2016 -0400 @@ -2364,6 +2364,49 @@ Asynchronous Iterators ---------------------- +.. note:: + + .. versionchanged:: 3.5.2 + Starting with CPython 3.5.2, ``__aiter__`` can directly return + :term:`asynchronous iterators `. Returning + an :term:`awaitable` object will result in a + :exc:`PendingDeprecationWarning`. + + The recommended way of writing backwards compatible code in + CPython 3.5.x is to continue returning awaitables from + ``__aiter__``. If you want to avoid the PendingDeprecationWarning + and keep the code backwards compatible, the following decorator + can be used:: + + import functools + import sys + + if sys.version_info < (3, 5, 2): + def aiter_compat(func): + @functools.wraps(func) + async def wrapper(self): + return func(self) + return wrapper + else: + def aiter_compat(func): + return func + + Example:: + + class AsyncIterator: + + @aiter_compat + def __aiter__(self): + return self + + async def __anext__(self): + ... + + Starting with CPython 3.6, the :exc:`PendingDeprecationWarning` + will be replaced with the :exc:`DeprecationWarning`. + In CPython 3.7, returning an awaitable from ``__aiter__`` will + result in a :exc:`RuntimeError`. + An *asynchronous iterable* is able to call asynchronous code in its ``__aiter__`` implementation, and an *asynchronous iterator* can call asynchronous code in its ``__anext__`` method. @@ -2372,7 +2415,7 @@ .. method:: object.__aiter__(self) - Must return an *awaitable* resulting in an *asynchronous iterator* object. + Must return an *asynchronous iterator* object. .. method:: object.__anext__(self) @@ -2385,7 +2428,7 @@ async def readline(self): ... - async def __aiter__(self): + def __aiter__(self): return self async def __anext__(self): diff -r 2fcfa9badfcd Include/genobject.h --- a/Include/genobject.h Wed Jun 08 13:32:49 2016 +0000 +++ b/Include/genobject.h Wed Jun 08 11:34:43 2016 -0400 @@ -54,6 +54,9 @@ PyAPI_DATA(PyTypeObject) PyCoro_Type; PyAPI_DATA(PyTypeObject) _PyCoroWrapper_Type; +PyAPI_DATA(PyTypeObject) _PyAIterWrapper_Type; +PyObject *_PyAIterWrapper_New(PyObject *aiter); + #define PyCoro_CheckExact(op) (Py_TYPE(op) == &PyCoro_Type) PyObject *_PyCoro_GetAwaitableIter(PyObject *o); PyAPI_FUNC(PyObject *) PyCoro_New(struct _frame *, diff -r 2fcfa9badfcd Lib/_collections_abc.py --- a/Lib/_collections_abc.py Wed Jun 08 13:32:49 2016 +0000 +++ b/Lib/_collections_abc.py Wed Jun 08 11:34:43 2016 -0400 @@ -156,7 +156,7 @@ __slots__ = () @abstractmethod - async def __aiter__(self): + def __aiter__(self): return AsyncIterator() @classmethod @@ -176,7 +176,7 @@ """Return the next item or raise StopAsyncIteration when exhausted.""" raise StopAsyncIteration - async def __aiter__(self): + def __aiter__(self): return self @classmethod diff -r 2fcfa9badfcd Lib/asyncio/compat.py --- a/Lib/asyncio/compat.py Wed Jun 08 13:32:49 2016 +0000 +++ b/Lib/asyncio/compat.py Wed Jun 08 11:34:43 2016 -0400 @@ -4,6 +4,7 @@ PY34 = sys.version_info >= (3, 4) PY35 = sys.version_info >= (3, 5) +PY352 = sys.version_info >= (3, 5, 2) def flatten_list_bytes(list_of_data): diff -r 2fcfa9badfcd Lib/asyncio/streams.py --- a/Lib/asyncio/streams.py Wed Jun 08 13:32:49 2016 +0000 +++ b/Lib/asyncio/streams.py Wed Jun 08 11:34:43 2016 -0400 @@ -689,3 +689,9 @@ if val == b'': raise StopAsyncIteration return val + + if compat.PY352: + # In Python 3.5.2 and greater, __aiter__ should return + # the asynchronous iterator directly. + def __aiter__(self): + return self diff -r 2fcfa9badfcd Lib/test/test_coroutines.py --- a/Lib/test/test_coroutines.py Wed Jun 08 13:32:49 2016 +0000 +++ b/Lib/test/test_coroutines.py Wed Jun 08 11:34:43 2016 -0400 @@ -1255,8 +1255,9 @@ buffer = [] async def test1(): - async for i1, i2 in AsyncIter(): - buffer.append(i1 + i2) + with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"): + async for i1, i2 in AsyncIter(): + buffer.append(i1 + i2) yielded, _ = run_async(test1()) # Make sure that __aiter__ was called only once @@ -1268,12 +1269,13 @@ buffer = [] async def test2(): nonlocal buffer - async for i in AsyncIter(): - buffer.append(i[0]) - if i[0] == 20: - break - else: - buffer.append('what?') + with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"): + async for i in AsyncIter(): + buffer.append(i[0]) + if i[0] == 20: + break + else: + buffer.append('what?') buffer.append('end') yielded, _ = run_async(test2()) @@ -1286,12 +1288,13 @@ buffer = [] async def test3(): nonlocal buffer - async for i in AsyncIter(): - if i[0] > 20: - continue - buffer.append(i[0]) - else: - buffer.append('what?') + with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"): + async for i in AsyncIter(): + if i[0] > 20: + continue + buffer.append(i[0]) + else: + buffer.append('what?') buffer.append('end') yielded, _ = run_async(test3()) @@ -1338,7 +1341,7 @@ def test_for_4(self): class I: - async def __aiter__(self): + def __aiter__(self): return self def __anext__(self): @@ -1368,8 +1371,9 @@ return 123 async def foo(): - async for i in I(): - print('never going to happen') + with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"): + async for i in I(): + print('never going to happen') with self.assertRaisesRegex( TypeError, @@ -1393,7 +1397,7 @@ def __init__(self): self.i = 0 - async def __aiter__(self): + def __aiter__(self): return self async def __anext__(self): @@ -1417,7 +1421,11 @@ I += 1 I += 1000 - run_async(main()) + with warnings.catch_warnings(): + warnings.simplefilter("error") + # Test that __aiter__ that returns an asyncronous iterator + # directly does not throw any warnings. + run_async(main()) self.assertEqual(I, 111011) self.assertEqual(sys.getrefcount(manager), mrefs_before) @@ -1472,13 +1480,63 @@ 1/0 async def foo(): nonlocal CNT + with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"): + async for i in AI(): + CNT += 1 + CNT += 10 + with self.assertRaises(ZeroDivisionError): + run_async(foo()) + self.assertEqual(CNT, 0) + + def test_for_8(self): + CNT = 0 + class AI: + def __aiter__(self): + 1/0 + async def foo(): + nonlocal CNT async for i in AI(): CNT += 1 CNT += 10 with self.assertRaises(ZeroDivisionError): - run_async(foo()) + with warnings.catch_warnings(): + warnings.simplefilter("error") + # Test that if __aiter__ raises an exception it propagates + # without any kind of warning. + run_async(foo()) self.assertEqual(CNT, 0) + def test_for_9(self): + # Test that PendingDeprecationWarning can safely be converted into + # an exception (__aiter__ should not have a chance to raise + # a ZeroDivisionError.) + class AI: + async def __aiter__(self): + 1/0 + async def foo(): + async for i in AI(): + pass + + with self.assertRaises(PendingDeprecationWarning): + with warnings.catch_warnings(): + warnings.simplefilter("error") + run_async(foo()) + + def test_for_10(self): + # Test that PendingDeprecationWarning can safely be converted into + # an exception. + class AI: + async def __aiter__(self): + pass + async def foo(): + async for i in AI(): + pass + + with self.assertRaises(PendingDeprecationWarning): + with warnings.catch_warnings(): + warnings.simplefilter("error") + run_async(foo()) + def test_copy(self): async def func(): pass coro = func() diff -r 2fcfa9badfcd Lib/test/test_grammar.py --- a/Lib/test/test_grammar.py Wed Jun 08 13:32:49 2016 +0000 +++ b/Lib/test/test_grammar.py Wed Jun 08 11:34:43 2016 -0400 @@ -1114,7 +1114,7 @@ class Done(Exception): pass class AIter: - async def __aiter__(self): + def __aiter__(self): return self async def __anext__(self): raise StopAsyncIteration diff -r 2fcfa9badfcd Objects/genobject.c --- a/Objects/genobject.c Wed Jun 08 13:32:49 2016 +0000 +++ b/Objects/genobject.c Wed Jun 08 11:34:43 2016 -0400 @@ -984,3 +984,97 @@ { return gen_new_with_qualname(&PyCoro_Type, f, name, qualname); } + + +/* __aiter__ wrapper; see http://bugs.python.org/issue27243 for details. */ + +typedef struct { + PyObject_HEAD + PyObject *aw_aiter; +} PyAIterWrapper; + + +static PyObject * +aiter_wrapper_iternext(PyAIterWrapper *aw) +{ + PyErr_SetObject(PyExc_StopIteration, aw->aw_aiter); + return NULL; +} + +static int +aiter_wrapper_traverse(PyAIterWrapper *aw, visitproc visit, void *arg) +{ + Py_VISIT((PyObject *)aw->aw_aiter); + return 0; +} + +static void +aiter_wrapper_dealloc(PyAIterWrapper *aw) +{ + _PyObject_GC_UNTRACK((PyObject *)aw); + Py_CLEAR(aw->aw_aiter); + PyObject_GC_Del(aw); +} + +static PyAsyncMethods aiter_wrapper_as_async = { + PyObject_SelfIter, /* am_await */ + 0, /* am_aiter */ + 0 /* am_anext */ +}; + +PyTypeObject _PyAIterWrapper_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "aiter_wrapper", + sizeof(PyAIterWrapper), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)aiter_wrapper_dealloc, /* destructor tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + &aiter_wrapper_as_async, /* tp_as_async */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + "A wrapper object for __aiter__ bakwards compatibility.", + (traverseproc)aiter_wrapper_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)aiter_wrapper_iternext, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + PyObject_Del, /* tp_free */ +}; + + +PyObject * +_PyAIterWrapper_New(PyObject *aiter) +{ + PyAIterWrapper *aw = PyObject_GC_New(PyAIterWrapper, + &_PyAIterWrapper_Type); + if (aw == NULL) { + return NULL; + } + Py_INCREF(aiter); + aw->aw_aiter = aiter; + _PyObject_GC_TRACK(aw); + return (PyObject *)aw; +} diff -r 2fcfa9badfcd Python/ceval.c --- a/Python/ceval.c Wed Jun 08 13:32:49 2016 +0000 +++ b/Python/ceval.c Wed Jun 08 11:34:43 2016 -0400 @@ -1935,8 +1935,9 @@ PyObject *obj = TOP(); PyTypeObject *type = Py_TYPE(obj); - if (type->tp_as_async != NULL) + if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; + } if (getter != NULL) { iter = (*getter)(obj); @@ -1957,6 +1958,27 @@ goto error; } + if (Py_TYPE(iter)->tp_as_async != NULL && + Py_TYPE(iter)->tp_as_async->am_anext != NULL) { + + /* Starting with CPython 3.5.2 __aiter__ should return + asynchronous iterators directly (not awaitables that + resolve to asynchronous generators.) + + Therefore, we check if the object that was returned + from __aiter__ has an __anext__ method. If it does, + we wrap it in an awaitable that resolves to `iter`. + + See http://bugs.python.org/issue27243 for more + details. + */ + + PyObject *wrapper = _PyAIterWrapper_New(iter); + Py_DECREF(iter); + SET_TOP(wrapper); + DISPATCH(); + } + awaitable = _PyCoro_GetAwaitableIter(iter); if (awaitable == NULL) { SET_TOP(NULL); @@ -1968,9 +1990,23 @@ Py_DECREF(iter); goto error; - } else + } else { Py_DECREF(iter); + if (PyErr_WarnFormat( + PyExc_PendingDeprecationWarning, 1, + "'%.100s' implements legacy __aiter__ protocol; " + "__aiter__ should return an asynchronous " + "iterator, not awaitable", + type->tp_name)) + { + /* Warning was converted to an error. */ + Py_DECREF(awaitable); + SET_TOP(NULL); + goto error; + } + } + SET_TOP(awaitable); DISPATCH(); }