From a6f6edbda8648698289a8ee7abef6a35c924151b Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 9 Jun 2016 15:08:31 -0400 Subject: [PATCH] Issue #27243: Fix __aiter__ protocol --- Doc/glossary.rst | 7 +-- Doc/reference/compound_stmts.rst | 2 +- Doc/reference/datamodel.rst | 48 ++++++++++++++- Doc/whatsnew/3.5.rst | 13 ++++ Include/genobject.h | 3 + Lib/_collections_abc.py | 4 +- Lib/asyncio/compat.py | 1 + Lib/asyncio/streams.py | 6 ++ Lib/test/test_coroutines.py | 100 ++++++++++++++++++++++++------- Lib/test/test_grammar.py | 2 +- Misc/NEWS | 5 ++ Objects/genobject.c | 94 +++++++++++++++++++++++++++++ Python/ceval.c | 40 ++++++++++++- 13 files changed, 292 insertions(+), 33 deletions(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 75b380b1c39..e7bcb6aecb7 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -76,13 +76,12 @@ Glossary asynchronous iterable An object, that can be used in an :keyword:`async for` statement. - Must return an :term:`awaitable` from its :meth:`__aiter__` method, - which should in turn be resolved in an :term:`asynchronous iterator` - object. Introduced by :pep:`492`. + Must return an :term:`asyncronous iterator` from its + :meth:`__aiter__` method. Introduced by :pep:`492`. asynchronous iterator An object that implements :meth:`__aiter__` and :meth:`__anext__` - methods, that must return :term:`awaitable` objects. + methods. ``__anext__`` must return an :term:`awaitable` object. :keyword:`async for` resolves awaitable returned from asynchronous iterator's :meth:`__anext__` method until it raises :exc:`StopAsyncIteration` exception. Introduced by :pep:`492`. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 8047673e190..24694225659 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -726,7 +726,7 @@ The following code:: Is semantically equivalent to:: iter = (ITER) - iter = await type(iter).__aiter__(iter) + iter = type(iter).__aiter__(iter) running = True while running: try: diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 3ddbd622d87..493acaaa494 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2359,6 +2359,7 @@ generators, coroutines do not directly support iteration. Coroutine objects are automatically closed using the above process when they are about to be destroyed. +.. _async-iterators: Asynchronous Iterators ---------------------- @@ -2371,7 +2372,7 @@ Asynchronous iterators can be used in an :keyword:`async for` statement. .. method:: object.__aiter__(self) - Must return an *awaitable* resulting in an *asynchronous iterator* object. + Must return an *asynchronous iterator* object. .. method:: object.__anext__(self) @@ -2384,7 +2385,7 @@ An example of an asynchronous iterable object:: async def readline(self): ... - async def __aiter__(self): + def __aiter__(self): return self async def __anext__(self): @@ -2395,6 +2396,49 @@ An example of an asynchronous iterable object:: .. versionadded:: 3.5 +.. note:: + + .. versionchanged:: 3.5.2 + Starting with CPython 3.5.2, ``__aiter__`` can directly return + :term:`asynchronous iterators `. Returning + an :term:`awaitable` object will result in a + :exc:`PendingDeprecationWarning`. + + The recommended way of writing backwards compatible code in + CPython 3.5.x is to continue returning awaitables from + ``__aiter__``. If you want to avoid the PendingDeprecationWarning + and keep the code backwards compatible, the following decorator + can be used:: + + import functools + import sys + + if sys.version_info < (3, 5, 2): + def aiter_compat(func): + @functools.wraps(func) + async def wrapper(self): + return func(self) + return wrapper + else: + def aiter_compat(func): + return func + + Example:: + + class AsyncIterator: + + @aiter_compat + def __aiter__(self): + return self + + async def __anext__(self): + ... + + Starting with CPython 3.6, the :exc:`PendingDeprecationWarning` + will be replaced with the :exc:`DeprecationWarning`. + In CPython 3.7, returning an awaitable from ``__aiter__`` will + result in a :exc:`RuntimeError`. + Asynchronous Context Managers ----------------------------- diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index 83d5ce694cc..2d7f8a4266f 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -247,6 +247,19 @@ be used inside a coroutine function declared with :keyword:`async def`. Coroutine functions are intended to be run inside a compatible event loop, such as the :ref:`asyncio loop `. + +.. note:: + + .. versionchanged:: 3.5.2 + Starting with CPython 3.5.2, ``__aiter__`` can directly return + :term:`asynchronous iterators `. Returning + an :term:`awaitable` object will result in a + :exc:`PendingDeprecationWarning`. + + See more details in the :ref:`async-iterators` documentation + section. + + .. seealso:: :pep:`492` -- Coroutines with async and await syntax diff --git a/Include/genobject.h b/Include/genobject.h index 30cb0232344..1ff32a8eafa 100644 --- a/Include/genobject.h +++ b/Include/genobject.h @@ -54,6 +54,9 @@ typedef struct { PyAPI_DATA(PyTypeObject) PyCoro_Type; PyAPI_DATA(PyTypeObject) _PyCoroWrapper_Type; +PyAPI_DATA(PyTypeObject) _PyAIterWrapper_Type; +PyObject *_PyAIterWrapper_New(PyObject *aiter); + #define PyCoro_CheckExact(op) (Py_TYPE(op) == &PyCoro_Type) PyObject *_PyCoro_GetAwaitableIter(PyObject *o); PyAPI_FUNC(PyObject *) PyCoro_New(struct _frame *, diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index f89bb6f04b5..fc9c9f1cc14 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -156,7 +156,7 @@ class AsyncIterable(metaclass=ABCMeta): __slots__ = () @abstractmethod - async def __aiter__(self): + def __aiter__(self): return AsyncIterator() @classmethod @@ -176,7 +176,7 @@ class AsyncIterator(AsyncIterable): """Return the next item or raise StopAsyncIteration when exhausted.""" raise StopAsyncIteration - async def __aiter__(self): + def __aiter__(self): return self @classmethod diff --git a/Lib/asyncio/compat.py b/Lib/asyncio/compat.py index 660b7e7e6c9..4790bb4a35f 100644 --- a/Lib/asyncio/compat.py +++ b/Lib/asyncio/compat.py @@ -4,6 +4,7 @@ import sys PY34 = sys.version_info >= (3, 4) PY35 = sys.version_info >= (3, 5) +PY352 = sys.version_info >= (3, 5, 2) def flatten_list_bytes(list_of_data): diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 6f465afde2e..c88a87cd096 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -689,3 +689,9 @@ class StreamReader: if val == b'': raise StopAsyncIteration return val + + if compat.PY352: + # In Python 3.5.2 and greater, __aiter__ should return + # the asynchronous iterator directly. + def __aiter__(self): + return self diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 187348d48e1..4f725aeab2b 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1255,8 +1255,9 @@ class CoroutineTest(unittest.TestCase): buffer = [] async def test1(): - async for i1, i2 in AsyncIter(): - buffer.append(i1 + i2) + with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"): + async for i1, i2 in AsyncIter(): + buffer.append(i1 + i2) yielded, _ = run_async(test1()) # Make sure that __aiter__ was called only once @@ -1268,12 +1269,13 @@ class CoroutineTest(unittest.TestCase): buffer = [] async def test2(): nonlocal buffer - async for i in AsyncIter(): - buffer.append(i[0]) - if i[0] == 20: - break - else: - buffer.append('what?') + with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"): + async for i in AsyncIter(): + buffer.append(i[0]) + if i[0] == 20: + break + else: + buffer.append('what?') buffer.append('end') yielded, _ = run_async(test2()) @@ -1286,12 +1288,13 @@ class CoroutineTest(unittest.TestCase): buffer = [] async def test3(): nonlocal buffer - async for i in AsyncIter(): - if i[0] > 20: - continue - buffer.append(i[0]) - else: - buffer.append('what?') + with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"): + async for i in AsyncIter(): + if i[0] > 20: + continue + buffer.append(i[0]) + else: + buffer.append('what?') buffer.append('end') yielded, _ = run_async(test3()) @@ -1338,7 +1341,7 @@ class CoroutineTest(unittest.TestCase): def test_for_4(self): class I: - async def __aiter__(self): + def __aiter__(self): return self def __anext__(self): @@ -1368,8 +1371,9 @@ class CoroutineTest(unittest.TestCase): return 123 async def foo(): - async for i in I(): - print('never going to happen') + with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"): + async for i in I(): + print('never going to happen') with self.assertRaisesRegex( TypeError, @@ -1393,7 +1397,7 @@ class CoroutineTest(unittest.TestCase): def __init__(self): self.i = 0 - async def __aiter__(self): + def __aiter__(self): return self async def __anext__(self): @@ -1417,7 +1421,11 @@ class CoroutineTest(unittest.TestCase): I += 1 I += 1000 - run_async(main()) + with warnings.catch_warnings(): + warnings.simplefilter("error") + # Test that __aiter__ that returns an asyncronous iterator + # directly does not throw any warnings. + run_async(main()) self.assertEqual(I, 111011) self.assertEqual(sys.getrefcount(manager), mrefs_before) @@ -1472,13 +1480,63 @@ class CoroutineTest(unittest.TestCase): 1/0 async def foo(): nonlocal CNT - async for i in AI(): - CNT += 1 + with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"): + async for i in AI(): + CNT += 1 CNT += 10 with self.assertRaises(ZeroDivisionError): run_async(foo()) self.assertEqual(CNT, 0) + def test_for_8(self): + CNT = 0 + class AI: + def __aiter__(self): + 1/0 + async def foo(): + nonlocal CNT + async for i in AI(): + CNT += 1 + CNT += 10 + with self.assertRaises(ZeroDivisionError): + with warnings.catch_warnings(): + warnings.simplefilter("error") + # Test that if __aiter__ raises an exception it propagates + # without any kind of warning. + run_async(foo()) + self.assertEqual(CNT, 0) + + def test_for_9(self): + # Test that PendingDeprecationWarning can safely be converted into + # an exception (__aiter__ should not have a chance to raise + # a ZeroDivisionError.) + class AI: + async def __aiter__(self): + 1/0 + async def foo(): + async for i in AI(): + pass + + with self.assertRaises(PendingDeprecationWarning): + with warnings.catch_warnings(): + warnings.simplefilter("error") + run_async(foo()) + + def test_for_10(self): + # Test that PendingDeprecationWarning can safely be converted into + # an exception. + class AI: + async def __aiter__(self): + pass + async def foo(): + async for i in AI(): + pass + + with self.assertRaises(PendingDeprecationWarning): + with warnings.catch_warnings(): + warnings.simplefilter("error") + run_async(foo()) + def test_copy(self): async def func(): pass coro = func() diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index d68cc7da7ca..154e3b608cb 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -1076,7 +1076,7 @@ class GrammarTests(unittest.TestCase): class Done(Exception): pass class AIter: - async def __aiter__(self): + def __aiter__(self): return self async def __anext__(self): raise StopAsyncIteration diff --git a/Misc/NEWS b/Misc/NEWS index e16fa196da7..e7abe739081 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -130,6 +130,11 @@ Core and Builtins - Issue #25887: Raise a RuntimeError when a coroutine object is awaited more than once. +- Issue #27243: Update the __aiter__ protocol: instead of returning + an awaitable that resolves to an asynchronous iterator, the asynchronous + iterator should be returned directly. Doing the former will trigger a + PendingDeprecationWarning. + Library ------- diff --git a/Objects/genobject.c b/Objects/genobject.c index f74d044dcf7..b3e0a46e8b3 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -992,3 +992,97 @@ PyCoro_New(PyFrameObject *f, PyObject *name, PyObject *qualname) { return gen_new_with_qualname(&PyCoro_Type, f, name, qualname); } + + +/* __aiter__ wrapper; see http://bugs.python.org/issue27243 for details. */ + +typedef struct { + PyObject_HEAD + PyObject *aw_aiter; +} PyAIterWrapper; + + +static PyObject * +aiter_wrapper_iternext(PyAIterWrapper *aw) +{ + PyErr_SetObject(PyExc_StopIteration, aw->aw_aiter); + return NULL; +} + +static int +aiter_wrapper_traverse(PyAIterWrapper *aw, visitproc visit, void *arg) +{ + Py_VISIT((PyObject *)aw->aw_aiter); + return 0; +} + +static void +aiter_wrapper_dealloc(PyAIterWrapper *aw) +{ + _PyObject_GC_UNTRACK((PyObject *)aw); + Py_CLEAR(aw->aw_aiter); + PyObject_GC_Del(aw); +} + +static PyAsyncMethods aiter_wrapper_as_async = { + PyObject_SelfIter, /* am_await */ + 0, /* am_aiter */ + 0 /* am_anext */ +}; + +PyTypeObject _PyAIterWrapper_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "aiter_wrapper", + sizeof(PyAIterWrapper), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)aiter_wrapper_dealloc, /* destructor tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + &aiter_wrapper_as_async, /* tp_as_async */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + "A wrapper object for __aiter__ bakwards compatibility.", + (traverseproc)aiter_wrapper_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)aiter_wrapper_iternext, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + PyObject_Del, /* tp_free */ +}; + + +PyObject * +_PyAIterWrapper_New(PyObject *aiter) +{ + PyAIterWrapper *aw = PyObject_GC_New(PyAIterWrapper, + &_PyAIterWrapper_Type); + if (aw == NULL) { + return NULL; + } + Py_INCREF(aiter); + aw->aw_aiter = aiter; + _PyObject_GC_TRACK(aw); + return (PyObject *)aw; +} diff --git a/Python/ceval.c b/Python/ceval.c index 3758b0936ab..3d690384448 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1933,8 +1933,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) PyObject *obj = TOP(); PyTypeObject *type = Py_TYPE(obj); - if (type->tp_as_async != NULL) + if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; + } if (getter != NULL) { iter = (*getter)(obj); @@ -1955,6 +1956,27 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) goto error; } + if (Py_TYPE(iter)->tp_as_async != NULL && + Py_TYPE(iter)->tp_as_async->am_anext != NULL) { + + /* Starting with CPython 3.5.2 __aiter__ should return + asynchronous iterators directly (not awaitables that + resolve to asynchronous iterators.) + + Therefore, we check if the object that was returned + from __aiter__ has an __anext__ method. If it does, + we wrap it in an awaitable that resolves to `iter`. + + See http://bugs.python.org/issue27243 for more + details. + */ + + PyObject *wrapper = _PyAIterWrapper_New(iter); + Py_DECREF(iter); + SET_TOP(wrapper); + DISPATCH(); + } + awaitable = _PyCoro_GetAwaitableIter(iter); if (awaitable == NULL) { SET_TOP(NULL); @@ -1966,9 +1988,23 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_DECREF(iter); goto error; - } else + } else { Py_DECREF(iter); + if (PyErr_WarnFormat( + PyExc_PendingDeprecationWarning, 1, + "'%.100s' implements legacy __aiter__ protocol; " + "__aiter__ should return an asynchronous " + "iterator, not awaitable", + type->tp_name)) + { + /* Warning was converted to an error. */ + Py_DECREF(awaitable); + SET_TOP(NULL); + goto error; + } + } + SET_TOP(awaitable); DISPATCH(); }