Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Issue #27243: Fix __aiter__ protocol

This commit is contained in:
Yury Selivanov 2016-06-09 15:08:31 -04:00
parent ebe95fdabb
commit a6f6edbda8
13 changed files with 292 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Doc/glossary.rst
View File

@ -76,13 +76,12 @@ Glossary
asynchronous iterable
An object, that can be used in an :keyword:`async for` statement.
Must return an :term:`awaitable` from its :meth:`__aiter__` method,
which should in turn be resolved in an :term:`asynchronous iterator`
object. Introduced by :pep:`492`.
Must return an :term:`asyncronous iterator` from its
:meth:`__aiter__` method. Introduced by :pep:`492`.
asynchronous iterator
An object that implements :meth:`__aiter__` and :meth:`__anext__`
methods, that must return :term:`awaitable` objects.
methods. ``__anext__`` must return an :term:`awaitable` object.
:keyword:`async for` resolves awaitable returned from asynchronous
iterator's :meth:`__anext__` method until it raises
:exc:`StopAsyncIteration` exception. Introduced by :pep:`492`.

2
Doc/reference/compound_stmts.rst
View File

@ -726,7 +726,7 @@ The following code::
Is semantically equivalent to::
iter = (ITER)
iter = await type(iter).__aiter__(iter)
iter = type(iter).__aiter__(iter)
running = True
while running:
try:

48
Doc/reference/datamodel.rst
View File

@ -2359,6 +2359,7 @@ generators, coroutines do not directly support iteration.
Coroutine objects are automatically closed using the above process when
they are about to be destroyed.
.. _async-iterators:
Asynchronous Iterators
----------------------
@ -2371,7 +2372,7 @@ Asynchronous iterators can be used in an :keyword:`async for` statement.
.. method:: object.__aiter__(self)
Must return an *awaitable* resulting in an *asynchronous iterator* object.
Must return an *asynchronous iterator* object.
.. method:: object.__anext__(self)
@ -2384,7 +2385,7 @@ An example of an asynchronous iterable object::
async def readline(self):
...
async def __aiter__(self):
def __aiter__(self):
return self
async def __anext__(self):
@ -2395,6 +2396,49 @@ An example of an asynchronous iterable object::
.. versionadded:: 3.5
.. note::
.. versionchanged:: 3.5.2
Starting with CPython 3.5.2, ``__aiter__`` can directly return
:term:`asynchronous iterators <asynchronous iterator>`. Returning
an :term:`awaitable` object will result in a
:exc:`PendingDeprecationWarning`.
The recommended way of writing backwards compatible code in
CPython 3.5.x is to continue returning awaitables from
``__aiter__``. If you want to avoid the PendingDeprecationWarning
and keep the code backwards compatible, the following decorator
can be used::
import functools
import sys
if sys.version_info < (3, 5, 2):
def aiter_compat(func):
@functools.wraps(func)
async def wrapper(self):
return func(self)
return wrapper
else:
def aiter_compat(func):
return func
Example::
class AsyncIterator:
@aiter_compat
def __aiter__(self):
return self
async def __anext__(self):
...
Starting with CPython 3.6, the :exc:`PendingDeprecationWarning`
will be replaced with the :exc:`DeprecationWarning`.
In CPython 3.7, returning an awaitable from ``__aiter__`` will
result in a :exc:`RuntimeError`.
Asynchronous Context Managers
-----------------------------

13
Doc/whatsnew/3.5.rst
View File

@ -247,6 +247,19 @@ be used inside a coroutine function declared with :keyword:`async def`.
Coroutine functions are intended to be run inside a compatible event loop,
such as the :ref:`asyncio loop <asyncio-event-loop>`.
.. note::
.. versionchanged:: 3.5.2
Starting with CPython 3.5.2, ``__aiter__`` can directly return
:term:`asynchronous iterators <asynchronous iterator>`. Returning
an :term:`awaitable` object will result in a
:exc:`PendingDeprecationWarning`.
See more details in the :ref:`async-iterators` documentation
section.
.. seealso::
:pep:`492` -- Coroutines with async and await syntax

3
Include/genobject.h
View File

@ -54,6 +54,9 @@ typedef struct {
PyAPI_DATA(PyTypeObject) PyCoro_Type;
PyAPI_DATA(PyTypeObject) _PyCoroWrapper_Type;
PyAPI_DATA(PyTypeObject) _PyAIterWrapper_Type;
PyObject *_PyAIterWrapper_New(PyObject *aiter);
#define PyCoro_CheckExact(op) (Py_TYPE(op) == &PyCoro_Type)
PyObject *_PyCoro_GetAwaitableIter(PyObject *o);
PyAPI_FUNC(PyObject *) PyCoro_New(struct _frame *,

4
Lib/_collections_abc.py
View File

@ -156,7 +156,7 @@ class AsyncIterable(metaclass=ABCMeta):
__slots__ = ()
@abstractmethod
async def __aiter__(self):
def __aiter__(self):
return AsyncIterator()
@classmethod
@ -176,7 +176,7 @@ class AsyncIterator(AsyncIterable):
"""Return the next item or raise StopAsyncIteration when exhausted."""
raise StopAsyncIteration
async def __aiter__(self):
def __aiter__(self):
return self
@classmethod

1
Lib/asyncio/compat.py
View File

@ -4,6 +4,7 @@ import sys
PY34 = sys.version_info >= (3, 4)
PY35 = sys.version_info >= (3, 5)
PY352 = sys.version_info >= (3, 5, 2)
def flatten_list_bytes(list_of_data):

6
Lib/asyncio/streams.py
View File

@ -689,3 +689,9 @@ class StreamReader:
if val == b'':
raise StopAsyncIteration
return val
if compat.PY352:
# In Python 3.5.2 and greater, __aiter__ should return
# the asynchronous iterator directly.
def __aiter__(self):
return self

100
Lib/test/test_coroutines.py
View File

@ -1255,8 +1255,9 @@ class CoroutineTest(unittest.TestCase):
buffer = []
async def test1():
async for i1, i2 in AsyncIter():
buffer.append(i1 + i2)
with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
async for i1, i2 in AsyncIter():
buffer.append(i1 + i2)
yielded, _ = run_async(test1())
# Make sure that __aiter__ was called only once
@ -1268,12 +1269,13 @@ class CoroutineTest(unittest.TestCase):
buffer = []
async def test2():
nonlocal buffer
async for i in AsyncIter():
buffer.append(i[0])
if i[0] == 20:
break
else:
buffer.append('what?')
with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
async for i in AsyncIter():
buffer.append(i[0])
if i[0] == 20:
break
else:
buffer.append('what?')
buffer.append('end')
yielded, _ = run_async(test2())
@ -1286,12 +1288,13 @@ class CoroutineTest(unittest.TestCase):
buffer = []
async def test3():
nonlocal buffer
async for i in AsyncIter():
if i[0] > 20:
continue
buffer.append(i[0])
else:
buffer.append('what?')
with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
async for i in AsyncIter():
if i[0] > 20:
continue
buffer.append(i[0])
else:
buffer.append('what?')
buffer.append('end')
yielded, _ = run_async(test3())
@ -1338,7 +1341,7 @@ class CoroutineTest(unittest.TestCase):
def test_for_4(self):
class I:
async def __aiter__(self):
def __aiter__(self):
return self
def __anext__(self):
@ -1368,8 +1371,9 @@ class CoroutineTest(unittest.TestCase):
return 123
async def foo():
async for i in I():
print('never going to happen')
with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
async for i in I():
print('never going to happen')
with self.assertRaisesRegex(
TypeError,
@ -1393,7 +1397,7 @@ class CoroutineTest(unittest.TestCase):
def __init__(self):
self.i = 0
async def __aiter__(self):
def __aiter__(self):
return self
async def __anext__(self):
@ -1417,7 +1421,11 @@ class CoroutineTest(unittest.TestCase):
I += 1
I += 1000
run_async(main())
with warnings.catch_warnings():
warnings.simplefilter("error")
# Test that __aiter__ that returns an asyncronous iterator
# directly does not throw any warnings.
run_async(main())
self.assertEqual(I, 111011)
self.assertEqual(sys.getrefcount(manager), mrefs_before)
@ -1472,13 +1480,63 @@ class CoroutineTest(unittest.TestCase):
1/0
async def foo():
nonlocal CNT
async for i in AI():
CNT += 1
with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
async for i in AI():
CNT += 1
CNT += 10
with self.assertRaises(ZeroDivisionError):
run_async(foo())
self.assertEqual(CNT, 0)
def test_for_8(self):
CNT = 0
class AI:
def __aiter__(self):
1/0
async def foo():
nonlocal CNT
async for i in AI():
CNT += 1
CNT += 10
with self.assertRaises(ZeroDivisionError):
with warnings.catch_warnings():
warnings.simplefilter("error")
# Test that if __aiter__ raises an exception it propagates
# without any kind of warning.
run_async(foo())
self.assertEqual(CNT, 0)
def test_for_9(self):
# Test that PendingDeprecationWarning can safely be converted into
# an exception (__aiter__ should not have a chance to raise
# a ZeroDivisionError.)
class AI:
async def __aiter__(self):
1/0
async def foo():
async for i in AI():
pass
with self.assertRaises(PendingDeprecationWarning):
with warnings.catch_warnings():
warnings.simplefilter("error")
run_async(foo())
def test_for_10(self):
# Test that PendingDeprecationWarning can safely be converted into
# an exception.
class AI:
async def __aiter__(self):
pass
async def foo():
async for i in AI():
pass
with self.assertRaises(PendingDeprecationWarning):
with warnings.catch_warnings():
warnings.simplefilter("error")
run_async(foo())
def test_copy(self):
async def func(): pass
coro = func()

2
Lib/test/test_grammar.py
View File

@ -1076,7 +1076,7 @@ class GrammarTests(unittest.TestCase):
class Done(Exception): pass
class AIter:
async def __aiter__(self):
def __aiter__(self):
return self
async def __anext__(self):
raise StopAsyncIteration

5
Misc/NEWS
View File

@ -130,6 +130,11 @@ Core and Builtins
- Issue #25887: Raise a RuntimeError when a coroutine object is awaited
more than once.
- Issue #27243: Update the __aiter__ protocol: instead of returning
an awaitable that resolves to an asynchronous iterator, the asynchronous
iterator should be returned directly. Doing the former will trigger a
PendingDeprecationWarning.
Library
-------

94
Objects/genobject.c
View File

@ -992,3 +992,97 @@ PyCoro_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
{
return gen_new_with_qualname(&PyCoro_Type, f, name, qualname);
}
/* __aiter__ wrapper; see http://bugs.python.org/issue27243 for details. */
typedef struct {
PyObject_HEAD
PyObject *aw_aiter;
} PyAIterWrapper;
static PyObject *
aiter_wrapper_iternext(PyAIterWrapper *aw)
{
PyErr_SetObject(PyExc_StopIteration, aw->aw_aiter);
return NULL;
}
static int
aiter_wrapper_traverse(PyAIterWrapper *aw, visitproc visit, void *arg)
{
Py_VISIT((PyObject *)aw->aw_aiter);
return 0;
}
static void
aiter_wrapper_dealloc(PyAIterWrapper *aw)
{
_PyObject_GC_UNTRACK((PyObject *)aw);
Py_CLEAR(aw->aw_aiter);
PyObject_GC_Del(aw);
}
static PyAsyncMethods aiter_wrapper_as_async = {
PyObject_SelfIter, /* am_await */
0, /* am_aiter */
0 /* am_anext */
};
PyTypeObject _PyAIterWrapper_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"aiter_wrapper",
sizeof(PyAIterWrapper), /* tp_basicsize */
0, /* tp_itemsize */
(destructor)aiter_wrapper_dealloc, /* destructor tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
&aiter_wrapper_as_async, /* tp_as_async */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
PyObject_GenericGetAttr, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
"A wrapper object for __aiter__ bakwards compatibility.",
(traverseproc)aiter_wrapper_traverse, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
PyObject_SelfIter, /* tp_iter */
(iternextfunc)aiter_wrapper_iternext, /* tp_iternext */
0, /* tp_methods */
0, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
0, /* tp_init */
0, /* tp_alloc */
0, /* tp_new */
PyObject_Del, /* tp_free */
};
PyObject *
_PyAIterWrapper_New(PyObject *aiter)
{
PyAIterWrapper *aw = PyObject_GC_New(PyAIterWrapper,
&_PyAIterWrapper_Type);
if (aw == NULL) {
return NULL;
}
Py_INCREF(aiter);
aw->aw_aiter = aiter;
_PyObject_GC_TRACK(aw);
return (PyObject *)aw;
}

40
Python/ceval.c
View File

@ -1933,8 +1933,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
PyObject *obj = TOP();
PyTypeObject *type = Py_TYPE(obj);
if (type->tp_as_async != NULL)
if (type->tp_as_async != NULL) {
getter = type->tp_as_async->am_aiter;
}
if (getter != NULL) {
iter = (*getter)(obj);
@ -1955,6 +1956,27 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
goto error;
}
if (Py_TYPE(iter)->tp_as_async != NULL &&
Py_TYPE(iter)->tp_as_async->am_anext != NULL) {
/* Starting with CPython 3.5.2 __aiter__ should return
asynchronous iterators directly (not awaitables that
resolve to asynchronous iterators.)
Therefore, we check if the object that was returned
from __aiter__ has an __anext__ method. If it does,
we wrap it in an awaitable that resolves to `iter`.
See http://bugs.python.org/issue27243 for more
details.
*/
PyObject *wrapper = _PyAIterWrapper_New(iter);
Py_DECREF(iter);
SET_TOP(wrapper);
DISPATCH();
}
awaitable = _PyCoro_GetAwaitableIter(iter);
if (awaitable == NULL) {
SET_TOP(NULL);
@ -1966,9 +1988,23 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
Py_DECREF(iter);
goto error;
} else
} else {
Py_DECREF(iter);
if (PyErr_WarnFormat(
PyExc_PendingDeprecationWarning, 1,
"'%.100s' implements legacy __aiter__ protocol; "
"__aiter__ should return an asynchronous "
"iterator, not awaitable",
type->tp_name))
{
/* Warning was converted to an error. */
Py_DECREF(awaitable);
SET_TOP(NULL);
goto error;
}
}
SET_TOP(awaitable);
DISPATCH();
}