Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Issue #28003: Implement PEP 525 -- Asynchronous Generators.

This commit is contained in:
Yury Selivanov 2016-09-08 22:01:51 -07:00
parent b96ef55d49
commit eb6364557f
27 changed files with 2189 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Include/ceval.h
View File

@ -25,6 +25,10 @@ PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *);
PyAPI_FUNC(void) PyEval_SetTrace(Py_tracefunc, PyObject *);
PyAPI_FUNC(void) _PyEval_SetCoroutineWrapper(PyObject *);
PyAPI_FUNC(PyObject *) _PyEval_GetCoroutineWrapper(void);
PyAPI_FUNC(void) _PyEval_SetAsyncGenFirstiter(PyObject *);
PyAPI_FUNC(PyObject *) _PyEval_GetAsyncGenFirstiter(void);
PyAPI_FUNC(void) _PyEval_SetAsyncGenFinalizer(PyObject *);
PyAPI_FUNC(PyObject *) _PyEval_GetAsyncGenFinalizer(void);
#endif
struct _frame; /* Avoid including frameobject.h */

1
Include/code.h
View File

@ -59,6 +59,7 @@ typedef struct {
``async def`` keywords) */
#define CO_COROUTINE 0x0080
#define CO_ITERABLE_COROUTINE 0x0100
#define CO_ASYNC_GENERATOR 0x0200
/* These are no longer used. */
#if 0

31
Include/genobject.h
View File

@ -61,6 +61,37 @@ PyObject *_PyAIterWrapper_New(PyObject *aiter);
PyObject *_PyCoro_GetAwaitableIter(PyObject *o);
PyAPI_FUNC(PyObject *) PyCoro_New(struct _frame *,
PyObject *name, PyObject *qualname);
/* Asynchronous Generators */
typedef struct {
_PyGenObject_HEAD(ag)
PyObject *ag_finalizer;
/* Flag is set to 1 when hooks set up by sys.set_asyncgen_hooks
were called on the generator, to avoid calling them more
than once. */
int ag_hooks_inited;
/* Flag is set to 1 when aclose() is called for the first time, or
when a StopAsyncIteration exception is raised. */
int ag_closed;
} PyAsyncGenObject;
PyAPI_DATA(PyTypeObject) PyAsyncGen_Type;
PyAPI_DATA(PyTypeObject) _PyAsyncGenASend_Type;
PyAPI_DATA(PyTypeObject) _PyAsyncGenWrappedValue_Type;
PyAPI_DATA(PyTypeObject) _PyAsyncGenAThrow_Type;
PyAPI_FUNC(PyObject *) PyAsyncGen_New(struct _frame *,
PyObject *name, PyObject *qualname);
#define PyAsyncGen_CheckExact(op) (Py_TYPE(op) == &PyAsyncGen_Type)
PyObject *_PyAsyncGenValueWrapperNew(PyObject *);
int PyAsyncGen_ClearFreeLists(void);
#endif
#undef _PyGenObject_HEAD

1
Include/pylifecycle.h
View File

@ -107,6 +107,7 @@ PyAPI_FUNC(void) _PyGC_Fini(void);
PyAPI_FUNC(void) PySlice_Fini(void);
PyAPI_FUNC(void) _PyType_Fini(void);
PyAPI_FUNC(void) _PyRandom_Fini(void);
PyAPI_FUNC(void) PyAsyncGen_Fini(void);
PyAPI_DATA(PyThreadState *) _Py_Finalizing;
#endif

3
Include/pystate.h
View File

@ -148,6 +148,9 @@ typedef struct _ts {
Py_ssize_t co_extra_user_count;
freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS];
PyObject *async_gen_firstiter;
PyObject *async_gen_finalizer;
/* XXX signal handlers should also be here */
} PyThreadState;

1
Include/symtable.h
View File

@ -48,6 +48,7 @@ typedef struct _symtable_entry {
unsigned ste_child_free : 1; /* true if a child block has free vars,
including free refs to globals */
unsigned ste_generator : 1; /* true if namespace is a generator */
unsigned ste_coroutine : 1; /* true if namespace is a coroutine */
unsigned ste_varargs : 1; /* true if block has varargs */
unsigned ste_varkeywords : 1; /* true if block has varkeywords */
unsigned ste_returns_value : 1; /* true if namespace uses return with

57
Lib/asyncio/base_events.py
View File

@ -13,7 +13,6 @@ conscious design decision, leaving the door open for keyword arguments
to modify the meaning of the API call itself.
"""
import collections
import concurrent.futures
import heapq
@ -28,6 +27,7 @@ import time
import traceback
import sys
import warnings
import weakref
from . import compat
from . import coroutines
@ -242,6 +242,13 @@ class BaseEventLoop(events.AbstractEventLoop):
self._task_factory = None
self._coroutine_wrapper_set = False
# A weak set of all asynchronous generators that are being iterated
# by the loop.
self._asyncgens = weakref.WeakSet()
# Set to True when `loop.shutdown_asyncgens` is called.
self._asyncgens_shutdown_called = False
def __repr__(self):
return ('<%s running=%s closed=%s debug=%s>'
% (self.__class__.__name__, self.is_running(),
@ -333,6 +340,46 @@ class BaseEventLoop(events.AbstractEventLoop):
if self._closed:
raise RuntimeError('Event loop is closed')
def _asyncgen_finalizer_hook(self, agen):
self._asyncgens.discard(agen)
if not self.is_closed():
self.create_task(agen.aclose())
def _asyncgen_firstiter_hook(self, agen):
if self._asyncgens_shutdown_called:
warnings.warn(
"asynchronous generator {!r} was scheduled after "
"loop.shutdown_asyncgens() call".format(agen),
ResourceWarning, source=self)
self._asyncgens.add(agen)
@coroutine
def shutdown_asyncgens(self):
"""Shutdown all active asynchronous generators."""
self._asyncgens_shutdown_called = True
if not len(self._asyncgens):
return
closing_agens = list(self._asyncgens)
self._asyncgens.clear()
shutdown_coro = tasks.gather(
*[ag.aclose() for ag in closing_agens],
return_exceptions=True,
loop=self)
results = yield from shutdown_coro
for result, agen in zip(results, closing_agens):
if isinstance(result, Exception):
self.call_exception_handler({
'message': 'an error occurred during closing of '
'asynchronous generator {!r}'.format(agen),
'exception': result,
'asyncgen': agen
})
def run_forever(self):
"""Run until stop() is called."""
self._check_closed()
@ -340,6 +387,9 @@ class BaseEventLoop(events.AbstractEventLoop):
raise RuntimeError('Event loop is running.')
self._set_coroutine_wrapper(self._debug)
self._thread_id = threading.get_ident()
old_agen_hooks = sys.get_asyncgen_hooks()
sys.set_asyncgen_hooks(firstiter=self._asyncgen_firstiter_hook,
finalizer=self._asyncgen_finalizer_hook)
try:
while True:
self._run_once()
@ -349,6 +399,7 @@ class BaseEventLoop(events.AbstractEventLoop):
self._stopping = False
self._thread_id = None
self._set_coroutine_wrapper(False)
sys.set_asyncgen_hooks(*old_agen_hooks)
def run_until_complete(self, future):
"""Run until the Future is done.
@ -1179,7 +1230,9 @@ class BaseEventLoop(events.AbstractEventLoop):
- 'handle' (optional): Handle instance;
- 'protocol' (optional): Protocol instance;
- 'transport' (optional): Transport instance;
- 'socket' (optional): Socket instance.
- 'socket' (optional): Socket instance;
- 'asyncgen' (optional): Asynchronous generator that caused
the exception.
New keys maybe introduced in the future.

5
Lib/asyncio/coroutines.py
View File

@ -276,7 +276,10 @@ def _format_coroutine(coro):
try:
coro_code = coro.gi_code
except AttributeError:
coro_code = coro.cr_code
try:
coro_code = coro.cr_code
except AttributeError:
return repr(coro)
try:
coro_frame = coro.gi_frame

4
Lib/asyncio/events.py
View File

@ -248,6 +248,10 @@ class AbstractEventLoop:
"""
raise NotImplementedError
def shutdown_asyncgens(self):
"""Shutdown all active asynchronous generators."""
raise NotImplementedError
# Methods scheduling callbacks. All these return Handles.
def _timer_handle_cancelled(self, handle):

1
Lib/dis.py
View File

@ -87,6 +87,7 @@ COMPILER_FLAG_NAMES = {
64: "NOFREE",
128: "COROUTINE",
256: "ITERABLE_COROUTINE",
512: "ASYNC_GENERATOR",
}
def pretty_flags(flags):

7
Lib/inspect.py
View File

@ -185,6 +185,13 @@ def iscoroutinefunction(object):
return bool((isfunction(object) or ismethod(object)) and
object.__code__.co_flags & CO_COROUTINE)
def isasyncgenfunction(object):
return bool((isfunction(object) or ismethod(object)) and
object.__code__.co_flags & CO_ASYNC_GENERATOR)
def isasyncgen(object):
return isinstance(object, types.AsyncGeneratorType)
def isgenerator(object):
"""Return true if the object is a generator.

2
Lib/test/badsyntax_async6.py
View File

@ -1,2 +0,0 @@
async def foo():
yield

823
Lib/test/test_asyncgen.py Normal file
View File

@ -0,0 +1,823 @@
import asyncio
import inspect
import sys
import types
import unittest
from unittest import mock
class AwaitException(Exception):
pass
@types.coroutine
def awaitable(*, throw=False):
if throw:
yield ('throw',)
else:
yield ('result',)
def run_until_complete(coro):
exc = False
while True:
try:
if exc:
exc = False
fut = coro.throw(AwaitException)
else:
fut = coro.send(None)
except StopIteration as ex:
return ex.args[0]
if fut == ('throw',):
exc = True
def to_list(gen):
async def iterate():
res = []
async for i in gen:
res.append(i)
return res
return run_until_complete(iterate())
class AsyncGenSyntaxTest(unittest.TestCase):
def test_async_gen_syntax_01(self):
code = '''async def foo():
await abc
yield from 123
'''
with self.assertRaisesRegex(SyntaxError, 'yield from.*inside async'):
exec(code, {}, {})
def test_async_gen_syntax_02(self):
code = '''async def foo():
yield from 123
'''
with self.assertRaisesRegex(SyntaxError, 'yield from.*inside async'):
exec(code, {}, {})
def test_async_gen_syntax_03(self):
code = '''async def foo():
await abc
yield
return 123
'''
with self.assertRaisesRegex(SyntaxError, 'return.*value.*async gen'):
exec(code, {}, {})
def test_async_gen_syntax_04(self):
code = '''async def foo():
yield
return 123
'''
with self.assertRaisesRegex(SyntaxError, 'return.*value.*async gen'):
exec(code, {}, {})
def test_async_gen_syntax_05(self):
code = '''async def foo():
if 0:
yield
return 12
'''
with self.assertRaisesRegex(SyntaxError, 'return.*value.*async gen'):
exec(code, {}, {})
class AsyncGenTest(unittest.TestCase):
def compare_generators(self, sync_gen, async_gen):
def sync_iterate(g):
res = []
while True:
try:
res.append(g.__next__())
except StopIteration:
res.append('STOP')
break
except Exception as ex:
res.append(str(type(ex)))
return res
def async_iterate(g):
res = []
while True:
try:
g.__anext__().__next__()
except StopAsyncIteration:
res.append('STOP')
break
except StopIteration as ex:
if ex.args:
res.append(ex.args[0])
else:
res.append('EMPTY StopIteration')
break
except Exception as ex:
res.append(str(type(ex)))
return res
sync_gen_result = sync_iterate(sync_gen)
async_gen_result = async_iterate(async_gen)
self.assertEqual(sync_gen_result, async_gen_result)
return async_gen_result
def test_async_gen_iteration_01(self):
async def gen():
await awaitable()
a = yield 123
self.assertIs(a, None)
await awaitable()
yield 456
await awaitable()
yield 789
self.assertEqual(to_list(gen()), [123, 456, 789])
def test_async_gen_iteration_02(self):
async def gen():
await awaitable()
yield 123
await awaitable()
g = gen()
ai = g.__aiter__()
self.assertEqual(ai.__anext__().__next__(), ('result',))
try:
ai.__anext__().__next__()
except StopIteration as ex:
self.assertEqual(ex.args[0], 123)
else:
self.fail('StopIteration was not raised')
self.assertEqual(ai.__anext__().__next__(), ('result',))
try:
ai.__anext__().__next__()
except StopAsyncIteration as ex:
self.assertFalse(ex.args)
else:
self.fail('StopAsyncIteration was not raised')
def test_async_gen_exception_03(self):
async def gen():
await awaitable()
yield 123
await awaitable(throw=True)
yield 456
with self.assertRaises(AwaitException):
to_list(gen())
def test_async_gen_exception_04(self):
async def gen():
await awaitable()
yield 123
1 / 0
g = gen()
ai = g.__aiter__()
self.assertEqual(ai.__anext__().__next__(), ('result',))
try:
ai.__anext__().__next__()
except StopIteration as ex:
self.assertEqual(ex.args[0], 123)
else:
self.fail('StopIteration was not raised')
with self.assertRaises(ZeroDivisionError):
ai.__anext__().__next__()
def test_async_gen_exception_05(self):
async def gen():
yield 123
raise StopAsyncIteration
with self.assertRaisesRegex(RuntimeError,
'async generator.*StopAsyncIteration'):
to_list(gen())
def test_async_gen_exception_06(self):
async def gen():
yield 123
raise StopIteration
with self.assertRaisesRegex(RuntimeError,
'async generator.*StopIteration'):
to_list(gen())
def test_async_gen_exception_07(self):
def sync_gen():
try:
yield 1
1 / 0
finally:
yield 2
yield 3
yield 100
async def async_gen():
try:
yield 1
1 / 0
finally:
yield 2
yield 3
yield 100
self.compare_generators(sync_gen(), async_gen())
def test_async_gen_exception_08(self):
def sync_gen():
try:
yield 1
finally:
yield 2
1 / 0
yield 3
yield 100
async def async_gen():
try:
yield 1
await awaitable()
finally:
await awaitable()
yield 2
1 / 0
yield 3
yield 100
self.compare_generators(sync_gen(), async_gen())
def test_async_gen_exception_09(self):
def sync_gen():
try:
yield 1
1 / 0
finally:
yield 2
yield 3
yield 100
async def async_gen():
try:
await awaitable()
yield 1
1 / 0
finally:
yield 2
await awaitable()
yield 3
yield 100
self.compare_generators(sync_gen(), async_gen())
def test_async_gen_exception_10(self):
async def gen():
yield 123
with self.assertRaisesRegex(TypeError,
"non-None value .* async generator"):
gen().__anext__().send(100)
def test_async_gen_api_01(self):
async def gen():
yield 123
g = gen()
self.assertEqual(g.__name__, 'gen')
g.__name__ = '123'
self.assertEqual(g.__name__, '123')
self.assertIn('.gen', g.__qualname__)
g.__qualname__ = '123'
self.assertEqual(g.__qualname__, '123')
self.assertIsNone(g.ag_await)
self.assertIsInstance(g.ag_frame, types.FrameType)
self.assertFalse(g.ag_running)
self.assertIsInstance(g.ag_code, types.CodeType)
self.assertTrue(inspect.isawaitable(g.aclose()))
class AsyncGenAsyncioTest(unittest.TestCase):
def setUp(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
def tearDown(self):
self.loop.close()
self.loop = None
async def to_list(self, gen):
res = []
async for i in gen:
res.append(i)
return res
def test_async_gen_asyncio_01(self):
async def gen():
yield 1
await asyncio.sleep(0.01, loop=self.loop)
yield 2
await asyncio.sleep(0.01, loop=self.loop)
return
yield 3
res = self.loop.run_until_complete(self.to_list(gen()))
self.assertEqual(res, [1, 2])
def test_async_gen_asyncio_02(self):
async def gen():
yield 1
await asyncio.sleep(0.01, loop=self.loop)
yield 2
1 / 0
yield 3
with self.assertRaises(ZeroDivisionError):
self.loop.run_until_complete(self.to_list(gen()))
def test_async_gen_asyncio_03(self):
loop = self.loop
class Gen:
async def __aiter__(self):
yield 1
await asyncio.sleep(0.01, loop=loop)
yield 2
res = loop.run_until_complete(self.to_list(Gen()))
self.assertEqual(res, [1, 2])
def test_async_gen_asyncio_anext_04(self):
async def foo():
yield 1
await asyncio.sleep(0.01, loop=self.loop)
try:
yield 2
yield 3
except ZeroDivisionError:
yield 1000
await asyncio.sleep(0.01, loop=self.loop)
yield 4
async def run1():
it = foo().__aiter__()
self.assertEqual(await it.__anext__(), 1)
self.assertEqual(await it.__anext__(), 2)
self.assertEqual(await it.__anext__(), 3)
self.assertEqual(await it.__anext__(), 4)
with self.assertRaises(StopAsyncIteration):
await it.__anext__()
with self.assertRaises(StopAsyncIteration):
await it.__anext__()
async def run2():
it = foo().__aiter__()
self.assertEqual(await it.__anext__(), 1)
self.assertEqual(await it.__anext__(), 2)
try:
it.__anext__().throw(ZeroDivisionError)
except StopIteration as ex:
self.assertEqual(ex.args[0], 1000)
else:
self.fail('StopIteration was not raised')
self.assertEqual(await it.__anext__(), 4)
with self.assertRaises(StopAsyncIteration):
await it.__anext__()
self.loop.run_until_complete(run1())
self.loop.run_until_complete(run2())
def test_async_gen_asyncio_anext_05(self):
async def foo():
v = yield 1
v = yield v
yield v * 100
async def run():
it = foo().__aiter__()
try:
it.__anext__().send(None)
except StopIteration as ex:
self.assertEqual(ex.args[0], 1)
else:
self.fail('StopIteration was not raised')
try:
it.__anext__().send(10)
except StopIteration as ex:
self.assertEqual(ex.args[0], 10)
else:
self.fail('StopIteration was not raised')
try:
it.__anext__().send(12)
except StopIteration as ex:
self.assertEqual(ex.args[0], 1200)
else:
self.fail('StopIteration was not raised')
with self.assertRaises(StopAsyncIteration):
await it.__anext__()
self.loop.run_until_complete(run())
def test_async_gen_asyncio_aclose_06(self):
async def foo():
try:
yield 1
1 / 0
finally:
await asyncio.sleep(0.01, loop=self.loop)
yield 12
async def run():
gen = foo()
it = gen.__aiter__()
await it.__anext__()
await gen.aclose()
with self.assertRaisesRegex(
RuntimeError,
"async generator ignored GeneratorExit"):
self.loop.run_until_complete(run())
def test_async_gen_asyncio_aclose_07(self):
DONE = 0
async def foo():
nonlocal DONE
try:
yield 1
1 / 0
finally:
await asyncio.sleep(0.01, loop=self.loop)
await asyncio.sleep(0.01, loop=self.loop)
DONE += 1
DONE += 1000
async def run():
gen = foo()
it = gen.__aiter__()
await it.__anext__()
await gen.aclose()
self.loop.run_until_complete(run())
self.assertEqual(DONE, 1)
def test_async_gen_asyncio_aclose_08(self):
DONE = 0
fut = asyncio.Future(loop=self.loop)
async def foo():
nonlocal DONE
try:
yield 1
await fut
DONE += 1000
yield 2
finally:
await asyncio.sleep(0.01, loop=self.loop)
await asyncio.sleep(0.01, loop=self.loop)
DONE += 1
DONE += 1000
async def run():
gen = foo()
it = gen.__aiter__()
self.assertEqual(await it.__anext__(), 1)
t = self.loop.create_task(it.__anext__())
await asyncio.sleep(0.01, loop=self.loop)
await gen.aclose()
return t
t = self.loop.run_until_complete(run())
self.assertEqual(DONE, 1)
# Silence ResourceWarnings
fut.cancel()
t.cancel()
self.loop.run_until_complete(asyncio.sleep(0.01, loop=self.loop))
def test_async_gen_asyncio_gc_aclose_09(self):
DONE = 0
async def gen():
nonlocal DONE
try:
while True:
yield 1
finally:
await asyncio.sleep(0.01, loop=self.loop)
await asyncio.sleep(0.01, loop=self.loop)
DONE = 1
async def run():
g = gen()
await g.__anext__()
await g.__anext__()
del g
await asyncio.sleep(0.1, loop=self.loop)
self.loop.run_until_complete(run())
self.assertEqual(DONE, 1)
def test_async_gen_asyncio_asend_01(self):
DONE = 0
# Sanity check:
def sgen():
v = yield 1
yield v * 2
sg = sgen()
v = sg.send(None)
self.assertEqual(v, 1)
v = sg.send(100)
self.assertEqual(v, 200)
async def gen():
nonlocal DONE
try:
await asyncio.sleep(0.01, loop=self.loop)
v = yield 1
await asyncio.sleep(0.01, loop=self.loop)
yield v * 2
await asyncio.sleep(0.01, loop=self.loop)
return
finally:
await asyncio.sleep(0.01, loop=self.loop)
await asyncio.sleep(0.01, loop=self.loop)
DONE = 1
async def run():
g = gen()
v = await g.asend(None)
self.assertEqual(v, 1)
v = await g.asend(100)
self.assertEqual(v, 200)
with self.assertRaises(StopAsyncIteration):
await g.asend(None)
self.loop.run_until_complete(run())
self.assertEqual(DONE, 1)
def test_async_gen_asyncio_asend_02(self):
DONE = 0
async def sleep_n_crash(delay):
await asyncio.sleep(delay, loop=self.loop)
1 / 0
async def gen():
nonlocal DONE
try:
await asyncio.sleep(0.01, loop=self.loop)
v = yield 1
await sleep_n_crash(0.01)
DONE += 1000
yield v * 2
finally:
await asyncio.sleep(0.01, loop=self.loop)
await asyncio.sleep(0.01, loop=self.loop)
DONE = 1
async def run():
g = gen()
v = await g.asend(None)
self.assertEqual(v, 1)
await g.asend(100)
with self.assertRaises(ZeroDivisionError):
self.loop.run_until_complete(run())
self.assertEqual(DONE, 1)
def test_async_gen_asyncio_asend_03(self):
DONE = 0
async def sleep_n_crash(delay):
fut = asyncio.ensure_future(asyncio.sleep(delay, loop=self.loop),
loop=self.loop)
self.loop.call_later(delay / 2, lambda: fut.cancel())
return await fut
async def gen():
nonlocal DONE
try:
await asyncio.sleep(0.01, loop=self.loop)
v = yield 1
await sleep_n_crash(0.01)
DONE += 1000
yield v * 2
finally:
await asyncio.sleep(0.01, loop=self.loop)
await asyncio.sleep(0.01, loop=self.loop)
DONE = 1
async def run():
g = gen()
v = await g.asend(None)
self.assertEqual(v, 1)
await g.asend(100)
with self.assertRaises(asyncio.CancelledError):
self.loop.run_until_complete(run())
self.assertEqual(DONE, 1)
def test_async_gen_asyncio_athrow_01(self):
DONE = 0
class FooEr(Exception):
pass
# Sanity check:
def sgen():
try:
v = yield 1
except FooEr:
v = 1000
yield v * 2
sg = sgen()
v = sg.send(None)
self.assertEqual(v, 1)
v = sg.throw(FooEr)
self.assertEqual(v, 2000)
with self.assertRaises(StopIteration):
sg.send(None)
async def gen():
nonlocal DONE
try:
await asyncio.sleep(0.01, loop=self.loop)
try:
v = yield 1
except FooEr:
v = 1000
await asyncio.sleep(0.01, loop=self.loop)
yield v * 2
await asyncio.sleep(0.01, loop=self.loop)
# return
finally:
await asyncio.sleep(0.01, loop=self.loop)
await asyncio.sleep(0.01, loop=self.loop)
DONE = 1
async def run():
g = gen()
v = await g.asend(None)
self.assertEqual(v, 1)
v = await g.athrow(FooEr)
self.assertEqual(v, 2000)
with self.assertRaises(StopAsyncIteration):
await g.asend(None)
self.loop.run_until_complete(run())
self.assertEqual(DONE, 1)
def test_async_gen_asyncio_athrow_02(self):
DONE = 0
class FooEr(Exception):
pass
async def sleep_n_crash(delay):
fut = asyncio.ensure_future(asyncio.sleep(delay, loop=self.loop),
loop=self.loop)
self.loop.call_later(delay / 2, lambda: fut.cancel())
return await fut
async def gen():
nonlocal DONE
try:
await asyncio.sleep(0.01, loop=self.loop)
try:
v = yield 1
except FooEr:
await sleep_n_crash(0.01)
yield v * 2
await asyncio.sleep(0.01, loop=self.loop)
# return
finally:
await asyncio.sleep(0.01, loop=self.loop)
await asyncio.sleep(0.01, loop=self.loop)
DONE = 1
async def run():
g = gen()
v = await g.asend(None)
self.assertEqual(v, 1)
try:
await g.athrow(FooEr)
except asyncio.CancelledError:
self.assertEqual(DONE, 1)
raise
else:
self.fail('CancelledError was not raised')
with self.assertRaises(asyncio.CancelledError):
self.loop.run_until_complete(run())
self.assertEqual(DONE, 1)
def test_async_gen_asyncio_shutdown_01(self):
finalized = 0
async def waiter(timeout):
nonlocal finalized
try:
await asyncio.sleep(timeout, loop=self.loop)
yield 1
finally:
await asyncio.sleep(0, loop=self.loop)
finalized += 1
async def wait():
async for _ in waiter(1):
pass
t1 = self.loop.create_task(wait())
t2 = self.loop.create_task(wait())
self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop))
self.loop.run_until_complete(self.loop.shutdown_asyncgens())
self.assertEqual(finalized, 2)
# Silence warnings
t1.cancel()
t2.cancel()
self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop))
def test_async_gen_asyncio_shutdown_02(self):
logged = 0
def logger(loop, context):
nonlocal logged
self.assertIn('asyncgen', context)
expected = 'an error occurred during closing of asynchronous'
if expected in context['message']:
logged += 1
async def waiter(timeout):
try:
await asyncio.sleep(timeout, loop=self.loop)
yield 1
finally:
1 / 0
async def wait():
async for _ in waiter(1):
pass
t = self.loop.create_task(wait())
self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop))
self.loop.set_exception_handler(logger)
self.loop.run_until_complete(self.loop.shutdown_asyncgens())
self.assertEqual(logged, 1)
# Silence warnings
t.cancel()
self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop))
if __name__ == "__main__":
unittest.main()

6
Lib/test/test_coroutines.py
View File

@ -88,12 +88,6 @@ class AsyncBadSyntaxTest(unittest.TestCase):
with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
import test.badsyntax_async5
def test_badsyntax_6(self):
with self.assertRaisesRegex(
SyntaxError, "'yield' inside async function"):
import test.badsyntax_async6
def test_badsyntax_7(self):
with self.assertRaisesRegex(
SyntaxError, "'yield from' inside async function"):

2
Lib/test/test_dis.py
View File

@ -574,7 +574,7 @@ Argument count: 0
Kw-only arguments: 0
Number of locals: 2
Stack size: 17
Flags: OPTIMIZED, NEWLOCALS, GENERATOR, NOFREE, COROUTINE
Flags: OPTIMIZED, NEWLOCALS, NOFREE, COROUTINE
Constants:
0: None
1: 1"""

12
Lib/test/test_inspect.py
View File

@ -65,7 +65,8 @@ class IsTestBase(unittest.TestCase):
inspect.isframe, inspect.isfunction, inspect.ismethod,
inspect.ismodule, inspect.istraceback,
inspect.isgenerator, inspect.isgeneratorfunction,
inspect.iscoroutine, inspect.iscoroutinefunction])
inspect.iscoroutine, inspect.iscoroutinefunction,
inspect.isasyncgen, inspect.isasyncgenfunction])
def istest(self, predicate, exp):
obj = eval(exp)
@ -73,6 +74,7 @@ class IsTestBase(unittest.TestCase):
for other in self.predicates - set([predicate]):
if (predicate == inspect.isgeneratorfunction or \
predicate == inspect.isasyncgenfunction or \
predicate == inspect.iscoroutinefunction) and \
other == inspect.isfunction:
continue
@ -82,6 +84,10 @@ def generator_function_example(self):
for i in range(2):
yield i
async def async_generator_function_example(self):
async for i in range(2):
yield i
async def coroutine_function_example(self):
return 'spam'
@ -122,6 +128,10 @@ class TestPredicates(IsTestBase):
self.istest(inspect.isdatadescriptor, 'collections.defaultdict.default_factory')
self.istest(inspect.isgenerator, '(x for x in range(2))')
self.istest(inspect.isgeneratorfunction, 'generator_function_example')
self.istest(inspect.isasyncgen,
'async_generator_function_example(1)')
self.istest(inspect.isasyncgenfunction,
'async_generator_function_example')
with warnings.catch_warnings():
warnings.simplefilter("ignore")

26
Lib/test/test_sys.py
View File

@ -1192,6 +1192,32 @@ class SizeofTest(unittest.TestCase):
# sys.flags
check(sys.flags, vsize('') + self.P * len(sys.flags))
def test_asyncgen_hooks(self):
old = sys.get_asyncgen_hooks()
self.assertIsNone(old.firstiter)
self.assertIsNone(old.finalizer)
firstiter = lambda *a: None
sys.set_asyncgen_hooks(firstiter=firstiter)
hooks = sys.get_asyncgen_hooks()
self.assertIs(hooks.firstiter, firstiter)
self.assertIs(hooks[0], firstiter)
self.assertIs(hooks.finalizer, None)
self.assertIs(hooks[1], None)
finalizer = lambda *a: None
sys.set_asyncgen_hooks(finalizer=finalizer)
hooks = sys.get_asyncgen_hooks()
self.assertIs(hooks.firstiter, firstiter)
self.assertIs(hooks[0], firstiter)
self.assertIs(hooks.finalizer, finalizer)
self.assertIs(hooks[1], finalizer)
sys.set_asyncgen_hooks(*old)
cur = sys.get_asyncgen_hooks()
self.assertIsNone(cur.firstiter)
self.assertIsNone(cur.finalizer)
def test_main():
test.support.run_unittest(SysModuleTest, SizeofTest)

5
Lib/types.py
View File

@ -24,6 +24,11 @@ _c = _c()
CoroutineType = type(_c)
_c.close() # Prevent ResourceWarning
async def _ag():
yield
_ag = _ag()
AsyncGeneratorType = type(_ag)
class _C:
def _m(self): pass
MethodType = type(_C()._m)

3
Misc/NEWS
View File

@ -103,6 +103,9 @@ Core and Builtins
- Issue #27985: Implement PEP 526 -- Syntax for Variable Annotations.
Patch by Ivan Levkivskyi.
- Issue #28003: Implement PEP 525 -- Asynchronous Generators.
Library
-------

1
Modules/gcmodule.c
View File

@ -892,6 +892,7 @@ clear_freelists(void)
(void)PyList_ClearFreeList();
(void)PyDict_ClearFreeList();
(void)PySet_ClearFreeList();
(void)PyAsyncGen_ClearFreeLists();
}
/* This is the main function. Read this to understand how the

1032
Objects/genobject.c
View File

File diff suppressed because it is too large Load Diff

114
Python/ceval.c
View File

@ -1204,7 +1204,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
f->f_stacktop = NULL; /* remains NULL unless yield suspends frame */
f->f_executing = 1;
if (co->co_flags & (CO_GENERATOR | CO_COROUTINE)) {
if (co->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
if (!throwflag && f->f_exc_type != NULL && f->f_exc_type != Py_None) {
/* We were in an except handler when we left,
restore the exception state which was put aside
@ -2083,36 +2083,45 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
PyObject *aiter = TOP();
PyTypeObject *type = Py_TYPE(aiter);
if (type->tp_as_async != NULL)
getter = type->tp_as_async->am_anext;
if (getter != NULL) {
next_iter = (*getter)(aiter);
if (next_iter == NULL) {
if (PyAsyncGen_CheckExact(aiter)) {
awaitable = type->tp_as_async->am_anext(aiter);
if (awaitable == NULL) {
goto error;
}
}
else {
PyErr_Format(
PyExc_TypeError,
"'async for' requires an iterator with "
"__anext__ method, got %.100s",
type->tp_name);
goto error;
}
} else {
if (type->tp_as_async != NULL){
getter = type->tp_as_async->am_anext;
}
awaitable = _PyCoro_GetAwaitableIter(next_iter);
if (awaitable == NULL) {
PyErr_Format(
PyExc_TypeError,
"'async for' received an invalid object "
"from __anext__: %.100s",
Py_TYPE(next_iter)->tp_name);
if (getter != NULL) {
next_iter = (*getter)(aiter);
if (next_iter == NULL) {
goto error;
}
}
else {
PyErr_Format(
PyExc_TypeError,
"'async for' requires an iterator with "
"__anext__ method, got %.100s",
type->tp_name);
goto error;
}
Py_DECREF(next_iter);
goto error;
} else
Py_DECREF(next_iter);
awaitable = _PyCoro_GetAwaitableIter(next_iter);
if (awaitable == NULL) {
PyErr_Format(
PyExc_TypeError,
"'async for' received an invalid object "
"from __anext__: %.100s",
Py_TYPE(next_iter)->tp_name);
Py_DECREF(next_iter);
goto error;
} else {
Py_DECREF(next_iter);
}
}
PUSH(awaitable);
PREDICT(LOAD_CONST);
@ -2187,6 +2196,17 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
TARGET(YIELD_VALUE) {
retval = POP();
if (co->co_flags & CO_ASYNC_GENERATOR) {
PyObject *w = _PyAsyncGenValueWrapperNew(retval);
Py_DECREF(retval);
if (w == NULL) {
retval = NULL;
goto error;
}
retval = w;
}
f->f_stacktop = stack_pointer;
why = WHY_YIELD;
goto fast_yield;
@ -3712,7 +3732,7 @@ fast_block_end:
assert((retval != NULL) ^ (PyErr_Occurred() != NULL));
fast_yield:
if (co->co_flags & (CO_GENERATOR | CO_COROUTINE)) {
if (co->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
/* The purpose of this block is to put aside the generator's exception
state and restore that of the calling frame. If the current
@ -4156,8 +4176,8 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals,
freevars[PyTuple_GET_SIZE(co->co_cellvars) + i] = o;
}
/* Handle generator/coroutine */
if (co->co_flags & (CO_GENERATOR | CO_COROUTINE)) {
/* Handle generator/coroutine/asynchronous generator */
if (co->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
PyObject *gen;
PyObject *coro_wrapper = tstate->coroutine_wrapper;
int is_coro = co->co_flags & CO_COROUTINE;
@ -4182,6 +4202,8 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals,
* and return that as the value. */
if (is_coro) {
gen = PyCoro_New(f, name, qualname);
} else if (co->co_flags & CO_ASYNC_GENERATOR) {
gen = PyAsyncGen_New(f, name, qualname);
} else {
gen = PyGen_NewWithQualName(f, name, qualname);
}
@ -4660,6 +4682,38 @@ _PyEval_GetCoroutineWrapper(void)
return tstate->coroutine_wrapper;
}
void
_PyEval_SetAsyncGenFirstiter(PyObject *firstiter)
{
PyThreadState *tstate = PyThreadState_GET();
Py_XINCREF(firstiter);
Py_XSETREF(tstate->async_gen_firstiter, firstiter);
}
PyObject *
_PyEval_GetAsyncGenFirstiter(void)
{
PyThreadState *tstate = PyThreadState_GET();
return tstate->async_gen_firstiter;
}
void
_PyEval_SetAsyncGenFinalizer(PyObject *finalizer)
{
PyThreadState *tstate = PyThreadState_GET();
Py_XINCREF(finalizer);
Py_XSETREF(tstate->async_gen_finalizer, finalizer);
}
PyObject *
_PyEval_GetAsyncGenFinalizer(void)
{
PyThreadState *tstate = PyThreadState_GET();
return tstate->async_gen_finalizer;
}
PyObject *
PyEval_GetBuiltins(void)
{

13
Python/compile.c
View File

@ -1886,8 +1886,6 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
return 0;
}
if (is_async)
co->co_flags |= CO_COROUTINE;
compiler_make_closure(c, co, funcflags, qualname);
Py_DECREF(qualname);
Py_DECREF(co);
@ -2801,6 +2799,9 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s)
if (c->u->u_ste->ste_type != FunctionBlock)
return compiler_error(c, "'return' outside function");
if (s->v.Return.value) {
if (c->u->u_ste->ste_coroutine && c->u->u_ste->ste_generator)
return compiler_error(
c, "'return' with value in async generator");
VISIT(c, expr, s->v.Return.value);
}
else
@ -4115,8 +4116,6 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
case Yield_kind:
if (c->u->u_ste->ste_type != FunctionBlock)
return compiler_error(c, "'yield' outside function");
if (c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION)
return compiler_error(c, "'yield' inside async function");
if (e->v.Yield.value) {
VISIT(c, expr, e->v.Yield.value);
}
@ -4992,8 +4991,12 @@ compute_code_flags(struct compiler *c)
flags |= CO_NEWLOCALS | CO_OPTIMIZED;
if (ste->ste_nested)
flags |= CO_NESTED;
if (ste->ste_generator)
if (ste->ste_generator && !ste->ste_coroutine)
flags |= CO_GENERATOR;
if (!ste->ste_generator && ste->ste_coroutine)
flags |= CO_COROUTINE;
if (ste->ste_generator && ste->ste_coroutine)
flags |= CO_ASYNC_GENERATOR;
if (ste->ste_varargs)
flags |= CO_VARARGS;
if (ste->ste_varkeywords)

1
Python/pylifecycle.c
View File

@ -694,6 +694,7 @@ Py_FinalizeEx(void)
_PyGC_Fini();
_PyRandom_Fini();
_PyArg_Fini();
PyAsyncGen_Fini();
/* Cleanup Unicode implementation */
_PyUnicode_Fini();

5
Python/pystate.c
View File

@ -229,6 +229,9 @@ new_threadstate(PyInterpreterState *interp, int init)
tstate->in_coroutine_wrapper = 0;
tstate->co_extra_user_count = 0;
tstate->async_gen_firstiter = NULL;
tstate->async_gen_finalizer = NULL;
if (init)
_PyThreadState_Init(tstate);
@ -408,6 +411,8 @@ PyThreadState_Clear(PyThreadState *tstate)
Py_CLEAR(tstate->c_traceobj);
Py_CLEAR(tstate->coroutine_wrapper);
Py_CLEAR(tstate->async_gen_firstiter);
Py_CLEAR(tstate->async_gen_finalizer);
}

6
Python/symtable.c
View File

@ -63,6 +63,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block,
ste->ste_nested = 1;
ste->ste_child_free = 0;
ste->ste_generator = 0;
ste->ste_coroutine = 0;
ste->ste_returns_value = 0;
ste->ste_needs_class_closure = 0;
@ -378,7 +379,7 @@ error_at_directive(PySTEntryObject *ste, PyObject *name)
PyLong_AsLong(PyTuple_GET_ITEM(data, 2)));
return 0;
}
}
}
PyErr_SetString(PyExc_RuntimeError,
"BUG: internal directive bookkeeping broken");
@ -1397,6 +1398,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
FunctionBlock, (void *)s, s->lineno,
s->col_offset))
VISIT_QUIT(st, 0);
st->st_cur->ste_coroutine = 1;
VISIT(st, arguments, s->v.AsyncFunctionDef.args);
VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body);
if (!symtable_exit_block(st, s))
@ -1492,7 +1494,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
break;
case Await_kind:
VISIT(st, expr, e->v.Await.value);
st->st_cur->ste_generator = 1;
st->st_cur->ste_coroutine = 1;
break;
case Compare_kind:
VISIT(st, expr, e->v.Compare.left);

119
Python/sysmodule.c
View File

@ -717,6 +717,113 @@ Return the wrapper for coroutine objects set by sys.set_coroutine_wrapper."
);
static PyTypeObject AsyncGenHooksType;
PyDoc_STRVAR(asyncgen_hooks_doc,
"asyncgen_hooks\n\
\n\
A struct sequence providing information about asynhronous\n\
generators hooks. The attributes are read only.");
static PyStructSequence_Field asyncgen_hooks_fields[] = {
{"firstiter", "Hook to intercept first iteration"},
{"finalizer", "Hook to intercept finalization"},
{0}
};
static PyStructSequence_Desc asyncgen_hooks_desc = {
"asyncgen_hooks", /* name */
asyncgen_hooks_doc, /* doc */
asyncgen_hooks_fields , /* fields */
2
};
static PyObject *
sys_set_asyncgen_hooks(PyObject *self, PyObject *args, PyObject *kw)
{
static char *keywords[] = {"firstiter", "finalizer", NULL};
PyObject *firstiter = NULL;
PyObject *finalizer = NULL;
if (!PyArg_ParseTupleAndKeywords(
args, kw, "|OO", keywords,
&firstiter, &finalizer)) {
return NULL;
}
if (finalizer && finalizer != Py_None) {
if (!PyCallable_Check(finalizer)) {
PyErr_Format(PyExc_TypeError,
"callable finalizer expected, got %.50s",
Py_TYPE(finalizer)->tp_name);
return NULL;
}
_PyEval_SetAsyncGenFinalizer(finalizer);
}
else if (finalizer == Py_None) {
_PyEval_SetAsyncGenFinalizer(NULL);
}
if (firstiter && firstiter != Py_None) {
if (!PyCallable_Check(firstiter)) {
PyErr_Format(PyExc_TypeError,
"callable firstiter expected, got %.50s",
Py_TYPE(firstiter)->tp_name);
return NULL;
}
_PyEval_SetAsyncGenFirstiter(firstiter);
}
else if (firstiter == Py_None) {
_PyEval_SetAsyncGenFirstiter(NULL);
}
Py_RETURN_NONE;
}
PyDoc_STRVAR(set_asyncgen_hooks_doc,
"set_asyncgen_hooks(*, firstiter=None, finalizer=None)\n\
\n\
Set a finalizer for async generators objects."
);
static PyObject *
sys_get_asyncgen_hooks(PyObject *self, PyObject *args)
{
PyObject *res;
PyObject *firstiter = _PyEval_GetAsyncGenFirstiter();
PyObject *finalizer = _PyEval_GetAsyncGenFinalizer();
res = PyStructSequence_New(&AsyncGenHooksType);
if (res == NULL) {
return NULL;
}
if (firstiter == NULL) {
firstiter = Py_None;
}
if (finalizer == NULL) {
finalizer = Py_None;
}
Py_INCREF(firstiter);
PyStructSequence_SET_ITEM(res, 0, firstiter);
Py_INCREF(finalizer);
PyStructSequence_SET_ITEM(res, 1, finalizer);
return res;
}
PyDoc_STRVAR(get_asyncgen_hooks_doc,
"get_asyncgen_hooks()\n\
\n\
Return a namedtuple of installed asynchronous generators hooks \
(firstiter, finalizer)."
);
static PyTypeObject Hash_InfoType;
PyDoc_STRVAR(hash_info_doc,
@ -1315,6 +1422,10 @@ static PyMethodDef sys_methods[] = {
set_coroutine_wrapper_doc},
{"get_coroutine_wrapper", sys_get_coroutine_wrapper, METH_NOARGS,
get_coroutine_wrapper_doc},
{"set_asyncgen_hooks", sys_set_asyncgen_hooks,
METH_VARARGS | METH_KEYWORDS, set_asyncgen_hooks_doc},
{"get_asyncgen_hooks", sys_get_asyncgen_hooks, METH_NOARGS,
get_asyncgen_hooks_doc},
{NULL, NULL} /* sentinel */
};
@ -1950,6 +2061,14 @@ _PySys_Init(void)
SET_SYS_FROM_STRING("thread_info", PyThread_GetInfo());
#endif
/* initialize asyncgen_hooks */
if (AsyncGenHooksType.tp_name == NULL) {
if (PyStructSequence_InitType2(
&AsyncGenHooksType, &asyncgen_hooks_desc) < 0) {
return NULL;
}
}
#undef SET_SYS_FROM_STRING
#undef SET_SYS_FROM_STRING_BORROW
if (PyErr_Occurred())