traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-34616: Add PyCF_ALLOW_TOP_LEVEL_AWAIT to allow top-level await (GH-13148) 

Co-Authored-By: Yury Selivanov <yury@magic.io>
This commit is contained in:
Matthias Bussonnier 2019-05-21 13:12:03 -07:00 committed by Yury Selivanov
parent aa32a7e111
commit 565b4f1ac7
7 changed files with 109 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Doc/library/functions.rst
View File

@ -257,6 +257,12 @@ are always available. They are listed here in alphabetical order.
can be found as the :attr:`~__future__._Feature.compiler_flag` attribute on
the :class:`~__future__._Feature` instance in the :mod:`__future__` module.
The optional argument *flags* also controls whether the compiled source is
allowed to contain top-level ``await``, ``async for`` and ``async with``.
When the bit ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` is set, the return code
object has ``CO_COROUTINE`` set in ``co_code``, and can be interactively
executed via ``await eval(code_object)``.
The argument *optimize* specifies the optimization level of the compiler; the
default value of ``-1`` selects the optimization level of the interpreter as
given by :option:`-O` options. Explicit levels are ``0`` (no optimization;
@ -290,6 +296,10 @@ are always available. They are listed here in alphabetical order.
Previously, :exc:`TypeError` was raised when null bytes were encountered
in *source*.
.. versionadded:: 3.8
``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable
support for top-level ``await``, ``async for``, and ``async with``.
.. class:: complex([real[, imag]])

1
Include/compile.h
View File

@ -23,6 +23,7 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
#define PyCF_ONLY_AST 0x0400
#define PyCF_IGNORE_COOKIE 0x0800
#define PyCF_TYPE_COMMENTS 0x1000
#define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000
#ifndef Py_LIMITED_API
typedef struct {

73
Lib/test/test_builtin.py
View File

@ -1,6 +1,7 @@
# Python test set -- built-in functions
import ast
import asyncio
import builtins
import collections
import decimal
@ -18,9 +19,14 @@ import types
import unittest
import warnings
from contextlib import ExitStack
from inspect import CO_COROUTINE
from itertools import product
from textwrap import dedent
from types import AsyncGeneratorType, FunctionType
from operator import neg
from test.support import (
EnvironmentVarGuard, TESTFN, check_warnings, swap_attr, unlink)
EnvironmentVarGuard, TESTFN, check_warnings, swap_attr, unlink,
maybe_get_event_loop_policy)
from test.support.script_helper import assert_python_ok
from unittest.mock import MagicMock, patch
try:
@ -358,6 +364,71 @@ class BuiltinTest(unittest.TestCase):
rv = ns['f']()
self.assertEqual(rv, tuple(expected))
def test_compile_top_level_await(self):
"""Test whether code some top level await can be compiled.
Make sure it compiles only with the PyCF_ALLOW_TOP_LEVEL_AWAIT flag set,
and make sure the generated code object has the CO_COROUTINE flag set in
order to execute it with `await eval(.....)` instead of exec, or via a
FunctionType.
"""
# helper function just to check we can run top=level async-for
async def arange(n):
for i in range(n):
yield i
modes = ('single', 'exec')
code_samples = ['''a = await asyncio.sleep(0, result=1)''',
'''async for i in arange(1):
a = 1''',
'''async with asyncio.Lock() as l:
a = 1''']
policy = maybe_get_event_loop_policy()
try:
for mode, code_sample in product(modes,code_samples):
source = dedent(code_sample)
with self.assertRaises(SyntaxError, msg=f"{source=} {mode=}"):
compile(source, '?' , mode)
co = compile(source,
'?',
mode,
flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE,
msg=f"{source=} {mode=}")
# test we can create and advance a function type
globals_ = {'asyncio': asyncio, 'a':0, 'arange': arange}
async_f = FunctionType(co, globals_)
asyncio.run(async_f())
self.assertEqual(globals_['a'], 1)
# test we can await-eval,
globals_ = {'asyncio': asyncio, 'a':0, 'arange': arange}
asyncio.run(eval(co, globals_))
self.assertEqual(globals_['a'], 1)
finally:
asyncio.set_event_loop_policy(policy)
def test_compile_async_generator(self):
"""
With the PyCF_ALLOW_TOP_LEVEL_AWAIT flag added in 3.8, we want to
make sure AsyncGenerators are still properly not marked with CO_COROUTINE
"""
code = dedent("""async def ticker():
for i in range(10):
yield i
await asyncio.sleep(0)""")
co = compile(code, '?', 'exec', flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
glob = {}
exec(co, glob)
self.assertEqual(type(glob['ticker']()), AsyncGeneratorType)
def test_delattr(self):
sys.spam = 1
delattr(sys, 'spam')

1
Misc/NEWS.d/next/Core and Builtins/2019-05-07-17-12-37.bpo-34616.0Y0_9r.rst Normal file
View File

@ -0,0 +1 @@
The ``compile()`` builtin functions now support the ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` flag, which allow to compile sources that contains top-level ``await``, ``async with`` or ``async for``. This is useful to evaluate async-code from with an already async functions; for example in a custom REPL.

2
Parser/asdl_c.py
View File

@ -1000,6 +1000,8 @@ class ASTModuleVisitor(PickleVisitor):
self.emit("if (!m) return NULL;", 1)
self.emit("d = PyModule_GetDict(m);", 1)
self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL;', 1)
self.emit('if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0)', 1)
self.emit("return NULL;", 2)
self.emit('if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0)', 1)
self.emit("return NULL;", 2)
self.emit('if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0)', 1)

2
Python/Python-ast.c generated
View File

@ -8776,6 +8776,8 @@ PyInit__ast(void)
if (!m) return NULL;
d = PyModule_GetDict(m);
if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL;
if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0)
return NULL;
if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0)
return NULL;
if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0)

28
Python/compile.c
View File

@ -2609,7 +2609,9 @@ static int
compiler_async_for(struct compiler *c, stmt_ty s)
{
basicblock *start, *except, *end;
if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
if (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT){
c->u->u_ste->ste_coroutine = 1;
} else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
return compiler_error(c, "'async for' outside async function");
}
@ -4564,7 +4566,9 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos)
withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos);
assert(s->kind == AsyncWith_kind);
if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
if (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT){
c->u->u_ste->ste_coroutine = 1;
} else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION){
return compiler_error(c, "'async with' outside async function");
}
@ -4773,12 +4777,16 @@ compiler_visit_expr1(struct compiler *c, expr_ty e)
ADDOP(c, YIELD_FROM);
break;
case Await_kind:
if (c->u->u_ste->ste_type != FunctionBlock)
return compiler_error(c, "'await' outside function");
if (!(c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT)){
if (c->u->u_ste->ste_type != FunctionBlock){
return compiler_error(c, "'await' outside function");
}
if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION &&
c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION)
return compiler_error(c, "'await' outside async function");
if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION &&
c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION){
return compiler_error(c, "'await' outside async function");
}
}
VISIT(c, expr, e->v.Await.value);
ADDOP(c, GET_AWAITABLE);
@ -5712,6 +5720,12 @@ compute_code_flags(struct compiler *c)
/* (Only) inherit compilerflags in PyCF_MASK */
flags |= (c->c_flags->cf_flags & PyCF_MASK);
if ((c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) &&
ste->ste_coroutine &&
!ste->ste_generator) {
flags |= CO_COROUTINE;
}
return flags;
}