traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

PEP 0492 -- Coroutines with async and await syntax. Issue #24017.

This commit is contained in:
Yury Selivanov 2015-05-11 22:57:16 -04:00
parent 4e6bf4b3da
commit 7544508f02
72 changed files with 9261 additions and 5739 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Doc/includes/typestruct.h
View File

@ -9,7 +9,7 @@ typedef struct _typeobject {
printfunc tp_print;
getattrfunc tp_getattr;
setattrfunc tp_setattr;
void *tp_reserved;
PyAsyncMethods *tp_as_async;
reprfunc tp_repr;
/* Method suites for standard classes */

42
Doc/library/dis.rst
View File

@ -508,6 +508,38 @@ the original TOS1.
Implements ``del TOS1[TOS]``.
**Coroutines opcodes**
.. opcode:: GET_AWAITABLE
Implements ``TOS = get_awaitable(TOS)``; where ``get_awaitable(o)``
returns ``o`` if ``o`` is a coroutine object; or resolved ``o.__await__``.
.. opcode:: GET_AITER
Implements ``TOS = get_awaitable(TOS.__aiter__())``. See ``GET_AWAITABLE``
for details about ``get_awaitable``
.. opcode:: GET_ANEXT
Implements ``PUSH(get_awaitable(TOS.__anext__()))``. See ``GET_AWAITABLE``
for details about ``get_awaitable``
.. opcode:: BEFORE_ASYNC_WITH
Resolves ``__aenter__`` and ``__aexit__`` from the object on top of the
stack. Pushes ``__aexit__`` and result of ``__aenter__()`` to the stack.
.. opcode:: SETUP_ASYNC_WITH
Creates a new frame object.
**Miscellaneous opcodes**
.. opcode:: PRINT_EXPR
@ -612,7 +644,7 @@ iterations of the loop.
:opcode:`UNPACK_SEQUENCE`).
.. opcode:: WITH_CLEANUP
.. opcode:: WITH_CLEANUP_START
Cleans up the stack when a :keyword:`with` statement block exits. TOS is the
context manager's :meth:`__exit__` bound method. Below TOS are 1--3 values
@ -624,7 +656,13 @@ iterations of the loop.
* (SECOND, THIRD, FOURTH) = exc_info()
In the last case, ``TOS(SECOND, THIRD, FOURTH)`` is called, otherwise
``TOS(None, None, None)``. In addition, TOS is removed from the stack.
``TOS(None, None, None)``. Pushes SECOND and result of the call
to the stack.
.. opcode:: WITH_CLEANUP_FINISH
Pops exception type and result of 'exit' function call from the stack.
If the stack represents an exception, *and* the function call returns a
'true' value, this information is "zapped" and replaced with a single

11
Grammar/Grammar
View File

@ -21,8 +21,11 @@ eval_input: testlist NEWLINE* ENDMARKER
decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
decorators: decorator+
decorated: decorators (classdef | funcdef)
decorated: decorators (classdef | funcdef | async_funcdef)
async_funcdef: ASYNC funcdef
funcdef: 'def' NAME parameters ['->' test] ':' suite
parameters: '(' [typedargslist] ')'
typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [','
['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]]
@ -65,7 +68,8 @@ global_stmt: 'global' NAME (',' NAME)*
nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
assert_stmt: 'assert' test [',' test]
compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
while_stmt: 'while' test ':' suite ['else' ':' suite]
for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
@ -99,7 +103,8 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)*
arith_expr: term (('+'|'-') term)*
term: factor (('*'|'@'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
power: atom trailer* ['**' factor]
power: atom_expr ['**' factor]
atom_expr: [AWAIT] atom trailer*
atom: ('(' [yield_expr|testlist_comp] ')' |
'[' [testlist_comp] ']' |
'{' [dictorsetmaker] '}' |

59
Include/Python-ast.h
View File

@ -63,12 +63,13 @@ struct _mod {
} v;
};
enum _stmt_kind {FunctionDef_kind=1, ClassDef_kind=2, Return_kind=3,
Delete_kind=4, Assign_kind=5, AugAssign_kind=6, For_kind=7,
While_kind=8, If_kind=9, With_kind=10, Raise_kind=11,
Try_kind=12, Assert_kind=13, Import_kind=14,
ImportFrom_kind=15, Global_kind=16, Nonlocal_kind=17,
Expr_kind=18, Pass_kind=19, Break_kind=20, Continue_kind=21};
enum _stmt_kind {FunctionDef_kind=1, AsyncFunctionDef_kind=2, ClassDef_kind=3,
Return_kind=4, Delete_kind=5, Assign_kind=6,
AugAssign_kind=7, For_kind=8, AsyncFor_kind=9, While_kind=10,
If_kind=11, With_kind=12, AsyncWith_kind=13, Raise_kind=14,
Try_kind=15, Assert_kind=16, Import_kind=17,
ImportFrom_kind=18, Global_kind=19, Nonlocal_kind=20,
Expr_kind=21, Pass_kind=22, Break_kind=23, Continue_kind=24};
struct _stmt {
enum _stmt_kind kind;
union {
@ -80,6 +81,14 @@ struct _stmt {
expr_ty returns;
} FunctionDef;
struct {
identifier name;
arguments_ty args;
asdl_seq *body;
asdl_seq *decorator_list;
expr_ty returns;
} AsyncFunctionDef;
struct {
identifier name;
asdl_seq *bases;
@ -114,6 +123,13 @@ struct _stmt {
asdl_seq *orelse;
} For;
struct {
expr_ty target;
expr_ty iter;
asdl_seq *body;
asdl_seq *orelse;
} AsyncFor;
struct {
expr_ty test;
asdl_seq *body;
@ -131,6 +147,11 @@ struct _stmt {
asdl_seq *body;
} With;
struct {
asdl_seq *items;
asdl_seq *body;
} AsyncWith;
struct {
expr_ty exc;
expr_ty cause;
@ -178,11 +199,11 @@ struct _stmt {
enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
IfExp_kind=5, Dict_kind=6, Set_kind=7, ListComp_kind=8,
SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11,
Yield_kind=12, YieldFrom_kind=13, Compare_kind=14,
Call_kind=15, Num_kind=16, Str_kind=17, Bytes_kind=18,
NameConstant_kind=19, Ellipsis_kind=20, Attribute_kind=21,
Subscript_kind=22, Starred_kind=23, Name_kind=24,
List_kind=25, Tuple_kind=26};
Await_kind=12, Yield_kind=13, YieldFrom_kind=14,
Compare_kind=15, Call_kind=16, Num_kind=17, Str_kind=18,
Bytes_kind=19, NameConstant_kind=20, Ellipsis_kind=21,
Attribute_kind=22, Subscript_kind=23, Starred_kind=24,
Name_kind=25, List_kind=26, Tuple_kind=27};
struct _expr {
enum _expr_kind kind;
union {
@ -243,6 +264,10 @@ struct _expr {
asdl_seq *generators;
} GeneratorExp;
struct {
expr_ty value;
} Await;
struct {
expr_ty value;
} Yield;
@ -402,6 +427,10 @@ mod_ty _Py_Suite(asdl_seq * body, PyArena *arena);
stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
asdl_seq * decorator_list, expr_ty returns, int lineno,
int col_offset, PyArena *arena);
#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
stmt_ty _Py_AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq *
body, asdl_seq * decorator_list, expr_ty returns,
int lineno, int col_offset, PyArena *arena);
#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7)
stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords,
asdl_seq * body, asdl_seq * decorator_list, int lineno,
@ -420,6 +449,9 @@ stmt_ty _Py_AugAssign(expr_ty target, operator_ty op, expr_ty value, int
#define For(a0, a1, a2, a3, a4, a5, a6) _Py_For(a0, a1, a2, a3, a4, a5, a6)
stmt_ty _Py_For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq *
orelse, int lineno, int col_offset, PyArena *arena);
#define AsyncFor(a0, a1, a2, a3, a4, a5, a6) _Py_AsyncFor(a0, a1, a2, a3, a4, a5, a6)
stmt_ty _Py_AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq *
orelse, int lineno, int col_offset, PyArena *arena);
#define While(a0, a1, a2, a3, a4, a5) _Py_While(a0, a1, a2, a3, a4, a5)
stmt_ty _Py_While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
int col_offset, PyArena *arena);
@ -429,6 +461,9 @@ stmt_ty _Py_If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
#define With(a0, a1, a2, a3, a4) _Py_With(a0, a1, a2, a3, a4)
stmt_ty _Py_With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset,
PyArena *arena);
#define AsyncWith(a0, a1, a2, a3, a4) _Py_AsyncWith(a0, a1, a2, a3, a4)
stmt_ty _Py_AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int
col_offset, PyArena *arena);
#define Raise(a0, a1, a2, a3, a4) _Py_Raise(a0, a1, a2, a3, a4)
stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset,
PyArena *arena);
@ -491,6 +526,8 @@ expr_ty _Py_DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int
#define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4)
expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
col_offset, PyArena *arena);
#define Await(a0, a1, a2, a3) _Py_Await(a0, a1, a2, a3)
expr_ty _Py_Await(expr_ty value, int lineno, int col_offset, PyArena *arena);
#define Yield(a0, a1, a2, a3) _Py_Yield(a0, a1, a2, a3)
expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, PyArena *arena);
#define YieldFrom(a0, a1, a2, a3) _Py_YieldFrom(a0, a1, a2, a3)

2
Include/ceval.h
View File

@ -23,6 +23,8 @@ PyAPI_FUNC(PyObject *) PyEval_CallMethod(PyObject *obj,
#ifndef Py_LIMITED_API
PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *);
PyAPI_FUNC(void) PyEval_SetTrace(Py_tracefunc, PyObject *);
PyAPI_FUNC(void) PyEval_SetCoroutineWrapper(PyObject *wrapper);
PyAPI_FUNC(PyObject *) PyEval_GetCoroutineWrapper();
#endif
struct _frame; /* Avoid including frameobject.h */

5
Include/code.h
View File

@ -51,6 +51,11 @@ typedef struct {
*/
#define CO_NOFREE 0x0040
/* The CO_COROUTINE flag is set for coroutine functions (defined with
``async def`` keywords) */
#define CO_COROUTINE 0x0080
#define CO_ITERABLE_COROUTINE 0x0100
/* These are no longer used. */
#if 0
#define CO_GENERATOR_ALLOWED 0x1000

7
Include/genobject.h
View File

@ -38,6 +38,12 @@ PyAPI_DATA(PyTypeObject) PyGen_Type;
#define PyGen_Check(op) PyObject_TypeCheck(op, &PyGen_Type)
#define PyGen_CheckExact(op) (Py_TYPE(op) == &PyGen_Type)
#define PyGen_CheckCoroutineExact(op) (PyGen_CheckExact(op) && \
(((PyCodeObject*) \
((PyGenObject*)op)->gi_code) \
->co_flags & (CO_ITERABLE_COROUTINE | \
CO_COROUTINE)))
PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *);
PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(struct _frame *,
PyObject *name, PyObject *qualname);
@ -46,6 +52,7 @@ PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **);
PyObject *_PyGen_Send(PyGenObject *, PyObject *);
PyAPI_FUNC(void) _PyGen_Finalize(PyObject *self);
PyObject *_PyGen_GetAwaitableIter(PyObject *o);
#ifdef __cplusplus
}

155
Include/graminit.h
View File

@ -6,79 +6,82 @@
#define decorator 259
#define decorators 260
#define decorated 261
#define funcdef 262
#define parameters 263
#define typedargslist 264
#define tfpdef 265
#define varargslist 266
#define vfpdef 267
#define stmt 268
#define simple_stmt 269
#define small_stmt 270
#define expr_stmt 271
#define testlist_star_expr 272
#define augassign 273
#define del_stmt 274
#define pass_stmt 275
#define flow_stmt 276
#define break_stmt 277
#define continue_stmt 278
#define return_stmt 279
#define yield_stmt 280
#define raise_stmt 281
#define import_stmt 282
#define import_name 283
#define import_from 284
#define import_as_name 285
#define dotted_as_name 286
#define import_as_names 287
#define dotted_as_names 288
#define dotted_name 289
#define global_stmt 290
#define nonlocal_stmt 291
#define assert_stmt 292
#define compound_stmt 293
#define if_stmt 294
#define while_stmt 295
#define for_stmt 296
#define try_stmt 297
#define with_stmt 298
#define with_item 299
#define except_clause 300
#define suite 301
#define test 302
#define test_nocond 303
#define lambdef 304
#define lambdef_nocond 305
#define or_test 306
#define and_test 307
#define not_test 308
#define comparison 309
#define comp_op 310
#define star_expr 311
#define expr 312
#define xor_expr 313
#define and_expr 314
#define shift_expr 315
#define arith_expr 316
#define term 317
#define factor 318
#define power 319
#define atom 320
#define testlist_comp 321
#define trailer 322
#define subscriptlist 323
#define subscript 324
#define sliceop 325
#define exprlist 326
#define testlist 327
#define dictorsetmaker 328
#define classdef 329
#define arglist 330
#define argument 331
#define comp_iter 332
#define comp_for 333
#define comp_if 334
#define encoding_decl 335
#define yield_expr 336
#define yield_arg 337
#define async_funcdef 262
#define funcdef 263
#define parameters 264
#define typedargslist 265
#define tfpdef 266
#define varargslist 267
#define vfpdef 268
#define stmt 269
#define simple_stmt 270
#define small_stmt 271
#define expr_stmt 272
#define testlist_star_expr 273
#define augassign 274
#define del_stmt 275
#define pass_stmt 276
#define flow_stmt 277
#define break_stmt 278
#define continue_stmt 279
#define return_stmt 280
#define yield_stmt 281
#define raise_stmt 282
#define import_stmt 283
#define import_name 284
#define import_from 285
#define import_as_name 286
#define dotted_as_name 287
#define import_as_names 288
#define dotted_as_names 289
#define dotted_name 290
#define global_stmt 291
#define nonlocal_stmt 292
#define assert_stmt 293
#define compound_stmt 294
#define async_stmt 295
#define if_stmt 296
#define while_stmt 297
#define for_stmt 298
#define try_stmt 299
#define with_stmt 300
#define with_item 301
#define except_clause 302
#define suite 303
#define test 304
#define test_nocond 305
#define lambdef 306
#define lambdef_nocond 307
#define or_test 308
#define and_test 309
#define not_test 310
#define comparison 311
#define comp_op 312
#define star_expr 313
#define expr 314
#define xor_expr 315
#define and_expr 316
#define shift_expr 317
#define arith_expr 318
#define term 319
#define factor 320
#define power 321
#define atom_expr 322
#define atom 323
#define testlist_comp 324
#define trailer 325
#define subscriptlist 326
#define subscript 327
#define sliceop 328
#define exprlist 329
#define testlist 330
#define dictorsetmaker 331
#define classdef 332
#define arglist 333
#define argument 334
#define comp_iter 335
#define comp_for 336
#define comp_if 337
#define encoding_decl 338
#define yield_expr 339
#define yield_arg 340

11
Include/object.h
View File

@ -173,6 +173,9 @@ typedef PyObject *(*ssizessizeargfunc)(PyObject *, Py_ssize_t, Py_ssize_t);
typedef int(*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *);
typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *);
typedef int(*objobjargproc)(PyObject *, PyObject *, PyObject *);
typedef PyObject *(*getawaitablefunc) (PyObject *);
typedef PyObject *(*getaiterfunc) (PyObject *);
typedef PyObject *(*aiternextfunc) (PyObject *);
#ifndef Py_LIMITED_API
/* buffer interface */
@ -301,6 +304,11 @@ typedef struct {
objobjargproc mp_ass_subscript;
} PyMappingMethods;
typedef struct {
getawaitablefunc am_await;
getaiterfunc am_aiter;
aiternextfunc am_anext;
} PyAsyncMethods;
typedef struct {
getbufferproc bf_getbuffer;
@ -346,7 +354,7 @@ typedef struct _typeobject {
printfunc tp_print;
getattrfunc tp_getattr;
setattrfunc tp_setattr;
void *tp_reserved; /* formerly known as tp_compare */
PyAsyncMethods *tp_as_async; /* formerly known as tp_compare or tp_reserved */
reprfunc tp_repr;
/* Method suites for standard classes */
@ -453,6 +461,7 @@ typedef struct _heaptypeobject {
/* Note: there's a dependency on the order of these members
in slotptr() in typeobject.c . */
PyTypeObject ht_type;
PyAsyncMethods as_async;
PyNumberMethods as_number;
PyMappingMethods as_mapping;
PySequenceMethods as_sequence; /* as_sequence comes after as_mapping,

8
Include/opcode.h
View File

@ -29,6 +29,9 @@ extern "C" {
#define BINARY_TRUE_DIVIDE 27
#define INPLACE_FLOOR_DIVIDE 28
#define INPLACE_TRUE_DIVIDE 29
#define GET_AITER 50
#define GET_ANEXT 51
#define BEFORE_ASYNC_WITH 52
#define STORE_MAP 54
#define INPLACE_ADD 55
#define INPLACE_SUBTRACT 56
@ -46,13 +49,15 @@ extern "C" {
#define PRINT_EXPR 70
#define LOAD_BUILD_CLASS 71
#define YIELD_FROM 72
#define GET_AWAITABLE 73
#define INPLACE_LSHIFT 75
#define INPLACE_RSHIFT 76
#define INPLACE_AND 77
#define INPLACE_XOR 78
#define INPLACE_OR 79
#define BREAK_LOOP 80
#define WITH_CLEANUP 81
#define WITH_CLEANUP_START 81
#define WITH_CLEANUP_FINISH 82
#define RETURN_VALUE 83
#define IMPORT_STAR 84
#define YIELD_VALUE 86
@ -116,6 +121,7 @@ extern "C" {
#define BUILD_MAP_UNPACK_WITH_CALL 151
#define BUILD_TUPLE_UNPACK 152
#define BUILD_SET_UNPACK 153
#define SETUP_ASYNC_WITH 154
/* EXCEPT_HANDLER is a special, implicit block type which is created when
entering an except handler. It is not an opcode but we define it here

1
Include/pyerrors.h
View File

@ -147,6 +147,7 @@ PyAPI_FUNC(void) _PyErr_ChainExceptions(PyObject *, PyObject *, PyObject *);
PyAPI_DATA(PyObject *) PyExc_BaseException;
PyAPI_DATA(PyObject *) PyExc_Exception;
PyAPI_DATA(PyObject *) PyExc_StopAsyncIteration;
PyAPI_DATA(PyObject *) PyExc_StopIteration;
PyAPI_DATA(PyObject *) PyExc_GeneratorExit;
PyAPI_DATA(PyObject *) PyExc_ArithmeticError;

2
Include/pystate.h
View File

@ -134,6 +134,8 @@ typedef struct _ts {
void (*on_delete)(void *);
void *on_delete_data;
PyObject *coroutine_wrapper;
/* XXX signal handlers should also be here */
} PyThreadState;

6
Include/token.h
View File

@ -64,8 +64,10 @@ extern "C" {
#define ELLIPSIS 52
/* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */
#define OP 53
#define ERRORTOKEN 54
#define N_TOKENS 55
#define AWAIT 54
#define ASYNC 55
#define ERRORTOKEN 56
#define N_TOKENS 57
/* Special definitions for cooperation with parser */

3
Include/typeslots.h
View File

@ -76,3 +76,6 @@
#define Py_tp_free 74
#define Py_nb_matrix_multiply 75
#define Py_nb_inplace_matrix_multiply 76
#define Py_am_await 77
#define Py_am_aiter 78
#define Py_am_anext 79

76
Lib/_collections_abc.py
View File

@ -9,7 +9,8 @@ Unit tests are in test_collections.
from abc import ABCMeta, abstractmethod
import sys
__all__ = ["Hashable", "Iterable", "Iterator", "Generator",
__all__ = ["Awaitable", "Coroutine",
"Hashable", "Iterable", "Iterator", "Generator",
"Sized", "Container", "Callable",
"Set", "MutableSet",
"Mapping", "MutableMapping",
@ -74,6 +75,79 @@ class Hashable(metaclass=ABCMeta):
return NotImplemented
class _CoroutineMeta(ABCMeta):
def __instancecheck__(cls, instance):
# 0x80 = CO_COROUTINE
# 0x100 = CO_ITERABLE_COROUTINE
# We don't want to import 'inspect' module, as
# a dependency for 'collections.abc'.
CO_COROUTINES = 0x80 | 0x100
if (isinstance(instance, generator) and
instance.gi_code.co_flags & CO_COROUTINES):
return True
return super().__instancecheck__(instance)
class Coroutine(metaclass=_CoroutineMeta):
__slots__ = ()
@abstractmethod
def send(self, value):
"""Send a value into the coroutine.
Return next yielded value or raise StopIteration.
"""
raise StopIteration
@abstractmethod
def throw(self, typ, val=None, tb=None):
"""Raise an exception in the coroutine.
Return next yielded value or raise StopIteration.
"""
if val is None:
if tb is None:
raise typ
val = typ()
if tb is not None:
val = val.with_traceback(tb)
raise val
def close(self):
"""Raise GeneratorExit inside coroutine.
"""
try:
self.throw(GeneratorExit)
except (GeneratorExit, StopIteration):
pass
else:
raise RuntimeError("coroutine ignored GeneratorExit")
class Awaitable(metaclass=_CoroutineMeta):
__slots__ = ()
@abstractmethod
def __await__(self):
yield
@classmethod
def __subclasshook__(cls, C):
if cls is Awaitable:
for B in C.__mro__:
if "__await__" in B.__dict__:
if B.__dict__["__await__"]:
return True
break
return NotImplemented
Awaitable.register(Coroutine)
class Iterable(metaclass=ABCMeta):
__slots__ = ()

2
Lib/dis.py
View File

@ -84,6 +84,8 @@ COMPILER_FLAG_NAMES = {
16: "NESTED",
32: "GENERATOR",
64: "NOFREE",
128: "COROUTINE",
256: "ITERABLE_COROUTINE",
}
def pretty_flags(flags):

26
Lib/inspect.py
View File

@ -33,6 +33,7 @@ __author__ = ('Ka-Ping Yee <ping@lfw.org>',
import ast
import dis
import collections.abc
import enum
import importlib.machinery
import itertools
@ -174,7 +175,22 @@ def isgeneratorfunction(object):
See help(isfunction) for attributes listing."""
return bool((isfunction(object) or ismethod(object)) and
object.__code__.co_flags & CO_GENERATOR)
object.__code__.co_flags & CO_GENERATOR and
not object.__code__.co_flags & CO_COROUTINE)
def iscoroutinefunction(object):
"""Return true if the object is a coroutine function.
Coroutine functions are defined with "async def" syntax,
or generators decorated with "types.coroutine".
"""
return bool((isfunction(object) or ismethod(object)) and
object.__code__.co_flags & (CO_ITERABLE_COROUTINE |
CO_COROUTINE))
def isawaitable(object):
"""Return true if the object can be used in "await" expression."""
return isinstance(object, collections.abc.Awaitable)
def isgenerator(object):
"""Return true if the object is a generator.
@ -191,7 +207,13 @@ def isgenerator(object):
send resumes the generator and "sends" a value that becomes
the result of the current yield-expression
throw used to raise an exception inside the generator"""
return isinstance(object, types.GeneratorType)
return (isinstance(object, types.GeneratorType) and
not object.gi_code.co_flags & CO_COROUTINE)
def iscoroutine(object):
"""Return true if the object is a coroutine."""
return (isinstance(object, types.GeneratorType) and
object.gi_code.co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))
def istraceback(object):
"""Return true if the object is a traceback.

8
Lib/lib2to3/Grammar.txt
View File

@ -33,7 +33,8 @@ eval_input: testlist NEWLINE* ENDMARKER
decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
decorators: decorator+
decorated: decorators (classdef | funcdef)
decorated: decorators (classdef | funcdef | async_funcdef)
async_funcdef: ASYNC funcdef
funcdef: 'def' NAME parameters ['->' test] ':' suite
parameters: '(' [typedargslist] ')'
typedargslist: ((tfpdef ['=' test] ',')*
@ -82,7 +83,8 @@ global_stmt: ('global' | 'nonlocal') NAME (',' NAME)*
exec_stmt: 'exec' expr ['in' test [',' test]]
assert_stmt: 'assert' test [',' test]
compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
while_stmt: 'while' test ':' suite ['else' ':' suite]
for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
@ -121,7 +123,7 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)*
arith_expr: term (('+'|'-') term)*
term: factor (('*'|'@'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
power: atom trailer* ['**' factor]
power: [AWAIT] atom trailer* ['**' factor]
atom: ('(' [yield_expr|testlist_gexp] ')' |
'[' [listmaker] ']' |
'{' [dictsetmaker] '}' |

6
Lib/lib2to3/pgen2/token.py
View File

@ -62,8 +62,10 @@ OP = 52
COMMENT = 53
NL = 54
RARROW = 55
ERRORTOKEN = 56
N_TOKENS = 57
AWAIT = 56
ASYNC = 57
ERRORTOKEN = 58
N_TOKENS = 59
NT_OFFSET = 256
#--end constants--

63
Lib/lib2to3/pgen2/tokenize.py
View File

@ -220,7 +220,7 @@ class Untokenizer:
for tok in iterable:
toknum, tokval = tok[:2]
if toknum in (NAME, NUMBER):
if toknum in (NAME, NUMBER, ASYNC, AWAIT):
tokval += ' '
if toknum == INDENT:
@ -366,6 +366,10 @@ def generate_tokens(readline):
contline = None
indents = [0]
# 'stashed' and 'ctx' are used for async/await parsing
stashed = None
ctx = [('sync', 0)]
while 1: # loop over lines in stream
try:
line = readline()
@ -406,6 +410,10 @@ def generate_tokens(readline):
pos = pos + 1
if pos == max: break
if stashed:
yield stashed
stashed = None
if line[pos] in '#\r\n': # skip comments or blank lines
if line[pos] == '#':
comment_token = line[pos:].rstrip('\r\n')
@ -449,9 +457,15 @@ def generate_tokens(readline):
newline = NEWLINE
if parenlev > 0:
newline = NL
if stashed:
yield stashed
stashed = None
yield (newline, token, spos, epos, line)
elif initial == '#':
assert not token.endswith("\n")
if stashed:
yield stashed
stashed = None
yield (COMMENT, token, spos, epos, line)
elif token in triple_quoted:
endprog = endprogs[token]
@ -459,6 +473,9 @@ def generate_tokens(readline):
if endmatch: # all on one line
pos = endmatch.end(0)
token = line[start:pos]
if stashed:
yield stashed
stashed = None
yield (STRING, token, spos, (lnum, pos), line)
else:
strstart = (lnum, start) # multiple lines
@ -476,22 +493,64 @@ def generate_tokens(readline):
contline = line
break
else: # ordinary string
if stashed:
yield stashed
stashed = None
yield (STRING, token, spos, epos, line)
elif initial in namechars: # ordinary name
yield (NAME, token, spos, epos, line)
if token in ('async', 'await'):
if ctx[-1][0] == 'async' and ctx[-1][1] < indents[-1]:
yield (ASYNC if token == 'async' else AWAIT,
token, spos, epos, line)
continue
tok = (NAME, token, spos, epos, line)
if token == 'async' and not stashed:
stashed = tok
continue
if token == 'def':
if (stashed
and stashed[0] == NAME
and stashed[1] == 'async'):
ctx.append(('async', indents[-1]))
yield (ASYNC, stashed[1],
stashed[2], stashed[3],
stashed[4])
stashed = None
else:
ctx.append(('sync', indents[-1]))
if stashed:
yield stashed
stashed = None
yield tok
elif initial == '\\': # continued stmt
# This yield is new; needed for better idempotency:
if stashed:
yield stashed
stashed = None
yield (NL, token, spos, (lnum, pos), line)
continued = 1
else:
if initial in '([{': parenlev = parenlev + 1
elif initial in ')]}': parenlev = parenlev - 1
if stashed:
yield stashed
stashed = None
yield (OP, token, spos, epos, line)
else:
yield (ERRORTOKEN, line[pos],
(lnum, pos), (lnum, pos+1), line)
pos = pos + 1
if stashed:
yield stashed
stashed = None
for indent in indents[1:]: # pop remaining indent levels
yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')

32
Lib/lib2to3/tests/test_parser.py
View File

@ -55,12 +55,42 @@ class TestMatrixMultiplication(GrammarTest):
class TestYieldFrom(GrammarTest):
def test_matrix_multiplication_operator(self):
def test_yield_from(self):
self.validate("yield from x")
self.validate("(yield from x) + y")
self.invalid_syntax("yield from")
class TestAsyncAwait(GrammarTest):
def test_await_expr(self):
self.validate("""async def foo():
await x
""")
self.invalid_syntax("await x")
self.invalid_syntax("""def foo():
await x""")
def test_async_var(self):
self.validate("""async = 1""")
self.validate("""await = 1""")
self.validate("""def async(): pass""")
def test_async_with(self):
self.validate("""async def foo():
async for a in b: pass""")
self.invalid_syntax("""def foo():
async for a in b: pass""")
def test_async_for(self):
self.validate("""async def foo():
async with a: pass""")
self.invalid_syntax("""def foo():
async with a: pass""")
class TestRaiseChanges(GrammarTest):
def test_2x_style_1(self):
self.validate("raise")

10
Lib/opcode.py
View File

@ -85,6 +85,10 @@ def_op('BINARY_TRUE_DIVIDE', 27)
def_op('INPLACE_FLOOR_DIVIDE', 28)
def_op('INPLACE_TRUE_DIVIDE', 29)
def_op('GET_AITER', 50)
def_op('GET_ANEXT', 51)
def_op('BEFORE_ASYNC_WITH', 52)
def_op('STORE_MAP', 54)
def_op('INPLACE_ADD', 55)
def_op('INPLACE_SUBTRACT', 56)
@ -104,6 +108,7 @@ def_op('GET_ITER', 68)
def_op('PRINT_EXPR', 70)
def_op('LOAD_BUILD_CLASS', 71)
def_op('YIELD_FROM', 72)
def_op('GET_AWAITABLE', 73)
def_op('INPLACE_LSHIFT', 75)
def_op('INPLACE_RSHIFT', 76)
@ -111,7 +116,8 @@ def_op('INPLACE_AND', 77)
def_op('INPLACE_XOR', 78)
def_op('INPLACE_OR', 79)
def_op('BREAK_LOOP', 80)
def_op('WITH_CLEANUP', 81)
def_op('WITH_CLEANUP_START', 81)
def_op('WITH_CLEANUP_FINISH', 82)
def_op('RETURN_VALUE', 83)
def_op('IMPORT_STAR', 84)
@ -197,6 +203,8 @@ def_op('MAP_ADD', 147)
def_op('LOAD_CLASSDEREF', 148)
hasfree.append(148)
jrel_op('SETUP_ASYNC_WITH', 154)
def_op('EXTENDED_ARG', 144)
EXTENDED_ARG = 144

3
Lib/test/badsyntax_async1.py Normal file
View File

@ -0,0 +1,3 @@
async def foo():
def foo(a=await something()):
pass

3
Lib/test/badsyntax_async2.py Normal file
View File

@ -0,0 +1,3 @@
async def foo():
def foo(a:await something()):
pass

2
Lib/test/badsyntax_async3.py Normal file
View File

@ -0,0 +1,2 @@
async def foo():
[i async for i in els]

2
Lib/test/badsyntax_async4.py Normal file
View File

@ -0,0 +1,2 @@
async def foo():
async def foo(): await something()

2
Lib/test/badsyntax_async5.py Normal file
View File

@ -0,0 +1,2 @@
def foo():
await something()

2
Lib/test/badsyntax_async6.py Normal file
View File

@ -0,0 +1,2 @@
async def foo():
yield

2
Lib/test/badsyntax_async7.py Normal file
View File

@ -0,0 +1,2 @@
async def foo():
yield from []

2
Lib/test/badsyntax_async8.py Normal file
View File

@ -0,0 +1,2 @@
async def foo():
await await fut

2
Lib/test/badsyntax_async9.py Normal file
View File

@ -0,0 +1,2 @@
async def foo():
await

1
Lib/test/exception_hierarchy.txt
View File

@ -4,6 +4,7 @@ BaseException
+-- GeneratorExit
+-- Exception
+-- StopIteration
+-- StopAsyncIteration
+-- ArithmeticError
| +-- FloatingPointError
| +-- OverflowError

9
Lib/test/test_ast.py
View File

@ -106,6 +106,12 @@ exec_tests = [
"{r for l in x if g}",
# setcomp with naked tuple
"{r for l,m in x}",
# AsyncFunctionDef
"async def f():\n await something()",
# AsyncFor
"async def f():\n async for e in i: 1\n else: 2",
# AsyncWith
"async def f():\n async with a as b: 1",
]
# These are compiled through "single"
@ -974,6 +980,9 @@ exec_results = [
('Module', [('Expr', (1, 0), ('DictComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [])]))]),
('Module', [('Expr', (1, 0), ('SetComp', (1, 1), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))])]))]),
('Module', [('Expr', (1, 0), ('SetComp', (1, 1), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [])]))]),
('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('Await', (2, 1), ('Call', (2, 7), ('Name', (2, 7), 'something', ('Load',)), [], [])))], [], None)]),
('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('AsyncFor', (2, 7), ('Name', (2, 11), 'e', ('Store',)), ('Name', (2, 16), 'i', ('Load',)), [('Expr', (2, 19), ('Num', (2, 19), 1))], [('Expr', (3, 7), ('Num', (3, 7), 2))])], [], None)]),
('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('AsyncWith', (2, 7), [('withitem', ('Name', (2, 12), 'a', ('Load',)), ('Name', (2, 17), 'b', ('Store',)))], [('Expr', (2, 20), ('Num', (2, 20), 1))])], [], None)]),
]
single_results = [
('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),

80
Lib/test/test_collections.py
View File

@ -11,9 +11,11 @@ from random import randrange, shuffle
import keyword
import re
import sys
import types
from collections import UserDict
from collections import ChainMap
from collections import deque
from collections.abc import Awaitable, Coroutine
from collections.abc import Hashable, Iterable, Iterator, Generator
from collections.abc import Sized, Container, Callable
from collections.abc import Set, MutableSet
@ -446,6 +448,84 @@ class ABCTestCase(unittest.TestCase):
class TestOneTrickPonyABCs(ABCTestCase):
def test_Awaitable(self):
def gen():
yield
@types.coroutine
def coro():
yield
async def new_coro():
pass
class Bar:
def __await__(self):
yield
class MinimalCoro(Coroutine):
def send(self, value):
return value
def throw(self, typ, val=None, tb=None):
super().throw(typ, val, tb)
non_samples = [None, int(), gen(), object()]
for x in non_samples:
self.assertNotIsInstance(x, Awaitable)
self.assertFalse(issubclass(type(x), Awaitable), repr(type(x)))
samples = [Bar(), MinimalCoro()]
for x in samples:
self.assertIsInstance(x, Awaitable)
self.assertTrue(issubclass(type(x), Awaitable))
c = coro()
self.assertIsInstance(c, Awaitable)
c.close() # awoid RuntimeWarning that coro() was not awaited
c = new_coro()
self.assertIsInstance(c, Awaitable)
c.close() # awoid RuntimeWarning that coro() was not awaited
def test_Coroutine(self):
def gen():
yield
@types.coroutine
def coro():
yield
async def new_coro():
pass
class Bar:
def __await__(self):
yield
class MinimalCoro(Coroutine):
def send(self, value):
return value
def throw(self, typ, val=None, tb=None):
super().throw(typ, val, tb)
non_samples = [None, int(), gen(), object(), Bar()]
for x in non_samples:
self.assertNotIsInstance(x, Coroutine)
self.assertFalse(issubclass(type(x), Coroutine), repr(type(x)))
samples = [MinimalCoro()]
for x in samples:
self.assertIsInstance(x, Awaitable)
self.assertTrue(issubclass(type(x), Awaitable))
c = coro()
self.assertIsInstance(c, Coroutine)
c.close() # awoid RuntimeWarning that coro() was not awaited
c = new_coro()
self.assertIsInstance(c, Coroutine)
c.close() # awoid RuntimeWarning that coro() was not awaited
def test_Hashable(self):
# Check some non-hashables
non_samples = [bytearray(), list(), set(), dict()]

862
Lib/test/test_coroutines.py Normal file
View File

@ -0,0 +1,862 @@
import contextlib
import gc
import sys
import types
import unittest
import warnings
from test import support
class AsyncYieldFrom:
def __init__(self, obj):
self.obj = obj
def __await__(self):
yield from self.obj
class AsyncYield:
def __init__(self, value):
self.value = value
def __await__(self):
yield self.value
def run_async(coro):
assert coro.__class__ is types.GeneratorType
buffer = []
result = None
while True:
try:
buffer.append(coro.send(None))
except StopIteration as ex:
result = ex.args[0] if ex.args else None
break
return buffer, result
@contextlib.contextmanager
def silence_coro_gc():
with warnings.catch_warnings():
warnings.simplefilter("ignore")
yield
support.gc_collect()
class AsyncBadSyntaxTest(unittest.TestCase):
def test_badsyntax_1(self):
with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
import test.badsyntax_async1
def test_badsyntax_2(self):
with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
import test.badsyntax_async2
def test_badsyntax_3(self):
with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
import test.badsyntax_async3
def test_badsyntax_4(self):
with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
import test.badsyntax_async4
def test_badsyntax_5(self):
with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
import test.badsyntax_async5
def test_badsyntax_6(self):
with self.assertRaisesRegex(
SyntaxError, "'yield' inside async function"):
import test.badsyntax_async6
def test_badsyntax_7(self):
with self.assertRaisesRegex(
SyntaxError, "'yield from' inside async function"):
import test.badsyntax_async7
def test_badsyntax_8(self):
with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
import test.badsyntax_async8
def test_badsyntax_9(self):
with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
import test.badsyntax_async9
class CoroutineTest(unittest.TestCase):
def test_gen_1(self):
def gen(): yield
self.assertFalse(hasattr(gen, '__await__'))
def test_func_1(self):
async def foo():
return 10
f = foo()
self.assertIsInstance(f, types.GeneratorType)
self.assertTrue(bool(foo.__code__.co_flags & 0x80))
self.assertTrue(bool(foo.__code__.co_flags & 0x20))
self.assertTrue(bool(f.gi_code.co_flags & 0x80))
self.assertTrue(bool(f.gi_code.co_flags & 0x20))
self.assertEqual(run_async(f), ([], 10))
def bar(): pass
self.assertFalse(bool(bar.__code__.co_flags & 0x80))
def test_func_2(self):
async def foo():
raise StopIteration
with self.assertRaisesRegex(
RuntimeError, "generator raised StopIteration"):
run_async(foo())
def test_func_3(self):
async def foo():
raise StopIteration
with silence_coro_gc():
self.assertRegex(repr(foo()), '^<coroutine object.* at 0x.*>$')
def test_func_4(self):
async def foo():
raise StopIteration
check = lambda: self.assertRaisesRegex(
TypeError, "coroutine-objects do not support iteration")
with check():
list(foo())
with check():
tuple(foo())
with check():
sum(foo())
with check():
iter(foo())
with check():
next(foo())
with silence_coro_gc(), check():
for i in foo():
pass
with silence_coro_gc(), check():
[i for i in foo()]
def test_func_5(self):
@types.coroutine
def bar():
yield 1
async def foo():
await bar()
check = lambda: self.assertRaisesRegex(
TypeError, "coroutine-objects do not support iteration")
with check():
for el in foo(): pass
# the following should pass without an error
for el in bar():
self.assertEqual(el, 1)
self.assertEqual([el for el in bar()], [1])
self.assertEqual(tuple(bar()), (1,))
self.assertEqual(next(iter(bar())), 1)
def test_func_6(self):
@types.coroutine
def bar():
yield 1
yield 2
async def foo():
await bar()
f = foo()
self.assertEquals(f.send(None), 1)
self.assertEquals(f.send(None), 2)
with self.assertRaises(StopIteration):
f.send(None)
def test_func_7(self):
async def bar():
return 10
def foo():
yield from bar()
with silence_coro_gc(), self.assertRaisesRegex(
TypeError,
"cannot 'yield from' a coroutine object from a generator"):
list(foo())
def test_func_8(self):
@types.coroutine
def bar():
return (yield from foo())
async def foo():
return 'spam'
self.assertEqual(run_async(bar()), ([], 'spam') )
def test_func_9(self):
async def foo(): pass
with self.assertWarnsRegex(
RuntimeWarning, "coroutine '.*test_func_9.*foo' was never awaited"):
foo()
support.gc_collect()
def test_await_1(self):
async def foo():
await 1
with self.assertRaisesRegex(TypeError, "object int can.t.*await"):
run_async(foo())
def test_await_2(self):
async def foo():
await []
with self.assertRaisesRegex(TypeError, "object list can.t.*await"):
run_async(foo())
def test_await_3(self):
async def foo():
await AsyncYieldFrom([1, 2, 3])
self.assertEqual(run_async(foo()), ([1, 2, 3], None))
def test_await_4(self):
async def bar():
return 42
async def foo():
return await bar()
self.assertEqual(run_async(foo()), ([], 42))
def test_await_5(self):
class Awaitable:
def __await__(self):
return
async def foo():
return (await Awaitable())
with self.assertRaisesRegex(
TypeError, "__await__.*returned non-iterator of type"):
run_async(foo())
def test_await_6(self):
class Awaitable:
def __await__(self):
return iter([52])
async def foo():
return (await Awaitable())
self.assertEqual(run_async(foo()), ([52], None))
def test_await_7(self):
class Awaitable:
def __await__(self):
yield 42
return 100
async def foo():
return (await Awaitable())
self.assertEqual(run_async(foo()), ([42], 100))
def test_await_8(self):
class Awaitable:
pass
async def foo():
return (await Awaitable())
with self.assertRaisesRegex(
TypeError, "object Awaitable can't be used in 'await' expression"):
run_async(foo())
def test_await_9(self):
def wrap():
return bar
async def bar():
return 42
async def foo():
b = bar()
db = {'b': lambda: wrap}
class DB:
b = wrap
return (await bar() + await wrap()() + await db['b']()()() +
await bar() * 1000 + await DB.b()())
async def foo2():
return -await bar()
self.assertEqual(run_async(foo()), ([], 42168))
self.assertEqual(run_async(foo2()), ([], -42))
def test_await_10(self):
async def baz():
return 42
async def bar():
return baz()
async def foo():
return await (await bar())
self.assertEqual(run_async(foo()), ([], 42))
def test_await_11(self):
def ident(val):
return val
async def bar():
return 'spam'
async def foo():
return ident(val=await bar())
async def foo2():
return await bar(), 'ham'
self.assertEqual(run_async(foo2()), ([], ('spam', 'ham')))
def test_await_12(self):
async def coro():
return 'spam'
class Awaitable:
def __await__(self):
return coro()
async def foo():
return await Awaitable()
with self.assertRaisesRegex(
TypeError, "__await__\(\) returned a coroutine"):
run_async(foo())
def test_await_13(self):
class Awaitable:
def __await__(self):
return self
async def foo():
return await Awaitable()
with self.assertRaisesRegex(
TypeError, "__await__.*returned non-iterator of type"):
run_async(foo())
def test_with_1(self):
class Manager:
def __init__(self, name):
self.name = name
async def __aenter__(self):
await AsyncYieldFrom(['enter-1-' + self.name,
'enter-2-' + self.name])
return self
async def __aexit__(self, *args):
await AsyncYieldFrom(['exit-1-' + self.name,
'exit-2-' + self.name])
if self.name == 'B':
return True
async def foo():
async with Manager("A") as a, Manager("B") as b:
await AsyncYieldFrom([('managers', a.name, b.name)])
1/0
f = foo()
result, _ = run_async(f)
self.assertEqual(
result, ['enter-1-A', 'enter-2-A', 'enter-1-B', 'enter-2-B',
('managers', 'A', 'B'),
'exit-1-B', 'exit-2-B', 'exit-1-A', 'exit-2-A']
)
async def foo():
async with Manager("A") as a, Manager("C") as c:
await AsyncYieldFrom([('managers', a.name, c.name)])
1/0
with self.assertRaises(ZeroDivisionError):
run_async(foo())
def test_with_2(self):
class CM:
def __aenter__(self):
pass
async def foo():
async with CM():
pass
with self.assertRaisesRegex(AttributeError, '__aexit__'):
run_async(foo())
def test_with_3(self):
class CM:
def __aexit__(self):
pass
async def foo():
async with CM():
pass
with self.assertRaisesRegex(AttributeError, '__aenter__'):
run_async(foo())
def test_with_4(self):
class CM:
def __enter__(self):
pass
def __exit__(self):
pass
async def foo():
async with CM():
pass
with self.assertRaisesRegex(AttributeError, '__aexit__'):
run_async(foo())
def test_with_5(self):
# While this test doesn't make a lot of sense,
# it's a regression test for an early bug with opcodes
# generation
class CM:
async def __aenter__(self):
return self
async def __aexit__(self, *exc):
pass
async def func():
async with CM():
assert (1, ) == 1
with self.assertRaises(AssertionError):
run_async(func())
def test_with_6(self):
class CM:
def __aenter__(self):
return 123
def __aexit__(self, *e):
return 456
async def foo():
async with CM():
pass
with self.assertRaisesRegex(
TypeError, "object int can't be used in 'await' expression"):
# it's important that __aexit__ wasn't called
run_async(foo())
def test_with_7(self):
class CM:
async def __aenter__(self):
return self
def __aexit__(self, *e):
return 456
async def foo():
async with CM():
pass
with self.assertRaisesRegex(
TypeError, "object int can't be used in 'await' expression"):
run_async(foo())
def test_for_1(self):
aiter_calls = 0
class AsyncIter:
def __init__(self):
self.i = 0
async def __aiter__(self):
nonlocal aiter_calls
aiter_calls += 1
return self
async def __anext__(self):
self.i += 1
if not (self.i % 10):
await AsyncYield(self.i * 10)
if self.i > 100:
raise StopAsyncIteration
return self.i, self.i
buffer = []
async def test1():
async for i1, i2 in AsyncIter():
buffer.append(i1 + i2)
yielded, _ = run_async(test1())
# Make sure that __aiter__ was called only once
self.assertEqual(aiter_calls, 1)
self.assertEqual(yielded, [i * 100 for i in range(1, 11)])
self.assertEqual(buffer, [i*2 for i in range(1, 101)])
buffer = []
async def test2():
nonlocal buffer
async for i in AsyncIter():
buffer.append(i[0])
if i[0] == 20:
break
else:
buffer.append('what?')
buffer.append('end')
yielded, _ = run_async(test2())
# Make sure that __aiter__ was called only once
self.assertEqual(aiter_calls, 2)
self.assertEqual(yielded, [100, 200])
self.assertEqual(buffer, [i for i in range(1, 21)] + ['end'])
buffer = []
async def test3():
nonlocal buffer
async for i in AsyncIter():
if i[0] > 20:
continue
buffer.append(i[0])
else:
buffer.append('what?')
buffer.append('end')
yielded, _ = run_async(test3())
# Make sure that __aiter__ was called only once
self.assertEqual(aiter_calls, 3)
self.assertEqual(yielded, [i * 100 for i in range(1, 11)])
self.assertEqual(buffer, [i for i in range(1, 21)] +
['what?', 'end'])
def test_for_2(self):
tup = (1, 2, 3)
refs_before = sys.getrefcount(tup)
async def foo():
async for i in tup:
print('never going to happen')
with self.assertRaisesRegex(
TypeError, "async for' requires an object.*__aiter__.*tuple"):
run_async(foo())
self.assertEqual(sys.getrefcount(tup), refs_before)
def test_for_3(self):
class I:
def __aiter__(self):
return self
aiter = I()
refs_before = sys.getrefcount(aiter)
async def foo():
async for i in aiter:
print('never going to happen')
with self.assertRaisesRegex(
TypeError,
"async for' received an invalid object.*__aiter.*\: I"):
run_async(foo())
self.assertEqual(sys.getrefcount(aiter), refs_before)
def test_for_4(self):
class I:
async def __aiter__(self):
return self
def __anext__(self):
return ()
aiter = I()
refs_before = sys.getrefcount(aiter)
async def foo():
async for i in aiter:
print('never going to happen')
with self.assertRaisesRegex(
TypeError,
"async for' received an invalid object.*__anext__.*tuple"):
run_async(foo())
self.assertEqual(sys.getrefcount(aiter), refs_before)
def test_for_5(self):
class I:
async def __aiter__(self):
return self
def __anext__(self):
return 123
async def foo():
async for i in I():
print('never going to happen')
with self.assertRaisesRegex(
TypeError,
"async for' received an invalid object.*__anext.*int"):
run_async(foo())
def test_for_6(self):
I = 0
class Manager:
async def __aenter__(self):
nonlocal I
I += 10000
async def __aexit__(self, *args):
nonlocal I
I += 100000
class Iterable:
def __init__(self):
self.i = 0
async def __aiter__(self):
return self
async def __anext__(self):
if self.i > 10:
raise StopAsyncIteration
self.i += 1
return self.i
##############
manager = Manager()
iterable = Iterable()
mrefs_before = sys.getrefcount(manager)
irefs_before = sys.getrefcount(iterable)
async def main():
nonlocal I
async with manager:
async for i in iterable:
I += 1
I += 1000
run_async(main())
self.assertEqual(I, 111011)
self.assertEqual(sys.getrefcount(manager), mrefs_before)
self.assertEqual(sys.getrefcount(iterable), irefs_before)
##############
async def main():
nonlocal I
async with Manager():
async for i in Iterable():
I += 1
I += 1000
async with Manager():
async for i in Iterable():
I += 1
I += 1000
run_async(main())
self.assertEqual(I, 333033)
##############
async def main():
nonlocal I
async with Manager():
I += 100
async for i in Iterable():
I += 1
else:
I += 10000000
I += 1000
async with Manager():
I += 100
async for i in Iterable():
I += 1
else:
I += 10000000
I += 1000
run_async(main())
self.assertEqual(I, 20555255)
class CoroAsyncIOCompatTest(unittest.TestCase):
def test_asyncio_1(self):
import asyncio
class MyException(Exception):
pass
buffer = []
class CM:
async def __aenter__(self):
buffer.append(1)
await asyncio.sleep(0.01)
buffer.append(2)
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
await asyncio.sleep(0.01)
buffer.append(exc_type.__name__)
async def f():
async with CM() as c:
await asyncio.sleep(0.01)
raise MyException
buffer.append('unreachable')
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(f())
except MyException:
pass
finally:
loop.close()
self.assertEqual(buffer, [1, 2, 'MyException'])
class SysSetCoroWrapperTest(unittest.TestCase):
def test_set_wrapper_1(self):
async def foo():
return 'spam'
wrapped = None
def wrap(gen):
nonlocal wrapped
wrapped = gen
return gen
self.assertIsNone(sys.get_coroutine_wrapper())
sys.set_coroutine_wrapper(wrap)
self.assertIs(sys.get_coroutine_wrapper(), wrap)
try:
f = foo()
self.assertTrue(wrapped)
self.assertEqual(run_async(f), ([], 'spam'))
finally:
sys.set_coroutine_wrapper(None)
self.assertIsNone(sys.get_coroutine_wrapper())
wrapped = None
with silence_coro_gc():
foo()
self.assertFalse(wrapped)
def test_set_wrapper_2(self):
self.assertIsNone(sys.get_coroutine_wrapper())
with self.assertRaisesRegex(TypeError, "callable expected, got int"):
sys.set_coroutine_wrapper(1)
self.assertIsNone(sys.get_coroutine_wrapper())
class CAPITest(unittest.TestCase):
def test_tp_await_1(self):
from _testcapi import awaitType as at
async def foo():
future = at(iter([1]))
return (await future)
self.assertEqual(foo().send(None), 1)
def test_tp_await_2(self):
# Test tp_await to __await__ mapping
from _testcapi import awaitType as at
future = at(iter([1]))
self.assertEqual(next(future.__await__()), 1)
def test_tp_await_3(self):
from _testcapi import awaitType as at
async def foo():
future = at(1)
return (await future)
with self.assertRaisesRegex(
TypeError, "__await__.*returned non-iterator of type 'int'"):
self.assertEqual(foo().send(None), 1)
def test_main():
support.run_unittest(AsyncBadSyntaxTest,
CoroutineTest,
CoroAsyncIOCompatTest,
SysSetCoroWrapperTest,
CAPITest)
if __name__=="__main__":
test_main()

46
Lib/test/test_dis.py
View File

@ -480,6 +480,24 @@ Constants:
Names:
0: x"""
async def async_def():
await 1
async for a in b: pass
async with c as d: pass
code_info_async_def = """\
Name: async_def
Filename: (.*)
Argument count: 0
Kw-only arguments: 0
Number of locals: 2
Stack size: 17
Flags: OPTIMIZED, NEWLOCALS, GENERATOR, NOFREE, COROUTINE
Constants:
0: None
1: 1"""
class CodeInfoTests(unittest.TestCase):
test_pairs = [
(dis.code_info, code_info_code_info),
@ -488,6 +506,7 @@ class CodeInfoTests(unittest.TestCase):
(expr_str, code_info_expr_str),
(simple_stmt_str, code_info_simple_stmt_str),
(compound_stmt_str, code_info_compound_stmt_str),
(async_def, code_info_async_def)
]
def test_code_info(self):
@ -697,7 +716,7 @@ expected_opinfo_jumpy = [
Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=135, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=138, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=141, starts_line=None, is_jump_target=False),
Instruction(opname='SETUP_FINALLY', opcode=122, arg=72, argval=217, argrepr='to 217', offset=142, starts_line=20, is_jump_target=True),
Instruction(opname='SETUP_FINALLY', opcode=122, arg=73, argval=218, argrepr='to 218', offset=142, starts_line=20, is_jump_target=True),
Instruction(opname='SETUP_EXCEPT', opcode=121, arg=12, argval=160, argrepr='to 160', offset=145, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=148, starts_line=21, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=151, starts_line=None, is_jump_target=False),
@ -717,7 +736,7 @@ expected_opinfo_jumpy = [
Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=179, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=182, starts_line=None, is_jump_target=False),
Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=183, starts_line=None, is_jump_target=False),
Instruction(opname='JUMP_FORWARD', opcode=110, arg=26, argval=213, argrepr='to 213', offset=184, starts_line=None, is_jump_target=False),
Instruction(opname='JUMP_FORWARD', opcode=110, arg=27, argval=214, argrepr='to 214', offset=184, starts_line=None, is_jump_target=False),
Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=187, starts_line=None, is_jump_target=True),
Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=188, starts_line=25, is_jump_target=True),
Instruction(opname='SETUP_WITH', opcode=143, arg=17, argval=211, argrepr='to 211', offset=191, starts_line=None, is_jump_target=False),
@ -728,17 +747,18 @@ expected_opinfo_jumpy = [
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=206, starts_line=None, is_jump_target=False),
Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=207, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=208, starts_line=None, is_jump_target=False),
Instruction(opname='WITH_CLEANUP', opcode=81, arg=None, argval=None, argrepr='', offset=211, starts_line=None, is_jump_target=True),
Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=212, starts_line=None, is_jump_target=False),
Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=213, starts_line=None, is_jump_target=True),
Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=214, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=217, starts_line=28, is_jump_target=True),
Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=220, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=223, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=226, starts_line=None, is_jump_target=False),
Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=227, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=228, starts_line=None, is_jump_target=False),
Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=231, starts_line=None, is_jump_target=False),
Instruction(opname='WITH_CLEANUP_START', opcode=81, arg=None, argval=None, argrepr='', offset=211, starts_line=None, is_jump_target=True),
Instruction(opname='WITH_CLEANUP_FINISH', opcode=82, arg=None, argval=None, argrepr='', offset=212, starts_line=None, is_jump_target=False),
Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=213, starts_line=None, is_jump_target=False),
Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=214, starts_line=None, is_jump_target=True),
Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=215, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=218, starts_line=28, is_jump_target=True),
Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=221, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=224, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=227, starts_line=None, is_jump_target=False),
Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=228, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=229, starts_line=None, is_jump_target=False),
Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=232, starts_line=None, is_jump_target=False),
]
# One last piece of inspect fodder to check the default line number handling

87
Lib/test/test_grammar.py
View File

@ -2,6 +2,7 @@
# This just tests whether the parser accepts them all.
from test.support import check_syntax_error
import inspect
import unittest
import sys
# testing import *
@ -1034,6 +1035,92 @@ class GrammarTests(unittest.TestCase):
m @= 42
self.assertEqual(m.other, 42)
def test_async_await(self):
async = 1
await = 2
self.assertEqual(async, 1)
def async():
nonlocal await
await = 10
async()
self.assertEqual(await, 10)
self.assertFalse(bool(async.__code__.co_flags & inspect.CO_COROUTINE))
async def test():
def sum():
async = 1
await = 41
return async + await
if 1:
await someobj()
self.assertEqual(test.__name__, 'test')
self.assertTrue(bool(test.__code__.co_flags & inspect.CO_COROUTINE))
def decorator(func):
setattr(func, '_marked', True)
return func
@decorator
async def test2():
return 22
self.assertTrue(test2._marked)
self.assertEqual(test2.__name__, 'test2')
self.assertTrue(bool(test2.__code__.co_flags & inspect.CO_COROUTINE))
def test_async_for(self):
class Done(Exception): pass
class AIter:
async def __aiter__(self):
return self
async def __anext__(self):
raise StopAsyncIteration
async def foo():
async for i in AIter():
pass
async for i, j in AIter():
pass
async for i in AIter():
pass
else:
pass
raise Done
with self.assertRaises(Done):
foo().send(None)
def test_async_with(self):
class Done(Exception): pass
class manager:
async def __aenter__(self):
return (1, 2)
async def __aexit__(self, *exc):
return False
async def foo():
async with manager():
pass
async with manager() as x:
pass
async with manager() as (x, y):
pass
async with manager(), manager():
pass
async with manager() as x, manager() as y:
pass
async with manager() as x, manager():
pass
raise Done
with self.assertRaises(Done):
foo().send(None)
if __name__ == '__main__':
unittest.main()

72
Lib/test/test_inspect.py
View File

@ -18,6 +18,7 @@ import textwrap
import unicodedata
import unittest
import unittest.mock
import warnings
try:
from concurrent.futures import ThreadPoolExecutor
@ -62,14 +63,16 @@ class IsTestBase(unittest.TestCase):
predicates = set([inspect.isbuiltin, inspect.isclass, inspect.iscode,
inspect.isframe, inspect.isfunction, inspect.ismethod,
inspect.ismodule, inspect.istraceback,
inspect.isgenerator, inspect.isgeneratorfunction])
inspect.isgenerator, inspect.isgeneratorfunction,
inspect.iscoroutine, inspect.iscoroutinefunction])
def istest(self, predicate, exp):
obj = eval(exp)
self.assertTrue(predicate(obj), '%s(%s)' % (predicate.__name__, exp))
for other in self.predicates - set([predicate]):
if predicate == inspect.isgeneratorfunction and\
if (predicate == inspect.isgeneratorfunction or \
predicate == inspect.iscoroutinefunction) and \
other == inspect.isfunction:
continue
self.assertFalse(other(obj), 'not %s(%s)' % (other.__name__, exp))
@ -78,13 +81,21 @@ def generator_function_example(self):
for i in range(2):
yield i
async def coroutine_function_example(self):
return 'spam'
@types.coroutine
def gen_coroutine_function_example(self):
yield
return 'spam'
class TestPredicates(IsTestBase):
def test_sixteen(self):
def test_eightteen(self):
count = len([x for x in dir(inspect) if x.startswith('is')])
# This test is here for remember you to update Doc/library/inspect.rst
# which claims there are 16 such functions
expected = 16
expected = 19
err_msg = "There are %d (not %d) is* functions" % (count, expected)
self.assertEqual(count, expected, err_msg)
@ -115,11 +126,64 @@ class TestPredicates(IsTestBase):
self.istest(inspect.isdatadescriptor, 'collections.defaultdict.default_factory')
self.istest(inspect.isgenerator, '(x for x in range(2))')
self.istest(inspect.isgeneratorfunction, 'generator_function_example')
with warnings.catch_warnings():
warnings.simplefilter("ignore")
self.istest(inspect.iscoroutine, 'coroutine_function_example(1)')
self.istest(inspect.iscoroutinefunction, 'coroutine_function_example')
if hasattr(types, 'MemberDescriptorType'):
self.istest(inspect.ismemberdescriptor, 'datetime.timedelta.days')
else:
self.assertFalse(inspect.ismemberdescriptor(datetime.timedelta.days))
def test_iscoroutine(self):
gen_coro = gen_coroutine_function_example(1)
coro = coroutine_function_example(1)
self.assertTrue(
inspect.iscoroutinefunction(gen_coroutine_function_example))
self.assertTrue(inspect.iscoroutine(gen_coro))
self.assertTrue(
inspect.isgeneratorfunction(gen_coroutine_function_example))
self.assertTrue(inspect.isgenerator(gen_coro))
self.assertTrue(
inspect.iscoroutinefunction(coroutine_function_example))
self.assertTrue(inspect.iscoroutine(coro))
self.assertFalse(
inspect.isgeneratorfunction(coroutine_function_example))
self.assertFalse(inspect.isgenerator(coro))
coro.close(); gen_coro.close() # silence warnings
def test_isawaitable(self):
def gen(): yield
self.assertFalse(inspect.isawaitable(gen()))
coro = coroutine_function_example(1)
gen_coro = gen_coroutine_function_example(1)
self.assertTrue(
inspect.isawaitable(coro))
self.assertTrue(
inspect.isawaitable(gen_coro))
class Future:
def __await__():
pass
self.assertTrue(inspect.isawaitable(Future()))
self.assertFalse(inspect.isawaitable(Future))
class NotFuture: pass
not_fut = NotFuture()
not_fut.__await__ = lambda: None
self.assertFalse(inspect.isawaitable(not_fut))
coro.close(); gen_coro.close() # silence warnings
def test_isroutine(self):
self.assertTrue(inspect.isroutine(mod.spam))
self.assertTrue(inspect.isroutine([].count))

15
Lib/test/test_minidom.py
View File

@ -49,6 +49,21 @@ class MinidomTest(unittest.TestCase):
t = node.wholeText
self.confirm(t == s, "looking for %r, found %r" % (s, t))
def testDocumentAsyncAttr(self):
doc = Document()
self.assertFalse(doc.async_)
with self.assertWarns(DeprecationWarning):
self.assertFalse(getattr(doc, 'async', True))
with self.assertWarns(DeprecationWarning):
setattr(doc, 'async', True)
with self.assertWarns(DeprecationWarning):
self.assertTrue(getattr(doc, 'async', False))
self.assertTrue(doc.async_)
self.assertFalse(Document.async_)
with self.assertWarns(DeprecationWarning):
self.assertFalse(getattr(Document, 'async', True))
def testParseFromBinaryFile(self):
with open(tstfile, 'rb') as file:
dom = parse(file)

16
Lib/test/test_parser.py
View File

@ -63,6 +63,22 @@ class RoundtripLegalSyntaxTestCase(unittest.TestCase):
" if (yield):\n"
" yield x\n")
def test_await_statement(self):
self.check_suite("async def f():\n await smth()")
self.check_suite("async def f():\n foo = await smth()")
self.check_suite("async def f():\n foo, bar = await smth()")
self.check_suite("async def f():\n (await smth())")
self.check_suite("async def f():\n foo((await smth()))")
self.check_suite("async def f():\n await foo(); return 42")
def test_async_with_statement(self):
self.check_suite("async def f():\n async with 1: pass")
self.check_suite("async def f():\n async with a as b, c as d: pass")
def test_async_for_statement(self):
self.check_suite("async def f():\n async for i in (): pass")
self.check_suite("async def f():\n async for i, b in (): pass")
def test_nonlocal_statement(self):
self.check_suite("def f():\n"
" x = 0\n"

4
Lib/test/test_pickle.py
View File

@ -351,7 +351,9 @@ class CompatPickleTests(unittest.TestCase):
for name, exc in get_exceptions(builtins):
with self.subTest(name):
if exc in (BlockingIOError, ResourceWarning):
if exc in (BlockingIOError,
ResourceWarning,
StopAsyncIteration):
continue
if exc is not OSError and issubclass(exc, OSError):
self.assertEqual(reverse_mapping('builtins', name),

4
Lib/test/test_sys.py
View File

@ -1025,9 +1025,9 @@ class SizeofTest(unittest.TestCase):
# static type: PyTypeObject
s = vsize('P2n15Pl4Pn9Pn11PIP')
check(int, s)
# (PyTypeObject + PyNumberMethods + PyMappingMethods +
# (PyTypeObject + PyAsyncMethods + PyNumberMethods + PyMappingMethods +
# PySequenceMethods + PyBufferProcs + 4P)
s = vsize('P2n17Pl4Pn9Pn11PIP') + struct.calcsize('34P 3P 10P 2P 4P')
s = vsize('P2n17Pl4Pn9Pn11PIP') + struct.calcsize('34P 3P 3P 10P 2P 4P')
# Separate block for PyDictKeysObject with 4 entries
s += struct.calcsize("2nPn") + 4*struct.calcsize("n2P")
# class

186
Lib/test/test_tokenize.py
View File

@ -641,6 +641,192 @@ Legacy unicode literals:
NAME 'grün' (2, 0) (2, 4)
OP '=' (2, 5) (2, 6)
STRING "U'green'" (2, 7) (2, 15)
Async/await extension:
>>> dump_tokens("async = 1")
ENCODING 'utf-8' (0, 0) (0, 0)
NAME 'async' (1, 0) (1, 5)
OP '=' (1, 6) (1, 7)
NUMBER '1' (1, 8) (1, 9)
>>> dump_tokens("a = (async = 1)")
ENCODING 'utf-8' (0, 0) (0, 0)
NAME 'a' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
OP '(' (1, 4) (1, 5)
NAME 'async' (1, 5) (1, 10)
OP '=' (1, 11) (1, 12)
NUMBER '1' (1, 13) (1, 14)
OP ')' (1, 14) (1, 15)
>>> dump_tokens("async()")
ENCODING 'utf-8' (0, 0) (0, 0)
NAME 'async' (1, 0) (1, 5)
OP '(' (1, 5) (1, 6)
OP ')' (1, 6) (1, 7)
>>> dump_tokens("class async(Bar):pass")
ENCODING 'utf-8' (0, 0) (0, 0)
NAME 'class' (1, 0) (1, 5)
NAME 'async' (1, 6) (1, 11)
OP '(' (1, 11) (1, 12)
NAME 'Bar' (1, 12) (1, 15)
OP ')' (1, 15) (1, 16)
OP ':' (1, 16) (1, 17)
NAME 'pass' (1, 17) (1, 21)
>>> dump_tokens("class async:pass")
ENCODING 'utf-8' (0, 0) (0, 0)
NAME 'class' (1, 0) (1, 5)
NAME 'async' (1, 6) (1, 11)
OP ':' (1, 11) (1, 12)
NAME 'pass' (1, 12) (1, 16)
>>> dump_tokens("await = 1")
ENCODING 'utf-8' (0, 0) (0, 0)
NAME 'await' (1, 0) (1, 5)
OP '=' (1, 6) (1, 7)
NUMBER '1' (1, 8) (1, 9)
>>> dump_tokens("foo.async")
ENCODING 'utf-8' (0, 0) (0, 0)
NAME 'foo' (1, 0) (1, 3)
OP '.' (1, 3) (1, 4)
NAME 'async' (1, 4) (1, 9)
>>> dump_tokens("async for a in b: pass")
ENCODING 'utf-8' (0, 0) (0, 0)
NAME 'async' (1, 0) (1, 5)
NAME 'for' (1, 6) (1, 9)
NAME 'a' (1, 10) (1, 11)
NAME 'in' (1, 12) (1, 14)
NAME 'b' (1, 15) (1, 16)
OP ':' (1, 16) (1, 17)
NAME 'pass' (1, 18) (1, 22)
>>> dump_tokens("async with a as b: pass")
ENCODING 'utf-8' (0, 0) (0, 0)
NAME 'async' (1, 0) (1, 5)
NAME 'with' (1, 6) (1, 10)
NAME 'a' (1, 11) (1, 12)
NAME 'as' (1, 13) (1, 15)
NAME 'b' (1, 16) (1, 17)
OP ':' (1, 17) (1, 18)
NAME 'pass' (1, 19) (1, 23)
>>> dump_tokens("async.foo")
ENCODING 'utf-8' (0, 0) (0, 0)
NAME 'async' (1, 0) (1, 5)
OP '.' (1, 5) (1, 6)
NAME 'foo' (1, 6) (1, 9)
>>> dump_tokens("async")
ENCODING 'utf-8' (0, 0) (0, 0)
NAME 'async' (1, 0) (1, 5)
>>> dump_tokens("async\\n#comment\\nawait")
ENCODING 'utf-8' (0, 0) (0, 0)
NAME 'async' (1, 0) (1, 5)
NEWLINE '\\n' (1, 5) (1, 6)
COMMENT '#comment' (2, 0) (2, 8)
NL '\\n' (2, 8) (2, 9)
NAME 'await' (3, 0) (3, 5)
>>> dump_tokens("async\\n...\\nawait")
ENCODING 'utf-8' (0, 0) (0, 0)
NAME 'async' (1, 0) (1, 5)
NEWLINE '\\n' (1, 5) (1, 6)
OP '...' (2, 0) (2, 3)
NEWLINE '\\n' (2, 3) (2, 4)
NAME 'await' (3, 0) (3, 5)
>>> dump_tokens("async\\nawait")
ENCODING 'utf-8' (0, 0) (0, 0)
NAME 'async' (1, 0) (1, 5)
NEWLINE '\\n' (1, 5) (1, 6)
NAME 'await' (2, 0) (2, 5)
>>> dump_tokens("foo.async + 1")
ENCODING 'utf-8' (0, 0) (0, 0)
NAME 'foo' (1, 0) (1, 3)
OP '.' (1, 3) (1, 4)
NAME 'async' (1, 4) (1, 9)
OP '+' (1, 10) (1, 11)
NUMBER '1' (1, 12) (1, 13)
>>> dump_tokens("async def foo(): pass")
ENCODING 'utf-8' (0, 0) (0, 0)
ASYNC 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
OP '(' (1, 13) (1, 14)
OP ')' (1, 14) (1, 15)
OP ':' (1, 15) (1, 16)
NAME 'pass' (1, 17) (1, 21)
>>> dump_tokens('''async def foo():
... def foo(await):
... await = 1
... if 1:
... await
... async += 1
... ''')
ENCODING 'utf-8' (0, 0) (0, 0)
ASYNC 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
OP '(' (1, 13) (1, 14)
OP ')' (1, 14) (1, 15)
OP ':' (1, 15) (1, 16)
NEWLINE '\\n' (1, 16) (1, 17)
INDENT ' ' (2, 0) (2, 2)
NAME 'def' (2, 2) (2, 5)
NAME 'foo' (2, 6) (2, 9)
OP '(' (2, 9) (2, 10)
NAME 'await' (2, 10) (2, 15)
OP ')' (2, 15) (2, 16)
OP ':' (2, 16) (2, 17)
NEWLINE '\\n' (2, 17) (2, 18)
INDENT ' ' (3, 0) (3, 4)
NAME 'await' (3, 4) (3, 9)
OP '=' (3, 10) (3, 11)
NUMBER '1' (3, 12) (3, 13)
NEWLINE '\\n' (3, 13) (3, 14)
DEDENT '' (4, 2) (4, 2)
NAME 'if' (4, 2) (4, 4)
NUMBER '1' (4, 5) (4, 6)
OP ':' (4, 6) (4, 7)
NEWLINE '\\n' (4, 7) (4, 8)
INDENT ' ' (5, 0) (5, 4)
AWAIT 'await' (5, 4) (5, 9)
NEWLINE '\\n' (5, 9) (5, 10)
DEDENT '' (6, 0) (6, 0)
DEDENT '' (6, 0) (6, 0)
NAME 'async' (6, 0) (6, 5)
OP '+=' (6, 6) (6, 8)
NUMBER '1' (6, 9) (6, 10)
NEWLINE '\\n' (6, 10) (6, 11)
>>> dump_tokens('''async def foo():
... async for i in 1: pass''')
ENCODING 'utf-8' (0, 0) (0, 0)
ASYNC 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
OP '(' (1, 13) (1, 14)
OP ')' (1, 14) (1, 15)
OP ':' (1, 15) (1, 16)
NEWLINE '\\n' (1, 16) (1, 17)
INDENT ' ' (2, 0) (2, 2)
ASYNC 'async' (2, 2) (2, 7)
NAME 'for' (2, 8) (2, 11)
NAME 'i' (2, 12) (2, 13)
NAME 'in' (2, 14) (2, 16)
NUMBER '1' (2, 17) (2, 18)
OP ':' (2, 18) (2, 19)
NAME 'pass' (2, 20) (2, 24)
DEDENT '' (3, 0) (3, 0)
"""
from test import support

35
Lib/test/test_types.py
View File

@ -1,7 +1,8 @@
# Python test set -- part 6, built-in types
from test.support import run_with_locale
import collections
import collections.abc
import inspect
import pickle
import locale
import sys
@ -1172,5 +1173,37 @@ class SimpleNamespaceTests(unittest.TestCase):
self.assertEqual(ns, ns_roundtrip, pname)
class CoroutineTests(unittest.TestCase):
def test_wrong_args(self):
class Foo:
def __call__(self):
pass
def bar(): pass
samples = [Foo, Foo(), bar, None, int, 1]
for sample in samples:
with self.assertRaisesRegex(TypeError, 'expects a generator'):
types.coroutine(sample)
def test_genfunc(self):
def gen():
yield
self.assertFalse(isinstance(gen(), collections.abc.Coroutine))
self.assertFalse(isinstance(gen(), collections.abc.Awaitable))
self.assertIs(types.coroutine(gen), gen)
self.assertTrue(gen.__code__.co_flags & inspect.CO_ITERABLE_COROUTINE)
self.assertFalse(gen.__code__.co_flags & inspect.CO_COROUTINE)
g = gen()
self.assertTrue(g.gi_code.co_flags & inspect.CO_ITERABLE_COROUTINE)
self.assertFalse(g.gi_code.co_flags & inspect.CO_COROUTINE)
self.assertTrue(isinstance(g, collections.abc.Coroutine))
self.assertTrue(isinstance(g, collections.abc.Awaitable))
g.close() # silence warning
if __name__ == '__main__':
unittest.main()

6
Lib/token.py
View File

@ -64,8 +64,10 @@ ATEQUAL = 50
RARROW = 51
ELLIPSIS = 52
OP = 53
ERRORTOKEN = 54
N_TOKENS = 55
AWAIT = 54
ASYNC = 55
ERRORTOKEN = 56
N_TOKENS = 57
NT_OFFSET = 256
#--end constants--

56
Lib/tokenize.py
View File

@ -274,7 +274,7 @@ class Untokenizer:
self.encoding = tokval
continue
if toknum in (NAME, NUMBER):
if toknum in (NAME, NUMBER, ASYNC, AWAIT):
tokval += ' '
# Insert a space between two consecutive strings
@ -477,6 +477,10 @@ def _tokenize(readline, encoding):
contline = None
indents = [0]
# 'stashed' and 'ctx' are used for async/await parsing
stashed = None
ctx = [('sync', 0)]
if encoding is not None:
if encoding == "utf-8-sig":
# BOM will already have been stripped.
@ -552,6 +556,11 @@ def _tokenize(readline, encoding):
"unindent does not match any outer indentation level",
("<tokenize>", lnum, pos, line))
indents = indents[:-1]
cur_indent = indents[-1]
while len(ctx) > 1 and ctx[-1][1] >= cur_indent:
ctx.pop()
yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
else: # continued statement
@ -572,10 +581,16 @@ def _tokenize(readline, encoding):
(initial == '.' and token != '.' and token != '...')):
yield TokenInfo(NUMBER, token, spos, epos, line)
elif initial in '\r\n':
if stashed:
yield stashed
stashed = None
yield TokenInfo(NL if parenlev > 0 else NEWLINE,
token, spos, epos, line)
elif initial == '#':
assert not token.endswith("\n")
if stashed:
yield stashed
stashed = None
yield TokenInfo(COMMENT, token, spos, epos, line)
elif token in triple_quoted:
endprog = _compile(endpats[token])
@ -603,7 +618,37 @@ def _tokenize(readline, encoding):
else: # ordinary string
yield TokenInfo(STRING, token, spos, epos, line)
elif initial.isidentifier(): # ordinary name
yield TokenInfo(NAME, token, spos, epos, line)
if token in ('async', 'await'):
if ctx[-1][0] == 'async' and ctx[-1][1] < indents[-1]:
yield TokenInfo(
ASYNC if token == 'async' else AWAIT,
token, spos, epos, line)
continue
tok = TokenInfo(NAME, token, spos, epos, line)
if token == 'async' and not stashed:
stashed = tok
continue
if token == 'def':
if (stashed
and stashed.type == NAME
and stashed.string == 'async'):
ctx.append(('async', indents[-1]))
yield TokenInfo(ASYNC, stashed.string,
stashed.start, stashed.end,
stashed.line)
stashed = None
else:
ctx.append(('sync', indents[-1]))
if stashed:
yield stashed
stashed = None
yield tok
elif initial == '\\': # continued stmt
continued = 1
else:
@ -611,12 +656,19 @@ def _tokenize(readline, encoding):
parenlev += 1
elif initial in ')]}':
parenlev -= 1
if stashed:
yield stashed
stashed = None
yield TokenInfo(OP, token, spos, epos, line)
else:
yield TokenInfo(ERRORTOKEN, line[pos],
(lnum, pos), (lnum, pos+1), line)
pos += 1
if stashed:
yield stashed
stashed = None
for indent in indents[1:]: # pop remaining indent levels
yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')

24
Lib/types.py
View File

@ -43,6 +43,30 @@ MemberDescriptorType = type(FunctionType.__globals__)
del sys, _f, _g, _C, # Not for export
_CO_GENERATOR = 0x20
_CO_ITERABLE_COROUTINE = 0x100
def coroutine(func):
"""Convert regular generator function to a coroutine."""
# TODO: Implement this in C.
if (not isinstance(func, (FunctionType, MethodType)) or
not isinstance(getattr(func, '__code__', None), CodeType) or
not (func.__code__.co_flags & _CO_GENERATOR)):
raise TypeError('coroutine() expects a generator function')
co = func.__code__
func.__code__ = CodeType(
co.co_argcount, co.co_kwonlyargcount, co.co_nlocals, co.co_stacksize,
co.co_flags | _CO_ITERABLE_COROUTINE,
co.co_code,
co.co_consts, co.co_names, co.co_varnames, co.co_filename, co.co_name,
co.co_firstlineno, co.co_lnotab, co.co_freevars, co.co_cellvars)
return func
# Provide a PEP 3115 compliant mechanism for class creation
def new_class(name, bases=(), kwds=None, exec_body=None):
"""Create a class object dynamically using the appropriate metaclass."""

26
Lib/xml/dom/xmlbuilder.py
View File

@ -1,6 +1,7 @@
"""Implementation of the DOM Level 3 'LS-Load' feature."""
import copy
import warnings
import xml.dom
from xml.dom.NodeFilter import NodeFilter
@ -331,13 +332,33 @@ class DOMBuilderFilter:
del NodeFilter
class _AsyncDeprecatedProperty:
def warn(self, cls):
clsname = cls.__name__
warnings.warn(
"{cls}.async is deprecated; use {cls}.async_".format(cls=clsname),
DeprecationWarning)
def __get__(self, instance, cls):
self.warn(cls)
if instance is not None:
return instance.async_
return False
def __set__(self, instance, value):
self.warn(type(instance))
setattr(instance, 'async_', value)
class DocumentLS:
"""Mixin to create documents that conform to the load/save spec."""
async = False
async = _AsyncDeprecatedProperty()
async_ = False
def _get_async(self):
return False
def _set_async(self, async):
if async:
raise xml.dom.NotSupportedErr(
@ -363,6 +384,9 @@ class DocumentLS:
return snode.toxml()
del _AsyncDeprecatedProperty
class DOMImplementationLS:
MODE_SYNCHRONOUS = 1
MODE_ASYNCHRONOUS = 2

97
Modules/_testcapimodule.c
View File

@ -3944,6 +3944,98 @@ static PyTypeObject matmulType = {
};
typedef struct {
PyObject_HEAD
PyObject *ao_iterator;
} awaitObject;
static PyObject *
awaitObject_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
PyObject *v;
awaitObject *ao;
if (!PyArg_UnpackTuple(args, "awaitObject", 1, 1, &v))
return NULL;
ao = (awaitObject *)type->tp_alloc(type, 0);
if (ao == NULL) {
return NULL;
}
Py_INCREF(v);
ao->ao_iterator = v;
return (PyObject *)ao;
}
static void
awaitObject_dealloc(awaitObject *ao)
{
Py_CLEAR(ao->ao_iterator);
Py_TYPE(ao)->tp_free(ao);
}
static PyObject *
awaitObject_await(awaitObject *ao)
{
Py_INCREF(ao->ao_iterator);
return ao->ao_iterator;
}
static PyAsyncMethods awaitType_as_async = {
(getawaitablefunc)awaitObject_await, /* am_await */
0, /* am_aiter */
0 /* am_anext */
};
static PyTypeObject awaitType = {
PyVarObject_HEAD_INIT(NULL, 0)
"awaitType",
sizeof(awaitObject), /* tp_basicsize */
0, /* tp_itemsize */
(destructor)awaitObject_dealloc, /* destructor tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
&awaitType_as_async, /* tp_as_async */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
PyObject_GenericGetAttr, /* tp_getattro */
PyObject_GenericSetAttr, /* tp_setattro */
0, /* tp_as_buffer */
0, /* tp_flags */
"C level type with tp_as_async",
0, /* traverseproc tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
0, /* tp_methods */
0, /* tp_members */
0,
0,
0,
0,
0,
0,
0,
0,
awaitObject_new, /* tp_new */
PyObject_Del, /* tp_free */
};
static struct PyModuleDef _testcapimodule = {
PyModuleDef_HEAD_INIT,
"_testcapi",
@ -3977,6 +4069,11 @@ PyInit__testcapi(void)
Py_INCREF(&matmulType);
PyModule_AddObject(m, "matmulType", (PyObject *)&matmulType);
if (PyType_Ready(&awaitType) < 0)
return NULL;
Py_INCREF(&awaitType);
PyModule_AddObject(m, "awaitType", (PyObject *)&awaitType);
PyModule_AddObject(m, "CHAR_MAX", PyLong_FromLong(CHAR_MAX));
PyModule_AddObject(m, "CHAR_MIN", PyLong_FromLong(CHAR_MIN));
PyModule_AddObject(m, "UCHAR_MAX", PyLong_FromLong(UCHAR_MAX));

111
Modules/parsermodule.c
View File

@ -1041,6 +1041,8 @@ VALIDATER(testlist_comp); VALIDATER(yield_expr);
VALIDATER(or_test);
VALIDATER(test_nocond); VALIDATER(lambdef_nocond);
VALIDATER(yield_arg);
VALIDATER(async_funcdef); VALIDATER(async_stmt);
VALIDATER(atom_expr);
#undef VALIDATER
@ -1608,6 +1610,7 @@ validate_compound_stmt(node *tree)
|| (ntype == try_stmt)
|| (ntype == with_stmt)
|| (ntype == funcdef)
|| (ntype == async_stmt)
|| (ntype == classdef)
|| (ntype == decorated))
res = validate_node(tree);
@ -2440,27 +2443,60 @@ validate_factor(node *tree)
/* power:
*
* power: atom trailer* ('**' factor)*
* power: atom_expr trailer* ['**' factor]
*/
static int
validate_power(node *tree)
{
int pos = 1;
int nch = NCH(tree);
int res = (validate_ntype(tree, power) && (nch >= 1)
&& validate_atom(CHILD(tree, 0)));
&& validate_atom_expr(CHILD(tree, 0)));
while (res && (pos < nch) && (TYPE(CHILD(tree, pos)) == trailer))
res = validate_trailer(CHILD(tree, pos++));
if (res && (pos < nch)) {
if (!is_even(nch - pos)) {
if (nch > 1) {
if (nch != 3) {
err_string("illegal number of nodes for 'power'");
return (0);
}
for ( ; res && (pos < (nch - 1)); pos += 2)
res = (validate_doublestar(CHILD(tree, pos))
&& validate_factor(CHILD(tree, pos + 1)));
res = (validate_doublestar(CHILD(tree, 1))
&& validate_factor(CHILD(tree, 2)));
}
return (res);
}
/* atom_expr:
*
* atom_expr: [AWAIT] atom trailer*
*/
static int
validate_atom_expr(node *tree)
{
int start = 0;
int nch = NCH(tree);
int res;
int pos;
res = validate_ntype(tree, atom_expr) && (nch >= 1);
if (!res) {
return (res);
}
if (TYPE(CHILD(tree, 0)) == AWAIT) {
start = 1;
if (nch < 2) {
err_string("illegal number of nodes for 'atom_expr'");
return (0);
}
}
res = validate_atom(CHILD(tree, start));
if (res) {
pos = start + 1;
while (res && (pos < nch) && (TYPE(CHILD(tree, pos)) == trailer))
res = validate_trailer(CHILD(tree, pos++));
}
return (res);
}
@ -2658,6 +2694,55 @@ validate_funcdef(node *tree)
return res;
}
/* async_funcdef: ASYNC funcdef */
static int
validate_async_funcdef(node *tree)
{
int nch = NCH(tree);
int res = validate_ntype(tree, async_funcdef);
if (res) {
if (nch == 2) {
res = (validate_ntype(CHILD(tree, 0), ASYNC)
&& validate_funcdef(CHILD(tree, 1)));
}
else {
res = 0;
err_string("illegal number of children for async_funcdef");
}
}
return res;
}
/* async_stmt: ASYNC (funcdef | with_stmt | for_stmt) */
static int
validate_async_stmt(node *tree)
{
int nch = NCH(tree);
int res = (validate_ntype(tree, async_stmt)
&& validate_ntype(CHILD(tree, 0), ASYNC));
if (nch != 2) {
res = 0;
err_string("illegal number of children for async_stmt");
} else {
if (TYPE(CHILD(tree, 1)) == funcdef) {
res = validate_funcdef(CHILD(tree, 1));
}
else if (TYPE(CHILD(tree, 1)) == with_stmt) {
res = validate_with_stmt(CHILD(tree, 1));
}
else if (TYPE(CHILD(tree, 1)) == for_stmt) {
res = validate_for(CHILD(tree, 1));
}
}
return res;
}
/* decorated
* decorators (classdef | funcdef)
@ -3085,6 +3170,12 @@ validate_node(node *tree)
/*
* Definition nodes.
*/
case async_funcdef:
res = validate_async_funcdef(tree);
break;
case async_stmt:
res = validate_async_stmt(tree);
break;
case funcdef:
res = validate_funcdef(tree);
break;

9
Objects/exceptions.c
View File

@ -472,6 +472,13 @@ SimpleExtendsException(PyExc_Exception, TypeError,
"Inappropriate argument type.");
/*
* StopAsyncIteration extends Exception
*/
SimpleExtendsException(PyExc_Exception, StopAsyncIteration,
"Signal the end from iterator.__anext__().");
/*
* StopIteration extends Exception
*/
@ -2468,6 +2475,7 @@ _PyExc_Init(PyObject *bltinmod)
PRE_INIT(BaseException)
PRE_INIT(Exception)
PRE_INIT(TypeError)
PRE_INIT(StopAsyncIteration)
PRE_INIT(StopIteration)
PRE_INIT(GeneratorExit)
PRE_INIT(SystemExit)
@ -2538,6 +2546,7 @@ _PyExc_Init(PyObject *bltinmod)
POST_INIT(BaseException)
POST_INIT(Exception)
POST_INIT(TypeError)
POST_INIT(StopAsyncIteration)
POST_INIT(StopIteration)
POST_INIT(GeneratorExit)
POST_INIT(SystemExit)

8
Objects/frameobject.c
View File

@ -196,6 +196,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno)
case SETUP_EXCEPT:
case SETUP_FINALLY:
case SETUP_WITH:
case SETUP_ASYNC_WITH:
blockstack[blockstack_top++] = addr;
in_finally[blockstack_top-1] = 0;
break;
@ -203,7 +204,8 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno)
case POP_BLOCK:
assert(blockstack_top > 0);
setup_op = code[blockstack[blockstack_top-1]];
if (setup_op == SETUP_FINALLY || setup_op == SETUP_WITH) {
if (setup_op == SETUP_FINALLY || setup_op == SETUP_WITH
|| setup_op == SETUP_ASYNC_WITH) {
in_finally[blockstack_top-1] = 1;
}
else {
@ -218,7 +220,8 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno)
* be seeing such an END_FINALLY.) */
if (blockstack_top > 0) {
setup_op = code[blockstack[blockstack_top-1]];
if (setup_op == SETUP_FINALLY || setup_op == SETUP_WITH) {
if (setup_op == SETUP_FINALLY || setup_op == SETUP_WITH
|| setup_op == SETUP_ASYNC_WITH) {
blockstack_top--;
}
}
@ -281,6 +284,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno)
case SETUP_EXCEPT:
case SETUP_FINALLY:
case SETUP_WITH:
case SETUP_ASYNC_WITH:
delta_iblock++;
break;

98
Objects/genobject.c
View File

@ -24,6 +24,19 @@ _PyGen_Finalize(PyObject *self)
PyObject *res;
PyObject *error_type, *error_value, *error_traceback;
/* If `gen` is a coroutine, and if it was never awaited on,
issue a RuntimeWarning. */
if (gen->gi_code != NULL
&& ((PyCodeObject *)gen->gi_code)->co_flags & (CO_COROUTINE
| CO_ITERABLE_COROUTINE)
&& gen->gi_frame != NULL
&& gen->gi_frame->f_lasti == -1
&& !PyErr_Occurred()
&& PyErr_WarnFormat(PyExc_RuntimeWarning, 1,
"coroutine '%.50S' was never awaited",
gen->gi_qualname))
return;
if (gen->gi_frame == NULL || gen->gi_frame->f_stacktop == NULL)
/* Generator isn't paused, so no need to close */
return;
@ -135,7 +148,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc)
* a leaking StopIteration into RuntimeError (with its cause
* set appropriately). */
if ((((PyCodeObject *)gen->gi_code)->co_flags &
CO_FUTURE_GENERATOR_STOP)
(CO_FUTURE_GENERATOR_STOP | CO_COROUTINE | CO_ITERABLE_COROUTINE))
&& PyErr_ExceptionMatches(PyExc_StopIteration))
{
PyObject *exc, *val, *val2, *tb;
@ -402,6 +415,12 @@ failed_throw:
static PyObject *
gen_iternext(PyGenObject *gen)
{
if (((PyCodeObject*)gen->gi_code)->co_flags & CO_COROUTINE) {
PyErr_SetString(PyExc_TypeError,
"coroutine-objects do not support iteration");
return NULL;
}
return gen_send_ex(gen, NULL, 0);
}
@ -459,8 +478,14 @@ _PyGen_FetchStopIterationValue(PyObject **pvalue) {
static PyObject *
gen_repr(PyGenObject *gen)
{
if (PyGen_CheckCoroutineExact(gen)) {
return PyUnicode_FromFormat("<coroutine object %S at %p>",
gen->gi_qualname, gen);
}
else {
return PyUnicode_FromFormat("<generator object %S at %p>",
gen->gi_qualname, gen);
}
}
static PyObject *
@ -496,6 +521,19 @@ gen_get_qualname(PyGenObject *op)
return op->gi_qualname;
}
static PyObject *
gen_get_iter(PyGenObject *gen)
{
if (((PyCodeObject*)gen->gi_code)->co_flags & CO_COROUTINE) {
PyErr_SetString(PyExc_TypeError,
"coroutine-objects do not support iteration");
return NULL;
}
Py_INCREF(gen);
return gen;
}
static int
gen_set_qualname(PyGenObject *op, PyObject *value)
{
@ -547,7 +585,7 @@ PyTypeObject PyGen_Type = {
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_reserved */
0, /* tp_as_async */
(reprfunc)gen_repr, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
@ -565,7 +603,7 @@ PyTypeObject PyGen_Type = {
0, /* tp_clear */
0, /* tp_richcompare */
offsetof(PyGenObject, gi_weakreflist), /* tp_weaklistoffset */
PyObject_SelfIter, /* tp_iter */
(getiterfunc)gen_get_iter, /* tp_iter */
(iternextfunc)gen_iternext, /* tp_iternext */
gen_methods, /* tp_methods */
gen_memberlist, /* tp_members */
@ -642,3 +680,57 @@ PyGen_NeedsFinalizing(PyGenObject *gen)
/* No blocks except loops, it's safe to skip finalization. */
return 0;
}
/*
* This helper function returns an awaitable for `o`:
* - `o` if `o` is a coroutine-object;
* - `type(o)->tp_as_async->am_await(o)`
*
* Raises a TypeError if it's not possible to return
* an awaitable and returns NULL.
*/
PyObject *
_PyGen_GetAwaitableIter(PyObject *o)
{
getawaitablefunc getter = NULL;
PyTypeObject *ot;
if (PyGen_CheckCoroutineExact(o)) {
/* Fast path. It's a central function for 'await'. */
Py_INCREF(o);
return o;
}
ot = Py_TYPE(o);
if (ot->tp_as_async != NULL) {
getter = ot->tp_as_async->am_await;
}
if (getter != NULL) {
PyObject *res = (*getter)(o);
if (res != NULL) {
if (!PyIter_Check(res)) {
PyErr_Format(PyExc_TypeError,
"__await__() returned non-iterator "
"of type '%.100s'",
Py_TYPE(res)->tp_name);
Py_CLEAR(res);
}
else {
if (PyGen_CheckCoroutineExact(res)) {
/* __await__ must return an *iterator*, not
a coroutine or another awaitable (see PEP 492) */
PyErr_SetString(PyExc_TypeError,
"__await__() returned a coroutine");
Py_CLEAR(res);
}
}
}
return res;
}
PyErr_Format(PyExc_TypeError,
"object %.100s can't be used in 'await' expression",
ot->tp_name);
return NULL;
}

91
Objects/typeobject.c
View File

@ -2506,6 +2506,7 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds)
type->tp_flags |= Py_TPFLAGS_HAVE_GC;
/* Initialize essential fields */
type->tp_as_async = &et->as_async;
type->tp_as_number = &et->as_number;
type->tp_as_sequence = &et->as_sequence;
type->tp_as_mapping = &et->as_mapping;
@ -2751,6 +2752,7 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
}
/* Initialize essential fields */
type->tp_as_async = &res->as_async;
type->tp_as_number = &res->as_number;
type->tp_as_sequence = &res->as_sequence;
type->tp_as_mapping = &res->as_mapping;
@ -4566,6 +4568,7 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base)
#define COPYSLOT(SLOT) \
if (!type->SLOT && SLOTDEFINED(SLOT)) type->SLOT = base->SLOT
#define COPYASYNC(SLOT) COPYSLOT(tp_as_async->SLOT)
#define COPYNUM(SLOT) COPYSLOT(tp_as_number->SLOT)
#define COPYSEQ(SLOT) COPYSLOT(tp_as_sequence->SLOT)
#define COPYMAP(SLOT) COPYSLOT(tp_as_mapping->SLOT)
@ -4615,6 +4618,15 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base)
COPYNUM(nb_inplace_matrix_multiply);
}
if (type->tp_as_async != NULL && base->tp_as_async != NULL) {
basebase = base->tp_base;
if (basebase->tp_as_async == NULL)
basebase = NULL;
COPYASYNC(am_await);
COPYASYNC(am_aiter);
COPYASYNC(am_anext);
}
if (type->tp_as_sequence != NULL && base->tp_as_sequence != NULL) {
basebase = base->tp_base;
if (basebase->tp_as_sequence == NULL)
@ -4884,6 +4896,8 @@ PyType_Ready(PyTypeObject *type)
/* Some more special stuff */
base = type->tp_base;
if (base != NULL) {
if (type->tp_as_async == NULL)
type->tp_as_async = base->tp_as_async;
if (type->tp_as_number == NULL)
type->tp_as_number = base->tp_as_number;
if (type->tp_as_sequence == NULL)
@ -4904,16 +4918,6 @@ PyType_Ready(PyTypeObject *type)
goto error;
}
/* Warn for a type that implements tp_compare (now known as
tp_reserved) but not tp_richcompare. */
if (type->tp_reserved && !type->tp_richcompare) {
PyErr_Format(PyExc_TypeError,
"Type %.100s defines tp_reserved (formerly tp_compare) "
"but not tp_richcompare. Comparisons may not behave as intended.",
type->tp_name);
goto error;
}
/* All done -- set the ready flag */
assert(type->tp_dict != NULL);
type->tp_flags =
@ -6265,6 +6269,59 @@ slot_tp_finalize(PyObject *self)
PyErr_Restore(error_type, error_value, error_traceback);
}
static PyObject *
slot_am_await(PyObject *self)
{
PyObject *func, *res;
_Py_IDENTIFIER(__await__);
func = lookup_method(self, &PyId___await__);
if (func != NULL) {
res = PyEval_CallObject(func, NULL);
Py_DECREF(func);
return res;
}
PyErr_Format(PyExc_AttributeError,
"object %.50s does not have __await__ method",
Py_TYPE(self)->tp_name);
return NULL;
}
static PyObject *
slot_am_aiter(PyObject *self)
{
PyObject *func, *res;
_Py_IDENTIFIER(__aiter__);
func = lookup_method(self, &PyId___aiter__);
if (func != NULL) {
res = PyEval_CallObject(func, NULL);
Py_DECREF(func);
return res;
}
PyErr_Format(PyExc_AttributeError,
"object %.50s does not have __aiter__ method",
Py_TYPE(self)->tp_name);
return NULL;
}
static PyObject *
slot_am_anext(PyObject *self)
{
PyObject *func, *res;
_Py_IDENTIFIER(__anext__);
func = lookup_method(self, &PyId___anext__);
if (func != NULL) {
res = PyEval_CallObject(func, NULL);
Py_DECREF(func);
return res;
}
PyErr_Format(PyExc_AttributeError,
"object %.50s does not have __anext__ method",
Py_TYPE(self)->tp_name);
return NULL;
}
/*
Table mapping __foo__ names to tp_foo offsets and slot_tp_foo wrapper functions.
@ -6281,6 +6338,7 @@ typedef struct wrapperbase slotdef;
#undef TPSLOT
#undef FLSLOT
#undef AMSLOT
#undef ETSLOT
#undef SQSLOT
#undef MPSLOT
@ -6299,6 +6357,8 @@ typedef struct wrapperbase slotdef;
#define ETSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \
{NAME, offsetof(PyHeapTypeObject, SLOT), (void *)(FUNCTION), WRAPPER, \
PyDoc_STR(DOC)}
#define AMSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \
ETSLOT(NAME, as_async.SLOT, FUNCTION, WRAPPER, DOC)
#define SQSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \
ETSLOT(NAME, as_sequence.SLOT, FUNCTION, WRAPPER, DOC)
#define MPSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \
@ -6378,6 +6438,13 @@ static slotdef slotdefs[] = {
"Create and return new object. See help(type) for accurate signature."),
TPSLOT("__del__", tp_finalize, slot_tp_finalize, (wrapperfunc)wrap_del, ""),
AMSLOT("__await__", am_await, slot_am_await, wrap_unaryfunc,
"__await__($self, /)\n--\n\nReturn an iterator to be used in await expression."),
AMSLOT("__aiter__", am_aiter, slot_am_aiter, wrap_unaryfunc,
"__aiter__($self, /)\n--\n\nReturn an awaitable, that resolves in asynchronous iterator."),
AMSLOT("__anext__", am_anext, slot_am_anext, wrap_unaryfunc,
"__anext__($self, /)\n--\n\nReturn a value or raise StopAsyncIteration."),
BINSLOT("__add__", nb_add, slot_nb_add,
"+"),
RBINSLOT("__radd__", nb_add, slot_nb_add,
@ -6530,6 +6597,10 @@ slotptr(PyTypeObject *type, int ioffset)
ptr = (char *)type->tp_as_number;
offset -= offsetof(PyHeapTypeObject, as_number);
}
else if ((size_t)offset >= offsetof(PyHeapTypeObject, as_async)) {
ptr = (char *)type->tp_as_async;
offset -= offsetof(PyHeapTypeObject, as_async);
}
else {
ptr = (char *)type;
}

3
Objects/typeslots.inc
View File

@ -75,3 +75,6 @@ offsetof(PyHeapTypeObject, ht_type.tp_getset),
offsetof(PyHeapTypeObject, ht_type.tp_free),
offsetof(PyHeapTypeObject, as_number.nb_matrix_multiply),
offsetof(PyHeapTypeObject, as_number.nb_inplace_matrix_multiply),
offsetof(PyHeapTypeObject, as_async.am_await),
offsetof(PyHeapTypeObject, as_async.am_aiter),
offsetof(PyHeapTypeObject, as_async.am_anext),

2
Objects/typeslots.py
View File

@ -12,6 +12,8 @@ for line in sys.stdin:
member = m.group(1)
if member.startswith("tp_"):
member = "ht_type."+member
elif member.startswith("am_"):
member = "as_async."+member
elif member.startswith("nb_"):
member = "as_number."+member
elif member.startswith("mp_"):

6
Parser/Python.asdl
View File

@ -11,6 +11,9 @@ module Python
stmt = FunctionDef(identifier name, arguments args,
stmt* body, expr* decorator_list, expr? returns)
| AsyncFunctionDef(identifier name, arguments args,
stmt* body, expr* decorator_list, expr? returns)
| ClassDef(identifier name,
expr* bases,
keyword* keywords,
@ -24,9 +27,11 @@ module Python
-- use 'orelse' because else is a keyword in target languages
| For(expr target, expr iter, stmt* body, stmt* orelse)
| AsyncFor(expr target, expr iter, stmt* body, stmt* orelse)
| While(expr test, stmt* body, stmt* orelse)
| If(expr test, stmt* body, stmt* orelse)
| With(withitem* items, stmt* body)
| AsyncWith(withitem* items, stmt* body)
| Raise(expr? exc, expr? cause)
| Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)
@ -57,6 +62,7 @@ module Python
| DictComp(expr key, expr value, comprehension* generators)
| GeneratorExp(expr elt, comprehension* generators)
-- the grammar constrains where yield expressions can occur
| Await(expr value)
| Yield(expr? value)
| YieldFrom(expr value)
-- need sequences for compare to distinguish between

68
Parser/tokenizer.c
View File

@ -103,6 +103,8 @@ const char *_PyParser_TokenNames[] = {
"ELLIPSIS",
/* This table must match the #defines in token.h! */
"OP",
"AWAIT",
"ASYNC",
"<ERRORTOKEN>",
"<N_TOKENS>"
};
@ -124,6 +126,11 @@ tok_new(void)
tok->tabsize = TABSIZE;
tok->indent = 0;
tok->indstack[0] = 0;
tok->def = 0;
tok->defstack[0] = 0;
tok->deftypestack[0] = 0;
tok->atbol = 1;
tok->pendin = 0;
tok->prompt = tok->nextprompt = NULL;
@ -1335,6 +1342,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
int c;
int blankline, nonascii;
int tok_len;
struct tok_state ahead_tok;
char *ahead_tok_start = NULL, *ahead_top_end = NULL;
int ahead_tok_kind;
*p_start = *p_end = NULL;
nextline:
tok->start = NULL;
@ -1422,6 +1434,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
if (tok->pendin != 0) {
if (tok->pendin < 0) {
tok->pendin++;
while (tok->def && tok->defstack[tok->def] >= tok->indent) {
tok->def--;
}
return DEDENT;
}
else {
@ -1481,6 +1498,57 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
return ERRORTOKEN;
*p_start = tok->start;
*p_end = tok->cur;
tok_len = tok->cur - tok->start;
if (tok_len == 3 && memcmp(tok->start, "def", 3) == 0) {
if (tok->def + 1 >= MAXINDENT) {
tok->done = E_TOODEEP;
tok->cur = tok->inp;
return ERRORTOKEN;
}
if (tok->def && tok->deftypestack[tok->def] == 3) {
tok->deftypestack[tok->def] = 2;
}
else {
tok->def++;
tok->defstack[tok->def] = tok->indent;
tok->deftypestack[tok->def] = 1;
}
}
else if (tok_len == 5) {
if (memcmp(tok->start, "async", 5) == 0) {
memcpy(&ahead_tok, tok, sizeof(ahead_tok));
ahead_tok_kind = tok_get(&ahead_tok, &ahead_tok_start,
&ahead_top_end);
if (ahead_tok_kind == NAME &&
ahead_tok.cur - ahead_tok.start == 3 &&
memcmp(ahead_tok.start, "def", 3) == 0) {
tok->def++;
tok->defstack[tok->def] = tok->indent;
tok->deftypestack[tok->def] = 3;
return ASYNC;
}
else if (tok->def && tok->deftypestack[tok->def] == 2
&& tok->defstack[tok->def] < tok->indent) {
return ASYNC;
}
}
else if (memcmp(tok->start, "await", 5) == 0
&& tok->def && tok->deftypestack[tok->def] == 2
&& tok->defstack[tok->def] < tok->indent) {
return AWAIT;
}
}
return NAME;
}

7
Parser/tokenizer.h
View File

@ -65,6 +65,13 @@ struct tok_state {
const char* enc; /* Encoding for the current str. */
const char* str;
const char* input; /* Tokenizer's newline translated copy of the string. */
int defstack[MAXINDENT]; /* stack if funcs & indents where they
were defined */
int deftypestack[MAXINDENT]; /* stack of func types
(0 not func; 1: "def name";
2: "async def name") */
int def; /* Length of stack of func types */
};
extern struct tok_state *PyTokenizer_FromString(const char *, int);

473
Python/Python-ast.c
View File

@ -45,6 +45,14 @@ static char *FunctionDef_fields[]={
"decorator_list",
"returns",
};
static PyTypeObject *AsyncFunctionDef_type;
static char *AsyncFunctionDef_fields[]={
"name",
"args",
"body",
"decorator_list",
"returns",
};
static PyTypeObject *ClassDef_type;
_Py_IDENTIFIER(bases);
_Py_IDENTIFIER(keywords);
@ -87,6 +95,13 @@ static char *For_fields[]={
"body",
"orelse",
};
static PyTypeObject *AsyncFor_type;
static char *AsyncFor_fields[]={
"target",
"iter",
"body",
"orelse",
};
static PyTypeObject *While_type;
_Py_IDENTIFIER(test);
static char *While_fields[]={
@ -106,6 +121,11 @@ static char *With_fields[]={
"items",
"body",
};
static PyTypeObject *AsyncWith_type;
static char *AsyncWith_fields[]={
"items",
"body",
};
static PyTypeObject *Raise_type;
_Py_IDENTIFIER(exc);
_Py_IDENTIFIER(cause);
@ -228,6 +248,10 @@ static char *GeneratorExp_fields[]={
"elt",
"generators",
};
static PyTypeObject *Await_type;
static char *Await_fields[]={
"value",
};
static PyTypeObject *Yield_type;
static char *Yield_fields[]={
"value",
@ -806,6 +830,9 @@ static int init_types(void)
FunctionDef_type = make_type("FunctionDef", stmt_type, FunctionDef_fields,
5);
if (!FunctionDef_type) return 0;
AsyncFunctionDef_type = make_type("AsyncFunctionDef", stmt_type,
AsyncFunctionDef_fields, 5);
if (!AsyncFunctionDef_type) return 0;
ClassDef_type = make_type("ClassDef", stmt_type, ClassDef_fields, 5);
if (!ClassDef_type) return 0;
Return_type = make_type("Return", stmt_type, Return_fields, 1);
@ -818,12 +845,16 @@ static int init_types(void)
if (!AugAssign_type) return 0;
For_type = make_type("For", stmt_type, For_fields, 4);
if (!For_type) return 0;
AsyncFor_type = make_type("AsyncFor", stmt_type, AsyncFor_fields, 4);
if (!AsyncFor_type) return 0;
While_type = make_type("While", stmt_type, While_fields, 3);
if (!While_type) return 0;
If_type = make_type("If", stmt_type, If_fields, 3);
if (!If_type) return 0;
With_type = make_type("With", stmt_type, With_fields, 2);
if (!With_type) return 0;
AsyncWith_type = make_type("AsyncWith", stmt_type, AsyncWith_fields, 2);
if (!AsyncWith_type) return 0;
Raise_type = make_type("Raise", stmt_type, Raise_fields, 2);
if (!Raise_type) return 0;
Try_type = make_type("Try", stmt_type, Try_fields, 4);
@ -872,6 +903,8 @@ static int init_types(void)
GeneratorExp_type = make_type("GeneratorExp", expr_type,
GeneratorExp_fields, 2);
if (!GeneratorExp_type) return 0;
Await_type = make_type("Await", expr_type, Await_fields, 1);
if (!Await_type) return 0;
Yield_type = make_type("Yield", expr_type, Yield_fields, 1);
if (!Yield_type) return 0;
YieldFrom_type = make_type("YieldFrom", expr_type, YieldFrom_fields, 1);
@ -1200,6 +1233,36 @@ FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
return p;
}
stmt_ty
AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq
* decorator_list, expr_ty returns, int lineno, int col_offset,
PyArena *arena)
{
stmt_ty p;
if (!name) {
PyErr_SetString(PyExc_ValueError,
"field name is required for AsyncFunctionDef");
return NULL;
}
if (!args) {
PyErr_SetString(PyExc_ValueError,
"field args is required for AsyncFunctionDef");
return NULL;
}
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
if (!p)
return NULL;
p->kind = AsyncFunctionDef_kind;
p->v.AsyncFunctionDef.name = name;
p->v.AsyncFunctionDef.args = args;
p->v.AsyncFunctionDef.body = body;
p->v.AsyncFunctionDef.decorator_list = decorator_list;
p->v.AsyncFunctionDef.returns = returns;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, asdl_seq *
body, asdl_seq * decorator_list, int lineno, int col_offset, PyArena
@ -1334,6 +1397,34 @@ For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int
return p;
}
stmt_ty
AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int
lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
if (!target) {
PyErr_SetString(PyExc_ValueError,
"field target is required for AsyncFor");
return NULL;
}
if (!iter) {
PyErr_SetString(PyExc_ValueError,
"field iter is required for AsyncFor");
return NULL;
}
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
if (!p)
return NULL;
p->kind = AsyncFor_kind;
p->v.AsyncFor.target = target;
p->v.AsyncFor.iter = iter;
p->v.AsyncFor.body = body;
p->v.AsyncFor.orelse = orelse;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
col_offset, PyArena *arena)
@ -1394,6 +1485,22 @@ With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, PyArena
return p;
}
stmt_ty
AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int col_offset,
PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
if (!p)
return NULL;
p->kind = AsyncWith_kind;
p->v.AsyncWith.items = items;
p->v.AsyncWith.body = body;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
stmt_ty
Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, PyArena *arena)
{
@ -1821,6 +1928,25 @@ GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
return p;
}
expr_ty
Await(expr_ty value, int lineno, int col_offset, PyArena *arena)
{
expr_ty p;
if (!value) {
PyErr_SetString(PyExc_ValueError,
"field value is required for Await");
return NULL;
}
p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
if (!p)
return NULL;
p->kind = Await_kind;
p->v.Await.value = value;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
Yield(expr_ty value, int lineno, int col_offset, PyArena *arena)
{
@ -2409,6 +2535,36 @@ ast2obj_stmt(void* _o)
goto failed;
Py_DECREF(value);
break;
case AsyncFunctionDef_kind:
result = PyType_GenericNew(AsyncFunctionDef_type, NULL, NULL);
if (!result) goto failed;
value = ast2obj_identifier(o->v.AsyncFunctionDef.name);
if (!value) goto failed;
if (_PyObject_SetAttrId(result, &PyId_name, value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_arguments(o->v.AsyncFunctionDef.args);
if (!value) goto failed;
if (_PyObject_SetAttrId(result, &PyId_args, value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->v.AsyncFunctionDef.body, ast2obj_stmt);
if (!value) goto failed;
if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->v.AsyncFunctionDef.decorator_list,
ast2obj_expr);
if (!value) goto failed;
if (_PyObject_SetAttrId(result, &PyId_decorator_list, value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_expr(o->v.AsyncFunctionDef.returns);
if (!value) goto failed;
if (_PyObject_SetAttrId(result, &PyId_returns, value) == -1)
goto failed;
Py_DECREF(value);
break;
case ClassDef_kind:
result = PyType_GenericNew(ClassDef_type, NULL, NULL);
if (!result) goto failed;
@ -2513,6 +2669,30 @@ ast2obj_stmt(void* _o)
goto failed;
Py_DECREF(value);
break;
case AsyncFor_kind:
result = PyType_GenericNew(AsyncFor_type, NULL, NULL);
if (!result) goto failed;
value = ast2obj_expr(o->v.AsyncFor.target);
if (!value) goto failed;
if (_PyObject_SetAttrId(result, &PyId_target, value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_expr(o->v.AsyncFor.iter);
if (!value) goto failed;
if (_PyObject_SetAttrId(result, &PyId_iter, value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->v.AsyncFor.body, ast2obj_stmt);
if (!value) goto failed;
if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->v.AsyncFor.orelse, ast2obj_stmt);
if (!value) goto failed;
if (_PyObject_SetAttrId(result, &PyId_orelse, value) == -1)
goto failed;
Py_DECREF(value);
break;
case While_kind:
result = PyType_GenericNew(While_type, NULL, NULL);
if (!result) goto failed;
@ -2565,6 +2745,20 @@ ast2obj_stmt(void* _o)
goto failed;
Py_DECREF(value);
break;
case AsyncWith_kind:
result = PyType_GenericNew(AsyncWith_type, NULL, NULL);
if (!result) goto failed;
value = ast2obj_list(o->v.AsyncWith.items, ast2obj_withitem);
if (!value) goto failed;
if (_PyObject_SetAttrId(result, &PyId_items, value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->v.AsyncWith.body, ast2obj_stmt);
if (!value) goto failed;
if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
goto failed;
Py_DECREF(value);
break;
case Raise_kind:
result = PyType_GenericNew(Raise_type, NULL, NULL);
if (!result) goto failed;
@ -2878,6 +3072,15 @@ ast2obj_expr(void* _o)
goto failed;
Py_DECREF(value);
break;
case Await_kind:
result = PyType_GenericNew(Await_type, NULL, NULL);
if (!result) goto failed;
value = ast2obj_expr(o->v.Await.value);
if (!value) goto failed;
if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
goto failed;
Py_DECREF(value);
break;
case Yield_kind:
result = PyType_GenericNew(Yield_type, NULL, NULL);
if (!result) goto failed;
@ -3832,6 +4035,102 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
if (*out == NULL) goto failed;
return 0;
}
isinstance = PyObject_IsInstance(obj, (PyObject*)AsyncFunctionDef_type);
if (isinstance == -1) {
return 1;
}
if (isinstance) {
identifier name;
arguments_ty args;
asdl_seq* body;
asdl_seq* decorator_list;
expr_ty returns;
if (_PyObject_HasAttrId(obj, &PyId_name)) {
int res;
tmp = _PyObject_GetAttrId(obj, &PyId_name);
if (tmp == NULL) goto failed;
res = obj2ast_identifier(tmp, &name, arena);
if (res != 0) goto failed;
Py_CLEAR(tmp);
} else {
PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from AsyncFunctionDef");
return 1;
}
if (_PyObject_HasAttrId(obj, &PyId_args)) {
int res;
tmp = _PyObject_GetAttrId(obj, &PyId_args);
if (tmp == NULL) goto failed;
res = obj2ast_arguments(tmp, &args, arena);
if (res != 0) goto failed;
Py_CLEAR(tmp);
} else {
PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from AsyncFunctionDef");
return 1;
}
if (_PyObject_HasAttrId(obj, &PyId_body)) {
int res;
Py_ssize_t len;
Py_ssize_t i;
tmp = _PyObject_GetAttrId(obj, &PyId_body);
if (tmp == NULL) goto failed;
if (!PyList_Check(tmp)) {
PyErr_Format(PyExc_TypeError, "AsyncFunctionDef field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
goto failed;
}
len = PyList_GET_SIZE(tmp);
body = _Py_asdl_seq_new(len, arena);
if (body == NULL) goto failed;
for (i = 0; i < len; i++) {
stmt_ty value;
res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena);
if (res != 0) goto failed;
asdl_seq_SET(body, i, value);
}
Py_CLEAR(tmp);
} else {
PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from AsyncFunctionDef");
return 1;
}
if (_PyObject_HasAttrId(obj, &PyId_decorator_list)) {
int res;
Py_ssize_t len;
Py_ssize_t i;
tmp = _PyObject_GetAttrId(obj, &PyId_decorator_list);
if (tmp == NULL) goto failed;
if (!PyList_Check(tmp)) {
PyErr_Format(PyExc_TypeError, "AsyncFunctionDef field \"decorator_list\" must be a list, not a %.200s", tmp->ob_type->tp_name);
goto failed;
}
len = PyList_GET_SIZE(tmp);
decorator_list = _Py_asdl_seq_new(len, arena);
if (decorator_list == NULL) goto failed;
for (i = 0; i < len; i++) {
expr_ty value;
res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena);
if (res != 0) goto failed;
asdl_seq_SET(decorator_list, i, value);
}
Py_CLEAR(tmp);
} else {
PyErr_SetString(PyExc_TypeError, "required field \"decorator_list\" missing from AsyncFunctionDef");
return 1;
}
if (exists_not_none(obj, &PyId_returns)) {
int res;
tmp = _PyObject_GetAttrId(obj, &PyId_returns);
if (tmp == NULL) goto failed;
res = obj2ast_expr(tmp, &returns, arena);
if (res != 0) goto failed;
Py_CLEAR(tmp);
} else {
returns = NULL;
}
*out = AsyncFunctionDef(name, args, body, decorator_list, returns,
lineno, col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
isinstance = PyObject_IsInstance(obj, (PyObject*)ClassDef_type);
if (isinstance == -1) {
return 1;
@ -4188,6 +4487,90 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
if (*out == NULL) goto failed;
return 0;
}
isinstance = PyObject_IsInstance(obj, (PyObject*)AsyncFor_type);
if (isinstance == -1) {
return 1;
}
if (isinstance) {
expr_ty target;
expr_ty iter;
asdl_seq* body;
asdl_seq* orelse;
if (_PyObject_HasAttrId(obj, &PyId_target)) {
int res;
tmp = _PyObject_GetAttrId(obj, &PyId_target);
if (tmp == NULL) goto failed;
res = obj2ast_expr(tmp, &target, arena);
if (res != 0) goto failed;
Py_CLEAR(tmp);
} else {
PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from AsyncFor");
return 1;
}
if (_PyObject_HasAttrId(obj, &PyId_iter)) {
int res;
tmp = _PyObject_GetAttrId(obj, &PyId_iter);
if (tmp == NULL) goto failed;
res = obj2ast_expr(tmp, &iter, arena);
if (res != 0) goto failed;
Py_CLEAR(tmp);
} else {
PyErr_SetString(PyExc_TypeError, "required field \"iter\" missing from AsyncFor");
return 1;
}
if (_PyObject_HasAttrId(obj, &PyId_body)) {
int res;
Py_ssize_t len;
Py_ssize_t i;
tmp = _PyObject_GetAttrId(obj, &PyId_body);
if (tmp == NULL) goto failed;
if (!PyList_Check(tmp)) {
PyErr_Format(PyExc_TypeError, "AsyncFor field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
goto failed;
}
len = PyList_GET_SIZE(tmp);
body = _Py_asdl_seq_new(len, arena);
if (body == NULL) goto failed;
for (i = 0; i < len; i++) {
stmt_ty value;
res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena);
if (res != 0) goto failed;
asdl_seq_SET(body, i, value);
}
Py_CLEAR(tmp);
} else {
PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from AsyncFor");
return 1;
}
if (_PyObject_HasAttrId(obj, &PyId_orelse)) {
int res;
Py_ssize_t len;
Py_ssize_t i;
tmp = _PyObject_GetAttrId(obj, &PyId_orelse);
if (tmp == NULL) goto failed;
if (!PyList_Check(tmp)) {
PyErr_Format(PyExc_TypeError, "AsyncFor field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name);
goto failed;
}
len = PyList_GET_SIZE(tmp);
orelse = _Py_asdl_seq_new(len, arena);
if (orelse == NULL) goto failed;
for (i = 0; i < len; i++) {
stmt_ty value;
res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena);
if (res != 0) goto failed;
asdl_seq_SET(orelse, i, value);
}
Py_CLEAR(tmp);
} else {
PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from AsyncFor");
return 1;
}
*out = AsyncFor(target, iter, body, orelse, lineno, col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
isinstance = PyObject_IsInstance(obj, (PyObject*)While_type);
if (isinstance == -1) {
return 1;
@ -4392,6 +4775,66 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
if (*out == NULL) goto failed;
return 0;
}
isinstance = PyObject_IsInstance(obj, (PyObject*)AsyncWith_type);
if (isinstance == -1) {
return 1;
}
if (isinstance) {
asdl_seq* items;
asdl_seq* body;
if (_PyObject_HasAttrId(obj, &PyId_items)) {
int res;
Py_ssize_t len;
Py_ssize_t i;
tmp = _PyObject_GetAttrId(obj, &PyId_items);
if (tmp == NULL) goto failed;
if (!PyList_Check(tmp)) {
PyErr_Format(PyExc_TypeError, "AsyncWith field \"items\" must be a list, not a %.200s", tmp->ob_type->tp_name);
goto failed;
}
len = PyList_GET_SIZE(tmp);
items = _Py_asdl_seq_new(len, arena);
if (items == NULL) goto failed;
for (i = 0; i < len; i++) {
withitem_ty value;
res = obj2ast_withitem(PyList_GET_ITEM(tmp, i), &value, arena);
if (res != 0) goto failed;
asdl_seq_SET(items, i, value);
}
Py_CLEAR(tmp);
} else {
PyErr_SetString(PyExc_TypeError, "required field \"items\" missing from AsyncWith");
return 1;
}
if (_PyObject_HasAttrId(obj, &PyId_body)) {
int res;
Py_ssize_t len;
Py_ssize_t i;
tmp = _PyObject_GetAttrId(obj, &PyId_body);
if (tmp == NULL) goto failed;
if (!PyList_Check(tmp)) {
PyErr_Format(PyExc_TypeError, "AsyncWith field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
goto failed;
}
len = PyList_GET_SIZE(tmp);
body = _Py_asdl_seq_new(len, arena);
if (body == NULL) goto failed;
for (i = 0; i < len; i++) {
stmt_ty value;
res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena);
if (res != 0) goto failed;
asdl_seq_SET(body, i, value);
}
Py_CLEAR(tmp);
} else {
PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from AsyncWith");
return 1;
}
*out = AsyncWith(items, body, lineno, col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
isinstance = PyObject_IsInstance(obj, (PyObject*)Raise_type);
if (isinstance == -1) {
return 1;
@ -5326,6 +5769,28 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
if (*out == NULL) goto failed;
return 0;
}
isinstance = PyObject_IsInstance(obj, (PyObject*)Await_type);
if (isinstance == -1) {
return 1;
}
if (isinstance) {
expr_ty value;
if (_PyObject_HasAttrId(obj, &PyId_value)) {
int res;
tmp = _PyObject_GetAttrId(obj, &PyId_value);
if (tmp == NULL) goto failed;
res = obj2ast_expr(tmp, &value, arena);
if (res != 0) goto failed;
Py_CLEAR(tmp);
} else {
PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Await");
return 1;
}
*out = Await(value, lineno, col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
isinstance = PyObject_IsInstance(obj, (PyObject*)Yield_type);
if (isinstance == -1) {
return 1;
@ -6782,6 +7247,8 @@ PyInit__ast(void)
if (PyDict_SetItemString(d, "stmt", (PyObject*)stmt_type) < 0) return NULL;
if (PyDict_SetItemString(d, "FunctionDef", (PyObject*)FunctionDef_type) <
0) return NULL;
if (PyDict_SetItemString(d, "AsyncFunctionDef",
(PyObject*)AsyncFunctionDef_type) < 0) return NULL;
if (PyDict_SetItemString(d, "ClassDef", (PyObject*)ClassDef_type) < 0)
return NULL;
if (PyDict_SetItemString(d, "Return", (PyObject*)Return_type) < 0) return
@ -6793,10 +7260,14 @@ PyInit__ast(void)
if (PyDict_SetItemString(d, "AugAssign", (PyObject*)AugAssign_type) < 0)
return NULL;
if (PyDict_SetItemString(d, "For", (PyObject*)For_type) < 0) return NULL;
if (PyDict_SetItemString(d, "AsyncFor", (PyObject*)AsyncFor_type) < 0)
return NULL;
if (PyDict_SetItemString(d, "While", (PyObject*)While_type) < 0) return
NULL;
if (PyDict_SetItemString(d, "If", (PyObject*)If_type) < 0) return NULL;
if (PyDict_SetItemString(d, "With", (PyObject*)With_type) < 0) return NULL;
if (PyDict_SetItemString(d, "AsyncWith", (PyObject*)AsyncWith_type) < 0)
return NULL;
if (PyDict_SetItemString(d, "Raise", (PyObject*)Raise_type) < 0) return
NULL;
if (PyDict_SetItemString(d, "Try", (PyObject*)Try_type) < 0) return NULL;
@ -6837,6 +7308,8 @@ PyInit__ast(void)
return NULL;
if (PyDict_SetItemString(d, "GeneratorExp", (PyObject*)GeneratorExp_type) <
0) return NULL;
if (PyDict_SetItemString(d, "Await", (PyObject*)Await_type) < 0) return
NULL;
if (PyDict_SetItemString(d, "Yield", (PyObject*)Yield_type) < 0) return
NULL;
if (PyDict_SetItemString(d, "YieldFrom", (PyObject*)YieldFrom_type) < 0)

168
Python/ast.c
View File

@ -219,6 +219,8 @@ validate_expr(expr_ty exp, expr_context_ty ctx)
return !exp->v.Yield.value || validate_expr(exp->v.Yield.value, Load);
case YieldFrom_kind:
return validate_expr(exp->v.YieldFrom.value, Load);
case Await_kind:
return validate_expr(exp->v.Await.value, Load);
case Compare_kind:
if (!asdl_seq_LEN(exp->v.Compare.comparators)) {
PyErr_SetString(PyExc_ValueError, "Compare with no comparators");
@ -336,6 +338,11 @@ validate_stmt(stmt_ty stmt)
validate_expr(stmt->v.For.iter, Load) &&
validate_body(stmt->v.For.body, "For") &&
validate_stmts(stmt->v.For.orelse);
case AsyncFor_kind:
return validate_expr(stmt->v.AsyncFor.target, Store) &&
validate_expr(stmt->v.AsyncFor.iter, Load) &&
validate_body(stmt->v.AsyncFor.body, "AsyncFor") &&
validate_stmts(stmt->v.AsyncFor.orelse);
case While_kind:
return validate_expr(stmt->v.While.test, Load) &&
validate_body(stmt->v.While.body, "While") &&
@ -354,6 +361,16 @@ validate_stmt(stmt_ty stmt)
return 0;
}
return validate_body(stmt->v.With.body, "With");
case AsyncWith_kind:
if (!validate_nonempty_seq(stmt->v.AsyncWith.items, "items", "AsyncWith"))
return 0;
for (i = 0; i < asdl_seq_LEN(stmt->v.AsyncWith.items); i++) {
withitem_ty item = asdl_seq_GET(stmt->v.AsyncWith.items, i);
if (!validate_expr(item->context_expr, Load) ||
(item->optional_vars && !validate_expr(item->optional_vars, Store)))
return 0;
}
return validate_body(stmt->v.AsyncWith.body, "AsyncWith");
case Raise_kind:
if (stmt->v.Raise.exc) {
return validate_expr(stmt->v.Raise.exc, Load) &&
@ -405,6 +422,12 @@ validate_stmt(stmt_ty stmt)
return validate_nonempty_seq(stmt->v.Nonlocal.names, "names", "Nonlocal");
case Expr_kind:
return validate_expr(stmt->v.Expr.value, Load);
case AsyncFunctionDef_kind:
return validate_body(stmt->v.AsyncFunctionDef.body, "AsyncFunctionDef") &&
validate_arguments(stmt->v.AsyncFunctionDef.args) &&
validate_exprs(stmt->v.AsyncFunctionDef.decorator_list, Load, 0) &&
(!stmt->v.AsyncFunctionDef.returns ||
validate_expr(stmt->v.AsyncFunctionDef.returns, Load));
case Pass_kind:
case Break_kind:
case Continue_kind:
@ -503,6 +526,9 @@ static asdl_seq *ast_for_exprlist(struct compiling *, const node *,
static expr_ty ast_for_testlist(struct compiling *, const node *);
static stmt_ty ast_for_classdef(struct compiling *, const node *, asdl_seq *);
static stmt_ty ast_for_with_stmt(struct compiling *, const node *, int);
static stmt_ty ast_for_for_stmt(struct compiling *, const node *, int);
/* Note different signature for ast_for_call */
static expr_ty ast_for_call(struct compiling *, const node *, expr_ty);
@ -941,6 +967,9 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n)
case YieldFrom_kind:
expr_name = "yield expression";
break;
case Await_kind:
expr_name = "await expression";
break;
case ListComp_kind:
expr_name = "list comprehension";
break;
@ -1480,7 +1509,8 @@ ast_for_decorators(struct compiling *c, const node *n)
}
static stmt_ty
ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
ast_for_funcdef_impl(struct compiling *c, const node *n,
asdl_seq *decorator_seq, int is_async)
{
/* funcdef: 'def' NAME parameters ['->' test] ':' suite */
identifier name;
@ -1509,14 +1539,68 @@ ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
if (!body)
return NULL;
return FunctionDef(name, args, body, decorator_seq, returns, LINENO(n),
if (is_async)
return AsyncFunctionDef(name, args, body, decorator_seq, returns,
LINENO(n),
n->n_col_offset, c->c_arena);
else
return FunctionDef(name, args, body, decorator_seq, returns,
LINENO(n),
n->n_col_offset, c->c_arena);
}
static stmt_ty
ast_for_async_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
{
/* async_funcdef: ASYNC funcdef */
REQ(n, async_funcdef);
REQ(CHILD(n, 0), ASYNC);
REQ(CHILD(n, 1), funcdef);
return ast_for_funcdef_impl(c, CHILD(n, 1), decorator_seq,
1 /* is_async */);
}
static stmt_ty
ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
{
/* funcdef: 'def' NAME parameters ['->' test] ':' suite */
return ast_for_funcdef_impl(c, n, decorator_seq,
0 /* is_async */);
}
static stmt_ty
ast_for_async_stmt(struct compiling *c, const node *n)
{
/* async_stmt: ASYNC (funcdef | with_stmt | for_stmt) */
REQ(n, async_stmt);
REQ(CHILD(n, 0), ASYNC);
switch (TYPE(CHILD(n, 1))) {
case funcdef:
return ast_for_funcdef_impl(c, CHILD(n, 1), NULL,
1 /* is_async */);
case with_stmt:
return ast_for_with_stmt(c, CHILD(n, 1),
1 /* is_async */);
case for_stmt:
return ast_for_for_stmt(c, CHILD(n, 1),
1 /* is_async */);
default:
PyErr_Format(PyExc_SystemError,
"invalid async stament: %s",
STR(CHILD(n, 1)));
return NULL;
}
}
static stmt_ty
ast_for_decorated(struct compiling *c, const node *n)
{
/* decorated: decorators (classdef | funcdef) */
/* decorated: decorators (classdef | funcdef | async_funcdef) */
stmt_ty thing = NULL;
asdl_seq *decorator_seq = NULL;
@ -1527,12 +1611,15 @@ ast_for_decorated(struct compiling *c, const node *n)
return NULL;
assert(TYPE(CHILD(n, 1)) == funcdef ||
TYPE(CHILD(n, 1)) == async_funcdef ||
TYPE(CHILD(n, 1)) == classdef);
if (TYPE(CHILD(n, 1)) == funcdef) {
thing = ast_for_funcdef(c, CHILD(n, 1), decorator_seq);
} else if (TYPE(CHILD(n, 1)) == classdef) {
thing = ast_for_classdef(c, CHILD(n, 1), decorator_seq);
} else if (TYPE(CHILD(n, 1)) == async_funcdef) {
thing = ast_for_async_funcdef(c, CHILD(n, 1), decorator_seq);
}
/* we count the decorators in when talking about the class' or
* function's line number */
@ -2271,19 +2358,29 @@ ast_for_factor(struct compiling *c, const node *n)
}
static expr_ty
ast_for_power(struct compiling *c, const node *n)
ast_for_atom_expr(struct compiling *c, const node *n)
{
/* power: atom trailer* ('**' factor)*
*/
int i;
int i, nch, start = 0;
expr_ty e, tmp;
REQ(n, power);
e = ast_for_atom(c, CHILD(n, 0));
REQ(n, atom_expr);
nch = NCH(n);
if (TYPE(CHILD(n, 0)) == AWAIT) {
start = 1;
assert(nch > 1);
}
e = ast_for_atom(c, CHILD(n, start));
if (!e)
return NULL;
if (NCH(n) == 1)
if (nch == 1)
return e;
for (i = 1; i < NCH(n); i++) {
if (start && nch == 2) {
return Await(e, LINENO(n), n->n_col_offset, c->c_arena);
}
for (i = start + 1; i < nch; i++) {
node *ch = CHILD(n, i);
if (TYPE(ch) != trailer)
break;
@ -2294,6 +2391,28 @@ ast_for_power(struct compiling *c, const node *n)
tmp->col_offset = e->col_offset;
e = tmp;
}
if (start) {
/* there was an AWAIT */
return Await(e, LINENO(n), n->n_col_offset, c->c_arena);
}
else {
return e;
}
}
static expr_ty
ast_for_power(struct compiling *c, const node *n)
{
/* power: atom trailer* ('**' factor)*
*/
expr_ty e;
REQ(n, power);
e = ast_for_atom_expr(c, CHILD(n, 0));
if (!e)
return NULL;
if (NCH(n) == 1)
return e;
if (TYPE(CHILD(n, NCH(n) - 1)) == factor) {
expr_ty f = ast_for_expr(c, CHILD(n, NCH(n) - 1));
if (!f)
@ -2338,7 +2457,9 @@ ast_for_expr(struct compiling *c, const node *n)
arith_expr: term (('+'|'-') term)*
term: factor (('*'|'@'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
power: atom trailer* ('**' factor)*
power: atom_expr ['**' factor]
atom_expr: [AWAIT] atom trailer*
yield_expr: 'yield' [yield_arg]
*/
asdl_seq *seq;
@ -3403,7 +3524,7 @@ ast_for_while_stmt(struct compiling *c, const node *n)
}
static stmt_ty
ast_for_for_stmt(struct compiling *c, const node *n)
ast_for_for_stmt(struct compiling *c, const node *n, int is_async)
{
asdl_seq *_target, *seq = NULL, *suite_seq;
expr_ty expression;
@ -3437,7 +3558,13 @@ ast_for_for_stmt(struct compiling *c, const node *n)
if (!suite_seq)
return NULL;
return For(target, expression, suite_seq, seq, LINENO(n), n->n_col_offset,
if (is_async)
return AsyncFor(target, expression, suite_seq, seq,
LINENO(n), n->n_col_offset,
c->c_arena);
else
return For(target, expression, suite_seq, seq,
LINENO(n), n->n_col_offset,
c->c_arena);
}
@ -3585,7 +3712,7 @@ ast_for_with_item(struct compiling *c, const node *n)
/* with_stmt: 'with' with_item (',' with_item)* ':' suite */
static stmt_ty
ast_for_with_stmt(struct compiling *c, const node *n)
ast_for_with_stmt(struct compiling *c, const node *n, int is_async)
{
int i, n_items;
asdl_seq *items, *body;
@ -3607,6 +3734,9 @@ ast_for_with_stmt(struct compiling *c, const node *n)
if (!body)
return NULL;
if (is_async)
return AsyncWith(items, body, LINENO(n), n->n_col_offset, c->c_arena);
else
return With(items, body, LINENO(n), n->n_col_offset, c->c_arena);
}
@ -3714,7 +3844,7 @@ ast_for_stmt(struct compiling *c, const node *n)
}
else {
/* compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt
| funcdef | classdef | decorated
| funcdef | classdef | decorated | async_stmt
*/
node *ch = CHILD(n, 0);
REQ(n, compound_stmt);
@ -3724,17 +3854,19 @@ ast_for_stmt(struct compiling *c, const node *n)
case while_stmt:
return ast_for_while_stmt(c, ch);
case for_stmt:
return ast_for_for_stmt(c, ch);
return ast_for_for_stmt(c, ch, 0);
case try_stmt:
return ast_for_try_stmt(c, ch);
case with_stmt:
return ast_for_with_stmt(c, ch);
return ast_for_with_stmt(c, ch, 0);
case funcdef:
return ast_for_funcdef(c, ch, NULL);
case classdef:
return ast_for_classdef(c, ch, NULL);
case decorated:
return ast_for_decorated(c, ch);
case async_stmt:
return ast_for_async_stmt(c, ch);
default:
PyErr_Format(PyExc_SystemError,
"unhandled small_stmt: TYPE=%d NCH=%d\n",

223
Python/ceval.c
View File

@ -1926,11 +1926,133 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
goto fast_block_end;
}
TARGET(GET_AITER) {
getaiterfunc getter = NULL;
PyObject *iter = NULL;
PyObject *awaitable = NULL;
PyObject *obj = TOP();
PyTypeObject *type = Py_TYPE(obj);
if (type->tp_as_async != NULL)
getter = type->tp_as_async->am_aiter;
if (getter != NULL) {
iter = (*getter)(obj);
Py_DECREF(obj);
if (iter == NULL) {
SET_TOP(NULL);
goto error;
}
}
else {
SET_TOP(NULL);
PyErr_Format(
PyExc_TypeError,
"'async for' requires an object with "
"__aiter__ method, got %.100s",
type->tp_name);
Py_DECREF(obj);
goto error;
}
awaitable = _PyGen_GetAwaitableIter(iter);
if (awaitable == NULL) {
SET_TOP(NULL);
PyErr_Format(
PyExc_TypeError,
"'async for' received an invalid object "
"from __aiter__: %.100s",
Py_TYPE(iter)->tp_name);
Py_DECREF(iter);
goto error;
} else
Py_DECREF(iter);
SET_TOP(awaitable);
DISPATCH();
}
TARGET(GET_ANEXT) {
aiternextfunc getter = NULL;
PyObject *next_iter = NULL;
PyObject *awaitable = NULL;
PyObject *aiter = TOP();
PyTypeObject *type = Py_TYPE(aiter);
if (type->tp_as_async != NULL)
getter = type->tp_as_async->am_anext;
if (getter != NULL) {
next_iter = (*getter)(aiter);
if (next_iter == NULL) {
goto error;
}
}
else {
PyErr_Format(
PyExc_TypeError,
"'async for' requires an iterator with "
"__anext__ method, got %.100s",
type->tp_name);
goto error;
}
awaitable = _PyGen_GetAwaitableIter(next_iter);
if (awaitable == NULL) {
PyErr_Format(
PyExc_TypeError,
"'async for' received an invalid object "
"from __anext__: %.100s",
Py_TYPE(next_iter)->tp_name);
Py_DECREF(next_iter);
goto error;
} else
Py_DECREF(next_iter);
PUSH(awaitable);
DISPATCH();
}
TARGET(GET_AWAITABLE) {
PyObject *iterable = TOP();
PyObject *iter = _PyGen_GetAwaitableIter(iterable);
Py_DECREF(iterable);
SET_TOP(iter); /* Even if it's NULL */
if (iter == NULL) {
goto error;
}
DISPATCH();
}
TARGET(YIELD_FROM) {
PyObject *v = POP();
PyObject *reciever = TOP();
int err;
if (PyGen_CheckExact(reciever)) {
if (
(((PyCodeObject*) \
((PyGenObject*)reciever)->gi_code)->co_flags &
CO_COROUTINE)
&& !(co->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE)))
{
/* If we're yielding-from a coroutine object from a regular
generator object - raise an error. */
Py_CLEAR(v);
Py_CLEAR(reciever);
SET_TOP(NULL);
PyErr_SetString(PyExc_TypeError,
"cannot 'yield from' a coroutine object "
"from a generator");
goto error;
}
retval = _PyGen_Send((PyGenObject *)reciever, v);
} else {
_Py_IDENTIFIER(send);
@ -2822,11 +2944,26 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
TARGET(GET_ITER) {
/* before: [obj]; after [getiter(obj)] */
PyObject *iterable = TOP();
PyObject *iter = PyObject_GetIter(iterable);
PyObject *iter;
/* If we have a generator object on top -- keep it there,
it's already an iterator.
This is needed to allow use of 'async def' coroutines
in 'yield from' expression from generator-based coroutines
(decorated with types.coroutine()).
'yield from' is compiled to GET_ITER..YIELD_FROM combination,
but since coroutines raise TypeError in their 'tp_iter' we
need a way for them to "pass through" the GET_ITER.
*/
if (!PyGen_CheckExact(iterable)) {
/* `iterable` is not a generator. */
iter = PyObject_GetIter(iterable);
Py_DECREF(iterable);
SET_TOP(iter);
if (iter == NULL)
goto error;
}
PREDICT(FOR_ITER);
DISPATCH();
}
@ -2883,6 +3020,39 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
DISPATCH();
}
TARGET(BEFORE_ASYNC_WITH) {
_Py_IDENTIFIER(__aexit__);
_Py_IDENTIFIER(__aenter__);
PyObject *mgr = TOP();
PyObject *exit = special_lookup(mgr, &PyId___aexit__),
*enter;
PyObject *res;
if (exit == NULL)
goto error;
SET_TOP(exit);
enter = special_lookup(mgr, &PyId___aenter__);
Py_DECREF(mgr);
if (enter == NULL)
goto error;
res = PyObject_CallFunctionObjArgs(enter, NULL);
Py_DECREF(enter);
if (res == NULL)
goto error;
PUSH(res);
DISPATCH();
}
TARGET(SETUP_ASYNC_WITH) {
PyObject *res = POP();
/* Setup the finally block before pushing the result
of __aenter__ on the stack. */
PyFrame_BlockSetup(f, SETUP_FINALLY, INSTR_OFFSET() + oparg,
STACK_LEVEL());
PUSH(res);
DISPATCH();
}
TARGET(SETUP_WITH) {
_Py_IDENTIFIER(__exit__);
_Py_IDENTIFIER(__enter__);
@ -2909,7 +3079,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
DISPATCH();
}
TARGET(WITH_CLEANUP) {
TARGET(WITH_CLEANUP_START) {
/* At the top of the stack are 1-6 values indicating
how/why we entered the finally clause:
- TOP = None
@ -2937,7 +3107,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
PyObject *exit_func;
PyObject *exc = TOP(), *val = Py_None, *tb = Py_None, *res;
int err;
if (exc == Py_None) {
(void)POP();
exit_func = TOP();
@ -2987,10 +3156,23 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
if (res == NULL)
goto error;
PUSH(exc);
PUSH(res);
PREDICT(WITH_CLEANUP_FINISH);
DISPATCH();
}
PREDICTED(WITH_CLEANUP_FINISH);
TARGET(WITH_CLEANUP_FINISH) {
PyObject *res = POP();
PyObject *exc = POP();
int err;
if (exc != Py_None)
err = PyObject_IsTrue(res);
else
err = 0;
Py_DECREF(res);
if (err < 0)
@ -3751,6 +3933,9 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals,
}
if (co->co_flags & CO_GENERATOR) {
PyObject *gen;
PyObject *coroutine_wrapper;
/* Don't need to keep the reference to f_back, it will be set
* when the generator is resumed. */
Py_CLEAR(f->f_back);
@ -3759,7 +3944,19 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals,
/* Create a new generator that owns the ready to run frame
* and return that as the value. */
return PyGen_NewWithQualName(f, name, qualname);
gen = PyGen_NewWithQualName(f, name, qualname);
if (gen == NULL)
return NULL;
if (co->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE)) {
coroutine_wrapper = PyEval_GetCoroutineWrapper();
if (coroutine_wrapper != NULL) {
PyObject *wrapped =
PyObject_CallFunction(coroutine_wrapper, "N", gen);
gen = wrapped;
}
}
return gen;
}
retval = PyEval_EvalFrameEx(f,0);
@ -4205,6 +4402,24 @@ PyEval_SetTrace(Py_tracefunc func, PyObject *arg)
|| (tstate->c_profilefunc != NULL));
}
void
PyEval_SetCoroutineWrapper(PyObject *wrapper)
{
PyThreadState *tstate = PyThreadState_GET();
Py_CLEAR(tstate->coroutine_wrapper);
Py_XINCREF(wrapper);
tstate->coroutine_wrapper = wrapper;
}
PyObject *
PyEval_GetCoroutineWrapper()
{
PyThreadState *tstate = PyThreadState_GET();
return tstate->coroutine_wrapper;
}
PyObject *
PyEval_GetBuiltins(void)
{

286
Python/compile.c
View File

@ -92,6 +92,7 @@ enum {
COMPILER_SCOPE_MODULE,
COMPILER_SCOPE_CLASS,
COMPILER_SCOPE_FUNCTION,
COMPILER_SCOPE_ASYNC_FUNCTION,
COMPILER_SCOPE_LAMBDA,
COMPILER_SCOPE_COMPREHENSION,
};
@ -193,6 +194,8 @@ static int inplace_binop(struct compiler *, operator_ty);
static int expr_constant(struct compiler *, expr_ty);
static int compiler_with(struct compiler *, stmt_ty, int);
static int compiler_async_with(struct compiler *, stmt_ty, int);
static int compiler_async_for(struct compiler *, stmt_ty);
static int compiler_call_helper(struct compiler *c, Py_ssize_t n,
asdl_seq *args,
asdl_seq *keywords);
@ -673,7 +676,9 @@ compiler_set_qualname(struct compiler *c)
parent = (struct compiler_unit *)PyCapsule_GetPointer(capsule, CAPSULE_NAME);
assert(parent);
if (u->u_scope_type == COMPILER_SCOPE_FUNCTION || u->u_scope_type == COMPILER_SCOPE_CLASS) {
if (u->u_scope_type == COMPILER_SCOPE_FUNCTION
|| u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION
|| u->u_scope_type == COMPILER_SCOPE_CLASS) {
assert(u->u_name);
mangled = _Py_Mangle(parent->u_private, u->u_name);
if (!mangled)
@ -687,6 +692,7 @@ compiler_set_qualname(struct compiler *c)
if (!force_global) {
if (parent->u_scope_type == COMPILER_SCOPE_FUNCTION
|| parent->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION
|| parent->u_scope_type == COMPILER_SCOPE_LAMBDA) {
dot_locals_str = _PyUnicode_FromId(&dot_locals);
if (dot_locals_str == NULL)
@ -927,7 +933,9 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg)
return 0;
case SETUP_WITH:
return 7;
case WITH_CLEANUP:
case WITH_CLEANUP_START:
return 1;
case WITH_CLEANUP_FINISH:
return -1; /* XXX Sometimes more */
case RETURN_VALUE:
return -1;
@ -1048,6 +1056,16 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg)
return -1;
case DELETE_DEREF:
return 0;
case GET_AWAITABLE:
return 0;
case SETUP_ASYNC_WITH:
return 6;
case BEFORE_ASYNC_WITH:
return 1;
case GET_AITER:
return 0;
case GET_ANEXT:
return 1;
default:
return PY_INVALID_STACK_EFFECT;
}
@ -1642,20 +1660,44 @@ error:
}
static int
compiler_function(struct compiler *c, stmt_ty s)
compiler_function(struct compiler *c, stmt_ty s, int is_async)
{
PyCodeObject *co;
PyObject *qualname, *first_const = Py_None;
arguments_ty args = s->v.FunctionDef.args;
expr_ty returns = s->v.FunctionDef.returns;
asdl_seq* decos = s->v.FunctionDef.decorator_list;
arguments_ty args;
expr_ty returns;
identifier name;
asdl_seq* decos;
asdl_seq *body;
stmt_ty st;
Py_ssize_t i, n, arglength;
int docstring, kw_default_count = 0;
int num_annotations;
int scope_type;
if (is_async) {
assert(s->kind == AsyncFunctionDef_kind);
args = s->v.AsyncFunctionDef.args;
returns = s->v.AsyncFunctionDef.returns;
decos = s->v.AsyncFunctionDef.decorator_list;
name = s->v.AsyncFunctionDef.name;
body = s->v.AsyncFunctionDef.body;
scope_type = COMPILER_SCOPE_ASYNC_FUNCTION;
} else {
assert(s->kind == FunctionDef_kind);
args = s->v.FunctionDef.args;
returns = s->v.FunctionDef.returns;
decos = s->v.FunctionDef.decorator_list;
name = s->v.FunctionDef.name;
body = s->v.FunctionDef.body;
scope_type = COMPILER_SCOPE_FUNCTION;
}
if (!compiler_decorators(c, decos))
return 0;
if (args->defaults)
@ -1672,12 +1714,12 @@ compiler_function(struct compiler *c, stmt_ty s)
return 0;
assert((num_annotations & 0xFFFF) == num_annotations);
if (!compiler_enter_scope(c, s->v.FunctionDef.name,
COMPILER_SCOPE_FUNCTION, (void *)s,
if (!compiler_enter_scope(c, name,
scope_type, (void *)s,
s->lineno))
return 0;
st = (stmt_ty)asdl_seq_GET(s->v.FunctionDef.body, 0);
st = (stmt_ty)asdl_seq_GET(body, 0);
docstring = compiler_isdocstring(st);
if (docstring && c->c_optimize < 2)
first_const = st->v.Expr.value->v.Str.s;
@ -1688,10 +1730,10 @@ compiler_function(struct compiler *c, stmt_ty s)
c->u->u_argcount = asdl_seq_LEN(args->args);
c->u->u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs);
n = asdl_seq_LEN(s->v.FunctionDef.body);
n = asdl_seq_LEN(body);
/* if there was a docstring, we need to skip the first statement */
for (i = docstring; i < n; i++) {
st = (stmt_ty)asdl_seq_GET(s->v.FunctionDef.body, i);
st = (stmt_ty)asdl_seq_GET(body, i);
VISIT_IN_SCOPE(c, stmt, st);
}
co = assemble(c, 1);
@ -1711,12 +1753,19 @@ compiler_function(struct compiler *c, stmt_ty s)
Py_DECREF(qualname);
Py_DECREF(co);
if (is_async) {
co->co_flags |= CO_COROUTINE;
/* An async function is always a generator, even
if there is no 'yield' expressions in it. */
co->co_flags |= CO_GENERATOR;
}
/* decorators */
for (i = 0; i < asdl_seq_LEN(decos); i++) {
ADDOP_I(c, CALL_FUNCTION, 1);
}
return compiler_nameop(c, s->v.FunctionDef.name, Store);
return compiler_nameop(c, name, Store);
}
static int
@ -1989,6 +2038,92 @@ compiler_for(struct compiler *c, stmt_ty s)
return 1;
}
static int
compiler_async_for(struct compiler *c, stmt_ty s)
{
static PyObject *stopiter_error = NULL;
basicblock *try, *except, *end, *after_try, *try_cleanup,
*after_loop, *after_loop_else;
if (stopiter_error == NULL) {
stopiter_error = PyUnicode_InternFromString("StopAsyncIteration");
if (stopiter_error == NULL)
return 0;
}
try = compiler_new_block(c);
except = compiler_new_block(c);
end = compiler_new_block(c);
after_try = compiler_new_block(c);
try_cleanup = compiler_new_block(c);
after_loop = compiler_new_block(c);
after_loop_else = compiler_new_block(c);
if (try == NULL || except == NULL || end == NULL
|| after_try == NULL || try_cleanup == NULL)
return 0;
ADDOP_JREL(c, SETUP_LOOP, after_loop);
if (!compiler_push_fblock(c, LOOP, try))
return 0;
VISIT(c, expr, s->v.AsyncFor.iter);
ADDOP(c, GET_AITER);
ADDOP_O(c, LOAD_CONST, Py_None, consts);
ADDOP(c, YIELD_FROM);
compiler_use_next_block(c, try);
ADDOP_JREL(c, SETUP_EXCEPT, except);
if (!compiler_push_fblock(c, EXCEPT, try))
return 0;
ADDOP(c, GET_ANEXT);
ADDOP_O(c, LOAD_CONST, Py_None, consts);
ADDOP(c, YIELD_FROM);
VISIT(c, expr, s->v.AsyncFor.target);
ADDOP(c, POP_BLOCK);
compiler_pop_fblock(c, EXCEPT, try);
ADDOP_JREL(c, JUMP_FORWARD, after_try);
compiler_use_next_block(c, except);
ADDOP(c, DUP_TOP);
ADDOP_O(c, LOAD_GLOBAL, stopiter_error, names);
ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH);
ADDOP_JABS(c, POP_JUMP_IF_FALSE, try_cleanup);
ADDOP(c, POP_TOP);
ADDOP(c, POP_TOP);
ADDOP(c, POP_TOP);
ADDOP(c, POP_EXCEPT); /* for SETUP_EXCEPT */
ADDOP(c, POP_BLOCK); /* for SETUP_LOOP */
ADDOP_JABS(c, JUMP_ABSOLUTE, after_loop_else);
compiler_use_next_block(c, try_cleanup);
ADDOP(c, END_FINALLY);
compiler_use_next_block(c, after_try);
VISIT_SEQ(c, stmt, s->v.AsyncFor.body);
ADDOP_JABS(c, JUMP_ABSOLUTE, try);
ADDOP(c, POP_BLOCK); /* for SETUP_LOOP */
compiler_pop_fblock(c, LOOP, try);
compiler_use_next_block(c, after_loop);
ADDOP_JABS(c, JUMP_ABSOLUTE, end);
compiler_use_next_block(c, after_loop_else);
VISIT_SEQ(c, stmt, s->v.For.orelse);
compiler_use_next_block(c, end);
return 1;
}
static int
compiler_while(struct compiler *c, stmt_ty s)
{
@ -2515,7 +2650,7 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s)
switch (s->kind) {
case FunctionDef_kind:
return compiler_function(c, s);
return compiler_function(c, s, 0);
case ClassDef_kind:
return compiler_class(c, s);
case Return_kind:
@ -2594,7 +2729,14 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s)
return compiler_continue(c);
case With_kind:
return compiler_with(c, s, 0);
case AsyncFunctionDef_kind:
return compiler_function(c, s, 1);
case AsyncWith_kind:
return compiler_async_with(c, s, 0);
case AsyncFor_kind:
return compiler_async_for(c, s);
}
return 1;
}
@ -3471,6 +3613,102 @@ expr_constant(struct compiler *c, expr_ty e)
}
}
/*
Implements the async with statement.
The semantics outlined in that PEP are as follows:
async with EXPR as VAR:
BLOCK
It is implemented roughly as:
context = EXPR
exit = context.__aexit__ # not calling it
value = await context.__aenter__()
try:
VAR = value # if VAR present in the syntax
BLOCK
finally:
if an exception was raised:
exc = copy of (exception, instance, traceback)
else:
exc = (None, None, None)
if not (await exit(*exc)):
raise
*/
static int
compiler_async_with(struct compiler *c, stmt_ty s, int pos)
{
basicblock *block, *finally;
withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos);
assert(s->kind == AsyncWith_kind);
block = compiler_new_block(c);
finally = compiler_new_block(c);
if (!block || !finally)
return 0;
/* Evaluate EXPR */
VISIT(c, expr, item->context_expr);
ADDOP(c, BEFORE_ASYNC_WITH);
ADDOP(c, GET_AWAITABLE);
ADDOP_O(c, LOAD_CONST, Py_None, consts);
ADDOP(c, YIELD_FROM);
ADDOP_JREL(c, SETUP_ASYNC_WITH, finally);
/* SETUP_ASYNC_WITH pushes a finally block. */
compiler_use_next_block(c, block);
if (!compiler_push_fblock(c, FINALLY_TRY, block)) {
return 0;
}
if (item->optional_vars) {
VISIT(c, expr, item->optional_vars);
}
else {
/* Discard result from context.__aenter__() */
ADDOP(c, POP_TOP);
}
pos++;
if (pos == asdl_seq_LEN(s->v.AsyncWith.items))
/* BLOCK code */
VISIT_SEQ(c, stmt, s->v.AsyncWith.body)
else if (!compiler_async_with(c, s, pos))
return 0;
/* End of try block; start the finally block */
ADDOP(c, POP_BLOCK);
compiler_pop_fblock(c, FINALLY_TRY, block);
ADDOP_O(c, LOAD_CONST, Py_None, consts);
compiler_use_next_block(c, finally);
if (!compiler_push_fblock(c, FINALLY_END, finally))
return 0;
/* Finally block starts; context.__exit__ is on the stack under
the exception or return information. Just issue our magic
opcode. */
ADDOP(c, WITH_CLEANUP_START);
ADDOP(c, GET_AWAITABLE);
ADDOP_O(c, LOAD_CONST, Py_None, consts);
ADDOP(c, YIELD_FROM);
ADDOP(c, WITH_CLEANUP_FINISH);
/* Finally block ends. */
ADDOP(c, END_FINALLY);
compiler_pop_fblock(c, FINALLY_END, finally);
return 1;
}
/*
Implements the with statement from PEP 343.
@ -3544,7 +3782,8 @@ compiler_with(struct compiler *c, stmt_ty s, int pos)
/* Finally block starts; context.__exit__ is on the stack under
the exception or return information. Just issue our magic
opcode. */
ADDOP(c, WITH_CLEANUP);
ADDOP(c, WITH_CLEANUP_START);
ADDOP(c, WITH_CLEANUP_FINISH);
/* Finally block ends. */
ADDOP(c, END_FINALLY);
@ -3595,6 +3834,8 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
case Yield_kind:
if (c->u->u_ste->ste_type != FunctionBlock)
return compiler_error(c, "'yield' outside function");
if (c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION)
return compiler_error(c, "'yield' inside async function");
if (e->v.Yield.value) {
VISIT(c, expr, e->v.Yield.value);
}
@ -3606,11 +3847,28 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
case YieldFrom_kind:
if (c->u->u_ste->ste_type != FunctionBlock)
return compiler_error(c, "'yield' outside function");
if (c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION)
return compiler_error(c, "'yield from' inside async function");
VISIT(c, expr, e->v.YieldFrom.value);
ADDOP(c, GET_ITER);
ADDOP_O(c, LOAD_CONST, Py_None, consts);
ADDOP(c, YIELD_FROM);
break;
case Await_kind:
if (c->u->u_ste->ste_type != FunctionBlock)
return compiler_error(c, "'await' outside function");
/* this check won't be triggered while we have AWAIT token */
if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION)
return compiler_error(c, "'await' outside async function");
VISIT(c, expr, e->v.Await.value);
ADDOP(c, GET_AWAITABLE);
ADDOP_O(c, LOAD_CONST, Py_None, consts);
ADDOP(c, YIELD_FROM);
break;
case Compare_kind:
return compiler_compare(c, e);
case Call_kind:

2714
Python/graminit.c
View File

File diff suppressed because it is too large Load Diff

3480
Python/importlib.h
View File

File diff suppressed because it is too large Load Diff

4773
Python/importlib_external.h
View File

File diff suppressed because it is too large Load Diff

14
Python/opcode_targets.h
View File

@ -49,9 +49,9 @@ static void *opcode_targets[256] = {
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_GET_AITER,
&&TARGET_GET_ANEXT,
&&TARGET_BEFORE_ASYNC_WITH,
&&_unknown_opcode,
&&TARGET_STORE_MAP,
&&TARGET_INPLACE_ADD,
@ -72,7 +72,7 @@ static void *opcode_targets[256] = {
&&TARGET_PRINT_EXPR,
&&TARGET_LOAD_BUILD_CLASS,
&&TARGET_YIELD_FROM,
&&_unknown_opcode,
&&TARGET_GET_AWAITABLE,
&&_unknown_opcode,
&&TARGET_INPLACE_LSHIFT,
&&TARGET_INPLACE_RSHIFT,
@ -80,8 +80,8 @@ static void *opcode_targets[256] = {
&&TARGET_INPLACE_XOR,
&&TARGET_INPLACE_OR,
&&TARGET_BREAK_LOOP,
&&TARGET_WITH_CLEANUP,
&&_unknown_opcode,
&&TARGET_WITH_CLEANUP_START,
&&TARGET_WITH_CLEANUP_FINISH,
&&TARGET_RETURN_VALUE,
&&TARGET_IMPORT_STAR,
&&_unknown_opcode,
@ -153,7 +153,7 @@ static void *opcode_targets[256] = {
&&TARGET_BUILD_MAP_UNPACK_WITH_CALL,
&&TARGET_BUILD_TUPLE_UNPACK,
&&TARGET_BUILD_SET_UNPACK,
&&_unknown_opcode,
&&TARGET_SETUP_ASYNC_WITH,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,

3
Python/peephole.c
View File

@ -319,6 +319,7 @@ markblocks(unsigned char *code, Py_ssize_t len)
case SETUP_EXCEPT:
case SETUP_FINALLY:
case SETUP_WITH:
case SETUP_ASYNC_WITH:
j = GETJUMPTGT(code, i);
blocks[j] = 1;
break;
@ -620,6 +621,7 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names,
case SETUP_EXCEPT:
case SETUP_FINALLY:
case SETUP_WITH:
case SETUP_ASYNC_WITH:
tgt = GETJUMPTGT(codestr, i);
/* Replace JUMP_* to a RETURN into just a RETURN */
if (UNCONDITIONAL_JUMP(opcode) &&
@ -704,6 +706,7 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names,
case SETUP_EXCEPT:
case SETUP_FINALLY:
case SETUP_WITH:
case SETUP_ASYNC_WITH:
j = addrmap[GETARG(codestr, i) + i + 3] - addrmap[i] - 3;
SETARG(codestr, i, j);
break;

4
Python/pystate.c
View File

@ -212,6 +212,8 @@ new_threadstate(PyInterpreterState *interp, int init)
tstate->on_delete = NULL;
tstate->on_delete_data = NULL;
tstate->coroutine_wrapper = NULL;
if (init)
_PyThreadState_Init(tstate);
@ -372,6 +374,8 @@ PyThreadState_Clear(PyThreadState *tstate)
tstate->c_tracefunc = NULL;
Py_CLEAR(tstate->c_profileobj);
Py_CLEAR(tstate->c_traceobj);
Py_CLEAR(tstate->coroutine_wrapper);
}

49
Python/symtable.c
View File

@ -180,7 +180,7 @@ static int symtable_visit_slice(struct symtable *st, slice_ty);
static int symtable_visit_params(struct symtable *st, asdl_seq *args);
static int symtable_visit_argannotations(struct symtable *st, asdl_seq *args);
static int symtable_implicit_arg(struct symtable *st, int pos);
static int symtable_visit_annotations(struct symtable *st, stmt_ty s);
static int symtable_visit_annotations(struct symtable *st, stmt_ty s, arguments_ty, expr_ty);
static int symtable_visit_withitem(struct symtable *st, withitem_ty item);
@ -1147,7 +1147,8 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults);
if (s->v.FunctionDef.args->kw_defaults)
VISIT_SEQ_WITH_NULL(st, expr, s->v.FunctionDef.args->kw_defaults);
if (!symtable_visit_annotations(st, s))
if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args,
s->v.FunctionDef.returns))
VISIT_QUIT(st, 0);
if (s->v.FunctionDef.decorator_list)
VISIT_SEQ(st, expr, s->v.FunctionDef.decorator_list);
@ -1315,6 +1316,39 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
VISIT_SEQ(st, withitem, s->v.With.items);
VISIT_SEQ(st, stmt, s->v.With.body);
break;
case AsyncFunctionDef_kind:
if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL))
VISIT_QUIT(st, 0);
if (s->v.AsyncFunctionDef.args->defaults)
VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.args->defaults);
if (s->v.AsyncFunctionDef.args->kw_defaults)
VISIT_SEQ_WITH_NULL(st, expr,
s->v.AsyncFunctionDef.args->kw_defaults);
if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args,
s->v.AsyncFunctionDef.returns))
VISIT_QUIT(st, 0);
if (s->v.AsyncFunctionDef.decorator_list)
VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.decorator_list);
if (!symtable_enter_block(st, s->v.AsyncFunctionDef.name,
FunctionBlock, (void *)s, s->lineno,
s->col_offset))
VISIT_QUIT(st, 0);
VISIT(st, arguments, s->v.AsyncFunctionDef.args);
VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body);
if (!symtable_exit_block(st, s))
VISIT_QUIT(st, 0);
break;
case AsyncWith_kind:
VISIT_SEQ(st, withitem, s->v.AsyncWith.items);
VISIT_SEQ(st, stmt, s->v.AsyncWith.body);
break;
case AsyncFor_kind:
VISIT(st, expr, s->v.AsyncFor.target);
VISIT(st, expr, s->v.AsyncFor.iter);
VISIT_SEQ(st, stmt, s->v.AsyncFor.body);
if (s->v.AsyncFor.orelse)
VISIT_SEQ(st, stmt, s->v.AsyncFor.orelse);
break;
}
VISIT_QUIT(st, 1);
}
@ -1392,6 +1426,10 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
VISIT(st, expr, e->v.YieldFrom.value);
st->st_cur->ste_generator = 1;
break;
case Await_kind:
VISIT(st, expr, e->v.Await.value);
st->st_cur->ste_generator = 1;
break;
case Compare_kind:
VISIT(st, expr, e->v.Compare.left);
VISIT_SEQ(st, expr, e->v.Compare.comparators);
@ -1492,10 +1530,9 @@ symtable_visit_argannotations(struct symtable *st, asdl_seq *args)
}
static int
symtable_visit_annotations(struct symtable *st, stmt_ty s)
symtable_visit_annotations(struct symtable *st, stmt_ty s,
arguments_ty a, expr_ty returns)
{
arguments_ty a = s->v.FunctionDef.args;
if (a->args && !symtable_visit_argannotations(st, a->args))
return 0;
if (a->vararg && a->vararg->annotation)
@ -1504,7 +1541,7 @@ symtable_visit_annotations(struct symtable *st, stmt_ty s)
VISIT(st, expr, a->kwarg->annotation);
if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs))
return 0;
if (s->v.FunctionDef.returns)
if (returns)
VISIT(st, expr, s->v.FunctionDef.returns);
return 1;
}

47
Python/sysmodule.c
View File

@ -645,6 +645,49 @@ sys_setrecursionlimit(PyObject *self, PyObject *args)
return Py_None;
}
static PyObject *
sys_set_coroutine_wrapper(PyObject *self, PyObject *wrapper)
{
if (wrapper != Py_None) {
if (!PyCallable_Check(wrapper)) {
PyErr_Format(PyExc_TypeError,
"callable expected, got %.50s",
Py_TYPE(wrapper)->tp_name);
return NULL;
}
PyEval_SetCoroutineWrapper(wrapper);
}
else
PyEval_SetCoroutineWrapper(NULL);
Py_INCREF(Py_None);
Py_RETURN_NONE;
}
PyDoc_STRVAR(set_coroutine_wrapper_doc,
"set_coroutine_wrapper(wrapper)\n\
\n\
Set a wrapper for coroutine objects."
);
static PyObject *
sys_get_coroutine_wrapper(PyObject *self, PyObject *args)
{
PyObject *wrapper = PyEval_GetCoroutineWrapper();
if (wrapper == NULL) {
wrapper = Py_None;
}
Py_INCREF(wrapper);
return wrapper;
}
PyDoc_STRVAR(get_coroutine_wrapper_doc,
"get_coroutine_wrapper()\n\
\n\
Return the wrapper for coroutine objects set by sys.set_coroutine_wrapper."
);
static PyTypeObject Hash_InfoType;
PyDoc_STRVAR(hash_info_doc,
@ -1215,6 +1258,10 @@ static PyMethodDef sys_methods[] = {
{"call_tracing", sys_call_tracing, METH_VARARGS, call_tracing_doc},
{"_debugmallocstats", sys_debugmallocstats, METH_NOARGS,
debugmallocstats_doc},
{"set_coroutine_wrapper", sys_set_coroutine_wrapper, METH_O,
set_coroutine_wrapper_doc},
{"get_coroutine_wrapper", sys_get_coroutine_wrapper, METH_NOARGS,
get_coroutine_wrapper_doc},
{NULL, NULL} /* sentinel */
};

32
Tools/parser/unparse.py
View File

@ -138,6 +138,14 @@ class Unparser:
self.fill("nonlocal ")
interleave(lambda: self.write(", "), self.write, t.names)
def _Await(self, t):
self.write("(")
self.write("await")
if t.value:
self.write(" ")
self.dispatch(t.value)
self.write(")")
def _Yield(self, t):
self.write("(")
self.write("yield")
@ -218,11 +226,18 @@ class Unparser:
self.leave()
def _FunctionDef(self, t):
self.__FunctionDef_helper(t, "def")
def _AsyncFunctionDef(self, t):
self.__FunctionDef_helper(t, "async def")
def __FunctionDef_helper(self, t, fill_suffix):
self.write("\n")
for deco in t.decorator_list:
self.fill("@")
self.dispatch(deco)
self.fill("def "+t.name + "(")
def_str = fill_suffix+" "+t.name + "("
self.fill(def_str)
self.dispatch(t.args)
self.write(")")
if t.returns:
@ -233,7 +248,13 @@ class Unparser:
self.leave()
def _For(self, t):
self.fill("for ")
self.__For_helper("for ", t)
def _AsyncFor(self, t):
self.__For_helper("async for ", t)
def __For_helper(self, fill, t):
self.fill(fill)
self.dispatch(t.target)
self.write(" in ")
self.dispatch(t.iter)
@ -287,6 +308,13 @@ class Unparser:
self.dispatch(t.body)
self.leave()
def _AsyncWith(self, t):
self.fill("async with ")
interleave(lambda: self.write(", "), self.dispatch, t.items)
self.enter()
self.dispatch(t.body)
self.leave()
# expr
def _Bytes(self, t):
self.write(repr(t.s))