PEP 0492 -- Coroutines with async and await syntax. Issue #24017.

Doc/includes/typestruct.h

@@ -9,7 +9,7 @@ typedef struct _typeobject {
     printfunc tp_print;
     getattrfunc tp_getattr;
     setattrfunc tp_setattr;
-    void *tp_reserved;
+    PyAsyncMethods *tp_as_async;
     reprfunc tp_repr;
 
     /* Method suites for standard classes */

Doc/library/dis.rst

@@ -508,6 +508,38 @@ the original TOS1.
    Implements ``del TOS1[TOS]``.
 
 
+**Coroutines opcodes**
+
+.. opcode:: GET_AWAITABLE
+
+   Implements ``TOS = get_awaitable(TOS)``; where ``get_awaitable(o)``
+   returns ``o`` if ``o`` is a coroutine object; or resolved ``o.__await__``.
+
+
+.. opcode:: GET_AITER
+
+   Implements ``TOS = get_awaitable(TOS.__aiter__())``.  See ``GET_AWAITABLE``
+   for details about ``get_awaitable``
+
+
+.. opcode:: GET_ANEXT
+
+   Implements ``PUSH(get_awaitable(TOS.__anext__()))``.  See ``GET_AWAITABLE``
+   for details about ``get_awaitable``
+
+
+.. opcode:: BEFORE_ASYNC_WITH
+
+   Resolves ``__aenter__`` and ``__aexit__`` from the object on top of the
+   stack.  Pushes ``__aexit__`` and result of ``__aenter__()`` to the stack.
+
+
+.. opcode:: SETUP_ASYNC_WITH
+
+   Creates a new frame object.
+
+
+
 **Miscellaneous opcodes**
 
 .. opcode:: PRINT_EXPR
@@ -612,7 +644,7 @@ iterations of the loop.
    :opcode:`UNPACK_SEQUENCE`).
 
 
-.. opcode:: WITH_CLEANUP
+.. opcode:: WITH_CLEANUP_START
 
    Cleans up the stack when a :keyword:`with` statement block exits.  TOS is the
    context manager's :meth:`__exit__` bound method. Below TOS are 1--3 values
@@ -624,7 +656,13 @@ iterations of the loop.
    * (SECOND, THIRD, FOURTH) = exc_info()
 
    In the last case, ``TOS(SECOND, THIRD, FOURTH)`` is called, otherwise
-   ``TOS(None, None, None)``.  In addition, TOS is removed from the stack.
+   ``TOS(None, None, None)``.  Pushes SECOND and result of the call
+   to the stack.
+
+
+.. opcode:: WITH_CLEANUP_FINISH
+
+   Pops exception type and result of 'exit' function call from the stack.
 
    If the stack represents an exception, *and* the function call returns a
    'true' value, this information is "zapped" and replaced with a single

Grammar/Grammar

@@ -21,8 +21,11 @@ eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
 decorators: decorator+
-decorated: decorators (classdef | funcdef)
+decorated: decorators (classdef | funcdef | async_funcdef)
+
+async_funcdef: ASYNC funcdef
 funcdef: 'def' NAME parameters ['->' test] ':' suite
+
 parameters: '(' [typedargslist] ')'
 typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [','
        ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]]
@@ -65,7 +68,8 @@ global_stmt: 'global' NAME (',' NAME)*
 nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
 assert_stmt: 'assert' test [',' test]
 
-compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
+async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
 if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
 while_stmt: 'while' test ':' suite ['else' ':' suite]
 for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
@@ -99,7 +103,8 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)*
 arith_expr: term (('+'|'-') term)*
 term: factor (('*'|'@'|'/'|'%'|'//') factor)*
 factor: ('+'|'-'|'~') factor | power
-power: atom trailer* ['**' factor]
+power: atom_expr ['**' factor]
+atom_expr: [AWAIT] atom trailer*
 atom: ('(' [yield_expr|testlist_comp] ')' |
        '[' [testlist_comp] ']' |
        '{' [dictorsetmaker] '}' |

Include/Python-ast.h

@@ -63,12 +63,13 @@ struct _mod {
     } v;
 };
 
-enum _stmt_kind {FunctionDef_kind=1, ClassDef_kind=2, Return_kind=3,
+enum _stmt_kind {FunctionDef_kind=1, AsyncFunctionDef_kind=2, ClassDef_kind=3,
-                  Delete_kind=4, Assign_kind=5, AugAssign_kind=6, For_kind=7,
+                  Return_kind=4, Delete_kind=5, Assign_kind=6,
-                  While_kind=8, If_kind=9, With_kind=10, Raise_kind=11,
+                  AugAssign_kind=7, For_kind=8, AsyncFor_kind=9, While_kind=10,
-                  Try_kind=12, Assert_kind=13, Import_kind=14,
+                  If_kind=11, With_kind=12, AsyncWith_kind=13, Raise_kind=14,
-                  ImportFrom_kind=15, Global_kind=16, Nonlocal_kind=17,
+                  Try_kind=15, Assert_kind=16, Import_kind=17,
-                  Expr_kind=18, Pass_kind=19, Break_kind=20, Continue_kind=21};
+                  ImportFrom_kind=18, Global_kind=19, Nonlocal_kind=20,
+                  Expr_kind=21, Pass_kind=22, Break_kind=23, Continue_kind=24};
 struct _stmt {
     enum _stmt_kind kind;
     union {
@@ -80,6 +81,14 @@ struct _stmt {
             expr_ty returns;
         } FunctionDef;
         
+        struct {
+            identifier name;
+            arguments_ty args;
+            asdl_seq *body;
+            asdl_seq *decorator_list;
+            expr_ty returns;
+        } AsyncFunctionDef;
+        
         struct {
             identifier name;
             asdl_seq *bases;
@@ -114,6 +123,13 @@ struct _stmt {
             asdl_seq *orelse;
         } For;
         
+        struct {
+            expr_ty target;
+            expr_ty iter;
+            asdl_seq *body;
+            asdl_seq *orelse;
+        } AsyncFor;
+        
         struct {
             expr_ty test;
             asdl_seq *body;
@@ -131,6 +147,11 @@ struct _stmt {
             asdl_seq *body;
         } With;
         
+        struct {
+            asdl_seq *items;
+            asdl_seq *body;
+        } AsyncWith;
+        
         struct {
             expr_ty exc;
             expr_ty cause;
@@ -178,11 +199,11 @@ struct _stmt {
 enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
                   IfExp_kind=5, Dict_kind=6, Set_kind=7, ListComp_kind=8,
                   SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11,
-                  Yield_kind=12, YieldFrom_kind=13, Compare_kind=14,
+                  Await_kind=12, Yield_kind=13, YieldFrom_kind=14,
-                  Call_kind=15, Num_kind=16, Str_kind=17, Bytes_kind=18,
+                  Compare_kind=15, Call_kind=16, Num_kind=17, Str_kind=18,
-                  NameConstant_kind=19, Ellipsis_kind=20, Attribute_kind=21,
+                  Bytes_kind=19, NameConstant_kind=20, Ellipsis_kind=21,
-                  Subscript_kind=22, Starred_kind=23, Name_kind=24,
+                  Attribute_kind=22, Subscript_kind=23, Starred_kind=24,
-                  List_kind=25, Tuple_kind=26};
+                  Name_kind=25, List_kind=26, Tuple_kind=27};
 struct _expr {
     enum _expr_kind kind;
     union {
@@ -243,6 +264,10 @@ struct _expr {
             asdl_seq *generators;
         } GeneratorExp;
         
+        struct {
+            expr_ty value;
+        } Await;
+        
         struct {
             expr_ty value;
         } Yield;
@@ -402,6 +427,10 @@ mod_ty _Py_Suite(asdl_seq * body, PyArena *arena);
 stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
                         asdl_seq * decorator_list, expr_ty returns, int lineno,
                         int col_offset, PyArena *arena);
+#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
+stmt_ty _Py_AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq *
+                             body, asdl_seq * decorator_list, expr_ty returns,
+                             int lineno, int col_offset, PyArena *arena);
 #define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7)
 stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords,
                      asdl_seq * body, asdl_seq * decorator_list, int lineno,
@@ -420,6 +449,9 @@ stmt_ty _Py_AugAssign(expr_ty target, operator_ty op, expr_ty value, int
 #define For(a0, a1, a2, a3, a4, a5, a6) _Py_For(a0, a1, a2, a3, a4, a5, a6)
 stmt_ty _Py_For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq *
                 orelse, int lineno, int col_offset, PyArena *arena);
+#define AsyncFor(a0, a1, a2, a3, a4, a5, a6) _Py_AsyncFor(a0, a1, a2, a3, a4, a5, a6)
+stmt_ty _Py_AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq *
+                     orelse, int lineno, int col_offset, PyArena *arena);
 #define While(a0, a1, a2, a3, a4, a5) _Py_While(a0, a1, a2, a3, a4, a5)
 stmt_ty _Py_While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
                   int col_offset, PyArena *arena);
@@ -429,6 +461,9 @@ stmt_ty _Py_If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
 #define With(a0, a1, a2, a3, a4) _Py_With(a0, a1, a2, a3, a4)
 stmt_ty _Py_With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset,
                  PyArena *arena);
+#define AsyncWith(a0, a1, a2, a3, a4) _Py_AsyncWith(a0, a1, a2, a3, a4)
+stmt_ty _Py_AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int
+                      col_offset, PyArena *arena);
 #define Raise(a0, a1, a2, a3, a4) _Py_Raise(a0, a1, a2, a3, a4)
 stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset,
                   PyArena *arena);
@@ -491,6 +526,8 @@ expr_ty _Py_DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int
 #define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4)
 expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
                          col_offset, PyArena *arena);
+#define Await(a0, a1, a2, a3) _Py_Await(a0, a1, a2, a3)
+expr_ty _Py_Await(expr_ty value, int lineno, int col_offset, PyArena *arena);
 #define Yield(a0, a1, a2, a3) _Py_Yield(a0, a1, a2, a3)
 expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, PyArena *arena);
 #define YieldFrom(a0, a1, a2, a3) _Py_YieldFrom(a0, a1, a2, a3)

Include/ceval.h

@@ -23,6 +23,8 @@ PyAPI_FUNC(PyObject *) PyEval_CallMethod(PyObject *obj,
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *);
 PyAPI_FUNC(void) PyEval_SetTrace(Py_tracefunc, PyObject *);
+PyAPI_FUNC(void) PyEval_SetCoroutineWrapper(PyObject *wrapper);
+PyAPI_FUNC(PyObject *) PyEval_GetCoroutineWrapper();
 #endif
 
 struct _frame; /* Avoid including frameobject.h */

Include/code.h

@@ -51,6 +51,11 @@ typedef struct {
 */
 #define CO_NOFREE       0x0040
 
+/* The CO_COROUTINE flag is set for coroutine functions (defined with
+   ``async def`` keywords) */
+#define CO_COROUTINE            0x0080
+#define CO_ITERABLE_COROUTINE   0x0100
+
 /* These are no longer used. */
 #if 0
 #define CO_GENERATOR_ALLOWED    0x1000

Include/genobject.h

@@ -38,6 +38,12 @@ PyAPI_DATA(PyTypeObject) PyGen_Type;
 #define PyGen_Check(op) PyObject_TypeCheck(op, &PyGen_Type)
 #define PyGen_CheckExact(op) (Py_TYPE(op) == &PyGen_Type)
 
+#define PyGen_CheckCoroutineExact(op) (PyGen_CheckExact(op) && \
+                                       (((PyCodeObject*) \
+                                           ((PyGenObject*)op)->gi_code) \
+                                         ->co_flags & (CO_ITERABLE_COROUTINE | \
+                                                       CO_COROUTINE)))
+
 PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *);
 PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(struct _frame *,
     PyObject *name, PyObject *qualname);
@@ -46,6 +52,7 @@ PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **);
 PyObject *_PyGen_Send(PyGenObject *, PyObject *);
 PyAPI_FUNC(void) _PyGen_Finalize(PyObject *self);
 
+PyObject *_PyGen_GetAwaitableIter(PyObject *o);
 
 #ifdef __cplusplus
 }

Include/graminit.h

@@ -6,79 +6,82 @@
 #define decorator 259
 #define decorators 260
 #define decorated 261
-#define funcdef 262
+#define async_funcdef 262
-#define parameters 263
+#define funcdef 263
-#define typedargslist 264
+#define parameters 264
-#define tfpdef 265
+#define typedargslist 265
-#define varargslist 266
+#define tfpdef 266
-#define vfpdef 267
+#define varargslist 267
-#define stmt 268
+#define vfpdef 268
-#define simple_stmt 269
+#define stmt 269
-#define small_stmt 270
+#define simple_stmt 270
-#define expr_stmt 271
+#define small_stmt 271
-#define testlist_star_expr 272
+#define expr_stmt 272
-#define augassign 273
+#define testlist_star_expr 273
-#define del_stmt 274
+#define augassign 274
-#define pass_stmt 275
+#define del_stmt 275
-#define flow_stmt 276
+#define pass_stmt 276
-#define break_stmt 277
+#define flow_stmt 277
-#define continue_stmt 278
+#define break_stmt 278
-#define return_stmt 279
+#define continue_stmt 279
-#define yield_stmt 280
+#define return_stmt 280
-#define raise_stmt 281
+#define yield_stmt 281
-#define import_stmt 282
+#define raise_stmt 282
-#define import_name 283
+#define import_stmt 283
-#define import_from 284
+#define import_name 284
-#define import_as_name 285
+#define import_from 285
-#define dotted_as_name 286
+#define import_as_name 286
-#define import_as_names 287
+#define dotted_as_name 287
-#define dotted_as_names 288
+#define import_as_names 288
-#define dotted_name 289
+#define dotted_as_names 289
-#define global_stmt 290
+#define dotted_name 290
-#define nonlocal_stmt 291
+#define global_stmt 291
-#define assert_stmt 292
+#define nonlocal_stmt 292
-#define compound_stmt 293
+#define assert_stmt 293
-#define if_stmt 294
+#define compound_stmt 294
-#define while_stmt 295
+#define async_stmt 295
-#define for_stmt 296
+#define if_stmt 296
-#define try_stmt 297
+#define while_stmt 297
-#define with_stmt 298
+#define for_stmt 298
-#define with_item 299
+#define try_stmt 299
-#define except_clause 300
+#define with_stmt 300
-#define suite 301
+#define with_item 301
-#define test 302
+#define except_clause 302
-#define test_nocond 303
+#define suite 303
-#define lambdef 304
+#define test 304
-#define lambdef_nocond 305
+#define test_nocond 305
-#define or_test 306
+#define lambdef 306
-#define and_test 307
+#define lambdef_nocond 307
-#define not_test 308
+#define or_test 308
-#define comparison 309
+#define and_test 309
-#define comp_op 310
+#define not_test 310
-#define star_expr 311
+#define comparison 311
-#define expr 312
+#define comp_op 312
-#define xor_expr 313
+#define star_expr 313
-#define and_expr 314
+#define expr 314
-#define shift_expr 315
+#define xor_expr 315
-#define arith_expr 316
+#define and_expr 316
-#define term 317
+#define shift_expr 317
-#define factor 318
+#define arith_expr 318
-#define power 319
+#define term 319
-#define atom 320
+#define factor 320
-#define testlist_comp 321
+#define power 321
-#define trailer 322
+#define atom_expr 322
-#define subscriptlist 323
+#define atom 323
-#define subscript 324
+#define testlist_comp 324
-#define sliceop 325
+#define trailer 325
-#define exprlist 326
+#define subscriptlist 326
-#define testlist 327
+#define subscript 327
-#define dictorsetmaker 328
+#define sliceop 328
-#define classdef 329
+#define exprlist 329
-#define arglist 330
+#define testlist 330
-#define argument 331
+#define dictorsetmaker 331
-#define comp_iter 332
+#define classdef 332
-#define comp_for 333
+#define arglist 333
-#define comp_if 334
+#define argument 334
-#define encoding_decl 335
+#define comp_iter 335
-#define yield_expr 336
+#define comp_for 336
-#define yield_arg 337
+#define comp_if 337
+#define encoding_decl 338
+#define yield_expr 339
+#define yield_arg 340

Include/object.h

@@ -173,6 +173,9 @@ typedef PyObject *(*ssizessizeargfunc)(PyObject *, Py_ssize_t, Py_ssize_t);
 typedef int(*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *);
 typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *);
 typedef int(*objobjargproc)(PyObject *, PyObject *, PyObject *);
+typedef PyObject *(*getawaitablefunc) (PyObject *);
+typedef PyObject *(*getaiterfunc) (PyObject *);
+typedef PyObject *(*aiternextfunc) (PyObject *);
 
 #ifndef Py_LIMITED_API
 /* buffer interface */
@@ -301,6 +304,11 @@ typedef struct {
     objobjargproc mp_ass_subscript;
 } PyMappingMethods;
 
+typedef struct {
+    getawaitablefunc am_await;
+    getaiterfunc am_aiter;
+    aiternextfunc am_anext;
+} PyAsyncMethods;
 
 typedef struct {
      getbufferproc bf_getbuffer;
@@ -346,7 +354,7 @@ typedef struct _typeobject {
     printfunc tp_print;
     getattrfunc tp_getattr;
     setattrfunc tp_setattr;
-    void *tp_reserved; /* formerly known as tp_compare */
+    PyAsyncMethods *tp_as_async; /* formerly known as tp_compare or tp_reserved */
     reprfunc tp_repr;
 
     /* Method suites for standard classes */
@@ -453,6 +461,7 @@ typedef struct _heaptypeobject {
     /* Note: there's a dependency on the order of these members
        in slotptr() in typeobject.c . */
     PyTypeObject ht_type;
+    PyAsyncMethods as_async;
     PyNumberMethods as_number;
     PyMappingMethods as_mapping;
     PySequenceMethods as_sequence; /* as_sequence comes after as_mapping,

Include/opcode.h

@@ -7,68 +7,73 @@ extern "C" {
 
 
     /* Instruction opcodes for compiled code */
-#define POP_TOP             	1  
+#define POP_TOP             	1
-#define ROT_TWO             	2  
+#define ROT_TWO             	2
-#define ROT_THREE           	3  
+#define ROT_THREE           	3
-#define DUP_TOP             	4  
+#define DUP_TOP             	4
-#define DUP_TOP_TWO         	5  
+#define DUP_TOP_TWO         	5
-#define NOP                 	9  
+#define NOP                 	9
-#define UNARY_POSITIVE      	10 
+#define UNARY_POSITIVE      	10
-#define UNARY_NEGATIVE      	11 
+#define UNARY_NEGATIVE      	11
-#define UNARY_NOT           	12 
+#define UNARY_NOT           	12
-#define UNARY_INVERT        	15 
+#define UNARY_INVERT        	15
-#define BINARY_MATRIX_MULTIPLY	16 
+#define BINARY_MATRIX_MULTIPLY	16
-#define INPLACE_MATRIX_MULTIPLY	17 
+#define INPLACE_MATRIX_MULTIPLY	17
-#define BINARY_POWER        	19 
+#define BINARY_POWER        	19
-#define BINARY_MULTIPLY     	20 
+#define BINARY_MULTIPLY     	20
-#define BINARY_MODULO       	22 
+#define BINARY_MODULO       	22
-#define BINARY_ADD          	23 
+#define BINARY_ADD          	23
-#define BINARY_SUBTRACT     	24 
+#define BINARY_SUBTRACT     	24
-#define BINARY_SUBSCR       	25 
+#define BINARY_SUBSCR       	25
-#define BINARY_FLOOR_DIVIDE 	26 
+#define BINARY_FLOOR_DIVIDE 	26
-#define BINARY_TRUE_DIVIDE  	27 
+#define BINARY_TRUE_DIVIDE  	27
-#define INPLACE_FLOOR_DIVIDE	28 
+#define INPLACE_FLOOR_DIVIDE	28
-#define INPLACE_TRUE_DIVIDE 	29 
+#define INPLACE_TRUE_DIVIDE 	29
-#define STORE_MAP           	54 
+#define GET_AITER           	50
-#define INPLACE_ADD         	55 
+#define GET_ANEXT           	51
-#define INPLACE_SUBTRACT    	56 
+#define BEFORE_ASYNC_WITH   	52
-#define INPLACE_MULTIPLY    	57 
+#define STORE_MAP           	54
-#define INPLACE_MODULO      	59 
+#define INPLACE_ADD         	55
-#define STORE_SUBSCR        	60 
+#define INPLACE_SUBTRACT    	56
-#define DELETE_SUBSCR       	61 
+#define INPLACE_MULTIPLY    	57
-#define BINARY_LSHIFT       	62 
+#define INPLACE_MODULO      	59
-#define BINARY_RSHIFT       	63 
+#define STORE_SUBSCR        	60
-#define BINARY_AND          	64 
+#define DELETE_SUBSCR       	61
-#define BINARY_XOR          	65 
+#define BINARY_LSHIFT       	62
-#define BINARY_OR           	66 
+#define BINARY_RSHIFT       	63
-#define INPLACE_POWER       	67 
+#define BINARY_AND          	64
-#define GET_ITER            	68 
+#define BINARY_XOR          	65
-#define PRINT_EXPR          	70 
+#define BINARY_OR           	66
-#define LOAD_BUILD_CLASS    	71 
+#define INPLACE_POWER       	67
-#define YIELD_FROM          	72 
+#define GET_ITER            	68
-#define INPLACE_LSHIFT      	75 
+#define PRINT_EXPR          	70
-#define INPLACE_RSHIFT      	76 
+#define LOAD_BUILD_CLASS    	71
-#define INPLACE_AND         	77 
+#define YIELD_FROM          	72
-#define INPLACE_XOR         	78 
+#define GET_AWAITABLE       	73
-#define INPLACE_OR          	79 
+#define INPLACE_LSHIFT      	75
-#define BREAK_LOOP          	80 
+#define INPLACE_RSHIFT      	76
-#define WITH_CLEANUP        	81 
+#define INPLACE_AND         	77
-#define RETURN_VALUE        	83 
+#define INPLACE_XOR         	78
-#define IMPORT_STAR         	84 
+#define INPLACE_OR          	79
-#define YIELD_VALUE         	86 
+#define BREAK_LOOP          	80
-#define POP_BLOCK           	87 
+#define WITH_CLEANUP_START  	81
-#define END_FINALLY         	88 
+#define WITH_CLEANUP_FINISH   	82
-#define POP_EXCEPT          	89 
+#define RETURN_VALUE        	83
-#define HAVE_ARGUMENT       	90 
+#define IMPORT_STAR         	84
-#define STORE_NAME          	90 
+#define YIELD_VALUE         	86
-#define DELETE_NAME         	91 
+#define POP_BLOCK           	87
-#define UNPACK_SEQUENCE     	92 
+#define END_FINALLY         	88
-#define FOR_ITER            	93 
+#define POP_EXCEPT          	89
-#define UNPACK_EX           	94 
+#define HAVE_ARGUMENT       	90
-#define STORE_ATTR          	95 
+#define STORE_NAME          	90
-#define DELETE_ATTR         	96 
+#define DELETE_NAME         	91
-#define STORE_GLOBAL        	97 
+#define UNPACK_SEQUENCE     	92
-#define DELETE_GLOBAL       	98 
+#define FOR_ITER            	93
+#define UNPACK_EX           	94
+#define STORE_ATTR          	95
+#define DELETE_ATTR         	96
+#define STORE_GLOBAL        	97
+#define DELETE_GLOBAL       	98
 #define LOAD_CONST          	100
 #define LOAD_NAME           	101
 #define BUILD_TUPLE         	102
@@ -116,6 +121,7 @@ extern "C" {
 #define BUILD_MAP_UNPACK_WITH_CALL	151
 #define BUILD_TUPLE_UNPACK  	152
 #define BUILD_SET_UNPACK    	153
+#define SETUP_ASYNC_WITH    	154
 
 /* EXCEPT_HANDLER is a special, implicit block type which is created when
    entering an except handler. It is not an opcode but we define it here

Include/pyerrors.h

@@ -147,6 +147,7 @@ PyAPI_FUNC(void) _PyErr_ChainExceptions(PyObject *, PyObject *, PyObject *);
 
 PyAPI_DATA(PyObject *) PyExc_BaseException;
 PyAPI_DATA(PyObject *) PyExc_Exception;
+PyAPI_DATA(PyObject *) PyExc_StopAsyncIteration;
 PyAPI_DATA(PyObject *) PyExc_StopIteration;
 PyAPI_DATA(PyObject *) PyExc_GeneratorExit;
 PyAPI_DATA(PyObject *) PyExc_ArithmeticError;

Include/pystate.h

@@ -134,6 +134,8 @@ typedef struct _ts {
     void (*on_delete)(void *);
     void *on_delete_data;
 
+    PyObject *coroutine_wrapper;
+
     /* XXX signal handlers should also be here */
 
 } PyThreadState;

Include/token.h

@@ -64,8 +64,10 @@ extern "C" {
 #define ELLIPSIS        52
 /* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */
 #define OP		53
-#define ERRORTOKEN	54
+#define AWAIT		54
-#define N_TOKENS	55
+#define ASYNC		55
+#define ERRORTOKEN	56
+#define N_TOKENS	57
 
 /* Special definitions for cooperation with parser */
 

Include/typeslots.h

@@ -76,3 +76,6 @@
 #define Py_tp_free 74
 #define Py_nb_matrix_multiply 75
 #define Py_nb_inplace_matrix_multiply 76
+#define Py_am_await 77
+#define Py_am_aiter 78
+#define Py_am_anext 79

Lib/_collections_abc.py

@@ -9,7 +9,8 @@ Unit tests are in test_collections.
 from abc import ABCMeta, abstractmethod
 import sys
 
-__all__ = ["Hashable", "Iterable", "Iterator", "Generator",
+__all__ = ["Awaitable", "Coroutine",
+           "Hashable", "Iterable", "Iterator", "Generator",
            "Sized", "Container", "Callable",
            "Set", "MutableSet",
            "Mapping", "MutableMapping",
@@ -74,6 +75,79 @@ class Hashable(metaclass=ABCMeta):
         return NotImplemented
 
 
+class _CoroutineMeta(ABCMeta):
+
+    def __instancecheck__(cls, instance):
+        # 0x80 = CO_COROUTINE
+        # 0x100 = CO_ITERABLE_COROUTINE
+        # We don't want to import 'inspect' module, as
+        # a dependency for 'collections.abc'.
+        CO_COROUTINES = 0x80 | 0x100
+
+        if (isinstance(instance, generator) and
+            instance.gi_code.co_flags & CO_COROUTINES):
+
+            return True
+
+        return super().__instancecheck__(instance)
+
+
+class Coroutine(metaclass=_CoroutineMeta):
+
+    __slots__ = ()
+
+    @abstractmethod
+    def send(self, value):
+        """Send a value into the coroutine.
+        Return next yielded value or raise StopIteration.
+        """
+        raise StopIteration
+
+    @abstractmethod
+    def throw(self, typ, val=None, tb=None):
+        """Raise an exception in the coroutine.
+        Return next yielded value or raise StopIteration.
+        """
+        if val is None:
+            if tb is None:
+                raise typ
+            val = typ()
+        if tb is not None:
+            val = val.with_traceback(tb)
+        raise val
+
+    def close(self):
+        """Raise GeneratorExit inside coroutine.
+        """
+        try:
+            self.throw(GeneratorExit)
+        except (GeneratorExit, StopIteration):
+            pass
+        else:
+            raise RuntimeError("coroutine ignored GeneratorExit")
+
+
+class Awaitable(metaclass=_CoroutineMeta):
+
+    __slots__ = ()
+
+    @abstractmethod
+    def __await__(self):
+        yield
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Awaitable:
+            for B in C.__mro__:
+                if "__await__" in B.__dict__:
+                    if B.__dict__["__await__"]:
+                        return True
+                    break
+        return NotImplemented
+
+Awaitable.register(Coroutine)
+
+
 class Iterable(metaclass=ABCMeta):
 
     __slots__ = ()

Lib/dis.py

@@ -84,6 +84,8 @@ COMPILER_FLAG_NAMES = {
     16: "NESTED",
     32: "GENERATOR",
     64: "NOFREE",
+   128: "COROUTINE",
+   256: "ITERABLE_COROUTINE",
 }
 
 def pretty_flags(flags):

Lib/inspect.py

@@ -33,6 +33,7 @@ __author__ = ('Ka-Ping Yee <ping@lfw.org>',
 
 import ast
 import dis
+import collections.abc
 import enum
 import importlib.machinery
 import itertools
@@ -174,7 +175,22 @@ def isgeneratorfunction(object):
 
     See help(isfunction) for attributes listing."""
     return bool((isfunction(object) or ismethod(object)) and
-                object.__code__.co_flags & CO_GENERATOR)
+                object.__code__.co_flags & CO_GENERATOR and
+                not object.__code__.co_flags & CO_COROUTINE)
+
+def iscoroutinefunction(object):
+    """Return true if the object is a coroutine function.
+
+    Coroutine functions are defined with "async def" syntax,
+    or generators decorated with "types.coroutine".
+    """
+    return bool((isfunction(object) or ismethod(object)) and
+                object.__code__.co_flags & (CO_ITERABLE_COROUTINE |
+                                            CO_COROUTINE))
+
+def isawaitable(object):
+    """Return true if the object can be used in "await" expression."""
+    return isinstance(object, collections.abc.Awaitable)
 
 def isgenerator(object):
     """Return true if the object is a generator.
@@ -191,7 +207,13 @@ def isgenerator(object):
         send            resumes the generator and "sends" a value that becomes
                         the result of the current yield-expression
         throw           used to raise an exception inside the generator"""
-    return isinstance(object, types.GeneratorType)
+    return (isinstance(object, types.GeneratorType) and
+            not object.gi_code.co_flags & CO_COROUTINE)
+
+def iscoroutine(object):
+    """Return true if the object is a coroutine."""
+    return (isinstance(object, types.GeneratorType) and
+            object.gi_code.co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))
 
 def istraceback(object):
     """Return true if the object is a traceback.

Lib/lib2to3/Grammar.txt

@@ -33,7 +33,8 @@ eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
 decorators: decorator+
-decorated: decorators (classdef | funcdef)
+decorated: decorators (classdef | funcdef | async_funcdef)
+async_funcdef: ASYNC funcdef
 funcdef: 'def' NAME parameters ['->' test] ':' suite
 parameters: '(' [typedargslist] ')'
 typedargslist: ((tfpdef ['=' test] ',')*
@@ -82,7 +83,8 @@ global_stmt: ('global' | 'nonlocal') NAME (',' NAME)*
 exec_stmt: 'exec' expr ['in' test [',' test]]
 assert_stmt: 'assert' test [',' test]
 
-compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
+async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
 if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
 while_stmt: 'while' test ':' suite ['else' ':' suite]
 for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
@@ -121,7 +123,7 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)*
 arith_expr: term (('+'|'-') term)*
 term: factor (('*'|'@'|'/'|'%'|'//') factor)*
 factor: ('+'|'-'|'~') factor | power
-power: atom trailer* ['**' factor]
+power: [AWAIT] atom trailer* ['**' factor]
 atom: ('(' [yield_expr|testlist_gexp] ')' |
        '[' [listmaker] ']' |
        '{' [dictsetmaker] '}' |

Lib/lib2to3/pgen2/token.py

@@ -62,8 +62,10 @@ OP = 52
 COMMENT = 53
 NL = 54
 RARROW = 55
-ERRORTOKEN = 56
+AWAIT = 56
-N_TOKENS = 57
+ASYNC = 57
+ERRORTOKEN = 58
+N_TOKENS = 59
 NT_OFFSET = 256
 #--end constants--
 

Lib/lib2to3/pgen2/tokenize.py

@@ -220,7 +220,7 @@ class Untokenizer:
         for tok in iterable:
             toknum, tokval = tok[:2]
 
-            if toknum in (NAME, NUMBER):
+            if toknum in (NAME, NUMBER, ASYNC, AWAIT):
                 tokval += ' '
 
             if toknum == INDENT:
@@ -366,6 +366,10 @@ def generate_tokens(readline):
     contline = None
     indents = [0]
 
+    # 'stashed' and 'ctx' are used for async/await parsing
+    stashed = None
+    ctx = [('sync', 0)]
+
     while 1:                                   # loop over lines in stream
         try:
             line = readline()
@@ -406,6 +410,10 @@ def generate_tokens(readline):
                 pos = pos + 1
             if pos == max: break
 
+            if stashed:
+                yield stashed
+                stashed = None
+
             if line[pos] in '#\r\n':           # skip comments or blank lines
                 if line[pos] == '#':
                     comment_token = line[pos:].rstrip('\r\n')
@@ -449,9 +457,15 @@ def generate_tokens(readline):
                     newline = NEWLINE
                     if parenlev > 0:
                         newline = NL
+                    if stashed:
+                        yield stashed
+                        stashed = None
                     yield (newline, token, spos, epos, line)
                 elif initial == '#':
                     assert not token.endswith("\n")
+                    if stashed:
+                        yield stashed
+                        stashed = None
                     yield (COMMENT, token, spos, epos, line)
                 elif token in triple_quoted:
                     endprog = endprogs[token]
@@ -459,6 +473,9 @@ def generate_tokens(readline):
                     if endmatch:                           # all on one line
                         pos = endmatch.end(0)
                         token = line[start:pos]
+                        if stashed:
+                            yield stashed
+                            stashed = None
                         yield (STRING, token, spos, (lnum, pos), line)
                     else:
                         strstart = (lnum, start)           # multiple lines
@@ -476,22 +493,64 @@ def generate_tokens(readline):
                         contline = line
                         break
                     else:                                  # ordinary string
+                        if stashed:
+                            yield stashed
+                            stashed = None
                         yield (STRING, token, spos, epos, line)
                 elif initial in namechars:                 # ordinary name
-                    yield (NAME, token, spos, epos, line)
+                    if token in ('async', 'await'):
+                        if ctx[-1][0] == 'async' and ctx[-1][1] < indents[-1]:
+                            yield (ASYNC if token == 'async' else AWAIT,
+                                   token, spos, epos, line)
+                            continue
+
+                    tok = (NAME, token, spos, epos, line)
+                    if token == 'async' and not stashed:
+                        stashed = tok
+                        continue
+
+                    if token == 'def':
+                        if (stashed
+                                and stashed[0] == NAME
+                                and stashed[1] == 'async'):
+
+                            ctx.append(('async', indents[-1]))
+
+                            yield (ASYNC, stashed[1],
+                                   stashed[2], stashed[3],
+                                   stashed[4])
+                            stashed = None
+                        else:
+                            ctx.append(('sync', indents[-1]))
+
+                    if stashed:
+                        yield stashed
+                        stashed = None
+
+                    yield tok
                 elif initial == '\\':                      # continued stmt
                     # This yield is new; needed for better idempotency:
+                    if stashed:
+                        yield stashed
+                        stashed = None
                     yield (NL, token, spos, (lnum, pos), line)
                     continued = 1
                 else:
                     if initial in '([{': parenlev = parenlev + 1
                     elif initial in ')]}': parenlev = parenlev - 1
+                    if stashed:
+                        yield stashed
+                        stashed = None
                     yield (OP, token, spos, epos, line)
             else:
                 yield (ERRORTOKEN, line[pos],
                            (lnum, pos), (lnum, pos+1), line)
                 pos = pos + 1
 
+    if stashed:
+        yield stashed
+        stashed = None
+
     for indent in indents[1:]:                 # pop remaining indent levels
         yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
     yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')

Lib/lib2to3/tests/test_parser.py

@@ -55,12 +55,42 @@ class TestMatrixMultiplication(GrammarTest):
 
 
 class TestYieldFrom(GrammarTest):
-    def test_matrix_multiplication_operator(self):
+    def test_yield_from(self):
         self.validate("yield from x")
         self.validate("(yield from x) + y")
         self.invalid_syntax("yield from")
 
 
+class TestAsyncAwait(GrammarTest):
+    def test_await_expr(self):
+        self.validate("""async def foo():
+                             await x
+                      """)
+
+        self.invalid_syntax("await x")
+        self.invalid_syntax("""def foo():
+                                   await x""")
+
+    def test_async_var(self):
+        self.validate("""async = 1""")
+        self.validate("""await = 1""")
+        self.validate("""def async(): pass""")
+
+    def test_async_with(self):
+        self.validate("""async def foo():
+                             async for a in b: pass""")
+
+        self.invalid_syntax("""def foo():
+                                   async for a in b: pass""")
+
+    def test_async_for(self):
+        self.validate("""async def foo():
+                             async with a: pass""")
+
+        self.invalid_syntax("""def foo():
+                                   async with a: pass""")
+
+
 class TestRaiseChanges(GrammarTest):
     def test_2x_style_1(self):
         self.validate("raise")

Lib/opcode.py

@@ -85,6 +85,10 @@ def_op('BINARY_TRUE_DIVIDE', 27)
 def_op('INPLACE_FLOOR_DIVIDE', 28)
 def_op('INPLACE_TRUE_DIVIDE', 29)
 
+def_op('GET_AITER', 50)
+def_op('GET_ANEXT', 51)
+def_op('BEFORE_ASYNC_WITH', 52)
+
 def_op('STORE_MAP', 54)
 def_op('INPLACE_ADD', 55)
 def_op('INPLACE_SUBTRACT', 56)
@@ -104,6 +108,7 @@ def_op('GET_ITER', 68)
 def_op('PRINT_EXPR', 70)
 def_op('LOAD_BUILD_CLASS', 71)
 def_op('YIELD_FROM', 72)
+def_op('GET_AWAITABLE', 73)
 
 def_op('INPLACE_LSHIFT', 75)
 def_op('INPLACE_RSHIFT', 76)
@@ -111,7 +116,8 @@ def_op('INPLACE_AND', 77)
 def_op('INPLACE_XOR', 78)
 def_op('INPLACE_OR', 79)
 def_op('BREAK_LOOP', 80)
-def_op('WITH_CLEANUP', 81)
+def_op('WITH_CLEANUP_START', 81)
+def_op('WITH_CLEANUP_FINISH', 82)
 
 def_op('RETURN_VALUE', 83)
 def_op('IMPORT_STAR', 84)
@@ -197,6 +203,8 @@ def_op('MAP_ADD', 147)
 def_op('LOAD_CLASSDEREF', 148)
 hasfree.append(148)
 
+jrel_op('SETUP_ASYNC_WITH', 154)
+
 def_op('EXTENDED_ARG', 144)
 EXTENDED_ARG = 144
 

Lib/test/badsyntax_async1.py

@@ -0,0 +1,3 @@
+async def foo():
+    def foo(a=await something()):
+        pass

Lib/test/badsyntax_async2.py

@@ -0,0 +1,3 @@
+async def foo():
+    def foo(a:await something()):
+        pass

Lib/test/badsyntax_async3.py

@@ -0,0 +1,2 @@
+async def foo():
+    [i async for i in els]

Lib/test/badsyntax_async4.py

@@ -0,0 +1,2 @@
+async def foo():
+    async def foo(): await something()

Lib/test/badsyntax_async5.py

@@ -0,0 +1,2 @@
+def foo():
+    await something()

Lib/test/badsyntax_async6.py

@@ -0,0 +1,2 @@
+async def foo():
+    yield

Lib/test/badsyntax_async7.py

@@ -0,0 +1,2 @@
+async def foo():
+    yield from []

Lib/test/badsyntax_async8.py

@@ -0,0 +1,2 @@
+async def foo():
+    await await fut

Lib/test/badsyntax_async9.py

@@ -0,0 +1,2 @@
+async def foo():
+    await

Lib/test/exception_hierarchy.txt

@@ -4,6 +4,7 @@ BaseException
  +-- GeneratorExit
  +-- Exception
       +-- StopIteration
+      +-- StopAsyncIteration
       +-- ArithmeticError
       |    +-- FloatingPointError
       |    +-- OverflowError

Lib/test/test_ast.py

@@ -106,6 +106,12 @@ exec_tests = [
     "{r for l in x if g}",
     # setcomp with naked tuple
     "{r for l,m in x}",
+    # AsyncFunctionDef
+    "async def f():\n await something()",
+    # AsyncFor
+    "async def f():\n async for e in i: 1\n else: 2",
+    # AsyncWith
+    "async def f():\n async with a as b: 1",
 ]
 
 # These are compiled through "single"
@@ -974,6 +980,9 @@ exec_results = [
 ('Module', [('Expr', (1, 0), ('DictComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [])]))]),
 ('Module', [('Expr', (1, 0), ('SetComp', (1, 1), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))])]))]),
 ('Module', [('Expr', (1, 0), ('SetComp', (1, 1), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [])]))]),
+('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('Await', (2, 1), ('Call', (2, 7), ('Name', (2, 7), 'something', ('Load',)), [], [])))], [], None)]),
+('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('AsyncFor', (2, 7), ('Name', (2, 11), 'e', ('Store',)), ('Name', (2, 16), 'i', ('Load',)), [('Expr', (2, 19), ('Num', (2, 19), 1))], [('Expr', (3, 7), ('Num', (3, 7), 2))])], [], None)]),
+('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('AsyncWith', (2, 7), [('withitem', ('Name', (2, 12), 'a', ('Load',)), ('Name', (2, 17), 'b', ('Store',)))], [('Expr', (2, 20), ('Num', (2, 20), 1))])], [], None)]),
 ]
 single_results = [
 ('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),

Lib/test/test_collections.py

@@ -11,9 +11,11 @@ from random import randrange, shuffle
 import keyword
 import re
 import sys
+import types
 from collections import UserDict
 from collections import ChainMap
 from collections import deque
+from collections.abc import Awaitable, Coroutine
 from collections.abc import Hashable, Iterable, Iterator, Generator
 from collections.abc import Sized, Container, Callable
 from collections.abc import Set, MutableSet
@@ -446,6 +448,84 @@ class ABCTestCase(unittest.TestCase):
 
 class TestOneTrickPonyABCs(ABCTestCase):
 
+    def test_Awaitable(self):
+        def gen():
+            yield
+
+        @types.coroutine
+        def coro():
+            yield
+
+        async def new_coro():
+            pass
+
+        class Bar:
+            def __await__(self):
+                yield
+
+        class MinimalCoro(Coroutine):
+            def send(self, value):
+                return value
+            def throw(self, typ, val=None, tb=None):
+                super().throw(typ, val, tb)
+
+        non_samples = [None, int(), gen(), object()]
+        for x in non_samples:
+            self.assertNotIsInstance(x, Awaitable)
+            self.assertFalse(issubclass(type(x), Awaitable), repr(type(x)))
+
+        samples = [Bar(), MinimalCoro()]
+        for x in samples:
+            self.assertIsInstance(x, Awaitable)
+            self.assertTrue(issubclass(type(x), Awaitable))
+
+        c = coro()
+        self.assertIsInstance(c, Awaitable)
+        c.close() # awoid RuntimeWarning that coro() was not awaited
+
+        c = new_coro()
+        self.assertIsInstance(c, Awaitable)
+        c.close() # awoid RuntimeWarning that coro() was not awaited
+
+    def test_Coroutine(self):
+        def gen():
+            yield
+
+        @types.coroutine
+        def coro():
+            yield
+
+        async def new_coro():
+            pass
+
+        class Bar:
+            def __await__(self):
+                yield
+
+        class MinimalCoro(Coroutine):
+            def send(self, value):
+                return value
+            def throw(self, typ, val=None, tb=None):
+                super().throw(typ, val, tb)
+
+        non_samples = [None, int(), gen(), object(), Bar()]
+        for x in non_samples:
+            self.assertNotIsInstance(x, Coroutine)
+            self.assertFalse(issubclass(type(x), Coroutine), repr(type(x)))
+
+        samples = [MinimalCoro()]
+        for x in samples:
+            self.assertIsInstance(x, Awaitable)
+            self.assertTrue(issubclass(type(x), Awaitable))
+
+        c = coro()
+        self.assertIsInstance(c, Coroutine)
+        c.close() # awoid RuntimeWarning that coro() was not awaited
+
+        c = new_coro()
+        self.assertIsInstance(c, Coroutine)
+        c.close() # awoid RuntimeWarning that coro() was not awaited
+
     def test_Hashable(self):
         # Check some non-hashables
         non_samples = [bytearray(), list(), set(), dict()]

Lib/test/test_coroutines.py

@@ -0,0 +1,862 @@
+import contextlib
+import gc
+import sys
+import types
+import unittest
+import warnings
+from test import support
+
+
+class AsyncYieldFrom:
+    def __init__(self, obj):
+        self.obj = obj
+
+    def __await__(self):
+        yield from self.obj
+
+
+class AsyncYield:
+    def __init__(self, value):
+        self.value = value
+
+    def __await__(self):
+        yield self.value
+
+
+def run_async(coro):
+    assert coro.__class__ is types.GeneratorType
+
+    buffer = []
+    result = None
+    while True:
+        try:
+            buffer.append(coro.send(None))
+        except StopIteration as ex:
+            result = ex.args[0] if ex.args else None
+            break
+    return buffer, result
+
+
+@contextlib.contextmanager
+def silence_coro_gc():
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        yield
+        support.gc_collect()
+
+
+class AsyncBadSyntaxTest(unittest.TestCase):
+
+    def test_badsyntax_1(self):
+        with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
+            import test.badsyntax_async1
+
+    def test_badsyntax_2(self):
+        with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
+            import test.badsyntax_async2
+
+    def test_badsyntax_3(self):
+        with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
+            import test.badsyntax_async3
+
+    def test_badsyntax_4(self):
+        with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
+            import test.badsyntax_async4
+
+    def test_badsyntax_5(self):
+        with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
+            import test.badsyntax_async5
+
+    def test_badsyntax_6(self):
+        with self.assertRaisesRegex(
+            SyntaxError, "'yield' inside async function"):
+
+            import test.badsyntax_async6
+
+    def test_badsyntax_7(self):
+        with self.assertRaisesRegex(
+            SyntaxError, "'yield from' inside async function"):
+
+            import test.badsyntax_async7
+
+    def test_badsyntax_8(self):
+        with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
+            import test.badsyntax_async8
+
+    def test_badsyntax_9(self):
+        with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
+            import test.badsyntax_async9
+
+
+class CoroutineTest(unittest.TestCase):
+
+    def test_gen_1(self):
+        def gen(): yield
+        self.assertFalse(hasattr(gen, '__await__'))
+
+    def test_func_1(self):
+        async def foo():
+            return 10
+
+        f = foo()
+        self.assertIsInstance(f, types.GeneratorType)
+        self.assertTrue(bool(foo.__code__.co_flags & 0x80))
+        self.assertTrue(bool(foo.__code__.co_flags & 0x20))
+        self.assertTrue(bool(f.gi_code.co_flags & 0x80))
+        self.assertTrue(bool(f.gi_code.co_flags & 0x20))
+        self.assertEqual(run_async(f), ([], 10))
+
+        def bar(): pass
+        self.assertFalse(bool(bar.__code__.co_flags & 0x80))
+
+    def test_func_2(self):
+        async def foo():
+            raise StopIteration
+
+        with self.assertRaisesRegex(
+                RuntimeError, "generator raised StopIteration"):
+
+            run_async(foo())
+
+    def test_func_3(self):
+        async def foo():
+            raise StopIteration
+
+        with silence_coro_gc():
+            self.assertRegex(repr(foo()), '^<coroutine object.* at 0x.*>$')
+
+    def test_func_4(self):
+        async def foo():
+            raise StopIteration
+
+        check = lambda: self.assertRaisesRegex(
+            TypeError, "coroutine-objects do not support iteration")
+
+        with check():
+            list(foo())
+
+        with check():
+            tuple(foo())
+
+        with check():
+            sum(foo())
+
+        with check():
+            iter(foo())
+
+        with check():
+            next(foo())
+
+        with silence_coro_gc(), check():
+            for i in foo():
+                pass
+
+        with silence_coro_gc(), check():
+            [i for i in foo()]
+
+    def test_func_5(self):
+        @types.coroutine
+        def bar():
+            yield 1
+
+        async def foo():
+            await bar()
+
+        check = lambda: self.assertRaisesRegex(
+            TypeError, "coroutine-objects do not support iteration")
+
+        with check():
+            for el in foo(): pass
+
+        # the following should pass without an error
+        for el in bar():
+            self.assertEqual(el, 1)
+        self.assertEqual([el for el in bar()], [1])
+        self.assertEqual(tuple(bar()), (1,))
+        self.assertEqual(next(iter(bar())), 1)
+
+    def test_func_6(self):
+        @types.coroutine
+        def bar():
+            yield 1
+            yield 2
+
+        async def foo():
+            await bar()
+
+        f = foo()
+        self.assertEquals(f.send(None), 1)
+        self.assertEquals(f.send(None), 2)
+        with self.assertRaises(StopIteration):
+            f.send(None)
+
+    def test_func_7(self):
+        async def bar():
+            return 10
+
+        def foo():
+            yield from bar()
+
+        with silence_coro_gc(), self.assertRaisesRegex(
+            TypeError,
+            "cannot 'yield from' a coroutine object from a generator"):
+
+            list(foo())
+
+    def test_func_8(self):
+        @types.coroutine
+        def bar():
+            return (yield from foo())
+
+        async def foo():
+            return 'spam'
+
+        self.assertEqual(run_async(bar()), ([], 'spam') )
+
+    def test_func_9(self):
+        async def foo(): pass
+
+        with self.assertWarnsRegex(
+            RuntimeWarning, "coroutine '.*test_func_9.*foo' was never awaited"):
+
+            foo()
+            support.gc_collect()
+
+    def test_await_1(self):
+
+        async def foo():
+            await 1
+        with self.assertRaisesRegex(TypeError, "object int can.t.*await"):
+            run_async(foo())
+
+    def test_await_2(self):
+        async def foo():
+            await []
+        with self.assertRaisesRegex(TypeError, "object list can.t.*await"):
+            run_async(foo())
+
+    def test_await_3(self):
+        async def foo():
+            await AsyncYieldFrom([1, 2, 3])
+
+        self.assertEqual(run_async(foo()), ([1, 2, 3], None))
+
+    def test_await_4(self):
+        async def bar():
+            return 42
+
+        async def foo():
+            return await bar()
+
+        self.assertEqual(run_async(foo()), ([], 42))
+
+    def test_await_5(self):
+        class Awaitable:
+            def __await__(self):
+                return
+
+        async def foo():
+            return (await Awaitable())
+
+        with self.assertRaisesRegex(
+            TypeError, "__await__.*returned non-iterator of type"):
+
+            run_async(foo())
+
+    def test_await_6(self):
+        class Awaitable:
+            def __await__(self):
+                return iter([52])
+
+        async def foo():
+            return (await Awaitable())
+
+        self.assertEqual(run_async(foo()), ([52], None))
+
+    def test_await_7(self):
+        class Awaitable:
+            def __await__(self):
+                yield 42
+                return 100
+
+        async def foo():
+            return (await Awaitable())
+
+        self.assertEqual(run_async(foo()), ([42], 100))
+
+    def test_await_8(self):
+        class Awaitable:
+            pass
+
+        async def foo():
+            return (await Awaitable())
+
+        with self.assertRaisesRegex(
+            TypeError, "object Awaitable can't be used in 'await' expression"):
+
+            run_async(foo())
+
+    def test_await_9(self):
+        def wrap():
+            return bar
+
+        async def bar():
+            return 42
+
+        async def foo():
+            b = bar()
+
+            db = {'b':  lambda: wrap}
+
+            class DB:
+                b = wrap
+
+            return (await bar() + await wrap()() + await db['b']()()() +
+                    await bar() * 1000 + await DB.b()())
+
+        async def foo2():
+            return -await bar()
+
+        self.assertEqual(run_async(foo()), ([], 42168))
+        self.assertEqual(run_async(foo2()), ([], -42))
+
+    def test_await_10(self):
+        async def baz():
+            return 42
+
+        async def bar():
+            return baz()
+
+        async def foo():
+            return await (await bar())
+
+        self.assertEqual(run_async(foo()), ([], 42))
+
+    def test_await_11(self):
+        def ident(val):
+            return val
+
+        async def bar():
+            return 'spam'
+
+        async def foo():
+            return ident(val=await bar())
+
+        async def foo2():
+            return await bar(), 'ham'
+
+        self.assertEqual(run_async(foo2()), ([], ('spam', 'ham')))
+
+    def test_await_12(self):
+        async def coro():
+            return 'spam'
+
+        class Awaitable:
+            def __await__(self):
+                return coro()
+
+        async def foo():
+            return await Awaitable()
+
+        with self.assertRaisesRegex(
+            TypeError, "__await__\(\) returned a coroutine"):
+
+            run_async(foo())
+
+    def test_await_13(self):
+        class Awaitable:
+            def __await__(self):
+                return self
+
+        async def foo():
+            return await Awaitable()
+
+        with self.assertRaisesRegex(
+            TypeError, "__await__.*returned non-iterator of type"):
+
+            run_async(foo())
+
+    def test_with_1(self):
+        class Manager:
+            def __init__(self, name):
+                self.name = name
+
+            async def __aenter__(self):
+                await AsyncYieldFrom(['enter-1-' + self.name,
+                                      'enter-2-' + self.name])
+                return self
+
+            async def __aexit__(self, *args):
+                await AsyncYieldFrom(['exit-1-' + self.name,
+                                      'exit-2-' + self.name])
+
+                if self.name == 'B':
+                    return True
+
+
+        async def foo():
+            async with Manager("A") as a, Manager("B") as b:
+                await AsyncYieldFrom([('managers', a.name, b.name)])
+                1/0
+
+        f = foo()
+        result, _ = run_async(f)
+
+        self.assertEqual(
+            result, ['enter-1-A', 'enter-2-A', 'enter-1-B', 'enter-2-B',
+                     ('managers', 'A', 'B'),
+                     'exit-1-B', 'exit-2-B', 'exit-1-A', 'exit-2-A']
+        )
+
+        async def foo():
+            async with Manager("A") as a, Manager("C") as c:
+                await AsyncYieldFrom([('managers', a.name, c.name)])
+                1/0
+
+        with self.assertRaises(ZeroDivisionError):
+            run_async(foo())
+
+    def test_with_2(self):
+        class CM:
+            def __aenter__(self):
+                pass
+
+        async def foo():
+            async with CM():
+                pass
+
+        with self.assertRaisesRegex(AttributeError, '__aexit__'):
+            run_async(foo())
+
+    def test_with_3(self):
+        class CM:
+            def __aexit__(self):
+                pass
+
+        async def foo():
+            async with CM():
+                pass
+
+        with self.assertRaisesRegex(AttributeError, '__aenter__'):
+            run_async(foo())
+
+    def test_with_4(self):
+        class CM:
+            def __enter__(self):
+                pass
+
+            def __exit__(self):
+                pass
+
+        async def foo():
+            async with CM():
+                pass
+
+        with self.assertRaisesRegex(AttributeError, '__aexit__'):
+            run_async(foo())
+
+    def test_with_5(self):
+        # While this test doesn't make a lot of sense,
+        # it's a regression test for an early bug with opcodes
+        # generation
+
+        class CM:
+            async def __aenter__(self):
+                return self
+
+            async def __aexit__(self, *exc):
+                pass
+
+        async def func():
+            async with CM():
+                assert (1, ) == 1
+
+        with self.assertRaises(AssertionError):
+            run_async(func())
+
+    def test_with_6(self):
+        class CM:
+            def __aenter__(self):
+                return 123
+
+            def __aexit__(self, *e):
+                return 456
+
+        async def foo():
+            async with CM():
+                pass
+
+        with self.assertRaisesRegex(
+            TypeError, "object int can't be used in 'await' expression"):
+            # it's important that __aexit__ wasn't called
+            run_async(foo())
+
+    def test_with_7(self):
+        class CM:
+            async def __aenter__(self):
+                return self
+
+            def __aexit__(self, *e):
+                return 456
+
+        async def foo():
+            async with CM():
+                pass
+
+        with self.assertRaisesRegex(
+            TypeError, "object int can't be used in 'await' expression"):
+
+            run_async(foo())
+
+    def test_for_1(self):
+        aiter_calls = 0
+
+        class AsyncIter:
+            def __init__(self):
+                self.i = 0
+
+            async def __aiter__(self):
+                nonlocal aiter_calls
+                aiter_calls += 1
+                return self
+
+            async def __anext__(self):
+                self.i += 1
+
+                if not (self.i % 10):
+                    await AsyncYield(self.i * 10)
+
+                if self.i > 100:
+                    raise StopAsyncIteration
+
+                return self.i, self.i
+
+
+        buffer = []
+        async def test1():
+            async for i1, i2 in AsyncIter():
+                buffer.append(i1 + i2)
+
+        yielded, _ = run_async(test1())
+        # Make sure that __aiter__ was called only once
+        self.assertEqual(aiter_calls, 1)
+        self.assertEqual(yielded, [i * 100 for i in range(1, 11)])
+        self.assertEqual(buffer, [i*2 for i in range(1, 101)])
+
+
+        buffer = []
+        async def test2():
+            nonlocal buffer
+            async for i in AsyncIter():
+                buffer.append(i[0])
+                if i[0] == 20:
+                    break
+            else:
+                buffer.append('what?')
+            buffer.append('end')
+
+        yielded, _ = run_async(test2())
+        # Make sure that __aiter__ was called only once
+        self.assertEqual(aiter_calls, 2)
+        self.assertEqual(yielded, [100, 200])
+        self.assertEqual(buffer, [i for i in range(1, 21)] + ['end'])
+
+
+        buffer = []
+        async def test3():
+            nonlocal buffer
+            async for i in AsyncIter():
+                if i[0] > 20:
+                    continue
+                buffer.append(i[0])
+            else:
+                buffer.append('what?')
+            buffer.append('end')
+
+        yielded, _ = run_async(test3())
+        # Make sure that __aiter__ was called only once
+        self.assertEqual(aiter_calls, 3)
+        self.assertEqual(yielded, [i * 100 for i in range(1, 11)])
+        self.assertEqual(buffer, [i for i in range(1, 21)] +
+                                 ['what?', 'end'])
+
+    def test_for_2(self):
+        tup = (1, 2, 3)
+        refs_before = sys.getrefcount(tup)
+
+        async def foo():
+            async for i in tup:
+                print('never going to happen')
+
+        with self.assertRaisesRegex(
+                TypeError, "async for' requires an object.*__aiter__.*tuple"):
+
+            run_async(foo())
+
+        self.assertEqual(sys.getrefcount(tup), refs_before)
+
+    def test_for_3(self):
+        class I:
+            def __aiter__(self):
+                return self
+
+        aiter = I()
+        refs_before = sys.getrefcount(aiter)
+
+        async def foo():
+            async for i in aiter:
+                print('never going to happen')
+
+        with self.assertRaisesRegex(
+                TypeError,
+                "async for' received an invalid object.*__aiter.*\: I"):
+
+            run_async(foo())
+
+        self.assertEqual(sys.getrefcount(aiter), refs_before)
+
+    def test_for_4(self):
+        class I:
+            async def __aiter__(self):
+                return self
+
+            def __anext__(self):
+                return ()
+
+        aiter = I()
+        refs_before = sys.getrefcount(aiter)
+
+        async def foo():
+            async for i in aiter:
+                print('never going to happen')
+
+        with self.assertRaisesRegex(
+                TypeError,
+                "async for' received an invalid object.*__anext__.*tuple"):
+
+            run_async(foo())
+
+        self.assertEqual(sys.getrefcount(aiter), refs_before)
+
+    def test_for_5(self):
+        class I:
+            async def __aiter__(self):
+                return self
+
+            def __anext__(self):
+                return 123
+
+        async def foo():
+            async for i in I():
+                print('never going to happen')
+
+        with self.assertRaisesRegex(
+                TypeError,
+                "async for' received an invalid object.*__anext.*int"):
+
+            run_async(foo())
+
+    def test_for_6(self):
+        I = 0
+
+        class Manager:
+            async def __aenter__(self):
+                nonlocal I
+                I += 10000
+
+            async def __aexit__(self, *args):
+                nonlocal I
+                I += 100000
+
+        class Iterable:
+            def __init__(self):
+                self.i = 0
+
+            async def __aiter__(self):
+                return self
+
+            async def __anext__(self):
+                if self.i > 10:
+                    raise StopAsyncIteration
+                self.i += 1
+                return self.i
+
+        ##############
+
+        manager = Manager()
+        iterable = Iterable()
+        mrefs_before = sys.getrefcount(manager)
+        irefs_before = sys.getrefcount(iterable)
+
+        async def main():
+            nonlocal I
+
+            async with manager:
+                async for i in iterable:
+                    I += 1
+            I += 1000
+
+        run_async(main())
+        self.assertEqual(I, 111011)
+
+        self.assertEqual(sys.getrefcount(manager), mrefs_before)
+        self.assertEqual(sys.getrefcount(iterable), irefs_before)
+
+        ##############
+
+        async def main():
+            nonlocal I
+
+            async with Manager():
+                async for i in Iterable():
+                    I += 1
+            I += 1000
+
+            async with Manager():
+                async for i in Iterable():
+                    I += 1
+            I += 1000
+
+        run_async(main())
+        self.assertEqual(I, 333033)
+
+        ##############
+
+        async def main():
+            nonlocal I
+
+            async with Manager():
+                I += 100
+                async for i in Iterable():
+                    I += 1
+                else:
+                    I += 10000000
+            I += 1000
+
+            async with Manager():
+                I += 100
+                async for i in Iterable():
+                    I += 1
+                else:
+                    I += 10000000
+            I += 1000
+
+        run_async(main())
+        self.assertEqual(I, 20555255)
+
+
+class CoroAsyncIOCompatTest(unittest.TestCase):
+
+    def test_asyncio_1(self):
+        import asyncio
+
+        class MyException(Exception):
+            pass
+
+        buffer = []
+
+        class CM:
+            async def __aenter__(self):
+                buffer.append(1)
+                await asyncio.sleep(0.01)
+                buffer.append(2)
+                return self
+
+            async def __aexit__(self, exc_type, exc_val, exc_tb):
+                await asyncio.sleep(0.01)
+                buffer.append(exc_type.__name__)
+
+        async def f():
+            async with CM() as c:
+                await asyncio.sleep(0.01)
+                raise MyException
+            buffer.append('unreachable')
+
+        loop = asyncio.get_event_loop()
+        try:
+            loop.run_until_complete(f())
+        except MyException:
+            pass
+        finally:
+            loop.close()
+
+        self.assertEqual(buffer, [1, 2, 'MyException'])
+
+
+class SysSetCoroWrapperTest(unittest.TestCase):
+
+    def test_set_wrapper_1(self):
+        async def foo():
+            return 'spam'
+
+        wrapped = None
+        def wrap(gen):
+            nonlocal wrapped
+            wrapped = gen
+            return gen
+
+        self.assertIsNone(sys.get_coroutine_wrapper())
+
+        sys.set_coroutine_wrapper(wrap)
+        self.assertIs(sys.get_coroutine_wrapper(), wrap)
+        try:
+            f = foo()
+            self.assertTrue(wrapped)
+
+            self.assertEqual(run_async(f), ([], 'spam'))
+        finally:
+            sys.set_coroutine_wrapper(None)
+
+        self.assertIsNone(sys.get_coroutine_wrapper())
+
+        wrapped = None
+        with silence_coro_gc():
+            foo()
+        self.assertFalse(wrapped)
+
+    def test_set_wrapper_2(self):
+        self.assertIsNone(sys.get_coroutine_wrapper())
+        with self.assertRaisesRegex(TypeError, "callable expected, got int"):
+            sys.set_coroutine_wrapper(1)
+        self.assertIsNone(sys.get_coroutine_wrapper())
+
+
+class CAPITest(unittest.TestCase):
+
+    def test_tp_await_1(self):
+        from _testcapi import awaitType as at
+
+        async def foo():
+            future = at(iter([1]))
+            return (await future)
+
+        self.assertEqual(foo().send(None), 1)
+
+    def test_tp_await_2(self):
+        # Test tp_await to __await__ mapping
+        from _testcapi import awaitType as at
+        future = at(iter([1]))
+        self.assertEqual(next(future.__await__()), 1)
+
+    def test_tp_await_3(self):
+        from _testcapi import awaitType as at
+
+        async def foo():
+            future = at(1)
+            return (await future)
+
+        with self.assertRaisesRegex(
+                TypeError, "__await__.*returned non-iterator of type 'int'"):
+            self.assertEqual(foo().send(None), 1)
+
+
+def test_main():
+    support.run_unittest(AsyncBadSyntaxTest,
+                         CoroutineTest,
+                         CoroAsyncIOCompatTest,
+                         SysSetCoroWrapperTest,
+                         CAPITest)
+
+
+if __name__=="__main__":
+    test_main()

Lib/test/test_dis.py

@@ -480,6 +480,24 @@ Constants:
 Names:
    0: x"""
 
+
+async def async_def():
+    await 1
+    async for a in b: pass
+    async with c as d: pass
+
+code_info_async_def = """\
+Name:              async_def
+Filename:          (.*)
+Argument count:    0
+Kw-only arguments: 0
+Number of locals:  2
+Stack size:        17
+Flags:             OPTIMIZED, NEWLOCALS, GENERATOR, NOFREE, COROUTINE
+Constants:
+   0: None
+   1: 1"""
+
 class CodeInfoTests(unittest.TestCase):
     test_pairs = [
       (dis.code_info, code_info_code_info),
@@ -488,6 +506,7 @@ class CodeInfoTests(unittest.TestCase):
       (expr_str, code_info_expr_str),
       (simple_stmt_str, code_info_simple_stmt_str),
       (compound_stmt_str, code_info_compound_stmt_str),
+      (async_def, code_info_async_def)
     ]
 
     def test_code_info(self):
@@ -697,7 +716,7 @@ expected_opinfo_jumpy = [
   Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=135, starts_line=None, is_jump_target=False),
   Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=138, starts_line=None, is_jump_target=False),
   Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=141, starts_line=None, is_jump_target=False),
-  Instruction(opname='SETUP_FINALLY', opcode=122, arg=72, argval=217, argrepr='to 217', offset=142, starts_line=20, is_jump_target=True),
+  Instruction(opname='SETUP_FINALLY', opcode=122, arg=73, argval=218, argrepr='to 218', offset=142, starts_line=20, is_jump_target=True),
   Instruction(opname='SETUP_EXCEPT', opcode=121, arg=12, argval=160, argrepr='to 160', offset=145, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=148, starts_line=21, is_jump_target=False),
   Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=151, starts_line=None, is_jump_target=False),
@@ -717,7 +736,7 @@ expected_opinfo_jumpy = [
   Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=179, starts_line=None, is_jump_target=False),
   Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=182, starts_line=None, is_jump_target=False),
   Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=183, starts_line=None, is_jump_target=False),
-  Instruction(opname='JUMP_FORWARD', opcode=110, arg=26, argval=213, argrepr='to 213', offset=184, starts_line=None, is_jump_target=False),
+  Instruction(opname='JUMP_FORWARD', opcode=110, arg=27, argval=214, argrepr='to 214', offset=184, starts_line=None, is_jump_target=False),
   Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=187, starts_line=None, is_jump_target=True),
   Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=188, starts_line=25, is_jump_target=True),
   Instruction(opname='SETUP_WITH', opcode=143, arg=17, argval=211, argrepr='to 211', offset=191, starts_line=None, is_jump_target=False),
@@ -728,17 +747,18 @@ expected_opinfo_jumpy = [
   Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=206, starts_line=None, is_jump_target=False),
   Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=207, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=208, starts_line=None, is_jump_target=False),
-  Instruction(opname='WITH_CLEANUP', opcode=81, arg=None, argval=None, argrepr='', offset=211, starts_line=None, is_jump_target=True),
+  Instruction(opname='WITH_CLEANUP_START', opcode=81, arg=None, argval=None, argrepr='', offset=211, starts_line=None, is_jump_target=True),
-  Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=212, starts_line=None, is_jump_target=False),
+  Instruction(opname='WITH_CLEANUP_FINISH', opcode=82, arg=None, argval=None, argrepr='', offset=212, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=213, starts_line=None, is_jump_target=True),
+  Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=213, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=214, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=214, starts_line=None, is_jump_target=True),
-  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=217, starts_line=28, is_jump_target=True),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=215, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=220, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=218, starts_line=28, is_jump_target=True),
-  Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=223, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=221, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=226, starts_line=None, is_jump_target=False),
+  Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=224, starts_line=None, is_jump_target=False),
-  Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=227, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=227, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=228, starts_line=None, is_jump_target=False),
+  Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=228, starts_line=None, is_jump_target=False),
-  Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=231, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=229, starts_line=None, is_jump_target=False),
+  Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=232, starts_line=None, is_jump_target=False),
 ]
 
 # One last piece of inspect fodder to check the default line number handling

Lib/test/test_grammar.py

@@ -2,6 +2,7 @@
 # This just tests whether the parser accepts them all.
 
 from test.support import check_syntax_error
+import inspect
 import unittest
 import sys
 # testing import *
@@ -1034,6 +1035,92 @@ class GrammarTests(unittest.TestCase):
         m @= 42
         self.assertEqual(m.other, 42)
 
+    def test_async_await(self):
+        async = 1
+        await = 2
+        self.assertEqual(async, 1)
+
+        def async():
+            nonlocal await
+            await = 10
+        async()
+        self.assertEqual(await, 10)
+
+        self.assertFalse(bool(async.__code__.co_flags & inspect.CO_COROUTINE))
+
+        async def test():
+            def sum():
+                async = 1
+                await = 41
+                return async + await
+
+            if 1:
+                await someobj()
+
+        self.assertEqual(test.__name__, 'test')
+        self.assertTrue(bool(test.__code__.co_flags & inspect.CO_COROUTINE))
+
+        def decorator(func):
+            setattr(func, '_marked', True)
+            return func
+
+        @decorator
+        async def test2():
+            return 22
+        self.assertTrue(test2._marked)
+        self.assertEqual(test2.__name__, 'test2')
+        self.assertTrue(bool(test2.__code__.co_flags & inspect.CO_COROUTINE))
+
+    def test_async_for(self):
+        class Done(Exception): pass
+
+        class AIter:
+            async def __aiter__(self):
+                return self
+            async def __anext__(self):
+                raise StopAsyncIteration
+
+        async def foo():
+            async for i in AIter():
+                pass
+            async for i, j in AIter():
+                pass
+            async for i in AIter():
+                pass
+            else:
+                pass
+            raise Done
+
+        with self.assertRaises(Done):
+            foo().send(None)
+
+    def test_async_with(self):
+        class Done(Exception): pass
+
+        class manager:
+            async def __aenter__(self):
+                return (1, 2)
+            async def __aexit__(self, *exc):
+                return False
+
+        async def foo():
+            async with manager():
+                pass
+            async with manager() as x:
+                pass
+            async with manager() as (x, y):
+                pass
+            async with manager(), manager():
+                pass
+            async with manager() as x, manager() as y:
+                pass
+            async with manager() as x, manager():
+                pass
+            raise Done
+
+        with self.assertRaises(Done):
+            foo().send(None)
+
 
 if __name__ == '__main__':
     unittest.main()

Lib/test/test_inspect.py

@@ -18,6 +18,7 @@ import textwrap
 import unicodedata
 import unittest
 import unittest.mock
+import warnings
 
 try:
     from concurrent.futures import ThreadPoolExecutor
@@ -62,14 +63,16 @@ class IsTestBase(unittest.TestCase):
     predicates = set([inspect.isbuiltin, inspect.isclass, inspect.iscode,
                       inspect.isframe, inspect.isfunction, inspect.ismethod,
                       inspect.ismodule, inspect.istraceback,
-                      inspect.isgenerator, inspect.isgeneratorfunction])
+                      inspect.isgenerator, inspect.isgeneratorfunction,
+                      inspect.iscoroutine, inspect.iscoroutinefunction])
 
     def istest(self, predicate, exp):
         obj = eval(exp)
         self.assertTrue(predicate(obj), '%s(%s)' % (predicate.__name__, exp))
 
         for other in self.predicates - set([predicate]):
-            if predicate == inspect.isgeneratorfunction and\
+            if (predicate == inspect.isgeneratorfunction or \
+               predicate == inspect.iscoroutinefunction) and \
                other == inspect.isfunction:
                 continue
             self.assertFalse(other(obj), 'not %s(%s)' % (other.__name__, exp))
@@ -78,13 +81,21 @@ def generator_function_example(self):
     for i in range(2):
         yield i
 
+async def coroutine_function_example(self):
+    return 'spam'
+
+@types.coroutine
+def gen_coroutine_function_example(self):
+    yield
+    return 'spam'
+
 
 class TestPredicates(IsTestBase):
-    def test_sixteen(self):
+    def test_eightteen(self):
         count = len([x for x in dir(inspect) if x.startswith('is')])
         # This test is here for remember you to update Doc/library/inspect.rst
         # which claims there are 16 such functions
-        expected = 16
+        expected = 19
         err_msg = "There are %d (not %d) is* functions" % (count, expected)
         self.assertEqual(count, expected, err_msg)
 
@@ -115,11 +126,64 @@ class TestPredicates(IsTestBase):
         self.istest(inspect.isdatadescriptor, 'collections.defaultdict.default_factory')
         self.istest(inspect.isgenerator, '(x for x in range(2))')
         self.istest(inspect.isgeneratorfunction, 'generator_function_example')
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            self.istest(inspect.iscoroutine, 'coroutine_function_example(1)')
+            self.istest(inspect.iscoroutinefunction, 'coroutine_function_example')
+
         if hasattr(types, 'MemberDescriptorType'):
             self.istest(inspect.ismemberdescriptor, 'datetime.timedelta.days')
         else:
             self.assertFalse(inspect.ismemberdescriptor(datetime.timedelta.days))
 
+    def test_iscoroutine(self):
+        gen_coro = gen_coroutine_function_example(1)
+        coro = coroutine_function_example(1)
+
+        self.assertTrue(
+            inspect.iscoroutinefunction(gen_coroutine_function_example))
+        self.assertTrue(inspect.iscoroutine(gen_coro))
+
+        self.assertTrue(
+            inspect.isgeneratorfunction(gen_coroutine_function_example))
+        self.assertTrue(inspect.isgenerator(gen_coro))
+
+        self.assertTrue(
+            inspect.iscoroutinefunction(coroutine_function_example))
+        self.assertTrue(inspect.iscoroutine(coro))
+
+        self.assertFalse(
+            inspect.isgeneratorfunction(coroutine_function_example))
+        self.assertFalse(inspect.isgenerator(coro))
+
+        coro.close(); gen_coro.close() # silence warnings
+
+    def test_isawaitable(self):
+        def gen(): yield
+        self.assertFalse(inspect.isawaitable(gen()))
+
+        coro = coroutine_function_example(1)
+        gen_coro = gen_coroutine_function_example(1)
+
+        self.assertTrue(
+            inspect.isawaitable(coro))
+        self.assertTrue(
+            inspect.isawaitable(gen_coro))
+
+        class Future:
+            def __await__():
+                pass
+        self.assertTrue(inspect.isawaitable(Future()))
+        self.assertFalse(inspect.isawaitable(Future))
+
+        class NotFuture: pass
+        not_fut = NotFuture()
+        not_fut.__await__ = lambda: None
+        self.assertFalse(inspect.isawaitable(not_fut))
+
+        coro.close(); gen_coro.close() # silence warnings
+
     def test_isroutine(self):
         self.assertTrue(inspect.isroutine(mod.spam))
         self.assertTrue(inspect.isroutine([].count))

Lib/test/test_minidom.py

@@ -49,6 +49,21 @@ class MinidomTest(unittest.TestCase):
         t = node.wholeText
         self.confirm(t == s, "looking for %r, found %r" % (s, t))
 
+    def testDocumentAsyncAttr(self):
+        doc = Document()
+        self.assertFalse(doc.async_)
+        with self.assertWarns(DeprecationWarning):
+            self.assertFalse(getattr(doc, 'async', True))
+        with self.assertWarns(DeprecationWarning):
+            setattr(doc, 'async', True)
+        with self.assertWarns(DeprecationWarning):
+            self.assertTrue(getattr(doc, 'async', False))
+        self.assertTrue(doc.async_)
+
+        self.assertFalse(Document.async_)
+        with self.assertWarns(DeprecationWarning):
+            self.assertFalse(getattr(Document, 'async', True))
+
     def testParseFromBinaryFile(self):
         with open(tstfile, 'rb') as file:
             dom = parse(file)

Lib/test/test_parser.py

@@ -63,6 +63,22 @@ class RoundtripLegalSyntaxTestCase(unittest.TestCase):
                          "    if (yield):\n"
                          "        yield x\n")
 
+    def test_await_statement(self):
+        self.check_suite("async def f():\n await smth()")
+        self.check_suite("async def f():\n foo = await smth()")
+        self.check_suite("async def f():\n foo, bar = await smth()")
+        self.check_suite("async def f():\n (await smth())")
+        self.check_suite("async def f():\n foo((await smth()))")
+        self.check_suite("async def f():\n await foo(); return 42")
+
+    def test_async_with_statement(self):
+        self.check_suite("async def f():\n async with 1: pass")
+        self.check_suite("async def f():\n async with a as b, c as d: pass")
+
+    def test_async_for_statement(self):
+        self.check_suite("async def f():\n async for i in (): pass")
+        self.check_suite("async def f():\n async for i, b in (): pass")
+
     def test_nonlocal_statement(self):
         self.check_suite("def f():\n"
                          "    x = 0\n"

Lib/test/test_pickle.py

@@ -351,7 +351,9 @@ class CompatPickleTests(unittest.TestCase):
 
         for name, exc in get_exceptions(builtins):
             with self.subTest(name):
-                if exc in (BlockingIOError, ResourceWarning):
+                if exc in (BlockingIOError,
+                           ResourceWarning,
+                           StopAsyncIteration):
                     continue
                 if exc is not OSError and issubclass(exc, OSError):
                     self.assertEqual(reverse_mapping('builtins', name),

Lib/test/test_sys.py

@@ -1025,9 +1025,9 @@ class SizeofTest(unittest.TestCase):
         # static type: PyTypeObject
         s = vsize('P2n15Pl4Pn9Pn11PIP')
         check(int, s)
-        # (PyTypeObject + PyNumberMethods + PyMappingMethods +
+        # (PyTypeObject + PyAsyncMethods + PyNumberMethods + PyMappingMethods +
         #  PySequenceMethods + PyBufferProcs + 4P)
-        s = vsize('P2n17Pl4Pn9Pn11PIP') + struct.calcsize('34P 3P 10P 2P 4P')
+        s = vsize('P2n17Pl4Pn9Pn11PIP') + struct.calcsize('34P 3P 3P 10P 2P 4P')
         # Separate block for PyDictKeysObject with 4 entries
         s += struct.calcsize("2nPn") + 4*struct.calcsize("n2P")
         # class

Lib/test/test_tokenize.py

@@ -641,6 +641,192 @@ Legacy unicode literals:
     NAME       'grün'        (2, 0) (2, 4)
     OP         '='           (2, 5) (2, 6)
     STRING     "U'green'"    (2, 7) (2, 15)
+
+Async/await extension:
+
+    >>> dump_tokens("async = 1")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    NAME       'async'       (1, 0) (1, 5)
+    OP         '='           (1, 6) (1, 7)
+    NUMBER     '1'           (1, 8) (1, 9)
+
+    >>> dump_tokens("a = (async = 1)")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    NAME       'a'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    OP         '('           (1, 4) (1, 5)
+    NAME       'async'       (1, 5) (1, 10)
+    OP         '='           (1, 11) (1, 12)
+    NUMBER     '1'           (1, 13) (1, 14)
+    OP         ')'           (1, 14) (1, 15)
+
+    >>> dump_tokens("async()")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    NAME       'async'       (1, 0) (1, 5)
+    OP         '('           (1, 5) (1, 6)
+    OP         ')'           (1, 6) (1, 7)
+
+    >>> dump_tokens("class async(Bar):pass")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    NAME       'class'       (1, 0) (1, 5)
+    NAME       'async'       (1, 6) (1, 11)
+    OP         '('           (1, 11) (1, 12)
+    NAME       'Bar'         (1, 12) (1, 15)
+    OP         ')'           (1, 15) (1, 16)
+    OP         ':'           (1, 16) (1, 17)
+    NAME       'pass'        (1, 17) (1, 21)
+
+    >>> dump_tokens("class async:pass")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    NAME       'class'       (1, 0) (1, 5)
+    NAME       'async'       (1, 6) (1, 11)
+    OP         ':'           (1, 11) (1, 12)
+    NAME       'pass'        (1, 12) (1, 16)
+
+    >>> dump_tokens("await = 1")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    NAME       'await'       (1, 0) (1, 5)
+    OP         '='           (1, 6) (1, 7)
+    NUMBER     '1'           (1, 8) (1, 9)
+
+    >>> dump_tokens("foo.async")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    NAME       'foo'         (1, 0) (1, 3)
+    OP         '.'           (1, 3) (1, 4)
+    NAME       'async'       (1, 4) (1, 9)
+
+    >>> dump_tokens("async for a in b: pass")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    NAME       'async'       (1, 0) (1, 5)
+    NAME       'for'         (1, 6) (1, 9)
+    NAME       'a'           (1, 10) (1, 11)
+    NAME       'in'          (1, 12) (1, 14)
+    NAME       'b'           (1, 15) (1, 16)
+    OP         ':'           (1, 16) (1, 17)
+    NAME       'pass'        (1, 18) (1, 22)
+
+    >>> dump_tokens("async with a as b: pass")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    NAME       'async'       (1, 0) (1, 5)
+    NAME       'with'        (1, 6) (1, 10)
+    NAME       'a'           (1, 11) (1, 12)
+    NAME       'as'          (1, 13) (1, 15)
+    NAME       'b'           (1, 16) (1, 17)
+    OP         ':'           (1, 17) (1, 18)
+    NAME       'pass'        (1, 19) (1, 23)
+
+    >>> dump_tokens("async.foo")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    NAME       'async'       (1, 0) (1, 5)
+    OP         '.'           (1, 5) (1, 6)
+    NAME       'foo'         (1, 6) (1, 9)
+
+    >>> dump_tokens("async")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    NAME       'async'       (1, 0) (1, 5)
+
+    >>> dump_tokens("async\\n#comment\\nawait")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    NAME       'async'       (1, 0) (1, 5)
+    NEWLINE    '\\n'          (1, 5) (1, 6)
+    COMMENT    '#comment'    (2, 0) (2, 8)
+    NL         '\\n'          (2, 8) (2, 9)
+    NAME       'await'       (3, 0) (3, 5)
+
+    >>> dump_tokens("async\\n...\\nawait")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    NAME       'async'       (1, 0) (1, 5)
+    NEWLINE    '\\n'          (1, 5) (1, 6)
+    OP         '...'         (2, 0) (2, 3)
+    NEWLINE    '\\n'          (2, 3) (2, 4)
+    NAME       'await'       (3, 0) (3, 5)
+
+    >>> dump_tokens("async\\nawait")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    NAME       'async'       (1, 0) (1, 5)
+    NEWLINE    '\\n'          (1, 5) (1, 6)
+    NAME       'await'       (2, 0) (2, 5)
+
+    >>> dump_tokens("foo.async + 1")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    NAME       'foo'         (1, 0) (1, 3)
+    OP         '.'           (1, 3) (1, 4)
+    NAME       'async'       (1, 4) (1, 9)
+    OP         '+'           (1, 10) (1, 11)
+    NUMBER     '1'           (1, 12) (1, 13)
+
+    >>> dump_tokens("async def foo(): pass")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    ASYNC      'async'       (1, 0) (1, 5)
+    NAME       'def'         (1, 6) (1, 9)
+    NAME       'foo'         (1, 10) (1, 13)
+    OP         '('           (1, 13) (1, 14)
+    OP         ')'           (1, 14) (1, 15)
+    OP         ':'           (1, 15) (1, 16)
+    NAME       'pass'        (1, 17) (1, 21)
+
+    >>> dump_tokens('''async def foo():
+    ...   def foo(await):
+    ...     await = 1
+    ...   if 1:
+    ...     await
+    ... async += 1
+    ... ''')
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    ASYNC      'async'       (1, 0) (1, 5)
+    NAME       'def'         (1, 6) (1, 9)
+    NAME       'foo'         (1, 10) (1, 13)
+    OP         '('           (1, 13) (1, 14)
+    OP         ')'           (1, 14) (1, 15)
+    OP         ':'           (1, 15) (1, 16)
+    NEWLINE    '\\n'          (1, 16) (1, 17)
+    INDENT     '  '          (2, 0) (2, 2)
+    NAME       'def'         (2, 2) (2, 5)
+    NAME       'foo'         (2, 6) (2, 9)
+    OP         '('           (2, 9) (2, 10)
+    NAME       'await'       (2, 10) (2, 15)
+    OP         ')'           (2, 15) (2, 16)
+    OP         ':'           (2, 16) (2, 17)
+    NEWLINE    '\\n'          (2, 17) (2, 18)
+    INDENT     '    '        (3, 0) (3, 4)
+    NAME       'await'       (3, 4) (3, 9)
+    OP         '='           (3, 10) (3, 11)
+    NUMBER     '1'           (3, 12) (3, 13)
+    NEWLINE    '\\n'          (3, 13) (3, 14)
+    DEDENT     ''            (4, 2) (4, 2)
+    NAME       'if'          (4, 2) (4, 4)
+    NUMBER     '1'           (4, 5) (4, 6)
+    OP         ':'           (4, 6) (4, 7)
+    NEWLINE    '\\n'          (4, 7) (4, 8)
+    INDENT     '    '        (5, 0) (5, 4)
+    AWAIT      'await'       (5, 4) (5, 9)
+    NEWLINE    '\\n'          (5, 9) (5, 10)
+    DEDENT     ''            (6, 0) (6, 0)
+    DEDENT     ''            (6, 0) (6, 0)
+    NAME       'async'       (6, 0) (6, 5)
+    OP         '+='          (6, 6) (6, 8)
+    NUMBER     '1'           (6, 9) (6, 10)
+    NEWLINE    '\\n'          (6, 10) (6, 11)
+
+    >>> dump_tokens('''async def foo():
+    ...   async for i in 1: pass''')
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    ASYNC      'async'       (1, 0) (1, 5)
+    NAME       'def'         (1, 6) (1, 9)
+    NAME       'foo'         (1, 10) (1, 13)
+    OP         '('           (1, 13) (1, 14)
+    OP         ')'           (1, 14) (1, 15)
+    OP         ':'           (1, 15) (1, 16)
+    NEWLINE    '\\n'          (1, 16) (1, 17)
+    INDENT     '  '          (2, 0) (2, 2)
+    ASYNC      'async'       (2, 2) (2, 7)
+    NAME       'for'         (2, 8) (2, 11)
+    NAME       'i'           (2, 12) (2, 13)
+    NAME       'in'          (2, 14) (2, 16)
+    NUMBER     '1'           (2, 17) (2, 18)
+    OP         ':'           (2, 18) (2, 19)
+    NAME       'pass'        (2, 20) (2, 24)
+    DEDENT     ''            (3, 0) (3, 0)
 """
 
 from test import support

Lib/test/test_types.py

@@ -1,7 +1,8 @@
 # Python test set -- part 6, built-in types
 
 from test.support import run_with_locale
-import collections
+import collections.abc
+import inspect
 import pickle
 import locale
 import sys
@@ -1172,5 +1173,37 @@ class SimpleNamespaceTests(unittest.TestCase):
             self.assertEqual(ns, ns_roundtrip, pname)
 
 
+class CoroutineTests(unittest.TestCase):
+    def test_wrong_args(self):
+        class Foo:
+            def __call__(self):
+                pass
+        def bar(): pass
+
+        samples = [Foo, Foo(), bar, None, int, 1]
+        for sample in samples:
+            with self.assertRaisesRegex(TypeError, 'expects a generator'):
+                types.coroutine(sample)
+
+    def test_genfunc(self):
+        def gen():
+            yield
+
+        self.assertFalse(isinstance(gen(), collections.abc.Coroutine))
+        self.assertFalse(isinstance(gen(), collections.abc.Awaitable))
+
+        self.assertIs(types.coroutine(gen), gen)
+
+        self.assertTrue(gen.__code__.co_flags & inspect.CO_ITERABLE_COROUTINE)
+        self.assertFalse(gen.__code__.co_flags & inspect.CO_COROUTINE)
+
+        g = gen()
+        self.assertTrue(g.gi_code.co_flags & inspect.CO_ITERABLE_COROUTINE)
+        self.assertFalse(g.gi_code.co_flags & inspect.CO_COROUTINE)
+        self.assertTrue(isinstance(g, collections.abc.Coroutine))
+        self.assertTrue(isinstance(g, collections.abc.Awaitable))
+        g.close() # silence warning
+
+
 if __name__ == '__main__':
     unittest.main()

Lib/token.py

@@ -64,8 +64,10 @@ ATEQUAL = 50
 RARROW = 51
 ELLIPSIS = 52
 OP = 53
-ERRORTOKEN = 54
+AWAIT = 54
-N_TOKENS = 55
+ASYNC = 55
+ERRORTOKEN = 56
+N_TOKENS = 57
 NT_OFFSET = 256
 #--end constants--
 

Lib/tokenize.py

@@ -274,7 +274,7 @@ class Untokenizer:
                 self.encoding = tokval
                 continue
 
-            if toknum in (NAME, NUMBER):
+            if toknum in (NAME, NUMBER, ASYNC, AWAIT):
                 tokval += ' '
 
             # Insert a space between two consecutive strings
@@ -477,6 +477,10 @@ def _tokenize(readline, encoding):
     contline = None
     indents = [0]
 
+    # 'stashed' and 'ctx' are used for async/await parsing
+    stashed = None
+    ctx = [('sync', 0)]
+
     if encoding is not None:
         if encoding == "utf-8-sig":
             # BOM will already have been stripped.
@@ -552,6 +556,11 @@ def _tokenize(readline, encoding):
                         "unindent does not match any outer indentation level",
                         ("<tokenize>", lnum, pos, line))
                 indents = indents[:-1]
+
+                cur_indent = indents[-1]
+                while len(ctx) > 1 and ctx[-1][1] >= cur_indent:
+                    ctx.pop()
+
                 yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
 
         else:                                  # continued statement
@@ -572,10 +581,16 @@ def _tokenize(readline, encoding):
                     (initial == '.' and token != '.' and token != '...')):
                     yield TokenInfo(NUMBER, token, spos, epos, line)
                 elif initial in '\r\n':
+                    if stashed:
+                        yield stashed
+                        stashed = None
                     yield TokenInfo(NL if parenlev > 0 else NEWLINE,
                            token, spos, epos, line)
                 elif initial == '#':
                     assert not token.endswith("\n")
+                    if stashed:
+                        yield stashed
+                        stashed = None
                     yield TokenInfo(COMMENT, token, spos, epos, line)
                 elif token in triple_quoted:
                     endprog = _compile(endpats[token])
@@ -603,7 +618,37 @@ def _tokenize(readline, encoding):
                     else:                                  # ordinary string
                         yield TokenInfo(STRING, token, spos, epos, line)
                 elif initial.isidentifier():               # ordinary name
-                    yield TokenInfo(NAME, token, spos, epos, line)
+                    if token in ('async', 'await'):
+                        if ctx[-1][0] == 'async' and ctx[-1][1] < indents[-1]:
+                            yield TokenInfo(
+                                ASYNC if token == 'async' else AWAIT,
+                                token, spos, epos, line)
+                            continue
+
+                    tok = TokenInfo(NAME, token, spos, epos, line)
+                    if token == 'async' and not stashed:
+                        stashed = tok
+                        continue
+
+                    if token == 'def':
+                        if (stashed
+                                and stashed.type == NAME
+                                and stashed.string == 'async'):
+
+                            ctx.append(('async', indents[-1]))
+
+                            yield TokenInfo(ASYNC, stashed.string,
+                                            stashed.start, stashed.end,
+                                            stashed.line)
+                            stashed = None
+                        else:
+                            ctx.append(('sync', indents[-1]))
+
+                    if stashed:
+                        yield stashed
+                        stashed = None
+
+                    yield tok
                 elif initial == '\\':                      # continued stmt
                     continued = 1
                 else:
@@ -611,12 +656,19 @@ def _tokenize(readline, encoding):
                         parenlev += 1
                     elif initial in ')]}':
                         parenlev -= 1
+                    if stashed:
+                        yield stashed
+                        stashed = None
                     yield TokenInfo(OP, token, spos, epos, line)
             else:
                 yield TokenInfo(ERRORTOKEN, line[pos],
                            (lnum, pos), (lnum, pos+1), line)
                 pos += 1
 
+    if stashed:
+        yield stashed
+        stashed = None
+
     for indent in indents[1:]:                 # pop remaining indent levels
         yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
     yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')

Lib/types.py

@@ -43,6 +43,30 @@ MemberDescriptorType = type(FunctionType.__globals__)
 del sys, _f, _g, _C,                              # Not for export
 
 
+_CO_GENERATOR = 0x20
+_CO_ITERABLE_COROUTINE = 0x100
+
+def coroutine(func):
+    """Convert regular generator function to a coroutine."""
+
+    # TODO: Implement this in C.
+
+    if (not isinstance(func, (FunctionType, MethodType)) or
+            not isinstance(getattr(func, '__code__', None), CodeType) or
+            not (func.__code__.co_flags & _CO_GENERATOR)):
+        raise TypeError('coroutine() expects a generator function')
+
+    co = func.__code__
+    func.__code__ = CodeType(
+        co.co_argcount, co.co_kwonlyargcount, co.co_nlocals, co.co_stacksize,
+        co.co_flags | _CO_ITERABLE_COROUTINE,
+        co.co_code,
+        co.co_consts, co.co_names, co.co_varnames, co.co_filename, co.co_name,
+        co.co_firstlineno, co.co_lnotab, co.co_freevars, co.co_cellvars)
+
+    return func
+
+
 # Provide a PEP 3115 compliant mechanism for class creation
 def new_class(name, bases=(), kwds=None, exec_body=None):
     """Create a class object dynamically using the appropriate metaclass."""

Lib/xml/dom/xmlbuilder.py

@@ -1,6 +1,7 @@
 """Implementation of the DOM Level 3 'LS-Load' feature."""
 
 import copy
+import warnings
 import xml.dom
 
 from xml.dom.NodeFilter import NodeFilter
@@ -331,13 +332,33 @@ class DOMBuilderFilter:
 del NodeFilter
 
 
+class _AsyncDeprecatedProperty:
+    def warn(self, cls):
+        clsname = cls.__name__
+        warnings.warn(
+            "{cls}.async is deprecated; use {cls}.async_".format(cls=clsname),
+            DeprecationWarning)
+
+    def __get__(self, instance, cls):
+        self.warn(cls)
+        if instance is not None:
+            return instance.async_
+        return False
+
+    def __set__(self, instance, value):
+        self.warn(type(instance))
+        setattr(instance, 'async_', value)
+
+
 class DocumentLS:
     """Mixin to create documents that conform to the load/save spec."""
 
-    async = False
+    async = _AsyncDeprecatedProperty()
+    async_ = False
 
     def _get_async(self):
         return False
+
     def _set_async(self, async):
         if async:
             raise xml.dom.NotSupportedErr(
@@ -363,6 +384,9 @@ class DocumentLS:
         return snode.toxml()
 
 
+del _AsyncDeprecatedProperty
+
+
 class DOMImplementationLS:
     MODE_SYNCHRONOUS = 1
     MODE_ASYNCHRONOUS = 2

Modules/_testcapimodule.c

@@ -3944,6 +3944,98 @@ static PyTypeObject matmulType = {
 };
 
 
+typedef struct {
+    PyObject_HEAD
+    PyObject *ao_iterator;
+} awaitObject;
+
+
+static PyObject *
+awaitObject_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+    PyObject *v;
+    awaitObject *ao;
+
+    if (!PyArg_UnpackTuple(args, "awaitObject", 1, 1, &v))
+        return NULL;
+
+    ao = (awaitObject *)type->tp_alloc(type, 0);
+    if (ao == NULL) {
+        return NULL;
+    }
+
+    Py_INCREF(v);
+    ao->ao_iterator = v;
+
+    return (PyObject *)ao;
+}
+
+
+static void
+awaitObject_dealloc(awaitObject *ao)
+{
+    Py_CLEAR(ao->ao_iterator);
+    Py_TYPE(ao)->tp_free(ao);
+}
+
+
+static PyObject *
+awaitObject_await(awaitObject *ao)
+{
+    Py_INCREF(ao->ao_iterator);
+    return ao->ao_iterator;
+}
+
+static PyAsyncMethods awaitType_as_async = {
+    (getawaitablefunc)awaitObject_await,    /* am_await */
+    0,                                      /* am_aiter */
+    0                                       /* am_anext */
+};
+
+
+static PyTypeObject awaitType = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "awaitType",
+    sizeof(awaitObject),                /* tp_basicsize */
+    0,                                  /* tp_itemsize */
+    (destructor)awaitObject_dealloc,    /* destructor tp_dealloc */
+    0,                                  /* tp_print */
+    0,                                  /* tp_getattr */
+    0,                                  /* tp_setattr */
+    &awaitType_as_async,                /* tp_as_async */
+    0,                                  /* tp_repr */
+    0,                                  /* tp_as_number */
+    0,                                  /* tp_as_sequence */
+    0,                                  /* tp_as_mapping */
+    0,                                  /* tp_hash */
+    0,                                  /* tp_call */
+    0,                                  /* tp_str */
+    PyObject_GenericGetAttr,            /* tp_getattro */
+    PyObject_GenericSetAttr,            /* tp_setattro */
+    0,                                  /* tp_as_buffer */
+    0,                                  /* tp_flags */
+    "C level type with tp_as_async",
+    0,                                  /* traverseproc tp_traverse */
+    0,                                  /* tp_clear */
+    0,                                  /* tp_richcompare */
+    0,                                  /* tp_weaklistoffset */
+    0,                                  /* tp_iter */
+    0,                                  /* tp_iternext */
+    0,                                  /* tp_methods */
+    0,                                  /* tp_members */
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    awaitObject_new,                    /* tp_new */
+    PyObject_Del,                       /* tp_free */
+};
+
+
 static struct PyModuleDef _testcapimodule = {
     PyModuleDef_HEAD_INIT,
     "_testcapi",
@@ -3977,6 +4069,11 @@ PyInit__testcapi(void)
     Py_INCREF(&matmulType);
     PyModule_AddObject(m, "matmulType", (PyObject *)&matmulType);
 
+    if (PyType_Ready(&awaitType) < 0)
+        return NULL;
+    Py_INCREF(&awaitType);
+    PyModule_AddObject(m, "awaitType", (PyObject *)&awaitType);
+
     PyModule_AddObject(m, "CHAR_MAX", PyLong_FromLong(CHAR_MAX));
     PyModule_AddObject(m, "CHAR_MIN", PyLong_FromLong(CHAR_MIN));
     PyModule_AddObject(m, "UCHAR_MAX", PyLong_FromLong(UCHAR_MAX));

Modules/parsermodule.c

@@ -1041,6 +1041,8 @@ VALIDATER(testlist_comp);       VALIDATER(yield_expr);
 VALIDATER(or_test);
 VALIDATER(test_nocond);         VALIDATER(lambdef_nocond);
 VALIDATER(yield_arg);
+VALIDATER(async_funcdef);       VALIDATER(async_stmt);
+VALIDATER(atom_expr);
 
 #undef VALIDATER
 
@@ -1608,6 +1610,7 @@ validate_compound_stmt(node *tree)
           || (ntype == try_stmt)
           || (ntype == with_stmt)
           || (ntype == funcdef)
+          || (ntype == async_stmt)
           || (ntype == classdef)
           || (ntype == decorated))
         res = validate_node(tree);
@@ -2440,27 +2443,60 @@ validate_factor(node *tree)
 
 /*  power:
  *
- *  power: atom trailer* ('**' factor)*
+ *  power: atom_expr trailer* ['**' factor]
  */
 static int
 validate_power(node *tree)
 {
-    int pos = 1;
     int nch = NCH(tree);
     int res = (validate_ntype(tree, power) && (nch >= 1)
-               && validate_atom(CHILD(tree, 0)));
+               && validate_atom_expr(CHILD(tree, 0)));
 
-    while (res && (pos < nch) && (TYPE(CHILD(tree, pos)) == trailer))
+    if (nch > 1) {
-        res = validate_trailer(CHILD(tree, pos++));
+        if (nch != 3) {
-    if (res && (pos < nch)) {
-        if (!is_even(nch - pos)) {
             err_string("illegal number of nodes for 'power'");
             return (0);
         }
-        for ( ; res && (pos < (nch - 1)); pos += 2)
+        res = (validate_doublestar(CHILD(tree, 1))
-            res = (validate_doublestar(CHILD(tree, pos))
+               && validate_factor(CHILD(tree, 2)));
-                   && validate_factor(CHILD(tree, pos + 1)));
     }
+
+    return (res);
+}
+
+
+/*  atom_expr:
+ *
+ *  atom_expr: [AWAIT] atom trailer*
+ */
+static int
+validate_atom_expr(node *tree)
+{
+    int start = 0;
+    int nch = NCH(tree);
+    int res;
+    int pos;
+
+    res = validate_ntype(tree, atom_expr) && (nch >= 1);
+    if (!res) {
+        return (res);
+    }
+
+    if (TYPE(CHILD(tree, 0)) == AWAIT) {
+        start = 1;
+        if (nch < 2) {
+            err_string("illegal number of nodes for 'atom_expr'");
+            return (0);
+        }
+    }
+
+    res = validate_atom(CHILD(tree, start));
+    if (res) {
+        pos = start + 1;
+        while (res && (pos < nch) && (TYPE(CHILD(tree, pos)) == trailer))
+            res = validate_trailer(CHILD(tree, pos++));
+    }
+
     return (res);
 }
 
@@ -2482,9 +2518,9 @@ validate_atom(node *tree)
 
             if (res && (nch == 3)) {
                 if (TYPE(CHILD(tree, 1))==yield_expr)
-                    res = validate_yield_expr(CHILD(tree, 1));
+                        res = validate_yield_expr(CHILD(tree, 1));
                 else
-                    res = validate_testlist_comp(CHILD(tree, 1));
+                        res = validate_testlist_comp(CHILD(tree, 1));
             }
             break;
           case LSQB:
@@ -2658,6 +2694,55 @@ validate_funcdef(node *tree)
     return res;
 }
 
+/* async_funcdef: ASYNC funcdef */
+
+static int
+validate_async_funcdef(node *tree)
+{
+    int nch = NCH(tree);
+    int res = validate_ntype(tree, async_funcdef);
+    if (res) {
+        if (nch == 2) {
+            res = (validate_ntype(CHILD(tree, 0), ASYNC)
+                   && validate_funcdef(CHILD(tree, 1)));
+        }
+        else {
+            res = 0;
+            err_string("illegal number of children for async_funcdef");
+        }
+    }
+    return res;
+}
+
+
+/* async_stmt: ASYNC (funcdef | with_stmt | for_stmt) */
+
+static int
+validate_async_stmt(node *tree)
+{
+    int nch = NCH(tree);
+    int res = (validate_ntype(tree, async_stmt)
+                && validate_ntype(CHILD(tree, 0), ASYNC));
+
+    if (nch != 2) {
+        res = 0;
+        err_string("illegal number of children for async_stmt");
+    } else {
+        if (TYPE(CHILD(tree, 1)) == funcdef) {
+            res = validate_funcdef(CHILD(tree, 1));
+        }
+        else if (TYPE(CHILD(tree, 1)) == with_stmt) {
+            res = validate_with_stmt(CHILD(tree, 1));
+        }
+        else if (TYPE(CHILD(tree, 1)) == for_stmt) {
+            res = validate_for(CHILD(tree, 1));
+        }
+    }
+
+    return res;
+}
+
+
 
 /* decorated
  *   decorators (classdef | funcdef)
@@ -3085,6 +3170,12 @@ validate_node(node *tree)
             /*
              *  Definition nodes.
              */
+          case async_funcdef:
+            res = validate_async_funcdef(tree);
+            break;
+          case async_stmt:
+            res = validate_async_stmt(tree);
+            break;
           case funcdef:
             res = validate_funcdef(tree);
             break;

Objects/exceptions.c

@@ -472,6 +472,13 @@ SimpleExtendsException(PyExc_Exception, TypeError,
                        "Inappropriate argument type.");
 
 
+/*
+ *    StopAsyncIteration extends Exception
+ */
+SimpleExtendsException(PyExc_Exception, StopAsyncIteration,
+                       "Signal the end from iterator.__anext__().");
+
+
 /*
  *    StopIteration extends Exception
  */
@@ -2468,6 +2475,7 @@ _PyExc_Init(PyObject *bltinmod)
     PRE_INIT(BaseException)
     PRE_INIT(Exception)
     PRE_INIT(TypeError)
+    PRE_INIT(StopAsyncIteration)
     PRE_INIT(StopIteration)
     PRE_INIT(GeneratorExit)
     PRE_INIT(SystemExit)
@@ -2538,6 +2546,7 @@ _PyExc_Init(PyObject *bltinmod)
     POST_INIT(BaseException)
     POST_INIT(Exception)
     POST_INIT(TypeError)
+    POST_INIT(StopAsyncIteration)
     POST_INIT(StopIteration)
     POST_INIT(GeneratorExit)
     POST_INIT(SystemExit)

Objects/frameobject.c

@@ -196,6 +196,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno)
         case SETUP_EXCEPT:
         case SETUP_FINALLY:
         case SETUP_WITH:
+        case SETUP_ASYNC_WITH:
             blockstack[blockstack_top++] = addr;
             in_finally[blockstack_top-1] = 0;
             break;
@@ -203,7 +204,8 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno)
         case POP_BLOCK:
             assert(blockstack_top > 0);
             setup_op = code[blockstack[blockstack_top-1]];
-            if (setup_op == SETUP_FINALLY || setup_op == SETUP_WITH) {
+            if (setup_op == SETUP_FINALLY || setup_op == SETUP_WITH
+                                    || setup_op == SETUP_ASYNC_WITH) {
                 in_finally[blockstack_top-1] = 1;
             }
             else {
@@ -218,7 +220,8 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno)
              * be seeing such an END_FINALLY.) */
             if (blockstack_top > 0) {
                 setup_op = code[blockstack[blockstack_top-1]];
-                if (setup_op == SETUP_FINALLY || setup_op == SETUP_WITH) {
+                if (setup_op == SETUP_FINALLY || setup_op == SETUP_WITH
+                                    || setup_op == SETUP_ASYNC_WITH) {
                     blockstack_top--;
                 }
             }
@@ -281,6 +284,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno)
         case SETUP_EXCEPT:
         case SETUP_FINALLY:
         case SETUP_WITH:
+        case SETUP_ASYNC_WITH:
             delta_iblock++;
             break;
 

Objects/genobject.c

@@ -24,6 +24,19 @@ _PyGen_Finalize(PyObject *self)
     PyObject *res;
     PyObject *error_type, *error_value, *error_traceback;
 
+    /* If `gen` is a coroutine, and if it was never awaited on,
+       issue a RuntimeWarning. */
+    if (gen->gi_code != NULL
+            && ((PyCodeObject *)gen->gi_code)->co_flags & (CO_COROUTINE
+                                                       | CO_ITERABLE_COROUTINE)
+            && gen->gi_frame != NULL
+            && gen->gi_frame->f_lasti == -1
+            && !PyErr_Occurred()
+            && PyErr_WarnFormat(PyExc_RuntimeWarning, 1,
+                                "coroutine '%.50S' was never awaited",
+                                gen->gi_qualname))
+        return;
+
     if (gen->gi_frame == NULL || gen->gi_frame->f_stacktop == NULL)
         /* Generator isn't paused, so no need to close */
         return;
@@ -135,7 +148,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc)
          * a leaking StopIteration into RuntimeError (with its cause
          * set appropriately). */
         if ((((PyCodeObject *)gen->gi_code)->co_flags &
-                                                CO_FUTURE_GENERATOR_STOP)
+              (CO_FUTURE_GENERATOR_STOP | CO_COROUTINE | CO_ITERABLE_COROUTINE))
             && PyErr_ExceptionMatches(PyExc_StopIteration))
         {
             PyObject *exc, *val, *val2, *tb;
@@ -402,6 +415,12 @@ failed_throw:
 static PyObject *
 gen_iternext(PyGenObject *gen)
 {
+    if (((PyCodeObject*)gen->gi_code)->co_flags & CO_COROUTINE) {
+        PyErr_SetString(PyExc_TypeError,
+                        "coroutine-objects do not support iteration");
+        return NULL;
+    }
+
     return gen_send_ex(gen, NULL, 0);
 }
 
@@ -459,8 +478,14 @@ _PyGen_FetchStopIterationValue(PyObject **pvalue) {
 static PyObject *
 gen_repr(PyGenObject *gen)
 {
-    return PyUnicode_FromFormat("<generator object %S at %p>",
+    if (PyGen_CheckCoroutineExact(gen)) {
-                                gen->gi_qualname, gen);
+        return PyUnicode_FromFormat("<coroutine object %S at %p>",
+                                    gen->gi_qualname, gen);
+    }
+    else {
+        return PyUnicode_FromFormat("<generator object %S at %p>",
+                                    gen->gi_qualname, gen);
+    }
 }
 
 static PyObject *
@@ -496,6 +521,19 @@ gen_get_qualname(PyGenObject *op)
     return op->gi_qualname;
 }
 
+static PyObject *
+gen_get_iter(PyGenObject *gen)
+{
+    if (((PyCodeObject*)gen->gi_code)->co_flags & CO_COROUTINE) {
+        PyErr_SetString(PyExc_TypeError,
+                        "coroutine-objects do not support iteration");
+        return NULL;
+    }
+
+    Py_INCREF(gen);
+    return gen;
+}
+
 static int
 gen_set_qualname(PyGenObject *op, PyObject *value)
 {
@@ -547,7 +585,7 @@ PyTypeObject PyGen_Type = {
     0,                                          /* tp_print */
     0,                                          /* tp_getattr */
     0,                                          /* tp_setattr */
-    0,                                          /* tp_reserved */
+    0,                                          /* tp_as_async */
     (reprfunc)gen_repr,                         /* tp_repr */
     0,                                          /* tp_as_number */
     0,                                          /* tp_as_sequence */
@@ -565,7 +603,7 @@ PyTypeObject PyGen_Type = {
     0,                                          /* tp_clear */
     0,                                          /* tp_richcompare */
     offsetof(PyGenObject, gi_weakreflist),      /* tp_weaklistoffset */
-    PyObject_SelfIter,                          /* tp_iter */
+    (getiterfunc)gen_get_iter,                  /* tp_iter */
     (iternextfunc)gen_iternext,                 /* tp_iternext */
     gen_methods,                                /* tp_methods */
     gen_memberlist,                             /* tp_members */
@@ -642,3 +680,57 @@ PyGen_NeedsFinalizing(PyGenObject *gen)
     /* No blocks except loops, it's safe to skip finalization. */
     return 0;
 }
+
+/*
+ *   This helper function returns an awaitable for `o`:
+ *     - `o` if `o` is a coroutine-object;
+ *     - `type(o)->tp_as_async->am_await(o)`
+ *
+ *   Raises a TypeError if it's not possible to return
+ *   an awaitable and returns NULL.
+ */
+PyObject *
+_PyGen_GetAwaitableIter(PyObject *o)
+{
+    getawaitablefunc getter = NULL;
+    PyTypeObject *ot;
+
+    if (PyGen_CheckCoroutineExact(o)) {
+        /* Fast path. It's a central function for 'await'. */
+        Py_INCREF(o);
+        return o;
+    }
+
+    ot = Py_TYPE(o);
+    if (ot->tp_as_async != NULL) {
+        getter = ot->tp_as_async->am_await;
+    }
+    if (getter != NULL) {
+        PyObject *res = (*getter)(o);
+        if (res != NULL) {
+            if (!PyIter_Check(res)) {
+                PyErr_Format(PyExc_TypeError,
+                             "__await__() returned non-iterator "
+                             "of type '%.100s'",
+                             Py_TYPE(res)->tp_name);
+                Py_CLEAR(res);
+            }
+            else {
+                if (PyGen_CheckCoroutineExact(res)) {
+                    /* __await__ must return an *iterator*, not
+                       a coroutine or another awaitable (see PEP 492) */
+                    PyErr_SetString(PyExc_TypeError,
+                                    "__await__() returned a coroutine");
+                    Py_CLEAR(res);
+                }
+            }
+        }
+        return res;
+    }
+
+    PyErr_Format(PyExc_TypeError,
+                 "object %.100s can't be used in 'await' expression",
+                 ot->tp_name);
+
+    return NULL;
+}

Objects/typeobject.c

@@ -2506,6 +2506,7 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds)
         type->tp_flags |= Py_TPFLAGS_HAVE_GC;
 
     /* Initialize essential fields */
+    type->tp_as_async = &et->as_async;
     type->tp_as_number = &et->as_number;
     type->tp_as_sequence = &et->as_sequence;
     type->tp_as_mapping = &et->as_mapping;
@@ -2751,6 +2752,7 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
     }
 
     /* Initialize essential fields */
+    type->tp_as_async = &res->as_async;
     type->tp_as_number = &res->as_number;
     type->tp_as_sequence = &res->as_sequence;
     type->tp_as_mapping = &res->as_mapping;
@@ -4566,6 +4568,7 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base)
 #define COPYSLOT(SLOT) \
     if (!type->SLOT && SLOTDEFINED(SLOT)) type->SLOT = base->SLOT
 
+#define COPYASYNC(SLOT) COPYSLOT(tp_as_async->SLOT)
 #define COPYNUM(SLOT) COPYSLOT(tp_as_number->SLOT)
 #define COPYSEQ(SLOT) COPYSLOT(tp_as_sequence->SLOT)
 #define COPYMAP(SLOT) COPYSLOT(tp_as_mapping->SLOT)
@@ -4615,6 +4618,15 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base)
         COPYNUM(nb_inplace_matrix_multiply);
     }
 
+    if (type->tp_as_async != NULL && base->tp_as_async != NULL) {
+        basebase = base->tp_base;
+        if (basebase->tp_as_async == NULL)
+            basebase = NULL;
+        COPYASYNC(am_await);
+        COPYASYNC(am_aiter);
+        COPYASYNC(am_anext);
+    }
+
     if (type->tp_as_sequence != NULL && base->tp_as_sequence != NULL) {
         basebase = base->tp_base;
         if (basebase->tp_as_sequence == NULL)
@@ -4884,6 +4896,8 @@ PyType_Ready(PyTypeObject *type)
     /* Some more special stuff */
     base = type->tp_base;
     if (base != NULL) {
+        if (type->tp_as_async == NULL)
+            type->tp_as_async = base->tp_as_async;
         if (type->tp_as_number == NULL)
             type->tp_as_number = base->tp_as_number;
         if (type->tp_as_sequence == NULL)
@@ -4904,16 +4918,6 @@ PyType_Ready(PyTypeObject *type)
             goto error;
     }
 
-    /* Warn for a type that implements tp_compare (now known as
-       tp_reserved) but not tp_richcompare. */
-    if (type->tp_reserved && !type->tp_richcompare) {
-        PyErr_Format(PyExc_TypeError,
-            "Type %.100s defines tp_reserved (formerly tp_compare) "
-            "but not tp_richcompare. Comparisons may not behave as intended.",
-            type->tp_name);
-        goto error;
-    }
-
     /* All done -- set the ready flag */
     assert(type->tp_dict != NULL);
     type->tp_flags =
@@ -6265,6 +6269,59 @@ slot_tp_finalize(PyObject *self)
     PyErr_Restore(error_type, error_value, error_traceback);
 }
 
+static PyObject *
+slot_am_await(PyObject *self)
+{
+    PyObject *func, *res;
+    _Py_IDENTIFIER(__await__);
+
+    func = lookup_method(self, &PyId___await__);
+    if (func != NULL) {
+        res = PyEval_CallObject(func, NULL);
+        Py_DECREF(func);
+        return res;
+    }
+    PyErr_Format(PyExc_AttributeError,
+                 "object %.50s does not have __await__ method",
+                 Py_TYPE(self)->tp_name);
+    return NULL;
+}
+
+static PyObject *
+slot_am_aiter(PyObject *self)
+{
+    PyObject *func, *res;
+    _Py_IDENTIFIER(__aiter__);
+
+    func = lookup_method(self, &PyId___aiter__);
+    if (func != NULL) {
+        res = PyEval_CallObject(func, NULL);
+        Py_DECREF(func);
+        return res;
+    }
+    PyErr_Format(PyExc_AttributeError,
+                 "object %.50s does not have __aiter__ method",
+                 Py_TYPE(self)->tp_name);
+    return NULL;
+}
+
+static PyObject *
+slot_am_anext(PyObject *self)
+{
+    PyObject *func, *res;
+    _Py_IDENTIFIER(__anext__);
+
+    func = lookup_method(self, &PyId___anext__);
+    if (func != NULL) {
+        res = PyEval_CallObject(func, NULL);
+        Py_DECREF(func);
+        return res;
+    }
+    PyErr_Format(PyExc_AttributeError,
+                 "object %.50s does not have __anext__ method",
+                 Py_TYPE(self)->tp_name);
+    return NULL;
+}
 
 /*
 Table mapping __foo__ names to tp_foo offsets and slot_tp_foo wrapper functions.
@@ -6281,6 +6338,7 @@ typedef struct wrapperbase slotdef;
 
 #undef TPSLOT
 #undef FLSLOT
+#undef AMSLOT
 #undef ETSLOT
 #undef SQSLOT
 #undef MPSLOT
@@ -6299,6 +6357,8 @@ typedef struct wrapperbase slotdef;
 #define ETSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \
     {NAME, offsetof(PyHeapTypeObject, SLOT), (void *)(FUNCTION), WRAPPER, \
      PyDoc_STR(DOC)}
+#define AMSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \
+    ETSLOT(NAME, as_async.SLOT, FUNCTION, WRAPPER, DOC)
 #define SQSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \
     ETSLOT(NAME, as_sequence.SLOT, FUNCTION, WRAPPER, DOC)
 #define MPSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \
@@ -6378,6 +6438,13 @@ static slotdef slotdefs[] = {
            "Create and return new object.  See help(type) for accurate signature."),
     TPSLOT("__del__", tp_finalize, slot_tp_finalize, (wrapperfunc)wrap_del, ""),
 
+    AMSLOT("__await__", am_await, slot_am_await, wrap_unaryfunc,
+           "__await__($self, /)\n--\n\nReturn an iterator to be used in await expression."),
+    AMSLOT("__aiter__", am_aiter, slot_am_aiter, wrap_unaryfunc,
+           "__aiter__($self, /)\n--\n\nReturn an awaitable, that resolves in asynchronous iterator."),
+    AMSLOT("__anext__", am_anext, slot_am_anext, wrap_unaryfunc,
+           "__anext__($self, /)\n--\n\nReturn a value or raise StopAsyncIteration."),
+
     BINSLOT("__add__", nb_add, slot_nb_add,
            "+"),
     RBINSLOT("__radd__", nb_add, slot_nb_add,
@@ -6530,6 +6597,10 @@ slotptr(PyTypeObject *type, int ioffset)
         ptr = (char *)type->tp_as_number;
         offset -= offsetof(PyHeapTypeObject, as_number);
     }
+    else if ((size_t)offset >= offsetof(PyHeapTypeObject, as_async)) {
+        ptr = (char *)type->tp_as_async;
+        offset -= offsetof(PyHeapTypeObject, as_async);
+    }
     else {
         ptr = (char *)type;
     }

Objects/typeslots.inc

@@ -75,3 +75,6 @@ offsetof(PyHeapTypeObject, ht_type.tp_getset),
 offsetof(PyHeapTypeObject, ht_type.tp_free),
 offsetof(PyHeapTypeObject, as_number.nb_matrix_multiply),
 offsetof(PyHeapTypeObject, as_number.nb_inplace_matrix_multiply),
+offsetof(PyHeapTypeObject, as_async.am_await),
+offsetof(PyHeapTypeObject, as_async.am_aiter),
+offsetof(PyHeapTypeObject, as_async.am_anext),

Objects/typeslots.py

@@ -12,6 +12,8 @@ for line in sys.stdin:
     member = m.group(1)
     if member.startswith("tp_"):
         member = "ht_type."+member
+    elif member.startswith("am_"):
+        member = "as_async."+member
     elif member.startswith("nb_"):
         member = "as_number."+member
     elif member.startswith("mp_"):

Parser/Python.asdl

@@ -10,7 +10,10 @@ module Python
         | Suite(stmt* body)
 
     stmt = FunctionDef(identifier name, arguments args,
-                           stmt* body, expr* decorator_list, expr? returns)
+                       stmt* body, expr* decorator_list, expr? returns)
+          | AsyncFunctionDef(identifier name, arguments args,
+                             stmt* body, expr* decorator_list, expr? returns)
+
           | ClassDef(identifier name,
              expr* bases,
              keyword* keywords,
@@ -24,9 +27,11 @@ module Python
 
           -- use 'orelse' because else is a keyword in target languages
           | For(expr target, expr iter, stmt* body, stmt* orelse)
+          | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse)
           | While(expr test, stmt* body, stmt* orelse)
           | If(expr test, stmt* body, stmt* orelse)
           | With(withitem* items, stmt* body)
+          | AsyncWith(withitem* items, stmt* body)
 
           | Raise(expr? exc, expr? cause)
           | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)
@@ -57,6 +62,7 @@ module Python
          | DictComp(expr key, expr value, comprehension* generators)
          | GeneratorExp(expr elt, comprehension* generators)
          -- the grammar constrains where yield expressions can occur
+         | Await(expr value)
          | Yield(expr? value)
          | YieldFrom(expr value)
          -- need sequences for compare to distinguish between

Parser/tokenizer.c

@@ -103,6 +103,8 @@ const char *_PyParser_TokenNames[] = {
     "ELLIPSIS",
     /* This table must match the #defines in token.h! */
     "OP",
+    "AWAIT",
+    "ASYNC",
     "<ERRORTOKEN>",
     "<N_TOKENS>"
 };
@@ -124,6 +126,11 @@ tok_new(void)
     tok->tabsize = TABSIZE;
     tok->indent = 0;
     tok->indstack[0] = 0;
+
+    tok->def = 0;
+    tok->defstack[0] = 0;
+    tok->deftypestack[0] = 0;
+
     tok->atbol = 1;
     tok->pendin = 0;
     tok->prompt = tok->nextprompt = NULL;
@@ -1335,6 +1342,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
     int c;
     int blankline, nonascii;
 
+    int tok_len;
+    struct tok_state ahead_tok;
+    char *ahead_tok_start = NULL, *ahead_top_end = NULL;
+    int ahead_tok_kind;
+
     *p_start = *p_end = NULL;
   nextline:
     tok->start = NULL;
@@ -1422,6 +1434,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
     if (tok->pendin != 0) {
         if (tok->pendin < 0) {
             tok->pendin++;
+
+            while (tok->def && tok->defstack[tok->def] >= tok->indent) {
+                tok->def--;
+            }
+
             return DEDENT;
         }
         else {
@@ -1481,6 +1498,57 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
             return ERRORTOKEN;
         *p_start = tok->start;
         *p_end = tok->cur;
+
+        tok_len = tok->cur - tok->start;
+        if (tok_len == 3 && memcmp(tok->start, "def", 3) == 0) {
+
+            if (tok->def + 1 >= MAXINDENT) {
+                tok->done = E_TOODEEP;
+                tok->cur = tok->inp;
+                return ERRORTOKEN;
+            }
+
+            if (tok->def && tok->deftypestack[tok->def] == 3) {
+                tok->deftypestack[tok->def] = 2;
+            }
+            else {
+                tok->def++;
+                tok->defstack[tok->def] = tok->indent;
+                tok->deftypestack[tok->def] = 1;
+            }
+        }
+        else if (tok_len == 5) {
+            if (memcmp(tok->start, "async", 5) == 0) {
+                memcpy(&ahead_tok, tok, sizeof(ahead_tok));
+
+                ahead_tok_kind = tok_get(&ahead_tok, &ahead_tok_start,
+                                         &ahead_top_end);
+
+                if (ahead_tok_kind == NAME &&
+                        ahead_tok.cur - ahead_tok.start == 3 &&
+                        memcmp(ahead_tok.start, "def", 3) == 0) {
+
+                    tok->def++;
+                    tok->defstack[tok->def] = tok->indent;
+                    tok->deftypestack[tok->def] = 3;
+
+                    return ASYNC;
+                }
+                else if (tok->def && tok->deftypestack[tok->def] == 2
+                         && tok->defstack[tok->def] < tok->indent) {
+
+                    return ASYNC;
+                }
+
+            }
+            else if (memcmp(tok->start, "await", 5) == 0
+                        && tok->def && tok->deftypestack[tok->def] == 2
+                        && tok->defstack[tok->def] < tok->indent) {
+
+                return AWAIT;
+            }
+        }
+
         return NAME;
     }
 

Parser/tokenizer.h

@@ -65,6 +65,13 @@ struct tok_state {
     const char* enc;        /* Encoding for the current str. */
     const char* str;
     const char* input; /* Tokenizer's newline translated copy of the string. */
+
+    int defstack[MAXINDENT];     /* stack if funcs & indents where they
+                                    were defined */
+    int deftypestack[MAXINDENT]; /* stack of func types
+                                    (0 not func; 1: "def name";
+                                     2: "async def name") */
+    int def;                     /* Length of stack of func types */
 };
 
 extern struct tok_state *PyTokenizer_FromString(const char *, int);

Python/Python-ast.c

@@ -45,6 +45,14 @@ static char *FunctionDef_fields[]={
     "decorator_list",
     "returns",
 };
+static PyTypeObject *AsyncFunctionDef_type;
+static char *AsyncFunctionDef_fields[]={
+    "name",
+    "args",
+    "body",
+    "decorator_list",
+    "returns",
+};
 static PyTypeObject *ClassDef_type;
 _Py_IDENTIFIER(bases);
 _Py_IDENTIFIER(keywords);
@@ -87,6 +95,13 @@ static char *For_fields[]={
     "body",
     "orelse",
 };
+static PyTypeObject *AsyncFor_type;
+static char *AsyncFor_fields[]={
+    "target",
+    "iter",
+    "body",
+    "orelse",
+};
 static PyTypeObject *While_type;
 _Py_IDENTIFIER(test);
 static char *While_fields[]={
@@ -106,6 +121,11 @@ static char *With_fields[]={
     "items",
     "body",
 };
+static PyTypeObject *AsyncWith_type;
+static char *AsyncWith_fields[]={
+    "items",
+    "body",
+};
 static PyTypeObject *Raise_type;
 _Py_IDENTIFIER(exc);
 _Py_IDENTIFIER(cause);
@@ -228,6 +248,10 @@ static char *GeneratorExp_fields[]={
     "elt",
     "generators",
 };
+static PyTypeObject *Await_type;
+static char *Await_fields[]={
+    "value",
+};
 static PyTypeObject *Yield_type;
 static char *Yield_fields[]={
     "value",
@@ -806,6 +830,9 @@ static int init_types(void)
     FunctionDef_type = make_type("FunctionDef", stmt_type, FunctionDef_fields,
                                  5);
     if (!FunctionDef_type) return 0;
+    AsyncFunctionDef_type = make_type("AsyncFunctionDef", stmt_type,
+                                      AsyncFunctionDef_fields, 5);
+    if (!AsyncFunctionDef_type) return 0;
     ClassDef_type = make_type("ClassDef", stmt_type, ClassDef_fields, 5);
     if (!ClassDef_type) return 0;
     Return_type = make_type("Return", stmt_type, Return_fields, 1);
@@ -818,12 +845,16 @@ static int init_types(void)
     if (!AugAssign_type) return 0;
     For_type = make_type("For", stmt_type, For_fields, 4);
     if (!For_type) return 0;
+    AsyncFor_type = make_type("AsyncFor", stmt_type, AsyncFor_fields, 4);
+    if (!AsyncFor_type) return 0;
     While_type = make_type("While", stmt_type, While_fields, 3);
     if (!While_type) return 0;
     If_type = make_type("If", stmt_type, If_fields, 3);
     if (!If_type) return 0;
     With_type = make_type("With", stmt_type, With_fields, 2);
     if (!With_type) return 0;
+    AsyncWith_type = make_type("AsyncWith", stmt_type, AsyncWith_fields, 2);
+    if (!AsyncWith_type) return 0;
     Raise_type = make_type("Raise", stmt_type, Raise_fields, 2);
     if (!Raise_type) return 0;
     Try_type = make_type("Try", stmt_type, Try_fields, 4);
@@ -872,6 +903,8 @@ static int init_types(void)
     GeneratorExp_type = make_type("GeneratorExp", expr_type,
                                   GeneratorExp_fields, 2);
     if (!GeneratorExp_type) return 0;
+    Await_type = make_type("Await", expr_type, Await_fields, 1);
+    if (!Await_type) return 0;
     Yield_type = make_type("Yield", expr_type, Yield_fields, 1);
     if (!Yield_type) return 0;
     YieldFrom_type = make_type("YieldFrom", expr_type, YieldFrom_fields, 1);
@@ -1200,6 +1233,36 @@ FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
     return p;
 }
 
+stmt_ty
+AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq
+                 * decorator_list, expr_ty returns, int lineno, int col_offset,
+                 PyArena *arena)
+{
+    stmt_ty p;
+    if (!name) {
+        PyErr_SetString(PyExc_ValueError,
+                        "field name is required for AsyncFunctionDef");
+        return NULL;
+    }
+    if (!args) {
+        PyErr_SetString(PyExc_ValueError,
+                        "field args is required for AsyncFunctionDef");
+        return NULL;
+    }
+    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
+    if (!p)
+        return NULL;
+    p->kind = AsyncFunctionDef_kind;
+    p->v.AsyncFunctionDef.name = name;
+    p->v.AsyncFunctionDef.args = args;
+    p->v.AsyncFunctionDef.body = body;
+    p->v.AsyncFunctionDef.decorator_list = decorator_list;
+    p->v.AsyncFunctionDef.returns = returns;
+    p->lineno = lineno;
+    p->col_offset = col_offset;
+    return p;
+}
+
 stmt_ty
 ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, asdl_seq *
          body, asdl_seq * decorator_list, int lineno, int col_offset, PyArena
@@ -1334,6 +1397,34 @@ For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int
     return p;
 }
 
+stmt_ty
+AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int
+         lineno, int col_offset, PyArena *arena)
+{
+    stmt_ty p;
+    if (!target) {
+        PyErr_SetString(PyExc_ValueError,
+                        "field target is required for AsyncFor");
+        return NULL;
+    }
+    if (!iter) {
+        PyErr_SetString(PyExc_ValueError,
+                        "field iter is required for AsyncFor");
+        return NULL;
+    }
+    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
+    if (!p)
+        return NULL;
+    p->kind = AsyncFor_kind;
+    p->v.AsyncFor.target = target;
+    p->v.AsyncFor.iter = iter;
+    p->v.AsyncFor.body = body;
+    p->v.AsyncFor.orelse = orelse;
+    p->lineno = lineno;
+    p->col_offset = col_offset;
+    return p;
+}
+
 stmt_ty
 While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
       col_offset, PyArena *arena)
@@ -1394,6 +1485,22 @@ With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, PyArena
     return p;
 }
 
+stmt_ty
+AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int col_offset,
+          PyArena *arena)
+{
+    stmt_ty p;
+    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
+    if (!p)
+        return NULL;
+    p->kind = AsyncWith_kind;
+    p->v.AsyncWith.items = items;
+    p->v.AsyncWith.body = body;
+    p->lineno = lineno;
+    p->col_offset = col_offset;
+    return p;
+}
+
 stmt_ty
 Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, PyArena *arena)
 {
@@ -1821,6 +1928,25 @@ GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
     return p;
 }
 
+expr_ty
+Await(expr_ty value, int lineno, int col_offset, PyArena *arena)
+{
+    expr_ty p;
+    if (!value) {
+        PyErr_SetString(PyExc_ValueError,
+                        "field value is required for Await");
+        return NULL;
+    }
+    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
+    if (!p)
+        return NULL;
+    p->kind = Await_kind;
+    p->v.Await.value = value;
+    p->lineno = lineno;
+    p->col_offset = col_offset;
+    return p;
+}
+
 expr_ty
 Yield(expr_ty value, int lineno, int col_offset, PyArena *arena)
 {
@@ -2409,6 +2535,36 @@ ast2obj_stmt(void* _o)
             goto failed;
         Py_DECREF(value);
         break;
+    case AsyncFunctionDef_kind:
+        result = PyType_GenericNew(AsyncFunctionDef_type, NULL, NULL);
+        if (!result) goto failed;
+        value = ast2obj_identifier(o->v.AsyncFunctionDef.name);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_name, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        value = ast2obj_arguments(o->v.AsyncFunctionDef.args);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_args, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        value = ast2obj_list(o->v.AsyncFunctionDef.body, ast2obj_stmt);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        value = ast2obj_list(o->v.AsyncFunctionDef.decorator_list,
+                             ast2obj_expr);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_decorator_list, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        value = ast2obj_expr(o->v.AsyncFunctionDef.returns);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_returns, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        break;
     case ClassDef_kind:
         result = PyType_GenericNew(ClassDef_type, NULL, NULL);
         if (!result) goto failed;
@@ -2513,6 +2669,30 @@ ast2obj_stmt(void* _o)
             goto failed;
         Py_DECREF(value);
         break;
+    case AsyncFor_kind:
+        result = PyType_GenericNew(AsyncFor_type, NULL, NULL);
+        if (!result) goto failed;
+        value = ast2obj_expr(o->v.AsyncFor.target);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_target, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        value = ast2obj_expr(o->v.AsyncFor.iter);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_iter, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        value = ast2obj_list(o->v.AsyncFor.body, ast2obj_stmt);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        value = ast2obj_list(o->v.AsyncFor.orelse, ast2obj_stmt);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_orelse, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        break;
     case While_kind:
         result = PyType_GenericNew(While_type, NULL, NULL);
         if (!result) goto failed;
@@ -2565,6 +2745,20 @@ ast2obj_stmt(void* _o)
             goto failed;
         Py_DECREF(value);
         break;
+    case AsyncWith_kind:
+        result = PyType_GenericNew(AsyncWith_type, NULL, NULL);
+        if (!result) goto failed;
+        value = ast2obj_list(o->v.AsyncWith.items, ast2obj_withitem);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_items, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        value = ast2obj_list(o->v.AsyncWith.body, ast2obj_stmt);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        break;
     case Raise_kind:
         result = PyType_GenericNew(Raise_type, NULL, NULL);
         if (!result) goto failed;
@@ -2878,6 +3072,15 @@ ast2obj_expr(void* _o)
             goto failed;
         Py_DECREF(value);
         break;
+    case Await_kind:
+        result = PyType_GenericNew(Await_type, NULL, NULL);
+        if (!result) goto failed;
+        value = ast2obj_expr(o->v.Await.value);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        break;
     case Yield_kind:
         result = PyType_GenericNew(Yield_type, NULL, NULL);
         if (!result) goto failed;
@@ -3832,6 +4035,102 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
+    isinstance = PyObject_IsInstance(obj, (PyObject*)AsyncFunctionDef_type);
+    if (isinstance == -1) {
+        return 1;
+    }
+    if (isinstance) {
+        identifier name;
+        arguments_ty args;
+        asdl_seq* body;
+        asdl_seq* decorator_list;
+        expr_ty returns;
+
+        if (_PyObject_HasAttrId(obj, &PyId_name)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_name);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_identifier(tmp, &name, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from AsyncFunctionDef");
+            return 1;
+        }
+        if (_PyObject_HasAttrId(obj, &PyId_args)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_args);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_arguments(tmp, &args, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from AsyncFunctionDef");
+            return 1;
+        }
+        if (_PyObject_HasAttrId(obj, &PyId_body)) {
+            int res;
+            Py_ssize_t len;
+            Py_ssize_t i;
+            tmp = _PyObject_GetAttrId(obj, &PyId_body);
+            if (tmp == NULL) goto failed;
+            if (!PyList_Check(tmp)) {
+                PyErr_Format(PyExc_TypeError, "AsyncFunctionDef field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                goto failed;
+            }
+            len = PyList_GET_SIZE(tmp);
+            body = _Py_asdl_seq_new(len, arena);
+            if (body == NULL) goto failed;
+            for (i = 0; i < len; i++) {
+                stmt_ty value;
+                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena);
+                if (res != 0) goto failed;
+                asdl_seq_SET(body, i, value);
+            }
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from AsyncFunctionDef");
+            return 1;
+        }
+        if (_PyObject_HasAttrId(obj, &PyId_decorator_list)) {
+            int res;
+            Py_ssize_t len;
+            Py_ssize_t i;
+            tmp = _PyObject_GetAttrId(obj, &PyId_decorator_list);
+            if (tmp == NULL) goto failed;
+            if (!PyList_Check(tmp)) {
+                PyErr_Format(PyExc_TypeError, "AsyncFunctionDef field \"decorator_list\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                goto failed;
+            }
+            len = PyList_GET_SIZE(tmp);
+            decorator_list = _Py_asdl_seq_new(len, arena);
+            if (decorator_list == NULL) goto failed;
+            for (i = 0; i < len; i++) {
+                expr_ty value;
+                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena);
+                if (res != 0) goto failed;
+                asdl_seq_SET(decorator_list, i, value);
+            }
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"decorator_list\" missing from AsyncFunctionDef");
+            return 1;
+        }
+        if (exists_not_none(obj, &PyId_returns)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_returns);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_expr(tmp, &returns, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            returns = NULL;
+        }
+        *out = AsyncFunctionDef(name, args, body, decorator_list, returns,
+                                lineno, col_offset, arena);
+        if (*out == NULL) goto failed;
+        return 0;
+    }
     isinstance = PyObject_IsInstance(obj, (PyObject*)ClassDef_type);
     if (isinstance == -1) {
         return 1;
@@ -4188,6 +4487,90 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
+    isinstance = PyObject_IsInstance(obj, (PyObject*)AsyncFor_type);
+    if (isinstance == -1) {
+        return 1;
+    }
+    if (isinstance) {
+        expr_ty target;
+        expr_ty iter;
+        asdl_seq* body;
+        asdl_seq* orelse;
+
+        if (_PyObject_HasAttrId(obj, &PyId_target)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_target);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_expr(tmp, &target, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from AsyncFor");
+            return 1;
+        }
+        if (_PyObject_HasAttrId(obj, &PyId_iter)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_iter);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_expr(tmp, &iter, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"iter\" missing from AsyncFor");
+            return 1;
+        }
+        if (_PyObject_HasAttrId(obj, &PyId_body)) {
+            int res;
+            Py_ssize_t len;
+            Py_ssize_t i;
+            tmp = _PyObject_GetAttrId(obj, &PyId_body);
+            if (tmp == NULL) goto failed;
+            if (!PyList_Check(tmp)) {
+                PyErr_Format(PyExc_TypeError, "AsyncFor field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                goto failed;
+            }
+            len = PyList_GET_SIZE(tmp);
+            body = _Py_asdl_seq_new(len, arena);
+            if (body == NULL) goto failed;
+            for (i = 0; i < len; i++) {
+                stmt_ty value;
+                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena);
+                if (res != 0) goto failed;
+                asdl_seq_SET(body, i, value);
+            }
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from AsyncFor");
+            return 1;
+        }
+        if (_PyObject_HasAttrId(obj, &PyId_orelse)) {
+            int res;
+            Py_ssize_t len;
+            Py_ssize_t i;
+            tmp = _PyObject_GetAttrId(obj, &PyId_orelse);
+            if (tmp == NULL) goto failed;
+            if (!PyList_Check(tmp)) {
+                PyErr_Format(PyExc_TypeError, "AsyncFor field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                goto failed;
+            }
+            len = PyList_GET_SIZE(tmp);
+            orelse = _Py_asdl_seq_new(len, arena);
+            if (orelse == NULL) goto failed;
+            for (i = 0; i < len; i++) {
+                stmt_ty value;
+                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena);
+                if (res != 0) goto failed;
+                asdl_seq_SET(orelse, i, value);
+            }
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from AsyncFor");
+            return 1;
+        }
+        *out = AsyncFor(target, iter, body, orelse, lineno, col_offset, arena);
+        if (*out == NULL) goto failed;
+        return 0;
+    }
     isinstance = PyObject_IsInstance(obj, (PyObject*)While_type);
     if (isinstance == -1) {
         return 1;
@@ -4392,6 +4775,66 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
+    isinstance = PyObject_IsInstance(obj, (PyObject*)AsyncWith_type);
+    if (isinstance == -1) {
+        return 1;
+    }
+    if (isinstance) {
+        asdl_seq* items;
+        asdl_seq* body;
+
+        if (_PyObject_HasAttrId(obj, &PyId_items)) {
+            int res;
+            Py_ssize_t len;
+            Py_ssize_t i;
+            tmp = _PyObject_GetAttrId(obj, &PyId_items);
+            if (tmp == NULL) goto failed;
+            if (!PyList_Check(tmp)) {
+                PyErr_Format(PyExc_TypeError, "AsyncWith field \"items\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                goto failed;
+            }
+            len = PyList_GET_SIZE(tmp);
+            items = _Py_asdl_seq_new(len, arena);
+            if (items == NULL) goto failed;
+            for (i = 0; i < len; i++) {
+                withitem_ty value;
+                res = obj2ast_withitem(PyList_GET_ITEM(tmp, i), &value, arena);
+                if (res != 0) goto failed;
+                asdl_seq_SET(items, i, value);
+            }
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"items\" missing from AsyncWith");
+            return 1;
+        }
+        if (_PyObject_HasAttrId(obj, &PyId_body)) {
+            int res;
+            Py_ssize_t len;
+            Py_ssize_t i;
+            tmp = _PyObject_GetAttrId(obj, &PyId_body);
+            if (tmp == NULL) goto failed;
+            if (!PyList_Check(tmp)) {
+                PyErr_Format(PyExc_TypeError, "AsyncWith field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                goto failed;
+            }
+            len = PyList_GET_SIZE(tmp);
+            body = _Py_asdl_seq_new(len, arena);
+            if (body == NULL) goto failed;
+            for (i = 0; i < len; i++) {
+                stmt_ty value;
+                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena);
+                if (res != 0) goto failed;
+                asdl_seq_SET(body, i, value);
+            }
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from AsyncWith");
+            return 1;
+        }
+        *out = AsyncWith(items, body, lineno, col_offset, arena);
+        if (*out == NULL) goto failed;
+        return 0;
+    }
     isinstance = PyObject_IsInstance(obj, (PyObject*)Raise_type);
     if (isinstance == -1) {
         return 1;
@@ -5326,6 +5769,28 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
+    isinstance = PyObject_IsInstance(obj, (PyObject*)Await_type);
+    if (isinstance == -1) {
+        return 1;
+    }
+    if (isinstance) {
+        expr_ty value;
+
+        if (_PyObject_HasAttrId(obj, &PyId_value)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_value);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_expr(tmp, &value, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Await");
+            return 1;
+        }
+        *out = Await(value, lineno, col_offset, arena);
+        if (*out == NULL) goto failed;
+        return 0;
+    }
     isinstance = PyObject_IsInstance(obj, (PyObject*)Yield_type);
     if (isinstance == -1) {
         return 1;
@@ -6782,6 +7247,8 @@ PyInit__ast(void)
     if (PyDict_SetItemString(d, "stmt", (PyObject*)stmt_type) < 0) return NULL;
     if (PyDict_SetItemString(d, "FunctionDef", (PyObject*)FunctionDef_type) <
         0) return NULL;
+    if (PyDict_SetItemString(d, "AsyncFunctionDef",
+        (PyObject*)AsyncFunctionDef_type) < 0) return NULL;
     if (PyDict_SetItemString(d, "ClassDef", (PyObject*)ClassDef_type) < 0)
         return NULL;
     if (PyDict_SetItemString(d, "Return", (PyObject*)Return_type) < 0) return
@@ -6793,10 +7260,14 @@ PyInit__ast(void)
     if (PyDict_SetItemString(d, "AugAssign", (PyObject*)AugAssign_type) < 0)
         return NULL;
     if (PyDict_SetItemString(d, "For", (PyObject*)For_type) < 0) return NULL;
+    if (PyDict_SetItemString(d, "AsyncFor", (PyObject*)AsyncFor_type) < 0)
+        return NULL;
     if (PyDict_SetItemString(d, "While", (PyObject*)While_type) < 0) return
         NULL;
     if (PyDict_SetItemString(d, "If", (PyObject*)If_type) < 0) return NULL;
     if (PyDict_SetItemString(d, "With", (PyObject*)With_type) < 0) return NULL;
+    if (PyDict_SetItemString(d, "AsyncWith", (PyObject*)AsyncWith_type) < 0)
+        return NULL;
     if (PyDict_SetItemString(d, "Raise", (PyObject*)Raise_type) < 0) return
         NULL;
     if (PyDict_SetItemString(d, "Try", (PyObject*)Try_type) < 0) return NULL;
@@ -6837,6 +7308,8 @@ PyInit__ast(void)
         return NULL;
     if (PyDict_SetItemString(d, "GeneratorExp", (PyObject*)GeneratorExp_type) <
         0) return NULL;
+    if (PyDict_SetItemString(d, "Await", (PyObject*)Await_type) < 0) return
+        NULL;
     if (PyDict_SetItemString(d, "Yield", (PyObject*)Yield_type) < 0) return
         NULL;
     if (PyDict_SetItemString(d, "YieldFrom", (PyObject*)YieldFrom_type) < 0)

Python/ast.c

@@ -219,6 +219,8 @@ validate_expr(expr_ty exp, expr_context_ty ctx)
         return !exp->v.Yield.value || validate_expr(exp->v.Yield.value, Load);
     case YieldFrom_kind:
         return validate_expr(exp->v.YieldFrom.value, Load);
+    case Await_kind:
+        return validate_expr(exp->v.Await.value, Load);
     case Compare_kind:
         if (!asdl_seq_LEN(exp->v.Compare.comparators)) {
             PyErr_SetString(PyExc_ValueError, "Compare with no comparators");
@@ -336,6 +338,11 @@ validate_stmt(stmt_ty stmt)
             validate_expr(stmt->v.For.iter, Load) &&
             validate_body(stmt->v.For.body, "For") &&
             validate_stmts(stmt->v.For.orelse);
+    case AsyncFor_kind:
+        return validate_expr(stmt->v.AsyncFor.target, Store) &&
+            validate_expr(stmt->v.AsyncFor.iter, Load) &&
+            validate_body(stmt->v.AsyncFor.body, "AsyncFor") &&
+            validate_stmts(stmt->v.AsyncFor.orelse);
     case While_kind:
         return validate_expr(stmt->v.While.test, Load) &&
             validate_body(stmt->v.While.body, "While") &&
@@ -354,6 +361,16 @@ validate_stmt(stmt_ty stmt)
                 return 0;
         }
         return validate_body(stmt->v.With.body, "With");
+    case AsyncWith_kind:
+        if (!validate_nonempty_seq(stmt->v.AsyncWith.items, "items", "AsyncWith"))
+            return 0;
+        for (i = 0; i < asdl_seq_LEN(stmt->v.AsyncWith.items); i++) {
+            withitem_ty item = asdl_seq_GET(stmt->v.AsyncWith.items, i);
+            if (!validate_expr(item->context_expr, Load) ||
+                (item->optional_vars && !validate_expr(item->optional_vars, Store)))
+                return 0;
+        }
+        return validate_body(stmt->v.AsyncWith.body, "AsyncWith");
     case Raise_kind:
         if (stmt->v.Raise.exc) {
             return validate_expr(stmt->v.Raise.exc, Load) &&
@@ -405,6 +422,12 @@ validate_stmt(stmt_ty stmt)
         return validate_nonempty_seq(stmt->v.Nonlocal.names, "names", "Nonlocal");
     case Expr_kind:
         return validate_expr(stmt->v.Expr.value, Load);
+    case AsyncFunctionDef_kind:
+        return validate_body(stmt->v.AsyncFunctionDef.body, "AsyncFunctionDef") &&
+            validate_arguments(stmt->v.AsyncFunctionDef.args) &&
+            validate_exprs(stmt->v.AsyncFunctionDef.decorator_list, Load, 0) &&
+            (!stmt->v.AsyncFunctionDef.returns ||
+             validate_expr(stmt->v.AsyncFunctionDef.returns, Load));
     case Pass_kind:
     case Break_kind:
     case Continue_kind:
@@ -503,6 +526,9 @@ static asdl_seq *ast_for_exprlist(struct compiling *, const node *,
 static expr_ty ast_for_testlist(struct compiling *, const node *);
 static stmt_ty ast_for_classdef(struct compiling *, const node *, asdl_seq *);
 
+static stmt_ty ast_for_with_stmt(struct compiling *, const node *, int);
+static stmt_ty ast_for_for_stmt(struct compiling *, const node *, int);
+
 /* Note different signature for ast_for_call */
 static expr_ty ast_for_call(struct compiling *, const node *, expr_ty);
 
@@ -941,6 +967,9 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n)
         case YieldFrom_kind:
             expr_name = "yield expression";
             break;
+        case Await_kind:
+            expr_name = "await expression";
+            break;
         case ListComp_kind:
             expr_name = "list comprehension";
             break;
@@ -1480,7 +1509,8 @@ ast_for_decorators(struct compiling *c, const node *n)
 }
 
 static stmt_ty
-ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
+ast_for_funcdef_impl(struct compiling *c, const node *n,
+                     asdl_seq *decorator_seq, int is_async)
 {
     /* funcdef: 'def' NAME parameters ['->' test] ':' suite */
     identifier name;
@@ -1509,14 +1539,68 @@ ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
     if (!body)
         return NULL;
 
-    return FunctionDef(name, args, body, decorator_seq, returns, LINENO(n),
+    if (is_async)
-                       n->n_col_offset, c->c_arena);
+        return AsyncFunctionDef(name, args, body, decorator_seq, returns,
+                                LINENO(n),
+                                n->n_col_offset, c->c_arena);
+    else
+        return FunctionDef(name, args, body, decorator_seq, returns,
+                           LINENO(n),
+                           n->n_col_offset, c->c_arena);
+}
+
+static stmt_ty
+ast_for_async_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
+{
+    /* async_funcdef: ASYNC funcdef */
+    REQ(n, async_funcdef);
+    REQ(CHILD(n, 0), ASYNC);
+    REQ(CHILD(n, 1), funcdef);
+
+    return ast_for_funcdef_impl(c, CHILD(n, 1), decorator_seq,
+                                1 /* is_async */);
+}
+
+static stmt_ty
+ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
+{
+    /* funcdef: 'def' NAME parameters ['->' test] ':' suite */
+    return ast_for_funcdef_impl(c, n, decorator_seq,
+                                0 /* is_async */);
+}
+
+
+static stmt_ty
+ast_for_async_stmt(struct compiling *c, const node *n)
+{
+    /* async_stmt: ASYNC (funcdef | with_stmt | for_stmt) */
+    REQ(n, async_stmt);
+    REQ(CHILD(n, 0), ASYNC);
+
+    switch (TYPE(CHILD(n, 1))) {
+        case funcdef:
+            return ast_for_funcdef_impl(c, CHILD(n, 1), NULL,
+                                        1 /* is_async */);
+        case with_stmt:
+            return ast_for_with_stmt(c, CHILD(n, 1),
+                                     1 /* is_async */);
+
+        case for_stmt:
+            return ast_for_for_stmt(c, CHILD(n, 1),
+                                    1 /* is_async */);
+
+        default:
+            PyErr_Format(PyExc_SystemError,
+                         "invalid async stament: %s",
+                         STR(CHILD(n, 1)));
+            return NULL;
+    }
 }
 
 static stmt_ty
 ast_for_decorated(struct compiling *c, const node *n)
 {
-    /* decorated: decorators (classdef | funcdef) */
+    /* decorated: decorators (classdef | funcdef | async_funcdef) */
     stmt_ty thing = NULL;
     asdl_seq *decorator_seq = NULL;
 
@@ -1527,12 +1611,15 @@ ast_for_decorated(struct compiling *c, const node *n)
       return NULL;
 
     assert(TYPE(CHILD(n, 1)) == funcdef ||
+           TYPE(CHILD(n, 1)) == async_funcdef ||
            TYPE(CHILD(n, 1)) == classdef);
 
     if (TYPE(CHILD(n, 1)) == funcdef) {
       thing = ast_for_funcdef(c, CHILD(n, 1), decorator_seq);
     } else if (TYPE(CHILD(n, 1)) == classdef) {
       thing = ast_for_classdef(c, CHILD(n, 1), decorator_seq);
+    } else if (TYPE(CHILD(n, 1)) == async_funcdef) {
+      thing = ast_for_async_funcdef(c, CHILD(n, 1), decorator_seq);
     }
     /* we count the decorators in when talking about the class' or
      * function's line number */
@@ -2271,19 +2358,29 @@ ast_for_factor(struct compiling *c, const node *n)
 }
 
 static expr_ty
-ast_for_power(struct compiling *c, const node *n)
+ast_for_atom_expr(struct compiling *c, const node *n)
 {
-    /* power: atom trailer* ('**' factor)*
+    int i, nch, start = 0;
-     */
-    int i;
     expr_ty e, tmp;
-    REQ(n, power);
+
-    e = ast_for_atom(c, CHILD(n, 0));
+    REQ(n, atom_expr);
+    nch = NCH(n);
+
+    if (TYPE(CHILD(n, 0)) == AWAIT) {
+        start = 1;
+        assert(nch > 1);
+    }
+
+    e = ast_for_atom(c, CHILD(n, start));
     if (!e)
         return NULL;
-    if (NCH(n) == 1)
+    if (nch == 1)
         return e;
-    for (i = 1; i < NCH(n); i++) {
+    if (start && nch == 2) {
+        return Await(e, LINENO(n), n->n_col_offset, c->c_arena);
+    }
+
+    for (i = start + 1; i < nch; i++) {
         node *ch = CHILD(n, i);
         if (TYPE(ch) != trailer)
             break;
@@ -2294,6 +2391,28 @@ ast_for_power(struct compiling *c, const node *n)
         tmp->col_offset = e->col_offset;
         e = tmp;
     }
+
+    if (start) {
+        /* there was an AWAIT */
+        return Await(e, LINENO(n), n->n_col_offset, c->c_arena);
+    }
+    else {
+        return e;
+    }
+}
+
+static expr_ty
+ast_for_power(struct compiling *c, const node *n)
+{
+    /* power: atom trailer* ('**' factor)*
+     */
+    expr_ty e;
+    REQ(n, power);
+    e = ast_for_atom_expr(c, CHILD(n, 0));
+    if (!e)
+        return NULL;
+    if (NCH(n) == 1)
+        return e;
     if (TYPE(CHILD(n, NCH(n) - 1)) == factor) {
         expr_ty f = ast_for_expr(c, CHILD(n, NCH(n) - 1));
         if (!f)
@@ -2338,7 +2457,9 @@ ast_for_expr(struct compiling *c, const node *n)
        arith_expr: term (('+'|'-') term)*
        term: factor (('*'|'@'|'/'|'%'|'//') factor)*
        factor: ('+'|'-'|'~') factor | power
-       power: atom trailer* ('**' factor)*
+       power: atom_expr ['**' factor]
+       atom_expr: [AWAIT] atom trailer*
+       yield_expr: 'yield' [yield_arg]
     */
 
     asdl_seq *seq;
@@ -3403,7 +3524,7 @@ ast_for_while_stmt(struct compiling *c, const node *n)
 }
 
 static stmt_ty
-ast_for_for_stmt(struct compiling *c, const node *n)
+ast_for_for_stmt(struct compiling *c, const node *n, int is_async)
 {
     asdl_seq *_target, *seq = NULL, *suite_seq;
     expr_ty expression;
@@ -3437,8 +3558,14 @@ ast_for_for_stmt(struct compiling *c, const node *n)
     if (!suite_seq)
         return NULL;
 
-    return For(target, expression, suite_seq, seq, LINENO(n), n->n_col_offset,
+    if (is_async)
-               c->c_arena);
+        return AsyncFor(target, expression, suite_seq, seq,
+                        LINENO(n), n->n_col_offset,
+                        c->c_arena);
+    else
+        return For(target, expression, suite_seq, seq,
+                   LINENO(n), n->n_col_offset,
+                   c->c_arena);
 }
 
 static excepthandler_ty
@@ -3585,7 +3712,7 @@ ast_for_with_item(struct compiling *c, const node *n)
 
 /* with_stmt: 'with' with_item (',' with_item)* ':' suite */
 static stmt_ty
-ast_for_with_stmt(struct compiling *c, const node *n)
+ast_for_with_stmt(struct compiling *c, const node *n, int is_async)
 {
     int i, n_items;
     asdl_seq *items, *body;
@@ -3607,7 +3734,10 @@ ast_for_with_stmt(struct compiling *c, const node *n)
     if (!body)
         return NULL;
 
-    return With(items, body, LINENO(n), n->n_col_offset, c->c_arena);
+    if (is_async)
+        return AsyncWith(items, body, LINENO(n), n->n_col_offset, c->c_arena);
+    else
+        return With(items, body, LINENO(n), n->n_col_offset, c->c_arena);
 }
 
 static stmt_ty
@@ -3714,7 +3844,7 @@ ast_for_stmt(struct compiling *c, const node *n)
     }
     else {
         /* compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt
-                        | funcdef | classdef | decorated
+                        | funcdef | classdef | decorated | async_stmt
         */
         node *ch = CHILD(n, 0);
         REQ(n, compound_stmt);
@@ -3724,17 +3854,19 @@ ast_for_stmt(struct compiling *c, const node *n)
             case while_stmt:
                 return ast_for_while_stmt(c, ch);
             case for_stmt:
-                return ast_for_for_stmt(c, ch);
+                return ast_for_for_stmt(c, ch, 0);
             case try_stmt:
                 return ast_for_try_stmt(c, ch);
             case with_stmt:
-                return ast_for_with_stmt(c, ch);
+                return ast_for_with_stmt(c, ch, 0);
             case funcdef:
                 return ast_for_funcdef(c, ch, NULL);
             case classdef:
                 return ast_for_classdef(c, ch, NULL);
             case decorated:
                 return ast_for_decorated(c, ch);
+            case async_stmt:
+                return ast_for_async_stmt(c, ch);
             default:
                 PyErr_Format(PyExc_SystemError,
                              "unhandled small_stmt: TYPE=%d NCH=%d\n",

Python/ceval.c

@@ -1926,11 +1926,133 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             goto fast_block_end;
         }
 
+        TARGET(GET_AITER) {
+            getaiterfunc getter = NULL;
+            PyObject *iter = NULL;
+            PyObject *awaitable = NULL;
+            PyObject *obj = TOP();
+            PyTypeObject *type = Py_TYPE(obj);
+
+            if (type->tp_as_async != NULL)
+                getter = type->tp_as_async->am_aiter;
+
+            if (getter != NULL) {
+                iter = (*getter)(obj);
+                Py_DECREF(obj);
+                if (iter == NULL) {
+                    SET_TOP(NULL);
+                    goto error;
+                }
+            }
+            else {
+                SET_TOP(NULL);
+                PyErr_Format(
+                    PyExc_TypeError,
+                    "'async for' requires an object with "
+                    "__aiter__ method, got %.100s",
+                    type->tp_name);
+                Py_DECREF(obj);
+                goto error;
+            }
+
+            awaitable = _PyGen_GetAwaitableIter(iter);
+            if (awaitable == NULL) {
+                SET_TOP(NULL);
+                PyErr_Format(
+                    PyExc_TypeError,
+                    "'async for' received an invalid object "
+                    "from __aiter__: %.100s",
+                    Py_TYPE(iter)->tp_name);
+
+                Py_DECREF(iter);
+                goto error;
+            } else
+                Py_DECREF(iter);
+
+            SET_TOP(awaitable);
+            DISPATCH();
+        }
+
+        TARGET(GET_ANEXT) {
+            aiternextfunc getter = NULL;
+            PyObject *next_iter = NULL;
+            PyObject *awaitable = NULL;
+            PyObject *aiter = TOP();
+            PyTypeObject *type = Py_TYPE(aiter);
+
+            if (type->tp_as_async != NULL)
+                getter = type->tp_as_async->am_anext;
+
+            if (getter != NULL) {
+                next_iter = (*getter)(aiter);
+                if (next_iter == NULL) {
+                    goto error;
+                }
+            }
+            else {
+                PyErr_Format(
+                    PyExc_TypeError,
+                    "'async for' requires an iterator with "
+                    "__anext__ method, got %.100s",
+                    type->tp_name);
+                goto error;
+            }
+
+            awaitable = _PyGen_GetAwaitableIter(next_iter);
+            if (awaitable == NULL) {
+                PyErr_Format(
+                    PyExc_TypeError,
+                    "'async for' received an invalid object "
+                    "from __anext__: %.100s",
+                    Py_TYPE(next_iter)->tp_name);
+
+                Py_DECREF(next_iter);
+                goto error;
+            } else
+                Py_DECREF(next_iter);
+
+            PUSH(awaitable);
+            DISPATCH();
+        }
+
+        TARGET(GET_AWAITABLE) {
+            PyObject *iterable = TOP();
+            PyObject *iter = _PyGen_GetAwaitableIter(iterable);
+
+            Py_DECREF(iterable);
+
+            SET_TOP(iter); /* Even if it's NULL */
+
+            if (iter == NULL) {
+                goto error;
+            }
+
+            DISPATCH();
+        }
+
         TARGET(YIELD_FROM) {
             PyObject *v = POP();
             PyObject *reciever = TOP();
             int err;
             if (PyGen_CheckExact(reciever)) {
+                if (
+                    (((PyCodeObject*) \
+                        ((PyGenObject*)reciever)->gi_code)->co_flags &
+                                                        CO_COROUTINE)
+                    && !(co->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE)))
+                {
+                    /* If we're yielding-from a coroutine object from a regular
+                       generator object - raise an error. */
+
+                    Py_CLEAR(v);
+                    Py_CLEAR(reciever);
+                    SET_TOP(NULL);
+
+                    PyErr_SetString(PyExc_TypeError,
+                                    "cannot 'yield from' a coroutine object "
+                                    "from a generator");
+                    goto error;
+                }
                 retval = _PyGen_Send((PyGenObject *)reciever, v);
             } else {
                 _Py_IDENTIFIER(send);
@@ -2822,11 +2944,26 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
         TARGET(GET_ITER) {
             /* before: [obj]; after [getiter(obj)] */
             PyObject *iterable = TOP();
-            PyObject *iter = PyObject_GetIter(iterable);
+            PyObject *iter;
-            Py_DECREF(iterable);
+            /* If we have a generator object on top -- keep it there,
-            SET_TOP(iter);
+               it's already an iterator.
-            if (iter == NULL)
+
-                goto error;
+               This is needed to allow use of 'async def' coroutines
+               in 'yield from' expression from generator-based coroutines
+               (decorated with types.coroutine()).
+
+               'yield from' is compiled to GET_ITER..YIELD_FROM combination,
+               but since coroutines raise TypeError in their 'tp_iter' we
+               need a way for them to "pass through" the GET_ITER.
+            */
+            if (!PyGen_CheckExact(iterable)) {
+                /* `iterable` is not a generator. */
+                iter = PyObject_GetIter(iterable);
+                Py_DECREF(iterable);
+                SET_TOP(iter);
+                if (iter == NULL)
+                    goto error;
+            }
             PREDICT(FOR_ITER);
             DISPATCH();
         }
@@ -2883,6 +3020,39 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             DISPATCH();
         }
 
+        TARGET(BEFORE_ASYNC_WITH) {
+            _Py_IDENTIFIER(__aexit__);
+            _Py_IDENTIFIER(__aenter__);
+
+            PyObject *mgr = TOP();
+            PyObject *exit = special_lookup(mgr, &PyId___aexit__),
+                     *enter;
+            PyObject *res;
+            if (exit == NULL)
+                goto error;
+            SET_TOP(exit);
+            enter = special_lookup(mgr, &PyId___aenter__);
+            Py_DECREF(mgr);
+            if (enter == NULL)
+                goto error;
+            res = PyObject_CallFunctionObjArgs(enter, NULL);
+            Py_DECREF(enter);
+            if (res == NULL)
+                goto error;
+            PUSH(res);
+            DISPATCH();
+        }
+
+        TARGET(SETUP_ASYNC_WITH) {
+            PyObject *res = POP();
+            /* Setup the finally block before pushing the result
+               of __aenter__ on the stack. */
+            PyFrame_BlockSetup(f, SETUP_FINALLY, INSTR_OFFSET() + oparg,
+                               STACK_LEVEL());
+            PUSH(res);
+            DISPATCH();
+        }
+
         TARGET(SETUP_WITH) {
             _Py_IDENTIFIER(__exit__);
             _Py_IDENTIFIER(__enter__);
@@ -2909,7 +3079,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             DISPATCH();
         }
 
-        TARGET(WITH_CLEANUP) {
+        TARGET(WITH_CLEANUP_START) {
             /* At the top of the stack are 1-6 values indicating
                how/why we entered the finally clause:
                - TOP = None
@@ -2937,7 +3107,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 
             PyObject *exit_func;
             PyObject *exc = TOP(), *val = Py_None, *tb = Py_None, *res;
-            int err;
             if (exc == Py_None) {
                 (void)POP();
                 exit_func = TOP();
@@ -2987,10 +3156,23 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             if (res == NULL)
                 goto error;
 
+            PUSH(exc);
+            PUSH(res);
+            PREDICT(WITH_CLEANUP_FINISH);
+            DISPATCH();
+        }
+
+        PREDICTED(WITH_CLEANUP_FINISH);
+        TARGET(WITH_CLEANUP_FINISH) {
+            PyObject *res = POP();
+            PyObject *exc = POP();
+            int err;
+
             if (exc != Py_None)
                 err = PyObject_IsTrue(res);
             else
                 err = 0;
+
             Py_DECREF(res);
 
             if (err < 0)
@@ -3751,6 +3933,9 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals,
     }
 
     if (co->co_flags & CO_GENERATOR) {
+        PyObject *gen;
+        PyObject *coroutine_wrapper;
+
         /* Don't need to keep the reference to f_back, it will be set
          * when the generator is resumed. */
         Py_CLEAR(f->f_back);
@@ -3759,7 +3944,19 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals,
 
         /* Create a new generator that owns the ready to run frame
          * and return that as the value. */
-        return PyGen_NewWithQualName(f, name, qualname);
+        gen = PyGen_NewWithQualName(f, name, qualname);
+        if (gen == NULL)
+            return NULL;
+
+        if (co->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE)) {
+            coroutine_wrapper = PyEval_GetCoroutineWrapper();
+            if (coroutine_wrapper != NULL) {
+                PyObject *wrapped =
+                            PyObject_CallFunction(coroutine_wrapper, "N", gen);
+                gen = wrapped;
+            }
+        }
+        return gen;
     }
 
     retval = PyEval_EvalFrameEx(f,0);
@@ -4205,6 +4402,24 @@ PyEval_SetTrace(Py_tracefunc func, PyObject *arg)
                            || (tstate->c_profilefunc != NULL));
 }
 
+void
+PyEval_SetCoroutineWrapper(PyObject *wrapper)
+{
+    PyThreadState *tstate = PyThreadState_GET();
+
+    Py_CLEAR(tstate->coroutine_wrapper);
+
+    Py_XINCREF(wrapper);
+    tstate->coroutine_wrapper = wrapper;
+}
+
+PyObject *
+PyEval_GetCoroutineWrapper()
+{
+    PyThreadState *tstate = PyThreadState_GET();
+    return tstate->coroutine_wrapper;
+}
+
 PyObject *
 PyEval_GetBuiltins(void)
 {

Python/compile.c

@@ -92,6 +92,7 @@ enum {
     COMPILER_SCOPE_MODULE,
     COMPILER_SCOPE_CLASS,
     COMPILER_SCOPE_FUNCTION,
+    COMPILER_SCOPE_ASYNC_FUNCTION,
     COMPILER_SCOPE_LAMBDA,
     COMPILER_SCOPE_COMPREHENSION,
 };
@@ -193,6 +194,8 @@ static int inplace_binop(struct compiler *, operator_ty);
 static int expr_constant(struct compiler *, expr_ty);
 
 static int compiler_with(struct compiler *, stmt_ty, int);
+static int compiler_async_with(struct compiler *, stmt_ty, int);
+static int compiler_async_for(struct compiler *, stmt_ty);
 static int compiler_call_helper(struct compiler *c, Py_ssize_t n,
                                 asdl_seq *args,
                                 asdl_seq *keywords);
@@ -673,7 +676,9 @@ compiler_set_qualname(struct compiler *c)
         parent = (struct compiler_unit *)PyCapsule_GetPointer(capsule, CAPSULE_NAME);
         assert(parent);
 
-        if (u->u_scope_type == COMPILER_SCOPE_FUNCTION || u->u_scope_type == COMPILER_SCOPE_CLASS) {
+        if (u->u_scope_type == COMPILER_SCOPE_FUNCTION
+            || u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION
+            || u->u_scope_type == COMPILER_SCOPE_CLASS) {
             assert(u->u_name);
             mangled = _Py_Mangle(parent->u_private, u->u_name);
             if (!mangled)
@@ -687,6 +692,7 @@ compiler_set_qualname(struct compiler *c)
 
         if (!force_global) {
             if (parent->u_scope_type == COMPILER_SCOPE_FUNCTION
+                || parent->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION
                 || parent->u_scope_type == COMPILER_SCOPE_LAMBDA) {
                 dot_locals_str = _PyUnicode_FromId(&dot_locals);
                 if (dot_locals_str == NULL)
@@ -927,7 +933,9 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg)
             return 0;
         case SETUP_WITH:
             return 7;
-        case WITH_CLEANUP:
+        case WITH_CLEANUP_START:
+            return 1;
+        case WITH_CLEANUP_FINISH:
             return -1; /* XXX Sometimes more */
         case RETURN_VALUE:
             return -1;
@@ -1048,6 +1056,16 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg)
             return -1;
         case DELETE_DEREF:
             return 0;
+        case GET_AWAITABLE:
+            return 0;
+        case SETUP_ASYNC_WITH:
+            return 6;
+        case BEFORE_ASYNC_WITH:
+            return 1;
+        case GET_AITER:
+            return 0;
+        case GET_ANEXT:
+            return 1;
         default:
             return PY_INVALID_STACK_EFFECT;
     }
@@ -1642,19 +1660,43 @@ error:
 }
 
 static int
-compiler_function(struct compiler *c, stmt_ty s)
+compiler_function(struct compiler *c, stmt_ty s, int is_async)
 {
     PyCodeObject *co;
     PyObject *qualname, *first_const = Py_None;
-    arguments_ty args = s->v.FunctionDef.args;
+    arguments_ty args;
-    expr_ty returns = s->v.FunctionDef.returns;
+    expr_ty returns;
-    asdl_seq* decos = s->v.FunctionDef.decorator_list;
+    identifier name;
+    asdl_seq* decos;
+    asdl_seq *body;
     stmt_ty st;
     Py_ssize_t i, n, arglength;
     int docstring, kw_default_count = 0;
     int num_annotations;
+    int scope_type;
 
-    assert(s->kind == FunctionDef_kind);
+
+    if (is_async) {
+        assert(s->kind == AsyncFunctionDef_kind);
+
+        args = s->v.AsyncFunctionDef.args;
+        returns = s->v.AsyncFunctionDef.returns;
+        decos = s->v.AsyncFunctionDef.decorator_list;
+        name = s->v.AsyncFunctionDef.name;
+        body = s->v.AsyncFunctionDef.body;
+
+        scope_type = COMPILER_SCOPE_ASYNC_FUNCTION;
+    } else {
+        assert(s->kind == FunctionDef_kind);
+
+        args = s->v.FunctionDef.args;
+        returns = s->v.FunctionDef.returns;
+        decos = s->v.FunctionDef.decorator_list;
+        name = s->v.FunctionDef.name;
+        body = s->v.FunctionDef.body;
+
+        scope_type = COMPILER_SCOPE_FUNCTION;
+    }
 
     if (!compiler_decorators(c, decos))
         return 0;
@@ -1672,12 +1714,12 @@ compiler_function(struct compiler *c, stmt_ty s)
         return 0;
     assert((num_annotations & 0xFFFF) == num_annotations);
 
-    if (!compiler_enter_scope(c, s->v.FunctionDef.name,
+    if (!compiler_enter_scope(c, name,
-                              COMPILER_SCOPE_FUNCTION, (void *)s,
+                              scope_type, (void *)s,
                               s->lineno))
         return 0;
 
-    st = (stmt_ty)asdl_seq_GET(s->v.FunctionDef.body, 0);
+    st = (stmt_ty)asdl_seq_GET(body, 0);
     docstring = compiler_isdocstring(st);
     if (docstring && c->c_optimize < 2)
         first_const = st->v.Expr.value->v.Str.s;
@@ -1688,10 +1730,10 @@ compiler_function(struct compiler *c, stmt_ty s)
 
     c->u->u_argcount = asdl_seq_LEN(args->args);
     c->u->u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs);
-    n = asdl_seq_LEN(s->v.FunctionDef.body);
+    n = asdl_seq_LEN(body);
     /* if there was a docstring, we need to skip the first statement */
     for (i = docstring; i < n; i++) {
-        st = (stmt_ty)asdl_seq_GET(s->v.FunctionDef.body, i);
+        st = (stmt_ty)asdl_seq_GET(body, i);
         VISIT_IN_SCOPE(c, stmt, st);
     }
     co = assemble(c, 1);
@@ -1711,12 +1753,19 @@ compiler_function(struct compiler *c, stmt_ty s)
     Py_DECREF(qualname);
     Py_DECREF(co);
 
+    if (is_async) {
+        co->co_flags |= CO_COROUTINE;
+        /* An async function is always a generator, even
+           if there is no 'yield' expressions in it. */
+        co->co_flags |= CO_GENERATOR;
+    }
+
     /* decorators */
     for (i = 0; i < asdl_seq_LEN(decos); i++) {
         ADDOP_I(c, CALL_FUNCTION, 1);
     }
 
-    return compiler_nameop(c, s->v.FunctionDef.name, Store);
+    return compiler_nameop(c, name, Store);
 }
 
 static int
@@ -1989,6 +2038,92 @@ compiler_for(struct compiler *c, stmt_ty s)
     return 1;
 }
 
+
+static int
+compiler_async_for(struct compiler *c, stmt_ty s)
+{
+    static PyObject *stopiter_error = NULL;
+    basicblock *try, *except, *end, *after_try, *try_cleanup,
+               *after_loop, *after_loop_else;
+
+    if (stopiter_error == NULL) {
+        stopiter_error = PyUnicode_InternFromString("StopAsyncIteration");
+        if (stopiter_error == NULL)
+            return 0;
+    }
+
+    try = compiler_new_block(c);
+    except = compiler_new_block(c);
+    end = compiler_new_block(c);
+    after_try = compiler_new_block(c);
+    try_cleanup = compiler_new_block(c);
+    after_loop = compiler_new_block(c);
+    after_loop_else = compiler_new_block(c);
+
+    if (try == NULL || except == NULL || end == NULL
+            || after_try == NULL || try_cleanup == NULL)
+        return 0;
+
+    ADDOP_JREL(c, SETUP_LOOP, after_loop);
+    if (!compiler_push_fblock(c, LOOP, try))
+        return 0;
+
+    VISIT(c, expr, s->v.AsyncFor.iter);
+    ADDOP(c, GET_AITER);
+    ADDOP_O(c, LOAD_CONST, Py_None, consts);
+    ADDOP(c, YIELD_FROM);
+
+    compiler_use_next_block(c, try);
+
+
+    ADDOP_JREL(c, SETUP_EXCEPT, except);
+    if (!compiler_push_fblock(c, EXCEPT, try))
+        return 0;
+
+    ADDOP(c, GET_ANEXT);
+    ADDOP_O(c, LOAD_CONST, Py_None, consts);
+    ADDOP(c, YIELD_FROM);
+    VISIT(c, expr, s->v.AsyncFor.target);
+    ADDOP(c, POP_BLOCK);
+    compiler_pop_fblock(c, EXCEPT, try);
+    ADDOP_JREL(c, JUMP_FORWARD, after_try);
+
+
+    compiler_use_next_block(c, except);
+    ADDOP(c, DUP_TOP);
+    ADDOP_O(c, LOAD_GLOBAL, stopiter_error, names);
+    ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH);
+    ADDOP_JABS(c, POP_JUMP_IF_FALSE, try_cleanup);
+
+    ADDOP(c, POP_TOP);
+    ADDOP(c, POP_TOP);
+    ADDOP(c, POP_TOP);
+    ADDOP(c, POP_EXCEPT); /* for SETUP_EXCEPT */
+    ADDOP(c, POP_BLOCK); /* for SETUP_LOOP */
+    ADDOP_JABS(c, JUMP_ABSOLUTE, after_loop_else);
+
+
+    compiler_use_next_block(c, try_cleanup);
+    ADDOP(c, END_FINALLY);
+
+    compiler_use_next_block(c, after_try);
+    VISIT_SEQ(c, stmt, s->v.AsyncFor.body);
+    ADDOP_JABS(c, JUMP_ABSOLUTE, try);
+
+    ADDOP(c, POP_BLOCK); /* for SETUP_LOOP */
+    compiler_pop_fblock(c, LOOP, try);
+
+    compiler_use_next_block(c, after_loop);
+    ADDOP_JABS(c, JUMP_ABSOLUTE, end);
+
+    compiler_use_next_block(c, after_loop_else);
+    VISIT_SEQ(c, stmt, s->v.For.orelse);
+
+    compiler_use_next_block(c, end);
+
+    return 1;
+}
+
 static int
 compiler_while(struct compiler *c, stmt_ty s)
 {
@@ -2515,7 +2650,7 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s)
 
     switch (s->kind) {
     case FunctionDef_kind:
-        return compiler_function(c, s);
+        return compiler_function(c, s, 0);
     case ClassDef_kind:
         return compiler_class(c, s);
     case Return_kind:
@@ -2594,7 +2729,14 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s)
         return compiler_continue(c);
     case With_kind:
         return compiler_with(c, s, 0);
+    case AsyncFunctionDef_kind:
+        return compiler_function(c, s, 1);
+    case AsyncWith_kind:
+        return compiler_async_with(c, s, 0);
+    case AsyncFor_kind:
+        return compiler_async_for(c, s);
     }
+
     return 1;
 }
 
@@ -3471,6 +3613,102 @@ expr_constant(struct compiler *c, expr_ty e)
     }
 }
 
+
+/*
+   Implements the async with statement.
+
+   The semantics outlined in that PEP are as follows:
+
+   async with EXPR as VAR:
+       BLOCK
+
+   It is implemented roughly as:
+
+   context = EXPR
+   exit = context.__aexit__  # not calling it
+   value = await context.__aenter__()
+   try:
+       VAR = value  # if VAR present in the syntax
+       BLOCK
+   finally:
+       if an exception was raised:
+       exc = copy of (exception, instance, traceback)
+       else:
+       exc = (None, None, None)
+       if not (await exit(*exc)):
+           raise
+ */
+static int
+compiler_async_with(struct compiler *c, stmt_ty s, int pos)
+{
+    basicblock *block, *finally;
+    withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos);
+
+    assert(s->kind == AsyncWith_kind);
+
+    block = compiler_new_block(c);
+    finally = compiler_new_block(c);
+    if (!block || !finally)
+        return 0;
+
+    /* Evaluate EXPR */
+    VISIT(c, expr, item->context_expr);
+
+    ADDOP(c, BEFORE_ASYNC_WITH);
+    ADDOP(c, GET_AWAITABLE);
+    ADDOP_O(c, LOAD_CONST, Py_None, consts);
+    ADDOP(c, YIELD_FROM);
+
+    ADDOP_JREL(c, SETUP_ASYNC_WITH, finally);
+
+    /* SETUP_ASYNC_WITH pushes a finally block. */
+    compiler_use_next_block(c, block);
+    if (!compiler_push_fblock(c, FINALLY_TRY, block)) {
+        return 0;
+    }
+
+    if (item->optional_vars) {
+        VISIT(c, expr, item->optional_vars);
+    }
+    else {
+    /* Discard result from context.__aenter__() */
+        ADDOP(c, POP_TOP);
+    }
+
+    pos++;
+    if (pos == asdl_seq_LEN(s->v.AsyncWith.items))
+        /* BLOCK code */
+        VISIT_SEQ(c, stmt, s->v.AsyncWith.body)
+    else if (!compiler_async_with(c, s, pos))
+            return 0;
+
+    /* End of try block; start the finally block */
+    ADDOP(c, POP_BLOCK);
+    compiler_pop_fblock(c, FINALLY_TRY, block);
+
+    ADDOP_O(c, LOAD_CONST, Py_None, consts);
+    compiler_use_next_block(c, finally);
+    if (!compiler_push_fblock(c, FINALLY_END, finally))
+        return 0;
+
+    /* Finally block starts; context.__exit__ is on the stack under
+       the exception or return information. Just issue our magic
+       opcode. */
+    ADDOP(c, WITH_CLEANUP_START);
+
+    ADDOP(c, GET_AWAITABLE);
+    ADDOP_O(c, LOAD_CONST, Py_None, consts);
+    ADDOP(c, YIELD_FROM);
+
+    ADDOP(c, WITH_CLEANUP_FINISH);
+
+    /* Finally block ends. */
+    ADDOP(c, END_FINALLY);
+    compiler_pop_fblock(c, FINALLY_END, finally);
+    return 1;
+}
+
+
 /*
    Implements the with statement from PEP 343.
 
@@ -3544,7 +3782,8 @@ compiler_with(struct compiler *c, stmt_ty s, int pos)
     /* Finally block starts; context.__exit__ is on the stack under
        the exception or return information. Just issue our magic
        opcode. */
-    ADDOP(c, WITH_CLEANUP);
+    ADDOP(c, WITH_CLEANUP_START);
+    ADDOP(c, WITH_CLEANUP_FINISH);
 
     /* Finally block ends. */
     ADDOP(c, END_FINALLY);
@@ -3595,6 +3834,8 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
     case Yield_kind:
         if (c->u->u_ste->ste_type != FunctionBlock)
             return compiler_error(c, "'yield' outside function");
+        if (c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION)
+            return compiler_error(c, "'yield' inside async function");
         if (e->v.Yield.value) {
             VISIT(c, expr, e->v.Yield.value);
         }
@@ -3606,11 +3847,28 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
     case YieldFrom_kind:
         if (c->u->u_ste->ste_type != FunctionBlock)
             return compiler_error(c, "'yield' outside function");
+
+        if (c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION)
+            return compiler_error(c, "'yield from' inside async function");
+
         VISIT(c, expr, e->v.YieldFrom.value);
         ADDOP(c, GET_ITER);
         ADDOP_O(c, LOAD_CONST, Py_None, consts);
         ADDOP(c, YIELD_FROM);
         break;
+    case Await_kind:
+        if (c->u->u_ste->ste_type != FunctionBlock)
+            return compiler_error(c, "'await' outside function");
+
+        /* this check won't be triggered while we have AWAIT token */
+        if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION)
+            return compiler_error(c, "'await' outside async function");
+
+        VISIT(c, expr, e->v.Await.value);
+        ADDOP(c, GET_AWAITABLE);
+        ADDOP_O(c, LOAD_CONST, Py_None, consts);
+        ADDOP(c, YIELD_FROM);
+        break;
     case Compare_kind:
         return compiler_compare(c, e);
     case Call_kind:

Python/opcode_targets.h

@@ -49,9 +49,9 @@ static void *opcode_targets[256] = {
     &&_unknown_opcode,
     &&_unknown_opcode,
     &&_unknown_opcode,
-    &&_unknown_opcode,
+    &&TARGET_GET_AITER,
-    &&_unknown_opcode,
+    &&TARGET_GET_ANEXT,
-    &&_unknown_opcode,
+    &&TARGET_BEFORE_ASYNC_WITH,
     &&_unknown_opcode,
     &&TARGET_STORE_MAP,
     &&TARGET_INPLACE_ADD,
@@ -72,7 +72,7 @@ static void *opcode_targets[256] = {
     &&TARGET_PRINT_EXPR,
     &&TARGET_LOAD_BUILD_CLASS,
     &&TARGET_YIELD_FROM,
-    &&_unknown_opcode,
+    &&TARGET_GET_AWAITABLE,
     &&_unknown_opcode,
     &&TARGET_INPLACE_LSHIFT,
     &&TARGET_INPLACE_RSHIFT,
@@ -80,8 +80,8 @@ static void *opcode_targets[256] = {
     &&TARGET_INPLACE_XOR,
     &&TARGET_INPLACE_OR,
     &&TARGET_BREAK_LOOP,
-    &&TARGET_WITH_CLEANUP,
+    &&TARGET_WITH_CLEANUP_START,
-    &&_unknown_opcode,
+    &&TARGET_WITH_CLEANUP_FINISH,
     &&TARGET_RETURN_VALUE,
     &&TARGET_IMPORT_STAR,
     &&_unknown_opcode,
@@ -153,7 +153,7 @@ static void *opcode_targets[256] = {
     &&TARGET_BUILD_MAP_UNPACK_WITH_CALL,
     &&TARGET_BUILD_TUPLE_UNPACK,
     &&TARGET_BUILD_SET_UNPACK,
-    &&_unknown_opcode,
+    &&TARGET_SETUP_ASYNC_WITH,
     &&_unknown_opcode,
     &&_unknown_opcode,
     &&_unknown_opcode,

Python/peephole.c

@@ -319,6 +319,7 @@ markblocks(unsigned char *code, Py_ssize_t len)
             case SETUP_EXCEPT:
             case SETUP_FINALLY:
             case SETUP_WITH:
+            case SETUP_ASYNC_WITH:
                 j = GETJUMPTGT(code, i);
                 blocks[j] = 1;
                 break;
@@ -620,6 +621,7 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names,
             case SETUP_EXCEPT:
             case SETUP_FINALLY:
             case SETUP_WITH:
+            case SETUP_ASYNC_WITH:
                 tgt = GETJUMPTGT(codestr, i);
                 /* Replace JUMP_* to a RETURN into just a RETURN */
                 if (UNCONDITIONAL_JUMP(opcode) &&
@@ -704,6 +706,7 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names,
             case SETUP_EXCEPT:
             case SETUP_FINALLY:
             case SETUP_WITH:
+            case SETUP_ASYNC_WITH:
                 j = addrmap[GETARG(codestr, i) + i + 3] - addrmap[i] - 3;
                 SETARG(codestr, i, j);
                 break;

Python/pystate.c

@@ -212,6 +212,8 @@ new_threadstate(PyInterpreterState *interp, int init)
         tstate->on_delete = NULL;
         tstate->on_delete_data = NULL;
 
+        tstate->coroutine_wrapper = NULL;
+
         if (init)
             _PyThreadState_Init(tstate);
 
@@ -372,6 +374,8 @@ PyThreadState_Clear(PyThreadState *tstate)
     tstate->c_tracefunc = NULL;
     Py_CLEAR(tstate->c_profileobj);
     Py_CLEAR(tstate->c_traceobj);
+
+    Py_CLEAR(tstate->coroutine_wrapper);
 }
 
 

Python/symtable.c

@@ -180,7 +180,7 @@ static int symtable_visit_slice(struct symtable *st, slice_ty);
 static int symtable_visit_params(struct symtable *st, asdl_seq *args);
 static int symtable_visit_argannotations(struct symtable *st, asdl_seq *args);
 static int symtable_implicit_arg(struct symtable *st, int pos);
-static int symtable_visit_annotations(struct symtable *st, stmt_ty s);
+static int symtable_visit_annotations(struct symtable *st, stmt_ty s, arguments_ty, expr_ty);
 static int symtable_visit_withitem(struct symtable *st, withitem_ty item);
 
 
@@ -1147,7 +1147,8 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
             VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults);
         if (s->v.FunctionDef.args->kw_defaults)
             VISIT_SEQ_WITH_NULL(st, expr, s->v.FunctionDef.args->kw_defaults);
-        if (!symtable_visit_annotations(st, s))
+        if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args,
+                                        s->v.FunctionDef.returns))
             VISIT_QUIT(st, 0);
         if (s->v.FunctionDef.decorator_list)
             VISIT_SEQ(st, expr, s->v.FunctionDef.decorator_list);
@@ -1315,6 +1316,39 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
         VISIT_SEQ(st, withitem, s->v.With.items);
         VISIT_SEQ(st, stmt, s->v.With.body);
         break;
+    case AsyncFunctionDef_kind:
+        if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL))
+            VISIT_QUIT(st, 0);
+        if (s->v.AsyncFunctionDef.args->defaults)
+            VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.args->defaults);
+        if (s->v.AsyncFunctionDef.args->kw_defaults)
+            VISIT_SEQ_WITH_NULL(st, expr,
+                                s->v.AsyncFunctionDef.args->kw_defaults);
+        if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args,
+                                        s->v.AsyncFunctionDef.returns))
+            VISIT_QUIT(st, 0);
+        if (s->v.AsyncFunctionDef.decorator_list)
+            VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.decorator_list);
+        if (!symtable_enter_block(st, s->v.AsyncFunctionDef.name,
+                                  FunctionBlock, (void *)s, s->lineno,
+                                  s->col_offset))
+            VISIT_QUIT(st, 0);
+        VISIT(st, arguments, s->v.AsyncFunctionDef.args);
+        VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body);
+        if (!symtable_exit_block(st, s))
+            VISIT_QUIT(st, 0);
+        break;
+    case AsyncWith_kind:
+        VISIT_SEQ(st, withitem, s->v.AsyncWith.items);
+        VISIT_SEQ(st, stmt, s->v.AsyncWith.body);
+        break;
+    case AsyncFor_kind:
+        VISIT(st, expr, s->v.AsyncFor.target);
+        VISIT(st, expr, s->v.AsyncFor.iter);
+        VISIT_SEQ(st, stmt, s->v.AsyncFor.body);
+        if (s->v.AsyncFor.orelse)
+            VISIT_SEQ(st, stmt, s->v.AsyncFor.orelse);
+        break;
     }
     VISIT_QUIT(st, 1);
 }
@@ -1392,6 +1426,10 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
         VISIT(st, expr, e->v.YieldFrom.value);
         st->st_cur->ste_generator = 1;
         break;
+    case Await_kind:
+        VISIT(st, expr, e->v.Await.value);
+        st->st_cur->ste_generator = 1;
+        break;
     case Compare_kind:
         VISIT(st, expr, e->v.Compare.left);
         VISIT_SEQ(st, expr, e->v.Compare.comparators);
@@ -1492,10 +1530,9 @@ symtable_visit_argannotations(struct symtable *st, asdl_seq *args)
 }
 
 static int
-symtable_visit_annotations(struct symtable *st, stmt_ty s)
+symtable_visit_annotations(struct symtable *st, stmt_ty s,
+                           arguments_ty a, expr_ty returns)
 {
-    arguments_ty a = s->v.FunctionDef.args;
-
     if (a->args && !symtable_visit_argannotations(st, a->args))
         return 0;
     if (a->vararg && a->vararg->annotation)
@@ -1504,7 +1541,7 @@ symtable_visit_annotations(struct symtable *st, stmt_ty s)
         VISIT(st, expr, a->kwarg->annotation);
     if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs))
         return 0;
-    if (s->v.FunctionDef.returns)
+    if (returns)
         VISIT(st, expr, s->v.FunctionDef.returns);
     return 1;
 }

Python/sysmodule.c

@@ -645,6 +645,49 @@ sys_setrecursionlimit(PyObject *self, PyObject *args)
     return Py_None;
 }
 
+static PyObject *
+sys_set_coroutine_wrapper(PyObject *self, PyObject *wrapper)
+{
+    if (wrapper != Py_None) {
+        if (!PyCallable_Check(wrapper)) {
+            PyErr_Format(PyExc_TypeError,
+                         "callable expected, got %.50s",
+                         Py_TYPE(wrapper)->tp_name);
+            return NULL;
+        }
+
+        PyEval_SetCoroutineWrapper(wrapper);
+    }
+    else
+        PyEval_SetCoroutineWrapper(NULL);
+    Py_INCREF(Py_None);
+    Py_RETURN_NONE;
+}
+
+PyDoc_STRVAR(set_coroutine_wrapper_doc,
+"set_coroutine_wrapper(wrapper)\n\
+\n\
+Set a wrapper for coroutine objects."
+);
+
+static PyObject *
+sys_get_coroutine_wrapper(PyObject *self, PyObject *args)
+{
+    PyObject *wrapper = PyEval_GetCoroutineWrapper();
+    if (wrapper == NULL) {
+        wrapper = Py_None;
+    }
+    Py_INCREF(wrapper);
+    return wrapper;
+}
+
+PyDoc_STRVAR(get_coroutine_wrapper_doc,
+"get_coroutine_wrapper()\n\
+\n\
+Return the wrapper for coroutine objects set by sys.set_coroutine_wrapper."
+);
+
+
 static PyTypeObject Hash_InfoType;
 
 PyDoc_STRVAR(hash_info_doc,
@@ -1215,6 +1258,10 @@ static PyMethodDef sys_methods[] = {
     {"call_tracing", sys_call_tracing, METH_VARARGS, call_tracing_doc},
     {"_debugmallocstats", sys_debugmallocstats, METH_NOARGS,
      debugmallocstats_doc},
+    {"set_coroutine_wrapper", sys_set_coroutine_wrapper, METH_O,
+     set_coroutine_wrapper_doc},
+    {"get_coroutine_wrapper", sys_get_coroutine_wrapper, METH_NOARGS,
+     get_coroutine_wrapper_doc},
     {NULL,              NULL}           /* sentinel */
 };
 

Tools/parser/unparse.py

@@ -138,6 +138,14 @@ class Unparser:
         self.fill("nonlocal ")
         interleave(lambda: self.write(", "), self.write, t.names)
 
+    def _Await(self, t):
+        self.write("(")
+        self.write("await")
+        if t.value:
+            self.write(" ")
+            self.dispatch(t.value)
+        self.write(")")
+
     def _Yield(self, t):
         self.write("(")
         self.write("yield")
@@ -218,11 +226,18 @@ class Unparser:
         self.leave()
 
     def _FunctionDef(self, t):
+        self.__FunctionDef_helper(t, "def")
+
+    def _AsyncFunctionDef(self, t):
+        self.__FunctionDef_helper(t, "async def")
+
+    def __FunctionDef_helper(self, t, fill_suffix):
         self.write("\n")
         for deco in t.decorator_list:
             self.fill("@")
             self.dispatch(deco)
-        self.fill("def "+t.name + "(")
+        def_str = fill_suffix+" "+t.name + "("
+        self.fill(def_str)
         self.dispatch(t.args)
         self.write(")")
         if t.returns:
@@ -233,7 +248,13 @@ class Unparser:
         self.leave()
 
     def _For(self, t):
-        self.fill("for ")
+        self.__For_helper("for ", t)
+
+    def _AsyncFor(self, t):
+        self.__For_helper("async for ", t)
+
+    def __For_helper(self, fill, t):
+        self.fill(fill)
         self.dispatch(t.target)
         self.write(" in ")
         self.dispatch(t.iter)
@@ -287,6 +308,13 @@ class Unparser:
         self.dispatch(t.body)
         self.leave()
 
+    def _AsyncWith(self, t):
+        self.fill("async with ")
+        interleave(lambda: self.write(", "), self.dispatch, t.items)
+        self.enter()
+        self.dispatch(t.body)
+        self.leave()
+
     # expr
     def _Bytes(self, t):
         self.write(repr(t.s))