traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-39220: Do not optimise annotation if 'from __future__ import annotations' is used (GH-17866) 

Do not apply AST-based optimizations if 'from __future__ import annotations' is used in order to
prevent information lost in the final version of the annotations.
This commit is contained in:
Pablo Galindo 2020-03-18 23:02:09 +00:00 committed by GitHub
parent 8849e5962b
commit d112c600ab
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 67 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Include/compile.h
View File

@ -83,7 +83,12 @@ PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name);
PyAPI_FUNC(int) PyCompile_OpcodeStackEffect(int opcode, int oparg); PyAPI_FUNC(int) PyCompile_OpcodeStackEffect(int opcode, int oparg);
PyAPI_FUNC(int) PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump); PyAPI_FUNC(int) PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump);
PyAPI_FUNC(int) _PyAST_Optimize(struct _mod *, PyArena *arena, int optimize); typedef struct {
int optimize;
int ff_features;
} _PyASTOptimizeState;
PyAPI_FUNC(int) _PyAST_Optimize(struct _mod *, PyArena *arena, _PyASTOptimizeState *state);
#ifdef __cplusplus #ifdef __cplusplus
} }

10
Lib/test/test_future.py
View File

@ -111,6 +111,10 @@ class AnnotationsFutureTestCase(unittest.TestCase):
... ...
def g(arg: {ann}) -> None: def g(arg: {ann}) -> None:
... ...
async def f2() -> {ann}:
...
async def g2(arg: {ann}) -> None:
...
var: {ann} var: {ann}
var2: {ann} = None var2: {ann} = None
""" """
@ -121,9 +125,13 @@ class AnnotationsFutureTestCase(unittest.TestCase):
exec(self.template.format(ann=annotation), {}, scope) exec(self.template.format(ann=annotation), {}, scope)
func_ret_ann = scope['f'].__annotations__['return'] func_ret_ann = scope['f'].__annotations__['return']
func_arg_ann = scope['g'].__annotations__['arg'] func_arg_ann = scope['g'].__annotations__['arg']
async_func_ret_ann = scope['f2'].__annotations__['return']
async_func_arg_ann = scope['g2'].__annotations__['arg']
var_ann1 = scope['__annotations__']['var'] var_ann1 = scope['__annotations__']['var']
var_ann2 = scope['__annotations__']['var2'] var_ann2 = scope['__annotations__']['var2']
self.assertEqual(func_ret_ann, func_arg_ann) self.assertEqual(func_ret_ann, func_arg_ann)
self.assertEqual(func_ret_ann, async_func_ret_ann)
self.assertEqual(func_ret_ann, async_func_arg_ann)
self.assertEqual(func_ret_ann, var_ann1) self.assertEqual(func_ret_ann, var_ann1)
self.assertEqual(func_ret_ann, var_ann2) self.assertEqual(func_ret_ann, var_ann2)
return func_ret_ann return func_ret_ann
@ -288,6 +296,7 @@ class AnnotationsFutureTestCase(unittest.TestCase):
eq('(((a, b)))', '(a, b)') eq('(((a, b)))', '(a, b)')
eq("(x:=10)") eq("(x:=10)")
eq("f'{(x:=10):=10}'") eq("f'{(x:=10):=10}'")
eq("1 + 2 + 3")
def test_fstring_debug_annotations(self): def test_fstring_debug_annotations(self):
# f-strings with '=' don't round trip very well, so set the expected # f-strings with '=' don't round trip very well, so set the expected
@ -299,6 +308,5 @@ class AnnotationsFutureTestCase(unittest.TestCase):
self.assertAnnotationEqual("f'{x=!a}'", expected="f'x={x!a}'") self.assertAnnotationEqual("f'{x=!a}'", expected="f'x={x!a}'")
self.assertAnnotationEqual("f'{x=!s:*^20}'", expected="f'x={x!s:*^20}'") self.assertAnnotationEqual("f'{x=!s:*^20}'", expected="f'x={x!s:*^20}'")
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()

2
Misc/NEWS.d/next/Core and Builtins/2020-01-06-13-58-37.bpo-39220.KGFovE.rst Normal file
View File

@ -0,0 +1,2 @@
Do not optimize annotations if 'from __future__ import annotations' is used.
Patch by Pablo Galindo.

82
Python/ast_opt.c
View File

@ -35,7 +35,7 @@ unary_not(PyObject *v)
} }
static int static int
fold_unaryop(expr_ty node, PyArena *arena, int optimize) fold_unaryop(expr_ty node, PyArena *arena, _PyASTOptimizeState *state)
{ {
expr_ty arg = node->v.UnaryOp.operand; expr_ty arg = node->v.UnaryOp.operand;
@ -212,7 +212,7 @@ safe_mod(PyObject *v, PyObject *w)
} }
static int static int
fold_binop(expr_ty node, PyArena *arena, int optimize) fold_binop(expr_ty node, PyArena *arena, _PyASTOptimizeState *state)
{ {
expr_ty lhs, rhs; expr_ty lhs, rhs;
lhs = node->v.BinOp.left; lhs = node->v.BinOp.left;
@ -294,7 +294,7 @@ make_const_tuple(asdl_seq *elts)
} }
static int static int
fold_tuple(expr_ty node, PyArena *arena, int optimize) fold_tuple(expr_ty node, PyArena *arena, _PyASTOptimizeState *state)
{ {
PyObject *newval; PyObject *newval;
@ -306,7 +306,7 @@ fold_tuple(expr_ty node, PyArena *arena, int optimize)
} }
static int static int
fold_subscr(expr_ty node, PyArena *arena, int optimize) fold_subscr(expr_ty node, PyArena *arena, _PyASTOptimizeState *state)
{ {
PyObject *newval; PyObject *newval;
expr_ty arg, idx; expr_ty arg, idx;
@ -331,7 +331,7 @@ fold_subscr(expr_ty node, PyArena *arena, int optimize)
in "for" loop and comprehensions. in "for" loop and comprehensions.
*/ */
static int static int
fold_iter(expr_ty arg, PyArena *arena, int optimize) fold_iter(expr_ty arg, PyArena *arena, _PyASTOptimizeState *state)
{ {
PyObject *newval; PyObject *newval;
if (arg->kind == List_kind) { if (arg->kind == List_kind) {
@ -364,7 +364,7 @@ fold_iter(expr_ty arg, PyArena *arena, int optimize)
} }
static int static int
fold_compare(expr_ty node, PyArena *arena, int optimize) fold_compare(expr_ty node, PyArena *arena, _PyASTOptimizeState *state)
{ {
asdl_int_seq *ops; asdl_int_seq *ops;
asdl_seq *args; asdl_seq *args;
@ -378,28 +378,28 @@ fold_compare(expr_ty node, PyArena *arena, int optimize)
i = asdl_seq_LEN(ops) - 1; i = asdl_seq_LEN(ops) - 1;
int op = asdl_seq_GET(ops, i); int op = asdl_seq_GET(ops, i);
if (op == In || op == NotIn) { if (op == In || op == NotIn) {
if (!fold_iter((expr_ty)asdl_seq_GET(args, i), arena, optimize)) { if (!fold_iter((expr_ty)asdl_seq_GET(args, i), arena, state)) {
return 0; return 0;
} }
} }
return 1; return 1;
} }
static int astfold_mod(mod_ty node_, PyArena *ctx_, int optimize_); static int astfold_mod(mod_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
static int astfold_stmt(stmt_ty node_, PyArena *ctx_, int optimize_); static int astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
static int astfold_expr(expr_ty node_, PyArena *ctx_, int optimize_); static int astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
static int astfold_arguments(arguments_ty node_, PyArena *ctx_, int optimize_); static int astfold_arguments(arguments_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
static int astfold_comprehension(comprehension_ty node_, PyArena *ctx_, int optimize_); static int astfold_comprehension(comprehension_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
static int astfold_keyword(keyword_ty node_, PyArena *ctx_, int optimize_); static int astfold_keyword(keyword_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
static int astfold_arg(arg_ty node_, PyArena *ctx_, int optimize_); static int astfold_arg(arg_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
static int astfold_withitem(withitem_ty node_, PyArena *ctx_, int optimize_); static int astfold_withitem(withitem_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
static int astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, int optimize_); static int astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
#define CALL(FUNC, TYPE, ARG) \ #define CALL(FUNC, TYPE, ARG) \
if (!FUNC((ARG), ctx_, optimize_)) \ if (!FUNC((ARG), ctx_, state)) \
return 0; return 0;
#define CALL_OPT(FUNC, TYPE, ARG) \ #define CALL_OPT(FUNC, TYPE, ARG) \
if ((ARG) != NULL && !FUNC((ARG), ctx_, optimize_)) \ if ((ARG) != NULL && !FUNC((ARG), ctx_, state)) \
return 0; return 0;
#define CALL_SEQ(FUNC, TYPE, ARG) { \ #define CALL_SEQ(FUNC, TYPE, ARG) { \
@ -407,7 +407,7 @@ static int astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, int opti
asdl_seq *seq = (ARG); /* avoid variable capture */ \ asdl_seq *seq = (ARG); /* avoid variable capture */ \
for (i = 0; i < asdl_seq_LEN(seq); i++) { \ for (i = 0; i < asdl_seq_LEN(seq); i++) { \
TYPE elt = (TYPE)asdl_seq_GET(seq, i); \ TYPE elt = (TYPE)asdl_seq_GET(seq, i); \
if (elt != NULL && !FUNC(elt, ctx_, optimize_)) \ if (elt != NULL && !FUNC(elt, ctx_, state)) \
return 0; \ return 0; \
} \ } \
} }
@ -417,13 +417,13 @@ static int astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, int opti
asdl_int_seq *seq = (ARG); /* avoid variable capture */ \ asdl_int_seq *seq = (ARG); /* avoid variable capture */ \
for (i = 0; i < asdl_seq_LEN(seq); i++) { \ for (i = 0; i < asdl_seq_LEN(seq); i++) { \
TYPE elt = (TYPE)asdl_seq_GET(seq, i); \ TYPE elt = (TYPE)asdl_seq_GET(seq, i); \
if (!FUNC(elt, ctx_, optimize_)) \ if (!FUNC(elt, ctx_, state)) \
return 0; \ return 0; \
} \ } \
} }
static int static int
astfold_body(asdl_seq *stmts, PyArena *ctx_, int optimize_) astfold_body(asdl_seq *stmts, PyArena *ctx_, _PyASTOptimizeState *state)
{ {
int docstring = _PyAST_GetDocString(stmts) != NULL; int docstring = _PyAST_GetDocString(stmts) != NULL;
CALL_SEQ(astfold_stmt, stmt_ty, stmts); CALL_SEQ(astfold_stmt, stmt_ty, stmts);
@ -445,7 +445,7 @@ astfold_body(asdl_seq *stmts, PyArena *ctx_, int optimize_)
} }
static int static int
astfold_mod(mod_ty node_, PyArena *ctx_, int optimize_) astfold_mod(mod_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
{ {
switch (node_->kind) { switch (node_->kind) {
case Module_kind: case Module_kind:
@ -464,7 +464,7 @@ astfold_mod(mod_ty node_, PyArena *ctx_, int optimize_)
} }
static int static int
astfold_expr(expr_ty node_, PyArena *ctx_, int optimize_) astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
{ {
switch (node_->kind) { switch (node_->kind) {
case BoolOp_kind: case BoolOp_kind:
@ -563,7 +563,7 @@ astfold_expr(expr_ty node_, PyArena *ctx_, int optimize_)
break; break;
case Name_kind: case Name_kind:
if (_PyUnicode_EqualToASCIIString(node_->v.Name.id, "__debug__")) { if (_PyUnicode_EqualToASCIIString(node_->v.Name.id, "__debug__")) {
return make_const(node_, PyBool_FromLong(!optimize_), ctx_); return make_const(node_, PyBool_FromLong(!state->optimize), ctx_);
} }
break; break;
default: default:
@ -573,14 +573,14 @@ astfold_expr(expr_ty node_, PyArena *ctx_, int optimize_)
} }
static int static int
astfold_keyword(keyword_ty node_, PyArena *ctx_, int optimize_) astfold_keyword(keyword_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
{ {
CALL(astfold_expr, expr_ty, node_->value); CALL(astfold_expr, expr_ty, node_->value);
return 1; return 1;
} }
static int static int
astfold_comprehension(comprehension_ty node_, PyArena *ctx_, int optimize_) astfold_comprehension(comprehension_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
{ {
CALL(astfold_expr, expr_ty, node_->target); CALL(astfold_expr, expr_ty, node_->target);
CALL(astfold_expr, expr_ty, node_->iter); CALL(astfold_expr, expr_ty, node_->iter);
@ -591,7 +591,7 @@ astfold_comprehension(comprehension_ty node_, PyArena *ctx_, int optimize_)
} }
static int static int
astfold_arguments(arguments_ty node_, PyArena *ctx_, int optimize_) astfold_arguments(arguments_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
{ {
CALL_SEQ(astfold_arg, arg_ty, node_->posonlyargs); CALL_SEQ(astfold_arg, arg_ty, node_->posonlyargs);
CALL_SEQ(astfold_arg, arg_ty, node_->args); CALL_SEQ(astfold_arg, arg_ty, node_->args);
@ -604,27 +604,33 @@ astfold_arguments(arguments_ty node_, PyArena *ctx_, int optimize_)
} }
static int static int
astfold_arg(arg_ty node_, PyArena *ctx_, int optimize_) astfold_arg(arg_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
{ {
CALL_OPT(astfold_expr, expr_ty, node_->annotation); if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) {
CALL_OPT(astfold_expr, expr_ty, node_->annotation);
}
return 1; return 1;
} }
static int static int
astfold_stmt(stmt_ty node_, PyArena *ctx_, int optimize_) astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
{ {
switch (node_->kind) { switch (node_->kind) {
case FunctionDef_kind: case FunctionDef_kind:
CALL(astfold_arguments, arguments_ty, node_->v.FunctionDef.args); CALL(astfold_arguments, arguments_ty, node_->v.FunctionDef.args);
CALL(astfold_body, asdl_seq, node_->v.FunctionDef.body); CALL(astfold_body, asdl_seq, node_->v.FunctionDef.body);
CALL_SEQ(astfold_expr, expr_ty, node_->v.FunctionDef.decorator_list); CALL_SEQ(astfold_expr, expr_ty, node_->v.FunctionDef.decorator_list);
CALL_OPT(astfold_expr, expr_ty, node_->v.FunctionDef.returns); if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) {
CALL_OPT(astfold_expr, expr_ty, node_->v.FunctionDef.returns);
}
break; break;
case AsyncFunctionDef_kind: case AsyncFunctionDef_kind:
CALL(astfold_arguments, arguments_ty, node_->v.AsyncFunctionDef.args); CALL(astfold_arguments, arguments_ty, node_->v.AsyncFunctionDef.args);
CALL(astfold_body, asdl_seq, node_->v.AsyncFunctionDef.body); CALL(astfold_body, asdl_seq, node_->v.AsyncFunctionDef.body);
CALL_SEQ(astfold_expr, expr_ty, node_->v.AsyncFunctionDef.decorator_list); CALL_SEQ(astfold_expr, expr_ty, node_->v.AsyncFunctionDef.decorator_list);
CALL_OPT(astfold_expr, expr_ty, node_->v.AsyncFunctionDef.returns); if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) {
CALL_OPT(astfold_expr, expr_ty, node_->v.AsyncFunctionDef.returns);
}
break; break;
case ClassDef_kind: case ClassDef_kind:
CALL_SEQ(astfold_expr, expr_ty, node_->v.ClassDef.bases); CALL_SEQ(astfold_expr, expr_ty, node_->v.ClassDef.bases);
@ -648,7 +654,9 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, int optimize_)
break; break;
case AnnAssign_kind: case AnnAssign_kind:
CALL(astfold_expr, expr_ty, node_->v.AnnAssign.target); CALL(astfold_expr, expr_ty, node_->v.AnnAssign.target);
CALL(astfold_expr, expr_ty, node_->v.AnnAssign.annotation); if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) {
CALL(astfold_expr, expr_ty, node_->v.AnnAssign.annotation);
}
CALL_OPT(astfold_expr, expr_ty, node_->v.AnnAssign.value); CALL_OPT(astfold_expr, expr_ty, node_->v.AnnAssign.value);
break; break;
case For_kind: case For_kind:
@ -707,7 +715,7 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, int optimize_)
} }
static int static int
astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, int optimize_) astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
{ {
switch (node_->kind) { switch (node_->kind) {
case ExceptHandler_kind: case ExceptHandler_kind:
@ -721,7 +729,7 @@ astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, int optimize_)
} }
static int static int
astfold_withitem(withitem_ty node_, PyArena *ctx_, int optimize_) astfold_withitem(withitem_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
{ {
CALL(astfold_expr, expr_ty, node_->context_expr); CALL(astfold_expr, expr_ty, node_->context_expr);
CALL_OPT(astfold_expr, expr_ty, node_->optional_vars); CALL_OPT(astfold_expr, expr_ty, node_->optional_vars);
@ -734,9 +742,9 @@ astfold_withitem(withitem_ty node_, PyArena *ctx_, int optimize_)
#undef CALL_INT_SEQ #undef CALL_INT_SEQ
int int
_PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize) _PyAST_Optimize(mod_ty mod, PyArena *arena, _PyASTOptimizeState *state)
{ {
int ret = astfold_mod(mod, arena, optimize); int ret = astfold_mod(mod, arena, state);
assert(ret || PyErr_Occurred()); assert(ret || PyErr_Occurred());
return ret; return ret;
} }

6
Python/compile.c
View File

@ -354,7 +354,11 @@ PyAST_CompileObject(mod_ty mod, PyObject *filename, PyCompilerFlags *flags,
c.c_nestlevel = 0; c.c_nestlevel = 0;
c.c_do_not_emit_bytecode = 0; c.c_do_not_emit_bytecode = 0;
if (!_PyAST_Optimize(mod, arena, c.c_optimize)) { _PyASTOptimizeState state;
state.optimize = c.c_optimize;
state.ff_features = merged;
if (!_PyAST_Optimize(mod, arena, &state)) {
goto finally; goto finally;
} }