traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-37757: Disallow PEP 572 cases that expose implementation details (GH-15131) 

- drop TargetScopeError in favour of raising SyntaxError directly
  as per the updated PEP 572
- comprehension iteration variables are explicitly local, but
  named expression targets in comprehensions are nonlocal or
  global. Raise SyntaxError as specified in PEP 572
- named expression targets in the outermost iterable of a
  comprehension have an ambiguous target scope. Avoid resolving
  that question now by raising SyntaxError. PEP 572
  originally required this only for cases where the bound name
  conflicts with the iteration variable in the comprehension,
  but CPython can't easily restrict the exception to that case
  (as it doesn't know the target variable names when visiting
  the outermost iterator expression)
This commit is contained in:
Nick Coghlan 2019-08-25 23:45:40 +10:00 committed by GitHub
parent ce6a070414
commit 5dbe0f59b7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 191 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Doc/c-api/exceptions.rst
View File

@ -809,7 +809,6 @@ the variables:
single: PyExc_SystemError
single: PyExc_SystemExit
single: PyExc_TabError
single: PyExc_TargetScopeError
single: PyExc_TimeoutError
single: PyExc_TypeError
single: PyExc_UnboundLocalError
@ -911,8 +910,6 @@ the variables:
+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_TabError` | :exc:`TabError` | |
+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_TargetScopeError` | :exc:`TargetScopeError` | |
+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | |
+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_TypeError` | :exc:`TypeError` | |

1
Include/pyerrors.h
View File

@ -97,7 +97,6 @@ PyAPI_DATA(PyObject *) PyExc_NotImplementedError;
PyAPI_DATA(PyObject *) PyExc_SyntaxError;
PyAPI_DATA(PyObject *) PyExc_IndentationError;
PyAPI_DATA(PyObject *) PyExc_TabError;
PyAPI_DATA(PyObject *) PyExc_TargetScopeError;
PyAPI_DATA(PyObject *) PyExc_ReferenceError;
PyAPI_DATA(PyObject *) PyExc_SystemError;
PyAPI_DATA(PyObject *) PyExc_SystemExit;

3
Include/symtable.h
View File

@ -58,6 +58,8 @@ typedef struct _symtable_entry {
unsigned ste_needs_class_closure : 1; /* for class scopes, true if a
closure over __class__
should be created */
unsigned ste_comp_iter_target : 1; /* true if visiting comprehension target */
int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */
int ste_lineno; /* first line of block */
int ste_col_offset; /* offset of first line of block */
int ste_opt_lineno; /* lineno of last exec or import * */
@ -94,6 +96,7 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
#define DEF_FREE_CLASS 2<<5 /* free variable from class's method */
#define DEF_IMPORT 2<<6 /* assignment occurred via import */
#define DEF_ANNOT 2<<7 /* this name is annotated */
#define DEF_COMP_ITER 2<<8 /* this name is a comprehension iteration variable */
#define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT)

1
Lib/_compat_pickle.py
View File

@ -128,7 +128,6 @@ PYTHON2_EXCEPTIONS = (
"SystemError",
"SystemExit",
"TabError",
"TargetScopeError",
"TypeError",
"UnboundLocalError",
"UnicodeDecodeError",

1
Lib/test/exception_hierarchy.txt
View File

@ -42,7 +42,6 @@ BaseException
| +-- NotImplementedError
| +-- RecursionError
+-- SyntaxError
| +-- TargetScopeError
| +-- IndentationError
| +-- TabError
+-- SystemError

120
Lib/test/test_named_expressions.py
View File

@ -104,15 +104,69 @@ class NamedExpressionInvalidTest(unittest.TestCase):
with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
exec(code, {}, {})
def test_named_expression_invalid_18(self):
def test_named_expression_invalid_in_class_body(self):
code = """class Foo():
[(42, 1 + ((( j := i )))) for i in range(5)]
"""
with self.assertRaisesRegex(TargetScopeError,
"named expression within a comprehension cannot be used in a class body"):
with self.assertRaisesRegex(SyntaxError,
"assignment expression within a comprehension cannot be used in a class body"):
exec(code, {}, {})
def test_named_expression_invalid_rebinding_comprehension_iteration_variable(self):
cases = [
("Local reuse", 'i', "[i := 0 for i in range(5)]"),
("Nested reuse", 'j', "[[(j := 0) for i in range(5)] for j in range(5)]"),
("Reuse inner loop target", 'j', "[(j := 0) for i in range(5) for j in range(5)]"),
("Unpacking reuse", 'i', "[i := 0 for i, j in [(0, 1)]]"),
("Reuse in loop condition", 'i', "[i+1 for i in range(5) if (i := 0)]"),
("Unreachable reuse", 'i', "[False or (i:=0) for i in range(5)]"),
("Unreachable nested reuse", 'i',
"[(i, j) for i in range(5) for j in range(5) if True or (i:=10)]"),
]
for case, target, code in cases:
msg = f"assignment expression cannot rebind comprehension iteration variable '{target}'"
with self.subTest(case=case):
with self.assertRaisesRegex(SyntaxError, msg):
exec(code, {}, {})
def test_named_expression_invalid_rebinding_comprehension_inner_loop(self):
cases = [
("Inner reuse", 'j', "[i for i in range(5) if (j := 0) for j in range(5)]"),
("Inner unpacking reuse", 'j', "[i for i in range(5) if (j := 0) for j, k in [(0, 1)]]"),
]
for case, target, code in cases:
msg = f"comprehension inner loop cannot rebind assignment expression target '{target}'"
with self.subTest(case=case):
with self.assertRaisesRegex(SyntaxError, msg):
exec(code, {}) # Module scope
with self.assertRaisesRegex(SyntaxError, msg):
exec(code, {}, {}) # Class scope
with self.assertRaisesRegex(SyntaxError, msg):
exec(f"lambda: {code}", {}) # Function scope
def test_named_expression_invalid_comprehension_iterable_expression(self):
cases = [
("Top level", "[i for i in (i := range(5))]"),
("Inside tuple", "[i for i in (2, 3, i := range(5))]"),
("Inside list", "[i for i in [2, 3, i := range(5)]]"),
("Different name", "[i for i in (j := range(5))]"),
("Lambda expression", "[i for i in (lambda:(j := range(5)))()]"),
("Inner loop", "[i for i in range(5) for j in (i := range(5))]"),
("Nested comprehension", "[i for i in [j for j in (k := range(5))]]"),
("Nested comprehension condition", "[i for i in [j for j in range(5) if (j := True)]]"),
("Nested comprehension body", "[i for i in [(j := True) for j in range(5)]]"),
]
msg = "assignment expression cannot be used in a comprehension iterable expression"
for case, code in cases:
with self.subTest(case=case):
with self.assertRaisesRegex(SyntaxError, msg):
exec(code, {}) # Module scope
with self.assertRaisesRegex(SyntaxError, msg):
exec(code, {}, {}) # Class scope
with self.assertRaisesRegex(SyntaxError, msg):
exec(f"lambda: {code}", {}) # Function scope
class NamedExpressionAssignmentTest(unittest.TestCase):
@ -306,39 +360,6 @@ print(a)"""
self.assertEqual(res, [0, 1, 2, 3, 4])
self.assertEqual(j, 4)
def test_named_expression_scope_12(self):
res = [i := i for i in range(5)]
self.assertEqual(res, [0, 1, 2, 3, 4])
self.assertEqual(i, 4)
def test_named_expression_scope_13(self):
res = [i := 0 for i, j in [(1, 2), (3, 4)]]
self.assertEqual(res, [0, 0])
self.assertEqual(i, 0)
def test_named_expression_scope_14(self):
res = [(i := 0, j := 1) for i, j in [(1, 2), (3, 4)]]
self.assertEqual(res, [(0, 1), (0, 1)])
self.assertEqual(i, 0)
self.assertEqual(j, 1)
def test_named_expression_scope_15(self):
res = [(i := i, j := j) for i, j in [(1, 2), (3, 4)]]
self.assertEqual(res, [(1, 2), (3, 4)])
self.assertEqual(i, 3)
self.assertEqual(j, 4)
def test_named_expression_scope_16(self):
res = [(i := j, j := i) for i, j in [(1, 2), (3, 4)]]
self.assertEqual(res, [(2, 2), (4, 4)])
self.assertEqual(i, 4)
self.assertEqual(j, 4)
def test_named_expression_scope_17(self):
b = 0
res = [b := i + b for i in range(5)]
@ -421,6 +442,33 @@ spam()"""
self.assertEqual(ns["a"], 20)
def test_named_expression_variable_reuse_in_comprehensions(self):
# The compiler is expected to raise syntax error for comprehension
# iteration variables, but should be fine with rebinding of other
# names (e.g. globals, nonlocals, other assignment expressions)
# The cases are all defined to produce the same expected result
# Each comprehension is checked at both function scope and module scope
rebinding = "[x := i for i in range(3) if (x := i) or not x]"
filter_ref = "[x := i for i in range(3) if x or not x]"
body_ref = "[x for i in range(3) if (x := i) or not x]"
nested_ref = "[j for i in range(3) if x or not x for j in range(3) if (x := i)][:-3]"
cases = [
("Rebind global", f"x = 1; result = {rebinding}"),
("Rebind nonlocal", f"result, x = (lambda x=1: ({rebinding}, x))()"),
("Filter global", f"x = 1; result = {filter_ref}"),
("Filter nonlocal", f"result, x = (lambda x=1: ({filter_ref}, x))()"),
("Body global", f"x = 1; result = {body_ref}"),
("Body nonlocal", f"result, x = (lambda x=1: ({body_ref}, x))()"),
("Nested global", f"x = 1; result = {nested_ref}"),
("Nested nonlocal", f"result, x = (lambda x=1: ({nested_ref}, x))()"),
]
for case, code in cases:
with self.subTest(case=case):
ns = {}
exec(code, ns)
self.assertEqual(ns["x"], 2)
self.assertEqual(ns["result"], [0, 1, 2])
if __name__ == "__main__":
unittest.main()

7
Misc/NEWS.d/next/Core and Builtins/2019-08-05-14-22-59.bpo-37757.lRv5HX.rst Normal file
View File

@ -0,0 +1,7 @@
:pep:`572`: As described in the PEP, assignment expressions now raise
:exc:`SyntaxError` when their interaction with comprehension scoping results
in an ambiguous target scope.
The ``TargetScopeError`` subclass originally proposed by the PEP has been
removed in favour of just raising regular syntax errors for the disallowed
cases.

9
Objects/exceptions.c
View File

@ -1521,13 +1521,6 @@ MiddlingExtendsException(PyExc_SyntaxError, IndentationError, SyntaxError,
"Improper indentation.");
/*
* TargetScopeError extends SyntaxError
*/
MiddlingExtendsException(PyExc_SyntaxError, TargetScopeError, SyntaxError,
"Improper scope target.");
/*
* TabError extends IndentationError
*/
@ -2539,7 +2532,6 @@ _PyExc_Init(void)
PRE_INIT(AttributeError);
PRE_INIT(SyntaxError);
PRE_INIT(IndentationError);
PRE_INIT(TargetScopeError);
PRE_INIT(TabError);
PRE_INIT(LookupError);
PRE_INIT(IndexError);
@ -2680,7 +2672,6 @@ _PyBuiltins_AddExceptions(PyObject *bltinmod)
POST_INIT(AttributeError);
POST_INIT(SyntaxError);
POST_INIT(IndentationError);
POST_INIT(TargetScopeError);
POST_INIT(TabError);
POST_INIT(LookupError);
POST_INIT(IndexError);

1
PC/python3.def
View File

@ -235,7 +235,6 @@ EXPORTS
PyExc_SystemError=python39.PyExc_SystemError DATA
PyExc_SystemExit=python39.PyExc_SystemExit DATA
PyExc_TabError=python39.PyExc_TabError DATA
PyExc_TargetScopeError=python39.PyExc_TargetScopeError DATA
PyExc_TimeoutError=python39.PyExc_TimeoutError DATA
PyExc_TypeError=python39.PyExc_TypeError DATA
PyExc_UnboundLocalError=python39.PyExc_UnboundLocalError DATA

121
Python/symtable.c
View File

@ -32,7 +32,16 @@
#define IMPORT_STAR_WARNING "import * only allowed at module level"
#define NAMED_EXPR_COMP_IN_CLASS \
"named expression within a comprehension cannot be used in a class body"
"assignment expression within a comprehension cannot be used in a class body"
#define NAMED_EXPR_COMP_CONFLICT \
"assignment expression cannot rebind comprehension iteration variable '%U'"
#define NAMED_EXPR_COMP_INNER_LOOP_CONFLICT \
"comprehension inner loop cannot rebind assignment expression target '%U'"
#define NAMED_EXPR_COMP_ITER_EXPR \
"assignment expression cannot be used in a comprehension iterable expression"
static PySTEntryObject *
ste_new(struct symtable *st, identifier name, _Py_block_ty block,
@ -81,6 +90,8 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block,
ste->ste_comprehension = 0;
ste->ste_returns_value = 0;
ste->ste_needs_class_closure = 0;
ste->ste_comp_iter_target = 0;
ste->ste_comp_iter_expr = 0;
ste->ste_symbols = PyDict_New();
ste->ste_varnames = PyList_New(0);
@ -373,14 +384,21 @@ PySymtable_Lookup(struct symtable *st, void *key)
return (PySTEntryObject *)v;
}
int
PyST_GetScope(PySTEntryObject *ste, PyObject *name)
static long
_PyST_GetSymbol(PySTEntryObject *ste, PyObject *name)
{
PyObject *v = PyDict_GetItem(ste->ste_symbols, name);
if (!v)
return 0;
assert(PyLong_Check(v));
return (PyLong_AS_LONG(v) >> SCOPE_OFFSET) & SCOPE_MASK;
return PyLong_AS_LONG(v);
}
int
PyST_GetScope(PySTEntryObject *ste, PyObject *name)
{
long symbol = _PyST_GetSymbol(ste, name);
return (symbol >> SCOPE_OFFSET) & SCOPE_MASK;
}
static int
@ -955,6 +973,13 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
return 0;
}
prev = st->st_cur;
/* bpo-37757: For now, disallow *all* assignment expressions in the
* outermost iterator expression of a comprehension, even those inside
* a nested comprehension or a lambda expression.
*/
if (prev) {
ste->ste_comp_iter_expr = prev->ste_comp_iter_expr;
}
/* The entry is owned by the stack. Borrow it for st_cur. */
Py_DECREF(ste);
st->st_cur = ste;
@ -971,15 +996,10 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
static long
symtable_lookup(struct symtable *st, PyObject *name)
{
PyObject *o;
PyObject *mangled = _Py_Mangle(st->st_private, name);
if (!mangled)
return 0;
o = PyDict_GetItem(st->st_cur->ste_symbols, mangled);
Py_DECREF(mangled);
if (!o)
return 0;
return PyLong_AsLong(o);
return _PyST_GetSymbol(st->st_cur, mangled);
}
static int
@ -1012,6 +1032,22 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s
else {
val = flag;
}
if (ste->ste_comp_iter_target) {
/* This name is an iteration variable in a comprehension,
* so check for a binding conflict with any named expressions.
* Otherwise, mark it as an iteration variable so subsequent
* named expressions can check for conflicts.
*/
if (val & (DEF_GLOBAL | DEF_NONLOCAL)) {
PyErr_Format(PyExc_SyntaxError,
NAMED_EXPR_COMP_INNER_LOOP_CONFLICT, name);
PyErr_SyntaxLocationObject(st->st_filename,
ste->ste_lineno,
ste->ste_col_offset + 1);
goto error;
}
val |= DEF_COMP_ITER;
}
o = PyLong_FromLong(val);
if (o == NULL)
goto error;
@ -1392,7 +1428,9 @@ static int
symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e)
{
assert(st->st_stack);
assert(e->kind == Name_kind);
PyObject *target_name = e->v.Name.id;
Py_ssize_t i, size;
struct _symtable_entry *ste;
size = PyList_GET_SIZE(st->st_stack);
@ -1402,32 +1440,42 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e)
for (i = size - 1; i >= 0; i--) {
ste = (struct _symtable_entry *) PyList_GET_ITEM(st->st_stack, i);
/* If our current entry is a comprehension, skip it */
/* If we find a comprehension scope, check for a target
* binding conflict with iteration variables, otherwise skip it
*/
if (ste->ste_comprehension) {
long target_in_scope = _PyST_GetSymbol(ste, target_name);
if (target_in_scope & DEF_COMP_ITER) {
PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_CONFLICT, target_name);
PyErr_SyntaxLocationObject(st->st_filename,
e->lineno,
e->col_offset);
VISIT_QUIT(st, 0);
}
continue;
}
/* If we find a FunctionBlock entry, add as NONLOCAL/LOCAL */
if (ste->ste_type == FunctionBlock) {
if (!symtable_add_def(st, e->v.Name.id, DEF_NONLOCAL))
if (!symtable_add_def(st, target_name, DEF_NONLOCAL))
VISIT_QUIT(st, 0);
if (!symtable_record_directive(st, e->v.Name.id, e->lineno, e->col_offset))
if (!symtable_record_directive(st, target_name, e->lineno, e->col_offset))
VISIT_QUIT(st, 0);
return symtable_add_def_helper(st, e->v.Name.id, DEF_LOCAL, ste);
return symtable_add_def_helper(st, target_name, DEF_LOCAL, ste);
}
/* If we find a ModuleBlock entry, add as GLOBAL */
if (ste->ste_type == ModuleBlock) {
if (!symtable_add_def(st, e->v.Name.id, DEF_GLOBAL))
if (!symtable_add_def(st, target_name, DEF_GLOBAL))
VISIT_QUIT(st, 0);
if (!symtable_record_directive(st, e->v.Name.id, e->lineno, e->col_offset))
if (!symtable_record_directive(st, target_name, e->lineno, e->col_offset))
VISIT_QUIT(st, 0);
return symtable_add_def_helper(st, e->v.Name.id, DEF_GLOBAL, ste);
return symtable_add_def_helper(st, target_name, DEF_GLOBAL, ste);
}
/* Disallow usage in ClassBlock */
if (ste->ste_type == ClassBlock) {
PyErr_Format(PyExc_TargetScopeError, NAMED_EXPR_COMP_IN_CLASS, e->v.Name.id);
PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_CLASS);
PyErr_SyntaxLocationObject(st->st_filename,
e->lineno,
e->col_offset);
@ -1442,6 +1490,26 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e)
return 0;
}
static int
symtable_handle_namedexpr(struct symtable *st, expr_ty e)
{
if (st->st_cur->ste_comp_iter_expr > 0) {
/* Assignment isn't allowed in a comprehension iterable expression */
PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_ITER_EXPR);
PyErr_SyntaxLocationObject(st->st_filename,
e->lineno,
e->col_offset);
VISIT_QUIT(st, 0);
}
if (st->st_cur->ste_comprehension) {
/* Inside a comprehension body, so find the right target scope */
if (!symtable_extend_namedexpr_scope(st, e->v.NamedExpr.target))
VISIT_QUIT(st, 0);
}
VISIT(st, expr, e->v.NamedExpr.value);
VISIT(st, expr, e->v.NamedExpr.target);
}
static int
symtable_visit_expr(struct symtable *st, expr_ty e)
{
@ -1452,12 +1520,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
}
switch (e->kind) {
case NamedExpr_kind:
if (st->st_cur->ste_comprehension) {
if (!symtable_extend_namedexpr_scope(st, e->v.NamedExpr.target))
VISIT_QUIT(st, 0);
}
VISIT(st, expr, e->v.NamedExpr.value);
VISIT(st, expr, e->v.NamedExpr.target);
symtable_handle_namedexpr(st, e);
break;
case BoolOp_kind:
VISIT_SEQ(st, expr, e->v.BoolOp.values);
@ -1739,8 +1802,12 @@ symtable_visit_alias(struct symtable *st, alias_ty a)
static int
symtable_visit_comprehension(struct symtable *st, comprehension_ty lc)
{
st->st_cur->ste_comp_iter_target = 1;
VISIT(st, expr, lc->target);
st->st_cur->ste_comp_iter_target = 0;
st->st_cur->ste_comp_iter_expr++;
VISIT(st, expr, lc->iter);
st->st_cur->ste_comp_iter_expr--;
VISIT_SEQ(st, expr, lc->ifs);
if (lc->is_async) {
st->st_cur->ste_coroutine = 1;
@ -1788,7 +1855,9 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e,
comprehension_ty outermost = ((comprehension_ty)
asdl_seq_GET(generators, 0));
/* Outermost iterator is evaluated in current scope */
st->st_cur->ste_comp_iter_expr++;
VISIT(st, expr, outermost->iter);
st->st_cur->ste_comp_iter_expr--;
/* Create comprehension scope for the rest */
if (!scope_name ||
!symtable_enter_block(st, scope_name, FunctionBlock, (void *)e,
@ -1805,7 +1874,11 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e,
symtable_exit_block(st, (void *)e);
return 0;
}
/* Visit iteration variable target, and mark them as such */
st->st_cur->ste_comp_iter_target = 1;
VISIT(st, expr, outermost->target);
st->st_cur->ste_comp_iter_target = 0;
/* Visit the rest of the comprehension body */
VISIT_SEQ(st, expr, outermost->ifs);
VISIT_SEQ_TAIL(st, comprehension, generators, 1);
if (value)