From 662aede68b0ea222cf3db4715b310e91c51b665f Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 17 May 2023 22:22:17 -0700 Subject: [PATCH] gh-104374: Remove access to class scopes for inlined comprehensions (#104528) Co-authored-by: Carl Meyer --- Lib/test/test_listcomps.py | 109 ++++++++++++++++++++++++++++++++++++- Python/compile.c | 20 +++++-- Python/symtable.c | 5 +- 3 files changed, 125 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 23e1b8c1ce3..985274dfd6c 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -200,7 +200,8 @@ class ListComprehensionTest(unittest.TestCase): y = [g for x in [1]] """ outputs = {"y": [2]} - self._check_in_scopes(code, outputs) + self._check_in_scopes(code, outputs, scopes=["module", "function"]) + self._check_in_scopes(code, scopes=["class"], raises=NameError) def test_inner_cell_shadows_outer_redefined(self): code = """ @@ -328,7 +329,8 @@ class ListComprehensionTest(unittest.TestCase): y = [x for [x ** x for x in range(x)][x - 1] in l] """ outputs = {"y": [3, 3, 3]} - self._check_in_scopes(code, outputs) + self._check_in_scopes(code, outputs, scopes=["module", "function"]) + self._check_in_scopes(code, scopes=["class"], raises=NameError) def test_nested_3(self): code = """ @@ -379,6 +381,109 @@ class ListComprehensionTest(unittest.TestCase): with self.assertRaises(UnboundLocalError): f() + def test_name_error_in_class_scope(self): + code = """ + y = 1 + [x + y for x in range(2)] + """ + self._check_in_scopes(code, raises=NameError, scopes=["class"]) + + def test_global_in_class_scope(self): + code = """ + y = 2 + vals = [(x, y) for x in range(2)] + """ + outputs = {"vals": [(0, 1), (1, 1)]} + self._check_in_scopes(code, outputs, ns={"y": 1}, scopes=["class"]) + + def test_in_class_scope_inside_function_1(self): + code = """ + class C: + y = 2 + vals = [(x, y) for x in range(2)] + vals = C.vals + """ + outputs = {"vals": [(0, 1), (1, 1)]} + self._check_in_scopes(code, outputs, ns={"y": 1}, scopes=["function"]) + + def test_in_class_scope_inside_function_2(self): + code = """ + y = 1 + class C: + y = 2 + vals = [(x, y) for x in range(2)] + vals = C.vals + """ + outputs = {"vals": [(0, 1), (1, 1)]} + self._check_in_scopes(code, outputs, scopes=["function"]) + + def test_in_class_scope_with_global(self): + code = """ + y = 1 + class C: + global y + y = 2 + # Ensure the listcomp uses the global, not the value in the + # class namespace + locals()['y'] = 3 + vals = [(x, y) for x in range(2)] + vals = C.vals + """ + outputs = {"vals": [(0, 2), (1, 2)]} + self._check_in_scopes(code, outputs, scopes=["module", "class"]) + outputs = {"vals": [(0, 1), (1, 1)]} + self._check_in_scopes(code, outputs, scopes=["function"]) + + def test_in_class_scope_with_nonlocal(self): + code = """ + y = 1 + class C: + nonlocal y + y = 2 + # Ensure the listcomp uses the global, not the value in the + # class namespace + locals()['y'] = 3 + vals = [(x, y) for x in range(2)] + vals = C.vals + """ + outputs = {"vals": [(0, 2), (1, 2)]} + self._check_in_scopes(code, outputs, scopes=["function"]) + + def test_nested_has_free_var(self): + code = """ + items = [a for a in [1] if [a for _ in [0]]] + """ + outputs = {"items": [1]} + self._check_in_scopes(code, outputs, scopes=["class"]) + + def test_nested_free_var_not_bound_in_outer_comp(self): + code = """ + z = 1 + items = [a for a in [1] if [x for x in [1] if z]] + """ + self._check_in_scopes(code, {"items": [1]}, scopes=["module", "function"]) + self._check_in_scopes(code, {"items": []}, ns={"z": 0}, scopes=["class"]) + + def test_nested_free_var_in_iter(self): + code = """ + items = [_C for _C in [1] for [0, 1][[x for x in [1] if _C][0]] in [2]] + """ + self._check_in_scopes(code, {"items": [1]}) + + def test_nested_free_var_in_expr(self): + code = """ + items = [(_C, [x for x in [1] if _C]) for _C in [0, 1]] + """ + self._check_in_scopes(code, {"items": [(0, []), (1, [1])]}) + + def test_nested_listcomp_in_lambda(self): + code = """ + f = [(z, lambda y: [(x, y, z) for x in [3]]) for z in [1]] + (z, func), = f + out = func(2) + """ + self._check_in_scopes(code, {"z": 1, "out": [(3, 2, 1)]}) + __test__ = {'doctests' : doctests} diff --git a/Python/compile.c b/Python/compile.c index 60c845a821b..07f8d668477 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -388,6 +388,8 @@ struct compiler_unit { instr_sequence u_instr_sequence; /* codegen output */ int u_nfblocks; + int u_in_inlined_comp; + struct fblockinfo u_fblock[CO_MAXBLOCKS]; _PyCompile_CodeUnitMetadata u_metadata; @@ -1290,6 +1292,7 @@ compiler_enter_scope(struct compiler *c, identifier name, } u->u_nfblocks = 0; + u->u_in_inlined_comp = 0; u->u_metadata.u_firstlineno = lineno; u->u_metadata.u_consts = PyDict_New(); if (!u->u_metadata.u_consts) { @@ -4137,7 +4140,7 @@ compiler_nameop(struct compiler *c, location loc, case OP_DEREF: switch (ctx) { case Load: - if (c->u->u_ste->ste_type == ClassBlock) { + if (c->u->u_ste->ste_type == ClassBlock && !c->u->u_in_inlined_comp) { op = LOAD_FROM_DICT_OR_DEREF; // First load the locals if (codegen_addop_noarg(INSTR_SEQUENCE(c), LOAD_LOCALS, loc) < 0) { @@ -4188,7 +4191,12 @@ compiler_nameop(struct compiler *c, location loc, break; case OP_NAME: switch (ctx) { - case Load: op = LOAD_NAME; break; + case Load: + op = (c->u->u_ste->ste_type == ClassBlock + && c->u->u_in_inlined_comp) + ? LOAD_GLOBAL + : LOAD_NAME; + break; case Store: op = STORE_NAME; break; case Del: op = DELETE_NAME; break; } @@ -5415,6 +5423,8 @@ push_inlined_comprehension_state(struct compiler *c, location loc, PySTEntryObject *entry, inlined_comprehension_state *state) { + int in_class_block = (c->u->u_ste->ste_type == ClassBlock) && !c->u->u_in_inlined_comp; + c->u->u_in_inlined_comp++; // iterate over names bound in the comprehension and ensure we isolate // them from the outer scope as needed PyObject *k, *v; @@ -5426,7 +5436,7 @@ push_inlined_comprehension_state(struct compiler *c, location loc, // at all; DEF_LOCAL | DEF_NONLOCAL can occur in the case of an // assignment expression to a nonlocal in the comprehension, these don't // need handling here since they shouldn't be isolated - if (symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) { + if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { if (!_PyST_IsFunctionLike(c->u->u_ste)) { // non-function scope: override this name to use fast locals PyObject *orig = PyDict_GetItem(c->u->u_metadata.u_fasthidden, k); @@ -5448,8 +5458,7 @@ push_inlined_comprehension_state(struct compiler *c, location loc, long scope = (symbol >> SCOPE_OFFSET) & SCOPE_MASK; PyObject *outv = PyDict_GetItemWithError(c->u->u_ste->ste_symbols, k); if (outv == NULL) { - assert(PyErr_Occurred()); - return ERROR; + outv = _PyLong_GetZero(); } assert(PyLong_Check(outv)); long outsc = (PyLong_AS_LONG(outv) >> SCOPE_OFFSET) & SCOPE_MASK; @@ -5523,6 +5532,7 @@ static int pop_inlined_comprehension_state(struct compiler *c, location loc, inlined_comprehension_state state) { + c->u->u_in_inlined_comp--; PyObject *k, *v; Py_ssize_t pos = 0; if (state.temp_symbols) { diff --git a/Python/symtable.c b/Python/symtable.c index f896f7cbe33..a319c239d99 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -674,8 +674,9 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, } // free vars in comprehension that are locals in outer scope can - // now simply be locals, unless they are free in comp children - if (!is_free_in_any_child(comp, k)) { + // now simply be locals, unless they are free in comp children, + // or if the outer scope is a class block + if (!is_free_in_any_child(comp, k) && ste->ste_type != ClassBlock) { if (PySet_Discard(comp_free, k) < 0) { return 0; }