Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-97933: (PEP 709) inline list/dict/set comprehensions (#101441) 

Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com>
Co-authored-by: Erlend E. Aasland <erlend.aasland@protonmail.com>
This commit is contained in:
Carl Meyer 2023-05-09 11:02:14 -06:00 committed by GitHub
parent 0aeda29793
commit c3b595e73e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 1243 additions and 695 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Doc/library/dis.rst
View File

@ -1196,6 +1196,14 @@ iterations of the loop.
.. versionadded:: 3.12
.. opcode:: LOAD_FAST_AND_CLEAR (var_num)
Pushes a reference to the local ``co_varnames[var_num]`` onto the stack (or
pushes ``NULL`` onto the stack if the local variable has not been
initialized) and sets ``co_varnames[var_num]`` to ``NULL``.
.. versionadded:: 3.12
.. opcode:: STORE_FAST (var_num)
Stores ``STACK.pop()`` into the local ``co_varnames[var_num]``.

24
Doc/whatsnew/3.12.rst
View File

@ -153,6 +153,30 @@ New Features
In Python 3.14, the default will switch to ``'data'``.
(Contributed by Petr Viktorin in :pep:`706`.)
.. _whatsnew312-pep709:
PEP 709: Comprehension inlining
-------------------------------
Dictionary, list, and set comprehensions are now inlined, rather than creating a
new single-use function object for each execution of the comprehension. This
speeds up execution of a comprehension by up to 2x.
Comprehension iteration variables remain isolated; they don't overwrite a
variable of the same name in the outer scope, nor are they visible after the
comprehension. This isolation is now maintained via stack/locals manipulation,
not via separate function scope.
Inlining does result in a few visible behavior changes:
* There is no longer a separate frame for the comprehension in tracebacks,
and tracing/profiling no longer shows the comprehension as a function call.
* Calling :func:`locals` inside a comprehension now includes variables
from outside the comprehension, and no longer includes the synthetic ``.0``
variable for the comprehension "argument".
Contributed by Carl Meyer and Vladimir Matveev in :pep:`709`.
PEP 688: Making the buffer protocol accessible in Python
--------------------------------------------------------

1
Include/internal/pycore_code.h
View File

@ -131,6 +131,7 @@ struct callable_cache {
// Note that these all fit within a byte, as do combinations.
// Later, we will use the smaller numbers to differentiate the different
// kinds of locals (e.g. pos-only arg, varkwargs, local-only).
#define CO_FAST_HIDDEN 0x10
#define CO_FAST_LOCAL 0x20
#define CO_FAST_CELL 0x40
#define CO_FAST_FREE 0x80

3
Include/internal/pycore_compile.h
View File

@ -70,6 +70,9 @@ typedef struct {
PyObject *u_varnames; /* local variables */
PyObject *u_cellvars; /* cell variables */
PyObject *u_freevars; /* free variables */
PyObject *u_fasthidden; /* dict; keys are names that are fast-locals only
temporarily within an inlined comprehension. When
value is True, treat as fast-local. */
Py_ssize_t u_argcount; /* number of arguments for block */
Py_ssize_t u_posonlyargcount; /* number of positional only arguments for block */

2
Include/internal/pycore_flowgraph.h
View File

@ -94,7 +94,7 @@ _PyCfgInstruction* _PyCfg_BasicblockLastInstr(const _PyCfgBasicblock *b);
int _PyCfg_OptimizeCodeUnit(_PyCfgBuilder *g, PyObject *consts, PyObject *const_cache,
int code_flags, int nlocals, int nparams, int firstlineno);
int _PyCfg_Stackdepth(_PyCfgBasicblock *entryblock, int code_flags);
void _PyCfg_ConvertExceptionHandlersToNops(_PyCfgBasicblock *entryblock);
void _PyCfg_ConvertPseudoOps(_PyCfgBasicblock *entryblock);
int _PyCfg_ResolveJumps(_PyCfgBuilder *g);

13
Include/internal/pycore_opcode.h generated
View File

@ -173,6 +173,7 @@ const uint8_t _PyOpcode_Deopt[256] = {
[LOAD_CONST__LOAD_FAST] = LOAD_CONST,
[LOAD_DEREF] = LOAD_DEREF,
[LOAD_FAST] = LOAD_FAST,
[LOAD_FAST_AND_CLEAR] = LOAD_FAST_AND_CLEAR,
[LOAD_FAST_CHECK] = LOAD_FAST_CHECK,
[LOAD_FAST__LOAD_CONST] = LOAD_FAST,
[LOAD_FAST__LOAD_FAST] = LOAD_FAST,
@ -239,7 +240,7 @@ const uint8_t _PyOpcode_Deopt[256] = {
#endif // NEED_OPCODE_TABLES
#ifdef Py_DEBUG
static const char *const _PyOpcode_OpName[266] = {
static const char *const _PyOpcode_OpName[267] = {
[CACHE] = "CACHE",
[POP_TOP] = "POP_TOP",
[PUSH_NULL] = "PUSH_NULL",
@ -383,7 +384,7 @@ static const char *const _PyOpcode_OpName[266] = {
[JUMP_BACKWARD] = "JUMP_BACKWARD",
[LOAD_SUPER_ATTR] = "LOAD_SUPER_ATTR",
[CALL_FUNCTION_EX] = "CALL_FUNCTION_EX",
[STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST",
[LOAD_FAST_AND_CLEAR] = "LOAD_FAST_AND_CLEAR",
[EXTENDED_ARG] = "EXTENDED_ARG",
[LIST_APPEND] = "LIST_APPEND",
[SET_ADD] = "SET_ADD",
@ -393,21 +394,21 @@ static const char *const _PyOpcode_OpName[266] = {
[YIELD_VALUE] = "YIELD_VALUE",
[RESUME] = "RESUME",
[MATCH_CLASS] = "MATCH_CLASS",
[STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST",
[STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST",
[STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT",
[FORMAT_VALUE] = "FORMAT_VALUE",
[BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP",
[BUILD_STRING] = "BUILD_STRING",
[STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT",
[STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT",
[UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST",
[UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE",
[UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE",
[LIST_EXTEND] = "LIST_EXTEND",
[SET_UPDATE] = "SET_UPDATE",
[DICT_MERGE] = "DICT_MERGE",
[DICT_UPDATE] = "DICT_UPDATE",
[UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE",
[SEND_GEN] = "SEND_GEN",
[167] = "<167>",
[168] = "<168>",
[169] = "<169>",
[170] = "<170>",
@ -506,11 +507,11 @@ static const char *const _PyOpcode_OpName[266] = {
[LOAD_SUPER_METHOD] = "LOAD_SUPER_METHOD",
[LOAD_ZERO_SUPER_METHOD] = "LOAD_ZERO_SUPER_METHOD",
[LOAD_ZERO_SUPER_ATTR] = "LOAD_ZERO_SUPER_ATTR",
[STORE_FAST_MAYBE_NULL] = "STORE_FAST_MAYBE_NULL",
};
#endif
#define EXTRA_CASES \
case 167: \
case 168: \
case 169: \
case 170: \

1
Include/internal/pycore_symtable.h
View File

@ -64,6 +64,7 @@ typedef struct _symtable_entry {
unsigned ste_needs_class_closure : 1; /* for class scopes, true if a
closure over __class__
should be created */
unsigned ste_comp_inlined : 1; /* true if this comprehension is inlined */
unsigned ste_comp_iter_target : 1; /* true if visiting comprehension target */
int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */
int ste_lineno; /* first line of block */

21
Include/opcode.h generated
View File

@ -97,6 +97,7 @@ extern "C" {
#define JUMP_BACKWARD 140
#define LOAD_SUPER_ATTR 141
#define CALL_FUNCTION_EX 142
#define LOAD_FAST_AND_CLEAR 143
#define EXTENDED_ARG 144
#define LIST_APPEND 145
#define SET_ADD 146
@ -146,7 +147,8 @@ extern "C" {
#define LOAD_SUPER_METHOD 263
#define LOAD_ZERO_SUPER_METHOD 264
#define LOAD_ZERO_SUPER_ATTR 265
#define MAX_PSEUDO_OPCODE 265
#define STORE_FAST_MAYBE_NULL 266
#define MAX_PSEUDO_OPCODE 266
#define BINARY_OP_ADD_FLOAT 6
#define BINARY_OP_ADD_INT 7
#define BINARY_OP_ADD_UNICODE 8
@ -202,14 +204,14 @@ extern "C" {
#define STORE_ATTR_INSTANCE_VALUE 111
#define STORE_ATTR_SLOT 112
#define STORE_ATTR_WITH_HINT 113
#define STORE_FAST__LOAD_FAST 143
#define STORE_FAST__STORE_FAST 153
#define STORE_SUBSCR_DICT 154
#define STORE_SUBSCR_LIST_INT 158
#define UNPACK_SEQUENCE_LIST 159
#define UNPACK_SEQUENCE_TUPLE 160
#define UNPACK_SEQUENCE_TWO_TUPLE 161
#define SEND_GEN 166
#define STORE_FAST__LOAD_FAST 153
#define STORE_FAST__STORE_FAST 154
#define STORE_SUBSCR_DICT 158
#define STORE_SUBSCR_LIST_INT 159
#define UNPACK_SEQUENCE_LIST 160
#define UNPACK_SEQUENCE_TUPLE 161
#define UNPACK_SEQUENCE_TWO_TUPLE 166
#define SEND_GEN 167
#define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\
|| ((op) == JUMP) \
@ -218,6 +220,7 @@ extern "C" {
|| ((op) == LOAD_SUPER_METHOD) \
|| ((op) == LOAD_ZERO_SUPER_METHOD) \
|| ((op) == LOAD_ZERO_SUPER_ATTR) \
|| ((op) == STORE_FAST_MAYBE_NULL) \
)
#define HAS_CONST(op) (false\

3
Lib/importlib/_bootstrap_external.py
View File

@ -442,6 +442,7 @@ _code_type = type(_write_atomic.__code__)
# Python 3.12b1 3526 (Add instrumentation support)
# Python 3.12b1 3527 (Add LOAD_SUPER_ATTR)
# Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization)
# Python 3.12b1 3529 (Inline list/dict/set comprehensions)
# Python 3.13 will start with 3550
@ -458,7 +459,7 @@ _code_type = type(_write_atomic.__code__)
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
# in PC/launcher.c must also be updated.
MAGIC_NUMBER = (3528).to_bytes(2, 'little') + b'\r\n'
MAGIC_NUMBER = (3529).to_bytes(2, 'little') + b'\r\n'
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c

4
Lib/opcode.py
View File

@ -198,6 +198,8 @@ hasfree.append(139)
jrel_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards)
name_op('LOAD_SUPER_ATTR', 141)
def_op('CALL_FUNCTION_EX', 142) # Flags
def_op('LOAD_FAST_AND_CLEAR', 143) # Local variable number
haslocal.append(143)
def_op('EXTENDED_ARG', 144)
EXTENDED_ARG = 144
@ -268,6 +270,8 @@ pseudo_op('LOAD_SUPER_METHOD', 263, ['LOAD_SUPER_ATTR'])
pseudo_op('LOAD_ZERO_SUPER_METHOD', 264, ['LOAD_SUPER_ATTR'])
pseudo_op('LOAD_ZERO_SUPER_ATTR', 265, ['LOAD_SUPER_ATTR'])
pseudo_op('STORE_FAST_MAYBE_NULL', 266, ['STORE_FAST'])
MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1
del def_op, name_op, jrel_op, jabs_op, pseudo_op

24
Lib/test/test_compile.py
View File

@ -1352,14 +1352,11 @@ class TestSourcePositions(unittest.TestCase):
and x != 50)]
""")
compiled_code, _ = self.check_positions_against_ast(snippet)
compiled_code = compiled_code.co_consts[0]
self.assertIsInstance(compiled_code, types.CodeType)
self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND',
line=1, end_line=2, column=1, end_column=8, occurrence=1)
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
line=1, end_line=2, column=1, end_column=8, occurrence=1)
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
line=1, end_line=6, column=0, end_column=32, occurrence=1)
def test_multiline_async_list_comprehension(self):
snippet = textwrap.dedent("""\
@ -1374,13 +1371,13 @@ class TestSourcePositions(unittest.TestCase):
compiled_code, _ = self.check_positions_against_ast(snippet)
g = {}
eval(compiled_code, g)
compiled_code = g['f'].__code__.co_consts[1]
compiled_code = g['f'].__code__
self.assertIsInstance(compiled_code, types.CodeType)
self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND',
line=2, end_line=3, column=5, end_column=12, occurrence=1)
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
line=2, end_line=3, column=5, end_column=12, occurrence=1)
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST',
line=2, end_line=7, column=4, end_column=36, occurrence=1)
def test_multiline_set_comprehension(self):
@ -1393,14 +1390,11 @@ class TestSourcePositions(unittest.TestCase):
and x != 50)}
""")
compiled_code, _ = self.check_positions_against_ast(snippet)
compiled_code = compiled_code.co_consts[0]
self.assertIsInstance(compiled_code, types.CodeType)
self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD',
line=1, end_line=2, column=1, end_column=8, occurrence=1)
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
line=1, end_line=2, column=1, end_column=8, occurrence=1)
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
line=1, end_line=6, column=0, end_column=32, occurrence=1)
def test_multiline_async_set_comprehension(self):
snippet = textwrap.dedent("""\
@ -1415,13 +1409,13 @@ class TestSourcePositions(unittest.TestCase):
compiled_code, _ = self.check_positions_against_ast(snippet)
g = {}
eval(compiled_code, g)
compiled_code = g['f'].__code__.co_consts[1]
compiled_code = g['f'].__code__
self.assertIsInstance(compiled_code, types.CodeType)
self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD',
line=2, end_line=3, column=5, end_column=12, occurrence=1)
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
line=2, end_line=3, column=5, end_column=12, occurrence=1)
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST',
line=2, end_line=7, column=4, end_column=36, occurrence=1)
def test_multiline_dict_comprehension(self):
@ -1434,14 +1428,11 @@ class TestSourcePositions(unittest.TestCase):
and x != 50)}
""")
compiled_code, _ = self.check_positions_against_ast(snippet)
compiled_code = compiled_code.co_consts[0]
self.assertIsInstance(compiled_code, types.CodeType)
self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD',
line=1, end_line=2, column=1, end_column=7, occurrence=1)
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
line=1, end_line=2, column=1, end_column=7, occurrence=1)
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
line=1, end_line=6, column=0, end_column=32, occurrence=1)
def test_multiline_async_dict_comprehension(self):
snippet = textwrap.dedent("""\
@ -1456,13 +1447,13 @@ class TestSourcePositions(unittest.TestCase):
compiled_code, _ = self.check_positions_against_ast(snippet)
g = {}
eval(compiled_code, g)
compiled_code = g['f'].__code__.co_consts[1]
compiled_code = g['f'].__code__
self.assertIsInstance(compiled_code, types.CodeType)
self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD',
line=2, end_line=3, column=5, end_column=11, occurrence=1)
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
line=2, end_line=3, column=5, end_column=11, occurrence=1)
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST',
line=2, end_line=7, column=4, end_column=36, occurrence=1)
def test_matchcase_sequence(self):
@ -1711,9 +1702,6 @@ class TestSourcePositions(unittest.TestCase):
for source in [
"lambda: a",
"(a for b in c)",
"[a for b in c]",
"{a for b in c}",
"{a: b for c in d}",
]:
with self.subTest(source):
code = compile(f"{source}, {source}", "<test>", "eval")

5
Lib/test/test_compiler_assemble.py
View File

@ -16,7 +16,7 @@ class IsolatedAssembleTests(AssemblerTestCase):
metadata.setdefault(key, key)
for key in ['consts']:
metadata.setdefault(key, [])
for key in ['names', 'varnames', 'cellvars', 'freevars']:
for key in ['names', 'varnames', 'cellvars', 'freevars', 'fasthidden']:
metadata.setdefault(key, {})
for key in ['argcount', 'posonlyargcount', 'kwonlyargcount']:
metadata.setdefault(key, 0)
@ -33,6 +33,9 @@ class IsolatedAssembleTests(AssemblerTestCase):
expected_metadata = {}
for key, value in metadata.items():
if key == "fasthidden":
# not exposed on code object
continue
if isinstance(value, list):
expected_metadata[key] = tuple(value)
elif isinstance(value, dict):

33
Lib/test/test_dis.py
View File

@ -154,7 +154,7 @@ dis_bug708901 = """\
def bug1333982(x=[]):
assert 0, ([s for s in x] +
assert 0, ((s for s in x) +
1)
pass
@ -162,7 +162,7 @@ dis_bug1333982 = """\
%3d RESUME 0
%3d LOAD_ASSERTION_ERROR
LOAD_CONST 1 (<code object <listcomp> at 0x..., file "%s", line %d>)
LOAD_CONST 1 (<code object <genexpr> at 0x..., file "%s", line %d>)
MAKE_FUNCTION 0
LOAD_FAST 0 (x)
GET_ITER
@ -675,7 +675,7 @@ async def _co(x):
def _h(y):
def foo(x):
'''funcdoc'''
return [x + z for z in y]
return list(x + z for z in y)
return foo
dis_nested_0 = """\
@ -705,13 +705,15 @@ Disassembly of <code object foo at 0x..., file "%s", line %d>:
%3d RESUME 0
%3d LOAD_CLOSURE 0 (x)
%3d LOAD_GLOBAL 1 (NULL + list)
LOAD_CLOSURE 0 (x)
BUILD_TUPLE 1
LOAD_CONST 1 (<code object <listcomp> at 0x..., file "%s", line %d>)
LOAD_CONST 1 (<code object <genexpr> at 0x..., file "%s", line %d>)
MAKE_FUNCTION 8 (closure)
LOAD_DEREF 1 (y)
GET_ITER
CALL 0
CALL 1
RETURN_VALUE
""" % (dis_nested_0,
__file__,
@ -723,21 +725,28 @@ Disassembly of <code object foo at 0x..., file "%s", line %d>:
)
dis_nested_2 = """%s
Disassembly of <code object <listcomp> at 0x..., file "%s", line %d>:
Disassembly of <code object <genexpr> at 0x..., file "%s", line %d>:
COPY_FREE_VARS 1
%3d RESUME 0
BUILD_LIST 0
%3d RETURN_GENERATOR
POP_TOP
RESUME 0
LOAD_FAST 0 (.0)
>> FOR_ITER 7 (to 26)
>> FOR_ITER 9 (to 32)
STORE_FAST 1 (z)
LOAD_DEREF 2 (x)
LOAD_FAST 1 (z)
BINARY_OP 0 (+)
LIST_APPEND 2
JUMP_BACKWARD 9 (to 8)
YIELD_VALUE 1
RESUME 1
POP_TOP
JUMP_BACKWARD 11 (to 10)
>> END_FOR
RETURN_VALUE
RETURN_CONST 0 (None)
>> CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR)
RERAISE 1
ExceptionTable:
1 row
""" % (dis_nested_1,
__file__,
_h.__code__.co_firstlineno + 3,

10
Lib/test/test_inspect.py
View File

@ -4280,14 +4280,14 @@ class TestSignatureBind(unittest.TestCase):
@cpython_only
def test_signature_bind_implicit_arg(self):
# Issue #19611: getcallargs should work with set comprehensions
# Issue #19611: getcallargs should work with comprehensions
def make_set():
return {z * z for z in range(5)}
setcomp_code = make_set.__code__.co_consts[1]
setcomp_func = types.FunctionType(setcomp_code, {})
return set(z * z for z in range(5))
gencomp_code = make_set.__code__.co_consts[1]
gencomp_func = types.FunctionType(gencomp_code, {})
iterator = iter(range(5))
self.assertEqual(self.call(setcomp_func, iterator), {0, 1, 4, 9, 16})
self.assertEqual(set(self.call(gencomp_func, iterator)), {0, 1, 4, 9, 16})
def test_signature_bind_posonly_kwargs(self):
def foo(bar, /, **kwargs):

277
Lib/test/test_listcomps.py
View File

@ -1,4 +1,5 @@
import doctest
import textwrap
import unittest
@ -87,65 +88,229 @@ Make sure that None is a valid return value
>>> [None for i in range(10)]
[None, None, None, None, None, None, None, None, None, None]
########### Tests for various scoping corner cases ############
Return lambdas that use the iteration variable as a default argument
>>> items = [(lambda i=i: i) for i in range(5)]
>>> [x() for x in items]
[0, 1, 2, 3, 4]
Same again, only this time as a closure variable
>>> items = [(lambda: i) for i in range(5)]
>>> [x() for x in items]
[4, 4, 4, 4, 4]
Another way to test that the iteration variable is local to the list comp
>>> items = [(lambda: i) for i in range(5)]
>>> i = 20
>>> [x() for x in items]
[4, 4, 4, 4, 4]
And confirm that a closure can jump over the list comp scope
>>> items = [(lambda: y) for i in range(5)]
>>> y = 2
>>> [x() for x in items]
[2, 2, 2, 2, 2]
We also repeat each of the above scoping tests inside a function
>>> def test_func():
... items = [(lambda i=i: i) for i in range(5)]
... return [x() for x in items]
>>> test_func()
[0, 1, 2, 3, 4]
>>> def test_func():
... items = [(lambda: i) for i in range(5)]
... return [x() for x in items]
>>> test_func()
[4, 4, 4, 4, 4]
>>> def test_func():
... items = [(lambda: i) for i in range(5)]
... i = 20
... return [x() for x in items]
>>> test_func()
[4, 4, 4, 4, 4]
>>> def test_func():
... items = [(lambda: y) for i in range(5)]
... y = 2
... return [x() for x in items]
>>> test_func()
[2, 2, 2, 2, 2]
"""
class ListComprehensionTest(unittest.TestCase):
def _check_in_scopes(self, code, outputs=None, ns=None, scopes=None, raises=()):
code = textwrap.dedent(code)
scopes = scopes or ["module", "class", "function"]
for scope in scopes:
with self.subTest(scope=scope):
if scope == "class":
newcode = textwrap.dedent("""
class _C:
{code}
""").format(code=textwrap.indent(code, " "))
def get_output(moddict, name):
return getattr(moddict["_C"], name)
elif scope == "function":
newcode = textwrap.dedent("""
def _f():
{code}
return locals()
_out = _f()
""").format(code=textwrap.indent(code, " "))
def get_output(moddict, name):
return moddict["_out"][name]
else:
newcode = code
def get_output(moddict, name):
return moddict[name]
ns = ns or {}
try:
exec(newcode, ns)
except raises as e:
# We care about e.g. NameError vs UnboundLocalError
self.assertIs(type(e), raises)
else:
for k, v in (outputs or {}).items():
self.assertEqual(get_output(ns, k), v)
def test_lambdas_with_iteration_var_as_default(self):
code = """
items = [(lambda i=i: i) for i in range(5)]
y = [x() for x in items]
"""
outputs = {"y": [0, 1, 2, 3, 4]}
self._check_in_scopes(code, outputs)
def test_lambdas_with_free_var(self):
code = """
items = [(lambda: i) for i in range(5)]
y = [x() for x in items]
"""
outputs = {"y": [4, 4, 4, 4, 4]}
self._check_in_scopes(code, outputs)
def test_class_scope_free_var_with_class_cell(self):
class C:
def method(self):
super()
return __class__
items = [(lambda: i) for i in range(5)]
y = [x() for x in items]
self.assertEqual(C.y, [4, 4, 4, 4, 4])
self.assertIs(C().method(), C)
def test_inner_cell_shadows_outer(self):
code = """
items = [(lambda: i) for i in range(5)]
i = 20
y = [x() for x in items]
"""
outputs = {"y": [4, 4, 4, 4, 4], "i": 20}
self._check_in_scopes(code, outputs)
def test_closure_can_jump_over_comp_scope(self):
code = """
items = [(lambda: y) for i in range(5)]
y = 2
z = [x() for x in items]
"""
outputs = {"z": [2, 2, 2, 2, 2]}
self._check_in_scopes(code, outputs)
def test_inner_cell_shadows_outer_redefined(self):
code = """
y = 10
items = [(lambda: y) for y in range(5)]
x = y
y = 20
out = [z() for z in items]
"""
outputs = {"x": 10, "out": [4, 4, 4, 4, 4]}
self._check_in_scopes(code, outputs)
def test_shadows_outer_cell(self):
code = """
def inner():
return g
[g for g in range(5)]
x = inner()
"""
outputs = {"x": -1}
self._check_in_scopes(code, outputs, ns={"g": -1})
def test_assignment_expression(self):
code = """
x = -1
items = [(x:=y) for y in range(3)]
"""
outputs = {"x": 2}
# assignment expression in comprehension is disallowed in class scope
self._check_in_scopes(code, outputs, scopes=["module", "function"])
def test_free_var_in_comp_child(self):
code = """
lst = range(3)
funcs = [lambda: x for x in lst]
inc = [x + 1 for x in lst]
[x for x in inc]
x = funcs[0]()
"""
outputs = {"x": 2}
self._check_in_scopes(code, outputs)
def test_shadow_with_free_and_local(self):
code = """
lst = range(3)
x = -1
funcs = [lambda: x for x in lst]
items = [x + 1 for x in lst]
"""
outputs = {"x": -1}
self._check_in_scopes(code, outputs)
def test_shadow_comp_iterable_name(self):
code = """
x = [1]
y = [x for x in x]
"""
outputs = {"x": [1]}
self._check_in_scopes(code, outputs)
def test_nested_free(self):
code = """
x = 1
def g():
[x for x in range(3)]
return x
g()
"""
outputs = {"x": 1}
self._check_in_scopes(code, outputs)
def test_introspecting_frame_locals(self):
code = """
import sys
[i for i in range(2)]
i = 20
sys._getframe().f_locals
"""
outputs = {"i": 20}
self._check_in_scopes(code, outputs)
def test_nested(self):
code = """
l = [2, 3]
y = [[x ** 2 for x in range(x)] for x in l]
"""
outputs = {"y": [[0, 1], [0, 1, 4]]}
self._check_in_scopes(code, outputs)
def test_nested_2(self):
code = """
l = [1, 2, 3]
x = 3
y = [x for [x ** x for x in range(x)][x - 1] in l]
"""
outputs = {"y": [3, 3, 3]}
self._check_in_scopes(code, outputs)
def test_nested_3(self):
code = """
l = [(1, 2), (3, 4), (5, 6)]
y = [x for (x, [x ** x for x in range(x)][x - 1]) in l]
"""
outputs = {"y": [1, 3, 5]}
self._check_in_scopes(code, outputs)
def test_nameerror(self):
code = """
[x for x in [1]]
x
"""
self._check_in_scopes(code, raises=NameError)
def test_dunder_name(self):
code = """
y = [__x for __x in [1]]
"""
outputs = {"y": [1]}
self._check_in_scopes(code, outputs)
def test_unbound_local_after_comprehension(self):
def f():
if False:
x = 0
[x for x in [1]]
return x
with self.assertRaises(UnboundLocalError):
f()
def test_unbound_local_inside_comprehension(self):
def f():
l = [None]
return [1 for (l[0], l) in [[1, 2]]]
with self.assertRaises(UnboundLocalError):
f()
__test__ = {'doctests' : doctests}
def load_tests(loader, tests, pattern):

4
Lib/test/test_trace.py
View File

@ -187,9 +187,7 @@ class TestLineCounts(unittest.TestCase):
firstlineno_called = get_firstlineno(traced_doubler)
expected = {
(self.my_py_filename, firstlineno_calling + 1): 1,
# List comprehensions work differently in 3.x, so the count
# below changed compared to 2.x.
(self.my_py_filename, firstlineno_calling + 2): 12,
(self.my_py_filename, firstlineno_calling + 2): 11,
(self.my_py_filename, firstlineno_calling + 3): 1,
(self.my_py_filename, firstlineno_called + 1): 10,
}

2
Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst Normal file
View File

@ -0,0 +1,2 @@
:pep:`709`: inline list, dict and set comprehensions to improve performance
and reduce bytecode size.

2
Modules/_testinternalcapi.c
View File

@ -669,12 +669,14 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module,
umd.u_varnames = PyDict_GetItemString(metadata, "varnames");
umd.u_cellvars = PyDict_GetItemString(metadata, "cellvars");
umd.u_freevars = PyDict_GetItemString(metadata, "freevars");
umd.u_fasthidden = PyDict_GetItemString(metadata, "fasthidden");
assert(PyDict_Check(umd.u_consts));
assert(PyDict_Check(umd.u_names));
assert(PyDict_Check(umd.u_varnames));
assert(PyDict_Check(umd.u_cellvars));
assert(PyDict_Check(umd.u_freevars));
assert(PyDict_Check(umd.u_fasthidden));
umd.u_argcount = get_nonnegative_int_from_dict(metadata, "argcount");
umd.u_posonlyargcount = get_nonnegative_int_from_dict(metadata, "posonlyargcount");

4
Objects/frameobject.c
View File

@ -1224,6 +1224,10 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame)
}
PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i);
_PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i);
if (kind & CO_FAST_HIDDEN) {
continue;
}
if (value == NULL) {
if (PyObject_DelItem(locals, name) != 0) {
if (PyErr_ExceptionMatches(PyExc_KeyError)) {

3
Python/assemble.c
View File

@ -456,6 +456,9 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus,
assert(offset < nlocalsplus);
// For now we do not distinguish arg kinds.
_PyLocals_Kind kind = CO_FAST_LOCAL;
if (PyDict_Contains(umd->u_fasthidden, k)) {
kind |= CO_FAST_HIDDEN;
}
if (PyDict_GetItem(umd->u_cellvars, k) != NULL) {
kind |= CO_FAST_CELL;
}

6
Python/bytecodes.c
View File

@ -194,6 +194,12 @@ dummy_func(
Py_INCREF(value);
}
inst(LOAD_FAST_AND_CLEAR, (-- value)) {
value = GETLOCAL(oparg);
// do not use SETLOCAL here, it decrefs the old value
GETLOCAL(oparg) = NULL;
}
inst(LOAD_CONST, (-- value)) {
value = GETITEM(frame->f_code->co_consts, oparg);
Py_INCREF(value);

361
Python/compile.c
View File

@ -381,7 +381,6 @@ struct compiler_unit {
int u_scope_type;
PyObject *u_private; /* for private name mangling */
instr_sequence u_instr_sequence; /* codegen output */
@ -485,13 +484,15 @@ static int compiler_sync_comprehension_generator(
struct compiler *c, location loc,
asdl_comprehension_seq *generators, int gen_index,
int depth,
expr_ty elt, expr_ty val, int type);
expr_ty elt, expr_ty val, int type,
int iter_on_stack);
static int compiler_async_comprehension_generator(
struct compiler *c, location loc,
asdl_comprehension_seq *generators, int gen_index,
int depth,
expr_ty elt, expr_ty val, int type);
expr_ty elt, expr_ty val, int type,
int iter_on_stack);
static int compiler_pattern(struct compiler *, pattern_ty, pattern_context *);
static int compiler_match(struct compiler *, stmt_ty);
@ -689,6 +690,7 @@ compiler_unit_free(struct compiler_unit *u)
Py_CLEAR(u->u_metadata.u_varnames);
Py_CLEAR(u->u_metadata.u_freevars);
Py_CLEAR(u->u_metadata.u_cellvars);
Py_CLEAR(u->u_metadata.u_fasthidden);
Py_CLEAR(u->u_private);
PyObject_Free(u);
}
@ -837,6 +839,8 @@ stack_effect(int opcode, int oparg, int jump)
* if an exception be raised. */
return jump ? 1 : 0;
case STORE_FAST_MAYBE_NULL:
return -1;
case LOAD_METHOD:
return 1;
case LOAD_SUPER_METHOD:
@ -1239,11 +1243,9 @@ compiler_enter_scope(struct compiler *c, identifier name,
}
if (u->u_ste->ste_needs_class_closure) {
/* Cook up an implicit __class__ cell. */
int res;
Py_ssize_t res;
assert(u->u_scope_type == COMPILER_SCOPE_CLASS);
assert(PyDict_GET_SIZE(u->u_metadata.u_cellvars) == 0);
res = PyDict_SetItem(u->u_metadata.u_cellvars, &_Py_ID(__class__),
_PyLong_GetZero());
res = dict_add_o(u->u_metadata.u_cellvars, &_Py_ID(__class__));
if (res < 0) {
compiler_unit_free(u);
return ERROR;
@ -1257,6 +1259,12 @@ compiler_enter_scope(struct compiler *c, identifier name,
return ERROR;
}
u->u_metadata.u_fasthidden = PyDict_New();
if (!u->u_metadata.u_fasthidden) {
compiler_unit_free(u);
return ERROR;
}
u->u_nfblocks = 0;
u->u_metadata.u_firstlineno = lineno;
u->u_metadata.u_consts = PyDict_New();
@ -2235,7 +2243,6 @@ compiler_class(struct compiler *c, stmt_ty s)
compiler_exit_scope(c);
return ERROR;
}
assert(i == 0);
ADDOP_I(c, NO_LOCATION, LOAD_CLOSURE, i);
ADDOP_I(c, NO_LOCATION, COPY, 1);
if (compiler_nameop(c, NO_LOCATION, &_Py_ID(__classcell__), Store) < 0) {
@ -2245,7 +2252,6 @@ compiler_class(struct compiler *c, stmt_ty s)
}
else {
/* No methods referenced __class__, so just return None */
assert(PyDict_GET_SIZE(c->u->u_metadata.u_cellvars) == 0);
ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None);
}
ADDOP_IN_SCOPE(c, NO_LOCATION, RETURN_VALUE);
@ -3718,7 +3724,8 @@ compiler_nameop(struct compiler *c, location loc,
optype = OP_DEREF;
break;
case LOCAL:
if (c->u->u_ste->ste_type == FunctionBlock)
if (c->u->u_ste->ste_type == FunctionBlock ||
(PyDict_GetItem(c->u->u_metadata.u_fasthidden, mangled) == Py_True))
optype = OP_FAST;
break;
case GLOBAL_IMPLICIT:
@ -4742,16 +4749,19 @@ static int
compiler_comprehension_generator(struct compiler *c, location loc,
asdl_comprehension_seq *generators, int gen_index,
int depth,
expr_ty elt, expr_ty val, int type)
expr_ty elt, expr_ty val, int type,
int iter_on_stack)
{
comprehension_ty gen;
gen = (comprehension_ty)asdl_seq_GET(generators, gen_index);
if (gen->is_async) {
return compiler_async_comprehension_generator(
c, loc, generators, gen_index, depth, elt, val, type);
c, loc, generators, gen_index, depth, elt, val, type,
iter_on_stack);
} else {
return compiler_sync_comprehension_generator(
c, loc, generators, gen_index, depth, elt, val, type);
c, loc, generators, gen_index, depth, elt, val, type,
iter_on_stack);
}
}
@ -4759,7 +4769,8 @@ static int
compiler_sync_comprehension_generator(struct compiler *c, location loc,
asdl_comprehension_seq *generators,
int gen_index, int depth,
expr_ty elt, expr_ty val, int type)
expr_ty elt, expr_ty val, int type,
int iter_on_stack)
{
/* generate code for the iterator, then each of the ifs,
and then write to the element */
@ -4771,37 +4782,39 @@ compiler_sync_comprehension_generator(struct compiler *c, location loc,
comprehension_ty gen = (comprehension_ty)asdl_seq_GET(generators,
gen_index);
if (gen_index == 0) {
/* Receive outermost iter as an implicit argument */
c->u->u_metadata.u_argcount = 1;
ADDOP_I(c, loc, LOAD_FAST, 0);
}
else {
/* Sub-iter - calculate on the fly */
/* Fast path for the temporary variable assignment idiom:
for y in [f(x)]
*/
asdl_expr_seq *elts;
switch (gen->iter->kind) {
case List_kind:
elts = gen->iter->v.List.elts;
break;
case Tuple_kind:
elts = gen->iter->v.Tuple.elts;
break;
default:
elts = NULL;
if (!iter_on_stack) {
if (gen_index == 0) {
/* Receive outermost iter as an implicit argument */
c->u->u_metadata.u_argcount = 1;
ADDOP_I(c, loc, LOAD_FAST, 0);
}
if (asdl_seq_LEN(elts) == 1) {
expr_ty elt = asdl_seq_GET(elts, 0);
if (elt->kind != Starred_kind) {
VISIT(c, expr, elt);
start = NO_LABEL;
else {
/* Sub-iter - calculate on the fly */
/* Fast path for the temporary variable assignment idiom:
for y in [f(x)]
*/
asdl_expr_seq *elts;
switch (gen->iter->kind) {
case List_kind:
elts = gen->iter->v.List.elts;
break;
case Tuple_kind:
elts = gen->iter->v.Tuple.elts;
break;
default:
elts = NULL;
}
if (asdl_seq_LEN(elts) == 1) {
expr_ty elt = asdl_seq_GET(elts, 0);
if (elt->kind != Starred_kind) {
VISIT(c, expr, elt);
start = NO_LABEL;
}
}
if (IS_LABEL(start)) {
VISIT(c, expr, gen->iter);
ADDOP(c, loc, GET_ITER);
}
}
if (IS_LABEL(start)) {
VISIT(c, expr, gen->iter);
ADDOP(c, loc, GET_ITER);
}
}
if (IS_LABEL(start)) {
@ -4822,7 +4835,7 @@ compiler_sync_comprehension_generator(struct compiler *c, location loc,
RETURN_IF_ERROR(
compiler_comprehension_generator(c, loc,
generators, gen_index, depth,
elt, val, type));
elt, val, type, 0));
}
location elt_loc = LOC(elt);
@ -4875,7 +4888,8 @@ static int
compiler_async_comprehension_generator(struct compiler *c, location loc,
asdl_comprehension_seq *generators,
int gen_index, int depth,
expr_ty elt, expr_ty val, int type)
expr_ty elt, expr_ty val, int type,
int iter_on_stack)
{
NEW_JUMP_TARGET_LABEL(c, start);
NEW_JUMP_TARGET_LABEL(c, except);
@ -4884,15 +4898,17 @@ compiler_async_comprehension_generator(struct compiler *c, location loc,
comprehension_ty gen = (comprehension_ty)asdl_seq_GET(generators,
gen_index);
if (gen_index == 0) {
/* Receive outermost iter as an implicit argument */
c->u->u_metadata.u_argcount = 1;
ADDOP_I(c, loc, LOAD_FAST, 0);
}
else {
/* Sub-iter - calculate on the fly */
VISIT(c, expr, gen->iter);
ADDOP(c, loc, GET_AITER);
if (!iter_on_stack) {
if (gen_index == 0) {
/* Receive outermost iter as an implicit argument */
c->u->u_metadata.u_argcount = 1;
ADDOP_I(c, loc, LOAD_FAST, 0);
}
else {
/* Sub-iter - calculate on the fly */
VISIT(c, expr, gen->iter);
ADDOP(c, loc, GET_AITER);
}
}
USE_LABEL(c, start);
@ -4919,7 +4935,7 @@ compiler_async_comprehension_generator(struct compiler *c, location loc,
RETURN_IF_ERROR(
compiler_comprehension_generator(c, loc,
generators, gen_index, depth,
elt, val, type));
elt, val, type, 0));
}
location elt_loc = LOC(elt);
@ -4968,26 +4984,212 @@ compiler_async_comprehension_generator(struct compiler *c, location loc,
return SUCCESS;
}
typedef struct {
PyObject *pushed_locals;
PyObject *temp_symbols;
PyObject *fast_hidden;
} inlined_comprehension_state;
static int
push_inlined_comprehension_state(struct compiler *c, location loc,
PySTEntryObject *entry,
inlined_comprehension_state *state)
{
// iterate over names bound in the comprehension and ensure we isolate
// them from the outer scope as needed
PyObject *k, *v;
Py_ssize_t pos = 0;
while (PyDict_Next(entry->ste_symbols, &pos, &k, &v)) {
assert(PyLong_Check(v));
long symbol = PyLong_AS_LONG(v);
// only values bound in the comprehension (DEF_LOCAL) need to be handled
// at all; DEF_LOCAL | DEF_NONLOCAL can occur in the case of an
// assignment expression to a nonlocal in the comprehension, these don't
// need handling here since they shouldn't be isolated
if (symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) {
if (c->u->u_ste->ste_type != FunctionBlock) {
// non-function scope: override this name to use fast locals
PyObject *orig = PyDict_GetItem(c->u->u_metadata.u_fasthidden, k);
if (orig != Py_True) {
if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_True) < 0) {
return ERROR;
}
if (state->fast_hidden == NULL) {
state->fast_hidden = PySet_New(NULL);
if (state->fast_hidden == NULL) {
return ERROR;
}
}
if (PySet_Add(state->fast_hidden, k) < 0) {
return ERROR;
}
}
}
long scope = (symbol >> SCOPE_OFFSET) & SCOPE_MASK;
PyObject *outv = PyDict_GetItemWithError(c->u->u_ste->ste_symbols, k);
if (outv == NULL) {
return ERROR;
}
assert(PyLong_Check(outv));
long outsc = (PyLong_AS_LONG(outv) >> SCOPE_OFFSET) & SCOPE_MASK;
if (scope != outsc) {
// If a name has different scope inside than outside the
// comprehension, we need to temporarily handle it with the
// right scope while compiling the comprehension.
if (state->temp_symbols == NULL) {
state->temp_symbols = PyDict_New();
if (state->temp_symbols == NULL) {
return ERROR;
}
}
// update the symbol to the in-comprehension version and save
// the outer version; we'll restore it after running the
// comprehension
Py_INCREF(outv);
if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v) < 0) {
Py_DECREF(outv);
return ERROR;
}
if (PyDict_SetItem(state->temp_symbols, k, outv) < 0) {
Py_DECREF(outv);
return ERROR;
}
Py_DECREF(outv);
}
if (outsc == LOCAL || outsc == CELL || outsc == FREE) {
// local names bound in comprehension must be isolated from
// outer scope; push existing value (which may be NULL if
// not defined) on stack
if (state->pushed_locals == NULL) {
state->pushed_locals = PyList_New(0);
if (state->pushed_locals == NULL) {
return ERROR;
}
}
// in the case of a cell, this will actually push the cell
// itself to the stack, then we'll create a new one for the
// comprehension and restore the original one after
ADDOP_NAME(c, loc, LOAD_FAST_AND_CLEAR, k, varnames);
if (scope == CELL) {
ADDOP_NAME(c, loc, MAKE_CELL, k, cellvars);
}
if (PyList_Append(state->pushed_locals, k) < 0) {
return ERROR;
}
}
}
}
if (state->pushed_locals) {
// Outermost iterable expression was already evaluated and is on the
// stack, we need to swap it back to TOS. This also rotates the order of
// `pushed_locals` on the stack, but this will be reversed when we swap
// out the comprehension result in pop_inlined_comprehension_state
ADDOP_I(c, loc, SWAP, PyList_GET_SIZE(state->pushed_locals) + 1);
}
return SUCCESS;
}
static int
pop_inlined_comprehension_state(struct compiler *c, location loc,
inlined_comprehension_state state)
{
PyObject *k, *v;
Py_ssize_t pos = 0;
if (state.temp_symbols) {
while (PyDict_Next(state.temp_symbols, &pos, &k, &v)) {
if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v)) {
return ERROR;
}
}
Py_CLEAR(state.temp_symbols);
}
if (state.pushed_locals) {
// pop names we pushed to stack earlier
Py_ssize_t npops = PyList_GET_SIZE(state.pushed_locals);
// Preserve the list/dict/set result of the comprehension as TOS. This
// reverses the SWAP we did in push_inlined_comprehension_state to get
// the outermost iterable to TOS, so we can still just iterate
// pushed_locals in simple reverse order
ADDOP_I(c, loc, SWAP, npops + 1);
for (Py_ssize_t i = npops - 1; i >= 0; --i) {
k = PyList_GetItem(state.pushed_locals, i);
if (k == NULL) {
return ERROR;
}
ADDOP_NAME(c, loc, STORE_FAST_MAYBE_NULL, k, varnames);
}
Py_CLEAR(state.pushed_locals);
}
if (state.fast_hidden) {
while (PySet_Size(state.fast_hidden) > 0) {
PyObject *k = PySet_Pop(state.fast_hidden);
if (k == NULL) {
return ERROR;
}
// we set to False instead of clearing, so we can track which names
// were temporarily fast-locals and should use CO_FAST_HIDDEN
if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_False)) {
Py_DECREF(k);
return ERROR;
}
Py_DECREF(k);
}
Py_CLEAR(state.fast_hidden);
}
return SUCCESS;
}
static inline int
compiler_comprehension_iter(struct compiler *c, location loc,
comprehension_ty comp)
{
VISIT(c, expr, comp->iter);
if (comp->is_async) {
ADDOP(c, loc, GET_AITER);
}
else {
ADDOP(c, loc, GET_ITER);
}
return SUCCESS;
}
static int
compiler_comprehension(struct compiler *c, expr_ty e, int type,
identifier name, asdl_comprehension_seq *generators, expr_ty elt,
expr_ty val)
{
PyCodeObject *co = NULL;
inlined_comprehension_state inline_state = {NULL, NULL};
comprehension_ty outermost;
int scope_type = c->u->u_scope_type;
int is_async_generator = 0;
int is_top_level_await = IS_TOP_LEVEL_AWAIT(c);
outermost = (comprehension_ty) asdl_seq_GET(generators, 0);
if (compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION,
(void *)e, e->lineno) < 0)
{
PySTEntryObject *entry = PySymtable_Lookup(c->c_st, (void *)e);
if (entry == NULL) {
goto error;
}
int is_inlined = entry->ste_comp_inlined;
int is_async_generator = entry->ste_coroutine;
location loc = LOC(e);
is_async_generator = c->u->u_ste->ste_coroutine;
outermost = (comprehension_ty) asdl_seq_GET(generators, 0);
if (is_inlined) {
if (compiler_comprehension_iter(c, loc, outermost)) {
goto error;
}
if (push_inlined_comprehension_state(c, loc, entry, &inline_state)) {
goto error;
}
}
else {
if (compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION,
(void *)e, e->lineno) < 0)
{
goto error;
}
}
Py_CLEAR(entry);
if (is_async_generator && type != COMP_GENEXP &&
scope_type != COMPILER_SCOPE_ASYNC_FUNCTION &&
@ -5018,13 +5220,23 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type,
}
ADDOP_I(c, loc, op, 0);
if (is_inlined) {
ADDOP_I(c, loc, SWAP, 2);
}
}
if (compiler_comprehension_generator(c, loc, generators, 0, 0,
elt, val, type) < 0) {
elt, val, type, is_inlined) < 0) {
goto error_in_scope;
}
if (is_inlined) {
if (pop_inlined_comprehension_state(c, loc, inline_state)) {
goto error;
}
return SUCCESS;
}
if (type != COMP_GENEXP) {
ADDOP(c, LOC(e), RETURN_VALUE);
}
@ -5047,15 +5259,10 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type,
if (compiler_make_closure(c, loc, co, 0) < 0) {
goto error;
}
Py_DECREF(co);
Py_CLEAR(co);
VISIT(c, expr, outermost->iter);
loc = LOC(e);
if (outermost->is_async) {
ADDOP(c, loc, GET_AITER);
} else {
ADDOP(c, loc, GET_ITER);
if (compiler_comprehension_iter(c, loc, outermost)) {
goto error;
}
ADDOP_I(c, loc, CALL, 0);
@ -5068,9 +5275,15 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type,
return SUCCESS;
error_in_scope:
compiler_exit_scope(c);
if (!is_inlined) {
compiler_exit_scope(c);
}
error:
Py_XDECREF(co);
Py_XDECREF(entry);
Py_XDECREF(inline_state.pushed_locals);
Py_XDECREF(inline_state.temp_symbols);
Py_XDECREF(inline_state.fast_hidden);
return ERROR;
}
@ -6989,7 +7202,7 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache,
goto error;
}
_PyCfg_ConvertExceptionHandlersToNops(g.g_entryblock);
_PyCfg_ConvertPseudoOps(g.g_entryblock);
/* Order of basic blocks must have been determined by now */
@ -7401,7 +7614,7 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename,
goto error;
}
_PyCfg_ConvertExceptionHandlersToNops(g.g_entryblock);
_PyCfg_ConvertPseudoOps(g.g_entryblock);
/* Order of basic blocks must have been determined by now */

16
Python/flowgraph.c
View File

@ -1289,7 +1289,9 @@ swaptimize(basicblock *block, int *ix)
// - can't invoke arbitrary code (besides finalizers)
// - only touch the TOS (and pop it when finished)
#define SWAPPABLE(opcode) \
((opcode) == STORE_FAST || (opcode) == POP_TOP)
((opcode) == STORE_FAST || \
(opcode) == STORE_FAST_MAYBE_NULL || \
(opcode) == POP_TOP)
static int
next_swappable_instruction(basicblock *block, int i, int lineno)
@ -1600,6 +1602,8 @@ scan_block_for_locals(basicblock *b, basicblock ***sp)
uint64_t bit = (uint64_t)1 << instr->i_oparg;
switch (instr->i_opcode) {
case DELETE_FAST:
case LOAD_FAST_AND_CLEAR:
case STORE_FAST_MAYBE_NULL:
unsafe_mask |= bit;
break;
case STORE_FAST:
@ -1639,7 +1643,8 @@ fast_scan_many_locals(basicblock *entryblock, int nlocals)
Py_ssize_t blocknum = 0;
// state[i - 64] == blocknum if local i is guaranteed to
// be initialized, i.e., if it has had a previous LOAD_FAST or
// STORE_FAST within that basicblock (not followed by DELETE_FAST).
// STORE_FAST within that basicblock (not followed by
// DELETE_FAST/LOAD_FAST_AND_CLEAR/STORE_FAST_MAYBE_NULL).
for (basicblock *b = entryblock; b != NULL; b = b->b_next) {
blocknum++;
for (int i = 0; i < b->b_iused; i++) {
@ -1653,6 +1658,8 @@ fast_scan_many_locals(basicblock *entryblock, int nlocals)
assert(arg >= 0);
switch (instr->i_opcode) {
case DELETE_FAST:
case LOAD_FAST_AND_CLEAR:
case STORE_FAST_MAYBE_NULL:
states[arg - 64] = blocknum - 1;
break;
case STORE_FAST:
@ -1975,7 +1982,7 @@ push_cold_blocks_to_end(cfg_builder *g, int code_flags) {
}
void
_PyCfg_ConvertExceptionHandlersToNops(basicblock *entryblock)
_PyCfg_ConvertPseudoOps(basicblock *entryblock)
{
for (basicblock *b = entryblock; b != NULL; b = b->b_next) {
for (int i = 0; i < b->b_iused; i++) {
@ -1983,6 +1990,9 @@ _PyCfg_ConvertExceptionHandlersToNops(basicblock *entryblock)
if (is_block_push(instr) || instr->i_opcode == POP_BLOCK) {
INSTR_SET_OP0(instr, NOP);
}
else if (instr->i_opcode == STORE_FAST_MAYBE_NULL) {
instr->i_opcode = STORE_FAST;
}
}
}
for (basicblock *b = entryblock; b != NULL; b = b->b_next) {

962
Python/generated_cases.c.h generated
View File

File diff suppressed because it is too large Load Diff

5
Python/opcode_metadata.h
View File

@ -21,6 +21,8 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) {
return 0;
case LOAD_FAST:
return 0;
case LOAD_FAST_AND_CLEAR:
return 0;
case LOAD_CONST:
return 0;
case STORE_FAST:
@ -409,6 +411,8 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) {
return 1;
case LOAD_FAST:
return 1;
case LOAD_FAST_AND_CLEAR:
return 1;
case LOAD_CONST:
return 1;
case STORE_FAST:
@ -795,6 +799,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = {
[LOAD_CLOSURE] = { true, INSTR_FMT_IB },
[LOAD_FAST_CHECK] = { true, INSTR_FMT_IB },
[LOAD_FAST] = { true, INSTR_FMT_IB },
[LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB },
[LOAD_CONST] = { true, INSTR_FMT_IB },
[STORE_FAST] = { true, INSTR_FMT_IB },
[LOAD_FAST__LOAD_FAST] = { true, INSTR_FMT_IBIB },

8
Python/opcode_targets.h generated
View File

@ -142,7 +142,7 @@ static void *opcode_targets[256] = {
&&TARGET_JUMP_BACKWARD,
&&TARGET_LOAD_SUPER_ATTR,
&&TARGET_CALL_FUNCTION_EX,
&&TARGET_STORE_FAST__LOAD_FAST,
&&TARGET_LOAD_FAST_AND_CLEAR,
&&TARGET_EXTENDED_ARG,
&&TARGET_LIST_APPEND,
&&TARGET_SET_ADD,
@ -152,24 +152,24 @@ static void *opcode_targets[256] = {
&&TARGET_YIELD_VALUE,
&&TARGET_RESUME,
&&TARGET_MATCH_CLASS,
&&TARGET_STORE_FAST__LOAD_FAST,
&&TARGET_STORE_FAST__STORE_FAST,
&&TARGET_STORE_SUBSCR_DICT,
&&TARGET_FORMAT_VALUE,
&&TARGET_BUILD_CONST_KEY_MAP,
&&TARGET_BUILD_STRING,
&&TARGET_STORE_SUBSCR_DICT,
&&TARGET_STORE_SUBSCR_LIST_INT,
&&TARGET_UNPACK_SEQUENCE_LIST,
&&TARGET_UNPACK_SEQUENCE_TUPLE,
&&TARGET_UNPACK_SEQUENCE_TWO_TUPLE,
&&TARGET_LIST_EXTEND,
&&TARGET_SET_UPDATE,
&&TARGET_DICT_MERGE,
&&TARGET_DICT_UPDATE,
&&TARGET_UNPACK_SEQUENCE_TWO_TUPLE,
&&TARGET_SEND_GEN,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_CALL,
&&TARGET_KW_NAMES,
&&TARGET_CALL_INTRINSIC_1,

136
Python/symtable.c
View File

@ -103,6 +103,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block,
ste->ste_comprehension = NoComprehension;
ste->ste_returns_value = 0;
ste->ste_needs_class_closure = 0;
ste->ste_comp_inlined = 0;
ste->ste_comp_iter_target = 0;
ste->ste_comp_iter_expr = 0;
@ -558,6 +559,67 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags,
return 1;
}
static int
is_free_in_any_child(PySTEntryObject *entry, PyObject *key)
{
for (Py_ssize_t i = 0; i < PyList_GET_SIZE(entry->ste_children); i++) {
PySTEntryObject *child_ste = (PySTEntryObject *)PyList_GET_ITEM(
entry->ste_children, i);
long scope = _PyST_GetScope(child_ste, key);
if (scope == FREE) {
return 1;
}
}
return 0;
}
static int
inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp,
PyObject *scopes, PyObject *comp_free)
{
PyObject *k, *v;
Py_ssize_t pos = 0;
while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) {
// skip comprehension parameter
long comp_flags = PyLong_AS_LONG(v);
if (comp_flags & DEF_PARAM) {
assert(_PyUnicode_EqualToASCIIString(k, ".0"));
continue;
}
int scope = (comp_flags >> SCOPE_OFFSET) & SCOPE_MASK;
int only_flags = comp_flags & ((1 << SCOPE_OFFSET) - 1);
PyObject *existing = PyDict_GetItemWithError(ste->ste_symbols, k);
if (existing == NULL && PyErr_Occurred()) {
return 0;
}
if (!existing) {
// name does not exist in scope, copy from comprehension
assert(scope != FREE || PySet_Contains(comp_free, k) == 1);
PyObject *v_flags = PyLong_FromLong(only_flags);
if (v_flags == NULL) {
return 0;
}
int ok = PyDict_SetItem(ste->ste_symbols, k, v_flags);
Py_DECREF(v_flags);
if (ok < 0) {
return 0;
}
SET_SCOPE(scopes, k, scope);
}
else {
// free vars in comprehension that are locals in outer scope can
// now simply be locals, unless they are free in comp children
if ((PyLong_AsLong(existing) & DEF_BOUND) &&
!is_free_in_any_child(comp, k)) {
if (PySet_Discard(comp_free, k) < 0) {
return 0;
}
}
}
}
return 1;
}
#undef SET_SCOPE
/* If a name is defined in free and also in locals, then this block
@ -727,17 +789,17 @@ error:
static int
analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free,
PyObject *global, PyObject* child_free);
PyObject *global, PyObject **child_free);
static int
analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free,
PyObject *global)
{
PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL;
PyObject *newglobal = NULL, *newfree = NULL, *allfree = NULL;
PyObject *newglobal = NULL, *newfree = NULL;
PyObject *temp;
int i, success = 0;
Py_ssize_t pos = 0;
int success = 0;
Py_ssize_t i, pos = 0;
local = PySet_New(NULL); /* collect new names bound in block */
if (!local)
@ -746,8 +808,8 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free,
if (!scopes)
goto error;
/* Allocate new global and bound variable dictionaries. These
dictionaries hold the names visible in nested blocks. For
/* Allocate new global, bound and free variable sets. These
sets hold the names visible in nested blocks. For
ClassBlocks, the bound and global names are initialized
before analyzing names, because class bindings aren't
visible in methods. For other blocks, they are initialized
@ -826,28 +888,55 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free,
newbound, newglobal now contain the names visible in
nested blocks. The free variables in the children will
be collected in allfree.
be added to newfree.
*/
allfree = PySet_New(NULL);
if (!allfree)
goto error;
for (i = 0; i < PyList_GET_SIZE(ste->ste_children); ++i) {
PyObject *child_free = NULL;
PyObject *c = PyList_GET_ITEM(ste->ste_children, i);
PySTEntryObject* entry;
assert(c && PySTEntry_Check(c));
entry = (PySTEntryObject*)c;
// we inline all non-generator-expression comprehensions
int inline_comp =
entry->ste_comprehension &&
!entry->ste_generator;
if (!analyze_child_block(entry, newbound, newfree, newglobal,
allfree))
&child_free))
{
goto error;
}
if (inline_comp) {
if (!inline_comprehension(ste, entry, scopes, child_free)) {
Py_DECREF(child_free);
goto error;
}
entry->ste_comp_inlined = 1;
}
temp = PyNumber_InPlaceOr(newfree, child_free);
Py_DECREF(child_free);
if (!temp)
goto error;
Py_DECREF(temp);
/* Check if any children have free variables */
if (entry->ste_free || entry->ste_child_free)
ste->ste_child_free = 1;
}
temp = PyNumber_InPlaceOr(newfree, allfree);
if (!temp)
goto error;
Py_DECREF(temp);
/* Splice children of inlined comprehensions into our children list */
for (i = PyList_GET_SIZE(ste->ste_children) - 1; i >= 0; --i) {
PyObject* c = PyList_GET_ITEM(ste->ste_children, i);
PySTEntryObject* entry;
assert(c && PySTEntry_Check(c));
entry = (PySTEntryObject*)c;
if (entry->ste_comp_inlined &&
PyList_SetSlice(ste->ste_children, i, i + 1,
entry->ste_children) < 0)
{
goto error;
}
}
/* Check if any local variables must be converted to cell variables */
if (ste->ste_type == FunctionBlock && !analyze_cells(scopes, newfree))
@ -870,7 +959,6 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free,
Py_XDECREF(newbound);
Py_XDECREF(newglobal);
Py_XDECREF(newfree);
Py_XDECREF(allfree);
if (!success)
assert(PyErr_Occurred());
return success;
@ -878,16 +966,15 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free,
static int
analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free,
PyObject *global, PyObject* child_free)
PyObject *global, PyObject** child_free)
{
PyObject *temp_bound = NULL, *temp_global = NULL, *temp_free = NULL;
PyObject *temp;
/* Copy the bound and global dictionaries.
/* Copy the bound/global/free sets.
These dictionaries are used by all blocks enclosed by the
These sets are used by all blocks enclosed by the
current block. The analyze_block() call modifies these
dictionaries.
sets.
*/
temp_bound = PySet_New(bound);
@ -902,12 +989,8 @@ analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free,
if (!analyze_block(entry, temp_bound, temp_free, temp_global))
goto error;
temp = PyNumber_InPlaceOr(child_free, temp_free);
if (!temp)
goto error;
Py_DECREF(temp);
*child_free = temp_free;
Py_DECREF(temp_bound);
Py_DECREF(temp_free);
Py_DECREF(temp_global);
return 1;
error:
@ -2216,4 +2299,3 @@ _Py_Mangle(PyObject *privateobj, PyObject *ident)
assert(_PyUnicode_CheckConsistency(result, 1));
return result;
}