From 40f3f11a773b854c6d94746aa3b1881c8ac71b0f Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 18 Jul 2023 19:42:44 +0100 Subject: [PATCH] gh-105481: Generate the opcode lists in dis from data extracted from bytecodes.c (#106758) --- .gitattributes | 1 - Doc/library/dis.rst | 28 ++- Include/cpython/compile.h | 4 + Include/internal/pycore_opcode_metadata.h | 36 ++-- Lib/opcode.py | 127 +++++------- Lib/test/test__opcode.py | 63 ++---- ...-07-17-16-46-00.gh-issue-105481.fek_Nn.rst | 1 + Modules/_opcode.c | 60 ++++++ Modules/clinic/_opcode.c.h | 196 +++++++++++++++++- Python/bytecodes.c | 2 +- Python/ceval_macros.h | 1 + Python/compile.c | 18 ++ Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 2 +- Tools/build/generate_opcode_h.py | 6 - Tools/cases_generator/generate_cases.py | 13 +- 16 files changed, 402 insertions(+), 158 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-17-16-46-00.gh-issue-105481.fek_Nn.rst diff --git a/.gitattributes b/.gitattributes index 2616da74b48..5d5558da711 100644 --- a/.gitattributes +++ b/.gitattributes @@ -87,7 +87,6 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated -Include/internal/pycore_opcode_metadata.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 099b6410f16..6beaad3825a 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1803,15 +1803,12 @@ instructions: Sequence of bytecodes that access an attribute by name. -.. data:: hasjrel +.. data:: hasjump - Sequence of bytecodes that have a relative jump target. - - -.. data:: hasjabs - - Sequence of bytecodes that have an absolute jump target. + Sequence of bytecodes that have a jump target. All jumps + are relative. + .. versionadded:: 3.13 .. data:: haslocal @@ -1827,3 +1824,20 @@ instructions: Sequence of bytecodes that set an exception handler. .. versionadded:: 3.12 + + +.. data:: hasjrel + + Sequence of bytecodes that have a relative jump target. + + .. deprecated:: 3.13 + All jumps are now relative. Use :data:`hasjump`. + + +.. data:: hasjabs + + Sequence of bytecodes that have an absolute jump target. + + .. deprecated:: 3.13 + All jumps are now relative. This list is empty. + diff --git a/Include/cpython/compile.h b/Include/cpython/compile.h index cd7fd7bd377..fd526978402 100644 --- a/Include/cpython/compile.h +++ b/Include/cpython/compile.h @@ -73,3 +73,7 @@ PyAPI_FUNC(int) PyUnstable_OpcodeHasArg(int opcode); PyAPI_FUNC(int) PyUnstable_OpcodeHasConst(int opcode); PyAPI_FUNC(int) PyUnstable_OpcodeHasName(int opcode); PyAPI_FUNC(int) PyUnstable_OpcodeHasJump(int opcode); +PyAPI_FUNC(int) PyUnstable_OpcodeHasFree(int opcode); +PyAPI_FUNC(int) PyUnstable_OpcodeHasLocal(int opcode); +PyAPI_FUNC(int) PyUnstable_OpcodeHasExc(int opcode); + diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index a5844b3135d..d525913f8a7 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -955,10 +955,14 @@ enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC00, INSTR_FMT #define HAS_CONST_FLAG (2) #define HAS_NAME_FLAG (4) #define HAS_JUMP_FLAG (8) +#define HAS_FREE_FLAG (16) +#define HAS_LOCAL_FLAG (32) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) #define OPCODE_HAS_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_JUMP_FLAG)) +#define OPCODE_HAS_FREE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_FREE_FLAG)) +#define OPCODE_HAS_LOCAL(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_LOCAL_FLAG)) struct opcode_metadata { bool valid_entry; @@ -995,16 +999,16 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [NOP] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_CLOSURE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [LOAD_CLOSURE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, - [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [STORE_FAST_MAYBE_NULL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_MAYBE_NULL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [POP_TOP] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, 0 }, [END_FOR] = { true, INSTR_FMT_IB, 0 }, @@ -1028,7 +1032,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IBC, 0 }, [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IBC, 0 }, [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IBC, 0 }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IB, 0 }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IB, HAS_LOCAL_FLAG }, [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, 0 }, [BINARY_SLICE] = { true, INSTR_FMT_IX, 0 }, [STORE_SLICE] = { true, INSTR_FMT_IX, 0 }, @@ -1080,12 +1084,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG }, [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, - [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, + [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, + [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, + [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, + [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, diff --git a/Lib/opcode.py b/Lib/opcode.py index 1b36300785a..08dfd2674dc 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -4,10 +4,12 @@ opcode module - potentially shared between dis and other modules which operate on bytecodes (e.g. peephole optimizers). """ -__all__ = ["cmp_op", "hasarg", "hasconst", "hasname", "hasjrel", "hasjabs", - "haslocal", "hascompare", "hasfree", "hasexc", "opname", "opmap", - "stack_effect", "HAVE_ARGUMENT", "EXTENDED_ARG"] +# Note that __all__ is further extended below +__all__ = ["cmp_op", "opname", "opmap", "stack_effect", "hascompare", + "HAVE_ARGUMENT", "EXTENDED_ARG"] + +import _opcode from _opcode import stack_effect import sys @@ -17,55 +19,24 @@ if sys.version_info[:2] >= (3, 13): cmp_op = ('<', '<=', '==', '!=', '>', '>=') -hasarg = [] -hasconst = [] -hasname = [] -hasjrel = [] -hasjabs = [] -haslocal = [] -hascompare = [] -hasfree = [] -hasexc = [] - ENABLE_SPECIALIZATION = True def is_pseudo(op): return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE -oplists = [hasarg, hasconst, hasname, hasjrel, hasjabs, - haslocal, hascompare, hasfree, hasexc] - opmap = {} -## pseudo opcodes (used in the compiler) mapped to the values -## they can become in the actual code. +# pseudo opcodes (used in the compiler) mapped to the values +# they can become in the actual code. _pseudo_ops = {} def def_op(name, op): opmap[name] = op -def name_op(name, op): - def_op(name, op) - hasname.append(op) - -def jrel_op(name, op): - def_op(name, op) - hasjrel.append(op) - -def jabs_op(name, op): - def_op(name, op) - hasjabs.append(op) - def pseudo_op(name, op, real_ops): def_op(name, op) _pseudo_ops[name] = real_ops - # add the pseudo opcode to the lists its targets are in - for oplist in oplists: - res = [opmap[rop] in oplist for rop in real_ops] - if any(res): - assert all(res) - oplist.append(op) # Instruction opcodes for compiled code @@ -137,74 +108,61 @@ def_op('POP_EXCEPT', 89) HAVE_ARGUMENT = 90 # real opcodes from here have an argument: -name_op('STORE_NAME', 90) # Index in name list -name_op('DELETE_NAME', 91) # "" +def_op('STORE_NAME', 90) # Index in name list +def_op('DELETE_NAME', 91) # "" def_op('UNPACK_SEQUENCE', 92) # Number of tuple items -jrel_op('FOR_ITER', 93) +def_op('FOR_ITER', 93) def_op('UNPACK_EX', 94) -name_op('STORE_ATTR', 95) # Index in name list -name_op('DELETE_ATTR', 96) # "" -name_op('STORE_GLOBAL', 97) # "" -name_op('DELETE_GLOBAL', 98) # "" +def_op('STORE_ATTR', 95) # Index in name list +def_op('DELETE_ATTR', 96) # "" +def_op('STORE_GLOBAL', 97) # "" +def_op('DELETE_GLOBAL', 98) # "" def_op('SWAP', 99) def_op('LOAD_CONST', 100) # Index in const list -hasconst.append(100) -name_op('LOAD_NAME', 101) # Index in name list +def_op('LOAD_NAME', 101) # Index in name list def_op('BUILD_TUPLE', 102) # Number of tuple items def_op('BUILD_LIST', 103) # Number of list items def_op('BUILD_SET', 104) # Number of set items def_op('BUILD_MAP', 105) # Number of dict entries -name_op('LOAD_ATTR', 106) # Index in name list +def_op('LOAD_ATTR', 106) # Index in name list def_op('COMPARE_OP', 107) # Comparison operator -hascompare.append(107) -name_op('IMPORT_NAME', 108) # Index in name list -name_op('IMPORT_FROM', 109) # Index in name list -jrel_op('JUMP_FORWARD', 110) # Number of words to skip +def_op('IMPORT_NAME', 108) # Index in name list +def_op('IMPORT_FROM', 109) # Index in name list +def_op('JUMP_FORWARD', 110) # Number of words to skip -jrel_op('POP_JUMP_IF_FALSE', 114) -jrel_op('POP_JUMP_IF_TRUE', 115) -name_op('LOAD_GLOBAL', 116) # Index in name list +def_op('POP_JUMP_IF_FALSE', 114) +def_op('POP_JUMP_IF_TRUE', 115) +def_op('LOAD_GLOBAL', 116) # Index in name list def_op('IS_OP', 117) def_op('CONTAINS_OP', 118) def_op('RERAISE', 119) def_op('COPY', 120) def_op('RETURN_CONST', 121) -hasconst.append(121) def_op('BINARY_OP', 122) -jrel_op('SEND', 123) # Number of words to skip +def_op('SEND', 123) # Number of words to skip def_op('LOAD_FAST', 124) # Local variable number, no null check -haslocal.append(124) def_op('STORE_FAST', 125) # Local variable number -haslocal.append(125) def_op('DELETE_FAST', 126) # Local variable number -haslocal.append(126) def_op('LOAD_FAST_CHECK', 127) # Local variable number -haslocal.append(127) -jrel_op('POP_JUMP_IF_NOT_NONE', 128) -jrel_op('POP_JUMP_IF_NONE', 129) +def_op('POP_JUMP_IF_NOT_NONE', 128) +def_op('POP_JUMP_IF_NONE', 129) def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) def_op('GET_AWAITABLE', 131) def_op('BUILD_SLICE', 133) # Number of items -jrel_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards) +def_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards) def_op('MAKE_CELL', 135) -hasfree.append(135) def_op('LOAD_DEREF', 137) -hasfree.append(137) def_op('STORE_DEREF', 138) -hasfree.append(138) def_op('DELETE_DEREF', 139) -hasfree.append(139) -jrel_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards) -name_op('LOAD_SUPER_ATTR', 141) +def_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards) +def_op('LOAD_SUPER_ATTR', 141) def_op('CALL_FUNCTION_EX', 142) # Flags def_op('LOAD_FAST_AND_CLEAR', 143) # Local variable number -haslocal.append(143) def_op('EXTENDED_ARG', 144) -EXTENDED_ARG = 144 +EXTENDED_ARG = opmap['EXTENDED_ARG'] def_op('LIST_APPEND', 145) def_op('SET_ADD', 146) def_op('MAP_ADD', 147) -hasfree.append(148) def_op('COPY_FREE_VARS', 149) def_op('YIELD_VALUE', 150) def_op('RESUME', 151) # This must be kept in sync with deepfreeze.py @@ -224,12 +182,10 @@ def_op('STORE_FAST_LOAD_FAST', 169) def_op('STORE_FAST_STORE_FAST', 170) def_op('CALL', 171) def_op('KW_NAMES', 172) -hasconst.append(172) def_op('CALL_INTRINSIC_1', 173) def_op('CALL_INTRINSIC_2', 174) -name_op('LOAD_FROM_DICT_OR_GLOBALS', 175) +def_op('LOAD_FROM_DICT_OR_GLOBALS', 175) def_op('LOAD_FROM_DICT_OR_DEREF', 176) -hasfree.append(176) def_op('SET_FUNCTION_ATTRIBUTE', 177) # Attribute # Optimizer hook @@ -258,16 +214,12 @@ def_op('INSTRUMENTED_INSTRUCTION', 253) def_op('INSTRUMENTED_LINE', 254) # 255 is reserved -hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) MIN_PSEUDO_OPCODE = 256 pseudo_op('SETUP_FINALLY', 256, ['NOP']) -hasexc.append(256) pseudo_op('SETUP_CLEANUP', 257, ['NOP']) -hasexc.append(257) pseudo_op('SETUP_WITH', 258, ['NOP']) -hasexc.append(258) pseudo_op('POP_BLOCK', 259, ['NOP']) pseudo_op('JUMP', 260, ['JUMP_FORWARD', 'JUMP_BACKWARD']) @@ -283,12 +235,29 @@ pseudo_op('LOAD_CLOSURE', 267, ['LOAD_FAST']) MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1 -del def_op, name_op, jrel_op, jabs_op, pseudo_op +del def_op, pseudo_op opname = ['<%r>' % (op,) for op in range(MAX_PSEUDO_OPCODE + 1)] for op, i in opmap.items(): opname[i] = op +# The build uses older versions of Python which do not have _opcode.has_* functions +if sys.version_info[:2] >= (3, 13): + # These lists are documented as part of the dis module's API + hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] + hasconst = [op for op in opmap.values() if _opcode.has_const(op)] + hasname = [op for op in opmap.values() if _opcode.has_name(op)] + hasjump = [op for op in opmap.values() if _opcode.has_jump(op)] + hasjrel = hasjump # for backward compatibility + hasjabs = [] + hasfree = [op for op in opmap.values() if _opcode.has_free(op)] + haslocal = [op for op in opmap.values() if _opcode.has_local(op)] + hasexc = [op for op in opmap.values() if _opcode.has_exc(op)] + + __all__.extend(["hasarg", "hasconst", "hasname", "hasjump", "hasjrel", + "hasjabs", "hasfree", "haslocal", "hasexc"]) + +hascompare = [opmap["COMPARE_OP"]] _nb_ops = [ ("NB_ADD", "+"), diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 7d9553d9e38..b3a9bcbe160 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -7,16 +7,7 @@ _opcode = import_module("_opcode") from _opcode import stack_effect -class OpcodeTests(unittest.TestCase): - - def check_bool_function_result(self, func, ops, expected): - for op in ops: - if isinstance(op, str): - op = dis.opmap[op] - with self.subTest(opcode=op, func=func): - self.assertIsInstance(func(op), bool) - self.assertEqual(func(op), expected) - +class OpListTests(unittest.TestCase): def test_invalid_opcodes(self): invalid = [-100, -1, 255, 512, 513, 1000] self.check_bool_function_result(_opcode.is_valid, invalid, False) @@ -24,6 +15,9 @@ class OpcodeTests(unittest.TestCase): self.check_bool_function_result(_opcode.has_const, invalid, False) self.check_bool_function_result(_opcode.has_name, invalid, False) self.check_bool_function_result(_opcode.has_jump, invalid, False) + self.check_bool_function_result(_opcode.has_free, invalid, False) + self.check_bool_function_result(_opcode.has_local, invalid, False) + self.check_bool_function_result(_opcode.has_exc, invalid, False) def test_is_valid(self): names = [ @@ -36,43 +30,24 @@ class OpcodeTests(unittest.TestCase): opcodes = [dis.opmap[opname] for opname in names] self.check_bool_function_result(_opcode.is_valid, opcodes, True) - def test_has_arg(self): - has_arg = ['SWAP', 'LOAD_FAST', 'INSTRUMENTED_POP_JUMP_IF_TRUE', 'JUMP'] - no_arg = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] - self.check_bool_function_result(_opcode.has_arg, has_arg, True) - self.check_bool_function_result(_opcode.has_arg, no_arg, False) + def test_oplists(self): + def check_function(self, func, expected): + for op in [-10, 520]: + with self.subTest(opcode=op, func=func): + res = func(op) + self.assertIsInstance(res, bool) + self.assertEqual(res, op in expected) - def test_has_const(self): - has_const = ['LOAD_CONST', 'RETURN_CONST', 'KW_NAMES'] - no_const = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] - self.check_bool_function_result(_opcode.has_const, has_const, True) - self.check_bool_function_result(_opcode.has_const, no_const, False) + check_function(self, _opcode.has_arg, dis.hasarg) + check_function(self, _opcode.has_const, dis.hasconst) + check_function(self, _opcode.has_name, dis.hasname) + check_function(self, _opcode.has_jump, dis.hasjump) + check_function(self, _opcode.has_free, dis.hasfree) + check_function(self, _opcode.has_local, dis.haslocal) + check_function(self, _opcode.has_exc, dis.hasexc) - def test_has_name(self): - has_name = ['STORE_NAME', 'DELETE_ATTR', 'STORE_GLOBAL', 'IMPORT_FROM', - 'LOAD_FROM_DICT_OR_GLOBALS'] - no_name = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] - self.check_bool_function_result(_opcode.has_name, has_name, True) - self.check_bool_function_result(_opcode.has_name, no_name, False) - - def test_has_jump(self): - has_jump = ['FOR_ITER', 'JUMP_FORWARD', 'JUMP', 'POP_JUMP_IF_TRUE', 'SEND'] - no_jump = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] - self.check_bool_function_result(_opcode.has_jump, has_jump, True) - self.check_bool_function_result(_opcode.has_jump, no_jump, False) - - # the following test is part of the refactor, it will be removed soon - def test_against_legacy_bool_values(self): - # limiting to ops up to ENTER_EXECUTOR, because everything after that - # is not currently categorized correctly in opcode.py. - for op in range(0, opcode.opmap['ENTER_EXECUTOR']): - with self.subTest(op=op): - if opcode.opname[op] != f'<{op}>': - self.assertEqual(op in dis.hasarg, _opcode.has_arg(op)) - self.assertEqual(op in dis.hasconst, _opcode.has_const(op)) - self.assertEqual(op in dis.hasname, _opcode.has_name(op)) - self.assertEqual(op in dis.hasjrel, _opcode.has_jump(op)) +class OpListTests(unittest.TestCase): def test_stack_effect(self): self.assertEqual(stack_effect(dis.opmap['POP_TOP']), -1) self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 0), -1) diff --git a/Misc/NEWS.d/next/Library/2023-07-17-16-46-00.gh-issue-105481.fek_Nn.rst b/Misc/NEWS.d/next/Library/2023-07-17-16-46-00.gh-issue-105481.fek_Nn.rst new file mode 100644 index 00000000000..d82eb987c83 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-17-16-46-00.gh-issue-105481.fek_Nn.rst @@ -0,0 +1 @@ +The various opcode lists in the :mod:`dis` module are now generated from bytecodes.c instead of explicitly constructed in opcode.py. diff --git a/Modules/_opcode.c b/Modules/_opcode.c index b3b9873d21a..daabdce1655 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -147,6 +147,63 @@ _opcode_has_jump_impl(PyObject *module, int opcode) /*[clinic input] +_opcode.has_free -> bool + + opcode: int + +Return True if the opcode accesses a free variable, False otherwise. + +Note that 'free' in this context refers to names in the current scope +that are referenced by inner scopes or names in outer scopes that are +referenced from this scope. It does not include references to global +or builtin scopes. +[clinic start generated code]*/ + +static int +_opcode_has_free_impl(PyObject *module, int opcode) +/*[clinic end generated code: output=d81ae4d79af0ee26 input=117dcd5c19c1139b]*/ +{ + return PyUnstable_OpcodeIsValid(opcode) && + PyUnstable_OpcodeHasFree(opcode); + +} + +/*[clinic input] + +_opcode.has_local -> bool + + opcode: int + +Return True if the opcode accesses a local variable, False otherwise. +[clinic start generated code]*/ + +static int +_opcode_has_local_impl(PyObject *module, int opcode) +/*[clinic end generated code: output=da5a8616b7a5097b input=9a798ee24aaef49d]*/ +{ + return PyUnstable_OpcodeIsValid(opcode) && + PyUnstable_OpcodeHasLocal(opcode); +} + +/*[clinic input] + +_opcode.has_exc -> bool + + opcode: int + +Return True if the opcode sets an exception handler, False otherwise. +[clinic start generated code]*/ + +static int +_opcode_has_exc_impl(PyObject *module, int opcode) +/*[clinic end generated code: output=41b68dff0ec82a52 input=db0e4bdb9bf13fa5]*/ +{ + return PyUnstable_OpcodeIsValid(opcode) && + PyUnstable_OpcodeHasExc(opcode); +} + +/*[clinic input] + _opcode.get_specialization_stats Return the specialization stats @@ -171,6 +228,9 @@ opcode_functions[] = { _OPCODE_HAS_CONST_METHODDEF _OPCODE_HAS_NAME_METHODDEF _OPCODE_HAS_JUMP_METHODDEF + _OPCODE_HAS_FREE_METHODDEF + _OPCODE_HAS_LOCAL_METHODDEF + _OPCODE_HAS_EXC_METHODDEF _OPCODE_GET_SPECIALIZATION_STATS_METHODDEF {NULL, NULL, 0, NULL} }; diff --git a/Modules/clinic/_opcode.c.h b/Modules/clinic/_opcode.c.h index 3eb050e470c..e6381fa73a5 100644 --- a/Modules/clinic/_opcode.c.h +++ b/Modules/clinic/_opcode.c.h @@ -401,6 +401,200 @@ exit: return return_value; } +PyDoc_STRVAR(_opcode_has_free__doc__, +"has_free($module, /, opcode)\n" +"--\n" +"\n" +"Return True if the opcode accesses a free variable, False otherwise.\n" +"\n" +"Note that \'free\' in this context refers to names in the current scope\n" +"that are referenced by inner scopes or names in outer scopes that are\n" +"referenced from this scope. It does not include references to global\n" +"or builtin scopes."); + +#define _OPCODE_HAS_FREE_METHODDEF \ + {"has_free", _PyCFunction_CAST(_opcode_has_free), METH_FASTCALL|METH_KEYWORDS, _opcode_has_free__doc__}, + +static int +_opcode_has_free_impl(PyObject *module, int opcode); + +static PyObject * +_opcode_has_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(opcode), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"opcode", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "has_free", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int opcode; + int _return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + opcode = _PyLong_AsInt(args[0]); + if (opcode == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = _opcode_has_free_impl(module, opcode); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(_opcode_has_local__doc__, +"has_local($module, /, opcode)\n" +"--\n" +"\n" +"Return True if the opcode accesses a local variable, False otherwise."); + +#define _OPCODE_HAS_LOCAL_METHODDEF \ + {"has_local", _PyCFunction_CAST(_opcode_has_local), METH_FASTCALL|METH_KEYWORDS, _opcode_has_local__doc__}, + +static int +_opcode_has_local_impl(PyObject *module, int opcode); + +static PyObject * +_opcode_has_local(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(opcode), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"opcode", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "has_local", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int opcode; + int _return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + opcode = _PyLong_AsInt(args[0]); + if (opcode == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = _opcode_has_local_impl(module, opcode); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(_opcode_has_exc__doc__, +"has_exc($module, /, opcode)\n" +"--\n" +"\n" +"Return True if the opcode sets an exception handler, False otherwise."); + +#define _OPCODE_HAS_EXC_METHODDEF \ + {"has_exc", _PyCFunction_CAST(_opcode_has_exc), METH_FASTCALL|METH_KEYWORDS, _opcode_has_exc__doc__}, + +static int +_opcode_has_exc_impl(PyObject *module, int opcode); + +static PyObject * +_opcode_has_exc(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(opcode), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"opcode", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "has_exc", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int opcode; + int _return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + opcode = _PyLong_AsInt(args[0]); + if (opcode == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = _opcode_has_exc_impl(module, opcode); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(_opcode_get_specialization_stats__doc__, "get_specialization_stats($module, /)\n" "--\n" @@ -418,4 +612,4 @@ _opcode_get_specialization_stats(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _opcode_get_specialization_stats_impl(module); } -/*[clinic end generated code: output=ae2b2ef56d582180 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e507bf14fb2796f8 input=a9049054013a1b77]*/ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 19fb138ee64..ea136a3fca2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3604,7 +3604,7 @@ dummy_func( _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { next_instr--; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 874bd45becf..c2c323317d1 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -234,6 +234,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* Local variable macros */ +#define LOCALS_ARRAY (frame->localsplus) #define GETLOCAL(i) (frame->localsplus[i]) /* The SETLOCAL() macro must not DECREF the local variable in-place and diff --git a/Python/compile.c b/Python/compile.c index 2a735382c0c..d5405b46561 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -896,6 +896,24 @@ PyUnstable_OpcodeHasJump(int opcode) return OPCODE_HAS_JUMP(opcode); } +int +PyUnstable_OpcodeHasFree(int opcode) +{ + return OPCODE_HAS_FREE(opcode); +} + +int +PyUnstable_OpcodeHasLocal(int opcode) +{ + return OPCODE_HAS_LOCAL(opcode); +} + +int +PyUnstable_OpcodeHasExc(int opcode) +{ + return IS_BLOCK_PUSH_OPCODE(opcode); +} + static int codegen_addop_noarg(instr_sequence *seq, int opcode, location loc) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index f492c1fa9d8..e1f8b9f208c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2449,7 +2449,7 @@ _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { next_instr--; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0a8e4da46b8..b2b0aa6ece4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4451,7 +4451,7 @@ _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { next_instr--; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 2e841e6097a..5b0560e6b21 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -84,13 +84,7 @@ def main(opcode_py, opcode = get_python_module_dict(opcode_py) opmap = opcode['opmap'] opname = opcode['opname'] - hasarg = opcode['hasarg'] - hasconst = opcode['hasconst'] - hasjrel = opcode['hasjrel'] - hasjabs = opcode['hasjabs'] is_pseudo = opcode['is_pseudo'] - _pseudo_ops = opcode['_pseudo_ops'] - ENABLE_SPECIALIZATION = opcode["ENABLE_SPECIALIZATION"] MIN_PSEUDO_OPCODE = opcode["MIN_PSEUDO_OPCODE"] diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 2713fc6774e..33eff548a18 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -261,6 +261,8 @@ class InstructionFlags: HAS_CONST_FLAG: bool HAS_NAME_FLAG: bool HAS_JUMP_FLAG: bool + HAS_FREE_FLAG: bool + HAS_LOCAL_FLAG: bool def __post_init__(self): self.bitmask = { @@ -269,16 +271,25 @@ class InstructionFlags: @staticmethod def fromInstruction(instr: "AnyInstruction"): + + has_free = (variable_used(instr, "PyCell_New") or + variable_used(instr, "PyCell_GET") or + variable_used(instr, "PyCell_SET")) + return InstructionFlags( HAS_ARG_FLAG=variable_used(instr, "oparg"), HAS_CONST_FLAG=variable_used(instr, "FRAME_CO_CONSTS"), HAS_NAME_FLAG=variable_used(instr, "FRAME_CO_NAMES"), HAS_JUMP_FLAG=variable_used(instr, "JUMPBY"), + HAS_FREE_FLAG=has_free, + HAS_LOCAL_FLAG=(variable_used(instr, "GETLOCAL") or + variable_used(instr, "SETLOCAL")) and + not has_free, ) @staticmethod def newEmpty(): - return InstructionFlags(False, False, False, False) + return InstructionFlags(False, False, False, False, False, False) def add(self, other: "InstructionFlags") -> None: for name, value in dataclasses.asdict(other).items():