From 72119d16a5f658939809febef29dadeca02cf34d Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 23 Aug 2023 18:39:00 +0100 Subject: [PATCH] gh-105481: remove regen-opcode. Generated _PyOpcode_Caches in regen-cases. (#108367) --- Include/internal/pycore_opcode.h | 40 ----------- Include/internal/pycore_opcode_metadata.h | 20 ++++++ Include/internal/pycore_opcode_utils.h | 3 +- Lib/opcode.py | 54 +++++++-------- Makefile.pre.in | 14 +--- ...-08-23-14-54-15.gh-issue-105481.40q-c4.rst | 2 + Objects/codeobject.c | 3 +- Objects/frameobject.c | 3 +- PCbuild/regen.targets | 14 +--- Python/assemble.c | 3 +- Python/ceval.c | 3 +- Python/compile.c | 1 + Python/executor.c | 2 + Python/instrumentation.c | 6 +- Python/optimizer.c | 1 - Python/optimizer_analysis.c | 1 - Python/specialize.c | 5 +- Tools/build/generate_opcode_h.py | 67 ------------------- Tools/cases_generator/generate_cases.py | 12 ++++ 19 files changed, 78 insertions(+), 176 deletions(-) delete mode 100644 Include/internal/pycore_opcode.h create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst delete mode 100644 Tools/build/generate_opcode_h.py diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h deleted file mode 100644 index b47e7964852..00000000000 --- a/Include/internal/pycore_opcode.h +++ /dev/null @@ -1,40 +0,0 @@ -// Auto-generated by Tools/build/generate_opcode_h.py from Lib/opcode.py - -#ifndef Py_INTERNAL_OPCODE_H -#define Py_INTERNAL_OPCODE_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -#include "opcode.h" - -extern const uint8_t _PyOpcode_Caches[256]; - -#ifdef NEED_OPCODE_TABLES - -const uint8_t _PyOpcode_Caches[256] = { - [LOAD_GLOBAL] = 4, - [BINARY_OP] = 1, - [UNPACK_SEQUENCE] = 1, - [COMPARE_OP] = 1, - [BINARY_SUBSCR] = 1, - [FOR_ITER] = 1, - [LOAD_SUPER_ATTR] = 1, - [LOAD_ATTR] = 9, - [STORE_ATTR] = 4, - [CALL] = 3, - [STORE_SUBSCR] = 1, - [SEND] = 1, - [JUMP_BACKWARD] = 1, - [TO_BOOL] = 3, -}; -#endif // NEED_OPCODE_TABLES - -#ifdef __cplusplus -} -#endif -#endif // !Py_INTERNAL_OPCODE_H diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index e35db0c4c5a..cc8894ad539 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1814,6 +1814,26 @@ const char *const _PyOpcode_OpName[268] = { }; #endif // NEED_OPCODE_METADATA +extern const uint8_t _PyOpcode_Caches[256]; +#ifdef NEED_OPCODE_METADATA +const uint8_t _PyOpcode_Caches[256] = { + [TO_BOOL] = 3, + [BINARY_OP] = 1, + [BINARY_SUBSCR] = 1, + [STORE_SUBSCR] = 1, + [SEND] = 1, + [UNPACK_SEQUENCE] = 1, + [STORE_ATTR] = 4, + [LOAD_GLOBAL] = 4, + [LOAD_SUPER_ATTR] = 1, + [LOAD_ATTR] = 9, + [COMPARE_OP] = 1, + [FOR_ITER] = 1, + [CALL] = 3, + [JUMP_BACKWARD] = 1, +}; +#endif // NEED_OPCODE_METADATA + extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index f17612908ce..c4acb00a4b2 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -8,8 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_opcode.h" // JUMP_FORWARD - +#include "opcode_ids.h" #define MAX_REAL_OPCODE 254 diff --git a/Lib/opcode.py b/Lib/opcode.py index f8487522bfd..386a2fba396 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -5,48 +5,40 @@ operate on bytecodes (e.g. peephole optimizers). """ -# Note that __all__ is further extended below -__all__ = ["cmp_op", "stack_effect", "hascompare"] +__all__ = ["cmp_op", "stack_effect", "hascompare", "opname", "opmap", + "HAVE_ARGUMENT", "EXTENDED_ARG", "hasarg", "hasconst", "hasname", + "hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc"] import _opcode from _opcode import stack_effect -import sys -# The build uses older versions of Python which do not have _opcode_metadata -if sys.version_info[:2] >= (3, 13): - from _opcode_metadata import _specializations, _specialized_opmap - from _opcode_metadata import opmap, HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE - EXTENDED_ARG = opmap['EXTENDED_ARG'] +from _opcode_metadata import (_specializations, _specialized_opmap, opmap, + HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE) +EXTENDED_ARG = opmap['EXTENDED_ARG'] - opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] - for op, i in opmap.items(): - opname[i] = op - - __all__.extend(["opname", "opmap", "HAVE_ARGUMENT", "EXTENDED_ARG"]) +opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] +for op, i in opmap.items(): + opname[i] = op cmp_op = ('<', '<=', '==', '!=', '>', '>=') -# The build uses older versions of Python which do not have _opcode.has_* functions -if sys.version_info[:2] >= (3, 13): - # These lists are documented as part of the dis module's API - hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] - hasconst = [op for op in opmap.values() if _opcode.has_const(op)] - hasname = [op for op in opmap.values() if _opcode.has_name(op)] - hasjump = [op for op in opmap.values() if _opcode.has_jump(op)] - hasjrel = hasjump # for backward compatibility - hasjabs = [] - hasfree = [op for op in opmap.values() if _opcode.has_free(op)] - haslocal = [op for op in opmap.values() if _opcode.has_local(op)] - hasexc = [op for op in opmap.values() if _opcode.has_exc(op)] +# These lists are documented as part of the dis module's API +hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] +hasconst = [op for op in opmap.values() if _opcode.has_const(op)] +hasname = [op for op in opmap.values() if _opcode.has_name(op)] +hasjump = [op for op in opmap.values() if _opcode.has_jump(op)] +hasjrel = hasjump # for backward compatibility +hasjabs = [] +hasfree = [op for op in opmap.values() if _opcode.has_free(op)] +haslocal = [op for op in opmap.values() if _opcode.has_local(op)] +hasexc = [op for op in opmap.values() if _opcode.has_exc(op)] - __all__.extend(["hasarg", "hasconst", "hasname", "hasjump", "hasjrel", - "hasjabs", "hasfree", "haslocal", "hasexc"]) - _intrinsic_1_descs = _opcode.get_intrinsic1_descs() - _intrinsic_2_descs = _opcode.get_intrinsic2_descs() - _nb_ops = _opcode.get_nb_ops() +_intrinsic_1_descs = _opcode.get_intrinsic1_descs() +_intrinsic_2_descs = _opcode.get_intrinsic2_descs() +_nb_ops = _opcode.get_nb_ops() - hascompare = [opmap["COMPARE_OP"]] +hascompare = [opmap["COMPARE_OP"]] _cache_format = { "LOAD_GLOBAL": { diff --git a/Makefile.pre.in b/Makefile.pre.in index d66764e6165..54b64e123cd 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1321,7 +1321,7 @@ regen-limited-abi: all # Regenerate all generated files .PHONY: regen-all -regen-all: regen-cases regen-opcode regen-typeslots \ +regen-all: regen-cases regen-typeslots \ regen-token regen-ast regen-keyword regen-sre regen-frozen clinic \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ regen-test-levenshtein regen-global-objects @@ -1425,15 +1425,6 @@ regen-ast: $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_ast_state.h $(srcdir)/Include/internal/pycore_ast_state.h.new $(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new -.PHONY: regen-opcode -regen-opcode: - # Regenerate Include/internal/pycore_opcode.h from Lib/opcode.py - # using Tools/build/generate_opcode_h.py - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_opcode_h.py \ - $(srcdir)/Lib/opcode.py \ - $(srcdir)/Include/internal/pycore_opcode.h.new - $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new - .PHONY: regen-token regen-token: # Regenerate Doc/library/token-list.inc from Grammar/Tokens @@ -1651,6 +1642,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/object.h \ $(srcdir)/Include/objimpl.h \ $(srcdir)/Include/opcode.h \ + $(srcdir)/Include/opcode_ids.h \ $(srcdir)/Include/osdefs.h \ $(srcdir)/Include/osmodule.h \ $(srcdir)/Include/patchlevel.h \ @@ -1790,7 +1782,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_object_state.h \ $(srcdir)/Include/internal/pycore_obmalloc.h \ $(srcdir)/Include/internal/pycore_obmalloc_init.h \ - $(srcdir)/Include/internal/pycore_opcode.h \ + $(srcdir)/Include/internal/pycore_opcode_metadata.h \ $(srcdir)/Include/internal/pycore_opcode_utils.h \ $(srcdir)/Include/internal/pycore_optimizer.h \ $(srcdir)/Include/internal/pycore_pathconfig.h \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst new file mode 100644 index 00000000000..19746ebb701 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst @@ -0,0 +1,2 @@ +The regen-opcode build stage was removed and its work is now done in +regen-cases. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index dca5804a91d..70a0c2ebd66 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -6,8 +6,7 @@ #include "pycore_code.h" // _PyCodeConstructor #include "pycore_frame.h" // FRAME_SPECIALS_SIZE #include "pycore_interp.h" // PyInterpreterState.co_extra_freefuncs -#include "pycore_opcode.h" // _PyOpcode_Caches -#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt +#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_tuple.h" // _PyTuple_ITEMS() diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 80e118e8a8a..28f5a5a1222 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -6,8 +6,7 @@ #include "pycore_function.h" // _PyFunction_FromConstructor() #include "pycore_moduleobject.h" // _PyModule_GetDict() #include "pycore_object.h" // _PyObject_GC_UNTRACK() -#include "pycore_opcode.h" // _PyOpcode_Caches -#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt +#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches #include "frameobject.h" // PyFrameObject diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 2ff18a8966f..cc9469c7ddd 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -13,8 +13,6 @@ <_ASTOutputs Include="$(PySourcePath)Python\Python-ast.c"> -C - <_OpcodeSources Include="$(PySourcePath)Tools\build\generate_opcode_h.py;$(PySourcePath)Lib\opcode.py" /> - <_OpcodeOutputs Include="$(PySourcePath)Include\internal\pycore_opcode.h" /> <_TokenSources Include="$(PySourcePath)Grammar\Tokens" /> <_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc"> rst @@ -34,7 +32,7 @@ - @@ -55,14 +53,6 @@ WorkingDirectory="$(PySourcePath)" /> - - - - - @@ -89,7 +79,7 @@ + DependsOnTargets="_TouchRegenSources;_RegenPegen;_RegenAST_H;_RegenTokens;_RegenKeywords;_RegenGlobalObjects"> diff --git a/Python/assemble.c b/Python/assemble.c index 4f66cf294e3..c770fd108d4 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -3,9 +3,8 @@ #include "Python.h" #include "pycore_code.h" // write_location_entry_start() #include "pycore_compile.h" -#include "pycore_opcode.h" // _PyOpcode_Caches[] and opcode category macros #include "pycore_opcode_utils.h" // IS_BACKWARDS_JUMP_OPCODE -#include "pycore_opcode_metadata.h" // IS_PSEUDO_INSTR +#include "pycore_opcode_metadata.h" // IS_PSEUDO_INSTR, _PyOpcode_Caches #define DEFAULT_CODE_SIZE 128 diff --git a/Python/ceval.c b/Python/ceval.c index f7dfaebcdd8..55dfe6b1efc 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -14,8 +14,7 @@ #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_object.h" // _PyObject_GC_TRACK() -#include "pycore_opcode.h" // EXTRA_CASES -#include "pycore_opcode_metadata.h" +#include "pycore_opcode_metadata.h" // EXTRA_CASES #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() diff --git a/Python/compile.c b/Python/compile.c index b67a1885ddc..6b816b4c6ed 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -24,6 +24,7 @@ #include #include "Python.h" +#include "opcode.h" #include "pycore_ast.h" // _PyAST_GetDocString() #define NEED_OPCODE_TABLES #include "pycore_opcode_utils.h" diff --git a/Python/executor.c b/Python/executor.c index 88c039da853..0ff5106fe44 100644 --- a/Python/executor.c +++ b/Python/executor.c @@ -1,5 +1,7 @@ #include "Python.h" +#include "opcode.h" + #include "pycore_call.h" #include "pycore_ceval.h" #include "pycore_dict.h" diff --git a/Python/instrumentation.c b/Python/instrumentation.c index f77c2e696f4..8c7a3a06c9b 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1,4 +1,7 @@ #include "Python.h" + +#include "opcode_ids.h" + #include "pycore_call.h" #include "pycore_frame.h" #include "pycore_interp.h" @@ -6,8 +9,7 @@ #include "pycore_modsupport.h" // _PyModule_CreateInitialized() #include "pycore_namespace.h" #include "pycore_object.h" -#include "pycore_opcode.h" -#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE +#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE, _PyOpcode_Caches #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() diff --git a/Python/optimizer.c b/Python/optimizer.c index 57518404c3f..bbc125954c7 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1,7 +1,6 @@ #include "Python.h" #include "opcode.h" #include "pycore_interp.h" -#include "pycore_opcode.h" #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_optimizer.h" diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index e48e018052c..2d177f14ff2 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -1,7 +1,6 @@ #include "Python.h" #include "opcode.h" #include "pycore_interp.h" -#include "pycore_opcode.h" #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() diff --git a/Python/specialize.c b/Python/specialize.c index 2d514c0dc47..a467f163f2c 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1,4 +1,7 @@ #include "Python.h" + +#include "opcode.h" + #include "pycore_code.h" #include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" @@ -7,7 +10,7 @@ #include "pycore_long.h" #include "pycore_moduleobject.h" #include "pycore_object.h" -#include "pycore_opcode.h" // _PyOpcode_Caches +#include "pycore_opcode_metadata.h" // _PyOpcode_Caches #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py deleted file mode 100644 index 643918c1bc2..00000000000 --- a/Tools/build/generate_opcode_h.py +++ /dev/null @@ -1,67 +0,0 @@ -# This script generates the pycore_opcode.h header file. - -import sys -import tokenize - -SCRIPT_NAME = "Tools/build/generate_opcode_h.py" -PYTHON_OPCODE = "Lib/opcode.py" - -internal_header = f""" -// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} - -#ifndef Py_INTERNAL_OPCODE_H -#define Py_INTERNAL_OPCODE_H -#ifdef __cplusplus -extern "C" {{ -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -#include "opcode.h" -""".lstrip() - -internal_footer = """ -#ifdef __cplusplus -} -#endif -#endif // !Py_INTERNAL_OPCODE_H -""" - -DEFINE = "#define {:<38} {:>3}\n" - -UINT32_MASK = (1<<32)-1 - -def get_python_module_dict(filename): - mod = {} - with tokenize.open(filename) as fp: - code = fp.read() - exec(code, mod) - return mod - -def main(opcode_py, - internal_opcode_h='Include/internal/pycore_opcode.h'): - - opcode = get_python_module_dict(opcode_py) - - with open(internal_opcode_h, 'w') as iobj: - iobj.write(internal_header) - - iobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n") - iobj.write("\n#ifdef NEED_OPCODE_TABLES\n") - - iobj.write("\nconst uint8_t _PyOpcode_Caches[256] = {\n") - for name, entries in opcode["_inline_cache_entries"].items(): - iobj.write(f" [{name}] = {entries},\n") - iobj.write("};\n") - - iobj.write("#endif // NEED_OPCODE_TABLES\n") - - iobj.write(internal_footer) - - print(f"{internal_opcode_h} regenerated from {opcode_py}") - - -if __name__ == '__main__': - main(sys.argv[1], sys.argv[2]) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index f605dcc5e46..8f9a6502e52 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -540,6 +540,18 @@ class Generator(Analyzer): for name in self.opmap: self.out.emit(f'[{name}] = "{name}",') + with self.metadata_item( + f"const uint8_t _PyOpcode_Caches[256]", + "=", + ";", + ): + for name, _ in self.families.items(): + instr = self.instrs[name] + if instr.cache_offset > 0: + self.out.emit(f'[{name}] = {instr.cache_offset},') + # Irregular case: + self.out.emit('[JUMP_BACKWARD] = 1,') + deoptcodes = {} for name, op in self.opmap.items(): if op < 256: