From 2bde6827ea4f136297b2d882480b981ff26262b6 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 21 Mar 2022 04:11:17 -0700 Subject: [PATCH] bpo-46841: Quicken code in-place (GH-31888) * Moves the bytecode to the end of the corresponding PyCodeObject, and quickens it in-place. * Removes the almost-always-unused co_varnames, co_freevars, and co_cellvars member caches * _PyOpcode_Deopt is a new mapping from all opcodes to their un-quickened forms. * _PyOpcode_InlineCacheEntries is renamed to _PyOpcode_Caches * _Py_IncrementCountAndMaybeQuicken is renamed to _PyCode_Warmup * _Py_Quicken is renamed to _PyCode_Quicken * _co_quickened is renamed to _co_code_adaptive (and is now a read-only memoryview). * Do not emit unused nonzero opargs anymore in the compiler. --- Include/cpython/code.h | 157 +++-- Include/internal/pycore_code.h | 17 +- Include/opcode.h | 590 ++++++++++++------ Lib/opcode.py | 174 +++--- Lib/test/test_compile.py | 3 +- .../2022-03-16-11-05-35.bpo-46841.yUoIHg.rst | 2 + Objects/clinic/codeobject.c.h | 10 +- Objects/codeobject.c | 213 +++---- Objects/frameobject.c | 39 +- Objects/genobject.c | 19 +- Objects/typeobject.c | 5 +- Python/ceval.c | 39 +- Python/compile.c | 8 +- Python/marshal.c | 8 +- Python/opcode_targets.h | 72 +-- Python/specialize.c | 100 +-- Tools/scripts/deepfreeze.py | 47 +- Tools/scripts/generate_opcode_h.py | 17 +- 18 files changed, 832 insertions(+), 688 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-03-16-11-05-35.bpo-46841.yUoIHg.rst diff --git a/Include/cpython/code.h b/Include/cpython/code.h index ab827c5ae87..15767831793 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -26,91 +26,80 @@ typedef uint16_t _Py_CODEUNIT; // Use "unsigned char" instead of "uint8_t" here to avoid illegal aliasing: #define _Py_SET_OPCODE(word, opcode) (((unsigned char *)&(word))[0] = (opcode)) +// To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are +// defined in this macro: +#define _PyCode_DEF(SIZE) { \ + PyObject_VAR_HEAD \ + \ + /* Note only the following fields are used in hash and/or comparisons \ + * \ + * - co_name \ + * - co_argcount \ + * - co_posonlyargcount \ + * - co_kwonlyargcount \ + * - co_nlocals \ + * - co_stacksize \ + * - co_flags \ + * - co_firstlineno \ + * - co_consts \ + * - co_names \ + * - co_localsplusnames \ + * This is done to preserve the name and line number for tracebacks \ + * and debuggers; otherwise, constant de-duplication would collapse \ + * identical functions/lambdas defined on different lines. \ + */ \ + \ + /* These fields are set with provided values on new code objects. */ \ + \ + /* The hottest fields (in the eval loop) are grouped here at the top. */ \ + PyObject *co_consts; /* list (constants used) */ \ + PyObject *co_names; /* list of strings (names used) */ \ + PyObject *co_exceptiontable; /* Byte string encoding exception handling \ + table */ \ + int co_flags; /* CO_..., see below */ \ + int co_warmup; /* Warmup counter for quickening */ \ + \ + /* The rest are not so impactful on performance. */ \ + int co_argcount; /* #arguments, except *args */ \ + int co_posonlyargcount; /* #positional only arguments */ \ + int co_kwonlyargcount; /* #keyword only arguments */ \ + int co_stacksize; /* #entries needed for evaluation stack */ \ + int co_firstlineno; /* first source line number */ \ + \ + /* redundant values (derived from co_localsplusnames and \ + co_localspluskinds) */ \ + int co_nlocalsplus; /* number of local + cell + free variables \ + */ \ + int co_nlocals; /* number of local variables */ \ + int co_nplaincellvars; /* number of non-arg cell variables */ \ + int co_ncellvars; /* total number of cell variables */ \ + int co_nfreevars; /* number of free variables */ \ + \ + PyObject *co_localsplusnames; /* tuple mapping offsets to names */ \ + PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte \ + per variable) */ \ + PyObject *co_filename; /* unicode (where it was loaded from) */ \ + PyObject *co_name; /* unicode (name, for reference) */ \ + PyObject *co_qualname; /* unicode (qualname, for reference) */ \ + PyObject *co_linetable; /* bytes (encoding addr<->lineno mapping) \ + See Objects/lnotab_notes.txt for details. \ + */ \ + PyObject *co_endlinetable; /* bytes object that holds end lineno for \ + instructions separated across different \ + lines */ \ + PyObject *co_columntable; /* bytes object that holds start/end column \ + offset each instruction */ \ + \ + PyObject *co_weakreflist; /* to support weakrefs to code objects */ \ + /* Scratch space for extra data relating to the code object. \ + Type is a void* to keep the format private in codeobject.c to force \ + people to go through the proper APIs. */ \ + void *co_extra; \ + char co_code_adaptive[(SIZE)]; \ +} /* Bytecode object */ -struct PyCodeObject { - PyObject_HEAD - - /* Note only the following fields are used in hash and/or comparisons - * - * - co_name - * - co_argcount - * - co_posonlyargcount - * - co_kwonlyargcount - * - co_nlocals - * - co_stacksize - * - co_flags - * - co_firstlineno - * - co_code - * - co_consts - * - co_names - * - co_varnames - * - co_freevars - * - co_cellvars - * - * This is done to preserve the name and line number for tracebacks - * and debuggers; otherwise, constant de-duplication would collapse - * identical functions/lambdas defined on different lines. - */ - - /* These fields are set with provided values on new code objects. */ - - // The hottest fields (in the eval loop) are grouped here at the top. - PyObject *co_consts; /* list (constants used) */ - PyObject *co_names; /* list of strings (names used) */ - _Py_CODEUNIT *co_firstinstr; /* Pointer to first instruction, used for quickening. - Unlike the other "hot" fields, this one is - actually derived from co_code. */ - PyObject *co_exceptiontable; /* Byte string encoding exception handling table */ - int co_flags; /* CO_..., see below */ - int co_warmup; /* Warmup counter for quickening */ - - // The rest are not so impactful on performance. - int co_argcount; /* #arguments, except *args */ - int co_posonlyargcount; /* #positional only arguments */ - int co_kwonlyargcount; /* #keyword only arguments */ - int co_stacksize; /* #entries needed for evaluation stack */ - int co_firstlineno; /* first source line number */ - PyObject *co_code; /* instruction opcodes */ - PyObject *co_localsplusnames; /* tuple mapping offsets to names */ - PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte per variable) */ - PyObject *co_filename; /* unicode (where it was loaded from) */ - PyObject *co_name; /* unicode (name, for reference) */ - PyObject *co_qualname; /* unicode (qualname, for reference) */ - PyObject *co_linetable; /* bytes (encoding addr<->lineno mapping) See - Objects/lnotab_notes.txt for details. */ - PyObject *co_endlinetable; /* bytes object that holds end lineno for - instructions separated across different - lines */ - PyObject *co_columntable; /* bytes object that holds start/end column - offset each instruction */ - - /* These fields are set with computed values on new code objects. */ - - // redundant values (derived from co_localsplusnames and co_localspluskinds) - int co_nlocalsplus; /* number of local + cell + free variables */ - int co_nlocals; /* number of local variables */ - int co_nplaincellvars; /* number of non-arg cell variables */ - int co_ncellvars; /* total number of cell variables */ - int co_nfreevars; /* number of free variables */ - // lazily-computed values - PyObject *co_varnames; /* tuple of strings (local variable names) */ - PyObject *co_cellvars; /* tuple of strings (cell variable names) */ - PyObject *co_freevars; /* tuple of strings (free variable names) */ - - /* The remaining fields are zeroed out on new code objects. */ - - PyObject *co_weakreflist; /* to support weakrefs to code objects */ - /* Scratch space for extra data relating to the code object. - Type is a void* to keep the format private in codeobject.c to force - people to go through the proper APIs. */ - void *co_extra; - /* Quickened instructions and cache, or NULL - This should be treated as opaque by all code except the specializer and - interpreter. */ - _Py_CODEUNIT *co_quickened; - -}; +struct PyCodeObject _PyCode_DEF(1); /* Masks for co_flags above */ #define CO_OPTIMIZED 0x0001 @@ -151,6 +140,8 @@ PyAPI_DATA(PyTypeObject) PyCode_Type; #define PyCode_Check(op) Py_IS_TYPE(op, &PyCode_Type) #define PyCode_GetNumFree(op) ((op)->co_nfreevars) +#define _PyCode_CODE(CO) ((_Py_CODEUNIT *)(CO)->co_code_adaptive) +#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) /* Public interface */ PyAPI_FUNC(PyCodeObject *) PyCode_New( diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 0d324e9e4c0..82dc9e4bdc6 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -92,30 +92,22 @@ typedef struct { #define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache) -/* Maximum size of code to quicken, in code units. */ -#define MAX_SIZE_TO_QUICKEN 10000 - #define QUICKENING_WARMUP_DELAY 8 /* We want to compare to zero for efficiency, so we offset values accordingly */ #define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY) -#define QUICKENING_WARMUP_COLDEST 1 -int _Py_Quicken(PyCodeObject *code); +void _PyCode_Quicken(PyCodeObject *code); -/* Returns 1 if quickening occurs. - * -1 if an error occurs - * 0 otherwise */ -static inline int -_Py_IncrementCountAndMaybeQuicken(PyCodeObject *code) +static inline void +_PyCode_Warmup(PyCodeObject *code) { if (code->co_warmup != 0) { code->co_warmup++; if (code->co_warmup == 0) { - return _Py_Quicken(code) ? -1 : 1; + _PyCode_Quicken(code); } } - return 0; } extern Py_ssize_t _Py_QuickenedCount; @@ -225,6 +217,7 @@ PyAPI_FUNC(PyCodeObject *) _PyCode_New(struct _PyCodeConstructor *); extern PyObject* _PyCode_GetVarnames(PyCodeObject *); extern PyObject* _PyCode_GetCellvars(PyCodeObject *); extern PyObject* _PyCode_GetFreevars(PyCodeObject *); +extern PyObject* _PyCode_GetCode(PyCodeObject *); /* Return the ending source code line number from a bytecode index. */ extern int _PyCode_Addr2EndLine(PyCodeObject *, int); diff --git a/Include/opcode.h b/Include/opcode.h index 7bf0ba70fd7..dfc7b72e3cd 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -7,185 +7,187 @@ extern "C" { /* Instruction opcodes for compiled code */ -#define CACHE 0 -#define POP_TOP 1 -#define PUSH_NULL 2 -#define NOP 9 -#define UNARY_POSITIVE 10 -#define UNARY_NEGATIVE 11 -#define UNARY_NOT 12 -#define UNARY_INVERT 15 -#define BINARY_SUBSCR 25 -#define GET_LEN 30 -#define MATCH_MAPPING 31 -#define MATCH_SEQUENCE 32 -#define MATCH_KEYS 33 -#define PUSH_EXC_INFO 35 -#define WITH_EXCEPT_START 49 -#define GET_AITER 50 -#define GET_ANEXT 51 -#define BEFORE_ASYNC_WITH 52 -#define BEFORE_WITH 53 -#define END_ASYNC_FOR 54 -#define STORE_SUBSCR 60 -#define DELETE_SUBSCR 61 -#define GET_ITER 68 -#define GET_YIELD_FROM_ITER 69 -#define PRINT_EXPR 70 -#define LOAD_BUILD_CLASS 71 -#define LOAD_ASSERTION_ERROR 74 -#define RETURN_GENERATOR 75 -#define LIST_TO_TUPLE 82 -#define RETURN_VALUE 83 -#define IMPORT_STAR 84 -#define SETUP_ANNOTATIONS 85 -#define YIELD_VALUE 86 -#define ASYNC_GEN_WRAP 87 -#define PREP_RERAISE_STAR 88 -#define POP_EXCEPT 89 -#define HAVE_ARGUMENT 90 -#define STORE_NAME 90 -#define DELETE_NAME 91 -#define UNPACK_SEQUENCE 92 -#define FOR_ITER 93 -#define UNPACK_EX 94 -#define STORE_ATTR 95 -#define DELETE_ATTR 96 -#define STORE_GLOBAL 97 -#define DELETE_GLOBAL 98 -#define SWAP 99 -#define LOAD_CONST 100 -#define LOAD_NAME 101 -#define BUILD_TUPLE 102 -#define BUILD_LIST 103 -#define BUILD_SET 104 -#define BUILD_MAP 105 -#define LOAD_ATTR 106 -#define COMPARE_OP 107 -#define IMPORT_NAME 108 -#define IMPORT_FROM 109 -#define JUMP_FORWARD 110 -#define JUMP_IF_FALSE_OR_POP 111 -#define JUMP_IF_TRUE_OR_POP 112 -#define JUMP_ABSOLUTE 113 -#define POP_JUMP_IF_FALSE 114 -#define POP_JUMP_IF_TRUE 115 -#define LOAD_GLOBAL 116 -#define IS_OP 117 -#define CONTAINS_OP 118 -#define RERAISE 119 -#define COPY 120 -#define JUMP_IF_NOT_EXC_MATCH 121 -#define BINARY_OP 122 -#define SEND 123 -#define LOAD_FAST 124 -#define STORE_FAST 125 -#define DELETE_FAST 126 -#define JUMP_IF_NOT_EG_MATCH 127 -#define POP_JUMP_IF_NOT_NONE 128 -#define POP_JUMP_IF_NONE 129 -#define RAISE_VARARGS 130 -#define GET_AWAITABLE 131 -#define MAKE_FUNCTION 132 -#define BUILD_SLICE 133 -#define JUMP_NO_INTERRUPT 134 -#define MAKE_CELL 135 -#define LOAD_CLOSURE 136 -#define LOAD_DEREF 137 -#define STORE_DEREF 138 -#define DELETE_DEREF 139 -#define CALL_FUNCTION_EX 142 -#define EXTENDED_ARG 144 -#define LIST_APPEND 145 -#define SET_ADD 146 -#define MAP_ADD 147 -#define LOAD_CLASSDEREF 148 -#define COPY_FREE_VARS 149 -#define RESUME 151 -#define MATCH_CLASS 152 -#define FORMAT_VALUE 155 -#define BUILD_CONST_KEY_MAP 156 -#define BUILD_STRING 157 -#define LOAD_METHOD 160 -#define LIST_EXTEND 162 -#define SET_UPDATE 163 -#define DICT_MERGE 164 -#define DICT_UPDATE 165 -#define PRECALL 166 -#define CALL 171 -#define KW_NAMES 172 -#define BINARY_OP_ADAPTIVE 3 -#define BINARY_OP_ADD_INT 4 -#define BINARY_OP_ADD_FLOAT 5 -#define BINARY_OP_ADD_UNICODE 6 -#define BINARY_OP_INPLACE_ADD_UNICODE 7 -#define BINARY_OP_MULTIPLY_INT 8 -#define BINARY_OP_MULTIPLY_FLOAT 13 -#define BINARY_OP_SUBTRACT_INT 14 -#define BINARY_OP_SUBTRACT_FLOAT 16 -#define COMPARE_OP_ADAPTIVE 17 -#define COMPARE_OP_FLOAT_JUMP 18 -#define COMPARE_OP_INT_JUMP 19 -#define COMPARE_OP_STR_JUMP 20 -#define BINARY_SUBSCR_ADAPTIVE 21 -#define BINARY_SUBSCR_GETITEM 22 -#define BINARY_SUBSCR_LIST_INT 23 -#define BINARY_SUBSCR_TUPLE_INT 24 -#define BINARY_SUBSCR_DICT 26 -#define STORE_SUBSCR_ADAPTIVE 27 -#define STORE_SUBSCR_LIST_INT 28 -#define STORE_SUBSCR_DICT 29 -#define CALL_ADAPTIVE 34 -#define CALL_PY_EXACT_ARGS 36 -#define CALL_PY_WITH_DEFAULTS 37 -#define JUMP_ABSOLUTE_QUICK 38 -#define LOAD_ATTR_ADAPTIVE 39 -#define LOAD_ATTR_INSTANCE_VALUE 40 -#define LOAD_ATTR_WITH_HINT 41 -#define LOAD_ATTR_SLOT 42 -#define LOAD_ATTR_MODULE 43 -#define LOAD_GLOBAL_ADAPTIVE 44 -#define LOAD_GLOBAL_MODULE 45 -#define LOAD_GLOBAL_BUILTIN 46 -#define LOAD_METHOD_ADAPTIVE 47 -#define LOAD_METHOD_CLASS 48 -#define LOAD_METHOD_MODULE 55 -#define LOAD_METHOD_NO_DICT 56 -#define LOAD_METHOD_WITH_DICT 57 -#define LOAD_METHOD_WITH_VALUES 58 -#define PRECALL_ADAPTIVE 59 -#define PRECALL_BUILTIN_CLASS 62 -#define PRECALL_NO_KW_BUILTIN_O 63 -#define PRECALL_NO_KW_BUILTIN_FAST 64 -#define PRECALL_BUILTIN_FAST_WITH_KEYWORDS 65 -#define PRECALL_NO_KW_LEN 66 -#define PRECALL_NO_KW_ISINSTANCE 67 -#define PRECALL_NO_KW_LIST_APPEND 72 -#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O 73 -#define PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 76 -#define PRECALL_NO_KW_STR_1 77 -#define PRECALL_NO_KW_TUPLE_1 78 -#define PRECALL_NO_KW_TYPE_1 79 -#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST 80 -#define PRECALL_BOUND_METHOD 81 -#define PRECALL_PYFUNC 140 -#define RESUME_QUICK 141 -#define STORE_ATTR_ADAPTIVE 143 -#define STORE_ATTR_INSTANCE_VALUE 150 -#define STORE_ATTR_SLOT 153 -#define STORE_ATTR_WITH_HINT 154 -#define UNPACK_SEQUENCE_ADAPTIVE 158 -#define UNPACK_SEQUENCE_LIST 159 -#define UNPACK_SEQUENCE_TUPLE 161 -#define UNPACK_SEQUENCE_TWO_TUPLE 167 -#define LOAD_FAST__LOAD_FAST 168 -#define STORE_FAST__LOAD_FAST 169 -#define LOAD_FAST__LOAD_CONST 170 -#define LOAD_CONST__LOAD_FAST 173 -#define STORE_FAST__STORE_FAST 174 -#define DO_TRACING 255 +#define CACHE 0 +#define POP_TOP 1 +#define PUSH_NULL 2 +#define NOP 9 +#define UNARY_POSITIVE 10 +#define UNARY_NEGATIVE 11 +#define UNARY_NOT 12 +#define UNARY_INVERT 15 +#define BINARY_SUBSCR 25 +#define GET_LEN 30 +#define MATCH_MAPPING 31 +#define MATCH_SEQUENCE 32 +#define MATCH_KEYS 33 +#define PUSH_EXC_INFO 35 +#define WITH_EXCEPT_START 49 +#define GET_AITER 50 +#define GET_ANEXT 51 +#define BEFORE_ASYNC_WITH 52 +#define BEFORE_WITH 53 +#define END_ASYNC_FOR 54 +#define STORE_SUBSCR 60 +#define DELETE_SUBSCR 61 +#define GET_ITER 68 +#define GET_YIELD_FROM_ITER 69 +#define PRINT_EXPR 70 +#define LOAD_BUILD_CLASS 71 +#define LOAD_ASSERTION_ERROR 74 +#define RETURN_GENERATOR 75 +#define LIST_TO_TUPLE 82 +#define RETURN_VALUE 83 +#define IMPORT_STAR 84 +#define SETUP_ANNOTATIONS 85 +#define YIELD_VALUE 86 +#define ASYNC_GEN_WRAP 87 +#define PREP_RERAISE_STAR 88 +#define POP_EXCEPT 89 +#define HAVE_ARGUMENT 90 +#define STORE_NAME 90 +#define DELETE_NAME 91 +#define UNPACK_SEQUENCE 92 +#define FOR_ITER 93 +#define UNPACK_EX 94 +#define STORE_ATTR 95 +#define DELETE_ATTR 96 +#define STORE_GLOBAL 97 +#define DELETE_GLOBAL 98 +#define SWAP 99 +#define LOAD_CONST 100 +#define LOAD_NAME 101 +#define BUILD_TUPLE 102 +#define BUILD_LIST 103 +#define BUILD_SET 104 +#define BUILD_MAP 105 +#define LOAD_ATTR 106 +#define COMPARE_OP 107 +#define IMPORT_NAME 108 +#define IMPORT_FROM 109 +#define JUMP_FORWARD 110 +#define JUMP_IF_FALSE_OR_POP 111 +#define JUMP_IF_TRUE_OR_POP 112 +#define JUMP_ABSOLUTE 113 +#define POP_JUMP_IF_FALSE 114 +#define POP_JUMP_IF_TRUE 115 +#define LOAD_GLOBAL 116 +#define IS_OP 117 +#define CONTAINS_OP 118 +#define RERAISE 119 +#define COPY 120 +#define JUMP_IF_NOT_EXC_MATCH 121 +#define BINARY_OP 122 +#define SEND 123 +#define LOAD_FAST 124 +#define STORE_FAST 125 +#define DELETE_FAST 126 +#define JUMP_IF_NOT_EG_MATCH 127 +#define POP_JUMP_IF_NOT_NONE 128 +#define POP_JUMP_IF_NONE 129 +#define RAISE_VARARGS 130 +#define GET_AWAITABLE 131 +#define MAKE_FUNCTION 132 +#define BUILD_SLICE 133 +#define JUMP_NO_INTERRUPT 134 +#define MAKE_CELL 135 +#define LOAD_CLOSURE 136 +#define LOAD_DEREF 137 +#define STORE_DEREF 138 +#define DELETE_DEREF 139 +#define CALL_FUNCTION_EX 142 +#define EXTENDED_ARG 144 +#define LIST_APPEND 145 +#define SET_ADD 146 +#define MAP_ADD 147 +#define LOAD_CLASSDEREF 148 +#define COPY_FREE_VARS 149 +#define RESUME 151 +#define MATCH_CLASS 152 +#define FORMAT_VALUE 155 +#define BUILD_CONST_KEY_MAP 156 +#define BUILD_STRING 157 +#define LOAD_METHOD 160 +#define LIST_EXTEND 162 +#define SET_UPDATE 163 +#define DICT_MERGE 164 +#define DICT_UPDATE 165 +#define PRECALL 166 +#define CALL 171 +#define KW_NAMES 172 +#define BINARY_OP_ADAPTIVE 3 +#define BINARY_OP_ADD_FLOAT 4 +#define BINARY_OP_ADD_INT 5 +#define BINARY_OP_ADD_UNICODE 6 +#define BINARY_OP_INPLACE_ADD_UNICODE 7 +#define BINARY_OP_MULTIPLY_FLOAT 8 +#define BINARY_OP_MULTIPLY_INT 13 +#define BINARY_OP_SUBTRACT_FLOAT 14 +#define BINARY_OP_SUBTRACT_INT 16 +#define BINARY_SUBSCR_ADAPTIVE 17 +#define BINARY_SUBSCR_DICT 18 +#define BINARY_SUBSCR_GETITEM 19 +#define BINARY_SUBSCR_LIST_INT 20 +#define BINARY_SUBSCR_TUPLE_INT 21 +#define CALL_ADAPTIVE 22 +#define CALL_PY_EXACT_ARGS 23 +#define CALL_PY_WITH_DEFAULTS 24 +#define COMPARE_OP_ADAPTIVE 26 +#define COMPARE_OP_FLOAT_JUMP 27 +#define COMPARE_OP_INT_JUMP 28 +#define COMPARE_OP_STR_JUMP 29 +#define JUMP_ABSOLUTE_QUICK 34 +#define LOAD_ATTR_ADAPTIVE 36 +#define LOAD_ATTR_INSTANCE_VALUE 37 +#define LOAD_ATTR_MODULE 38 +#define LOAD_ATTR_SLOT 39 +#define LOAD_ATTR_WITH_HINT 40 +#define LOAD_CONST__LOAD_FAST 41 +#define LOAD_FAST__LOAD_CONST 42 +#define LOAD_FAST__LOAD_FAST 43 +#define LOAD_GLOBAL_ADAPTIVE 44 +#define LOAD_GLOBAL_BUILTIN 45 +#define LOAD_GLOBAL_MODULE 46 +#define LOAD_METHOD_ADAPTIVE 47 +#define LOAD_METHOD_CLASS 48 +#define LOAD_METHOD_MODULE 55 +#define LOAD_METHOD_NO_DICT 56 +#define LOAD_METHOD_WITH_DICT 57 +#define LOAD_METHOD_WITH_VALUES 58 +#define PRECALL_ADAPTIVE 59 +#define PRECALL_BOUND_METHOD 62 +#define PRECALL_BUILTIN_CLASS 63 +#define PRECALL_BUILTIN_FAST_WITH_KEYWORDS 64 +#define PRECALL_NO_KW_BUILTIN_FAST 65 +#define PRECALL_NO_KW_BUILTIN_O 66 +#define PRECALL_NO_KW_ISINSTANCE 67 +#define PRECALL_NO_KW_LEN 72 +#define PRECALL_NO_KW_LIST_APPEND 73 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST 76 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 77 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O 78 +#define PRECALL_NO_KW_STR_1 79 +#define PRECALL_NO_KW_TUPLE_1 80 +#define PRECALL_NO_KW_TYPE_1 81 +#define PRECALL_PYFUNC 140 +#define RESUME_QUICK 141 +#define STORE_ATTR_ADAPTIVE 143 +#define STORE_ATTR_INSTANCE_VALUE 150 +#define STORE_ATTR_SLOT 153 +#define STORE_ATTR_WITH_HINT 154 +#define STORE_FAST__LOAD_FAST 158 +#define STORE_FAST__STORE_FAST 159 +#define STORE_SUBSCR_ADAPTIVE 161 +#define STORE_SUBSCR_DICT 167 +#define STORE_SUBSCR_LIST_INT 168 +#define UNPACK_SEQUENCE_ADAPTIVE 169 +#define UNPACK_SEQUENCE_LIST 170 +#define UNPACK_SEQUENCE_TUPLE 173 +#define UNPACK_SEQUENCE_TWO_TUPLE 174 +#define DO_TRACING 255 -extern const uint8_t _PyOpcode_InlineCacheEntries[256]; +extern const uint8_t _PyOpcode_Caches[256]; + +extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_TABLES static const uint32_t _PyOpcode_RelativeJump[8] = { @@ -209,7 +211,7 @@ static const uint32_t _PyOpcode_Jump[8] = { 0U, }; -const uint8_t _PyOpcode_InlineCacheEntries[256] = { +const uint8_t _PyOpcode_Caches[256] = { [BINARY_SUBSCR] = 4, [STORE_SUBSCR] = 1, [UNPACK_SEQUENCE] = 1, @@ -222,6 +224,184 @@ const uint8_t _PyOpcode_InlineCacheEntries[256] = { [PRECALL] = 1, [CALL] = 4, }; + +const uint8_t _PyOpcode_Deopt[256] = { + [ASYNC_GEN_WRAP] = ASYNC_GEN_WRAP, + [BEFORE_ASYNC_WITH] = BEFORE_ASYNC_WITH, + [BEFORE_WITH] = BEFORE_WITH, + [BINARY_OP] = BINARY_OP, + [BINARY_OP_ADAPTIVE] = BINARY_OP, + [BINARY_OP_ADD_FLOAT] = BINARY_OP, + [BINARY_OP_ADD_INT] = BINARY_OP, + [BINARY_OP_ADD_UNICODE] = BINARY_OP, + [BINARY_OP_INPLACE_ADD_UNICODE] = BINARY_OP, + [BINARY_OP_MULTIPLY_FLOAT] = BINARY_OP, + [BINARY_OP_MULTIPLY_INT] = BINARY_OP, + [BINARY_OP_SUBTRACT_FLOAT] = BINARY_OP, + [BINARY_OP_SUBTRACT_INT] = BINARY_OP, + [BINARY_SUBSCR] = BINARY_SUBSCR, + [BINARY_SUBSCR_ADAPTIVE] = BINARY_SUBSCR, + [BINARY_SUBSCR_DICT] = BINARY_SUBSCR, + [BINARY_SUBSCR_GETITEM] = BINARY_SUBSCR, + [BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR, + [BINARY_SUBSCR_TUPLE_INT] = BINARY_SUBSCR, + [BUILD_CONST_KEY_MAP] = BUILD_CONST_KEY_MAP, + [BUILD_LIST] = BUILD_LIST, + [BUILD_MAP] = BUILD_MAP, + [BUILD_SET] = BUILD_SET, + [BUILD_SLICE] = BUILD_SLICE, + [BUILD_STRING] = BUILD_STRING, + [BUILD_TUPLE] = BUILD_TUPLE, + [CACHE] = CACHE, + [CALL] = CALL, + [CALL_ADAPTIVE] = CALL, + [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, + [CALL_PY_EXACT_ARGS] = CALL, + [CALL_PY_WITH_DEFAULTS] = CALL, + [COMPARE_OP] = COMPARE_OP, + [COMPARE_OP_ADAPTIVE] = COMPARE_OP, + [COMPARE_OP_FLOAT_JUMP] = COMPARE_OP, + [COMPARE_OP_INT_JUMP] = COMPARE_OP, + [COMPARE_OP_STR_JUMP] = COMPARE_OP, + [CONTAINS_OP] = CONTAINS_OP, + [COPY] = COPY, + [COPY_FREE_VARS] = COPY_FREE_VARS, + [DELETE_ATTR] = DELETE_ATTR, + [DELETE_DEREF] = DELETE_DEREF, + [DELETE_FAST] = DELETE_FAST, + [DELETE_GLOBAL] = DELETE_GLOBAL, + [DELETE_NAME] = DELETE_NAME, + [DELETE_SUBSCR] = DELETE_SUBSCR, + [DICT_MERGE] = DICT_MERGE, + [DICT_UPDATE] = DICT_UPDATE, + [END_ASYNC_FOR] = END_ASYNC_FOR, + [EXTENDED_ARG] = EXTENDED_ARG, + [FORMAT_VALUE] = FORMAT_VALUE, + [FOR_ITER] = FOR_ITER, + [GET_AITER] = GET_AITER, + [GET_ANEXT] = GET_ANEXT, + [GET_AWAITABLE] = GET_AWAITABLE, + [GET_ITER] = GET_ITER, + [GET_LEN] = GET_LEN, + [GET_YIELD_FROM_ITER] = GET_YIELD_FROM_ITER, + [IMPORT_FROM] = IMPORT_FROM, + [IMPORT_NAME] = IMPORT_NAME, + [IMPORT_STAR] = IMPORT_STAR, + [IS_OP] = IS_OP, + [JUMP_ABSOLUTE] = JUMP_ABSOLUTE, + [JUMP_ABSOLUTE_QUICK] = JUMP_ABSOLUTE, + [JUMP_FORWARD] = JUMP_FORWARD, + [JUMP_IF_FALSE_OR_POP] = JUMP_IF_FALSE_OR_POP, + [JUMP_IF_NOT_EG_MATCH] = JUMP_IF_NOT_EG_MATCH, + [JUMP_IF_NOT_EXC_MATCH] = JUMP_IF_NOT_EXC_MATCH, + [JUMP_IF_TRUE_OR_POP] = JUMP_IF_TRUE_OR_POP, + [JUMP_NO_INTERRUPT] = JUMP_NO_INTERRUPT, + [KW_NAMES] = KW_NAMES, + [LIST_APPEND] = LIST_APPEND, + [LIST_EXTEND] = LIST_EXTEND, + [LIST_TO_TUPLE] = LIST_TO_TUPLE, + [LOAD_ASSERTION_ERROR] = LOAD_ASSERTION_ERROR, + [LOAD_ATTR] = LOAD_ATTR, + [LOAD_ATTR_ADAPTIVE] = LOAD_ATTR, + [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR, + [LOAD_ATTR_MODULE] = LOAD_ATTR, + [LOAD_ATTR_SLOT] = LOAD_ATTR, + [LOAD_ATTR_WITH_HINT] = LOAD_ATTR, + [LOAD_BUILD_CLASS] = LOAD_BUILD_CLASS, + [LOAD_CLASSDEREF] = LOAD_CLASSDEREF, + [LOAD_CLOSURE] = LOAD_CLOSURE, + [LOAD_CONST] = LOAD_CONST, + [LOAD_CONST__LOAD_FAST] = LOAD_CONST, + [LOAD_DEREF] = LOAD_DEREF, + [LOAD_FAST] = LOAD_FAST, + [LOAD_FAST__LOAD_CONST] = LOAD_FAST, + [LOAD_FAST__LOAD_FAST] = LOAD_FAST, + [LOAD_GLOBAL] = LOAD_GLOBAL, + [LOAD_GLOBAL_ADAPTIVE] = LOAD_GLOBAL, + [LOAD_GLOBAL_BUILTIN] = LOAD_GLOBAL, + [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL, + [LOAD_METHOD] = LOAD_METHOD, + [LOAD_METHOD_ADAPTIVE] = LOAD_METHOD, + [LOAD_METHOD_CLASS] = LOAD_METHOD, + [LOAD_METHOD_MODULE] = LOAD_METHOD, + [LOAD_METHOD_NO_DICT] = LOAD_METHOD, + [LOAD_METHOD_WITH_DICT] = LOAD_METHOD, + [LOAD_METHOD_WITH_VALUES] = LOAD_METHOD, + [LOAD_NAME] = LOAD_NAME, + [MAKE_CELL] = MAKE_CELL, + [MAKE_FUNCTION] = MAKE_FUNCTION, + [MAP_ADD] = MAP_ADD, + [MATCH_CLASS] = MATCH_CLASS, + [MATCH_KEYS] = MATCH_KEYS, + [MATCH_MAPPING] = MATCH_MAPPING, + [MATCH_SEQUENCE] = MATCH_SEQUENCE, + [NOP] = NOP, + [POP_EXCEPT] = POP_EXCEPT, + [POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE, + [POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, + [POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, + [POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE, + [POP_TOP] = POP_TOP, + [PRECALL] = PRECALL, + [PRECALL_ADAPTIVE] = PRECALL, + [PRECALL_BOUND_METHOD] = PRECALL, + [PRECALL_BUILTIN_CLASS] = PRECALL, + [PRECALL_BUILTIN_FAST_WITH_KEYWORDS] = PRECALL, + [PRECALL_NO_KW_BUILTIN_FAST] = PRECALL, + [PRECALL_NO_KW_BUILTIN_O] = PRECALL, + [PRECALL_NO_KW_ISINSTANCE] = PRECALL, + [PRECALL_NO_KW_LEN] = PRECALL, + [PRECALL_NO_KW_LIST_APPEND] = PRECALL, + [PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST] = PRECALL, + [PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = PRECALL, + [PRECALL_NO_KW_METHOD_DESCRIPTOR_O] = PRECALL, + [PRECALL_NO_KW_STR_1] = PRECALL, + [PRECALL_NO_KW_TUPLE_1] = PRECALL, + [PRECALL_NO_KW_TYPE_1] = PRECALL, + [PRECALL_PYFUNC] = PRECALL, + [PREP_RERAISE_STAR] = PREP_RERAISE_STAR, + [PRINT_EXPR] = PRINT_EXPR, + [PUSH_EXC_INFO] = PUSH_EXC_INFO, + [PUSH_NULL] = PUSH_NULL, + [RAISE_VARARGS] = RAISE_VARARGS, + [RERAISE] = RERAISE, + [RESUME] = RESUME, + [RESUME_QUICK] = RESUME, + [RETURN_GENERATOR] = RETURN_GENERATOR, + [RETURN_VALUE] = RETURN_VALUE, + [SEND] = SEND, + [SETUP_ANNOTATIONS] = SETUP_ANNOTATIONS, + [SET_ADD] = SET_ADD, + [SET_UPDATE] = SET_UPDATE, + [STORE_ATTR] = STORE_ATTR, + [STORE_ATTR_ADAPTIVE] = STORE_ATTR, + [STORE_ATTR_INSTANCE_VALUE] = STORE_ATTR, + [STORE_ATTR_SLOT] = STORE_ATTR, + [STORE_ATTR_WITH_HINT] = STORE_ATTR, + [STORE_DEREF] = STORE_DEREF, + [STORE_FAST] = STORE_FAST, + [STORE_FAST__LOAD_FAST] = STORE_FAST, + [STORE_FAST__STORE_FAST] = STORE_FAST, + [STORE_GLOBAL] = STORE_GLOBAL, + [STORE_NAME] = STORE_NAME, + [STORE_SUBSCR] = STORE_SUBSCR, + [STORE_SUBSCR_ADAPTIVE] = STORE_SUBSCR, + [STORE_SUBSCR_DICT] = STORE_SUBSCR, + [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR, + [SWAP] = SWAP, + [UNARY_INVERT] = UNARY_INVERT, + [UNARY_NEGATIVE] = UNARY_NEGATIVE, + [UNARY_NOT] = UNARY_NOT, + [UNARY_POSITIVE] = UNARY_POSITIVE, + [UNPACK_EX] = UNPACK_EX, + [UNPACK_SEQUENCE] = UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_ADAPTIVE] = UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_LIST] = UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_TUPLE] = UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_TWO_TUPLE] = UNPACK_SEQUENCE, + [WITH_EXCEPT_START] = WITH_EXCEPT_START, + [YIELD_VALUE] = YIELD_VALUE, +}; #endif /* OPCODE_TABLES */ #define HAS_CONST(op) (false\ @@ -229,32 +409,32 @@ const uint8_t _PyOpcode_InlineCacheEntries[256] = { || ((op) == 172) \ ) -#define NB_ADD 0 -#define NB_AND 1 -#define NB_FLOOR_DIVIDE 2 -#define NB_LSHIFT 3 -#define NB_MATRIX_MULTIPLY 4 -#define NB_MULTIPLY 5 -#define NB_REMAINDER 6 -#define NB_OR 7 -#define NB_POWER 8 -#define NB_RSHIFT 9 -#define NB_SUBTRACT 10 -#define NB_TRUE_DIVIDE 11 -#define NB_XOR 12 -#define NB_INPLACE_ADD 13 -#define NB_INPLACE_AND 14 -#define NB_INPLACE_FLOOR_DIVIDE 15 -#define NB_INPLACE_LSHIFT 16 -#define NB_INPLACE_MATRIX_MULTIPLY 17 -#define NB_INPLACE_MULTIPLY 18 -#define NB_INPLACE_REMAINDER 19 -#define NB_INPLACE_OR 20 -#define NB_INPLACE_POWER 21 -#define NB_INPLACE_RSHIFT 22 -#define NB_INPLACE_SUBTRACT 23 -#define NB_INPLACE_TRUE_DIVIDE 24 -#define NB_INPLACE_XOR 25 +#define NB_ADD 0 +#define NB_AND 1 +#define NB_FLOOR_DIVIDE 2 +#define NB_LSHIFT 3 +#define NB_MATRIX_MULTIPLY 4 +#define NB_MULTIPLY 5 +#define NB_REMAINDER 6 +#define NB_OR 7 +#define NB_POWER 8 +#define NB_RSHIFT 9 +#define NB_SUBTRACT 10 +#define NB_TRUE_DIVIDE 11 +#define NB_XOR 12 +#define NB_INPLACE_ADD 13 +#define NB_INPLACE_AND 14 +#define NB_INPLACE_FLOOR_DIVIDE 15 +#define NB_INPLACE_LSHIFT 16 +#define NB_INPLACE_MATRIX_MULTIPLY 17 +#define NB_INPLACE_MULTIPLY 18 +#define NB_INPLACE_REMAINDER 19 +#define NB_INPLACE_OR 20 +#define NB_INPLACE_POWER 21 +#define NB_INPLACE_RSHIFT 22 +#define NB_INPLACE_SUBTRACT 23 +#define NB_INPLACE_TRUE_DIVIDE 24 +#define NB_INPLACE_XOR 25 #define HAS_ARG(op) ((op) >= HAVE_ARGUMENT) diff --git a/Lib/opcode.py b/Lib/opcode.py index eb9dd35fabf..7a52c13579a 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -229,77 +229,111 @@ _nb_ops = [ ("NB_INPLACE_XOR", "^="), ] +_specializations = { + "BINARY_OP": [ + "BINARY_OP_ADAPTIVE", + "BINARY_OP_ADD_FLOAT", + "BINARY_OP_ADD_INT", + "BINARY_OP_ADD_UNICODE", + "BINARY_OP_INPLACE_ADD_UNICODE", + "BINARY_OP_MULTIPLY_FLOAT", + "BINARY_OP_MULTIPLY_INT", + "BINARY_OP_SUBTRACT_FLOAT", + "BINARY_OP_SUBTRACT_INT", + ], + "BINARY_SUBSCR": [ + "BINARY_SUBSCR_ADAPTIVE", + "BINARY_SUBSCR_DICT", + "BINARY_SUBSCR_GETITEM", + "BINARY_SUBSCR_LIST_INT", + "BINARY_SUBSCR_TUPLE_INT", + ], + "CALL": [ + "CALL_ADAPTIVE", + "CALL_PY_EXACT_ARGS", + "CALL_PY_WITH_DEFAULTS", + ], + "COMPARE_OP": [ + "COMPARE_OP_ADAPTIVE", + "COMPARE_OP_FLOAT_JUMP", + "COMPARE_OP_INT_JUMP", + "COMPARE_OP_STR_JUMP", + ], + "JUMP_ABSOLUTE": [ + "JUMP_ABSOLUTE_QUICK", + ], + "LOAD_ATTR": [ + "LOAD_ATTR_ADAPTIVE", + "LOAD_ATTR_INSTANCE_VALUE", + "LOAD_ATTR_MODULE", + "LOAD_ATTR_SLOT", + "LOAD_ATTR_WITH_HINT", + ], + "LOAD_CONST": [ + "LOAD_CONST__LOAD_FAST", + ], + "LOAD_FAST": [ + "LOAD_FAST__LOAD_CONST", + "LOAD_FAST__LOAD_FAST", + ], + "LOAD_GLOBAL": [ + "LOAD_GLOBAL_ADAPTIVE", + "LOAD_GLOBAL_BUILTIN", + "LOAD_GLOBAL_MODULE", + ], + "LOAD_METHOD": [ + "LOAD_METHOD_ADAPTIVE", + "LOAD_METHOD_CLASS", + "LOAD_METHOD_MODULE", + "LOAD_METHOD_NO_DICT", + "LOAD_METHOD_WITH_DICT", + "LOAD_METHOD_WITH_VALUES", + ], + "PRECALL": [ + "PRECALL_ADAPTIVE", + "PRECALL_BOUND_METHOD", + "PRECALL_BUILTIN_CLASS", + "PRECALL_BUILTIN_FAST_WITH_KEYWORDS", + "PRECALL_NO_KW_BUILTIN_FAST", + "PRECALL_NO_KW_BUILTIN_O", + "PRECALL_NO_KW_ISINSTANCE", + "PRECALL_NO_KW_LEN", + "PRECALL_NO_KW_LIST_APPEND", + "PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST", + "PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", + "PRECALL_NO_KW_METHOD_DESCRIPTOR_O", + "PRECALL_NO_KW_STR_1", + "PRECALL_NO_KW_TUPLE_1", + "PRECALL_NO_KW_TYPE_1", + "PRECALL_PYFUNC", + ], + "RESUME": [ + "RESUME_QUICK", + ], + "STORE_ATTR": [ + "STORE_ATTR_ADAPTIVE", + "STORE_ATTR_INSTANCE_VALUE", + "STORE_ATTR_SLOT", + "STORE_ATTR_WITH_HINT", + ], + "STORE_FAST": [ + "STORE_FAST__LOAD_FAST", + "STORE_FAST__STORE_FAST", + ], + "STORE_SUBSCR": [ + "STORE_SUBSCR_ADAPTIVE", + "STORE_SUBSCR_DICT", + "STORE_SUBSCR_LIST_INT", + ], + "UNPACK_SEQUENCE": [ + "UNPACK_SEQUENCE_ADAPTIVE", + "UNPACK_SEQUENCE_LIST", + "UNPACK_SEQUENCE_TUPLE", + "UNPACK_SEQUENCE_TWO_TUPLE", + ], +} _specialized_instructions = [ - "BINARY_OP_ADAPTIVE", - "BINARY_OP_ADD_INT", - "BINARY_OP_ADD_FLOAT", - "BINARY_OP_ADD_UNICODE", - "BINARY_OP_INPLACE_ADD_UNICODE", - "BINARY_OP_MULTIPLY_INT", - "BINARY_OP_MULTIPLY_FLOAT", - "BINARY_OP_SUBTRACT_INT", - "BINARY_OP_SUBTRACT_FLOAT", - "COMPARE_OP_ADAPTIVE", - "COMPARE_OP_FLOAT_JUMP", - "COMPARE_OP_INT_JUMP", - "COMPARE_OP_STR_JUMP", - "BINARY_SUBSCR_ADAPTIVE", - "BINARY_SUBSCR_GETITEM", - "BINARY_SUBSCR_LIST_INT", - "BINARY_SUBSCR_TUPLE_INT", - "BINARY_SUBSCR_DICT", - "STORE_SUBSCR_ADAPTIVE", - "STORE_SUBSCR_LIST_INT", - "STORE_SUBSCR_DICT", - "CALL_ADAPTIVE", - "CALL_PY_EXACT_ARGS", - "CALL_PY_WITH_DEFAULTS", - "JUMP_ABSOLUTE_QUICK", - "LOAD_ATTR_ADAPTIVE", - "LOAD_ATTR_INSTANCE_VALUE", - "LOAD_ATTR_WITH_HINT", - "LOAD_ATTR_SLOT", - "LOAD_ATTR_MODULE", - "LOAD_GLOBAL_ADAPTIVE", - "LOAD_GLOBAL_MODULE", - "LOAD_GLOBAL_BUILTIN", - "LOAD_METHOD_ADAPTIVE", - "LOAD_METHOD_CLASS", - "LOAD_METHOD_MODULE", - "LOAD_METHOD_NO_DICT", - "LOAD_METHOD_WITH_DICT", - "LOAD_METHOD_WITH_VALUES", - "PRECALL_ADAPTIVE", - "PRECALL_BUILTIN_CLASS", - "PRECALL_NO_KW_BUILTIN_O", - "PRECALL_NO_KW_BUILTIN_FAST", - "PRECALL_BUILTIN_FAST_WITH_KEYWORDS", - "PRECALL_NO_KW_LEN", - "PRECALL_NO_KW_ISINSTANCE", - "PRECALL_NO_KW_LIST_APPEND", - "PRECALL_NO_KW_METHOD_DESCRIPTOR_O", - "PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", - "PRECALL_NO_KW_STR_1", - "PRECALL_NO_KW_TUPLE_1", - "PRECALL_NO_KW_TYPE_1", - "PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST", - "PRECALL_BOUND_METHOD", - "PRECALL_PYFUNC", - "RESUME_QUICK", - "STORE_ATTR_ADAPTIVE", - "STORE_ATTR_INSTANCE_VALUE", - "STORE_ATTR_SLOT", - "STORE_ATTR_WITH_HINT", - "UNPACK_SEQUENCE_ADAPTIVE", - "UNPACK_SEQUENCE_LIST", - "UNPACK_SEQUENCE_TUPLE", - "UNPACK_SEQUENCE_TWO_TUPLE", - # Super instructions - "LOAD_FAST__LOAD_FAST", - "STORE_FAST__LOAD_FAST", - "LOAD_FAST__LOAD_CONST", - "LOAD_CONST__LOAD_FAST", - "STORE_FAST__STORE_FAST", + opcode for family in _specializations.values() for opcode in family ] _specialization_stats = [ "success", diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 8499d2855bd..a4e80805d3e 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -641,7 +641,7 @@ if 1: self.check_constant(f1, frozenset({0})) self.assertTrue(f1(0)) - # Merging equal co_linetable and co_code is not a strict requirement + # Merging equal co_linetable is not a strict requirement # for the Python semantics, it's a more an implementation detail. @support.cpython_only def test_merge_code_attrs(self): @@ -650,7 +650,6 @@ if 1: f2 = lambda a: a.b.c self.assertIs(f1.__code__.co_linetable, f2.__code__.co_linetable) - self.assertIs(f1.__code__.co_code, f2.__code__.co_code) # Stripping unused constants is not a strict requirement for the # Python semantics, it's a more an implementation detail. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-03-16-11-05-35.bpo-46841.yUoIHg.rst b/Misc/NEWS.d/next/Core and Builtins/2022-03-16-11-05-35.bpo-46841.yUoIHg.rst new file mode 100644 index 00000000000..99fad382d13 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-03-16-11-05-35.bpo-46841.yUoIHg.rst @@ -0,0 +1,2 @@ +Quicken bytecode in-place by storing it as part of the corresponding +``PyCodeObject``. diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h index ee425f61bb1..272bcd6ea17 100644 --- a/Objects/clinic/codeobject.c.h +++ b/Objects/clinic/codeobject.c.h @@ -203,12 +203,12 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje int co_stacksize = self->co_stacksize; int co_flags = self->co_flags; int co_firstlineno = self->co_firstlineno; - PyBytesObject *co_code = (PyBytesObject *)self->co_code; + PyBytesObject *co_code = NULL; PyObject *co_consts = self->co_consts; PyObject *co_names = self->co_names; - PyObject *co_varnames = self->co_varnames; - PyObject *co_freevars = self->co_freevars; - PyObject *co_cellvars = self->co_cellvars; + PyObject *co_varnames = NULL; + PyObject *co_freevars = NULL; + PyObject *co_cellvars = NULL; PyObject *co_filename = self->co_filename; PyObject *co_name = self->co_name; PyObject *co_qualname = self->co_qualname; @@ -456,4 +456,4 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n exit: return return_value; } -/*[clinic end generated code: output=9e8c4a19474ec520 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b1b83a70ffc5b7cd input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 5279f6ce170..224493edb19 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -305,9 +305,6 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_qualname = con->qualname; co->co_flags = con->flags; - Py_INCREF(con->code); - co->co_code = con->code; - co->co_firstinstr = (_Py_CODEUNIT *)PyBytes_AS_STRING(con->code); co->co_firstlineno = con->firstlineno; Py_INCREF(con->linetable); co->co_linetable = con->linetable; @@ -341,16 +338,14 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_nplaincellvars = nplaincellvars; co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; - co->co_varnames = NULL; - co->co_cellvars = NULL; - co->co_freevars = NULL; /* not set */ co->co_weakreflist = NULL; co->co_extra = NULL; co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE; - co->co_quickened = NULL; + memcpy(_PyCode_CODE(co), PyBytes_AS_STRING(con->code), + PyBytes_GET_SIZE(con->code)); } /* The caller is responsible for ensuring that the given data is valid. */ @@ -386,7 +381,8 @@ _PyCode_New(struct _PyCodeConstructor *con) con->columntable = Py_None; } - PyCodeObject *co = PyObject_New(PyCodeObject, &PyCode_Type); + Py_ssize_t size = PyBytes_GET_SIZE(con->code) / sizeof(_Py_CODEUNIT); + PyCodeObject *co = PyObject_NewVar(PyCodeObject, &PyCode_Type, size); if (co == NULL) { PyErr_NoMemory(); return NULL; @@ -521,13 +517,6 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, goto error; } - Py_INCREF(varnames); - co->co_varnames = varnames; - Py_INCREF(cellvars); - co->co_cellvars = cellvars; - Py_INCREF(freevars); - co->co_freevars = freevars; - error: Py_XDECREF(localsplusnames); Py_XDECREF(localspluskinds); @@ -611,7 +600,7 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq) if (addrq < 0) { return co->co_firstlineno; } - assert(addrq >= 0 && addrq < PyBytes_GET_SIZE(co->co_code)); + assert(addrq >= 0 && addrq < _PyCode_NBYTES(co)); PyCodeAddressRange bounds; _PyCode_InitAddressRange(co, &bounds); return _PyCode_CheckLineNumber(addrq, &bounds); @@ -639,7 +628,7 @@ _PyCode_Addr2EndLine(PyCodeObject* co, int addrq) return -1; } - assert(addrq >= 0 && addrq < PyBytes_GET_SIZE(co->co_code)); + assert(addrq >= 0 && addrq < _PyCode_NBYTES(co)); PyCodeAddressRange bounds; _PyCode_InitEndAddressRange(co, &bounds); return _PyCode_CheckLineNumber(addrq, &bounds); @@ -995,7 +984,7 @@ _source_offset_converter(int* value) { static PyObject* positionsiter_next(positionsiterator* pi) { - if (pi->pi_offset >= PyBytes_GET_SIZE(pi->pi_code->co_code)) { + if (pi->pi_offset >= _PyCode_NBYTES(pi->pi_code)) { return NULL; } @@ -1151,46 +1140,39 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) PyObject * _PyCode_GetVarnames(PyCodeObject *co) { - if (co->co_varnames == NULL) { - // PyCodeObject owns this reference. - co->co_varnames = get_localsplus_names(co, CO_FAST_LOCAL, - co->co_nlocals); - if (co->co_varnames == NULL) { - return NULL; - } - } - Py_INCREF(co->co_varnames); - return co->co_varnames; + return get_localsplus_names(co, CO_FAST_LOCAL, co->co_nlocals); } PyObject * _PyCode_GetCellvars(PyCodeObject *co) { - if (co->co_cellvars == NULL) { - // PyCodeObject owns this reference. - co->co_cellvars = get_localsplus_names(co, CO_FAST_CELL, - co->co_ncellvars); - if (co->co_cellvars == NULL) { - return NULL; - } - } - Py_INCREF(co->co_cellvars); - return co->co_cellvars; + return get_localsplus_names(co, CO_FAST_CELL, co->co_ncellvars); } PyObject * _PyCode_GetFreevars(PyCodeObject *co) { - if (co->co_freevars == NULL) { - // PyCodeObject owns this reference. - co->co_freevars = get_localsplus_names(co, CO_FAST_FREE, - co->co_nfreevars); - if (co->co_freevars == NULL) { - return NULL; + return get_localsplus_names(co, CO_FAST_FREE, co->co_nfreevars); +} + +PyObject * +_PyCode_GetCode(PyCodeObject *co) +{ + PyObject *code = PyBytes_FromStringAndSize(NULL, _PyCode_NBYTES(co)); + if (code == NULL) { + return NULL; + } + _Py_CODEUNIT *instructions = (_Py_CODEUNIT *)PyBytes_AS_STRING(code); + for (int i = 0; i < Py_SIZE(co); i++) { + _Py_CODEUNIT instruction = _PyCode_CODE(co)[i]; + int opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)]; + int caches = _PyOpcode_Caches[opcode]; + instructions[i] = _Py_MAKECODEUNIT(opcode, _Py_OPARG(instruction)); + while (caches--) { + instructions[++i] = _Py_MAKECODEUNIT(CACHE, 0); } } - Py_INCREF(co->co_freevars); - return co->co_freevars; + return code; } @@ -1348,14 +1330,10 @@ code_dealloc(PyCodeObject *co) PyMem_Free(co_extra); } - Py_XDECREF(co->co_code); Py_XDECREF(co->co_consts); Py_XDECREF(co->co_names); Py_XDECREF(co->co_localsplusnames); Py_XDECREF(co->co_localspluskinds); - Py_XDECREF(co->co_varnames); - Py_XDECREF(co->co_freevars); - Py_XDECREF(co->co_cellvars); Py_XDECREF(co->co_filename); Py_XDECREF(co->co_name); Py_XDECREF(co->co_qualname); @@ -1363,10 +1341,10 @@ code_dealloc(PyCodeObject *co) Py_XDECREF(co->co_endlinetable); Py_XDECREF(co->co_columntable); Py_XDECREF(co->co_exceptiontable); - if (co->co_weakreflist != NULL) + if (co->co_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject*)co); - if (co->co_quickened) { - PyMem_Free(co->co_quickened); + } + if (co->co_warmup == 0) { _Py_QuickenedCount--; } PyObject_Free(co); @@ -1420,8 +1398,21 @@ code_richcompare(PyObject *self, PyObject *other, int op) if (!eq) goto unequal; eq = co->co_firstlineno == cp->co_firstlineno; if (!eq) goto unequal; - eq = PyObject_RichCompareBool(co->co_code, cp->co_code, Py_EQ); - if (eq <= 0) goto unequal; + PyObject *co_code = _PyCode_GetCode(co); + if (co_code == NULL) { + return NULL; + } + PyObject *cp_code = _PyCode_GetCode(cp); + if (cp_code == NULL) { + Py_DECREF(co_code); + return NULL; + } + eq = PyObject_RichCompareBool(co_code, cp_code, Py_EQ); + Py_DECREF(co_code); + Py_DECREF(cp_code); + if (eq <= 0) { + goto unequal; + } /* compare constants */ consts1 = _PyCode_ConstantKey(co->co_consts); @@ -1465,18 +1456,16 @@ code_richcompare(PyObject *self, PyObject *other, int op) static Py_hash_t code_hash(PyCodeObject *co) { - Py_hash_t h, h0, h1, h2, h3, h4; + Py_hash_t h, h0, h1, h2, h3; h0 = PyObject_Hash(co->co_name); if (h0 == -1) return -1; - h1 = PyObject_Hash(co->co_code); + h1 = PyObject_Hash(co->co_consts); if (h1 == -1) return -1; - h2 = PyObject_Hash(co->co_consts); + h2 = PyObject_Hash(co->co_names); if (h2 == -1) return -1; - h3 = PyObject_Hash(co->co_names); + h3 = PyObject_Hash(co->co_localsplusnames); if (h3 == -1) return -1; - h4 = PyObject_Hash(co->co_localsplusnames); - if (h4 == -1) return -1; - h = h0 ^ h1 ^ h2 ^ h3 ^ h4 ^ + h = h0 ^ h1 ^ h2 ^ h3 ^ co->co_argcount ^ co->co_posonlyargcount ^ co->co_kwonlyargcount ^ co->co_flags; if (h == -1) h = -2; @@ -1487,22 +1476,22 @@ code_hash(PyCodeObject *co) #define OFF(x) offsetof(PyCodeObject, x) static PyMemberDef code_memberlist[] = { - {"co_argcount", T_INT, OFF(co_argcount), READONLY}, - {"co_posonlyargcount", T_INT, OFF(co_posonlyargcount), READONLY}, - {"co_kwonlyargcount", T_INT, OFF(co_kwonlyargcount), READONLY}, - {"co_stacksize",T_INT, OFF(co_stacksize), READONLY}, - {"co_flags", T_INT, OFF(co_flags), READONLY}, - {"co_code", T_OBJECT, OFF(co_code), READONLY}, - {"co_consts", T_OBJECT, OFF(co_consts), READONLY}, - {"co_names", T_OBJECT, OFF(co_names), READONLY}, - {"co_filename", T_OBJECT, OFF(co_filename), READONLY}, - {"co_name", T_OBJECT, OFF(co_name), READONLY}, - {"co_qualname", T_OBJECT, OFF(co_qualname), READONLY}, - {"co_firstlineno", T_INT, OFF(co_firstlineno), READONLY}, - {"co_linetable", T_OBJECT, OFF(co_linetable), READONLY}, - {"co_endlinetable", T_OBJECT, OFF(co_endlinetable), READONLY}, - {"co_columntable", T_OBJECT, OFF(co_columntable), READONLY}, - {"co_exceptiontable", T_OBJECT, OFF(co_exceptiontable), READONLY}, + {"co_argcount", T_INT, OFF(co_argcount), READONLY}, + {"co_posonlyargcount", T_INT, OFF(co_posonlyargcount), READONLY}, + {"co_kwonlyargcount", T_INT, OFF(co_kwonlyargcount), READONLY}, + {"co_stacksize", T_INT, OFF(co_stacksize), READONLY}, + {"co_flags", T_INT, OFF(co_flags), READONLY}, + {"co_nlocals", T_INT, OFF(co_nlocals), READONLY}, + {"co_consts", T_OBJECT, OFF(co_consts), READONLY}, + {"co_names", T_OBJECT, OFF(co_names), READONLY}, + {"co_filename", T_OBJECT, OFF(co_filename), READONLY}, + {"co_name", T_OBJECT, OFF(co_name), READONLY}, + {"co_qualname", T_OBJECT, OFF(co_qualname), READONLY}, + {"co_firstlineno", T_INT, OFF(co_firstlineno), READONLY}, + {"co_linetable", T_OBJECT, OFF(co_linetable), READONLY}, + {"co_endlinetable", T_OBJECT, OFF(co_endlinetable), READONLY}, + {"co_columntable", T_OBJECT, OFF(co_columntable), READONLY}, + {"co_exceptiontable", T_OBJECT, OFF(co_exceptiontable), READONLY}, {NULL} /* Sentinel */ }; @@ -1513,12 +1502,6 @@ code_getlnotab(PyCodeObject *code, void *closure) return decode_linetable(code); } -static PyObject * -code_getnlocals(PyCodeObject *code, void *closure) -{ - return PyLong_FromLong(code->co_nlocals); -} - static PyObject * code_getvarnames(PyCodeObject *code, void *closure) { @@ -1538,23 +1521,26 @@ code_getfreevars(PyCodeObject *code, void *closure) } static PyObject * -code_getquickened(PyCodeObject *code, void *closure) +code_getcodeadaptive(PyCodeObject *code, void *closure) { - if (code->co_quickened == NULL) { - Py_RETURN_NONE; - } - return PyBytes_FromStringAndSize((char *)code->co_firstinstr, - PyBytes_Size(code->co_code)); + return PyMemoryView_FromMemory(code->co_code_adaptive, _PyCode_NBYTES(code), + PyBUF_READ); +} + +static PyObject * +code_getcode(PyCodeObject *code, void *closure) +{ + return _PyCode_GetCode(code); } static PyGetSetDef code_getsetlist[] = { - {"co_lnotab", (getter)code_getlnotab, NULL, NULL}, + {"co_lnotab", (getter)code_getlnotab, NULL, NULL}, + {"_co_code_adaptive", (getter)code_getcodeadaptive, NULL, NULL}, // The following old names are kept for backward compatibility. - {"co_nlocals", (getter)code_getnlocals, NULL, NULL}, - {"co_varnames", (getter)code_getvarnames, NULL, NULL}, - {"co_cellvars", (getter)code_getcellvars, NULL, NULL}, - {"co_freevars", (getter)code_getfreevars, NULL, NULL}, - {"_co_quickened", (getter)code_getquickened, NULL, NULL}, + {"co_varnames", (getter)code_getvarnames, NULL, NULL}, + {"co_cellvars", (getter)code_getcellvars, NULL, NULL}, + {"co_freevars", (getter)code_getfreevars, NULL, NULL}, + {"co_code", (getter)code_getcode, NULL, NULL}, {0} }; @@ -1562,7 +1548,7 @@ static PyGetSetDef code_getsetlist[] = { static PyObject * code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args)) { - Py_ssize_t res = _PyObject_SIZE(Py_TYPE(co)); + Py_ssize_t res = _PyObject_VAR_SIZE(Py_TYPE(co), Py_SIZE(co)); _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) co->co_extra; if (co_extra != NULL) { @@ -1570,10 +1556,6 @@ code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args)) (co_extra->ce_size-1) * sizeof(co_extra->ce_extras[0]); } - if (co->co_quickened != NULL) { - res += PyBytes_GET_SIZE(co->co_code); - } - return PyLong_FromSsize_t(res); } @@ -1594,12 +1576,12 @@ code.replace co_stacksize: int(c_default="self->co_stacksize") = -1 co_flags: int(c_default="self->co_flags") = -1 co_firstlineno: int(c_default="self->co_firstlineno") = -1 - co_code: PyBytesObject(c_default="(PyBytesObject *)self->co_code") = None + co_code: PyBytesObject(c_default="NULL") = None co_consts: object(subclass_of="&PyTuple_Type", c_default="self->co_consts") = None co_names: object(subclass_of="&PyTuple_Type", c_default="self->co_names") = None - co_varnames: object(subclass_of="&PyTuple_Type", c_default="self->co_varnames") = None - co_freevars: object(subclass_of="&PyTuple_Type", c_default="self->co_freevars") = None - co_cellvars: object(subclass_of="&PyTuple_Type", c_default="self->co_cellvars") = None + co_varnames: object(subclass_of="&PyTuple_Type", c_default="NULL") = None + co_freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None + co_cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None co_filename: unicode(c_default="self->co_filename") = None co_name: unicode(c_default="self->co_name") = None co_qualname: unicode(c_default="self->co_qualname") = None @@ -1622,7 +1604,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount, PyObject *co_name, PyObject *co_qualname, PyBytesObject *co_linetable, PyObject *co_endlinetable, PyObject *co_columntable, PyBytesObject *co_exceptiontable) -/*[clinic end generated code: output=f046bf0be3bab91f input=a63d09f248f00794]*/ +/*[clinic end generated code: output=f046bf0be3bab91f input=78dbe204dbd06c2f]*/ { #define CHECK_INT_ARG(ARG) \ if (ARG < 0) { \ @@ -1641,6 +1623,15 @@ code_replace_impl(PyCodeObject *self, int co_argcount, #undef CHECK_INT_ARG + PyObject *code = NULL; + if (co_code == NULL) { + code = _PyCode_GetCode(self); + if (code == NULL) { + return NULL; + } + co_code = (PyBytesObject *)code; + } + if (PySys_Audit("code.__new__", "OOOiiiiii", co_code, co_filename, co_name, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, @@ -1694,6 +1685,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount, (PyObject*)co_exceptiontable); error: + Py_XDECREF(code); Py_XDECREF(varnames); Py_XDECREF(cellvars); Py_XDECREF(freevars); @@ -1737,8 +1729,8 @@ static struct PyMethodDef code_methods[] = { PyTypeObject PyCode_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "code", - sizeof(PyCodeObject), - 0, + offsetof(PyCodeObject, co_code_adaptive), + sizeof(_Py_CODEUNIT), (destructor)code_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ @@ -1913,15 +1905,12 @@ _PyCode_ConstantKey(PyObject *op) void _PyStaticCode_Dealloc(PyCodeObject *co) { - if (co->co_quickened) { - PyMem_Free(co->co_quickened); - co->co_quickened = NULL; + if (co->co_warmup == 0) { _Py_QuickenedCount--; } co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE; PyMem_Free(co->co_extra); co->co_extra = NULL; - co->co_firstinstr = (_Py_CODEUNIT *)PyBytes_AS_STRING(co->co_code); if (co->co_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *)co); co->co_weakreflist = NULL; diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 73b6c3d9f8a..7ccd300e2b4 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -105,8 +105,9 @@ frame_getback(PyFrameObject *f, void *closure) return res; } -/* Given the index of the effective opcode, - scan back to construct the oparg with EXTENDED_ARG */ +// Given the index of the effective opcode, scan back to construct the oparg +// with EXTENDED_ARG. This only works correctly with *unquickened* code, +// obtained via a call to _PyCode_GetCode! static unsigned int get_arg(const _Py_CODEUNIT *codestr, Py_ssize_t i) { @@ -170,13 +171,17 @@ top_of_stack(int64_t stack) static int64_t * mark_stacks(PyCodeObject *code_obj, int len) { - const _Py_CODEUNIT *code = - (const _Py_CODEUNIT *)PyBytes_AS_STRING(code_obj->co_code); + PyObject *co_code = _PyCode_GetCode(code_obj); + if (co_code == NULL) { + return NULL; + } + _Py_CODEUNIT *code = (_Py_CODEUNIT *)PyBytes_AS_STRING(co_code); int64_t *stacks = PyMem_New(int64_t, len+1); int i, j, opcode; if (stacks == NULL) { PyErr_NoMemory(); + Py_DECREF(co_code); return NULL; } for (int i = 1; i <= len; i++) { @@ -304,6 +309,7 @@ mark_stacks(PyCodeObject *code_obj, int len) } } } + Py_DECREF(co_code); return stacks; } @@ -493,7 +499,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore /* PyCode_NewWithPosOnlyArgs limits co_code to be under INT_MAX so this * should never overflow. */ - int len = (int)(PyBytes_GET_SIZE(f->f_frame->f_code->co_code) / sizeof(_Py_CODEUNIT)); + int len = (int)Py_SIZE(f->f_frame->f_code); int *lines = marklines(f->f_frame->f_code, len); if (lines == NULL) { return -1; @@ -838,12 +844,23 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, static int _PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg) { - const _Py_CODEUNIT *code = - (const _Py_CODEUNIT *)PyBytes_AS_STRING(frame->f_code->co_code); + // This only works when opcode is a non-quickened form: + assert(_PyOpcode_Deopt[opcode] == opcode); + int check_oparg = 0; for (int i = 0; i < frame->f_lasti; i++) { - if (_Py_OPCODE(code[i]) == opcode && _Py_OPARG(code[i]) == oparg) { + _Py_CODEUNIT instruction = _PyCode_CODE(frame->f_code)[i]; + int check_opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)]; + check_oparg |= _Py_OPARG(instruction); + if (check_opcode == opcode && check_oparg == oparg) { return 1; } + if (check_opcode == EXTENDED_ARG) { + check_oparg <<= 8; + } + else { + check_oparg = 0; + } + i += _PyOpcode_Caches[check_opcode]; } return 0; } @@ -862,7 +879,10 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) { } co = frame->f_code; fast = _PyFrame_GetLocalsArray(frame); - if (frame->f_lasti < 0 && _Py_OPCODE(co->co_firstinstr[0]) == COPY_FREE_VARS) { + // COPY_FREE_VARS has no quickened forms, so no need to use _PyOpcode_Deopt + // here: + if (frame->f_lasti < 0 && _Py_OPCODE(_PyCode_CODE(co)[0]) == COPY_FREE_VARS) + { /* Free vars have not been initialized -- Do that */ PyCodeObject *co = frame->f_code; PyObject *closure = frame->f_func->func_closure; @@ -872,6 +892,7 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) { Py_INCREF(o); frame->localsplus[offset + i] = o; } + // COPY_FREE_VARS doesn't have inline CACHEs, either: frame->f_lasti = 0; } for (int i = 0; i < co->co_nlocalsplus; i++) { diff --git a/Objects/genobject.c b/Objects/genobject.c index 6551b939c45..3ad8dc1c459 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -349,19 +349,17 @@ _PyGen_yf(PyGenObject *gen) if (gen->gi_frame_valid) { _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; - PyObject *bytecode = gen->gi_code->co_code; - unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode); if (frame->f_lasti < 1) { /* Return immediately if the frame didn't start yet. SEND always come after LOAD_CONST: a code object should not start with SEND */ - assert(code[0] != SEND); + assert(_Py_OPCODE(_PyCode_CODE(gen->gi_code)[0]) != SEND); return NULL; } - int opcode = code[(frame->f_lasti+1)*sizeof(_Py_CODEUNIT)]; - int oparg = code[(frame->f_lasti+1)*sizeof(_Py_CODEUNIT)+1]; - if (opcode != RESUME || oparg < 2) { + _Py_CODEUNIT next = _PyCode_CODE(gen->gi_code)[frame->f_lasti + 1]; + if (_PyOpcode_Deopt[_Py_OPCODE(next)] != RESUME || _Py_OPARG(next) < 2) + { /* Not in a yield from */ return NULL; } @@ -485,14 +483,15 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, ret = _PyFrame_StackPop((_PyInterpreterFrame *)gen->gi_iframe); assert(ret == yf); Py_DECREF(ret); + // XXX: Performing this jump ourselves is awkward and problematic. + // See https://github.com/python/cpython/pull/31968. /* Termination repetition of SEND loop */ assert(frame->f_lasti >= 0); - PyObject *bytecode = gen->gi_code->co_code; - unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode); + _Py_CODEUNIT *code = _PyCode_CODE(gen->gi_code); /* Backup to SEND */ frame->f_lasti--; - assert(code[frame->f_lasti*sizeof(_Py_CODEUNIT)] == SEND); - int jump = code[frame->f_lasti*sizeof(_Py_CODEUNIT)+1]; + assert(_Py_OPCODE(code[frame->f_lasti]) == SEND); + int jump = _Py_OPARG(code[frame->f_lasti]); frame->f_lasti += jump; if (_PyGen_FetchStopIterationValue(&val) == 0) { ret = gen_send(gen, val); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 78795150756..4bed3ef4928 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8949,7 +8949,10 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co, // "firstarg" is a cell here unless (very unlikely) super() // was called from the C-API before the first MAKE_CELL op. if (cframe->f_lasti >= 0) { - assert(_Py_OPCODE(*co->co_firstinstr) == MAKE_CELL || _Py_OPCODE(*co->co_firstinstr) == COPY_FREE_VARS); + // MAKE_CELL and COPY_FREE_VARS have no quickened forms, so no need + // to use _PyOpcode_Deopt here: + assert(_Py_OPCODE(_PyCode_CODE(co)[0]) == MAKE_CELL || + _Py_OPCODE(_PyCode_CODE(co)[0]) == COPY_FREE_VARS); assert(PyCell_Check(firstarg)); firstarg = PyCell_GET(firstarg); } diff --git a/Python/ceval.c b/Python/ceval.c index 6f449e3172d..7c6bfd46900 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1327,9 +1327,8 @@ eval_frame_handle_pending(PyThreadState *tstate) /* Get opcode and oparg from original instructions, not quickened form. */ #define TRACING_NEXTOPARG() do { \ - _Py_CODEUNIT word = ((_Py_CODEUNIT *)PyBytes_AS_STRING(frame->f_code->co_code))[INSTR_OFFSET()]; \ - opcode = _Py_OPCODE(word); \ - oparg = _Py_OPARG(word); \ + NEXTOPARG(); \ + opcode = _PyOpcode_Deopt[opcode]; \ } while (0) /* OpCode prediction macros @@ -1650,9 +1649,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyCodeObject *co = frame->f_code; \ names = co->co_names; \ consts = co->co_consts; \ - first_instr = co->co_firstinstr; \ + first_instr = _PyCode_CODE(co); \ } \ assert(frame->f_lasti >= -1); \ + /* Jump back to the last instruction executed... */ \ next_instr = first_instr + frame->f_lasti + 1; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ /* Set stackdepth to -1. \ @@ -1722,16 +1722,7 @@ handle_eval_breaker: } TARGET(RESUME) { - int err = _Py_IncrementCountAndMaybeQuicken(frame->f_code); - if (err) { - if (err < 0) { - goto error; - } - /* Update first_instr and next_instr to point to newly quickened code */ - int nexti = INSTR_OFFSET(); - first_instr = frame->f_code->co_firstinstr; - next_instr = first_instr + nexti; - } + _PyCode_Warmup(frame->f_code); JUMP_TO_INSTRUCTION(RESUME_QUICK); } @@ -4067,16 +4058,7 @@ handle_eval_breaker: TARGET(JUMP_ABSOLUTE) { PREDICTED(JUMP_ABSOLUTE); - int err = _Py_IncrementCountAndMaybeQuicken(frame->f_code); - if (err) { - if (err < 0) { - goto error; - } - /* Update first_instr and next_instr to point to newly quickened code */ - int nexti = INSTR_OFFSET(); - first_instr = frame->f_code->co_firstinstr; - next_instr = first_instr + nexti; - } + _PyCode_Warmup(frame->f_code); JUMP_TO_INSTRUCTION(JUMP_ABSOLUTE_QUICK); } @@ -5425,6 +5407,7 @@ handle_eval_breaker: } TARGET(EXTENDED_ARG) { + assert(oparg); int oldoparg = oparg; NEXTOPARG(); oparg |= oldoparg << 8; @@ -6739,8 +6722,8 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj, */ initialize_trace_info(&tstate->trace_info, frame); int entry_point = 0; - _Py_CODEUNIT *code = (_Py_CODEUNIT *)PyBytes_AS_STRING(frame->f_code->co_code); - while (_Py_OPCODE(code[entry_point]) != RESUME) { + _Py_CODEUNIT *code = _PyCode_CODE(frame->f_code); + while (_PyOpcode_Deopt[_Py_OPCODE(code[entry_point])] != RESUME) { entry_point++; } int lastline; @@ -6759,7 +6742,9 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj, /* Trace backward edges (except in 'yield from') or if line number has changed */ int trace = line != lastline || (frame->f_lasti < instr_prev && - _Py_OPCODE(frame->f_code->co_firstinstr[frame->f_lasti]) != SEND); + // SEND has no quickened forms, so no need to use _PyOpcode_Deopt + // here: + _Py_OPCODE(_PyCode_CODE(frame->f_code)[frame->f_lasti]) != SEND); if (trace) { result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None); } diff --git a/Python/compile.c b/Python/compile.c index 950c44a749f..e24f425229b 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -134,9 +134,9 @@ static int instr_size(struct instr *instruction) { int opcode = instruction->i_opcode; - int oparg = instruction->i_oparg; + int oparg = HAS_ARG(opcode) ? instruction->i_oparg : 0; int extended_args = (0xFFFFFF < oparg) + (0xFFFF < oparg) + (0xFF < oparg); - int caches = _PyOpcode_InlineCacheEntries[opcode]; + int caches = _PyOpcode_Caches[opcode]; return extended_args + 1 + caches; } @@ -144,8 +144,8 @@ static void write_instr(_Py_CODEUNIT *codestr, struct instr *instruction, int ilen) { int opcode = instruction->i_opcode; - int oparg = instruction->i_oparg; - int caches = _PyOpcode_InlineCacheEntries[opcode]; + int oparg = HAS_ARG(opcode) ? instruction->i_oparg : 0; + int caches = _PyOpcode_Caches[opcode]; switch (ilen - caches) { case 4: *codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG, (oparg >> 24) & 0xFF); diff --git a/Python/marshal.c b/Python/marshal.c index 810244ba8ac..e7cf6553bd1 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -544,13 +544,18 @@ w_complex_object(PyObject *v, char flag, WFILE *p) } else if (PyCode_Check(v)) { PyCodeObject *co = (PyCodeObject *)v; + PyObject *co_code = _PyCode_GetCode(co); + if (co_code == NULL) { + p->error = WFERR_NOMEMORY; + return; + } W_TYPE(TYPE_CODE, p); w_long(co->co_argcount, p); w_long(co->co_posonlyargcount, p); w_long(co->co_kwonlyargcount, p); w_long(co->co_stacksize, p); w_long(co->co_flags, p); - w_object(co->co_code, p); + w_object(co_code, p); w_object(co->co_consts, p); w_object(co->co_names, p); w_object(co->co_localsplusnames, p); @@ -563,6 +568,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(co->co_endlinetable, p); w_object(co->co_columntable, p); w_object(co->co_exceptiontable, p); + Py_DECREF(co_code); } else if (PyObject_CheckBuffer(v)) { /* Write unknown bytes-like objects as a bytes object */ diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 7c94999bfdf..2aa6471abf9 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -3,49 +3,49 @@ static void *opcode_targets[256] = { &&TARGET_POP_TOP, &&TARGET_PUSH_NULL, &&TARGET_BINARY_OP_ADAPTIVE, - &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_FLOAT, + &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, - &&TARGET_BINARY_OP_MULTIPLY_INT, + &&TARGET_BINARY_OP_MULTIPLY_FLOAT, &&TARGET_NOP, &&TARGET_UNARY_POSITIVE, &&TARGET_UNARY_NEGATIVE, &&TARGET_UNARY_NOT, - &&TARGET_BINARY_OP_MULTIPLY_FLOAT, - &&TARGET_BINARY_OP_SUBTRACT_INT, - &&TARGET_UNARY_INVERT, + &&TARGET_BINARY_OP_MULTIPLY_INT, &&TARGET_BINARY_OP_SUBTRACT_FLOAT, + &&TARGET_UNARY_INVERT, + &&TARGET_BINARY_OP_SUBTRACT_INT, + &&TARGET_BINARY_SUBSCR_ADAPTIVE, + &&TARGET_BINARY_SUBSCR_DICT, + &&TARGET_BINARY_SUBSCR_GETITEM, + &&TARGET_BINARY_SUBSCR_LIST_INT, + &&TARGET_BINARY_SUBSCR_TUPLE_INT, + &&TARGET_CALL_ADAPTIVE, + &&TARGET_CALL_PY_EXACT_ARGS, + &&TARGET_CALL_PY_WITH_DEFAULTS, + &&TARGET_BINARY_SUBSCR, &&TARGET_COMPARE_OP_ADAPTIVE, &&TARGET_COMPARE_OP_FLOAT_JUMP, &&TARGET_COMPARE_OP_INT_JUMP, &&TARGET_COMPARE_OP_STR_JUMP, - &&TARGET_BINARY_SUBSCR_ADAPTIVE, - &&TARGET_BINARY_SUBSCR_GETITEM, - &&TARGET_BINARY_SUBSCR_LIST_INT, - &&TARGET_BINARY_SUBSCR_TUPLE_INT, - &&TARGET_BINARY_SUBSCR, - &&TARGET_BINARY_SUBSCR_DICT, - &&TARGET_STORE_SUBSCR_ADAPTIVE, - &&TARGET_STORE_SUBSCR_LIST_INT, - &&TARGET_STORE_SUBSCR_DICT, &&TARGET_GET_LEN, &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_MATCH_KEYS, - &&TARGET_CALL_ADAPTIVE, - &&TARGET_PUSH_EXC_INFO, - &&TARGET_CALL_PY_EXACT_ARGS, - &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_JUMP_ABSOLUTE_QUICK, + &&TARGET_PUSH_EXC_INFO, &&TARGET_LOAD_ATTR_ADAPTIVE, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, - &&TARGET_LOAD_ATTR_WITH_HINT, - &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_LOAD_ATTR_SLOT, + &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_LOAD_GLOBAL_ADAPTIVE, - &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_LOAD_METHOD_ADAPTIVE, &&TARGET_LOAD_METHOD_CLASS, &&TARGET_WITH_EXCEPT_START, @@ -61,26 +61,26 @@ static void *opcode_targets[256] = { &&TARGET_PRECALL_ADAPTIVE, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, + &&TARGET_PRECALL_BOUND_METHOD, &&TARGET_PRECALL_BUILTIN_CLASS, - &&TARGET_PRECALL_NO_KW_BUILTIN_O, - &&TARGET_PRECALL_NO_KW_BUILTIN_FAST, &&TARGET_PRECALL_BUILTIN_FAST_WITH_KEYWORDS, - &&TARGET_PRECALL_NO_KW_LEN, + &&TARGET_PRECALL_NO_KW_BUILTIN_FAST, + &&TARGET_PRECALL_NO_KW_BUILTIN_O, &&TARGET_PRECALL_NO_KW_ISINSTANCE, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_PRECALL_NO_KW_LEN, &&TARGET_PRECALL_NO_KW_LIST_APPEND, - &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_O, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST, &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, + &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_O, &&TARGET_PRECALL_NO_KW_STR_1, &&TARGET_PRECALL_NO_KW_TUPLE_1, &&TARGET_PRECALL_NO_KW_TYPE_1, - &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST, - &&TARGET_PRECALL_BOUND_METHOD, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, @@ -157,23 +157,23 @@ static void *opcode_targets[256] = { &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&TARGET_UNPACK_SEQUENCE_ADAPTIVE, - &&TARGET_UNPACK_SEQUENCE_LIST, + &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_LOAD_METHOD, - &&TARGET_UNPACK_SEQUENCE_TUPLE, + &&TARGET_STORE_SUBSCR_ADAPTIVE, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, &&TARGET_PRECALL, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, - &&TARGET_LOAD_FAST__LOAD_FAST, - &&TARGET_STORE_FAST__LOAD_FAST, - &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_STORE_SUBSCR_DICT, + &&TARGET_STORE_SUBSCR_LIST_INT, + &&TARGET_UNPACK_SEQUENCE_ADAPTIVE, + &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_CALL, &&TARGET_KW_NAMES, - &&TARGET_LOAD_CONST__LOAD_FAST, - &&TARGET_STORE_FAST__STORE_FAST, + &&TARGET_UNPACK_SEQUENCE_TUPLE, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index d84adac3520..ce091a2c5f0 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -15,31 +15,6 @@ * ./adaptive.md */ - -/* We layout the quickened data as a bi-directional array: - * Instructions upwards, cache entries downwards. - * first_instr is aligned to a SpecializedCacheEntry. - * The nth instruction is located at first_instr[n] - * The nth cache is located at ((SpecializedCacheEntry *)first_instr)[-1-n] - * The first (index 0) cache entry is reserved for the count, to enable finding - * the first instruction from the base pointer. - * The cache_count argument must include space for the count. - * We use the SpecializedCacheOrInstruction union to refer to the data - * to avoid type punning. - - Layout of quickened data, each line 8 bytes for M cache entries and N instructions: - - <---- co->co_quickened - - - ... - - <--- co->co_first_instr - - ... - -*/ - /* Map from opcode to adaptive opcode. Values of zero are ignored. */ static uint8_t adaptive_opcodes[256] = { @@ -275,26 +250,14 @@ _Py_PrintSpecializationStats(int to_file) #define SPECIALIZATION_FAIL(opcode, kind) ((void)0) #endif -static _Py_CODEUNIT * -allocate(int instruction_count) +// Insert adaptive instructions and superinstructions. This cannot fail. +void +_PyCode_Quicken(PyCodeObject *code) { - assert(instruction_count > 0); - void *array = PyMem_Malloc(sizeof(_Py_CODEUNIT) * instruction_count); - if (array == NULL) { - PyErr_NoMemory(); - return NULL; - } _Py_QuickenedCount++; - return (_Py_CODEUNIT *)array; -} - - -// Insert adaptive instructions and superinstructions. -static void -optimize(_Py_CODEUNIT *instructions, int len) -{ int previous_opcode = -1; - for(int i = 0; i < len; i++) { + _Py_CODEUNIT *instructions = _PyCode_CODE(code); + for (int i = 0; i < Py_SIZE(code); i++) { int opcode = _Py_OPCODE(instructions[i]); uint8_t adaptive_opcode = adaptive_opcodes[opcode]; if (adaptive_opcode) { @@ -302,10 +265,10 @@ optimize(_Py_CODEUNIT *instructions, int len) // Make sure the adaptive counter is zero: assert(instructions[i + 1] == 0); previous_opcode = -1; - i += _PyOpcode_InlineCacheEntries[opcode]; + i += _PyOpcode_Caches[opcode]; } else { - assert(!_PyOpcode_InlineCacheEntries[opcode]); + assert(!_PyOpcode_Caches[opcode]); switch (opcode) { case JUMP_ABSOLUTE: _Py_SET_OPCODE(instructions[i], JUMP_ABSOLUTE_QUICK); @@ -347,28 +310,6 @@ optimize(_Py_CODEUNIT *instructions, int len) } } -int -_Py_Quicken(PyCodeObject *code) { - if (code->co_quickened) { - return 0; - } - Py_ssize_t size = PyBytes_GET_SIZE(code->co_code); - int instr_count = (int)(size/sizeof(_Py_CODEUNIT)); - if (instr_count > MAX_SIZE_TO_QUICKEN) { - code->co_warmup = QUICKENING_WARMUP_COLDEST; - return 0; - } - _Py_CODEUNIT *quickened = allocate(instr_count); - if (quickened == NULL) { - return -1; - } - memcpy(quickened, code->co_firstinstr, size); - optimize(quickened, instr_count); - code->co_quickened = quickened; - code->co_firstinstr = quickened; - return 0; -} - static inline int initial_counter_value(void) { /* Starting value for the counter. @@ -705,8 +646,7 @@ specialize_dict_access( int _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) { - assert(_PyOpcode_InlineCacheEntries[LOAD_ATTR] == - INLINE_CACHE_ENTRIES_LOAD_ATTR); + assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR); _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); if (PyModule_CheckExact(owner)) { int err = specialize_module_load_attr(owner, instr, name, LOAD_ATTR, @@ -804,8 +744,7 @@ success: int _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) { - assert(_PyOpcode_InlineCacheEntries[STORE_ATTR] == - INLINE_CACHE_ENTRIES_STORE_ATTR); + assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR); _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyTypeObject *type = Py_TYPE(owner); if (PyModule_CheckExact(owner)) { @@ -965,8 +904,7 @@ typedef enum { int _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) { - assert(_PyOpcode_InlineCacheEntries[LOAD_METHOD] == - INLINE_CACHE_ENTRIES_LOAD_METHOD); + assert(_PyOpcode_Caches[LOAD_METHOD] == INLINE_CACHE_ENTRIES_LOAD_METHOD); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1); PyTypeObject *owner_cls = Py_TYPE(owner); @@ -1098,8 +1036,7 @@ _Py_Specialize_LoadGlobal( PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name) { - assert(_PyOpcode_InlineCacheEntries[LOAD_GLOBAL] == - INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + assert(_PyOpcode_Caches[LOAD_GLOBAL] == INLINE_CACHE_ENTRIES_LOAD_GLOBAL); /* Use inline cache */ _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)(instr + 1); assert(PyUnicode_CheckExact(name)); @@ -1235,7 +1172,7 @@ int _Py_Specialize_BinarySubscr( PyObject *container, PyObject *sub, _Py_CODEUNIT *instr) { - assert(_PyOpcode_InlineCacheEntries[BINARY_SUBSCR] == + assert(_PyOpcode_Caches[BINARY_SUBSCR] == INLINE_CACHE_ENTRIES_BINARY_SUBSCR); _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)(instr + 1); PyTypeObject *container_type = Py_TYPE(container); @@ -1663,8 +1600,7 @@ int _Py_Specialize_Precall(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames, int oparg) { - assert(_PyOpcode_InlineCacheEntries[PRECALL] == - INLINE_CACHE_ENTRIES_PRECALL); + assert(_PyOpcode_Caches[PRECALL] == INLINE_CACHE_ENTRIES_PRECALL); _PyPrecallCache *cache = (_PyPrecallCache *)(instr + 1); int fail; if (PyCFunction_CheckExact(callable)) { @@ -1710,7 +1646,7 @@ int _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames) { - assert(_PyOpcode_InlineCacheEntries[CALL] == INLINE_CACHE_ENTRIES_CALL); + assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL); _PyCallCache *cache = (_PyCallCache *)(instr + 1); int fail; if (PyFunction_Check(callable)) { @@ -1808,8 +1744,7 @@ void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, int oparg) { - assert(_PyOpcode_InlineCacheEntries[BINARY_OP] == - INLINE_CACHE_ENTRIES_BINARY_OP); + assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP); _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1); switch (oparg) { case NB_ADD: @@ -1936,8 +1871,7 @@ void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, int oparg) { - assert(_PyOpcode_InlineCacheEntries[COMPARE_OP] == - INLINE_CACHE_ENTRIES_COMPARE_OP); + assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP); _PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1); int next_opcode = _Py_OPCODE(instr[INLINE_CACHE_ENTRIES_COMPARE_OP + 1]); if (next_opcode != POP_JUMP_IF_FALSE && next_opcode != POP_JUMP_IF_TRUE) { @@ -2019,7 +1953,7 @@ unpack_sequence_fail_kind(PyObject *seq) void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) { - assert(_PyOpcode_InlineCacheEntries[UNPACK_SEQUENCE] == + assert(_PyOpcode_Caches[UNPACK_SEQUENCE] == INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)(instr + 1); if (PyTuple_CheckExact(seq)) { diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 954fca81b51..d208258dbc5 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -229,12 +229,8 @@ class Printer: def generate_code(self, name: str, code: types.CodeType) -> str: # The ordering here matches PyCode_NewWithPosOnlyArgs() # (but see below). - co_code = self.generate(name + "_code", code.co_code) co_consts = self.generate(name + "_consts", code.co_consts) co_names = self.generate(name + "_names", code.co_names) - co_varnames = self.generate(name + "_varnames", code.co_varnames) - co_freevars = self.generate(name + "_freevars", code.co_freevars) - co_cellvars = self.generate(name + "_cellvars", code.co_cellvars) co_filename = self.generate(name + "_filename", code.co_filename) co_name = self.generate(name + "_name", code.co_name) co_qualname = self.generate(name + "_qualname", code.co_qualname) @@ -249,14 +245,17 @@ class Printer: # Derived values nlocals, nplaincellvars, ncellvars, nfreevars = \ get_localsplus_counts(code, localsplusnames, localspluskinds) - with self.block(f"static struct PyCodeObject {name} =", ";"): - self.object_head("PyCode_Type") + co_code_adaptive = make_string_literal(code.co_code) + self.write("static") + with self.indent(): + self.write(f"struct _PyCode_DEF({len(code.co_code)})") + with self.block(f"{name} =", ";"): + self.object_var_head("PyCode_Type", len(code.co_code) // 2) # But the ordering here must match that in cpython/code.h # (which is a pain because we tend to reorder those for perf) # otherwise MSVC doesn't like it. self.write(f".co_consts = {co_consts},") self.write(f".co_names = {co_names},") - self.write(f".co_firstinstr = (_Py_CODEUNIT *) {removesuffix(co_code, '.ob_base.ob_base')}.ob_sval,") self.write(f".co_exceptiontable = {co_exceptiontable},") self.field(code, "co_flags") self.write(".co_warmup = QUICKENING_INITIAL_WARMUP_VALUE,") @@ -265,7 +264,11 @@ class Printer: self.field(code, "co_kwonlyargcount") self.field(code, "co_stacksize") self.field(code, "co_firstlineno") - self.write(f".co_code = {co_code},") + self.write(f".co_nlocalsplus = {len(localsplusnames)},") + self.field(code, "co_nlocals") + self.write(f".co_nplaincellvars = {nplaincellvars},") + self.write(f".co_ncellvars = {ncellvars},") + self.write(f".co_nfreevars = {nfreevars},") self.write(f".co_localsplusnames = {co_localsplusnames},") self.write(f".co_localspluskinds = {co_localspluskinds},") self.write(f".co_filename = {co_filename},") @@ -274,17 +277,11 @@ class Printer: self.write(f".co_linetable = {co_linetable},") self.write(f".co_endlinetable = {co_endlinetable},") self.write(f".co_columntable = {co_columntable},") - self.write(f".co_nlocalsplus = {len(localsplusnames)},") - self.field(code, "co_nlocals") - self.write(f".co_nplaincellvars = {nplaincellvars},") - self.write(f".co_ncellvars = {ncellvars},") - self.write(f".co_nfreevars = {nfreevars},") - self.write(f".co_varnames = {co_varnames},") - self.write(f".co_cellvars = {co_cellvars},") - self.write(f".co_freevars = {co_freevars},") - self.deallocs.append(f"_PyStaticCode_Dealloc(&{name});") - self.interns.append(f"_PyStaticCode_InternStrings(&{name})") - return f"& {name}.ob_base" + self.write(f".co_code_adaptive = {co_code_adaptive},") + name_as_code = f"(PyCodeObject *)&{name}" + self.deallocs.append(f"_PyStaticCode_Dealloc({name_as_code});") + self.interns.append(f"_PyStaticCode_InternStrings({name_as_code})") + return f"& {name}.ob_base.ob_base" def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: if len(t) == 0: @@ -450,13 +447,13 @@ def generate(args: list[str], output: TextIO) -> None: code = compile(fd.read(), f"", "exec") printer.generate_file(modname, code) with printer.block(f"void\n_Py_Deepfreeze_Fini(void)"): - for p in printer.deallocs: - printer.write(p) + for p in printer.deallocs: + printer.write(p) with printer.block(f"int\n_Py_Deepfreeze_Init(void)"): - for p in printer.interns: - with printer.block(f"if ({p} < 0)"): - printer.write("return -1;") - printer.write("return 0;") + for p in printer.interns: + with printer.block(f"if ({p} < 0)"): + printer.write("return -1;") + printer.write("return 0;") if verbose: print(f"Cache hits: {printer.hits}, misses: {printer.misses}") diff --git a/Tools/scripts/generate_opcode_h.py b/Tools/scripts/generate_opcode_h.py index 75a9c3f3bfa..3b79dc6b735 100644 --- a/Tools/scripts/generate_opcode_h.py +++ b/Tools/scripts/generate_opcode_h.py @@ -28,7 +28,7 @@ footer = """ #endif /* !Py_OPCODE_H */ """ -DEFINE = "#define {:<31} {:>3}\n" +DEFINE = "#define {:<38} {:>3}\n" UINT32_MASK = (1<<32)-1 @@ -75,16 +75,27 @@ def main(opcode_py, outfile='Include/opcode.h'): fobj.write(DEFINE.format(name, next_op)) used[next_op] = True fobj.write(DEFINE.format('DO_TRACING', 255)) - fobj.write("\nextern const uint8_t _PyOpcode_InlineCacheEntries[256];\n") + fobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n") + fobj.write("\nextern const uint8_t _PyOpcode_Deopt[256];\n") fobj.write("\n#ifdef NEED_OPCODE_TABLES\n") write_int_array_from_ops("_PyOpcode_RelativeJump", opcode['hasjrel'], fobj) write_int_array_from_ops("_PyOpcode_Jump", opcode['hasjrel'] + opcode['hasjabs'], fobj) - fobj.write("\nconst uint8_t _PyOpcode_InlineCacheEntries[256] = {\n") + fobj.write("\nconst uint8_t _PyOpcode_Caches[256] = {\n") for i, entries in enumerate(opcode["_inline_cache_entries"]): if entries: fobj.write(f" [{opname[i]}] = {entries},\n") fobj.write("};\n") + deoptcodes = {} + for basic in opmap: + deoptcodes[basic] = basic + for basic, family in opcode["_specializations"].items(): + for specialized in family: + deoptcodes[specialized] = basic + fobj.write("\nconst uint8_t _PyOpcode_Deopt[256] = {\n") + for opt, deopt in sorted(deoptcodes.items()): + fobj.write(f" [{opt}] = {deopt},\n") + fobj.write("};\n") fobj.write("#endif /* OPCODE_TABLES */\n") fobj.write("\n")