#include "Python.h" #include "pycore_backoff.h" #include "pycore_call.h" #include "pycore_ceval.h" #include "pycore_cell.h" #include "pycore_dict.h" #include "pycore_emscripten_signal.h" #include "pycore_intrinsics.h" #include "pycore_jit.h" #include "pycore_long.h" #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_optimizer.h" #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_range.h" #include "pycore_setobject.h" #include "pycore_sliceobject.h" #include "pycore_descrobject.h" #include "pycore_stackref.h" #include "ceval_macros.h" #undef CURRENT_OPARG #define CURRENT_OPARG() (_oparg) #undef CURRENT_OPERAND #define CURRENT_OPERAND() (_operand) #undef DEOPT_IF #define DEOPT_IF(COND, INSTNAME) \ do { \ if ((COND)) { \ goto deoptimize; \ } \ } while (0) #undef ENABLE_SPECIALIZATION #define ENABLE_SPECIALIZATION (0) #undef GOTO_ERROR #define GOTO_ERROR(LABEL) \ do { \ goto LABEL ## _tier_two; \ } while (0) #undef GOTO_TIER_TWO #define GOTO_TIER_TWO(EXECUTOR) \ do { \ OPT_STAT_INC(traces_executed); \ __attribute__((musttail)) \ return ((jit_func)((EXECUTOR)->jit_side_entry))(frame, stack_pointer, tstate); \ } while (0) #undef GOTO_TIER_ONE #define GOTO_TIER_ONE(TARGET) \ do { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ return TARGET; \ } while (0) #undef LOAD_IP #define LOAD_IP(UNUSED) \ do { \ } while (0) #define PATCH_VALUE(TYPE, NAME, ALIAS) \ PyAPI_DATA(void) ALIAS; \ TYPE NAME = (TYPE)(uintptr_t)&ALIAS; #define PATCH_JUMP(ALIAS) \ do { \ PyAPI_DATA(void) ALIAS; \ __attribute__((musttail)) \ return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); \ } while (0) #undef JUMP_TO_JUMP_TARGET #define JUMP_TO_JUMP_TARGET() PATCH_JUMP(_JIT_JUMP_TARGET) #undef JUMP_TO_ERROR #define JUMP_TO_ERROR() PATCH_JUMP(_JIT_ERROR_TARGET) #undef WITHIN_STACK_BOUNDS #define WITHIN_STACK_BOUNDS() 1 #define TIER_TWO 2 _Py_CODEUNIT * _JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) { // Locals that the instruction implementations expect to exist: PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR) int oparg; int uopcode = _JIT_OPCODE; _Py_CODEUNIT *next_instr; // Other stuff we need handy: PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) #if SIZEOF_VOID_P == 8 PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND) #else assert(SIZEOF_VOID_P == 4); PATCH_VALUE(uint32_t, _operand_hi, _JIT_OPERAND_HI) PATCH_VALUE(uint32_t, _operand_lo, _JIT_OPERAND_LO) uint64_t _operand = ((uint64_t)_operand_hi << 32) | _operand_lo; #endif PATCH_VALUE(uint32_t, _target, _JIT_TARGET) OPT_STAT_INC(uops_executed); UOP_STAT_INC(uopcode, execution_count); switch (uopcode) { // The actual instruction definition gets inserted here: CASE default: Py_UNREACHABLE(); } PATCH_JUMP(_JIT_CONTINUE); // Labels that the instruction implementations expect to exist: error_tier_two: tstate->previous_executor = (PyObject *)current_executor; GOTO_TIER_ONE(NULL); exit_to_tier1: tstate->previous_executor = (PyObject *)current_executor; GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target); exit_to_tier1_dynamic: tstate->previous_executor = (PyObject *)current_executor; GOTO_TIER_ONE(frame->instr_ptr); }