From 64857d849f3079a73367525ce93fd7a463b83908 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 26 Jul 2024 09:40:15 -0700 Subject: [PATCH] GH-122294: Burn in the addresses of side exits (GH-122295) --- Include/internal/pycore_uop_metadata.h | 4 ++-- Python/bytecodes.c | 12 ++++++------ Python/executor_cases.c.h | 12 ++++++------ Python/optimizer.c | 10 ++++++---- Python/optimizer_bytecodes.c | 3 ++- Python/optimizer_cases.c.h | 2 ++ 6 files changed, 24 insertions(+), 19 deletions(-) diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index fd0d4a67d93..d23a4e2ea14 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -252,7 +252,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SET_IP] = 0, [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, - [_EXIT_TRACE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_EXIT_TRACE] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, @@ -261,7 +261,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG, [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, - [_DYNAMIC_EXIT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, [_START_EXECUTOR] = 0, [_FATAL_ERROR] = 0, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 871e2dbf358..d74f2aae048 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4609,8 +4609,8 @@ dummy_func( #endif } - tier2 op(_EXIT_TRACE, (--)) { - _PyExitData *exit = ¤t_executor->exits[oparg]; + tier2 op(_EXIT_TRACE, (exit_p/4 --)) { + _PyExitData *exit = (_PyExitData *)exit_p; PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; #if defined(Py_DEBUG) && !defined(_Py_JIT) @@ -4619,7 +4619,7 @@ dummy_func( printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", - oparg, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.as_counter, (int)(target - _PyCode_CODE(code)), _PyOpcode_OpName[target->op.code]); } @@ -4698,9 +4698,9 @@ dummy_func( exe->count++; } - tier2 op(_DYNAMIC_EXIT, (--)) { + tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { tstate->previous_executor = (PyObject *)current_executor; - _PyExitData *exit = (_PyExitData *)¤t_executor->exits[oparg]; + _PyExitData *exit = (_PyExitData *)exit_p; _Py_CODEUNIT *target = frame->instr_ptr; #if defined(Py_DEBUG) && !defined(_Py_JIT) OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); @@ -4708,7 +4708,7 @@ dummy_func( printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", - oparg, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.as_counter, (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))), _PyOpcode_OpName[target->op.code]); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 1ced8b951b5..6e3f6cc62fe 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -5044,8 +5044,8 @@ } case _EXIT_TRACE: { - oparg = CURRENT_OPARG(); - _PyExitData *exit = ¤t_executor->exits[oparg]; + PyObject *exit_p = (PyObject *)CURRENT_OPERAND(); + _PyExitData *exit = (_PyExitData *)exit_p; PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; #if defined(Py_DEBUG) && !defined(_Py_JIT) @@ -5054,7 +5054,7 @@ printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", - oparg, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.as_counter, (int)(target - _PyCode_CODE(code)), _PyOpcode_OpName[target->op.code]); } @@ -5182,9 +5182,9 @@ } case _DYNAMIC_EXIT: { - oparg = CURRENT_OPARG(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND(); tstate->previous_executor = (PyObject *)current_executor; - _PyExitData *exit = (_PyExitData *)¤t_executor->exits[oparg]; + _PyExitData *exit = (_PyExitData *)exit_p; _Py_CODEUNIT *target = frame->instr_ptr; #if defined(Py_DEBUG) && !defined(_Py_JIT) OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); @@ -5192,7 +5192,7 @@ printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", - oparg, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.as_counter, (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))), _PyOpcode_OpName[target->op.code]); } diff --git a/Python/optimizer.c b/Python/optimizer.c index f0793b8c8f2..7b875af2aae 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1153,13 +1153,15 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil *dest = buffer[i]; assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE); if (opcode == _EXIT_TRACE) { - executor->exits[next_exit].target = buffer[i].target; - dest->oparg = next_exit; + _PyExitData *exit = &executor->exits[next_exit]; + exit->target = buffer[i].target; + dest->operand = (uint64_t)exit; next_exit--; } if (opcode == _DYNAMIC_EXIT) { - executor->exits[next_exit].target = 0; - dest->oparg = next_exit; + _PyExitData *exit = &executor->exits[next_exit]; + exit->target = 0; + dest->operand = (uint64_t)exit; next_exit--; } } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index a506f9948fd..4d4f89301c7 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -788,7 +788,8 @@ dummy_func(void) { ctx->done = true; } - op(_EXIT_TRACE, (--)) { + op(_EXIT_TRACE, (exit_p/4 --)) { + (void)exit_p; ctx->done = true; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 166b1674bc3..fae93ce89e8 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2163,6 +2163,8 @@ } case _EXIT_TRACE: { + PyObject *exit_p = (PyObject *)this_instr->operand; + (void)exit_p; ctx->done = true; break; }