diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 4ae15e71e8d..774c0f99379 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1285,11 +1285,11 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case _INIT_CALL_PY_EXACT_ARGS: return 1; case _PUSH_FRAME: - return 1; + return 0; case CALL_BOUND_METHOD_EXACT_ARGS: - return 1; + return 0; case CALL_PY_EXACT_ARGS: - return 1; + return 0; case CALL_PY_WITH_DEFAULTS: return 1; case CALL_TYPE_1: diff --git a/Makefile.pre.in b/Makefile.pre.in index b5edb4e6748..6ac68d59c8c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1587,7 +1587,6 @@ regen-cases: $(PYTHON_FOR_REGEN) \ $(srcdir)/Tools/cases_generator/generate_cases.py \ $(CASESFLAG) \ - -o $(srcdir)/Python/generated_cases.c.h.new \ -n $(srcdir)/Include/opcode_ids.h.new \ -t $(srcdir)/Python/opcode_targets.h.new \ -m $(srcdir)/Include/internal/pycore_opcode_metadata.h.new \ @@ -1595,6 +1594,8 @@ regen-cases: -p $(srcdir)/Lib/_opcode_metadata.py.new \ -a $(srcdir)/Python/abstract_interp_cases.c.h.new \ $(srcdir)/Python/bytecodes.c + $(PYTHON_FOR_REGEN) \ + $(srcdir)/Tools/cases_generator/tier1_generator.py -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new $(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index 0d7fbe8a39a..96ac0aabd1b 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -774,7 +774,8 @@ } case _PUSH_FRAME: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(0)), true); break; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2075c195df3..bcad8dcf0e7 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -800,11 +800,11 @@ dummy_func( // We also push it onto the stack on exit, but that's a // different frame, and it's accounted for by _PUSH_FRAME. op(_POP_FRAME, (retval --)) { - assert(EMPTY()); #if TIER_ONE assert(frame != &entry_frame); #endif STORE_SP(); + assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; @@ -1165,7 +1165,6 @@ dummy_func( } } - inst(STORE_NAME, (v -- )) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); @@ -3130,7 +3129,7 @@ dummy_func( // The 'unused' output effect represents the return value // (which will be pushed when the frame returns). // It is needed so CALL_PY_EXACT_ARGS matches its family. - op(_PUSH_FRAME, (new_frame: _PyInterpreterFrame* -- unused)) { + op(_PUSH_FRAME, (new_frame: _PyInterpreterFrame* -- unused if (0))) { // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 547be6f1323..974e3f28a41 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -661,11 +661,11 @@ PyObject *retval; retval = stack_pointer[-1]; STACK_SHRINK(1); - assert(EMPTY()); #if TIER_ONE assert(frame != &entry_frame); #endif STORE_SP(); + assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0ac99e759de..24243ecfb5b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1,6 +1,6 @@ -// This file is generated by Tools/cases_generator/generate_cases.py +// This file is generated by Tools/cases_generator/tier1_generator.py // from: -// Python/bytecodes.c +// ['./Python/bytecodes.c'] // Do not edit! #ifdef TIER_TWO @@ -8,6 +8,7 @@ #endif #define TIER_ONE 1 + TARGET(BEFORE_ASYNC_WITH) { frame->instr_ptr = next_instr; next_instr += 1; @@ -45,9 +46,9 @@ Py_DECREF(exit); if (true) goto pop_1_error; } - STACK_GROW(1); - stack_pointer[-2] = exit; - stack_pointer[-1] = res; + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -91,9 +92,9 @@ Py_DECREF(exit); if (true) goto pop_1_error; } - STACK_GROW(1); - stack_pointer[-2] = exit; - stack_pointer[-1] = res; + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -103,7 +104,6 @@ INSTRUCTION_STATS(BINARY_OP); PREDICTED(BINARY_OP); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *rhs; PyObject *lhs; PyObject *res; @@ -133,8 +133,8 @@ Py_DECREF(rhs); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -142,6 +142,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -156,12 +157,12 @@ { STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; + ((PyFloatObject *)left)->ob_fval + + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -169,6 +170,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -187,8 +189,8 @@ _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -196,6 +198,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -214,8 +217,8 @@ _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -223,6 +226,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; // _GUARD_BOTH_UNICODE @@ -258,7 +262,7 @@ assert(next_instr->op.code == STORE_FAST); SKIP_OVER(1); } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -266,6 +270,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -280,12 +285,12 @@ { STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; + ((PyFloatObject *)left)->ob_fval * + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -293,6 +298,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -311,8 +317,8 @@ _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -320,6 +326,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -334,12 +341,12 @@ { STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; + ((PyFloatObject *)left)->ob_fval - + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -347,6 +354,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -365,8 +373,8 @@ _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -393,8 +401,8 @@ } Py_DECREF(container); if (res == NULL) goto pop_3_error; - STACK_SHRINK(2); - stack_pointer[-1] = res; + stack_pointer[-3] = res; + stack_pointer += -2; DISPATCH(); } @@ -404,7 +412,6 @@ INSTRUCTION_STATS(BINARY_SUBSCR); PREDICTED(BINARY_SUBSCR); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; PyObject *res; @@ -431,8 +438,8 @@ Py_DECREF(sub); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -440,6 +447,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *dict; PyObject *res; @@ -454,8 +462,9 @@ Py_DECREF(dict); Py_DECREF(sub); if (rc <= 0) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + // not found or error + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -463,6 +472,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; sub = stack_pointer[-1]; @@ -494,6 +504,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *list; PyObject *res; @@ -501,7 +512,6 @@ list = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyList_Size(list) DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; @@ -512,8 +522,8 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -521,6 +531,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *str; PyObject *res; @@ -538,8 +549,8 @@ res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(str); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -547,6 +558,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *tuple; PyObject *res; @@ -554,7 +566,6 @@ tuple = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyTuple_Size(list) DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; @@ -565,8 +576,8 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -578,7 +589,7 @@ PyObject **values; PyObject *map; keys = stack_pointer[-1]; - values = stack_pointer - 1 - oparg; + values = &stack_pointer[-1 - oparg]; if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -586,15 +597,15 @@ GOTO_ERROR(error); // Pop the keys and values. } map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); + &PyTuple_GET_ITEM(keys, 0), 1, + values, 1, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); - if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - STACK_SHRINK(oparg); - stack_pointer[-1] = map; + if (map == NULL) { stack_pointer += -1 - oparg; goto error; } + stack_pointer[-1 - oparg] = map; + stack_pointer += -oparg; DISPATCH(); } @@ -604,12 +615,11 @@ INSTRUCTION_STATS(BUILD_LIST); PyObject **values; PyObject *list; - values = stack_pointer - oparg; + values = &stack_pointer[-oparg]; list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = list; + if (list == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; DISPATCH(); } @@ -619,18 +629,17 @@ INSTRUCTION_STATS(BUILD_MAP); PyObject **values; PyObject *map; - values = stack_pointer - oparg*2; + values = &stack_pointer[-oparg*2]; map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); + values, 2, + values+1, 2, + oparg); for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - STACK_SHRINK(oparg*2); - STACK_GROW(1); - stack_pointer[-1] = map; + if (map == NULL) { stack_pointer += -oparg*2; goto error; } + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; DISPATCH(); } @@ -640,24 +649,23 @@ INSTRUCTION_STATS(BUILD_SET); PyObject **values; PyObject *set; - values = stack_pointer - oparg; + values = &stack_pointer[-oparg]; set = PySet_New(NULL); if (set == NULL) - GOTO_ERROR(error); + GOTO_ERROR(error); int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; if (err == 0) - err = PySet_Add(set, item); + err = PySet_Add(set, item); Py_DECREF(item); } if (err != 0) { Py_DECREF(set); - if (true) { STACK_SHRINK(oparg); goto error; } + if (true) { stack_pointer += -oparg; goto error; } } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = set; + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; DISPATCH(); } @@ -669,17 +677,16 @@ PyObject *stop; PyObject *start; PyObject *slice; - if (oparg == 3) { step = stack_pointer[-(oparg == 3 ? 1 : 0)]; } - stop = stack_pointer[-1 - (oparg == 3 ? 1 : 0)]; - start = stack_pointer[-2 - (oparg == 3 ? 1 : 0)]; + if (oparg == 3) { step = stack_pointer[-(((oparg == 3) ? 1 : 0))]; } + stop = stack_pointer[-1 - (((oparg == 3) ? 1 : 0))]; + start = stack_pointer[-2 - (((oparg == 3) ? 1 : 0))]; slice = PySlice_New(start, stop, step); Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - STACK_SHRINK(((oparg == 3) ? 1 : 0)); - STACK_SHRINK(1); - stack_pointer[-1] = slice; + if (slice == NULL) { stack_pointer += -2 - (((oparg == 3) ? 1 : 0)); goto error; } + stack_pointer[-2 - (((oparg == 3) ? 1 : 0))] = slice; + stack_pointer += -1 - (((oparg == 3) ? 1 : 0)); DISPATCH(); } @@ -689,15 +696,14 @@ INSTRUCTION_STATS(BUILD_STRING); PyObject **pieces; PyObject *str; - pieces = stack_pointer - oparg; + pieces = &stack_pointer[-oparg]; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - if (str == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = str; + if (str == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; DISPATCH(); } @@ -707,12 +713,11 @@ INSTRUCTION_STATS(BUILD_TUPLE); PyObject **values; PyObject *tup; - values = stack_pointer - oparg; + values = &stack_pointer[-oparg]; tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = tup; + if (tup == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; DISPATCH(); } @@ -730,13 +735,12 @@ INSTRUCTION_STATS(CALL); PREDICTED(CALL); _Py_CODEUNIT *this_instr = next_instr - 4; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; // _SPECIALIZE_CALL - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { @@ -793,12 +797,12 @@ } /* Callable is not a normal Python function */ res = PyObject_Vectorcall( - callable, args, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); + callable, args, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; + &_PyInstrumentation_MISSING : args[0]; if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -818,11 +822,10 @@ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -831,10 +834,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *null; PyObject *callable; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* This instruction does the following: @@ -890,13 +894,15 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject *null; PyObject *callable; + PyObject *func; PyObject *self; PyObject *self_or_null; - PyObject *func; PyObject **args; _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ // _CHECK_PEP_523 { DEOPT_IF(tstate->interp->eval_frame, CALL); @@ -936,7 +942,8 @@ DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; { int argcount = oparg; if (self_or_null != NULL) { @@ -960,26 +967,26 @@ #endif } // _PUSH_FRAME - STACK_SHRINK(oparg); - STACK_SHRINK(2); { // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); - STORE_SP(); + stack_pointer += -2 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); - #if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } - #endif + #endif } + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } @@ -987,11 +994,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -1009,10 +1017,9 @@ Py_DECREF(args[i]); } Py_DECREF(tp); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1021,11 +1028,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ @@ -1044,21 +1052,19 @@ args, total_args); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + /* Not deopting because this doesn't mean our optimization was + wrong. `res` can be NULL for valid reasons. Eg. getattr(x, + 'invalid'). In those cases an exception is set, so we must + handle it. + */ + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1067,11 +1073,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ @@ -1085,20 +1092,18 @@ STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); + (_PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable); res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1107,11 +1112,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ @@ -1134,13 +1140,11 @@ res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1155,9 +1159,9 @@ PyObject *callargs; PyObject *func; PyObject *result; - if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1 ? 1 : 0)]; } - callargs = stack_pointer[-1 - (oparg & 1 ? 1 : 0)]; - func = stack_pointer[-3 - (oparg & 1 ? 1 : 0)]; + if (oparg & 1) { kwargs = stack_pointer[-((oparg & 1))]; } + callargs = stack_pointer[-1 - ((oparg & 1))]; + func = stack_pointer[-3 - ((oparg & 1))]; // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -1177,7 +1181,7 @@ !PyFunction_Check(func) && !PyMethod_Check(func) ) { PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : Py_None; + PyTuple_GET_ITEM(callargs, 0) : Py_None; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); @@ -1205,10 +1209,9 @@ Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)func, locals, - nargs, callargs, kwargs); + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { @@ -1224,10 +1227,9 @@ Py_DECREF(callargs); Py_XDECREF(kwargs); assert(PEEK(2 + (oparg & 1)) == NULL); - if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - STACK_SHRINK(((oparg & 1) ? 1 : 0)); - STACK_SHRINK(2); - stack_pointer[-1] = result; + if (result == NULL) { stack_pointer += -3 - ((oparg & 1)); goto error; } + stack_pointer[-3 - ((oparg & 1))] = result; + stack_pointer += -2 - ((oparg & 1)); CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1261,8 +1263,8 @@ Py_DECREF(value2); Py_DECREF(value1); if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -1270,11 +1272,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* isinstance(o, o2) */ @@ -1295,14 +1298,12 @@ } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(inst); Py_DECREF(cls); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; DISPATCH(); } @@ -1318,7 +1319,7 @@ PyObject *callable; PyObject *res; kwnames = stack_pointer[-1]; - args = stack_pointer - 1 - oparg; + args = &stack_pointer[-1 - oparg]; self_or_null = stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; // oparg counts all of the args, but *not* self: @@ -1363,12 +1364,12 @@ } /* Callable is not a normal Python function */ res = PyObject_Vectorcall( - callable, args, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); + callable, args, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; + &_PyInstrumentation_MISSING : args[0]; if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -1389,10 +1390,9 @@ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } - if (res == NULL) { STACK_SHRINK(oparg); goto pop_3_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(2); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -3 - oparg; goto error; } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1401,11 +1401,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LEN); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* len(o) */ @@ -1425,13 +1426,11 @@ } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); Py_DECREF(arg); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; DISPATCH(); } @@ -1439,10 +1438,11 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self; PyObject *callable; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); @@ -1467,11 +1467,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -1488,7 +1489,7 @@ DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + (_PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; res = cfunc(self, args + 1, nargs); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -1497,10 +1498,9 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1509,11 +1509,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -1531,19 +1532,17 @@ STAT_INC(CALL, hit); int nargs = total_args - 1; _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; res = cfunc(self, args + 1, nargs, NULL); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1552,11 +1551,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 0 || oparg == 1); @@ -1584,10 +1584,9 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1596,11 +1595,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -1629,10 +1629,9 @@ Py_DECREF(self); Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1641,10 +1640,12 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject *self_or_null; PyObject *callable; PyObject **args; _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ // _CHECK_PEP_523 { DEOPT_IF(tstate->interp->eval_frame, CALL); @@ -1668,7 +1669,8 @@ DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; { int argcount = oparg; if (self_or_null != NULL) { @@ -1692,26 +1694,26 @@ #endif } // _PUSH_FRAME - STACK_SHRINK(oparg); - STACK_SHRINK(2); { // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); - STORE_SP(); + stack_pointer += -2 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); - #if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } - #endif + #endif } + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } @@ -1719,10 +1721,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); @@ -1763,11 +1766,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); @@ -1778,10 +1782,9 @@ res = PyObject_Str(arg); Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1790,11 +1793,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); @@ -1805,10 +1809,9 @@ res = PySequence_Tuple(arg); Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1817,11 +1820,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); @@ -1832,9 +1836,8 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; DISPATCH(); } @@ -1853,18 +1856,15 @@ Py_DECREF(match_type); if (true) goto pop_2_error; } - match = NULL; rest = NULL; int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); + &match, &rest); Py_DECREF(exc_value); Py_DECREF(match_type); if (res < 0) goto pop_2_error; - assert((match == NULL) == (rest == NULL)); if (match == NULL) goto pop_2_error; - if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } @@ -1884,10 +1884,9 @@ left = stack_pointer[-2]; assert(PyExceptionInstance_Check(left)); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); - if (true) goto pop_1_error; + Py_DECREF(right); + if (true) goto pop_1_error; } - int res = PyErr_GivenExceptionMatches(left, right); Py_DECREF(right); b = res ? Py_True : Py_False; @@ -1922,9 +1921,9 @@ monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } - STACK_SHRINK(1); - stack_pointer[-2] = none; - stack_pointer[-1] = value; + stack_pointer[-3] = none; + stack_pointer[-2] = value; + stack_pointer += -1; DISPATCH(); } @@ -1934,7 +1933,6 @@ INSTRUCTION_STATS(COMPARE_OP); PREDICTED(COMPARE_OP); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -1968,8 +1966,8 @@ res = res_bool ? Py_True : Py_False; } } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -1977,6 +1975,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -1993,8 +1992,8 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -2002,6 +2001,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -2022,8 +2022,8 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -2031,6 +2031,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -2048,8 +2049,8 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -2067,8 +2068,8 @@ Py_DECREF(right); if (res < 0) goto pop_2_error; b = (res ^ oparg) ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; + stack_pointer[-2] = b; + stack_pointer += -1; DISPATCH(); } @@ -2098,8 +2099,8 @@ bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = Py_NewRef(bottom); - STACK_GROW(1); - stack_pointer[-1] = top; + stack_pointer[0] = top; + stack_pointer += 1; DISPATCH(); } @@ -2130,7 +2131,7 @@ int err = PyObject_DelAttr(owner, name); Py_DECREF(owner); if (err) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -2172,7 +2173,7 @@ if (err != 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + NAME_ERROR_MSG, name); } GOTO_ERROR(error); } @@ -2195,8 +2196,8 @@ // Can't use ERROR_IF here. if (err != 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); + NAME_ERROR_MSG, + name); GOTO_ERROR(error); } DISPATCH(); @@ -2215,7 +2216,7 @@ Py_DECREF(container); Py_DECREF(sub); if (err) goto pop_2_error; - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -2235,7 +2236,7 @@ if (true) goto pop_1_error; } Py_DECREF(update); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -2250,14 +2251,14 @@ if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); + "'%.200s' object is not a mapping", + Py_TYPE(update)->tp_name); } Py_DECREF(update); if (true) goto pop_1_error; } Py_DECREF(update); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -2281,7 +2282,7 @@ monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -2290,17 +2291,17 @@ next_instr += 1; INSTRUCTION_STATS(END_FOR); PyObject *value; - // POP_TOP + // _POP_TOP value = stack_pointer[-1]; { Py_DECREF(value); } - // POP_TOP + // _POP_TOP value = stack_pointer[-2]; { Py_DECREF(value); } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -2313,8 +2314,8 @@ value = stack_pointer[-1]; receiver = stack_pointer[-2]; Py_DECREF(receiver); - STACK_SHRINK(1); - stack_pointer[-1] = value; + stack_pointer[-2] = value; + stack_pointer += -1; DISPATCH(); } @@ -2324,7 +2325,6 @@ INSTRUCTION_STATS(ENTER_EXECUTOR); TIER_ONE_ONLY CHECK_EVAL_BREAKER(); - PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); @@ -2352,11 +2352,11 @@ assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); GOTO_ERROR(error); } - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -2405,8 +2405,8 @@ Py_DECREF(value); Py_DECREF(fmt_spec); if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -2416,7 +2416,6 @@ INSTRUCTION_STATS(FOR_ITER); PREDICTED(FOR_ITER); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; // _SPECIALIZE_FOR_ITER @@ -2448,7 +2447,7 @@ } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR instruction */ @@ -2457,8 +2456,8 @@ } // Common case: no jump, leave it to the code generator } - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } @@ -2466,6 +2465,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; iter = stack_pointer[-1]; DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); @@ -2489,8 +2489,10 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; + /* Skip 1 cache entry */ // _ITER_CHECK_LIST iter = stack_pointer[-1]; { @@ -2523,8 +2525,8 @@ assert(it->it_index < PyList_GET_SIZE(seq)); next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); } - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } @@ -2532,8 +2534,10 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; + /* Skip 1 cache entry */ // _ITER_CHECK_RANGE iter = stack_pointer[-1]; { @@ -2564,8 +2568,8 @@ next = PyLong_FromLong(value); if (next == NULL) goto error; } - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } @@ -2573,8 +2577,10 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; + /* Skip 1 cache entry */ // _ITER_CHECK_TUPLE iter = stack_pointer[-1]; { @@ -2607,8 +2613,8 @@ assert(it->it_index < PyTuple_GET_SIZE(seq)); next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); } - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } @@ -2621,11 +2627,9 @@ obj = stack_pointer[-1]; unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); - if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; } - if (getter == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " @@ -2634,14 +2638,11 @@ Py_DECREF(obj); if (true) goto pop_1_error; } - iter = (*getter)(obj); Py_DECREF(obj); if (iter == NULL) goto pop_1_error; - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - + Py_TYPE(iter)->tp_as_async->am_anext == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", @@ -2663,7 +2664,6 @@ unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); - if (PyAsyncGen_CheckExact(aiter)) { awaitable = type->tp_as_async->am_anext(aiter); if (awaitable == NULL) { @@ -2673,7 +2673,6 @@ if (type->tp_as_async != NULL){ getter = type->tp_as_async->am_anext; } - if (getter != NULL) { next_iter = (*getter)(aiter); if (next_iter == NULL) { @@ -2687,7 +2686,6 @@ type->tp_name); GOTO_ERROR(error); } - awaitable = _PyCoro_GetAwaitableIter(next_iter); if (awaitable == NULL) { _PyErr_FormatFromCause( @@ -2695,15 +2693,14 @@ "'async for' received an invalid object " "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); GOTO_ERROR(error); } else { Py_DECREF(next_iter); } } - STACK_GROW(1); - stack_pointer[-1] = awaitable; + stack_pointer[0] = awaitable; + stack_pointer += 1; DISPATCH(); } @@ -2715,13 +2712,10 @@ PyObject *iter; iterable = stack_pointer[-1]; iter = _PyCoro_GetAwaitableIter(iterable); - if (iter == NULL) { _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); } - Py_DECREF(iterable); - if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); if (yf != NULL) { @@ -2735,7 +2729,6 @@ /* The code below jumps to `error` if `iter` is NULL. */ } } - if (iter == NULL) goto pop_1_error; stack_pointer[-1] = iter; DISPATCH(); @@ -2768,8 +2761,8 @@ if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = len_o; + stack_pointer[0] = len_o; + stack_pointer += 1; DISPATCH(); } @@ -2819,8 +2812,8 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -2839,8 +2832,8 @@ Py_DECREF(level); Py_DECREF(fromlist); if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -2852,10 +2845,10 @@ int total_args = oparg + is_meth; PyObject *function = PEEK(oparg + 2); PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); + &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); if (err) goto error; INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); GO_TO_INSTRUCTION(CALL); @@ -2876,10 +2869,10 @@ int total_args = oparg + is_meth; PyObject *function = PEEK(oparg + 3); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING - : PEEK(total_args + 1); + : PEEK(total_args + 1); int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); if (err) goto error; GO_TO_INSTRUCTION(CALL_KW); } @@ -2904,7 +2897,7 @@ } Py_DECREF(receiver); Py_DECREF(value); - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -2925,8 +2918,8 @@ PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); - STACK_SHRINK(1); - stack_pointer[-1] = value; + stack_pointer[-2] = value; + stack_pointer += -1; DISPATCH(); } @@ -3094,7 +3087,7 @@ } _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); + tstate, oparg > 0, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; if (frame->instr_ptr != this_instr) { @@ -3112,8 +3105,8 @@ INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); if (err) GOTO_ERROR(error); Py_INCREF(retval); assert(EMPTY()); @@ -3136,8 +3129,8 @@ PyObject *retval; retval = stack_pointer[-1]; int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); if (err) GOTO_ERROR(error); STACK_SHRINK(1); assert(EMPTY()); @@ -3167,8 +3160,8 @@ gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyFrame_SetStackPointer(frame, stack_pointer - 1); int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, retval); + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, retval); if (err) GOTO_ERROR(error); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -3211,8 +3204,8 @@ Py_DECREF(left); Py_DECREF(right); b = res ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; + stack_pointer[-2] = b; + stack_pointer += -1; DISPATCH(); } @@ -3286,7 +3279,7 @@ v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -3301,19 +3294,19 @@ PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { _PyErr_Clear(tstate); _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); } Py_DECREF(iterable); if (true) goto pop_1_error; } assert(Py_IsNone(none_val)); Py_DECREF(iterable); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -3323,8 +3316,8 @@ INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); PyObject *value; value = Py_NewRef(PyExc_AssertionError); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } @@ -3334,7 +3327,6 @@ INSTRUCTION_STATS(LOAD_ATTR); PREDICTED(LOAD_ATTR); _Py_CODEUNIT *this_instr = next_instr - 10; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *self_or_null = NULL; @@ -3374,7 +3366,7 @@ the second element of the stack to NULL, to signal CALL that it's not a method call. NULL | meth | arg1 | ... | argN - */ + */ Py_DECREF(owner); if (attr == NULL) goto pop_1_error; self_or_null = NULL; @@ -3387,9 +3379,9 @@ if (attr == NULL) goto pop_1_error; } } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += ((oparg & 1)); DISPATCH(); } @@ -3397,9 +3389,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS owner = stack_pointer[-1]; { @@ -3408,6 +3402,7 @@ assert(type_version != 0); DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); } + /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS { PyObject *descr = read_obj(&this_instr[6].cache); @@ -3417,9 +3412,9 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } @@ -3427,6 +3422,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); @@ -3445,7 +3441,6 @@ assert(code->co_argcount == 2); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); Py_INCREF(f); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); @@ -3461,9 +3456,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3490,9 +3487,10 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } @@ -3500,9 +3498,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; - PyObject *self; + PyObject *self = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3519,6 +3519,7 @@ /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); } + /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_LAZY_DICT { PyObject *descr = read_obj(&this_instr[6].cache); @@ -3529,9 +3530,9 @@ attr = Py_NewRef(descr); self = owner; } - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } @@ -3539,9 +3540,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; - PyObject *self; + PyObject *self = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3550,6 +3553,7 @@ assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } + /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT { PyObject *descr = read_obj(&this_instr[6].cache); @@ -3561,9 +3565,9 @@ attr = Py_NewRef(descr); self = owner; } - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } @@ -3571,9 +3575,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; - PyObject *self; + PyObject *self = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3606,9 +3612,9 @@ assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); self = owner; } - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } @@ -3616,9 +3622,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _CHECK_ATTR_MODULE owner = stack_pointer[-1]; { @@ -3642,9 +3650,10 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } @@ -3652,8 +3661,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3662,6 +3673,7 @@ assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } + /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT { PyObject *descr = read_obj(&this_instr[6].cache); @@ -3673,6 +3685,7 @@ attr = Py_NewRef(descr); } stack_pointer[-1] = attr; + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } @@ -3680,8 +3693,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3713,6 +3728,7 @@ attr = Py_NewRef(descr); } stack_pointer[-1] = attr; + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } @@ -3720,6 +3736,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); @@ -3727,7 +3744,6 @@ PyObject *fget = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -3752,9 +3768,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3774,9 +3792,10 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } @@ -3784,9 +3803,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3827,9 +3848,10 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } @@ -3844,8 +3866,8 @@ "__build_class__ not found"); if (true) goto error; } - STACK_GROW(1); - stack_pointer[-1] = bc; + stack_pointer[0] = bc; + stack_pointer += 1; DISPATCH(); } @@ -3856,8 +3878,8 @@ PyObject *value; value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } @@ -3873,8 +3895,8 @@ if (true) goto error; } Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } @@ -3886,8 +3908,8 @@ value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } @@ -3899,8 +3921,8 @@ value = GETLOCAL(oparg); // do not use SETLOCAL here, it decrefs the old value GETLOCAL(oparg) = NULL; - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } @@ -3912,8 +3934,8 @@ value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } @@ -3929,9 +3951,9 @@ value2 = GETLOCAL(oparg2); Py_INCREF(value1); Py_INCREF(value2); - STACK_GROW(2); - stack_pointer[-2] = value1; - stack_pointer[-1] = value2; + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; DISPATCH(); } @@ -3984,8 +4006,8 @@ } if (v == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); GOTO_ERROR(error); } } @@ -4001,7 +4023,6 @@ INSTRUCTION_STATS(LOAD_GLOBAL); PREDICTED(LOAD_GLOBAL); _Py_CODEUNIT *this_instr = next_instr - 5; - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; // _SPECIALIZE_LOAD_GLOBAL @@ -4026,14 +4047,14 @@ && PyDict_CheckExact(BUILTINS())) { res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); + (PyDictObject *)BUILTINS(), + name); if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + NAME_ERROR_MSG, name); } if (true) goto error; } @@ -4048,18 +4069,17 @@ if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; if (res == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); if (true) goto error; } } } null = NULL; } - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); DISPATCH(); } @@ -4067,8 +4087,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&this_instr[2].cache); @@ -4096,10 +4118,9 @@ STAT_INC(LOAD_GLOBAL, hit); null = NULL; } - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); DISPATCH(); } @@ -4107,8 +4128,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&this_instr[2].cache); @@ -4117,6 +4140,7 @@ DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } + /* Skip 1 cache entry */ // _LOAD_GLOBAL_MODULE { uint16_t index = read_u16(&this_instr[4].cache); @@ -4128,10 +4152,9 @@ STAT_INC(LOAD_GLOBAL, hit); null = NULL; } - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); DISPATCH(); } @@ -4147,8 +4170,8 @@ if (true) goto error; } Py_INCREF(locals); - STACK_GROW(1); - stack_pointer[-1] = locals; + stack_pointer[0] = locals; + stack_pointer += 1; DISPATCH(); } @@ -4177,14 +4200,14 @@ } if (v == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); GOTO_ERROR(error); } } } - STACK_GROW(1); - stack_pointer[-1] = v; + stack_pointer[0] = v; + stack_pointer += 1; DISPATCH(); } @@ -4194,7 +4217,6 @@ INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED(LOAD_SUPER_ATTR); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); PyObject *class; PyObject *global_super; PyObject *self; @@ -4224,8 +4246,8 @@ if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); if (err) goto pop_3_error; } // we make no attempt to optimize here; specializations should @@ -4258,10 +4280,9 @@ if (attr == NULL) goto pop_3_error; null = NULL; } - STACK_SHRINK(2); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-3] = attr; + if (oparg & 1) stack_pointer[-2] = null; + stack_pointer += -2 + ((oparg & 1)); DISPATCH(); } @@ -4269,6 +4290,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); PyObject *self; PyObject *class; PyObject *global_super; @@ -4286,8 +4308,8 @@ Py_DECREF(class); Py_DECREF(self); if (attr == NULL) goto pop_3_error; - STACK_SHRINK(2); - stack_pointer[-1] = attr; + stack_pointer[-3] = attr; + stack_pointer += -2 + (((0) ? 1 : 0)); DISPATCH(); } @@ -4295,6 +4317,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); PyObject *self; PyObject *class; PyObject *global_super; @@ -4324,9 +4347,9 @@ Py_DECREF(self); self_or_null = NULL; } - STACK_SHRINK(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self_or_null; + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; DISPATCH(); } @@ -4352,17 +4375,14 @@ PyObject *codeobj; PyObject *func; codeobj = stack_pointer[-1]; - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - + PyFunction_New(codeobj, GLOBALS()); Py_DECREF(codeobj); if (func_obj == NULL) { GOTO_ERROR(error); } - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); + func_obj, ((PyCodeObject *)codeobj)->co_version); func = (PyObject *)func_obj; stack_pointer[-1] = func; DISPATCH(); @@ -4382,7 +4402,7 @@ /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -4409,10 +4429,11 @@ } else { if (_PyErr_Occurred(tstate)) goto pop_3_error; + // Error! attrs = Py_None; // Failure! } - STACK_SHRINK(2); - stack_pointer[-1] = attrs; + stack_pointer[-3] = attrs; + stack_pointer += -2; DISPATCH(); } @@ -4428,8 +4449,8 @@ // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = _PyEval_MatchKeys(tstate, subject, keys); if (values_or_none == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = values_or_none; + stack_pointer[0] = values_or_none; + stack_pointer += 1; DISPATCH(); } @@ -4442,8 +4463,8 @@ subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -4456,8 +4477,8 @@ subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -4476,7 +4497,7 @@ exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4492,7 +4513,7 @@ this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4524,7 +4545,7 @@ #endif JUMPBY(oparg * flag); } - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4556,7 +4577,7 @@ #endif JUMPBY(oparg * flag); } - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4572,7 +4593,7 @@ this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4583,7 +4604,7 @@ PyObject *value; value = stack_pointer[-1]; Py_DECREF(value); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4603,9 +4624,9 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - STACK_GROW(1); - stack_pointer[-2] = prev_exc; - stack_pointer[-1] = new_exc; + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; DISPATCH(); } @@ -4615,8 +4636,8 @@ INSTRUCTION_STATS(PUSH_NULL); PyObject *res; res = NULL; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -4625,29 +4646,29 @@ next_instr += 1; INSTRUCTION_STATS(RAISE_VARARGS); PyObject **args; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; TIER_ONE_ONLY PyObject *cause = NULL, *exc = NULL; switch (oparg) { - case 2: + case 2: cause = args[1]; /* fall through */ - case 1: + case 1: exc = args[0]; /* fall through */ - case 0: + case 0: if (do_raise(tstate, exc, cause)) { assert(oparg == 0); monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } break; - default: + default: _PyErr_SetString(tstate, PyExc_SystemError, "bad RAISE_VARARGS oparg"); break; } - if (true) { STACK_SHRINK(oparg); goto error; } + if (true) { stack_pointer += -oparg; goto error; } } TARGET(RERAISE) { @@ -4657,7 +4678,7 @@ PyObject *exc; PyObject **values; exc = stack_pointer[-1]; - values = stack_pointer - 1 - oparg; + values = &stack_pointer[-1 - oparg]; TIER_ONE_ONLY assert(oparg >= 0 && oparg <= 2); if (oparg) { @@ -4693,12 +4714,11 @@ INSTRUCTION_STATS(RESUME); PREDICTED(RESUME); _Py_CODEUNIT *this_instr = next_instr - 1; - static_assert(0 == 0, "incorrect cache size"); TIER_ONE_ONLY assert(frame == tstate->current_frame); uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; + _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((code_version & 255) == 0); if (code_version != global_version) { @@ -4719,10 +4739,11 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME_CHECK); -#if defined(__EMSCRIPTEN__) + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; -#endif + #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); @@ -4736,7 +4757,7 @@ INSTRUCTION_STATS(RETURN_CONST); PyObject *value; PyObject *retval; - // LOAD_CONST + // _LOAD_CONST { value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); @@ -4744,11 +4765,11 @@ // _POP_FRAME retval = value; { - assert(EMPTY()); #if TIER_ONE assert(frame != &entry_frame); #endif - STORE_SP(); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; @@ -4757,12 +4778,12 @@ _PyFrame_StackPush(frame, retval); LOAD_SP(); LOAD_IP(frame->return_offset); - #if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } - #endif + #endif } DISPATCH(); } @@ -4801,12 +4822,12 @@ INSTRUCTION_STATS(RETURN_VALUE); PyObject *retval; retval = stack_pointer[-1]; - STACK_SHRINK(1); - assert(EMPTY()); #if TIER_ONE assert(frame != &entry_frame); #endif - STORE_SP(); + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; @@ -4815,12 +4836,12 @@ _PyFrame_StackPush(frame, retval); LOAD_SP(); LOAD_IP(frame->return_offset); -#if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } -#endif + #endif DISPATCH(); } @@ -4830,7 +4851,6 @@ INSTRUCTION_STATS(SEND); PREDICTED(SEND); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); PyObject *receiver; PyObject *v; PyObject *retval; @@ -4897,6 +4917,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); PyObject *v; PyObject *receiver; v = stack_pointer[-1]; @@ -4955,7 +4976,7 @@ int err = PySet_Add(set, v); Py_DECREF(v); if (err) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4971,28 +4992,28 @@ PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; + assert(func_obj->func_closure == NULL); + func_obj->func_closure = attr; + break; case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; + assert(func_obj->func_annotations == NULL); + func_obj->func_annotations = attr; + break; case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; + assert(PyDict_CheckExact(attr)); + assert(func_obj->func_kwdefaults == NULL); + func_obj->func_kwdefaults = attr; + break; case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; + assert(PyTuple_CheckExact(attr)); + assert(func_obj->func_defaults == NULL); + func_obj->func_defaults = attr; + break; default: - Py_UNREACHABLE(); + Py_UNREACHABLE(); } - STACK_SHRINK(1); - stack_pointer[-1] = func; + stack_pointer[-2] = func; + stack_pointer += -1; DISPATCH(); } @@ -5007,7 +5028,7 @@ int err = _PySet_Update(set, iterable); Py_DECREF(iterable); if (err < 0) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -5017,7 +5038,6 @@ INSTRUCTION_STATS(STORE_ATTR); PREDICTED(STORE_ATTR); _Py_CODEUNIT *this_instr = next_instr - 5; - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *v; // _SPECIALIZE_STORE_ATTR @@ -5045,7 +5065,7 @@ Py_DECREF(owner); if (err) goto pop_2_error; } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -5053,8 +5073,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *value; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -5086,7 +5108,7 @@ } Py_DECREF(owner); } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -5094,8 +5116,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *value; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -5115,7 +5139,7 @@ Py_XDECREF(old_value); Py_DECREF(owner); } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -5123,6 +5147,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *value; owner = stack_pointer[-1]; @@ -5167,7 +5192,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -5181,7 +5206,7 @@ PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -5192,7 +5217,7 @@ PyObject *value; value = stack_pointer[-1]; SETLOCAL(oparg, value); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -5224,7 +5249,7 @@ uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); SETLOCAL(oparg2, value2); - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -5238,7 +5263,7 @@ int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); if (err) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -5258,12 +5283,12 @@ if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); + err = PyDict_SetItem(ns, name, v); else - err = PyObject_SetItem(ns, name, v); + err = PyObject_SetItem(ns, name, v); Py_DECREF(v); if (err) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -5291,7 +5316,7 @@ Py_DECREF(v); Py_DECREF(container); if (err) goto pop_4_error; - STACK_SHRINK(4); + stack_pointer += -4; DISPATCH(); } @@ -5301,7 +5326,6 @@ INSTRUCTION_STATS(STORE_SUBSCR); PREDICTED(STORE_SUBSCR); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; PyObject *v; @@ -5331,7 +5355,7 @@ Py_DECREF(sub); if (err) goto pop_3_error; } - STACK_SHRINK(3); + stack_pointer += -3; DISPATCH(); } @@ -5339,6 +5363,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *dict; PyObject *value; @@ -5350,7 +5375,7 @@ int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error; - STACK_SHRINK(3); + stack_pointer += -3; DISPATCH(); } @@ -5358,6 +5383,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *list; PyObject *value; @@ -5366,21 +5392,19 @@ value = stack_pointer[-3]; DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); - // Ensure nonnegative, zero-or-one-digit ints. DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; // Ensure index < len(list) DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); assert(old_value != NULL); Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - STACK_SHRINK(3); + stack_pointer += -3; DISPATCH(); } @@ -5404,7 +5428,6 @@ INSTRUCTION_STATS(TO_BOOL); PREDICTED(TO_BOOL); _Py_CODEUNIT *this_instr = next_instr - 4; - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; // _SPECIALIZE_TO_BOOL @@ -5437,6 +5460,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -5455,6 +5479,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; value = stack_pointer[-1]; DEOPT_IF(!PyBool_Check(value), TO_BOOL); @@ -5466,6 +5491,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_INT); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -5487,6 +5513,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -5502,6 +5529,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_NONE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -5517,6 +5545,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_STR); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -5587,7 +5616,7 @@ int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq); if (res == 0) goto pop_1_error; - STACK_GROW((oparg & 0xFF) + (oparg >> 8)); + stack_pointer += (oparg >> 8) + (oparg & 0xFF); DISPATCH(); } @@ -5597,7 +5626,6 @@ INSTRUCTION_STATS(UNPACK_SEQUENCE); PREDICTED(UNPACK_SEQUENCE); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; // _SPECIALIZE_UNPACK_SEQUENCE seq = stack_pointer[-1]; @@ -5623,8 +5651,7 @@ Py_DECREF(seq); if (res == 0) goto pop_1_error; } - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; DISPATCH(); } @@ -5632,10 +5659,11 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; - values = stack_pointer - 1; + values = &stack_pointer[-1]; DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -5644,8 +5672,7 @@ *values++ = Py_NewRef(items[i]); } Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; DISPATCH(); } @@ -5653,10 +5680,11 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; - values = stack_pointer - 1; + values = &stack_pointer[-1]; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -5665,8 +5693,7 @@ *values++ = Py_NewRef(items[i]); } Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; DISPATCH(); } @@ -5674,10 +5701,11 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; - values = stack_pointer - 1; + values = &stack_pointer[-1]; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); @@ -5685,8 +5713,7 @@ values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; DISPATCH(); } @@ -5708,9 +5735,8 @@ - exit_func: FOURTH = the context.__exit__ bound method We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). Then we push the __exit__ return value. - */ + */ PyObject *exc, *tb; - assert(val && PyExceptionInstance_Check(val)); exc = PyExceptionInstance_Class(val); tb = PyException_GetTraceback(val); @@ -5724,10 +5750,10 @@ (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[4] = {NULL, exc, val, tb}; res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -5759,5 +5785,4 @@ LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); goto resume_frame; } - #undef TIER_ONE diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py new file mode 100644 index 00000000000..027f9861a1c --- /dev/null +++ b/Tools/cases_generator/analyzer.py @@ -0,0 +1,456 @@ +from dataclasses import dataclass +import lexer +import parser +from typing import Optional + + +@dataclass +class Properties: + escapes: bool + infallible: bool + deopts: bool + oparg: bool + jumps: bool + ends_with_eval_breaker: bool + needs_this: bool + always_exits: bool + stores_sp: bool + + def dump(self, indent: str) -> None: + print(indent, end="") + text = ", ".join([f"{key}: {value}" for (key, value) in self.__dict__.items()]) + print(indent, text, sep="") + + @staticmethod + def from_list(properties: list["Properties"]) -> "Properties": + return Properties( + escapes=any(p.escapes for p in properties), + infallible=all(p.infallible for p in properties), + deopts=any(p.deopts for p in properties), + oparg=any(p.oparg for p in properties), + jumps=any(p.jumps for p in properties), + ends_with_eval_breaker=any(p.ends_with_eval_breaker for p in properties), + needs_this=any(p.needs_this for p in properties), + always_exits=any(p.always_exits for p in properties), + stores_sp=any(p.stores_sp for p in properties), + ) + + +SKIP_PROPERTIES = Properties( + escapes=False, + infallible=True, + deopts=False, + oparg=False, + jumps=False, + ends_with_eval_breaker=False, + needs_this=False, + always_exits=False, + stores_sp=False, +) + + +@dataclass +class Skip: + "Unused cache entry" + size: int + + @property + def name(self) -> str: + return f"unused/{self.size}" + + @property + def properties(self) -> Properties: + return SKIP_PROPERTIES + + +@dataclass +class StackItem: + name: str + type: str | None + condition: str | None + size: str + peek: bool = False + + def __str__(self) -> str: + cond = f" if ({self.condition})" if self.condition else "" + size = f"[{self.size}]" if self.size != "1" else "" + type = "" if self.type is None else f"{self.type} " + return f"{type}{self.name}{size}{cond} {self.peek}" + + def is_array(self) -> bool: + return self.type == "PyObject **" + + +@dataclass +class StackEffect: + inputs: list[StackItem] + outputs: list[StackItem] + + def __str__(self) -> str: + return f"({', '.join([str(i) for i in self.inputs])} -- {', '.join([str(i) for i in self.outputs])})" + + +@dataclass +class CacheEntry: + name: str + size: int + + def __str__(self) -> str: + return f"{self.name}/{self.size}" + + +@dataclass +class Uop: + name: str + context: parser.Context | None + annotations: list[str] + stack: StackEffect + caches: list[CacheEntry] + body: list[lexer.Token] + properties: Properties + _size: int = -1 + + def dump(self, indent: str) -> None: + print( + indent, self.name, ", ".join(self.annotations) if self.annotations else "" + ) + print(indent, self.stack, ", ".join([str(c) for c in self.caches])) + self.properties.dump(" " + indent) + + @property + def size(self) -> int: + if self._size < 0: + self._size = sum(c.size for c in self.caches) + return self._size + + +Part = Uop | Skip + + +@dataclass +class Instruction: + name: str + parts: list[Part] + _properties: Properties | None + is_target: bool = False + family: Optional["Family"] = None + + @property + def properties(self) -> Properties: + if self._properties is None: + self._properties = self._compute_properties() + return self._properties + + def _compute_properties(self) -> Properties: + return Properties.from_list([part.properties for part in self.parts]) + + def dump(self, indent: str) -> None: + print(indent, self.name, "=", ", ".join([part.name for part in self.parts])) + self.properties.dump(" " + indent) + + @property + def size(self) -> int: + return 1 + sum(part.size for part in self.parts) + + +@dataclass +class PseudoInstruction: + name: str + targets: list[Instruction] + flags: list[str] + + def dump(self, indent: str) -> None: + print(indent, self.name, "->", " or ".join([t.name for t in self.targets])) + + +@dataclass +class Family: + name: str + size: str + members: list[Instruction] + + def dump(self, indent: str) -> None: + print(indent, self.name, "= ", ", ".join([m.name for m in self.members])) + + +@dataclass +class Analysis: + instructions: dict[str, Instruction] + uops: dict[str, Uop] + families: dict[str, Family] + pseudos: dict[str, PseudoInstruction] + + +def analysis_error(message: str, tkn: lexer.Token) -> SyntaxError: + # To do -- support file and line output + # Construct a SyntaxError instance from message and token + return lexer.make_syntax_error(message, "", tkn.line, tkn.column, "") + + +def override_error( + name: str, + context: parser.Context | None, + prev_context: parser.Context | None, + token: lexer.Token, +) -> SyntaxError: + return analysis_error( + f"Duplicate definition of '{name}' @ {context} " + f"previous definition @ {prev_context}", + token, + ) + + +def convert_stack_item(item: parser.StackEffect) -> StackItem: + return StackItem(item.name, item.type, item.cond, (item.size or "1")) + + +def analyze_stack(op: parser.InstDef) -> StackEffect: + inputs: list[StackItem] = [ + convert_stack_item(i) for i in op.inputs if isinstance(i, parser.StackEffect) + ] + outputs: list[StackItem] = [convert_stack_item(i) for i in op.outputs] + for input, output in zip(inputs, outputs): + if input.name == output.name: + input.peek = output.peek = True + return StackEffect(inputs, outputs) + + +def analyze_caches(op: parser.InstDef) -> list[CacheEntry]: + caches: list[parser.CacheEffect] = [ + i for i in op.inputs if isinstance(i, parser.CacheEffect) + ] + return [CacheEntry(i.name, int(i.size)) for i in caches] + + +def variable_used(node: parser.InstDef, name: str) -> bool: + """Determine whether a variable with a given name is used in a node.""" + return any( + token.kind == "IDENTIFIER" and token.text == name for token in node.tokens + ) + + +def is_infallible(op: parser.InstDef) -> bool: + return not ( + variable_used(op, "ERROR_IF") + or variable_used(op, "error") + or variable_used(op, "pop_1_error") + or variable_used(op, "exception_unwind") + or variable_used(op, "resume_with_error") + ) + + +from flags import makes_escaping_api_call + +EXITS = { + "DISPATCH", + "GO_TO_INSTRUCTION", + "Py_UNREACHABLE", + "DISPATCH_INLINED", + "DISPATCH_GOTO", +} + + +def eval_breaker_at_end(op: parser.InstDef) -> bool: + return op.tokens[-5].text == "CHECK_EVAL_BREAKER" + + +def always_exits(op: parser.InstDef) -> bool: + depth = 0 + tkn_iter = iter(op.tokens) + for tkn in tkn_iter: + if tkn.kind == "LBRACE": + depth += 1 + elif tkn.kind == "RBRACE": + depth -= 1 + elif depth > 1: + continue + elif tkn.kind == "GOTO" or tkn.kind == "RETURN": + return True + elif tkn.kind == "KEYWORD": + if tkn.text in EXITS: + return True + elif tkn.kind == "IDENTIFIER": + if tkn.text in EXITS: + return True + if tkn.text == "DEOPT_IF" or tkn.text == "ERROR_IF": + next(tkn_iter) # '(' + t = next(tkn_iter) + if t.text == "true": + return True + return False + + +def compute_properties(op: parser.InstDef) -> Properties: + return Properties( + escapes=makes_escaping_api_call(op), + infallible=is_infallible(op), + deopts=variable_used(op, "DEOPT_IF"), + oparg=variable_used(op, "oparg"), + jumps=variable_used(op, "JUMPBY"), + ends_with_eval_breaker=eval_breaker_at_end(op), + needs_this=variable_used(op, "this_instr"), + always_exits=always_exits(op), + stores_sp=variable_used(op, "STORE_SP"), + ) + + +def make_uop(name: str, op: parser.InstDef) -> Uop: + return Uop( + name=name, + context=op.context, + annotations=op.annotations, + stack=analyze_stack(op), + caches=analyze_caches(op), + body=op.block.tokens, + properties=compute_properties(op), + ) + + +def add_op(op: parser.InstDef, uops: dict[str, Uop]) -> None: + assert op.kind == "op" + if op.name in uops: + if "override" not in op.annotations: + raise override_error( + op.name, op.context, uops[op.name].context, op.tokens[0] + ) + uops[op.name] = make_uop(op.name, op) + + +def add_instruction( + name: str, parts: list[Part], instructions: dict[str, Instruction] +) -> None: + instructions[name] = Instruction(name, parts, None) + + +def desugar_inst( + inst: parser.InstDef, instructions: dict[str, Instruction], uops: dict[str, Uop] +) -> None: + assert inst.kind == "inst" + name = inst.name + uop = make_uop("_" + inst.name, inst) + uops[inst.name] = uop + add_instruction(name, [uop], instructions) + + +def add_macro( + macro: parser.Macro, instructions: dict[str, Instruction], uops: dict[str, Uop] +) -> None: + parts: list[Uop | Skip] = [] + for part in macro.uops: + match part: + case parser.OpName(): + if part.name not in uops: + analysis_error(f"No Uop named {part.name}", macro.tokens[0]) + parts.append(uops[part.name]) + case parser.CacheEffect(): + parts.append(Skip(part.size)) + case _: + assert False + assert parts + add_instruction(macro.name, parts, instructions) + + +def add_family( + pfamily: parser.Family, + instructions: dict[str, Instruction], + families: dict[str, Family], +) -> None: + family = Family( + pfamily.name, + pfamily.size, + [instructions[member_name] for member_name in pfamily.members], + ) + for member in family.members: + member.family = family + # The head of the family is an implicit jump target for DEOPTs + instructions[family.name].is_target = True + families[family.name] = family + + +def add_pseudo( + pseudo: parser.Pseudo, + instructions: dict[str, Instruction], + pseudos: dict[str, PseudoInstruction], +) -> None: + pseudos[pseudo.name] = PseudoInstruction( + pseudo.name, + [instructions[target] for target in pseudo.targets], + pseudo.flags, + ) + + +def analyze_forest(forest: list[parser.AstNode]) -> Analysis: + instructions: dict[str, Instruction] = {} + uops: dict[str, Uop] = {} + families: dict[str, Family] = {} + pseudos: dict[str, PseudoInstruction] = {} + for node in forest: + match node: + case parser.InstDef(name): + if node.kind == "inst": + desugar_inst(node, instructions, uops) + else: + assert node.kind == "op" + add_op(node, uops) + case parser.Macro(): + pass + case parser.Family(): + pass + case parser.Pseudo(): + pass + case _: + assert False + for node in forest: + if isinstance(node, parser.Macro): + add_macro(node, instructions, uops) + for node in forest: + match node: + case parser.Family(): + add_family(node, instructions, families) + case parser.Pseudo(): + add_pseudo(node, instructions, pseudos) + case _: + pass + for uop in uops.values(): + tkn_iter = iter(uop.body) + for tkn in tkn_iter: + if tkn.kind == "IDENTIFIER" and tkn.text == "GO_TO_INSTRUCTION": + if next(tkn_iter).kind != "LPAREN": + continue + target = next(tkn_iter) + if target.kind != "IDENTIFIER": + continue + if target.text in instructions: + instructions[target.text].is_target = True + # Hack + instructions["BINARY_OP_INPLACE_ADD_UNICODE"].family = families["BINARY_OP"] + return Analysis(instructions, uops, families, pseudos) + + +def analyze_files(filenames: list[str]) -> Analysis: + return analyze_forest(parser.parse_files(filenames)) + + +def dump_analysis(analysis: Analysis) -> None: + print("Uops:") + for u in analysis.uops.values(): + u.dump(" ") + print("Instructions:") + for i in analysis.instructions.values(): + i.dump(" ") + print("Families:") + for f in analysis.families.values(): + f.dump(" ") + print("Pseudos:") + for p in analysis.pseudos.values(): + p.dump(" ") + + +if __name__ == "__main__": + import sys + + if len(sys.argv) < 2: + print("No input") + else: + filenames = sys.argv[1:] + dump_analysis(analyze_files(filenames)) diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py new file mode 100644 index 00000000000..0b7edd03fd9 --- /dev/null +++ b/Tools/cases_generator/cwriter.py @@ -0,0 +1,111 @@ +from lexer import Token +from typing import TextIO + + +class CWriter: + "A writer that understands tokens and how to format C code" + + last_token: Token | None + + def __init__(self, out: TextIO, indent: int, line_directives: bool): + self.out = out + self.base_column = indent * 4 + self.indents = [i * 4 for i in range(indent + 1)] + self.line_directives = line_directives + self.last_token = None + self.newline = True + + def set_position(self, tkn: Token) -> None: + if self.last_token is not None: + if self.last_token.line < tkn.line: + self.out.write("\n") + if self.line_directives: + self.out.write(f'#line {tkn.line} "{tkn.filename}"\n') + self.out.write(" " * self.indents[-1]) + else: + gap = tkn.column - self.last_token.end_column + self.out.write(" " * gap) + elif self.newline: + self.out.write(" " * self.indents[-1]) + self.last_token = tkn + self.newline = False + + def emit_at(self, txt: str, where: Token) -> None: + self.set_position(where) + self.out.write(txt) + + def maybe_dedent(self, txt: str) -> None: + parens = txt.count("(") - txt.count(")") + if parens < 0: + self.indents.pop() + elif "}" in txt or is_label(txt): + self.indents.pop() + + def maybe_indent(self, txt: str) -> None: + parens = txt.count("(") - txt.count(")") + if parens > 0 and self.last_token: + offset = self.last_token.end_column - 1 + if offset <= self.indents[-1] or offset > 40: + offset = self.indents[-1] + 4 + self.indents.append(offset) + elif "{" in txt or is_label(txt): + self.indents.append(self.indents[-1] + 4) + + def emit_text(self, txt: str) -> None: + self.out.write(txt) + + def emit_multiline_comment(self, tkn: Token) -> None: + self.set_position(tkn) + lines = tkn.text.splitlines(True) + first = True + for line in lines: + text = line.lstrip() + if first: + spaces = 0 + else: + spaces = self.indents[-1] + if text.startswith("*"): + spaces += 1 + else: + spaces += 3 + first = False + self.out.write(" " * spaces) + self.out.write(text) + + def emit_token(self, tkn: Token) -> None: + if tkn.kind == "COMMENT" and "\n" in tkn.text: + return self.emit_multiline_comment(tkn) + self.maybe_dedent(tkn.text) + self.set_position(tkn) + self.emit_text(tkn.text) + self.maybe_indent(tkn.text) + + def emit_str(self, txt: str) -> None: + self.maybe_dedent(txt) + if self.newline and txt: + if txt[0] != "\n": + self.out.write(" " * self.indents[-1]) + self.newline = False + self.emit_text(txt) + if txt.endswith("\n"): + self.newline = True + self.maybe_indent(txt) + self.last_token = None + + def emit(self, txt: str | Token) -> None: + if isinstance(txt, Token): + self.emit_token(txt) + elif isinstance(txt, str): + self.emit_str(txt) + else: + assert False + + def start_line(self) -> None: + if not self.newline: + self.out.write("\n") + self.newline = True + self.last_token = None + + +def is_label(txt: str) -> bool: + return not txt.startswith("//") and txt.endswith(":") diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 1f94c1fedb2..4b7f028970b 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -883,7 +883,6 @@ def main() -> None: return # These raise OSError if output can't be written - a.write_instructions(args.output, args.emit_line_directives) a.assign_opcode_ids() a.write_opcode_ids(args.opcode_ids_h, args.opcode_targets_h) diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 1185c855785..c3c2954a420 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -112,7 +112,7 @@ STRING = "STRING" char = r"\'.\'" # TODO: escape sequence CHARACTER = "CHARACTER" -comment_re = r"//.*|/\*([^*]|\*[^/])*\*/" +comment_re = r"(//.*)|/\*([^*]|\*[^/])*\*/" COMMENT = "COMMENT" newline = r"\n" @@ -234,6 +234,7 @@ def make_syntax_error( @dataclass(slots=True) class Token: + filename: str kind: str text: str begin: tuple[int, int] @@ -261,7 +262,7 @@ class Token: def replaceText(self, txt: str) -> "Token": assert isinstance(txt, str) - return Token(self.kind, txt, self.begin, self.end) + return Token(self.filename, self.kind, txt, self.begin, self.end) def __repr__(self) -> str: b0, b1 = self.begin @@ -272,7 +273,7 @@ class Token: return f"{self.kind}({self.text!r}, {b0}:{b1}, {e0}:{e1})" -def tokenize(src: str, line: int = 1, filename: str | None = None) -> Iterator[Token]: +def tokenize(src: str, line: int = 1, filename: str = "") -> Iterator[Token]: linestart = -1 for m in matcher.finditer(src): start, end = m.span() @@ -323,7 +324,7 @@ def tokenize(src: str, line: int = 1, filename: str | None = None) -> Iterator[T else: begin = line, start - linestart if kind != "\n": - yield Token(kind, text, begin, (line, start - linestart + len(text))) + yield Token(filename, kind, text, begin, (line, start - linestart + len(text))) def to_text(tkns: list[Token], dedent: int = 0) -> str: diff --git a/Tools/cases_generator/mypy.ini b/Tools/cases_generator/mypy.ini index e7175e26335..8e5a31851c5 100644 --- a/Tools/cases_generator/mypy.ini +++ b/Tools/cases_generator/mypy.ini @@ -11,3 +11,5 @@ strict = True strict_concatenate = True enable_error_code = ignore-without-code,redundant-expr,truthy-bool,possibly-undefined warn_unreachable = True +allow_redefinition = True +implicit_reexport = True diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py new file mode 100644 index 00000000000..12173a61199 --- /dev/null +++ b/Tools/cases_generator/parser.py @@ -0,0 +1,55 @@ +from parsing import ( + InstDef, + Macro, + Pseudo, + Family, + Parser, + Context, + CacheEffect, + StackEffect, + OpName, + AstNode, +) +from formatting import prettify_filename + + +BEGIN_MARKER = "// BEGIN BYTECODES //" +END_MARKER = "// END BYTECODES //" + + +def parse_files(filenames: list[str]) -> list[AstNode]: + result: list[AstNode] = [] + for filename in filenames: + with open(filename) as file: + src = file.read() + + psr = Parser(src, filename=prettify_filename(filename)) + + # Skip until begin marker + while tkn := psr.next(raw=True): + if tkn.text == BEGIN_MARKER: + break + else: + raise psr.make_syntax_error( + f"Couldn't find {BEGIN_MARKER!r} in {psr.filename}" + ) + start = psr.getpos() + + # Find end marker, then delete everything after it + while tkn := psr.next(raw=True): + if tkn.text == END_MARKER: + break + del psr.tokens[psr.getpos() - 1 :] + + # Parse from start + psr.setpos(start) + thing_first_token = psr.peek() + while node := psr.definition(): + assert node is not None + result.append(node) # type: ignore[arg-type] + if not psr.eof(): + psr.backup() + raise psr.make_syntax_error( + f"Extra stuff at the end of {filename}", psr.next(True) + ) + return result diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 7800adf1679..60c185dcef5 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -141,10 +141,11 @@ class Pseudo(Node): flags: list[str] # instr flags to set on the pseudo instruction targets: list[str] # opcodes this can be replaced by +AstNode = InstDef | Macro | Pseudo | Family class Parser(PLexer): @contextual - def definition(self) -> InstDef | Macro | Pseudo | Family | None: + def definition(self) -> AstNode | None: if macro := self.macro_def(): return macro if family := self.family_def(): diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py new file mode 100644 index 00000000000..9cd8e1a3b2e --- /dev/null +++ b/Tools/cases_generator/stack.py @@ -0,0 +1,81 @@ +import sys +from analyzer import StackItem +from dataclasses import dataclass +from formatting import maybe_parenthesize + + +def var_size(var: StackItem) -> str: + if var.condition: + # Special case simplification + if var.condition == "oparg & 1" and var.size == "1": + return f"({var.condition})" + else: + return f"(({var.condition}) ? {var.size} : 0)" + else: + return var.size + + +class StackOffset: + "The stack offset of the virtual base of the stack from the physical stack pointer" + + def __init__(self) -> None: + self.popped: list[str] = [] + self.pushed: list[str] = [] + + def pop(self, item: StackItem) -> None: + self.popped.append(var_size(item)) + + def push(self, item: StackItem) -> None: + self.pushed.append(var_size(item)) + + def simplify(self) -> None: + "Remove matching values from both the popped and pushed list" + if not self.popped or not self.pushed: + return + # Sort the list so the lexically largest element is last. + popped = sorted(self.popped) + pushed = sorted(self.pushed) + self.popped = [] + self.pushed = [] + while popped and pushed: + pop = popped.pop() + push = pushed.pop() + if pop == push: + pass + elif pop > push: + # if pop > push, there can be no element in pushed matching pop. + self.popped.append(pop) + pushed.append(push) + else: + self.pushed.append(push) + popped.append(pop) + self.popped.extend(popped) + self.pushed.extend(pushed) + + def to_c(self) -> str: + self.simplify() + int_offset = 0 + symbol_offset = "" + for item in self.popped: + try: + int_offset -= int(item) + except ValueError: + symbol_offset += f" - {maybe_parenthesize(item)}" + for item in self.pushed: + try: + int_offset += int(item) + except ValueError: + symbol_offset += f" + {maybe_parenthesize(item)}" + if symbol_offset and not int_offset: + res = symbol_offset + else: + res = f"{int_offset}{symbol_offset}" + if res.startswith(" + "): + res = res[3:] + if res.startswith(" - "): + res = "-" + res[3:] + return res + + def clear(self) -> None: + self.popped = [] + self.pushed = [] diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py new file mode 100644 index 00000000000..eba926435d2 --- /dev/null +++ b/Tools/cases_generator/tier1_generator.py @@ -0,0 +1,417 @@ +"""Generate the main interpreter switch. +Reads the instruction definitions from bytecodes.c. +Writes the cases to generated_cases.c.h, which is #included in ceval.c. +""" + +import argparse +import os.path +import sys + +from analyzer import ( + Analysis, + Instruction, + Uop, + Part, + analyze_files, + Skip, + StackItem, + analysis_error, +) +from cwriter import CWriter +from typing import TextIO, Iterator +from lexer import Token +from stack import StackOffset + + +HERE = os.path.dirname(__file__) +ROOT = os.path.join(HERE, "../..") +THIS = os.path.relpath(__file__, ROOT).replace(os.path.sep, "/") + +DEFAULT_INPUT = os.path.relpath(os.path.join(ROOT, "Python/bytecodes.c")) +DEFAULT_OUTPUT = os.path.relpath(os.path.join(ROOT, "Python/generated_cases.c.h")) + + +def write_header(filename: str, outfile: TextIO) -> None: + outfile.write( + f"""// This file is generated by {THIS} +// from: +// {filename} +// Do not edit! + +#ifdef TIER_TWO + #error "This file is for Tier 1 only" +#endif +#define TIER_ONE 1 +""" + ) + + +FOOTER = "#undef TIER_ONE\n" + + +class SizeMismatch(Exception): + pass + + +class Stack: + def __init__(self) -> None: + self.top_offset = StackOffset() + self.base_offset = StackOffset() + self.peek_offset = StackOffset() + self.variables: list[StackItem] = [] + self.defined: set[str] = set() + + def pop(self, var: StackItem) -> str: + self.top_offset.pop(var) + if not var.peek: + self.peek_offset.pop(var) + indirect = "&" if var.is_array() else "" + if self.variables: + popped = self.variables.pop() + if popped.size != var.size: + raise SizeMismatch( + f"Size mismatch when popping '{popped.name}' from stack to assign to {var.name}. " + f"Expected {var.size} got {popped.size}" + ) + if popped.name == var.name: + return "" + elif popped.name == "unused": + self.defined.add(var.name) + return ( + f"{var.name} = {indirect}stack_pointer[{self.top_offset.to_c()}];\n" + ) + elif var.name == "unused": + return "" + else: + self.defined.add(var.name) + return f"{var.name} = {popped.name};\n" + self.base_offset.pop(var) + if var.name == "unused": + return "" + else: + self.defined.add(var.name) + assign = f"{var.name} = {indirect}stack_pointer[{self.base_offset.to_c()}];" + if var.condition: + return f"if ({var.condition}) {{ {assign} }}\n" + return f"{assign}\n" + + def push(self, var: StackItem) -> str: + self.variables.append(var) + if var.is_array() and var.name not in self.defined and var.name != "unused": + c_offset = self.top_offset.to_c() + self.top_offset.push(var) + self.defined.add(var.name) + return f"{var.name} = &stack_pointer[{c_offset}];\n" + else: + self.top_offset.push(var) + return "" + + def flush(self, out: CWriter) -> None: + for var in self.variables: + if not var.peek: + if var.name != "unused" and not var.is_array(): + if var.condition: + out.emit(f" if ({var.condition}) ") + out.emit( + f"stack_pointer[{self.base_offset.to_c()}] = {var.name};\n" + ) + self.base_offset.push(var) + if self.base_offset.to_c() != self.top_offset.to_c(): + print("base", self.base_offset.to_c(), "top", self.top_offset.to_c()) + assert False + number = self.base_offset.to_c() + if number != "0": + out.emit(f"stack_pointer += {number};\n") + self.variables = [] + self.base_offset.clear() + self.top_offset.clear() + self.peek_offset.clear() + + def as_comment(self) -> str: + return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */" + + +def declare_variables(inst: Instruction, out: CWriter) -> None: + variables = {"unused"} + for uop in inst.parts: + if isinstance(uop, Uop): + for var in reversed(uop.stack.inputs): + if var.name not in variables: + type = var.type if var.type else "PyObject *" + variables.add(var.name) + if var.condition: + out.emit(f"{type}{var.name} = NULL;\n") + else: + out.emit(f"{type}{var.name};\n") + for var in uop.stack.outputs: + if var.name not in variables: + variables.add(var.name) + type = var.type if var.type else "PyObject *" + if var.condition: + out.emit(f"{type}{var.name} = NULL;\n") + else: + out.emit(f"{type}{var.name};\n") + + +def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None: + parens = 0 + for tkn in tkn_iter: + if tkn.kind == end and parens == 0: + return + if tkn.kind == "LPAREN": + parens += 1 + if tkn.kind == "RPAREN": + parens -= 1 + out.emit(tkn) + + +def replace_deopt( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction, +) -> None: + out.emit_at("DEOPT_IF", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + out.emit(", ") + assert inst.family is not None + out.emit(inst.family.name) + out.emit(");\n") + + +def replace_error( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction, +) -> None: + out.emit_at("if ", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "COMMA") + label = next(tkn_iter).text + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + out.emit(") ") + c_offset = stack.peek_offset.to_c() + try: + offset = -int(c_offset) + close = ";\n" + except ValueError: + offset = None + out.emit(f"{{ stack_pointer += {c_offset}; ") + close = "; }\n" + out.emit("goto ") + if offset: + out.emit(f"pop_{offset}_") + out.emit(label) + out.emit(close) + + +def replace_decrefs( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction, +) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + out.emit_at("", tkn) + for var in uop.stack.inputs: + if var.name == "unused" or var.name == "null" or var.peek: + continue + if var.size != "1": + out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + out.emit(f"Py_DECREF({var.name}[_i]);\n") + out.emit("}\n") + elif var.condition: + out.emit(f"Py_XDECREF({var.name});\n") + else: + out.emit(f"Py_DECREF({var.name});\n") + + +def replace_store_sp( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction, +) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + out.emit_at("", tkn) + stack.flush(out) + out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n") + + +def replace_check_eval_breaker( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction, +) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + if not uop.properties.ends_with_eval_breaker: + out.emit_at("CHECK_EVAL_BREAKER();", tkn) + + +REPLACEMENT_FUNCTIONS = { + "DEOPT_IF": replace_deopt, + "ERROR_IF": replace_error, + "DECREF_INPUTS": replace_decrefs, + "CHECK_EVAL_BREAKER": replace_check_eval_breaker, + "STORE_SP": replace_store_sp, +} + + +# Move this to formatter +def emit_tokens(out: CWriter, uop: Uop, stack: Stack, inst: Instruction) -> None: + tkns = uop.body[1:-1] + if not tkns: + return + tkn_iter = iter(tkns) + out.start_line() + for tkn in tkn_iter: + if tkn.kind == "IDENTIFIER" and tkn.text in REPLACEMENT_FUNCTIONS: + REPLACEMENT_FUNCTIONS[tkn.text](out, tkn, tkn_iter, uop, stack, inst) + else: + out.emit(tkn) + + +def write_uop( + uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruction, braces: bool +) -> int: + # out.emit(stack.as_comment() + "\n") + if isinstance(uop, Skip): + entries = "entries" if uop.size > 1 else "entry" + out.emit(f"/* Skip {uop.size} cache {entries} */\n") + return offset + uop.size + try: + out.start_line() + if braces: + out.emit(f"// {uop.name}\n") + for var in reversed(uop.stack.inputs): + out.emit(stack.pop(var)) + if braces: + out.emit("{\n") + if not uop.properties.stores_sp: + for i, var in enumerate(uop.stack.outputs): + out.emit(stack.push(var)) + for cache in uop.caches: + if cache.name != "unused": + if cache.size == 4: + type = "PyObject *" + reader = "read_obj" + else: + type = f"uint{cache.size*16}_t " + reader = f"read_u{cache.size*16}" + out.emit( + f"{type}{cache.name} = {reader}(&this_instr[{offset}].cache);\n" + ) + offset += cache.size + emit_tokens(out, uop, stack, inst) + if uop.properties.stores_sp: + for i, var in enumerate(uop.stack.outputs): + out.emit(stack.push(var)) + if braces: + out.start_line() + out.emit("}\n") + # out.emit(stack.as_comment() + "\n") + return offset + except SizeMismatch as ex: + raise analysis_error(ex.args[0], uop.body[0]) + + +def uses_this(inst: Instruction) -> bool: + if inst.properties.needs_this: + return True + for uop in inst.parts: + if isinstance(uop, Skip): + continue + for cache in uop.caches: + if cache.name != "unused": + return True + return False + + +def generate_tier1( + filenames: str, analysis: Analysis, outfile: TextIO, lines: bool +) -> None: + write_header(filenames, outfile) + out = CWriter(outfile, 2, lines) + out.emit("\n") + for name, inst in sorted(analysis.instructions.items()): + needs_this = uses_this(inst) + out.emit("\n") + out.emit(f"TARGET({name}) {{\n") + if needs_this and not inst.is_target: + out.emit(f"_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;\n") + else: + out.emit(f"frame->instr_ptr = next_instr;\n") + out.emit(f"next_instr += {inst.size};\n") + out.emit(f"INSTRUCTION_STATS({name});\n") + if inst.is_target: + out.emit(f"PREDICTED({name});\n") + if needs_this: + out.emit(f"_Py_CODEUNIT *this_instr = next_instr - {inst.size};\n") + if inst.family is not None: + out.emit( + f"static_assert({inst.family.size} == {inst.size-1}" + ', "incorrect cache size");\n' + ) + declare_variables(inst, out) + offset = 1 # The instruction itself + stack = Stack() + for part in inst.parts: + # Only emit braces if more than one uop + offset = write_uop(part, out, offset, stack, inst, len(inst.parts) > 1) + out.start_line() + if not inst.parts[-1].properties.always_exits: + stack.flush(out) + if inst.parts[-1].properties.ends_with_eval_breaker: + out.emit("CHECK_EVAL_BREAKER();\n") + out.emit("DISPATCH();\n") + out.start_line() + out.emit("}") + out.emit("\n") + outfile.write(FOOTER) + + +arg_parser = argparse.ArgumentParser( + description="Generate the code for the interpreter switch.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_tier1(args.input, data, outfile, args.emit_line_directives)