GH-113710: Add a tier 2 peephole optimization pass. (GH-114487)

* Convert _LOAD_CONST to inline versions

* Remove PEP 523 checks

Include/internal/pycore_uop_ids.h

@@ -230,9 +230,10 @@ extern "C" {
 #define _JUMP_TO_TOP 377
 #define _SAVE_RETURN_OFFSET 378
 #define _CHECK_VALIDITY 379
-#define _LOAD_CONST_INLINE_BORROW 380
+#define _LOAD_CONST_INLINE 380
-#define _INTERNAL_INCREMENT_OPT_COUNTER 381
+#define _LOAD_CONST_INLINE_BORROW 381
-#define MAX_UOP_ID 381
+#define _INTERNAL_INCREMENT_OPT_COUNTER 382
+#define MAX_UOP_ID 382
 
 #ifdef __cplusplus
 }

Include/internal/pycore_uop_metadata.h

@@ -202,6 +202,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
     [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG,
     [_EXIT_TRACE] = HAS_DEOPT_FLAG,
     [_CHECK_VALIDITY] = HAS_DEOPT_FLAG,
+    [_LOAD_CONST_INLINE] = 0,
     [_LOAD_CONST_INLINE_BORROW] = 0,
     [_INTERNAL_INCREMENT_OPT_COUNTER] = 0,
 };
@@ -329,6 +330,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
     [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT",
     [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS",
     [_LOAD_CONST] = "_LOAD_CONST",
+    [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE",
     [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW",
     [_LOAD_DEREF] = "_LOAD_DEREF",
     [_LOAD_FAST] = "_LOAD_FAST",

Python/bytecodes.c

@@ -4070,6 +4070,10 @@ dummy_func(
             DEOPT_IF(!current_executor->vm_data.valid);
         }
 
+        op(_LOAD_CONST_INLINE, (ptr/4 -- value)) {
+            value = Py_NewRef(ptr);
+        }
+
         op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) {
             value = ptr;
         }

Python/executor_cases.c.h

@@ -3390,6 +3390,15 @@
             break;
         }
 
+        case _LOAD_CONST_INLINE: {
+            PyObject *value;
+            PyObject *ptr = (PyObject *)CURRENT_OPERAND();
+            value = Py_NewRef(ptr);
+            stack_pointer[0] = value;
+            stack_pointer += 1;
+            break;
+        }
+
         case _LOAD_CONST_INLINE_BORROW: {
             PyObject *value;
             PyObject *ptr = (PyObject *)CURRENT_OPERAND();

Python/optimizer.c

@@ -588,6 +588,9 @@ top:  // Jump here after _PUSH_FRAME or likely branches
                         ADD_TO_TRACE(uop, oparg, operand, target);
                         if (uop == _POP_FRAME) {
                             TRACE_STACK_POP();
+                            /* Set the operand to the code object returned to,
+                             * to assist optimization passes */
+                            trace[trace_length-1].operand = (uintptr_t)code;
                             DPRINTF(2,
                                 "Returning to %s (%s:%d) at byte offset %d\n",
                                 PyUnicode_AsUTF8(code->co_qualname),
@@ -629,6 +632,9 @@ top:  // Jump here after _PUSH_FRAME or likely branches
                                 instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1;
                                 TRACE_STACK_PUSH();
                                 _Py_BloomFilter_Add(dependencies, new_code);
+                                /* Set the operand to the callee's code object,
+                                * to assist optimization passes */
+                                trace[trace_length-1].operand = (uintptr_t)new_code;
                                 code = new_code;
                                 instr = _PyCode_CODE(code);
                                 DPRINTF(2,

Python/optimizer_analysis.c

@@ -12,6 +12,39 @@
 #include <stddef.h>
 #include "pycore_optimizer.h"
 
+static void
+peephole_opt(PyCodeObject *co, _PyUOpInstruction *buffer, int buffer_size)
+{
+    for (int pc = 0; pc < buffer_size; pc++) {
+        int opcode = buffer[pc].opcode;
+        switch(opcode) {
+            case _LOAD_CONST: {
+                assert(co != NULL);
+                PyObject *val = PyTuple_GET_ITEM(co->co_consts, buffer[pc].oparg);
+                buffer[pc].opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE;
+                buffer[pc].operand = (uintptr_t)val;
+                break;
+            }
+            case _CHECK_PEP_523:
+            {
+                /* Setting the eval frame function invalidates
+                 * all executors, so no need to check dynamically */
+                if (_PyInterpreterState_GET()->eval_frame == NULL) {
+                    buffer[pc].opcode = _NOP;
+                }
+                break;
+            }
+            case _PUSH_FRAME:
+            case _POP_FRAME:
+                co = (PyCodeObject *)buffer[pc].operand;
+                break;
+            case _JUMP_TO_TOP:
+            case _EXIT_TRACE:
+                return;
+        }
+    }
+}
+
 static void
 remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size)
 {
@@ -59,6 +92,7 @@ _Py_uop_analyze_and_optimize(
     int curr_stacklen
 )
 {
+    peephole_opt(co, buffer, buffer_size);
     remove_unneeded_uops(buffer, buffer_size);
     return 0;
 }

Python/pystate.c

@@ -2608,11 +2608,15 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp,
                                      _PyFrameEvalFunction eval_frame)
 {
     if (eval_frame == _PyEval_EvalFrameDefault) {
-        interp->eval_frame = NULL;
+        eval_frame = NULL;
     }
-    else {
+    if (eval_frame == interp->eval_frame) {
-        interp->eval_frame = eval_frame;
+        return;
     }
+    if (eval_frame != NULL) {
+        _Py_Executors_InvalidateAll(interp);
+    }
+    interp->eval_frame = eval_frame;
 }