GH-96421: Insert shim frame on entry to interpreter (GH-96319)

* Adds EXIT_INTERPRETER instruction to exit PyEval_EvalDefault()

* Simplifies RETURN_VALUE, YIELD_VALUE and RETURN_GENERATOR instructions as they no longer need to check for entry frames.

Include/internal/pycore_code.h

@@ -456,6 +456,16 @@ _PyCode_LineNumberFromArray(PyCodeObject *co, int index)
     }
 }
 
+typedef struct _PyShimCodeDef {
+    const uint8_t *code;
+    int codelen;
+    int stacksize;
+    const char *cname;
+} _PyShimCodeDef;
+
+extern PyCodeObject *
+_Py_MakeShimCode(const _PyShimCodeDef *code);
+
 
 #ifdef __cplusplus
 }

Include/internal/pycore_frame.h

@@ -42,17 +42,18 @@ typedef enum _framestate {
 enum _frameowner {
     FRAME_OWNED_BY_THREAD = 0,
     FRAME_OWNED_BY_GENERATOR = 1,
-    FRAME_OWNED_BY_FRAME_OBJECT = 2
+    FRAME_OWNED_BY_FRAME_OBJECT = 2,
+    FRAME_OWNED_BY_CSTACK = 3,
 };
 
 typedef struct _PyInterpreterFrame {
     /* "Specials" section */
-    PyObject *f_funcobj; /* Strong reference */
+    PyObject *f_funcobj; /* Strong reference. Only valid if not on C stack */
-    PyObject *f_globals; /* Borrowed reference */
+    PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */
-    PyObject *f_builtins; /* Borrowed reference */
+    PyObject *f_builtins; /* Borrowed reference. Only valid if not on C stack */
-    PyObject *f_locals; /* Strong reference, may be NULL */
+    PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */
     PyCodeObject *f_code; /* Strong reference */
-    PyFrameObject *frame_obj; /* Strong reference, may be NULL */
+    PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */
     /* Linkage section */
     struct _PyInterpreterFrame *previous;
     // NOTE: This is not necessarily the last instruction started in the given
@@ -60,9 +61,8 @@ typedef struct _PyInterpreterFrame {
     // example, it may be an inline CACHE entry, an instruction we just jumped
     // over, or (in the case of a newly-created frame) a totally invalid value:
     _Py_CODEUNIT *prev_instr;
-    int stacktop;     /* Offset of TOS from localsplus  */
+    int stacktop;  /* Offset of TOS from localsplus  */
     uint16_t yield_offset;
-    bool is_entry;  // Whether this is the "root" frame for the current _PyCFrame.
     char owner;
     /* Locals and stack */
     PyObject *localsplus[1];
@@ -110,7 +110,6 @@ _PyFrame_InitializeSpecials(
     frame->stacktop = code->co_nlocalsplus;
     frame->frame_obj = NULL;
     frame->prev_instr = _PyCode_CODE(code) - 1;
-    frame->is_entry = false;
     frame->yield_offset = 0;
     frame->owner = FRAME_OWNED_BY_THREAD;
 }

Include/internal/pycore_global_objects_fini_generated.h

@@ -564,6 +564,7 @@ _PyStaticObjects_CheckRefcnt(void) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(newline));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(open_br));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(percent));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(shim_name));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CANCELLED));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(FINISHED));

Include/internal/pycore_global_strings.h

@@ -48,6 +48,7 @@ struct _Py_global_strings {
         STRUCT_FOR_STR(newline, "\n")
         STRUCT_FOR_STR(open_br, "{")
         STRUCT_FOR_STR(percent, "%")
+        STRUCT_FOR_STR(shim_name, "<shim>")
         STRUCT_FOR_STR(utf_8, "utf-8")
     } literals;
 

Include/internal/pycore_interp.h

@@ -186,6 +186,7 @@ struct _is {
     struct ast_state ast;
     struct types_state types;
     struct callable_cache callable_cache;
+    PyCodeObject *interpreter_trampoline;
 
     /* The following fields are here to avoid allocation during init.
        The data is exposed through PyInterpreterState pointer fields.

Include/internal/pycore_opcode.h

@@ -136,6 +136,7 @@ const uint8_t _PyOpcode_Deopt[256] = {
     [IMPORT_FROM] = IMPORT_FROM,
     [IMPORT_NAME] = IMPORT_NAME,
     [IMPORT_STAR] = IMPORT_STAR,
+    [INTERPRETER_EXIT] = INTERPRETER_EXIT,
     [IS_OP] = IS_OP,
     [JUMP_BACKWARD] = JUMP_BACKWARD,
     [JUMP_BACKWARD_NO_INTERRUPT] = JUMP_BACKWARD_NO_INTERRUPT,
@@ -235,19 +236,20 @@ static const char *const _PyOpcode_OpName[263] = {
     [CACHE] = "CACHE",
     [POP_TOP] = "POP_TOP",
     [PUSH_NULL] = "PUSH_NULL",
-    [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT",
+    [INTERPRETER_EXIT] = "INTERPRETER_EXIT",
     [END_FOR] = "END_FOR",
+    [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT",
     [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT",
     [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE",
     [BINARY_OP_GENERIC] = "BINARY_OP_GENERIC",
-    [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE",
     [NOP] = "NOP",
     [UNARY_POSITIVE] = "UNARY_POSITIVE",
     [UNARY_NEGATIVE] = "UNARY_NEGATIVE",
     [UNARY_NOT] = "UNARY_NOT",
+    [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE",
     [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT",
-    [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT",
     [UNARY_INVERT] = "UNARY_INVERT",
+    [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT",
     [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT",
     [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT",
     [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT",
@@ -256,20 +258,20 @@ static const char *const _PyOpcode_OpName[263] = {
     [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT",
     [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS",
     [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS",
-    [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS",
     [BINARY_SUBSCR] = "BINARY_SUBSCR",
     [BINARY_SLICE] = "BINARY_SLICE",
     [STORE_SLICE] = "STORE_SLICE",
+    [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS",
     [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS",
-    [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS",
     [GET_LEN] = "GET_LEN",
     [MATCH_MAPPING] = "MATCH_MAPPING",
     [MATCH_SEQUENCE] = "MATCH_SEQUENCE",
     [MATCH_KEYS] = "MATCH_KEYS",
-    [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS",
+    [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS",
     [PUSH_EXC_INFO] = "PUSH_EXC_INFO",
     [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH",
     [CHECK_EG_MATCH] = "CHECK_EG_MATCH",
+    [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS",
     [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST",
     [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O",
     [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE",
@@ -280,7 +282,6 @@ static const char *const _PyOpcode_OpName[263] = {
     [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O",
     [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1",
     [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1",
-    [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1",
     [WITH_EXCEPT_START] = "WITH_EXCEPT_START",
     [GET_AITER] = "GET_AITER",
     [GET_ANEXT] = "GET_ANEXT",
@@ -288,37 +289,37 @@ static const char *const _PyOpcode_OpName[263] = {
     [BEFORE_WITH] = "BEFORE_WITH",
     [END_ASYNC_FOR] = "END_ASYNC_FOR",
     [CLEANUP_THROW] = "CLEANUP_THROW",
+    [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1",
     [COMPARE_OP_FLOAT_JUMP] = "COMPARE_OP_FLOAT_JUMP",
     [COMPARE_OP_GENERIC] = "COMPARE_OP_GENERIC",
     [COMPARE_OP_INT_JUMP] = "COMPARE_OP_INT_JUMP",
-    [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP",
     [STORE_SUBSCR] = "STORE_SUBSCR",
     [DELETE_SUBSCR] = "DELETE_SUBSCR",
-    [FOR_ITER_LIST] = "FOR_ITER_LIST",
+    [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP",
     [STOPITERATION_ERROR] = "STOPITERATION_ERROR",
+    [FOR_ITER_LIST] = "FOR_ITER_LIST",
     [FOR_ITER_RANGE] = "FOR_ITER_RANGE",
     [FOR_ITER_GEN] = "FOR_ITER_GEN",
     [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS",
-    [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN",
     [GET_ITER] = "GET_ITER",
     [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER",
     [PRINT_EXPR] = "PRINT_EXPR",
     [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS",
+    [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN",
     [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE",
-    [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE",
     [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR",
     [RETURN_GENERATOR] = "RETURN_GENERATOR",
+    [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE",
     [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY",
     [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT",
     [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT",
     [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT",
     [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT",
-    [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT",
     [LIST_TO_TUPLE] = "LIST_TO_TUPLE",
     [RETURN_VALUE] = "RETURN_VALUE",
     [IMPORT_STAR] = "IMPORT_STAR",
     [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS",
-    [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES",
+    [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT",
     [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP",
     [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR",
     [POP_EXCEPT] = "POP_EXCEPT",
@@ -345,7 +346,7 @@ static const char *const _PyOpcode_OpName[263] = {
     [JUMP_FORWARD] = "JUMP_FORWARD",
     [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP",
     [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP",
-    [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST",
+    [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES",
     [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE",
     [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE",
     [LOAD_GLOBAL] = "LOAD_GLOBAL",
@@ -353,7 +354,7 @@ static const char *const _PyOpcode_OpName[263] = {
     [CONTAINS_OP] = "CONTAINS_OP",
     [RERAISE] = "RERAISE",
     [COPY] = "COPY",
-    [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST",
+    [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST",
     [BINARY_OP] = "BINARY_OP",
     [SEND] = "SEND",
     [LOAD_FAST] = "LOAD_FAST",
@@ -373,9 +374,9 @@ static const char *const _PyOpcode_OpName[263] = {
     [STORE_DEREF] = "STORE_DEREF",
     [DELETE_DEREF] = "DELETE_DEREF",
     [JUMP_BACKWARD] = "JUMP_BACKWARD",
-    [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST",
+    [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST",
     [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX",
-    [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN",
+    [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST",
     [EXTENDED_ARG] = "EXTENDED_ARG",
     [LIST_APPEND] = "LIST_APPEND",
     [SET_ADD] = "SET_ADD",
@@ -385,27 +386,27 @@ static const char *const _PyOpcode_OpName[263] = {
     [YIELD_VALUE] = "YIELD_VALUE",
     [RESUME] = "RESUME",
     [MATCH_CLASS] = "MATCH_CLASS",
+    [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN",
     [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE",
-    [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE",
     [FORMAT_VALUE] = "FORMAT_VALUE",
     [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP",
     [BUILD_STRING] = "BUILD_STRING",
+    [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE",
     [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT",
     [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT",
     [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST",
-    [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST",
     [LIST_EXTEND] = "LIST_EXTEND",
     [SET_UPDATE] = "SET_UPDATE",
     [DICT_MERGE] = "DICT_MERGE",
     [DICT_UPDATE] = "DICT_UPDATE",
+    [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST",
     [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT",
     [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT",
     [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST",
     [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE",
-    [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE",
     [CALL] = "CALL",
     [KW_NAMES] = "KW_NAMES",
-    [173] = "<173>",
+    [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE",
     [174] = "<174>",
     [175] = "<175>",
     [176] = "<176>",
@@ -499,7 +500,6 @@ static const char *const _PyOpcode_OpName[263] = {
 #endif
 
 #define EXTRA_CASES \
-    case 173: \
     case 174: \
     case 175: \
     case 176: \

Include/internal/pycore_runtime_init_generated.h

@@ -558,6 +558,7 @@ extern "C" {
                 INIT_STR(newline, "\n"), \
                 INIT_STR(open_br, "{"), \
                 INIT_STR(percent, "%"), \
+                INIT_STR(shim_name, "<shim>"), \
                 INIT_STR(utf_8, "utf-8"), \
             }, \
             .identifiers = { \

Include/opcode.h

@@ -11,6 +11,7 @@ extern "C" {
 #define CACHE                                    0
 #define POP_TOP                                  1
 #define PUSH_NULL                                2
+#define INTERPRETER_EXIT                         3
 #define END_FOR                                  4
 #define NOP                                      9
 #define UNARY_POSITIVE                          10
@@ -128,69 +129,69 @@ extern "C" {
 #define JUMP_NO_INTERRUPT                      261
 #define LOAD_METHOD                            262
 #define MAX_PSEUDO_OPCODE                      262
-#define BINARY_OP_ADD_FLOAT                      3
+#define BINARY_OP_ADD_FLOAT                      5
-#define BINARY_OP_ADD_INT                        5
+#define BINARY_OP_ADD_INT                        6
-#define BINARY_OP_ADD_UNICODE                    6
+#define BINARY_OP_ADD_UNICODE                    7
-#define BINARY_OP_GENERIC                        7
+#define BINARY_OP_GENERIC                        8
-#define BINARY_OP_INPLACE_ADD_UNICODE            8
+#define BINARY_OP_INPLACE_ADD_UNICODE           13
-#define BINARY_OP_MULTIPLY_FLOAT                13
+#define BINARY_OP_MULTIPLY_FLOAT                14
-#define BINARY_OP_MULTIPLY_INT                  14
+#define BINARY_OP_MULTIPLY_INT                  16
-#define BINARY_OP_SUBTRACT_FLOAT                16
+#define BINARY_OP_SUBTRACT_FLOAT                17
-#define BINARY_OP_SUBTRACT_INT                  17
+#define BINARY_OP_SUBTRACT_INT                  18
-#define BINARY_SUBSCR_DICT                      18
+#define BINARY_SUBSCR_DICT                      19
-#define BINARY_SUBSCR_GETITEM                   19
+#define BINARY_SUBSCR_GETITEM                   20
-#define BINARY_SUBSCR_LIST_INT                  20
+#define BINARY_SUBSCR_LIST_INT                  21
-#define BINARY_SUBSCR_TUPLE_INT                 21
+#define BINARY_SUBSCR_TUPLE_INT                 22
-#define CALL_PY_EXACT_ARGS                      22
+#define CALL_PY_EXACT_ARGS                      23
-#define CALL_PY_WITH_DEFAULTS                   23
+#define CALL_PY_WITH_DEFAULTS                   24
-#define CALL_BOUND_METHOD_EXACT_ARGS            24
+#define CALL_BOUND_METHOD_EXACT_ARGS            28
-#define CALL_BUILTIN_CLASS                      28
+#define CALL_BUILTIN_CLASS                      29
-#define CALL_BUILTIN_FAST_WITH_KEYWORDS         29
+#define CALL_BUILTIN_FAST_WITH_KEYWORDS         34
-#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS  34
+#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS  38
-#define CALL_NO_KW_BUILTIN_FAST                 38
+#define CALL_NO_KW_BUILTIN_FAST                 39
-#define CALL_NO_KW_BUILTIN_O                    39
+#define CALL_NO_KW_BUILTIN_O                    40
-#define CALL_NO_KW_ISINSTANCE                   40
+#define CALL_NO_KW_ISINSTANCE                   41
-#define CALL_NO_KW_LEN                          41
+#define CALL_NO_KW_LEN                          42
-#define CALL_NO_KW_LIST_APPEND                  42
+#define CALL_NO_KW_LIST_APPEND                  43
-#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST       43
+#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST       44
-#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS     44
+#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS     45
-#define CALL_NO_KW_METHOD_DESCRIPTOR_O          45
+#define CALL_NO_KW_METHOD_DESCRIPTOR_O          46
-#define CALL_NO_KW_STR_1                        46
+#define CALL_NO_KW_STR_1                        47
-#define CALL_NO_KW_TUPLE_1                      47
+#define CALL_NO_KW_TUPLE_1                      48
-#define CALL_NO_KW_TYPE_1                       48
+#define CALL_NO_KW_TYPE_1                       56
-#define COMPARE_OP_FLOAT_JUMP                   56
+#define COMPARE_OP_FLOAT_JUMP                   57
-#define COMPARE_OP_GENERIC                      57
+#define COMPARE_OP_GENERIC                      58
-#define COMPARE_OP_INT_JUMP                     58
+#define COMPARE_OP_INT_JUMP                     59
-#define COMPARE_OP_STR_JUMP                     59
+#define COMPARE_OP_STR_JUMP                     62
-#define FOR_ITER_LIST                           62
+#define FOR_ITER_LIST                           64
-#define FOR_ITER_RANGE                          64
+#define FOR_ITER_RANGE                          65
-#define FOR_ITER_GEN                            65
+#define FOR_ITER_GEN                            66
-#define LOAD_ATTR_CLASS                         66
+#define LOAD_ATTR_CLASS                         67
-#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN       67
+#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN       72
-#define LOAD_ATTR_INSTANCE_VALUE                72
+#define LOAD_ATTR_INSTANCE_VALUE                73
-#define LOAD_ATTR_MODULE                        73
+#define LOAD_ATTR_MODULE                        76
-#define LOAD_ATTR_PROPERTY                      76
+#define LOAD_ATTR_PROPERTY                      77
-#define LOAD_ATTR_SLOT                          77
+#define LOAD_ATTR_SLOT                          78
-#define LOAD_ATTR_WITH_HINT                     78
+#define LOAD_ATTR_WITH_HINT                     79
-#define LOAD_ATTR_METHOD_LAZY_DICT              79
+#define LOAD_ATTR_METHOD_LAZY_DICT              80
-#define LOAD_ATTR_METHOD_NO_DICT                80
+#define LOAD_ATTR_METHOD_NO_DICT                81
-#define LOAD_ATTR_METHOD_WITH_DICT              81
+#define LOAD_ATTR_METHOD_WITH_DICT              86
-#define LOAD_ATTR_METHOD_WITH_VALUES            86
+#define LOAD_ATTR_METHOD_WITH_VALUES           113
-#define LOAD_CONST__LOAD_FAST                  113
+#define LOAD_CONST__LOAD_FAST                  121
-#define LOAD_FAST__LOAD_CONST                  121
+#define LOAD_FAST__LOAD_CONST                  141
-#define LOAD_FAST__LOAD_FAST                   141
+#define LOAD_FAST__LOAD_FAST                   143
-#define LOAD_GLOBAL_BUILTIN                    143
+#define LOAD_GLOBAL_BUILTIN                    153
-#define LOAD_GLOBAL_MODULE                     153
+#define LOAD_GLOBAL_MODULE                     154
-#define STORE_ATTR_INSTANCE_VALUE              154
+#define STORE_ATTR_INSTANCE_VALUE              158
-#define STORE_ATTR_SLOT                        158
+#define STORE_ATTR_SLOT                        159
-#define STORE_ATTR_WITH_HINT                   159
+#define STORE_ATTR_WITH_HINT                   160
-#define STORE_FAST__LOAD_FAST                  160
+#define STORE_FAST__LOAD_FAST                  161
-#define STORE_FAST__STORE_FAST                 161
+#define STORE_FAST__STORE_FAST                 166
-#define STORE_SUBSCR_DICT                      166
+#define STORE_SUBSCR_DICT                      167
-#define STORE_SUBSCR_LIST_INT                  167
+#define STORE_SUBSCR_LIST_INT                  168
-#define UNPACK_SEQUENCE_LIST                   168
+#define UNPACK_SEQUENCE_LIST                   169
-#define UNPACK_SEQUENCE_TUPLE                  169
+#define UNPACK_SEQUENCE_TUPLE                  170
-#define UNPACK_SEQUENCE_TWO_TUPLE              170
+#define UNPACK_SEQUENCE_TWO_TUPLE              173
 #define DO_TRACING                             255
 
 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\

Lib/opcode.py

@@ -77,6 +77,7 @@ def pseudo_op(name, op, real_ops):
 def_op('CACHE', 0)
 def_op('POP_TOP', 1)
 def_op('PUSH_NULL', 2)
+def_op('INTERPRETER_EXIT', 3)
 
 def_op('END_FOR', 4)
 

Misc/NEWS.d/next/Core and Builtins/2022-10-19-15-59-08.gh-issue-96421.e22y3r.rst

@@ -0,0 +1,13 @@
+When calling into Python code from C code, through :c:func:`PyEval_EvalFrameEx` or
+a related C-API function, a shim frame in inserted into the call stack.
+This occurs in the ``_PyEval_EvalFrameDefault()`` function.
+The extra frame should be invisible to all Python and most C extensions,
+but out-of-process profilers and debuggers need to be aware of it.
+These shim frames can be detected by checking
+``frame->owner == FRAME_OWNED_BY_CSTACK``.
+
+Extensions implementing their own interpreters using PEP 523 need to be
+aware of this shim frame and the changes to the semantics of
+:opcode:`RETURN_VALUE`, :opcode:`YIELD_VALUE`, and :opcode:`RETURN_GENERATOR`,
+which now clear the frame.
+

Objects/codeobject.c

@@ -2264,3 +2264,78 @@ _PyStaticCode_Init(PyCodeObject *co)
     _PyCode_Quicken(co);
     return 0;
 }
+
+#define MAX_CODE_UNITS_PER_LOC_ENTRY 8
+
+PyCodeObject *
+_Py_MakeShimCode(const _PyShimCodeDef *codedef)
+{
+    PyObject *name = NULL;
+    PyObject *co_code = NULL;
+    PyObject *lines = NULL;
+    PyCodeObject *codeobj = NULL;
+    uint8_t *loc_table = NULL;
+
+    name = _PyUnicode_FromASCII(codedef->cname, strlen(codedef->cname));
+    if (name == NULL) {
+        goto cleanup;
+    }
+    co_code = PyBytes_FromStringAndSize(
+        (const char *)codedef->code, codedef->codelen);
+    if (co_code == NULL) {
+        goto cleanup;
+    }
+    int code_units = codedef->codelen / sizeof(_Py_CODEUNIT);
+    int loc_entries = (code_units + MAX_CODE_UNITS_PER_LOC_ENTRY - 1) /
+                      MAX_CODE_UNITS_PER_LOC_ENTRY;
+    loc_table = PyMem_Malloc(loc_entries);
+    if (loc_table == NULL) {
+        PyErr_NoMemory();
+        goto cleanup;
+    }
+    for (int i = 0; i < loc_entries-1; i++) {
+         loc_table[i] = 0x80 | (PY_CODE_LOCATION_INFO_NONE << 3) | 7;
+         code_units -= MAX_CODE_UNITS_PER_LOC_ENTRY;
+    }
+    assert(loc_entries > 0);
+    assert(code_units > 0 && code_units <= MAX_CODE_UNITS_PER_LOC_ENTRY);
+    loc_table[loc_entries-1] = 0x80 |
+        (PY_CODE_LOCATION_INFO_NONE << 3) | (code_units-1);
+    lines = PyBytes_FromStringAndSize((const char *)loc_table, loc_entries);
+    PyMem_Free(loc_table);
+    if (lines == NULL) {
+        goto cleanup;
+    }
+    _Py_DECLARE_STR(shim_name, "<shim>");
+    struct _PyCodeConstructor con = {
+        .filename = &_Py_STR(shim_name),
+        .name = name,
+        .qualname = name,
+        .flags = CO_NEWLOCALS | CO_OPTIMIZED,
+
+        .code = co_code,
+        .firstlineno = 1,
+        .linetable = lines,
+
+        .consts = (PyObject *)&_Py_SINGLETON(tuple_empty),
+        .names = (PyObject *)&_Py_SINGLETON(tuple_empty),
+
+        .localsplusnames = (PyObject *)&_Py_SINGLETON(tuple_empty),
+        .localspluskinds = (PyObject *)&_Py_SINGLETON(bytes_empty),
+
+        .argcount = 0,
+        .posonlyargcount = 0,
+        .kwonlyargcount = 0,
+
+        .stacksize = codedef->stacksize,
+
+        .exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty),
+    };
+
+    codeobj = _PyCode_New(&con);
+cleanup:
+    Py_XDECREF(name);
+    Py_XDECREF(co_code);
+    Py_XDECREF(lines);
+    return codeobj;
+}

Objects/frame_layout.md

@@ -9,10 +9,10 @@ results in poor locality of reference.
 
 In 3.11, rather than have these frames scattered about memory,
 as happens for heap-allocated objects, frames are allocated
-contiguously in a per-thread stack. 
+contiguously in a per-thread stack.
 This improves performance significantly for two reasons:
 * It reduces allocation overhead to a pointer comparison and increment.
-* Stack allocated data has the best possible locality and will always be in 
+* Stack allocated data has the best possible locality and will always be in
   CPU cache.
 
 Generator and coroutines still need heap allocated activation records, but
@@ -63,7 +63,7 @@ We may implement this in the future.
 
 > In a contiguous stack, we would need to save one fewer registers, as the
 > top of the caller's activation record would be the same at the base of the
-> callee's. However, since some activation records are kept on the heap we 
+> callee's. However, since some activation records are kept on the heap we
 > cannot do this.
 
 ### Generators and Coroutines
@@ -85,7 +85,7 @@ and builtins, than strong references to both globals and builtins.
 ### Frame objects
 
 When creating a backtrace or when calling `sys._getframe()` the frame becomes
-visible to Python code. When this happens a new `PyFrameObject` is created 
+visible to Python code. When this happens a new `PyFrameObject` is created
 and a strong reference to it placed in the `frame_obj` field of the specials
 section. The `frame_obj` field is initially `NULL`.
 
@@ -104,7 +104,7 @@ Generator objects have a `_PyInterpreterFrame` embedded in them.
 This means that creating a generator requires only a single allocation,
 reducing allocation overhead and improving locality of reference.
 The embedded frame is linked into the per-thread frame when iterated or
-awaited. 
+awaited.
 
 If a frame object associated with a generator outlives the generator, then
 the embedded `_PyInterpreterFrame` is copied into the frame object.
@@ -119,4 +119,14 @@ Thus, some of the field names may be a bit misleading.
 
 For example the `f_globals` field has a `f_` prefix implying it belongs to the
 `PyFrameObject` struct, although it belongs to the `_PyInterpreterFrame` struct.
-We may rationalize this naming scheme for 3.12.
+We may rationalize this naming scheme for 3.12.
+
+
+### Shim frames
+
+On entry to `_PyEval_EvalFrameDefault()` a shim `_PyInterpreterFrame` is pushed.
+This frame is stored on the C stack, and popped when `_PyEval_EvalFrameDefault()`
+returns. This extra frame is inserted so that `RETURN_VALUE`, `YIELD_VALUE`, and
+`RETURN_GENERATOR` do not need to check whether the current frame is the entry frame.
+The shim frame points to a special code object containing the `INTERPRETER_EXIT`
+instruction which cleans up the shim frame and returns.

Objects/frameobject.c

@@ -1329,15 +1329,15 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear)
     }
 }
 
-
+int
-int _PyFrame_IsEntryFrame(PyFrameObject *frame)
+_PyFrame_IsEntryFrame(PyFrameObject *frame)
 {
     assert(frame != NULL);
-    assert(!_PyFrame_IsIncomplete(frame->f_frame));
+    _PyInterpreterFrame *f = frame->f_frame;
-    return frame->f_frame->is_entry;
+    assert(!_PyFrame_IsIncomplete(f));
+    return f->previous && f->previous->owner == FRAME_OWNED_BY_CSTACK;
 }
 
-
 PyCodeObject *
 PyFrame_GetCode(PyFrameObject *frame)
 {

Objects/genobject.c

@@ -207,8 +207,6 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
     Py_INCREF(result);
     _PyFrame_StackPush(frame, result);
 
-    frame->previous = tstate->cframe->current_frame;
-
     _PyErr_StackItem *prev_exc_info = tstate->exc_info;
     gen->gi_exc_state.previous_item = prev_exc_info;
     tstate->exc_info = &gen->gi_exc_state;
@@ -223,14 +221,8 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
     result = _PyEval_EvalFrame(tstate, frame, exc);
     assert(tstate->exc_info == prev_exc_info);
     assert(gen->gi_exc_state.previous_item == NULL);
-    if (gen->gi_frame_state == FRAME_EXECUTING) {
+    assert(gen->gi_frame_state != FRAME_EXECUTING);
-        gen->gi_frame_state = FRAME_COMPLETED;
+    assert(frame->previous == NULL);
-    }
-    assert(tstate->cframe->current_frame == frame->previous);
-    /* Don't keep the reference to previous any longer than necessary.  It
-     * may keep a chain of frames alive or it could create a reference
-     * cycle. */
-    frame->previous = NULL;
 
     /* If the generator just returned (as opposed to yielding), signal
      * that the generator is exhausted. */
@@ -255,8 +247,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
     /* first clean reference cycle through stored exception traceback */
     _PyErr_ClearExcState(&gen->gi_exc_state);
 
-    gen->gi_frame_state = FRAME_CLEARED;
+    assert(gen->gi_frame_state == FRAME_CLEARED);
-    _PyFrame_Clear(frame);
     *presult = result;
     return result ? PYGEN_RETURN : PYGEN_ERROR;
 }

Python/bytecodes.c

@@ -97,8 +97,7 @@ dummy_func(
     PyObject *consts,
     _Py_CODEUNIT *next_instr,
     PyObject **stack_pointer,
-    CallShape call_shape,
+    PyObject *kwnames,
-    _Py_CODEUNIT *first_instr,
     int throwflag,
     binaryfunc binary_ops[]
 )
@@ -617,6 +616,21 @@ dummy_func(
             goto error;
         }
 
+        // stack effect: (__0 -- )
+        inst(INTERPRETER_EXIT) {
+            assert(frame == &entry_frame);
+            assert(_PyFrame_IsIncomplete(frame));
+            PyObject *retval = POP();
+            assert(EMPTY());
+            /* Restore previous cframe and return. */
+            tstate->cframe = cframe.previous;
+            tstate->cframe->use_tracing = cframe.use_tracing;
+            assert(tstate->cframe->current_frame == frame->previous);
+            assert(!_PyErr_Occurred(tstate));
+            _Py_LeaveRecursiveCallTstate(tstate);
+            return retval;
+        }
+
         // stack effect: (__0 -- )
         inst(RETURN_VALUE) {
             PyObject *retval = POP();
@@ -625,23 +639,10 @@ dummy_func(
             TRACE_FUNCTION_EXIT();
             DTRACE_FUNCTION_EXIT();
             _Py_LeaveRecursiveCallPy(tstate);
-            if (!frame->is_entry) {
+            assert(frame != &entry_frame);
-                frame = cframe.current_frame = pop_frame(tstate, frame);
+            frame = cframe.current_frame = pop_frame(tstate, frame);
-                _PyFrame_StackPush(frame, retval);
+            _PyFrame_StackPush(frame, retval);
-                goto resume_frame;
+            goto resume_frame;
-            }
-            _Py_LeaveRecursiveCallTstate(tstate);
-            if (frame->owner == FRAME_OWNED_BY_GENERATOR) {
-                PyGenObject *gen = _PyFrame_GetGenerator(frame);
-                tstate->exc_info = gen->gi_exc_state.previous_item;
-                gen->gi_exc_state.previous_item = NULL;
-            }
-            /* Restore previous cframe and return. */
-            tstate->cframe = cframe.previous;
-            tstate->cframe->use_tracing = cframe.use_tracing;
-            assert(tstate->cframe->current_frame == frame->previous);
-            assert(!_PyErr_Occurred(tstate));
-            return retval;
         }
 
         // stack effect: ( -- )
@@ -775,6 +776,7 @@ dummy_func(
 
         // error: SEND stack effect depends on jump flag
         inst(SEND) {
+            assert(frame != &entry_frame);
             assert(STACK_LEVEL() >= 2);
             PyObject *v = POP();
             PyObject *receiver = TOP();
@@ -839,6 +841,7 @@ dummy_func(
             // The compiler treats any exception raised here as a failed close()
             // or throw() call.
             assert(oparg == STACK_LEVEL());
+            assert(frame != &entry_frame);
             PyObject *retval = POP();
             PyGenObject *gen = _PyFrame_GetGenerator(frame);
             gen->gi_frame_state = FRAME_SUSPENDED;
@@ -848,19 +851,12 @@ dummy_func(
             tstate->exc_info = gen->gi_exc_state.previous_item;
             gen->gi_exc_state.previous_item = NULL;
             _Py_LeaveRecursiveCallPy(tstate);
-            if (!frame->is_entry) {
+            _PyInterpreterFrame *gen_frame = frame;
-                frame = cframe.current_frame = frame->previous;
+            frame = cframe.current_frame = frame->previous;
-                frame->prev_instr -= frame->yield_offset;
+            gen_frame->previous = NULL;
-                _PyFrame_StackPush(frame, retval);
+            frame->prev_instr -= frame->yield_offset;
-                goto resume_frame;
+            _PyFrame_StackPush(frame, retval);
-            }
+            goto resume_frame;
-            _Py_LeaveRecursiveCallTstate(tstate);
-            /* Restore previous cframe and return. */
-            tstate->cframe = cframe.previous;
-            tstate->cframe->use_tracing = cframe.use_tracing;
-            assert(tstate->cframe->current_frame == frame->previous);
-            assert(!_PyErr_Occurred(tstate));
-            return retval;
         }
 
         // stack effect: (__0 -- )
@@ -876,7 +872,7 @@ dummy_func(
             if (oparg) {
                 PyObject *lasti = PEEK(oparg + 1);
                 if (PyLong_Check(lasti)) {
-                    frame->prev_instr = first_instr + PyLong_AsLong(lasti);
+                    frame->prev_instr = _PyCode_CODE(frame->f_code) + PyLong_AsLong(lasti);
                     assert(!_PyErr_Occurred(tstate));
                 }
                 else {
@@ -2696,12 +2692,10 @@ dummy_func(
             gen->gi_exc_state.previous_item = tstate->exc_info;
             tstate->exc_info = &gen->gi_exc_state;
             gen_frame->previous = frame;
-            gen_frame->is_entry = false;
             frame = cframe.current_frame = gen_frame;
             goto start_frame;
         }
 
-
         // stack effect: ( -- __0)
         inst(BEFORE_ASYNC_WITH) {
             PyObject *mgr = TOP();
@@ -2929,9 +2923,9 @@ dummy_func(
 
         // stack effect: ( -- )
         inst(KW_NAMES) {
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             assert(oparg < PyTuple_GET_SIZE(consts));
-            call_shape.kwnames = GETITEM(consts, oparg);
+            kwnames = GETITEM(consts, oparg);
         }
 
         // stack effect: (__0, __array[oparg] -- )
@@ -2943,7 +2937,7 @@ dummy_func(
                 int nargs = oparg + is_meth;
                 PyObject *callable = PEEK(nargs + 1);
                 next_instr--;
-                _Py_Specialize_Call(callable, next_instr, nargs, call_shape.kwnames);
+                _Py_Specialize_Call(callable, next_instr, nargs, kwnames);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(CALL, deferred);
@@ -2972,9 +2966,9 @@ dummy_func(
                 STACK_SHRINK(total_args);
                 _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
                     tstate, (PyFunctionObject *)function, locals,
-                    stack_pointer, positional_args, call_shape.kwnames
+                    stack_pointer, positional_args, kwnames
                 );
-                call_shape.kwnames = NULL;
+                kwnames = NULL;
                 STACK_SHRINK(2-is_meth);
                 // The frame has stolen all the arguments from the stack,
                 // so there is no need to clean them up.
@@ -2994,15 +2988,15 @@ dummy_func(
             if (cframe.use_tracing) {
                 res = trace_call_function(
                     tstate, function, stack_pointer-total_args,
-                    positional_args, call_shape.kwnames);
+                    positional_args, kwnames);
             }
             else {
                 res = PyObject_Vectorcall(
                     function, stack_pointer-total_args,
                     positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
-                    call_shape.kwnames);
+                    kwnames);
             }
-            call_shape.kwnames = NULL;
+            kwnames = NULL;
             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
             Py_DECREF(function);
             /* Clear the stack */
@@ -3021,7 +3015,7 @@ dummy_func(
 
         // stack effect: (__0, __array[oparg] -- )
         inst(CALL_PY_EXACT_ARGS) {
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             DEOPT_IF(tstate->interp->eval_frame, CALL);
             _PyCallCache *cache = (_PyCallCache *)next_instr;
             int is_meth = is_method(stack_pointer, oparg);
@@ -3054,7 +3048,7 @@ dummy_func(
 
         // stack effect: (__0, __array[oparg] -- )
         inst(CALL_PY_WITH_DEFAULTS) {
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             DEOPT_IF(tstate->interp->eval_frame, CALL);
             _PyCallCache *cache = (_PyCallCache *)next_instr;
             int is_meth = is_method(stack_pointer, oparg);
@@ -3094,7 +3088,7 @@ dummy_func(
 
         // stack effect: (__0, __array[oparg] -- )
         inst(CALL_NO_KW_TYPE_1) {
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             assert(cframe.use_tracing == 0);
             assert(oparg == 1);
             DEOPT_IF(is_method(stack_pointer, 1), CALL);
@@ -3112,7 +3106,7 @@ dummy_func(
 
         // stack effect: (__0, __array[oparg] -- )
         inst(CALL_NO_KW_STR_1) {
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             assert(cframe.use_tracing == 0);
             assert(oparg == 1);
             DEOPT_IF(is_method(stack_pointer, 1), CALL);
@@ -3134,7 +3128,7 @@ dummy_func(
 
         // stack effect: (__0, __array[oparg] -- )
         inst(CALL_NO_KW_TUPLE_1) {
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             assert(oparg == 1);
             DEOPT_IF(is_method(stack_pointer, 1), CALL);
             PyObject *callable = PEEK(2);
@@ -3166,8 +3160,8 @@ dummy_func(
             JUMPBY(INLINE_CACHE_ENTRIES_CALL);
             STACK_SHRINK(total_args);
             PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer,
-                                              total_args-kwnames_len, call_shape.kwnames);
+                                              total_args-kwnames_len, kwnames);
-            call_shape.kwnames = NULL;
+            kwnames = NULL;
             /* Free the arguments. */
             for (int i = 0; i < total_args; i++) {
                 Py_DECREF(stack_pointer[i]);
@@ -3185,7 +3179,7 @@ dummy_func(
         inst(CALL_NO_KW_BUILTIN_O) {
             assert(cframe.use_tracing == 0);
             /* Builtin METH_O functions */
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             int is_meth = is_method(stack_pointer, oparg);
             int total_args = oparg + is_meth;
             DEOPT_IF(total_args != 1, CALL);
@@ -3219,7 +3213,7 @@ dummy_func(
         inst(CALL_NO_KW_BUILTIN_FAST) {
             assert(cframe.use_tracing == 0);
             /* Builtin METH_FASTCALL functions, without keywords */
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             int is_meth = is_method(stack_pointer, oparg);
             int total_args = oparg + is_meth;
             PyObject *callable = PEEK(total_args + 1);
@@ -3276,10 +3270,10 @@ dummy_func(
                 PyCFunction_GET_SELF(callable),
                 stack_pointer,
                 total_args - KWNAMES_LEN(),
-                call_shape.kwnames
+                kwnames
             );
             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
-            call_shape.kwnames = NULL;
+            kwnames = NULL;
 
             /* Free the arguments. */
             for (int i = 0; i < total_args; i++) {
@@ -3297,7 +3291,7 @@ dummy_func(
         // stack effect: (__0, __array[oparg] -- )
         inst(CALL_NO_KW_LEN) {
             assert(cframe.use_tracing == 0);
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             /* len(o) */
             int is_meth = is_method(stack_pointer, oparg);
             int total_args = oparg + is_meth;
@@ -3327,7 +3321,7 @@ dummy_func(
         // stack effect: (__0, __array[oparg] -- )
         inst(CALL_NO_KW_ISINSTANCE) {
             assert(cframe.use_tracing == 0);
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             /* isinstance(o, o2) */
             int is_meth = is_method(stack_pointer, oparg);
             int total_args = oparg + is_meth;
@@ -3360,7 +3354,7 @@ dummy_func(
         // stack effect: (__0, __array[oparg] -- )
         inst(CALL_NO_KW_LIST_APPEND) {
             assert(cframe.use_tracing == 0);
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             assert(oparg == 1);
             PyObject *callable = PEEK(3);
             PyInterpreterState *interp = _PyInterpreterState_GET();
@@ -3382,7 +3376,7 @@ dummy_func(
 
         // stack effect: (__0, __array[oparg] -- )
         inst(CALL_NO_KW_METHOD_DESCRIPTOR_O) {
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             int is_meth = is_method(stack_pointer, oparg);
             int total_args = oparg + is_meth;
             PyMethodDescrObject *callable =
@@ -3435,9 +3429,9 @@ dummy_func(
             _PyCFunctionFastWithKeywords cfunc =
                 (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
             PyObject *res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(),
-                                  call_shape.kwnames);
+                                  kwnames);
             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
-            call_shape.kwnames = NULL;
+            kwnames = NULL;
 
             /* Free the arguments. */
             for (int i = 0; i < nargs; i++) {
@@ -3455,7 +3449,7 @@ dummy_func(
 
         // stack effect: (__0, __array[oparg] -- )
         inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) {
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             assert(oparg == 0 || oparg == 1);
             int is_meth = is_method(stack_pointer, oparg);
             int total_args = oparg + is_meth;
@@ -3489,7 +3483,7 @@ dummy_func(
 
         // stack effect: (__0, __array[oparg] -- )
         inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) {
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             int is_meth = is_method(stack_pointer, oparg);
             int total_args = oparg + is_meth;
             PyMethodDescrObject *callable =
@@ -3606,25 +3600,12 @@ dummy_func(
             gen->gi_frame_state = FRAME_CREATED;
             gen_frame->owner = FRAME_OWNED_BY_GENERATOR;
             _Py_LeaveRecursiveCallPy(tstate);
-            if (!frame->is_entry) {
+            assert(frame != &entry_frame);
-                _PyInterpreterFrame *prev = frame->previous;
+            _PyInterpreterFrame *prev = frame->previous;
-                _PyThreadState_PopFrame(tstate, frame);
+            _PyThreadState_PopFrame(tstate, frame);
-                frame = cframe.current_frame = prev;
+            frame = cframe.current_frame = prev;
-                _PyFrame_StackPush(frame, (PyObject *)gen);
+            _PyFrame_StackPush(frame, (PyObject *)gen);
-                goto resume_frame;
+            goto resume_frame;
-            }
-            _Py_LeaveRecursiveCallTstate(tstate);
-            /* Make sure that frame is in a valid state */
-            frame->stacktop = 0;
-            frame->f_locals = NULL;
-            Py_INCREF(frame->f_funcobj);
-            Py_INCREF(frame->f_code);
-            /* Restore previous cframe and return. */
-            tstate->cframe = cframe.previous;
-            tstate->cframe->use_tracing = cframe.use_tracing;
-            assert(tstate->cframe->current_frame == frame->previous);
-            assert(!_PyErr_Occurred(tstate));
-            return (PyObject *)gen;
         }
 
         // error: BUILD_SLICE has irregular stack effect

Python/ceval.c

@@ -155,7 +155,10 @@ static void
 lltrace_resume_frame(_PyInterpreterFrame *frame)
 {
     PyObject *fobj = frame->f_funcobj;
-    if (fobj == NULL || !PyFunction_Check(fobj)) {
+    if (frame->owner == FRAME_OWNED_BY_CSTACK ||
+        fobj == NULL ||
+        !PyFunction_Check(fobj)
+    ) {
         printf("\nResuming frame.");
         return;
     }
@@ -733,13 +736,13 @@ GETITEM(PyObject *v, Py_ssize_t i) {
 /* Code access macros */
 
 /* The integer overflow is checked by an assertion below. */
-#define INSTR_OFFSET() ((int)(next_instr - first_instr))
+#define INSTR_OFFSET() ((int)(next_instr - _PyCode_CODE(frame->f_code)))
 #define NEXTOPARG()  do { \
         _Py_CODEUNIT word = *next_instr; \
         opcode = _Py_OPCODE(word); \
         oparg = _Py_OPARG(word); \
     } while (0)
-#define JUMPTO(x)       (next_instr = first_instr + (x))
+#define JUMPTO(x)       (next_instr = _PyCode_CODE(frame->f_code) + (x))
 #define JUMPBY(x)       (next_instr += (x))
 
 /* OpCode prediction macros
@@ -1037,18 +1040,11 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate)  {
 }
 
 
-/* It is only between the KW_NAMES instruction and the following CALL,
- * that this has any meaning.
- */
-typedef struct {
-    PyObject *kwnames;
-} CallShape;
-
 // GH-89279: Must be a macro to be sure it's inlined by MSVC.
 #define is_method(stack_pointer, args) (PEEK((args)+2) != NULL)
 
 #define KWNAMES_LEN() \
-    (call_shape.kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(call_shape.kwnames)))
+    (kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(kwnames)))
 
 PyObject* _Py_HOT_FUNCTION
 _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag)
@@ -1074,8 +1070,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
 #endif
 
     _PyCFrame cframe;
-    CallShape call_shape;
+    _PyInterpreterFrame  entry_frame;
-    call_shape.kwnames = NULL; // Borrowed reference. Reset by CALL instructions.
+    PyObject *kwnames = NULL; // Borrowed reference. Reset by CALL instructions.
 
     /* WARNING: Because the _PyCFrame lives on the C stack,
      * but can be accessed from a heap allocated object (tstate)
@@ -1086,9 +1082,24 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
     cframe.previous = prev_cframe;
     tstate->cframe = &cframe;
 
-    frame->is_entry = true;
+    assert(tstate->interp->interpreter_trampoline != NULL);
+#ifdef Py_DEBUG
+    /* Set these to invalid but identifiable values for debugging. */
+    entry_frame.f_funcobj = (PyObject*)0xaaa0;
+    entry_frame.f_locals = (PyObject*)0xaaa1;
+    entry_frame.frame_obj = (PyFrameObject*)0xaaa2;
+    entry_frame.f_globals = (PyObject*)0xaaa3;
+    entry_frame.f_builtins = (PyObject*)0xaaa4;
+#endif
+    entry_frame.f_code = tstate->interp->interpreter_trampoline;
+    entry_frame.prev_instr =
+        _PyCode_CODE(tstate->interp->interpreter_trampoline);
+    entry_frame.stacktop = 0;
+    entry_frame.owner = FRAME_OWNED_BY_CSTACK;
+    entry_frame.yield_offset = 0;
     /* Push frame */
-    frame->previous = prev_cframe->current_frame;
+    entry_frame.previous = prev_cframe->current_frame;
+    frame->previous = &entry_frame;
     cframe.current_frame = frame;
 
     if (_Py_EnterRecursiveCallTstate(tstate, "")) {
@@ -1112,7 +1123,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
 
     PyObject *names;
     PyObject *consts;
-    _Py_CODEUNIT *first_instr;
     _Py_CODEUNIT *next_instr;
     PyObject **stack_pointer;
 
@@ -1122,7 +1132,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
         PyCodeObject *co = frame->f_code; \
         names = co->co_names; \
         consts = co->co_consts; \
-        first_instr = _PyCode_CODE(co); \
     } \
     assert(_PyInterpreterFrame_LASTI(frame) >= -1); \
     /* Jump back to the last instruction executed... */ \
@@ -1147,14 +1156,16 @@ resume_frame:
 
 #ifdef LLTRACE
     {
-        int r = PyDict_Contains(GLOBALS(), &_Py_ID(__lltrace__));
+        if (frame != &entry_frame) {
-        if (r < 0) {
+            int r = PyDict_Contains(GLOBALS(), &_Py_ID(__lltrace__));
-            goto exit_unwind;
+            if (r < 0) {
+                goto exit_unwind;
+            }
+            lltrace = r;
+        }
+        if (lltrace) {
+            lltrace_resume_frame(frame);
         }
-        lltrace = r;
-    }
-    if (lltrace) {
-        lltrace_resume_frame(frame);
     }
 #endif
 
@@ -1313,7 +1324,7 @@ pop_2_error:
 pop_1_error:
     STACK_SHRINK(1);
 error:
-        call_shape.kwnames = NULL;
+        kwnames = NULL;
         /* Double-check exception status. */
 #ifdef NDEBUG
         if (!_PyErr_Occurred(tstate)) {
@@ -1325,6 +1336,7 @@ error:
 #endif
 
         /* Log traceback info. */
+        assert(frame != &entry_frame);
         if (!_PyFrame_IsIncomplete(frame)) {
             PyFrameObject *f = _PyFrame_GetFrameObject(frame);
             if (f != NULL) {
@@ -1397,12 +1409,9 @@ exception_unwind:
 exit_unwind:
     assert(_PyErr_Occurred(tstate));
     _Py_LeaveRecursiveCallPy(tstate);
-    if (frame->is_entry) {
+    assert(frame != &entry_frame);
-        if (frame->owner == FRAME_OWNED_BY_GENERATOR) {
+    frame = cframe.current_frame = pop_frame(tstate, frame);
-            PyGenObject *gen = _PyFrame_GetGenerator(frame);
+    if (frame == &entry_frame) {
-            tstate->exc_info = gen->gi_exc_state.previous_item;
-            gen->gi_exc_state.previous_item = NULL;
-        }
         /* Restore previous cframe and exit */
         tstate->cframe = cframe.previous;
         tstate->cframe->use_tracing = cframe.use_tracing;
@@ -1410,7 +1419,6 @@ exit_unwind:
         _Py_LeaveRecursiveCallTstate(tstate);
         return NULL;
     }
-    frame = cframe.current_frame = pop_frame(tstate, frame);
 
 resume_with_error:
     SET_LOCALS_FROM_FRAME();
@@ -2038,13 +2046,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func,
         return NULL;
     }
     EVAL_CALL_STAT_INC(EVAL_CALL_VECTOR);
-    PyObject *retval = _PyEval_EvalFrame(tstate, frame, 0);
+    return _PyEval_EvalFrame(tstate, frame, 0);
-    assert(
-        _PyFrame_GetStackPointer(frame) == _PyFrame_Stackbase(frame) ||
-        _PyFrame_GetStackPointer(frame) == frame->localsplus
-    );
-    _PyEvalFrameClearAndPop(tstate, frame);
-    return retval;
 }
 
 /* Legacy API */

Python/compile.c

@@ -1256,6 +1256,8 @@ stack_effect(int opcode, int oparg, int jump)
             return 1;
         case BINARY_OP:
             return -1;
+        case INTERPRETER_EXIT:
+            return -1;
         default:
             return PY_INVALID_STACK_EFFECT;
     }

Python/frame.c

@@ -80,6 +80,7 @@ _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest)
 static void
 take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame)
 {
+    assert(frame->owner != FRAME_OWNED_BY_CSTACK);
     assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT);
     assert(frame->owner != FRAME_CLEARED);
     Py_ssize_t size = ((char*)&frame->localsplus[frame->stacktop]) - (char *)frame;
@@ -99,7 +100,9 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame)
     while (prev && _PyFrame_IsIncomplete(prev)) {
         prev = prev->previous;
     }
+    frame->previous = NULL;
     if (prev) {
+        assert(prev->owner != FRAME_OWNED_BY_CSTACK);
         /* Link PyFrameObjects.f_back and remove link through _PyInterpreterFrame.previous */
         PyFrameObject *back = _PyFrame_GetFrameObject(prev);
         if (back == NULL) {
@@ -111,7 +114,6 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame)
         else {
             f->f_back = (PyFrameObject *)Py_NewRef(back);
         }
-        frame->previous = NULL;
     }
     if (!_PyObject_GC_IS_TRACKED((PyObject *)f)) {
         _PyObject_GC_TRACK((PyObject *)f);

Python/generated_cases.c.h

@@ -611,6 +611,20 @@
             goto error;
         }
 
+        TARGET(INTERPRETER_EXIT) {
+            assert(frame == &entry_frame);
+            assert(_PyFrame_IsIncomplete(frame));
+            PyObject *retval = POP();
+            assert(EMPTY());
+            /* Restore previous cframe and return. */
+            tstate->cframe = cframe.previous;
+            tstate->cframe->use_tracing = cframe.use_tracing;
+            assert(tstate->cframe->current_frame == frame->previous);
+            assert(!_PyErr_Occurred(tstate));
+            _Py_LeaveRecursiveCallTstate(tstate);
+            return retval;
+        }
+
         TARGET(RETURN_VALUE) {
             PyObject *retval = POP();
             assert(EMPTY());
@@ -618,23 +632,10 @@
             TRACE_FUNCTION_EXIT();
             DTRACE_FUNCTION_EXIT();
             _Py_LeaveRecursiveCallPy(tstate);
-            if (!frame->is_entry) {
+            assert(frame != &entry_frame);
-                frame = cframe.current_frame = pop_frame(tstate, frame);
+            frame = cframe.current_frame = pop_frame(tstate, frame);
-                _PyFrame_StackPush(frame, retval);
+            _PyFrame_StackPush(frame, retval);
-                goto resume_frame;
+            goto resume_frame;
-            }
-            _Py_LeaveRecursiveCallTstate(tstate);
-            if (frame->owner == FRAME_OWNED_BY_GENERATOR) {
-                PyGenObject *gen = _PyFrame_GetGenerator(frame);
-                tstate->exc_info = gen->gi_exc_state.previous_item;
-                gen->gi_exc_state.previous_item = NULL;
-            }
-            /* Restore previous cframe and return. */
-            tstate->cframe = cframe.previous;
-            tstate->cframe->use_tracing = cframe.use_tracing;
-            assert(tstate->cframe->current_frame == frame->previous);
-            assert(!_PyErr_Occurred(tstate));
-            return retval;
         }
 
         TARGET(GET_AITER) {
@@ -768,6 +769,7 @@
         }
 
         TARGET(SEND) {
+            assert(frame != &entry_frame);
             assert(STACK_LEVEL() >= 2);
             PyObject *v = POP();
             PyObject *receiver = TOP();
@@ -832,6 +834,7 @@
             // The compiler treats any exception raised here as a failed close()
             // or throw() call.
             assert(oparg == STACK_LEVEL());
+            assert(frame != &entry_frame);
             PyObject *retval = POP();
             PyGenObject *gen = _PyFrame_GetGenerator(frame);
             gen->gi_frame_state = FRAME_SUSPENDED;
@@ -841,19 +844,12 @@
             tstate->exc_info = gen->gi_exc_state.previous_item;
             gen->gi_exc_state.previous_item = NULL;
             _Py_LeaveRecursiveCallPy(tstate);
-            if (!frame->is_entry) {
+            _PyInterpreterFrame *gen_frame = frame;
-                frame = cframe.current_frame = frame->previous;
+            frame = cframe.current_frame = frame->previous;
-                frame->prev_instr -= frame->yield_offset;
+            gen_frame->previous = NULL;
-                _PyFrame_StackPush(frame, retval);
+            frame->prev_instr -= frame->yield_offset;
-                goto resume_frame;
+            _PyFrame_StackPush(frame, retval);
-            }
+            goto resume_frame;
-            _Py_LeaveRecursiveCallTstate(tstate);
-            /* Restore previous cframe and return. */
-            tstate->cframe = cframe.previous;
-            tstate->cframe->use_tracing = cframe.use_tracing;
-            assert(tstate->cframe->current_frame == frame->previous);
-            assert(!_PyErr_Occurred(tstate));
-            return retval;
         }
 
         TARGET(POP_EXCEPT) {
@@ -868,7 +864,7 @@
             if (oparg) {
                 PyObject *lasti = PEEK(oparg + 1);
                 if (PyLong_Check(lasti)) {
-                    frame->prev_instr = first_instr + PyLong_AsLong(lasti);
+                    frame->prev_instr = _PyCode_CODE(frame->f_code) + PyLong_AsLong(lasti);
                     assert(!_PyErr_Occurred(tstate));
                 }
                 else {
@@ -2690,7 +2686,6 @@
             gen->gi_exc_state.previous_item = tstate->exc_info;
             tstate->exc_info = &gen->gi_exc_state;
             gen_frame->previous = frame;
-            gen_frame->is_entry = false;
             frame = cframe.current_frame = gen_frame;
             goto start_frame;
         }
@@ -2919,9 +2914,9 @@
         }
 
         TARGET(KW_NAMES) {
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             assert(oparg < PyTuple_GET_SIZE(consts));
-            call_shape.kwnames = GETITEM(consts, oparg);
+            kwnames = GETITEM(consts, oparg);
             DISPATCH();
         }
 
@@ -2934,7 +2929,7 @@
                 int nargs = oparg + is_meth;
                 PyObject *callable = PEEK(nargs + 1);
                 next_instr--;
-                _Py_Specialize_Call(callable, next_instr, nargs, call_shape.kwnames);
+                _Py_Specialize_Call(callable, next_instr, nargs, kwnames);
                 DISPATCH_SAME_OPARG();
             }
             STAT_INC(CALL, deferred);
@@ -2963,9 +2958,9 @@
                 STACK_SHRINK(total_args);
                 _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
                     tstate, (PyFunctionObject *)function, locals,
-                    stack_pointer, positional_args, call_shape.kwnames
+                    stack_pointer, positional_args, kwnames
                 );
-                call_shape.kwnames = NULL;
+                kwnames = NULL;
                 STACK_SHRINK(2-is_meth);
                 // The frame has stolen all the arguments from the stack,
                 // so there is no need to clean them up.
@@ -2985,15 +2980,15 @@
             if (cframe.use_tracing) {
                 res = trace_call_function(
                     tstate, function, stack_pointer-total_args,
-                    positional_args, call_shape.kwnames);
+                    positional_args, kwnames);
             }
             else {
                 res = PyObject_Vectorcall(
                     function, stack_pointer-total_args,
                     positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
-                    call_shape.kwnames);
+                    kwnames);
             }
-            call_shape.kwnames = NULL;
+            kwnames = NULL;
             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
             Py_DECREF(function);
             /* Clear the stack */
@@ -3013,7 +3008,7 @@
 
         TARGET(CALL_PY_EXACT_ARGS) {
             PREDICTED(CALL_PY_EXACT_ARGS);
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             DEOPT_IF(tstate->interp->eval_frame, CALL);
             _PyCallCache *cache = (_PyCallCache *)next_instr;
             int is_meth = is_method(stack_pointer, oparg);
@@ -3045,7 +3040,7 @@
         }
 
         TARGET(CALL_PY_WITH_DEFAULTS) {
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             DEOPT_IF(tstate->interp->eval_frame, CALL);
             _PyCallCache *cache = (_PyCallCache *)next_instr;
             int is_meth = is_method(stack_pointer, oparg);
@@ -3084,7 +3079,7 @@
         }
 
         TARGET(CALL_NO_KW_TYPE_1) {
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             assert(cframe.use_tracing == 0);
             assert(oparg == 1);
             DEOPT_IF(is_method(stack_pointer, 1), CALL);
@@ -3102,7 +3097,7 @@
         }
 
         TARGET(CALL_NO_KW_STR_1) {
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             assert(cframe.use_tracing == 0);
             assert(oparg == 1);
             DEOPT_IF(is_method(stack_pointer, 1), CALL);
@@ -3124,7 +3119,7 @@
         }
 
         TARGET(CALL_NO_KW_TUPLE_1) {
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             assert(oparg == 1);
             DEOPT_IF(is_method(stack_pointer, 1), CALL);
             PyObject *callable = PEEK(2);
@@ -3156,8 +3151,8 @@
             JUMPBY(INLINE_CACHE_ENTRIES_CALL);
             STACK_SHRINK(total_args);
             PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer,
-                                              total_args-kwnames_len, call_shape.kwnames);
+                                              total_args-kwnames_len, kwnames);
-            call_shape.kwnames = NULL;
+            kwnames = NULL;
             /* Free the arguments. */
             for (int i = 0; i < total_args; i++) {
                 Py_DECREF(stack_pointer[i]);
@@ -3175,7 +3170,7 @@
         TARGET(CALL_NO_KW_BUILTIN_O) {
             assert(cframe.use_tracing == 0);
             /* Builtin METH_O functions */
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             int is_meth = is_method(stack_pointer, oparg);
             int total_args = oparg + is_meth;
             DEOPT_IF(total_args != 1, CALL);
@@ -3209,7 +3204,7 @@
         TARGET(CALL_NO_KW_BUILTIN_FAST) {
             assert(cframe.use_tracing == 0);
             /* Builtin METH_FASTCALL functions, without keywords */
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             int is_meth = is_method(stack_pointer, oparg);
             int total_args = oparg + is_meth;
             PyObject *callable = PEEK(total_args + 1);
@@ -3266,10 +3261,10 @@
                 PyCFunction_GET_SELF(callable),
                 stack_pointer,
                 total_args - KWNAMES_LEN(),
-                call_shape.kwnames
+                kwnames
             );
             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
-            call_shape.kwnames = NULL;
+            kwnames = NULL;
 
             /* Free the arguments. */
             for (int i = 0; i < total_args; i++) {
@@ -3287,7 +3282,7 @@
 
         TARGET(CALL_NO_KW_LEN) {
             assert(cframe.use_tracing == 0);
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             /* len(o) */
             int is_meth = is_method(stack_pointer, oparg);
             int total_args = oparg + is_meth;
@@ -3317,7 +3312,7 @@
 
         TARGET(CALL_NO_KW_ISINSTANCE) {
             assert(cframe.use_tracing == 0);
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             /* isinstance(o, o2) */
             int is_meth = is_method(stack_pointer, oparg);
             int total_args = oparg + is_meth;
@@ -3350,7 +3345,7 @@
 
         TARGET(CALL_NO_KW_LIST_APPEND) {
             assert(cframe.use_tracing == 0);
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             assert(oparg == 1);
             PyObject *callable = PEEK(3);
             PyInterpreterState *interp = _PyInterpreterState_GET();
@@ -3372,7 +3367,7 @@
         }
 
         TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) {
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             int is_meth = is_method(stack_pointer, oparg);
             int total_args = oparg + is_meth;
             PyMethodDescrObject *callable =
@@ -3425,9 +3420,9 @@
             _PyCFunctionFastWithKeywords cfunc =
                 (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
             PyObject *res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(),
-                                  call_shape.kwnames);
+                                  kwnames);
             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
-            call_shape.kwnames = NULL;
+            kwnames = NULL;
 
             /* Free the arguments. */
             for (int i = 0; i < nargs; i++) {
@@ -3445,7 +3440,7 @@
         }
 
         TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) {
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             assert(oparg == 0 || oparg == 1);
             int is_meth = is_method(stack_pointer, oparg);
             int total_args = oparg + is_meth;
@@ -3479,7 +3474,7 @@
         }
 
         TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) {
-            assert(call_shape.kwnames == NULL);
+            assert(kwnames == NULL);
             int is_meth = is_method(stack_pointer, oparg);
             int total_args = oparg + is_meth;
             PyMethodDescrObject *callable =
@@ -3597,25 +3592,12 @@
             gen->gi_frame_state = FRAME_CREATED;
             gen_frame->owner = FRAME_OWNED_BY_GENERATOR;
             _Py_LeaveRecursiveCallPy(tstate);
-            if (!frame->is_entry) {
+            assert(frame != &entry_frame);
-                _PyInterpreterFrame *prev = frame->previous;
+            _PyInterpreterFrame *prev = frame->previous;
-                _PyThreadState_PopFrame(tstate, frame);
+            _PyThreadState_PopFrame(tstate, frame);
-                frame = cframe.current_frame = prev;
+            frame = cframe.current_frame = prev;
-                _PyFrame_StackPush(frame, (PyObject *)gen);
+            _PyFrame_StackPush(frame, (PyObject *)gen);
-                goto resume_frame;
+            goto resume_frame;
-            }
-            _Py_LeaveRecursiveCallTstate(tstate);
-            /* Make sure that frame is in a valid state */
-            frame->stacktop = 0;
-            frame->f_locals = NULL;
-            Py_INCREF(frame->f_funcobj);
-            Py_INCREF(frame->f_code);
-            /* Restore previous cframe and return. */
-            tstate->cframe = cframe.previous;
-            tstate->cframe->use_tracing = cframe.use_tracing;
-            assert(tstate->cframe->current_frame == frame->previous);
-            assert(!_PyErr_Occurred(tstate));
-            return (PyObject *)gen;
         }
 
         TARGET(BUILD_SLICE) {

Python/opcode_targets.h

@@ -2,19 +2,20 @@ static void *opcode_targets[256] = {
     &&TARGET_CACHE,
     &&TARGET_POP_TOP,
     &&TARGET_PUSH_NULL,
-    &&TARGET_BINARY_OP_ADD_FLOAT,
+    &&TARGET_INTERPRETER_EXIT,
     &&TARGET_END_FOR,
+    &&TARGET_BINARY_OP_ADD_FLOAT,
     &&TARGET_BINARY_OP_ADD_INT,
     &&TARGET_BINARY_OP_ADD_UNICODE,
     &&TARGET_BINARY_OP_GENERIC,
-    &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE,
     &&TARGET_NOP,
     &&TARGET_UNARY_POSITIVE,
     &&TARGET_UNARY_NEGATIVE,
     &&TARGET_UNARY_NOT,
+    &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE,
     &&TARGET_BINARY_OP_MULTIPLY_FLOAT,
-    &&TARGET_BINARY_OP_MULTIPLY_INT,
     &&TARGET_UNARY_INVERT,
+    &&TARGET_BINARY_OP_MULTIPLY_INT,
     &&TARGET_BINARY_OP_SUBTRACT_FLOAT,
     &&TARGET_BINARY_OP_SUBTRACT_INT,
     &&TARGET_BINARY_SUBSCR_DICT,
@@ -23,20 +24,20 @@ static void *opcode_targets[256] = {
     &&TARGET_BINARY_SUBSCR_TUPLE_INT,
     &&TARGET_CALL_PY_EXACT_ARGS,
     &&TARGET_CALL_PY_WITH_DEFAULTS,
-    &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS,
     &&TARGET_BINARY_SUBSCR,
     &&TARGET_BINARY_SLICE,
     &&TARGET_STORE_SLICE,
+    &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS,
     &&TARGET_CALL_BUILTIN_CLASS,
-    &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS,
     &&TARGET_GET_LEN,
     &&TARGET_MATCH_MAPPING,
     &&TARGET_MATCH_SEQUENCE,
     &&TARGET_MATCH_KEYS,
-    &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS,
+    &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS,
     &&TARGET_PUSH_EXC_INFO,
     &&TARGET_CHECK_EXC_MATCH,
     &&TARGET_CHECK_EG_MATCH,
+    &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS,
     &&TARGET_CALL_NO_KW_BUILTIN_FAST,
     &&TARGET_CALL_NO_KW_BUILTIN_O,
     &&TARGET_CALL_NO_KW_ISINSTANCE,
@@ -47,7 +48,6 @@ static void *opcode_targets[256] = {
     &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O,
     &&TARGET_CALL_NO_KW_STR_1,
     &&TARGET_CALL_NO_KW_TUPLE_1,
-    &&TARGET_CALL_NO_KW_TYPE_1,
     &&TARGET_WITH_EXCEPT_START,
     &&TARGET_GET_AITER,
     &&TARGET_GET_ANEXT,
@@ -55,37 +55,37 @@ static void *opcode_targets[256] = {
     &&TARGET_BEFORE_WITH,
     &&TARGET_END_ASYNC_FOR,
     &&TARGET_CLEANUP_THROW,
+    &&TARGET_CALL_NO_KW_TYPE_1,
     &&TARGET_COMPARE_OP_FLOAT_JUMP,
     &&TARGET_COMPARE_OP_GENERIC,
     &&TARGET_COMPARE_OP_INT_JUMP,
-    &&TARGET_COMPARE_OP_STR_JUMP,
     &&TARGET_STORE_SUBSCR,
     &&TARGET_DELETE_SUBSCR,
-    &&TARGET_FOR_ITER_LIST,
+    &&TARGET_COMPARE_OP_STR_JUMP,
     &&TARGET_STOPITERATION_ERROR,
+    &&TARGET_FOR_ITER_LIST,
     &&TARGET_FOR_ITER_RANGE,
     &&TARGET_FOR_ITER_GEN,
     &&TARGET_LOAD_ATTR_CLASS,
-    &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN,
     &&TARGET_GET_ITER,
     &&TARGET_GET_YIELD_FROM_ITER,
     &&TARGET_PRINT_EXPR,
     &&TARGET_LOAD_BUILD_CLASS,
+    &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN,
     &&TARGET_LOAD_ATTR_INSTANCE_VALUE,
-    &&TARGET_LOAD_ATTR_MODULE,
     &&TARGET_LOAD_ASSERTION_ERROR,
     &&TARGET_RETURN_GENERATOR,
+    &&TARGET_LOAD_ATTR_MODULE,
     &&TARGET_LOAD_ATTR_PROPERTY,
     &&TARGET_LOAD_ATTR_SLOT,
     &&TARGET_LOAD_ATTR_WITH_HINT,
     &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT,
     &&TARGET_LOAD_ATTR_METHOD_NO_DICT,
-    &&TARGET_LOAD_ATTR_METHOD_WITH_DICT,
     &&TARGET_LIST_TO_TUPLE,
     &&TARGET_RETURN_VALUE,
     &&TARGET_IMPORT_STAR,
     &&TARGET_SETUP_ANNOTATIONS,
-    &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES,
+    &&TARGET_LOAD_ATTR_METHOD_WITH_DICT,
     &&TARGET_ASYNC_GEN_WRAP,
     &&TARGET_PREP_RERAISE_STAR,
     &&TARGET_POP_EXCEPT,
@@ -112,7 +112,7 @@ static void *opcode_targets[256] = {
     &&TARGET_JUMP_FORWARD,
     &&TARGET_JUMP_IF_FALSE_OR_POP,
     &&TARGET_JUMP_IF_TRUE_OR_POP,
-    &&TARGET_LOAD_CONST__LOAD_FAST,
+    &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES,
     &&TARGET_POP_JUMP_IF_FALSE,
     &&TARGET_POP_JUMP_IF_TRUE,
     &&TARGET_LOAD_GLOBAL,
@@ -120,7 +120,7 @@ static void *opcode_targets[256] = {
     &&TARGET_CONTAINS_OP,
     &&TARGET_RERAISE,
     &&TARGET_COPY,
-    &&TARGET_LOAD_FAST__LOAD_CONST,
+    &&TARGET_LOAD_CONST__LOAD_FAST,
     &&TARGET_BINARY_OP,
     &&TARGET_SEND,
     &&TARGET_LOAD_FAST,
@@ -140,9 +140,9 @@ static void *opcode_targets[256] = {
     &&TARGET_STORE_DEREF,
     &&TARGET_DELETE_DEREF,
     &&TARGET_JUMP_BACKWARD,
-    &&TARGET_LOAD_FAST__LOAD_FAST,
+    &&TARGET_LOAD_FAST__LOAD_CONST,
     &&TARGET_CALL_FUNCTION_EX,
-    &&TARGET_LOAD_GLOBAL_BUILTIN,
+    &&TARGET_LOAD_FAST__LOAD_FAST,
     &&TARGET_EXTENDED_ARG,
     &&TARGET_LIST_APPEND,
     &&TARGET_SET_ADD,
@@ -152,27 +152,27 @@ static void *opcode_targets[256] = {
     &&TARGET_YIELD_VALUE,
     &&TARGET_RESUME,
     &&TARGET_MATCH_CLASS,
+    &&TARGET_LOAD_GLOBAL_BUILTIN,
     &&TARGET_LOAD_GLOBAL_MODULE,
-    &&TARGET_STORE_ATTR_INSTANCE_VALUE,
     &&TARGET_FORMAT_VALUE,
     &&TARGET_BUILD_CONST_KEY_MAP,
     &&TARGET_BUILD_STRING,
+    &&TARGET_STORE_ATTR_INSTANCE_VALUE,
     &&TARGET_STORE_ATTR_SLOT,
     &&TARGET_STORE_ATTR_WITH_HINT,
     &&TARGET_STORE_FAST__LOAD_FAST,
-    &&TARGET_STORE_FAST__STORE_FAST,
     &&TARGET_LIST_EXTEND,
     &&TARGET_SET_UPDATE,
     &&TARGET_DICT_MERGE,
     &&TARGET_DICT_UPDATE,
+    &&TARGET_STORE_FAST__STORE_FAST,
     &&TARGET_STORE_SUBSCR_DICT,
     &&TARGET_STORE_SUBSCR_LIST_INT,
     &&TARGET_UNPACK_SEQUENCE_LIST,
     &&TARGET_UNPACK_SEQUENCE_TUPLE,
-    &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE,
     &&TARGET_CALL,
     &&TARGET_KW_NAMES,
-    &&_unknown_opcode,
+    &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE,
     &&_unknown_opcode,
     &&_unknown_opcode,
     &&_unknown_opcode,

Python/pylifecycle.c

@@ -29,6 +29,7 @@
 #include "pycore_tuple.h"         // _PyTuple_InitTypes()
 #include "pycore_typeobject.h"    // _PyTypes_InitTypes()
 #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes()
+#include "opcode.h"
 
 extern void _PyIO_Fini(void);
 
@@ -779,6 +780,21 @@ pycore_init_types(PyInterpreterState *interp)
     return _PyStatus_OK();
 }
 
+static const uint8_t INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = {
+    /* Put a NOP at the start, so that the IP points into
+    * the code, rather than before it */
+    NOP, 0,
+    INTERPRETER_EXIT, 0,
+    /* RESUME at end makes sure that the frame appears incomplete */
+    RESUME, 0
+};
+
+static const _PyShimCodeDef INTERPRETER_TRAMPOLINE_CODEDEF = {
+    INTERPRETER_TRAMPOLINE_INSTRUCTIONS,
+    sizeof(INTERPRETER_TRAMPOLINE_INSTRUCTIONS),
+    1,
+    "<interpreter trampoline>"
+};
 
 static PyStatus
 pycore_init_builtins(PyThreadState *tstate)
@@ -812,7 +828,10 @@ pycore_init_builtins(PyThreadState *tstate)
     PyObject *object__getattribute__ = _PyType_Lookup(&PyBaseObject_Type, &_Py_ID(__getattribute__));
     assert(object__getattribute__);
     interp->callable_cache.object__getattribute__ = object__getattribute__;
-
+    interp->interpreter_trampoline = _Py_MakeShimCode(&INTERPRETER_TRAMPOLINE_CODEDEF);
+    if (interp->interpreter_trampoline == NULL) {
+        return _PyStatus_ERR("failed to create interpreter trampoline.");
+    }
     if (_PyBuiltins_AddExceptions(bimod) < 0) {
         return _PyStatus_ERR("failed to add exceptions to builtins");
     }

Python/pystate.c

@@ -454,6 +454,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate)
     PyDict_Clear(interp->builtins);
     Py_CLEAR(interp->sysdict);
     Py_CLEAR(interp->builtins);
+    Py_CLEAR(interp->interpreter_trampoline);
 
     for (int i=0; i < DICT_MAX_WATCHERS; i++) {
         interp->dict_watchers[i] = NULL;

Python/traceback.c

@@ -1224,6 +1224,15 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header)
         if (frame == NULL) {
             break;
         }
+        if (frame->owner == FRAME_OWNED_BY_CSTACK) {
+            /* Trampoline frame */
+            frame = frame->previous;
+        }
+        if (frame == NULL) {
+            break;
+        }
+        /* Can't have more than one shim frame in a row */
+        assert(frame->owner != FRAME_OWNED_BY_CSTACK);
         depth++;
     }
 }

Tools/gdb/libpython.py

@@ -82,6 +82,8 @@ Py_TPFLAGS_DICT_SUBCLASS     = (1 << 29)
 Py_TPFLAGS_BASE_EXC_SUBCLASS = (1 << 30)
 Py_TPFLAGS_TYPE_SUBCLASS     = (1 << 31)
 
+#From pycore_frame.h
+FRAME_OWNED_BY_CSTACK = 3
 
 MAX_OUTPUT_LEN=1024
 
@@ -1077,8 +1079,8 @@ class PyFramePtr:
         first_instr = self._f_code().field("co_code_adaptive").cast(codeunit_p)
         return int(prev_instr - first_instr)
 
-    def is_entry(self):
+    def is_shim(self):
-        return self._f_special("is_entry", bool)
+        return self._f_special("owner", int) == FRAME_OWNED_BY_CSTACK
 
     def previous(self):
         return self._f_special("previous", PyFramePtr)
@@ -1821,14 +1823,14 @@ class Frame(object):
             interp_frame = self.get_pyop()
             while True:
                 if interp_frame:
+                    if interp_frame.is_shim():
+                        break
                     line = interp_frame.get_truncated_repr(MAX_OUTPUT_LEN)
                     sys.stdout.write('#%i %s\n' % (self.get_index(), line))
                     if not interp_frame.is_optimized_out():
                         line = interp_frame.current_line()
                         if line is not None:
                             sys.stdout.write('    %s\n' % line.strip())
-                    if interp_frame.is_entry():
-                        break
                 else:
                     sys.stdout.write('#%i (unable to read python frame information)\n' % self.get_index())
                     break
@@ -1845,13 +1847,13 @@ class Frame(object):
             interp_frame = self.get_pyop()
             while True:
                 if interp_frame:
+                    if interp_frame.is_shim():
+                        break
                     interp_frame.print_traceback()
                     if not interp_frame.is_optimized_out():
                         line = interp_frame.current_line()
                         if line is not None:
                             sys.stdout.write('    %s\n' % line.strip())
-                    if interp_frame.is_entry():
-                        break
                 else:
                     sys.stdout.write('  (unable to read python frame information)\n')
                     break
@@ -2106,6 +2108,8 @@ class PyLocals(gdb.Command):
         while True:
             if not pyop_frame:
                 print(UNABLE_READ_INFO_PYTHON_FRAME)
+            if pyop_frame.is_shim():
+                break
 
             sys.stdout.write('Locals for %s\n' % (pyop_frame.co_name.proxyval(set())))
 
@@ -2114,8 +2118,6 @@ class PyLocals(gdb.Command):
                     % (pyop_name.proxyval(set()),
                         pyop_value.get_truncated_repr(MAX_OUTPUT_LEN)))
 
-            if pyop_frame.is_entry():
-                break
 
             pyop_frame = pyop_frame.previous()