diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 2e76e9f64d4..a722a2e2527 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -593,6 +593,10 @@ extern _Py_CODEUNIT _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); extern int _PyInstruction_GetLength(PyCodeObject *code, int offset); +struct _PyCode8 _PyCode_DEF(8); + +PyAPI_DATA(const struct _PyCode8) _Py_InitCleanup; + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index d3d74557466..a6f7c1735b3 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -144,8 +144,9 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * static inline void _PyFrame_Initialize( _PyInterpreterFrame *frame, PyFunctionObject *func, - PyObject *locals, PyCodeObject *code, int null_locals_from) + PyObject *locals, PyCodeObject *code, int null_locals_from, _PyInterpreterFrame *previous) { + frame->previous = previous; frame->f_funcobj = (PyObject *)func; frame->f_executable = Py_NewRef(code); frame->f_builtins = func->func_builtins; @@ -298,26 +299,27 @@ PyAPI_FUNC(void) _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFr * Must be guarded by _PyThreadState_HasStackSpace() * Consumes reference to func. */ static inline _PyInterpreterFrame * -_PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_locals_from) +_PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_locals_from, _PyInterpreterFrame * previous) { CALL_STAT_INC(frames_pushed); PyCodeObject *code = (PyCodeObject *)func->func_code; _PyInterpreterFrame *new_frame = (_PyInterpreterFrame *)tstate->datastack_top; tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); - _PyFrame_Initialize(new_frame, func, NULL, code, null_locals_from); + _PyFrame_Initialize(new_frame, func, NULL, code, null_locals_from, previous); return new_frame; } /* Pushes a trampoline frame without checking for space. * Must be guarded by _PyThreadState_HasStackSpace() */ static inline _PyInterpreterFrame * -_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth) +_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth, _PyInterpreterFrame * previous) { CALL_STAT_INC(frames_pushed); _PyInterpreterFrame *frame = (_PyInterpreterFrame *)tstate->datastack_top; tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); + frame->previous = previous; frame->f_funcobj = Py_None; frame->f_executable = Py_NewRef(code); #ifdef Py_DEBUG @@ -344,7 +346,8 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int PyAPI_FUNC(_PyInterpreterFrame *) _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, _PyStackRef const* args, - size_t argcount, PyObject *kwnames); + size_t argcount, PyObject *kwnames, + _PyInterpreterFrame *previous); #ifdef __cplusplus } diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 2e0b5fa09c8..ee33da77f01 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -761,7 +761,7 @@ static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { } extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); -extern PyObject *_PyType_NewManagedObject(PyTypeObject *type); +PyAPI_FUNC(PyObject *) _PyType_NewManagedObject(PyTypeObject *type); extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 8e38855175a..236e6b41461 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -537,7 +537,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CALL: return 1; case CALL_ALLOC_AND_ENTER_INIT: - return 1; + return 0; case CALL_BOUND_METHOD_EXACT_ARGS: return 0; case CALL_BOUND_METHOD_GENERAL: @@ -1261,6 +1261,7 @@ _PyOpcode_macro_expansion[256] = { [BUILD_SLICE] = { .nuops = 1, .uops = { { _BUILD_SLICE, 0, 0 } } }, [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } }, + [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 4, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_AND_ALLOCATE_OBJECT, 2, 1 }, { _CREATE_INIT_FRAME, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 9, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, 0, 0 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_BOUND_METHOD_GENERAL] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, 0, 0 }, { _PY_FRAME_GENERAL, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 2a255143c3e..226f4a63460 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -34,7 +34,6 @@ extern "C" { #define _BUILD_SLICE BUILD_SLICE #define _BUILD_STRING BUILD_STRING #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT #define _CALL_BUILTIN_CLASS 314 #define _CALL_BUILTIN_FAST 315 #define _CALL_BUILTIN_FAST_WITH_KEYWORDS 316 @@ -53,97 +52,99 @@ extern "C" { #define _CALL_STR_1 324 #define _CALL_TUPLE_1 325 #define _CALL_TYPE_1 CALL_TYPE_1 -#define _CHECK_ATTR_CLASS 326 -#define _CHECK_ATTR_METHOD_LAZY_DICT 327 -#define _CHECK_ATTR_MODULE 328 -#define _CHECK_ATTR_WITH_HINT 329 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 330 +#define _CHECK_AND_ALLOCATE_OBJECT 326 +#define _CHECK_ATTR_CLASS 327 +#define _CHECK_ATTR_METHOD_LAZY_DICT 328 +#define _CHECK_ATTR_MODULE 329 +#define _CHECK_ATTR_WITH_HINT 330 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 331 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 331 -#define _CHECK_FUNCTION_EXACT_ARGS 332 -#define _CHECK_FUNCTION_VERSION 333 -#define _CHECK_FUNCTION_VERSION_KW 334 -#define _CHECK_IS_NOT_PY_CALLABLE 335 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 336 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 337 -#define _CHECK_METHOD_VERSION 338 -#define _CHECK_METHOD_VERSION_KW 339 -#define _CHECK_PEP_523 340 -#define _CHECK_PERIODIC 341 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 342 -#define _CHECK_STACK_SPACE 343 -#define _CHECK_STACK_SPACE_OPERAND 344 -#define _CHECK_VALIDITY 345 -#define _CHECK_VALIDITY_AND_SET_IP 346 -#define _COMPARE_OP 347 -#define _COMPARE_OP_FLOAT 348 -#define _COMPARE_OP_INT 349 -#define _COMPARE_OP_STR 350 -#define _CONTAINS_OP 351 +#define _CHECK_FUNCTION 332 +#define _CHECK_FUNCTION_EXACT_ARGS 333 +#define _CHECK_FUNCTION_VERSION 334 +#define _CHECK_FUNCTION_VERSION_KW 335 +#define _CHECK_IS_NOT_PY_CALLABLE 336 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 337 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 338 +#define _CHECK_METHOD_VERSION 339 +#define _CHECK_METHOD_VERSION_KW 340 +#define _CHECK_PEP_523 341 +#define _CHECK_PERIODIC 342 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 343 +#define _CHECK_STACK_SPACE 344 +#define _CHECK_STACK_SPACE_OPERAND 345 +#define _CHECK_VALIDITY 346 +#define _CHECK_VALIDITY_AND_SET_IP 347 +#define _COMPARE_OP 348 +#define _COMPARE_OP_FLOAT 349 +#define _COMPARE_OP_INT 350 +#define _COMPARE_OP_STR 351 +#define _CONTAINS_OP 352 #define _CONTAINS_OP_DICT CONTAINS_OP_DICT #define _CONTAINS_OP_SET CONTAINS_OP_SET #define _CONVERT_VALUE CONVERT_VALUE #define _COPY COPY #define _COPY_FREE_VARS COPY_FREE_VARS +#define _CREATE_INIT_FRAME 353 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 352 +#define _DEOPT 354 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 353 -#define _DO_CALL_KW 354 -#define _DYNAMIC_EXIT 355 +#define _DO_CALL 355 +#define _DO_CALL_KW 356 +#define _DYNAMIC_EXIT 357 #define _END_SEND END_SEND -#define _ERROR_POP_N 356 +#define _ERROR_POP_N 358 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 357 -#define _EXPAND_METHOD_KW 358 -#define _FATAL_ERROR 359 +#define _EXPAND_METHOD 359 +#define _EXPAND_METHOD_KW 360 +#define _FATAL_ERROR 361 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 360 -#define _FOR_ITER_GEN_FRAME 361 -#define _FOR_ITER_TIER_TWO 362 +#define _FOR_ITER 362 +#define _FOR_ITER_GEN_FRAME 363 +#define _FOR_ITER_TIER_TWO 364 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 363 -#define _GUARD_BOTH_INT 364 -#define _GUARD_BOTH_UNICODE 365 -#define _GUARD_BUILTINS_VERSION 366 -#define _GUARD_DORV_NO_DICT 367 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 368 -#define _GUARD_GLOBALS_VERSION 369 -#define _GUARD_IS_FALSE_POP 370 -#define _GUARD_IS_NONE_POP 371 -#define _GUARD_IS_NOT_NONE_POP 372 -#define _GUARD_IS_TRUE_POP 373 -#define _GUARD_KEYS_VERSION 374 -#define _GUARD_NOS_FLOAT 375 -#define _GUARD_NOS_INT 376 -#define _GUARD_NOT_EXHAUSTED_LIST 377 -#define _GUARD_NOT_EXHAUSTED_RANGE 378 -#define _GUARD_NOT_EXHAUSTED_TUPLE 379 -#define _GUARD_TOS_FLOAT 380 -#define _GUARD_TOS_INT 381 -#define _GUARD_TYPE_VERSION 382 +#define _GUARD_BOTH_FLOAT 365 +#define _GUARD_BOTH_INT 366 +#define _GUARD_BOTH_UNICODE 367 +#define _GUARD_BUILTINS_VERSION 368 +#define _GUARD_DORV_NO_DICT 369 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 370 +#define _GUARD_GLOBALS_VERSION 371 +#define _GUARD_IS_FALSE_POP 372 +#define _GUARD_IS_NONE_POP 373 +#define _GUARD_IS_NOT_NONE_POP 374 +#define _GUARD_IS_TRUE_POP 375 +#define _GUARD_KEYS_VERSION 376 +#define _GUARD_NOS_FLOAT 377 +#define _GUARD_NOS_INT 378 +#define _GUARD_NOT_EXHAUSTED_LIST 379 +#define _GUARD_NOT_EXHAUSTED_RANGE 380 +#define _GUARD_NOT_EXHAUSTED_TUPLE 381 +#define _GUARD_TOS_FLOAT 382 +#define _GUARD_TOS_INT 383 +#define _GUARD_TYPE_VERSION 384 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 383 -#define _INIT_CALL_PY_EXACT_ARGS 384 -#define _INIT_CALL_PY_EXACT_ARGS_0 385 -#define _INIT_CALL_PY_EXACT_ARGS_1 386 -#define _INIT_CALL_PY_EXACT_ARGS_2 387 -#define _INIT_CALL_PY_EXACT_ARGS_3 388 -#define _INIT_CALL_PY_EXACT_ARGS_4 389 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 385 +#define _INIT_CALL_PY_EXACT_ARGS 386 +#define _INIT_CALL_PY_EXACT_ARGS_0 387 +#define _INIT_CALL_PY_EXACT_ARGS_1 388 +#define _INIT_CALL_PY_EXACT_ARGS_2 389 +#define _INIT_CALL_PY_EXACT_ARGS_3 390 +#define _INIT_CALL_PY_EXACT_ARGS_4 391 #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER @@ -155,65 +156,65 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 390 -#define _IS_NONE 391 +#define _INTERNAL_INCREMENT_OPT_COUNTER 392 +#define _IS_NONE 393 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 392 -#define _ITER_CHECK_RANGE 393 -#define _ITER_CHECK_TUPLE 394 -#define _ITER_JUMP_LIST 395 -#define _ITER_JUMP_RANGE 396 -#define _ITER_JUMP_TUPLE 397 -#define _ITER_NEXT_LIST 398 -#define _ITER_NEXT_RANGE 399 -#define _ITER_NEXT_TUPLE 400 -#define _JUMP_TO_TOP 401 +#define _ITER_CHECK_LIST 394 +#define _ITER_CHECK_RANGE 395 +#define _ITER_CHECK_TUPLE 396 +#define _ITER_JUMP_LIST 397 +#define _ITER_JUMP_RANGE 398 +#define _ITER_JUMP_TUPLE 399 +#define _ITER_NEXT_LIST 400 +#define _ITER_NEXT_RANGE 401 +#define _ITER_NEXT_TUPLE 402 +#define _JUMP_TO_TOP 403 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 402 -#define _LOAD_ATTR_CLASS 403 -#define _LOAD_ATTR_CLASS_0 404 -#define _LOAD_ATTR_CLASS_1 405 +#define _LOAD_ATTR 404 +#define _LOAD_ATTR_CLASS 405 +#define _LOAD_ATTR_CLASS_0 406 +#define _LOAD_ATTR_CLASS_1 407 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 406 -#define _LOAD_ATTR_INSTANCE_VALUE_0 407 -#define _LOAD_ATTR_INSTANCE_VALUE_1 408 -#define _LOAD_ATTR_METHOD_LAZY_DICT 409 -#define _LOAD_ATTR_METHOD_NO_DICT 410 -#define _LOAD_ATTR_METHOD_WITH_VALUES 411 -#define _LOAD_ATTR_MODULE 412 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 413 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 414 -#define _LOAD_ATTR_PROPERTY_FRAME 415 -#define _LOAD_ATTR_SLOT 416 -#define _LOAD_ATTR_SLOT_0 417 -#define _LOAD_ATTR_SLOT_1 418 -#define _LOAD_ATTR_WITH_HINT 419 +#define _LOAD_ATTR_INSTANCE_VALUE 408 +#define _LOAD_ATTR_INSTANCE_VALUE_0 409 +#define _LOAD_ATTR_INSTANCE_VALUE_1 410 +#define _LOAD_ATTR_METHOD_LAZY_DICT 411 +#define _LOAD_ATTR_METHOD_NO_DICT 412 +#define _LOAD_ATTR_METHOD_WITH_VALUES 413 +#define _LOAD_ATTR_MODULE 414 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 415 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 416 +#define _LOAD_ATTR_PROPERTY_FRAME 417 +#define _LOAD_ATTR_SLOT 418 +#define _LOAD_ATTR_SLOT_0 419 +#define _LOAD_ATTR_SLOT_1 420 +#define _LOAD_ATTR_WITH_HINT 421 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 420 -#define _LOAD_CONST_INLINE_BORROW 421 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 422 -#define _LOAD_CONST_INLINE_WITH_NULL 423 +#define _LOAD_CONST_INLINE 422 +#define _LOAD_CONST_INLINE_BORROW 423 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 424 +#define _LOAD_CONST_INLINE_WITH_NULL 425 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 424 -#define _LOAD_FAST_0 425 -#define _LOAD_FAST_1 426 -#define _LOAD_FAST_2 427 -#define _LOAD_FAST_3 428 -#define _LOAD_FAST_4 429 -#define _LOAD_FAST_5 430 -#define _LOAD_FAST_6 431 -#define _LOAD_FAST_7 432 +#define _LOAD_FAST 426 +#define _LOAD_FAST_0 427 +#define _LOAD_FAST_1 428 +#define _LOAD_FAST_2 429 +#define _LOAD_FAST_3 430 +#define _LOAD_FAST_4 431 +#define _LOAD_FAST_5 432 +#define _LOAD_FAST_6 433 +#define _LOAD_FAST_7 434 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 433 -#define _LOAD_GLOBAL_BUILTINS 434 -#define _LOAD_GLOBAL_MODULE 435 +#define _LOAD_GLOBAL 435 +#define _LOAD_GLOBAL_BUILTINS 436 +#define _LOAD_GLOBAL_MODULE 437 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SPECIAL LOAD_SPECIAL @@ -226,59 +227,59 @@ extern "C" { #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 436 -#define _MONITOR_CALL 437 -#define _MONITOR_JUMP_BACKWARD 438 -#define _MONITOR_RESUME 439 +#define _MAYBE_EXPAND_METHOD 438 +#define _MONITOR_CALL 439 +#define _MONITOR_JUMP_BACKWARD 440 +#define _MONITOR_RESUME 441 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 440 -#define _POP_JUMP_IF_TRUE 441 +#define _POP_JUMP_IF_FALSE 442 +#define _POP_JUMP_IF_TRUE 443 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 442 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 444 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 443 +#define _PUSH_FRAME 445 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 444 -#define _PY_FRAME_KW 445 -#define _QUICKEN_RESUME 446 -#define _REPLACE_WITH_TRUE 447 +#define _PY_FRAME_GENERAL 446 +#define _PY_FRAME_KW 447 +#define _QUICKEN_RESUME 448 +#define _REPLACE_WITH_TRUE 449 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 448 -#define _SEND 449 -#define _SEND_GEN_FRAME 450 +#define _SAVE_RETURN_OFFSET 450 +#define _SEND 451 +#define _SEND_GEN_FRAME 452 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 451 -#define _STORE_ATTR 452 -#define _STORE_ATTR_INSTANCE_VALUE 453 -#define _STORE_ATTR_SLOT 454 -#define _STORE_ATTR_WITH_HINT 455 +#define _START_EXECUTOR 453 +#define _STORE_ATTR 454 +#define _STORE_ATTR_INSTANCE_VALUE 455 +#define _STORE_ATTR_SLOT 456 +#define _STORE_ATTR_WITH_HINT 457 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 456 -#define _STORE_FAST_0 457 -#define _STORE_FAST_1 458 -#define _STORE_FAST_2 459 -#define _STORE_FAST_3 460 -#define _STORE_FAST_4 461 -#define _STORE_FAST_5 462 -#define _STORE_FAST_6 463 -#define _STORE_FAST_7 464 +#define _STORE_FAST 458 +#define _STORE_FAST_0 459 +#define _STORE_FAST_1 460 +#define _STORE_FAST_2 461 +#define _STORE_FAST_3 462 +#define _STORE_FAST_4 463 +#define _STORE_FAST_5 464 +#define _STORE_FAST_6 465 +#define _STORE_FAST_7 466 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 465 +#define _STORE_SUBSCR 467 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 466 -#define _TO_BOOL 467 +#define _TIER2_RESUME_CHECK 468 +#define _TO_BOOL 469 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -288,14 +289,14 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 468 +#define _UNPACK_SEQUENCE 470 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE #define __DO_CALL_FUNCTION_EX _DO_CALL_FUNCTION_EX -#define MAX_UOP_ID 468 +#define MAX_UOP_ID 470 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 98757b6c8d8..15c0ac11030 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -225,6 +225,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_AND_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -327,6 +329,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CALL_STR_1] = "_CALL_STR_1", [_CALL_TUPLE_1] = "_CALL_TUPLE_1", [_CALL_TYPE_1] = "_CALL_TYPE_1", + [_CHECK_AND_ALLOCATE_OBJECT] = "_CHECK_AND_ALLOCATE_OBJECT", [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", [_CHECK_ATTR_MODULE] = "_CHECK_ATTR_MODULE", @@ -360,6 +363,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CONVERT_VALUE] = "_CONVERT_VALUE", [_COPY] = "_COPY", [_COPY_FREE_VARS] = "_COPY_FREE_VARS", + [_CREATE_INIT_FRAME] = "_CREATE_INIT_FRAME", [_DELETE_ATTR] = "_DELETE_ATTR", [_DELETE_DEREF] = "_DELETE_DEREF", [_DELETE_FAST] = "_DELETE_FAST", @@ -960,6 +964,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 3; case _CALL_TUPLE_1: return 3; + case _CHECK_AND_ALLOCATE_OBJECT: + return 2 + oparg; + case _CREATE_INIT_FRAME: + return 2 + oparg; case _EXIT_INIT_CHECK: return 1; case _CALL_BUILTIN_CLASS: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-19-15-13-13.gh-issue-118093.dLZ8qS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-19-15-13-13.gh-issue-118093.dLZ8qS.rst new file mode 100644 index 00000000000..d8127d8b505 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-19-15-13-13.gh-issue-118093.dLZ8qS.rst @@ -0,0 +1,3 @@ +Break up ``CALL_ALLOC_AND_ENTER_INIT`` into micro-ops and relax +requirement for exact args, in order to increase the amount of code +supported by tier 2. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 621f8996ccb..85c24550d0b 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1796,8 +1796,7 @@ init_frame(_PyInterpreterFrame *frame, PyFunctionObject *func, PyObject *locals) { PyCodeObject *code = (PyCodeObject *)func->func_code; _PyFrame_Initialize(frame, (PyFunctionObject*)Py_NewRef(func), - Py_XNewRef(locals), code, 0); - frame->previous = NULL; + Py_XNewRef(locals), code, 0, NULL); } PyFrameObject* diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ec57c07104d..ec0f6ab8fde 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -797,7 +797,7 @@ dummy_func( PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2); + new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2, frame); SYNC_SP(); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; @@ -1071,6 +1071,8 @@ dummy_func( tstate->exc_info = &gen->gi_exc_state; assert(next_instr - this_instr + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; DISPATCH_INLINED(gen_frame); } if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) { @@ -1113,6 +1115,7 @@ dummy_func( tstate->exc_info = &gen->gi_exc_state; assert(1 + INLINE_CACHE_ENTRIES_SEND + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); + gen_frame->previous = frame; } macro(SEND_GEN) = @@ -2143,7 +2146,7 @@ dummy_func( DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(LOAD_ATTR, hit); Py_INCREF(fget); - new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame); new_frame->localsplus[0] = owner; } @@ -2175,7 +2178,7 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); Py_INCREF(f); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2, frame); // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; @@ -2956,6 +2959,7 @@ dummy_func( gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; + gen_frame->previous = frame; // oparg is the return offset from the next instruction. frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); } @@ -3225,7 +3229,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); @@ -3315,7 +3319,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. @@ -3454,7 +3458,7 @@ dummy_func( int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3468,7 +3472,7 @@ dummy_func( assert(tstate->interp->eval_frame == NULL); SYNC_SP(); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -3550,58 +3554,51 @@ dummy_func( _CALL_TUPLE_1 + _CHECK_PERIODIC; - inst(CALL_ALLOC_AND_ENTER_INIT, (unused/1, unused/2, callable, null, args[oparg] -- unused)) { + op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, null, args[oparg] -- self, init, args[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - /* This instruction does the following: - * 1. Creates the object (by calling ``object.__new__``) - * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) - * 3. Pushes the frame for ``__init__`` to the frame stack - * */ - _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; DEOPT_IF(!PyStackRef_IsNull(null)); DEOPT_IF(!PyType_Check(callable_o)); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version)); + DEOPT_IF(tp->tp_version_tag != type_version); assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; - PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(code->co_argcount != oparg+1); + PyFunctionObject *init_func = (PyFunctionObject *)cls->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init_func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)); STAT_INC(CALL, hit); - PyObject *self = _PyType_NewManagedObject(tp); - if (self == NULL) { + self = PyStackRef_FromPyObjectSteal(_PyType_NewManagedObject(tp)); + if (PyStackRef_IsNull(self)) { ERROR_NO_POP(); } PyStackRef_CLOSE(callable); + init = PyStackRef_FromPyObjectNew(init_func); + } + + op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _PyInterpreterFrame *)) { _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1); + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ - Py_INCREF(self); - shim->localsplus[0] = PyStackRef_FromPyObjectSteal(self); - Py_INCREF(init); - _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); - /* Copy self followed by args to __init__ frame */ - init_frame->localsplus[0] = PyStackRef_FromPyObjectSteal(self); - for (int i = 0; i < oparg; i++) { - init_frame->localsplus[i+1] = args[i]; - } - frame->return_offset = (uint16_t)(next_instr - this_instr); - STACK_SHRINK(oparg+2); - _PyFrame_SetStackPointer(frame, stack_pointer); - /* Link frames */ - init_frame->previous = shim; - shim->previous = frame; - frame = tstate->current_frame = init_frame; - CALL_STAT_INC(inlined_py_calls); + shim->localsplus[0] = PyStackRef_DUP(self); + PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init); + args[-1] = self; + init_frame = _PyEvalFramePushAndInit( + tstate, init_func, NULL, args-1, oparg+1, NULL, shim); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + SYNC_SP(); /* Account for pushing the extra frame. * We don't check recursion depth here, * as it will be checked after start_frame */ tstate->py_recursion_remaining--; - goto start_frame; } + macro(CALL_ALLOC_AND_ENTER_INIT) = + unused/1 + + _CHECK_PEP_523 + + _CHECK_AND_ALLOCATE_OBJECT + + _CREATE_INIT_FRAME + + _PUSH_FRAME; + inst(EXIT_INIT_CHECK, (should_be_none -- )) { assert(STACK_LEVEL() == 2); if (!PyStackRef_Is(should_be_none, PyStackRef_None)) { @@ -4060,7 +4057,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, positional_args, kwnames_o + args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); // Manipulate stack directly since we leave using DISPATCH_INLINED(). @@ -4129,7 +4126,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, positional_args, kwnames_o + args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, @@ -4315,7 +4312,7 @@ dummy_func( _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, - nargs, callargs, kwargs); + nargs, callargs, kwargs, frame); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { diff --git a/Python/ceval.c b/Python/ceval.c index c685a95b2ef..b615deec3c4 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -246,7 +246,7 @@ static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg static int get_exception_handler(PyCodeObject *, int, int*, int*, int*); static _PyInterpreterFrame * _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs); + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous); #ifdef HAVE_ERRNO_H #include @@ -671,8 +671,6 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { { .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START } }; -extern const struct _PyCode_DEF(8) _Py_InitCleanup; - #ifdef Py_DEBUG extern void _PyUOpPrint(const _PyUOpInstruction *uop); #endif @@ -1691,7 +1689,7 @@ _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) _PyInterpreterFrame * _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, _PyStackRef const* args, - size_t argcount, PyObject *kwnames) + size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous) { PyCodeObject * code = (PyCodeObject *)func->func_code; CALL_STAT_INC(frames_pushed); @@ -1699,7 +1697,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, if (frame == NULL) { goto fail; } - _PyFrame_Initialize(frame, func, locals, code, 0); + _PyFrame_Initialize(frame, func, locals, code, 0, previous); if (initialize_locals(tstate, func, frame->localsplus, args, argcount, kwnames)) { assert(frame->owner == FRAME_OWNED_BY_THREAD); clear_thread_frame(tstate, frame); @@ -1726,7 +1724,7 @@ fail: static _PyInterpreterFrame * _PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, PyObject *const* args, - size_t argcount, PyObject *kwnames) + size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous) { #if defined(Py_GIL_DISABLED) size_t kw_count = kwnames == NULL ? 0 : PyTuple_GET_SIZE(kwnames); @@ -1742,11 +1740,11 @@ _PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func, for (size_t i = 0; i < kw_count; i++) { tagged_args_buffer[argcount + i] = PyStackRef_FromPyObjectSteal(args[argcount + i]); } - _PyInterpreterFrame *res = _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)tagged_args_buffer, argcount, kwnames); + _PyInterpreterFrame *res = _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)tagged_args_buffer, argcount, kwnames, previous); PyMem_Free(tagged_args_buffer); return res; #else - return _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)args, argcount, kwnames); + return _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)args, argcount, kwnames, previous); #endif } @@ -1755,7 +1753,7 @@ _PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func, */ static _PyInterpreterFrame * _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs) + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous) { bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); PyObject *kwnames = NULL; @@ -1776,7 +1774,7 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, } _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_UnTagged( tstate, (PyFunctionObject *)func, locals, - newargs, nargs, kwnames + newargs, nargs, kwnames, previous ); if (has_dict) { _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); @@ -1813,7 +1811,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, } } _PyInterpreterFrame *frame = _PyEvalFramePushAndInit_UnTagged( - tstate, func, locals, args, argcount, kwnames); + tstate, func, locals, args, argcount, kwnames, NULL); if (frame == NULL) { return NULL; } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index ed146a10b2a..9e1540674d4 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -124,7 +124,7 @@ do { \ do { \ assert(tstate->interp->eval_frame == NULL); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - (NEW_FRAME)->previous = frame; \ + assert((NEW_FRAME)->previous == frame); \ frame = tstate->current_frame = (NEW_FRAME); \ CALL_STAT_INC(inlined_py_calls); \ goto start_frame; \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d699e8d7994..9226b7a4e5d 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1031,7 +1031,7 @@ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2); + new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2, frame); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); new_frame->localsplus[0] = container; @@ -1319,6 +1319,7 @@ tstate->exc_info = &gen->gi_exc_state; assert(1 + INLINE_CACHE_ENTRIES_SEND + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); + gen_frame->previous = frame; stack_pointer[-1].bits = (uintptr_t)gen_frame; break; } @@ -2552,7 +2553,7 @@ } STAT_INC(LOAD_ATTR, hit); Py_INCREF(fget); - new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame); new_frame->localsplus[0] = owner; stack_pointer[-1].bits = (uintptr_t)new_frame; break; @@ -3316,6 +3317,7 @@ gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; + gen_frame->previous = frame; // oparg is the return offset from the next instruction. frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); stack_pointer[0].bits = (uintptr_t)gen_frame; @@ -3604,7 +3606,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. @@ -3838,7 +3840,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3864,7 +3866,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3890,7 +3892,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3916,7 +3918,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3942,7 +3944,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3967,7 +3969,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -3988,7 +3990,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -4087,7 +4089,81 @@ break; } - /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + case _CHECK_AND_ALLOCATE_OBJECT: { + _PyStackRef *args; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef self; + _PyStackRef init; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t type_version = (uint32_t)CURRENT_OPERAND(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + if (tp->tp_version_tag != type_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)cls->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(CALL, hit); + self = PyStackRef_FromPyObjectSteal(_PyType_NewManagedObject(tp)); + if (PyStackRef_IsNull(self)) { + JUMP_TO_ERROR(); + } + PyStackRef_CLOSE(callable); + init = PyStackRef_FromPyObjectNew(init_func); + stack_pointer[-1 - oparg] = init; + stack_pointer[-2 - oparg] = self; + break; + } + + case _CREATE_INIT_FRAME: { + _PyStackRef *args; + _PyStackRef init; + _PyStackRef self; + _PyInterpreterFrame *init_frame; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + init = stack_pointer[-1 - oparg]; + self = stack_pointer[-2 - oparg]; + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + shim->localsplus[0] = PyStackRef_DUP(self); + PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init); + args[-1] = self; + init_frame = _PyEvalFramePushAndInit( + tstate, init_func, NULL, args-1, oparg+1, NULL, shim); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + /* Account for pushing the extra frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; + stack_pointer[0].bits = (uintptr_t)init_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _EXIT_INIT_CHECK: { _PyStackRef should_be_none; @@ -4705,7 +4781,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, positional_args, kwnames_o + args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 486d356dfb9..27971ce7546 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -504,7 +504,7 @@ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2); + new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2, frame); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); new_frame->localsplus[0] = container; @@ -517,7 +517,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -888,7 +888,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); @@ -975,60 +975,75 @@ _PyStackRef callable; _PyStackRef null; _PyStackRef *args; + _PyStackRef self; + _PyStackRef init; + _PyInterpreterFrame *init_frame; + _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ - /* Skip 2 cache entries */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_AND_ALLOCATE_OBJECT args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - /* This instruction does the following: - * 1. Creates the object (by calling ``object.__new__``) - * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) - * 3. Pushes the frame for ``__init__`` to the frame stack - * */ - _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(!PyType_Check(callable_o), CALL); - PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); - assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; - PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(code->co_argcount != oparg+1, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); - STAT_INC(CALL, hit); - PyObject *self = _PyType_NewManagedObject(tp); - if (self == NULL) { - goto error; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; + DEOPT_IF(tp->tp_version_tag != type_version, CALL); + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)cls->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + self = PyStackRef_FromPyObjectSteal(_PyType_NewManagedObject(tp)); + if (PyStackRef_IsNull(self)) { + goto error; + } + PyStackRef_CLOSE(callable); + init = PyStackRef_FromPyObjectNew(init_func); + stack_pointer[-1 - oparg] = init; } - PyStackRef_CLOSE(callable); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1); - assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); - /* Push self onto stack of shim */ - Py_INCREF(self); - shim->localsplus[0] = PyStackRef_FromPyObjectSteal(self); - Py_INCREF(init); - _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); - /* Copy self followed by args to __init__ frame */ - init_frame->localsplus[0] = PyStackRef_FromPyObjectSteal(self); - for (int i = 0; i < oparg; i++) { - init_frame->localsplus[i+1] = args[i]; + // _CREATE_INIT_FRAME + { + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + shim->localsplus[0] = PyStackRef_DUP(self); + PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init); + args[-1] = self; + init_frame = _PyEvalFramePushAndInit( + tstate, init_func, NULL, args-1, oparg+1, NULL, shim); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + /* Account for pushing the extra frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; } - frame->return_offset = (uint16_t)(next_instr - this_instr); - STACK_SHRINK(oparg+2); - _PyFrame_SetStackPointer(frame, stack_pointer); - /* Link frames */ - init_frame->previous = shim; - shim->previous = frame; - frame = tstate->current_frame = init_frame; - CALL_STAT_INC(inlined_py_calls); - /* Account for pushing the extra frame. - * We don't check recursion depth here, - * as it will be checked after start_frame */ - tstate->py_recursion_remaining--; - goto start_frame; + // _PUSH_FRAME + new_frame = init_frame; + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { @@ -1099,7 +1114,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -1123,7 +1138,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -1194,7 +1209,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. @@ -1219,7 +1234,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -1595,7 +1610,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, - nargs, callargs, kwargs); + nargs, callargs, kwargs, frame); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { @@ -1781,7 +1796,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, positional_args, kwnames_o + args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); // Manipulate stack directly since we leave using DISPATCH_INLINED(). @@ -1916,7 +1931,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, positional_args, kwnames_o + args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, @@ -1942,7 +1957,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -2086,7 +2101,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, positional_args, kwnames_o + args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, @@ -2112,7 +2127,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -2635,7 +2650,7 @@ int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { @@ -2659,7 +2674,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -2710,7 +2725,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. @@ -2735,7 +2750,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -3667,6 +3682,7 @@ gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; + gen_frame->previous = frame; // oparg is the return offset from the next instruction. frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); } @@ -3677,7 +3693,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -4090,7 +4106,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, total_args, NULL, frame ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); @@ -4944,7 +4960,7 @@ STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); Py_INCREF(f); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2, frame); // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; @@ -5274,7 +5290,7 @@ DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(fget); - new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame); new_frame->localsplus[0] = owner; } // _SAVE_RETURN_OFFSET @@ -5294,7 +5310,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -6551,6 +6567,8 @@ tstate->exc_info = &gen->gi_exc_state; assert(next_instr - this_instr + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; DISPATCH_INLINED(gen_frame); } if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) { @@ -6611,6 +6629,7 @@ tstate->exc_info = &gen->gi_exc_state; assert(1 + INLINE_CACHE_ENTRIES_SEND + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); + gen_frame->previous = frame; } // _PUSH_FRAME new_frame = gen_frame; @@ -6621,7 +6640,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; @@ -7650,7 +7669,7 @@ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, - nargs, callargs, kwargs); + nargs, callargs, kwargs, frame); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 97e4c642225..6e46d9bed11 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -627,7 +627,7 @@ dummy_func(void) { ctx->done = true; } - op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame*)) { + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame *)) { (void)callable; (void)self_or_null; (void)args; @@ -636,6 +636,23 @@ dummy_func(void) { ctx->done = true; } + op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, null, args[oparg] -- self, init, args[oparg])) { + (void)type_version; + (void)callable; + (void)null; + (void)args; + self = sym_new_not_null(ctx); + init = sym_new_not_null(ctx); + } + + op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { + (void)self; + (void)init; + (void)args; + init_frame = NULL; + ctx->done = true; + } + op(_RETURN_VALUE, (retval -- res)) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 3ec2e6985d6..e5be9d0e3b5 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1866,7 +1866,46 @@ break; } - /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 */ + case _CHECK_AND_ALLOCATE_OBJECT: { + _Py_UopsSymbol **args; + _Py_UopsSymbol *null; + _Py_UopsSymbol *callable; + _Py_UopsSymbol *self; + _Py_UopsSymbol *init; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-oparg]; + uint32_t type_version = (uint32_t)this_instr->operand; + (void)type_version; + (void)callable; + (void)null; + (void)args; + self = sym_new_not_null(ctx); + init = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = self; + stack_pointer[-1 - oparg] = init; + break; + } + + case _CREATE_INIT_FRAME: { + _Py_UopsSymbol **args; + _Py_UopsSymbol *init; + _Py_UopsSymbol *self; + _Py_UOpsAbstractFrame *init_frame; + args = &stack_pointer[-oparg]; + init = stack_pointer[-1 - oparg]; + self = stack_pointer[-2 - oparg]; + (void)self; + (void)init; + (void)args; + init_frame = NULL; + ctx->done = true; + stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)init_frame; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _EXIT_INIT_CHECK: { stack_pointer += -1; diff --git a/Python/specialize.c b/Python/specialize.c index 4fec5a36fc9..b3a2e07c3bb 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1809,10 +1809,6 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) return -1; } if (init != NULL) { - if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); - return -1; - } _PyCallCache *cache = (_PyCallCache *)(instr + 1); write_u32(cache->func_version, tp->tp_version_tag); _Py_SET_OPCODE(*instr, CALL_ALLOC_AND_ENTER_INIT); @@ -2654,7 +2650,7 @@ static const PyBytesObject no_location = { .ob_sval = { NO_LOC_4 } }; -const struct _PyCode_DEF(8) _Py_InitCleanup = { +const struct _PyCode8 _Py_InitCleanup = { _PyVarObject_HEAD_INIT(&PyCode_Type, 3), .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty), .co_names = (PyObject *)&_Py_SINGLETON(tuple_empty), diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 63b640e465a..b91b11763fb 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -742,3 +742,7 @@ Modules/clinic/md5module.c.h _md5_md5 _keywords - Modules/clinic/grpmodule.c.h grp_getgrgid _keywords - Modules/clinic/grpmodule.c.h grp_getgrnam _keywords - Objects/object.c - constants static PyObject*[] + + +## False positives +Python/specialize.c - _Py_InitCleanup -