From 626c414995bad1dab51c7222a6f7bf388255eb9e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 20 Feb 2024 10:50:59 +0000 Subject: [PATCH] GH-115457: Support splitting and replication of micro ops. (GH-115558) --- Include/internal/pycore_opcode_metadata.h | 22 +- Include/internal/pycore_uop_ids.h | 463 +++++++++--------- Include/internal/pycore_uop_metadata.h | 67 ++- Lib/test/test_capi/test_opt.py | 2 +- Python/bytecodes.c | 16 +- Python/ceval.c | 2 +- Python/executor_cases.c.h | 405 ++++++++++++++- Python/generated_cases.c.h | 18 +- Python/optimizer.c | 15 + Python/tier2_redundancy_eliminator_cases.c.h | 17 +- Tools/cases_generator/analyzer.py | 89 +++- Tools/cases_generator/generators_common.py | 7 +- Tools/cases_generator/lexer.py | 2 + .../opcode_metadata_generator.py | 1 + Tools/cases_generator/parsing.py | 8 +- Tools/cases_generator/stack.py | 20 +- .../tier2_abstract_generator.py | 2 + Tools/cases_generator/tier2_generator.py | 62 ++- Tools/cases_generator/uop_id_generator.py | 12 +- .../cases_generator/uop_metadata_generator.py | 7 + 20 files changed, 918 insertions(+), 319 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 177dd302f73..f45e5f1901b 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -519,7 +519,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CALL_ALLOC_AND_ENTER_INIT: return 1; case CALL_BOUND_METHOD_EXACT_ARGS: - return ((0) ? 1 : 0); + return 0; case CALL_BUILTIN_CLASS: return 1; case CALL_BUILTIN_FAST: @@ -551,7 +551,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CALL_METHOD_DESCRIPTOR_O: return 1; case CALL_PY_EXACT_ARGS: - return ((0) ? 1 : 0); + return 0; case CALL_PY_WITH_DEFAULTS: return 1; case CALL_STR_1: @@ -697,23 +697,23 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case LOAD_ATTR_CLASS: return 1 + (oparg & 1); case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_ATTR_INSTANCE_VALUE: return 1 + (oparg & 1); case LOAD_ATTR_METHOD_LAZY_DICT: - return 1 + ((1) ? 1 : 0); + return 2; case LOAD_ATTR_METHOD_NO_DICT: - return 1 + ((1) ? 1 : 0); + return 2; case LOAD_ATTR_METHOD_WITH_VALUES: - return 1 + ((1) ? 1 : 0); + return 2; case LOAD_ATTR_MODULE: return 1 + (oparg & 1); case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_ATTR_PROPERTY: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_ATTR_SLOT: return 1 + (oparg & 1); case LOAD_ATTR_WITH_HINT: @@ -749,7 +749,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case LOAD_SUPER_ATTR: return 1 + (oparg & 1); case LOAD_SUPER_ATTR_ATTR: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_SUPER_ATTR_METHOD: return 2; case MAKE_CELL: @@ -912,6 +912,7 @@ enum InstructionFormat { #define HAS_EXIT_FLAG (1024) #define HAS_PURE_FLAG (2048) #define HAS_PASSTHROUGH_FLAG (4096) +#define HAS_OPARG_AND_1_FLAG (8192) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -925,6 +926,7 @@ enum InstructionFormat { #define OPCODE_HAS_EXIT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EXIT_FLAG)) #define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG)) #define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) +#define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) #define OPARG_FULL 0 #define OPARG_CACHE_1 1 diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index ed800a73796..e098852d941 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -11,236 +11,263 @@ extern "C" { #define _EXIT_TRACE 300 #define _SET_IP 301 -#define _NOP NOP -#define _RESUME_CHECK RESUME_CHECK +#define _BEFORE_ASYNC_WITH BEFORE_ASYNC_WITH +#define _BEFORE_WITH BEFORE_WITH +#define _BINARY_OP 302 +#define _BINARY_OP_ADD_FLOAT 303 +#define _BINARY_OP_ADD_INT 304 +#define _BINARY_OP_ADD_UNICODE 305 +#define _BINARY_OP_MULTIPLY_FLOAT 306 +#define _BINARY_OP_MULTIPLY_INT 307 +#define _BINARY_OP_SUBTRACT_FLOAT 308 +#define _BINARY_OP_SUBTRACT_INT 309 +#define _BINARY_SLICE BINARY_SLICE +#define _BINARY_SUBSCR 310 +#define _BINARY_SUBSCR_DICT BINARY_SUBSCR_DICT +#define _BINARY_SUBSCR_GETITEM BINARY_SUBSCR_GETITEM +#define _BINARY_SUBSCR_LIST_INT BINARY_SUBSCR_LIST_INT +#define _BINARY_SUBSCR_STR_INT BINARY_SUBSCR_STR_INT +#define _BINARY_SUBSCR_TUPLE_INT BINARY_SUBSCR_TUPLE_INT +#define _BUILD_CONST_KEY_MAP BUILD_CONST_KEY_MAP +#define _BUILD_LIST BUILD_LIST +#define _BUILD_MAP BUILD_MAP +#define _BUILD_SET BUILD_SET +#define _BUILD_SLICE BUILD_SLICE +#define _BUILD_STRING BUILD_STRING +#define _BUILD_TUPLE BUILD_TUPLE +#define _CALL 311 +#define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT +#define _CALL_BUILTIN_CLASS CALL_BUILTIN_CLASS +#define _CALL_BUILTIN_FAST CALL_BUILTIN_FAST +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS CALL_BUILTIN_FAST_WITH_KEYWORDS +#define _CALL_BUILTIN_O CALL_BUILTIN_O +#define _CALL_FUNCTION_EX CALL_FUNCTION_EX +#define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 +#define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 +#define _CALL_ISINSTANCE CALL_ISINSTANCE +#define _CALL_KW CALL_KW +#define _CALL_LEN CALL_LEN +#define _CALL_METHOD_DESCRIPTOR_FAST CALL_METHOD_DESCRIPTOR_FAST +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS +#define _CALL_METHOD_DESCRIPTOR_NOARGS CALL_METHOD_DESCRIPTOR_NOARGS +#define _CALL_METHOD_DESCRIPTOR_O CALL_METHOD_DESCRIPTOR_O +#define _CALL_PY_WITH_DEFAULTS CALL_PY_WITH_DEFAULTS +#define _CALL_STR_1 CALL_STR_1 +#define _CALL_TUPLE_1 CALL_TUPLE_1 +#define _CALL_TYPE_1 CALL_TYPE_1 +#define _CHECK_ATTR_CLASS 312 +#define _CHECK_ATTR_METHOD_LAZY_DICT 313 +#define _CHECK_ATTR_MODULE 314 +#define _CHECK_ATTR_WITH_HINT 315 +#define _CHECK_BUILTINS 316 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 317 +#define _CHECK_EG_MATCH CHECK_EG_MATCH +#define _CHECK_EXC_MATCH CHECK_EXC_MATCH +#define _CHECK_FUNCTION_EXACT_ARGS 318 +#define _CHECK_GLOBALS 319 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 320 +#define _CHECK_PEP_523 321 +#define _CHECK_STACK_SPACE 322 +#define _CHECK_VALIDITY 323 +#define _CHECK_VALIDITY_AND_SET_IP 324 +#define _COLD_EXIT 325 +#define _COMPARE_OP 326 +#define _COMPARE_OP_FLOAT COMPARE_OP_FLOAT +#define _COMPARE_OP_INT COMPARE_OP_INT +#define _COMPARE_OP_STR COMPARE_OP_STR +#define _CONTAINS_OP CONTAINS_OP +#define _CONVERT_VALUE CONVERT_VALUE +#define _COPY COPY +#define _COPY_FREE_VARS COPY_FREE_VARS +#define _DELETE_ATTR DELETE_ATTR +#define _DELETE_DEREF DELETE_DEREF +#define _DELETE_FAST DELETE_FAST +#define _DELETE_GLOBAL DELETE_GLOBAL +#define _DELETE_NAME DELETE_NAME +#define _DELETE_SUBSCR DELETE_SUBSCR +#define _DICT_MERGE DICT_MERGE +#define _DICT_UPDATE DICT_UPDATE +#define _END_SEND END_SEND +#define _EXIT_INIT_CHECK EXIT_INIT_CHECK +#define _FATAL_ERROR 327 +#define _FORMAT_SIMPLE FORMAT_SIMPLE +#define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC +#define _FOR_ITER 328 +#define _FOR_ITER_GEN FOR_ITER_GEN +#define _FOR_ITER_TIER_TWO 329 +#define _GET_AITER GET_AITER +#define _GET_ANEXT GET_ANEXT +#define _GET_AWAITABLE GET_AWAITABLE +#define _GET_ITER GET_ITER +#define _GET_LEN GET_LEN +#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER +#define _GUARD_BOTH_FLOAT 330 +#define _GUARD_BOTH_INT 331 +#define _GUARD_BOTH_UNICODE 332 +#define _GUARD_BUILTINS_VERSION 333 +#define _GUARD_DORV_VALUES 334 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 335 +#define _GUARD_GLOBALS_VERSION 336 +#define _GUARD_IS_FALSE_POP 337 +#define _GUARD_IS_NONE_POP 338 +#define _GUARD_IS_NOT_NONE_POP 339 +#define _GUARD_IS_TRUE_POP 340 +#define _GUARD_KEYS_VERSION 341 +#define _GUARD_NOT_EXHAUSTED_LIST 342 +#define _GUARD_NOT_EXHAUSTED_RANGE 343 +#define _GUARD_NOT_EXHAUSTED_TUPLE 344 +#define _GUARD_TYPE_VERSION 345 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 346 +#define _INIT_CALL_PY_EXACT_ARGS 347 +#define _INIT_CALL_PY_EXACT_ARGS_0 348 +#define _INIT_CALL_PY_EXACT_ARGS_1 349 +#define _INIT_CALL_PY_EXACT_ARGS_2 350 +#define _INIT_CALL_PY_EXACT_ARGS_3 351 +#define _INIT_CALL_PY_EXACT_ARGS_4 352 +#define _INSTRUMENTED_CALL INSTRUMENTED_CALL +#define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX +#define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW +#define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER +#define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION +#define _INSTRUMENTED_JUMP_BACKWARD INSTRUMENTED_JUMP_BACKWARD +#define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD +#define _INSTRUMENTED_LOAD_SUPER_ATTR INSTRUMENTED_LOAD_SUPER_ATTR +#define _INSTRUMENTED_POP_JUMP_IF_FALSE INSTRUMENTED_POP_JUMP_IF_FALSE +#define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE +#define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE #define _INSTRUMENTED_RESUME INSTRUMENTED_RESUME -#define _LOAD_FAST_CHECK LOAD_FAST_CHECK -#define _LOAD_FAST LOAD_FAST -#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST +#define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST +#define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE +#define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE +#define _INTERNAL_INCREMENT_OPT_COUNTER 353 +#define _IS_NONE 354 +#define _IS_OP IS_OP +#define _ITER_CHECK_LIST 355 +#define _ITER_CHECK_RANGE 356 +#define _ITER_CHECK_TUPLE 357 +#define _ITER_JUMP_LIST 358 +#define _ITER_JUMP_RANGE 359 +#define _ITER_JUMP_TUPLE 360 +#define _ITER_NEXT_LIST 361 +#define _ITER_NEXT_RANGE 362 +#define _ITER_NEXT_TUPLE 363 +#define _JUMP_TO_TOP 364 +#define _LIST_APPEND LIST_APPEND +#define _LIST_EXTEND LIST_EXTEND +#define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR +#define _LOAD_ATTR 365 +#define _LOAD_ATTR_CLASS 366 +#define _LOAD_ATTR_CLASS_0 367 +#define _LOAD_ATTR_CLASS_1 368 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN +#define _LOAD_ATTR_INSTANCE_VALUE 369 +#define _LOAD_ATTR_INSTANCE_VALUE_0 370 +#define _LOAD_ATTR_INSTANCE_VALUE_1 371 +#define _LOAD_ATTR_METHOD_LAZY_DICT 372 +#define _LOAD_ATTR_METHOD_NO_DICT 373 +#define _LOAD_ATTR_METHOD_WITH_VALUES 374 +#define _LOAD_ATTR_MODULE 375 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 376 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 377 +#define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY +#define _LOAD_ATTR_SLOT 378 +#define _LOAD_ATTR_SLOT_0 379 +#define _LOAD_ATTR_SLOT_1 380 +#define _LOAD_ATTR_WITH_HINT 381 +#define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_CONST LOAD_CONST -#define _STORE_FAST STORE_FAST +#define _LOAD_CONST_INLINE 382 +#define _LOAD_CONST_INLINE_BORROW 383 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 384 +#define _LOAD_CONST_INLINE_WITH_NULL 385 +#define _LOAD_DEREF LOAD_DEREF +#define _LOAD_FAST 386 +#define _LOAD_FAST_0 387 +#define _LOAD_FAST_1 388 +#define _LOAD_FAST_2 389 +#define _LOAD_FAST_3 390 +#define _LOAD_FAST_4 391 +#define _LOAD_FAST_5 392 +#define _LOAD_FAST_6 393 +#define _LOAD_FAST_7 394 +#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR +#define _LOAD_FAST_CHECK LOAD_FAST_CHECK +#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST +#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF +#define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS +#define _LOAD_GLOBAL 395 +#define _LOAD_GLOBAL_BUILTINS 396 +#define _LOAD_GLOBAL_MODULE 397 +#define _LOAD_LOCALS LOAD_LOCALS +#define _LOAD_NAME LOAD_NAME +#define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR +#define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD +#define _MAKE_CELL MAKE_CELL +#define _MAKE_FUNCTION MAKE_FUNCTION +#define _MAP_ADD MAP_ADD +#define _MATCH_CLASS MATCH_CLASS +#define _MATCH_KEYS MATCH_KEYS +#define _MATCH_MAPPING MATCH_MAPPING +#define _MATCH_SEQUENCE MATCH_SEQUENCE +#define _NOP NOP +#define _POP_EXCEPT POP_EXCEPT +#define _POP_FRAME 398 +#define _POP_JUMP_IF_FALSE 399 +#define _POP_JUMP_IF_TRUE 400 +#define _POP_TOP POP_TOP +#define _PUSH_EXC_INFO PUSH_EXC_INFO +#define _PUSH_FRAME 401 +#define _PUSH_NULL PUSH_NULL +#define _RESUME_CHECK RESUME_CHECK +#define _SAVE_RETURN_OFFSET 402 +#define _SEND 403 +#define _SEND_GEN SEND_GEN +#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS +#define _SET_ADD SET_ADD +#define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE +#define _SET_UPDATE SET_UPDATE +#define _START_EXECUTOR 404 +#define _STORE_ATTR 405 +#define _STORE_ATTR_INSTANCE_VALUE 406 +#define _STORE_ATTR_SLOT 407 +#define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT +#define _STORE_DEREF STORE_DEREF +#define _STORE_FAST 408 +#define _STORE_FAST_0 409 +#define _STORE_FAST_1 410 +#define _STORE_FAST_2 411 +#define _STORE_FAST_3 412 +#define _STORE_FAST_4 413 +#define _STORE_FAST_5 414 +#define _STORE_FAST_6 415 +#define _STORE_FAST_7 416 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST -#define _POP_TOP POP_TOP -#define _PUSH_NULL PUSH_NULL -#define _END_SEND END_SEND -#define _UNARY_NEGATIVE UNARY_NEGATIVE -#define _UNARY_NOT UNARY_NOT -#define _TO_BOOL 302 +#define _STORE_GLOBAL STORE_GLOBAL +#define _STORE_NAME STORE_NAME +#define _STORE_SLICE STORE_SLICE +#define _STORE_SUBSCR 417 +#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT +#define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT +#define _SWAP SWAP +#define _TO_BOOL 418 +#define _TO_BOOL_ALWAYS_TRUE TO_BOOL_ALWAYS_TRUE #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST #define _TO_BOOL_NONE TO_BOOL_NONE #define _TO_BOOL_STR TO_BOOL_STR -#define _TO_BOOL_ALWAYS_TRUE TO_BOOL_ALWAYS_TRUE #define _UNARY_INVERT UNARY_INVERT -#define _GUARD_BOTH_INT 303 -#define _BINARY_OP_MULTIPLY_INT 304 -#define _BINARY_OP_ADD_INT 305 -#define _BINARY_OP_SUBTRACT_INT 306 -#define _GUARD_BOTH_FLOAT 307 -#define _BINARY_OP_MULTIPLY_FLOAT 308 -#define _BINARY_OP_ADD_FLOAT 309 -#define _BINARY_OP_SUBTRACT_FLOAT 310 -#define _GUARD_BOTH_UNICODE 311 -#define _BINARY_OP_ADD_UNICODE 312 -#define _BINARY_SUBSCR 313 -#define _BINARY_SLICE BINARY_SLICE -#define _STORE_SLICE STORE_SLICE -#define _BINARY_SUBSCR_LIST_INT BINARY_SUBSCR_LIST_INT -#define _BINARY_SUBSCR_STR_INT BINARY_SUBSCR_STR_INT -#define _BINARY_SUBSCR_TUPLE_INT BINARY_SUBSCR_TUPLE_INT -#define _BINARY_SUBSCR_DICT BINARY_SUBSCR_DICT -#define _BINARY_SUBSCR_GETITEM BINARY_SUBSCR_GETITEM -#define _LIST_APPEND LIST_APPEND -#define _SET_ADD SET_ADD -#define _STORE_SUBSCR 314 -#define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT -#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT -#define _DELETE_SUBSCR DELETE_SUBSCR -#define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 -#define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 -#define _POP_FRAME 315 -#define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE -#define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST -#define _GET_AITER GET_AITER -#define _GET_ANEXT GET_ANEXT -#define _GET_AWAITABLE GET_AWAITABLE -#define _SEND 316 -#define _SEND_GEN SEND_GEN -#define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _POP_EXCEPT POP_EXCEPT -#define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR -#define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _STORE_NAME STORE_NAME -#define _DELETE_NAME DELETE_NAME -#define _UNPACK_SEQUENCE 317 -#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE -#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE -#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST +#define _UNARY_NEGATIVE UNARY_NEGATIVE +#define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _STORE_ATTR 318 -#define _DELETE_ATTR DELETE_ATTR -#define _STORE_GLOBAL STORE_GLOBAL -#define _DELETE_GLOBAL DELETE_GLOBAL -#define _LOAD_LOCALS LOAD_LOCALS -#define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_NAME LOAD_NAME -#define _LOAD_GLOBAL 319 -#define _GUARD_GLOBALS_VERSION 320 -#define _GUARD_BUILTINS_VERSION 321 -#define _LOAD_GLOBAL_MODULE 322 -#define _LOAD_GLOBAL_BUILTINS 323 -#define _DELETE_FAST DELETE_FAST -#define _MAKE_CELL MAKE_CELL -#define _DELETE_DEREF DELETE_DEREF -#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF -#define _LOAD_DEREF LOAD_DEREF -#define _STORE_DEREF STORE_DEREF -#define _COPY_FREE_VARS COPY_FREE_VARS -#define _BUILD_STRING BUILD_STRING -#define _BUILD_TUPLE BUILD_TUPLE -#define _BUILD_LIST BUILD_LIST -#define _LIST_EXTEND LIST_EXTEND -#define _SET_UPDATE SET_UPDATE -#define _BUILD_SET BUILD_SET -#define _BUILD_MAP BUILD_MAP -#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS -#define _BUILD_CONST_KEY_MAP BUILD_CONST_KEY_MAP -#define _DICT_UPDATE DICT_UPDATE -#define _DICT_MERGE DICT_MERGE -#define _MAP_ADD MAP_ADD -#define _INSTRUMENTED_LOAD_SUPER_ATTR INSTRUMENTED_LOAD_SUPER_ATTR -#define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR -#define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _LOAD_ATTR 324 -#define _GUARD_TYPE_VERSION 325 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 326 -#define _LOAD_ATTR_INSTANCE_VALUE 327 -#define _CHECK_ATTR_MODULE 328 -#define _LOAD_ATTR_MODULE 329 -#define _CHECK_ATTR_WITH_HINT 330 -#define _LOAD_ATTR_WITH_HINT 331 -#define _LOAD_ATTR_SLOT 332 -#define _CHECK_ATTR_CLASS 333 -#define _LOAD_ATTR_CLASS 334 -#define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _GUARD_DORV_VALUES 335 -#define _STORE_ATTR_INSTANCE_VALUE 336 -#define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT -#define _STORE_ATTR_SLOT 337 -#define _COMPARE_OP 338 -#define _COMPARE_OP_FLOAT COMPARE_OP_FLOAT -#define _COMPARE_OP_INT COMPARE_OP_INT -#define _COMPARE_OP_STR COMPARE_OP_STR -#define _IS_OP IS_OP -#define _CONTAINS_OP CONTAINS_OP -#define _CHECK_EG_MATCH CHECK_EG_MATCH -#define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _POP_JUMP_IF_FALSE 339 -#define _POP_JUMP_IF_TRUE 340 -#define _IS_NONE 341 -#define _GET_LEN GET_LEN -#define _MATCH_CLASS MATCH_CLASS -#define _MATCH_MAPPING MATCH_MAPPING -#define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MATCH_KEYS MATCH_KEYS -#define _GET_ITER GET_ITER -#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _FOR_ITER 342 -#define _FOR_ITER_TIER_TWO 343 -#define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER -#define _ITER_CHECK_LIST 344 -#define _ITER_JUMP_LIST 345 -#define _GUARD_NOT_EXHAUSTED_LIST 346 -#define _ITER_NEXT_LIST 347 -#define _ITER_CHECK_TUPLE 348 -#define _ITER_JUMP_TUPLE 349 -#define _GUARD_NOT_EXHAUSTED_TUPLE 350 -#define _ITER_NEXT_TUPLE 351 -#define _ITER_CHECK_RANGE 352 -#define _ITER_JUMP_RANGE 353 -#define _GUARD_NOT_EXHAUSTED_RANGE 354 -#define _ITER_NEXT_RANGE 355 -#define _FOR_ITER_GEN FOR_ITER_GEN -#define _BEFORE_ASYNC_WITH BEFORE_ASYNC_WITH -#define _BEFORE_WITH BEFORE_WITH +#define _UNPACK_SEQUENCE 419 +#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST +#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE +#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 356 -#define _GUARD_KEYS_VERSION 357 -#define _LOAD_ATTR_METHOD_WITH_VALUES 358 -#define _LOAD_ATTR_METHOD_NO_DICT 359 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 360 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 361 -#define _CHECK_ATTR_METHOD_LAZY_DICT 362 -#define _LOAD_ATTR_METHOD_LAZY_DICT 363 -#define _INSTRUMENTED_CALL INSTRUMENTED_CALL -#define _CALL 364 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 365 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 366 -#define _CHECK_PEP_523 367 -#define _CHECK_FUNCTION_EXACT_ARGS 368 -#define _CHECK_STACK_SPACE 369 -#define _INIT_CALL_PY_EXACT_ARGS 370 -#define _PUSH_FRAME 371 -#define _CALL_PY_WITH_DEFAULTS CALL_PY_WITH_DEFAULTS -#define _CALL_TYPE_1 CALL_TYPE_1 -#define _CALL_STR_1 CALL_STR_1 -#define _CALL_TUPLE_1 CALL_TUPLE_1 -#define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT -#define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _CALL_BUILTIN_CLASS CALL_BUILTIN_CLASS -#define _CALL_BUILTIN_O CALL_BUILTIN_O -#define _CALL_BUILTIN_FAST CALL_BUILTIN_FAST -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS CALL_BUILTIN_FAST_WITH_KEYWORDS -#define _CALL_LEN CALL_LEN -#define _CALL_ISINSTANCE CALL_ISINSTANCE -#define _CALL_METHOD_DESCRIPTOR_O CALL_METHOD_DESCRIPTOR_O -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS -#define _CALL_METHOD_DESCRIPTOR_NOARGS CALL_METHOD_DESCRIPTOR_NOARGS -#define _CALL_METHOD_DESCRIPTOR_FAST CALL_METHOD_DESCRIPTOR_FAST -#define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW -#define _CALL_KW CALL_KW -#define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX -#define _CALL_FUNCTION_EX CALL_FUNCTION_EX -#define _MAKE_FUNCTION MAKE_FUNCTION -#define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE -#define _BUILD_SLICE BUILD_SLICE -#define _CONVERT_VALUE CONVERT_VALUE -#define _FORMAT_SIMPLE FORMAT_SIMPLE -#define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _COPY COPY -#define _BINARY_OP 372 -#define _SWAP SWAP -#define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION -#define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD -#define _INSTRUMENTED_JUMP_BACKWARD INSTRUMENTED_JUMP_BACKWARD -#define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _INSTRUMENTED_POP_JUMP_IF_FALSE INSTRUMENTED_POP_JUMP_IF_FALSE -#define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE -#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE -#define _GUARD_IS_TRUE_POP 373 -#define _GUARD_IS_FALSE_POP 374 -#define _GUARD_IS_NONE_POP 375 -#define _GUARD_IS_NOT_NONE_POP 376 -#define _JUMP_TO_TOP 377 -#define _SAVE_RETURN_OFFSET 378 -#define _CHECK_VALIDITY 379 -#define _LOAD_CONST_INLINE 380 -#define _LOAD_CONST_INLINE_BORROW 381 -#define _LOAD_CONST_INLINE_WITH_NULL 382 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 383 -#define _CHECK_GLOBALS 384 -#define _CHECK_BUILTINS 385 -#define _INTERNAL_INCREMENT_OPT_COUNTER 386 -#define _COLD_EXIT 387 -#define _START_EXECUTOR 388 -#define _FATAL_ERROR 389 -#define _CHECK_VALIDITY_AND_SET_IP 390 -#define MAX_UOP_ID 390 +#define MAX_UOP_ID 419 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 1e2dfecd9cf..c9def0ecdc1 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -12,6 +12,7 @@ extern "C" { #include #include "pycore_uop_ids.h" extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; +extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1]; extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; #ifdef NEED_OPCODE_METADATA @@ -19,10 +20,26 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_NOP] = HAS_PURE_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, + [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_3] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_4] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_5] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_6] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_0] = HAS_LOCAL_FLAG, + [_STORE_FAST_1] = HAS_LOCAL_FLAG, + [_STORE_FAST_2] = HAS_LOCAL_FLAG, + [_STORE_FAST_3] = HAS_LOCAL_FLAG, + [_STORE_FAST_4] = HAS_LOCAL_FLAG, + [_STORE_FAST_5] = HAS_LOCAL_FLAG, + [_STORE_FAST_6] = HAS_LOCAL_FLAG, + [_STORE_FAST_7] = HAS_LOCAL_FLAG, [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, @@ -114,14 +131,20 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_TYPE_VERSION] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE_0] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE_1] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, [_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_ATTR_SLOT_0] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_SLOT_1] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, [_CHECK_ATTR_CLASS] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG, + [_LOAD_ATTR_CLASS_0] = 0, + [_LOAD_ATTR_CLASS_1] = 0, + [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG, [_GUARD_DORV_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = HAS_ESCAPES_FLAG, [_STORE_ATTR_SLOT] = HAS_ESCAPES_FLAG, @@ -168,6 +191,11 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_PEP_523] = HAS_DEOPT_FLAG, [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_CHECK_STACK_SPACE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_0] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_1] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_2] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_PUSH_FRAME] = HAS_ESCAPES_FLAG, [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, @@ -215,6 +243,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, }; +const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { + [_LOAD_FAST] = 8, + [_STORE_FAST] = 8, + [_INIT_CALL_PY_EXACT_ARGS] = 5, +}; + const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BEFORE_ASYNC_WITH] = "_BEFORE_ASYNC_WITH", [_BEFORE_WITH] = "_BEFORE_WITH", @@ -317,6 +351,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", + [_INIT_CALL_PY_EXACT_ARGS_0] = "_INIT_CALL_PY_EXACT_ARGS_0", + [_INIT_CALL_PY_EXACT_ARGS_1] = "_INIT_CALL_PY_EXACT_ARGS_1", + [_INIT_CALL_PY_EXACT_ARGS_2] = "_INIT_CALL_PY_EXACT_ARGS_2", + [_INIT_CALL_PY_EXACT_ARGS_3] = "_INIT_CALL_PY_EXACT_ARGS_3", + [_INIT_CALL_PY_EXACT_ARGS_4] = "_INIT_CALL_PY_EXACT_ARGS_4", [_INTERNAL_INCREMENT_OPT_COUNTER] = "_INTERNAL_INCREMENT_OPT_COUNTER", [_IS_NONE] = "_IS_NONE", [_IS_OP] = "_IS_OP", @@ -332,7 +371,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_ASSERTION_ERROR] = "_LOAD_ASSERTION_ERROR", [_LOAD_ATTR] = "_LOAD_ATTR", [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", + [_LOAD_ATTR_CLASS_0] = "_LOAD_ATTR_CLASS_0", + [_LOAD_ATTR_CLASS_1] = "_LOAD_ATTR_CLASS_1", [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", + [_LOAD_ATTR_INSTANCE_VALUE_0] = "_LOAD_ATTR_INSTANCE_VALUE_0", + [_LOAD_ATTR_INSTANCE_VALUE_1] = "_LOAD_ATTR_INSTANCE_VALUE_1", [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", @@ -340,6 +383,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", + [_LOAD_ATTR_SLOT_0] = "_LOAD_ATTR_SLOT_0", + [_LOAD_ATTR_SLOT_1] = "_LOAD_ATTR_SLOT_1", [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", [_LOAD_CONST] = "_LOAD_CONST", @@ -349,6 +394,14 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_CONST_INLINE_WITH_NULL] = "_LOAD_CONST_INLINE_WITH_NULL", [_LOAD_DEREF] = "_LOAD_DEREF", [_LOAD_FAST] = "_LOAD_FAST", + [_LOAD_FAST_0] = "_LOAD_FAST_0", + [_LOAD_FAST_1] = "_LOAD_FAST_1", + [_LOAD_FAST_2] = "_LOAD_FAST_2", + [_LOAD_FAST_3] = "_LOAD_FAST_3", + [_LOAD_FAST_4] = "_LOAD_FAST_4", + [_LOAD_FAST_5] = "_LOAD_FAST_5", + [_LOAD_FAST_6] = "_LOAD_FAST_6", + [_LOAD_FAST_7] = "_LOAD_FAST_7", [_LOAD_FAST_AND_CLEAR] = "_LOAD_FAST_AND_CLEAR", [_LOAD_FAST_CHECK] = "_LOAD_FAST_CHECK", [_LOAD_FAST_LOAD_FAST] = "_LOAD_FAST_LOAD_FAST", @@ -388,6 +441,14 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", [_STORE_DEREF] = "_STORE_DEREF", [_STORE_FAST] = "_STORE_FAST", + [_STORE_FAST_0] = "_STORE_FAST_0", + [_STORE_FAST_1] = "_STORE_FAST_1", + [_STORE_FAST_2] = "_STORE_FAST_2", + [_STORE_FAST_3] = "_STORE_FAST_3", + [_STORE_FAST_4] = "_STORE_FAST_4", + [_STORE_FAST_5] = "_STORE_FAST_5", + [_STORE_FAST_6] = "_STORE_FAST_6", + [_STORE_FAST_7] = "_STORE_FAST_7", [_STORE_FAST_LOAD_FAST] = "_STORE_FAST_LOAD_FAST", [_STORE_FAST_STORE_FAST] = "_STORE_FAST_STORE_FAST", [_STORE_GLOBAL] = "_STORE_GLOBAL", diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 66860c67966..9d19b6c3ad3 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -216,7 +216,7 @@ class TestUops(unittest.TestCase): self.assertIsNotNone(ex) uops = {opname for opname, _, _ in ex} self.assertIn("_SET_IP", uops) - self.assertIn("_LOAD_FAST", uops) + self.assertIn("_LOAD_FAST_0", uops) def test_extended_arg(self): "Check EXTENDED_ARG handling in superblock creation" diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2e0008e63f6..27c439b71fa 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -54,6 +54,8 @@ #define guard #define override #define specializing +#define split +#define replicate(TIMES) // Dummy variables for stack effects. static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, *sub; @@ -208,7 +210,7 @@ dummy_func( Py_INCREF(value); } - pure inst(LOAD_FAST, (-- value)) { + replicate(8) pure inst(LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); @@ -234,7 +236,7 @@ dummy_func( Py_INCREF(value); } - inst(STORE_FAST, (value --)) { + replicate(8) inst(STORE_FAST, (value --)) { SETLOCAL(oparg, value); } @@ -1914,7 +1916,7 @@ dummy_func( DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); } - op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { + split op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; DEOPT_IF(attr == NULL); @@ -1995,7 +1997,7 @@ dummy_func( _LOAD_ATTR_WITH_HINT + unused/5; - op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { + split op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { char *addr = (char *)owner + index; attr = *(PyObject **)addr; DEOPT_IF(attr == NULL); @@ -2018,7 +2020,7 @@ dummy_func( } - op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { + split op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = Py_NewRef(descr); @@ -2888,7 +2890,7 @@ dummy_func( DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version); } - op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { + split op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { assert(oparg & 1); /* Cached method object */ STAT_INC(LOAD_ATTR, hit); @@ -3130,7 +3132,7 @@ dummy_func( DEOPT_IF(tstate->py_recursion_remaining <= 1); } - pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { int argcount = oparg; if (self_or_null != NULL) { args--; diff --git a/Python/ceval.c b/Python/ceval.c index adccf8fc00f..6f647cfdd53 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1029,7 +1029,7 @@ enter_tier_two: #ifdef Py_DEBUG { fprintf(stderr, "Unknown uop %d, oparg %d, operand %" PRIu64 " @ %d\n", - opcode, next_uop[-1].oparg, next_uop[-1].operand, + next_uop[-1].opcode, next_uop[-1].oparg, next_uop[-1].operand, (int)(next_uop - current_executor->trace - 1)); Py_FatalError("Unknown uop"); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index a18284d89ab..b46885e0d5c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -37,6 +37,102 @@ break; } + case _LOAD_FAST_0: { + PyObject *value; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_1: { + PyObject *value; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_2: { + PyObject *value; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_3: { + PyObject *value; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_4: { + PyObject *value; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_5: { + PyObject *value; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_6: { + PyObject *value; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_7: { + PyObject *value; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + case _LOAD_FAST: { PyObject *value; oparg = CURRENT_OPARG(); @@ -69,6 +165,86 @@ break; } + case _STORE_FAST_0: { + PyObject *value; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_1: { + PyObject *value; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_2: { + PyObject *value; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_3: { + PyObject *value; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_4: { + PyObject *value; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_5: { + PyObject *value; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_6: { + PyObject *value; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_7: { + PyObject *value; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + case _STORE_FAST: { PyObject *value; oparg = CURRENT_OPARG(); @@ -1534,7 +1710,7 @@ Py_DECREF(self); if (attr == NULL) goto pop_3_error_tier_two; stack_pointer[-3] = attr; - stack_pointer += -2 + ((0) ? 1 : 0); + stack_pointer += -2; break; } @@ -1637,11 +1813,11 @@ break; } - case _LOAD_ATTR_INSTANCE_VALUE: { + case _LOAD_ATTR_INSTANCE_VALUE_0: { PyObject *owner; PyObject *attr; PyObject *null = NULL; - oparg = CURRENT_OPARG(); + (void)null; owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); @@ -1652,11 +1828,31 @@ null = NULL; Py_DECREF(owner); stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); break; } + case _LOAD_ATTR_INSTANCE_VALUE_1: { + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + (void)null; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)CURRENT_OPERAND(); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + attr = _PyDictOrValues_GetValues(dorv)->values[index]; + if (attr == NULL) goto deoptimize; + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + stack_pointer[-1] = attr; + stack_pointer[0] = null; + stack_pointer += 1; + break; + } + + /* _LOAD_ATTR_INSTANCE_VALUE is split on (oparg & 1) */ + case _CHECK_ATTR_MODULE: { PyObject *owner; owner = stack_pointer[-1]; @@ -1735,11 +1931,11 @@ break; } - case _LOAD_ATTR_SLOT: { + case _LOAD_ATTR_SLOT_0: { PyObject *owner; PyObject *attr; PyObject *null = NULL; - oparg = CURRENT_OPARG(); + (void)null; owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); char *addr = (char *)owner + index; @@ -1750,11 +1946,31 @@ null = NULL; Py_DECREF(owner); stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); break; } + case _LOAD_ATTR_SLOT_1: { + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + (void)null; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)CURRENT_OPERAND(); + char *addr = (char *)owner + index; + attr = *(PyObject **)addr; + if (attr == NULL) goto deoptimize; + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + stack_pointer[-1] = attr; + stack_pointer[0] = null; + stack_pointer += 1; + break; + } + + /* _LOAD_ATTR_SLOT is split on (oparg & 1) */ + case _CHECK_ATTR_CLASS: { PyObject *owner; owner = stack_pointer[-1]; @@ -1765,11 +1981,11 @@ break; } - case _LOAD_ATTR_CLASS: { + case _LOAD_ATTR_CLASS_0: { PyObject *owner; PyObject *attr; PyObject *null = NULL; - oparg = CURRENT_OPARG(); + (void)null; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND(); STAT_INC(LOAD_ATTR, hit); @@ -1778,11 +1994,29 @@ null = NULL; Py_DECREF(owner); stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); break; } + case _LOAD_ATTR_CLASS_1: { + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + (void)null; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)CURRENT_OPERAND(); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + null = NULL; + Py_DECREF(owner); + stack_pointer[-1] = attr; + stack_pointer[0] = null; + stack_pointer += 1; + break; + } + + /* _LOAD_ATTR_CLASS is split on (oparg & 1) */ + /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 */ /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ @@ -2464,8 +2698,8 @@ assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); self = owner; stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; break; } @@ -2484,8 +2718,8 @@ attr = Py_NewRef(descr); self = owner; stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; break; } @@ -2501,7 +2735,6 @@ Py_DECREF(owner); attr = Py_NewRef(descr); stack_pointer[-1] = attr; - stack_pointer += ((0) ? 1 : 0); break; } @@ -2518,7 +2751,6 @@ Py_DECREF(owner); attr = Py_NewRef(descr); stack_pointer[-1] = attr; - stack_pointer += ((0) ? 1 : 0); break; } @@ -2547,8 +2779,8 @@ attr = Py_NewRef(descr); self = owner; stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; break; } @@ -2615,6 +2847,136 @@ break; } + case _INIT_CALL_PY_EXACT_ARGS_0: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer += -1 - oparg; + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_1: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer += -1 - oparg; + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_2: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer += -1 - oparg; + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_3: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer += -1 - oparg; + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_4: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer += -1 - oparg; + break; + } + case _INIT_CALL_PY_EXACT_ARGS: { PyObject **args; PyObject *self_or_null; @@ -2660,7 +3022,6 @@ goto exit_unwind; } #endif - stack_pointer += ((0) ? 1 : 0); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a520d042a4a..324e53dca63 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1005,7 +1005,6 @@ } #endif } - stack_pointer += ((0) ? 1 : 0); DISPATCH(); } @@ -1755,7 +1754,6 @@ } #endif } - stack_pointer += ((0) ? 1 : 0); DISPATCH(); } @@ -3597,8 +3595,8 @@ self = owner; } stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; DISPATCH(); } @@ -3632,8 +3630,8 @@ self = owner; } stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; DISPATCH(); } @@ -3679,8 +3677,8 @@ self = owner; } stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; DISPATCH(); } @@ -3751,7 +3749,6 @@ attr = Py_NewRef(descr); } stack_pointer[-1] = attr; - stack_pointer += ((0) ? 1 : 0); DISPATCH(); } @@ -3794,7 +3791,6 @@ attr = Py_NewRef(descr); } stack_pointer[-1] = attr; - stack_pointer += ((0) ? 1 : 0); DISPATCH(); } @@ -4380,7 +4376,7 @@ Py_DECREF(self); if (attr == NULL) goto pop_3_error; stack_pointer[-3] = attr; - stack_pointer += -2 + ((0) ? 1 : 0); + stack_pointer += -2; DISPATCH(); } diff --git a/Python/optimizer.c b/Python/optimizer.c index acc1d545d2b..df8f0ed234b 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -963,6 +963,21 @@ uop_optimize( } } assert(err == 1); + /* Fix up */ + for (int pc = 0; pc < UOP_MAX_TRACE_LENGTH; pc++) { + int opcode = buffer[pc].opcode; + int oparg = buffer[pc].oparg; + if (_PyUop_Flags[opcode] & HAS_OPARG_AND_1_FLAG) { + buffer[pc].opcode = opcode + 1 + (oparg & 1); + } + else if (oparg < _PyUop_Replication[opcode]) { + buffer[pc].opcode = opcode + oparg + 1; + } + else if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) { + break; + } + assert(_PyOpcode_uop_name[buffer[pc].opcode]); + } _PyExecutorObject *executor = make_executor_from_uops(buffer, &dependencies); if (executor == NULL) { return -1; diff --git a/Python/tier2_redundancy_eliminator_cases.c.h b/Python/tier2_redundancy_eliminator_cases.c.h index 98f0bdca01f..904700a0bbe 100644 --- a/Python/tier2_redundancy_eliminator_cases.c.h +++ b/Python/tier2_redundancy_eliminator_cases.c.h @@ -834,7 +834,7 @@ attr = sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-3] = attr; - stack_pointer += -2 + ((0) ? 1 : 0); + stack_pointer += -2; break; } @@ -1264,8 +1264,8 @@ self = sym_new_unknown(ctx); if (self == NULL) goto out_of_space; stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; break; } @@ -1277,8 +1277,8 @@ self = sym_new_unknown(ctx); if (self == NULL) goto out_of_space; stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; break; } @@ -1287,7 +1287,6 @@ attr = sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-1] = attr; - stack_pointer += ((0) ? 1 : 0); break; } @@ -1296,7 +1295,6 @@ attr = sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-1] = attr; - stack_pointer += ((0) ? 1 : 0); break; } @@ -1312,8 +1310,8 @@ self = sym_new_unknown(ctx); if (self == NULL) goto out_of_space; stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; break; } @@ -1409,7 +1407,6 @@ ctx->frame = new_frame; ctx->curr_frame_depth++; stack_pointer = new_frame->stack_pointer; - stack_pointer += ((0) ? 1 : 0); break; } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index bcffd75269a..49b8b426444 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -1,6 +1,7 @@ from dataclasses import dataclass, field import lexer import parser +import re from typing import Optional @@ -22,9 +23,10 @@ class Properties: uses_locals: bool has_free: bool side_exit: bool - pure: bool passthrough: bool + oparg_and_1: bool = False + const_oparg: int = -1 def dump(self, indent: str) -> None: print(indent, end="") @@ -141,6 +143,8 @@ class Uop: properties: Properties _size: int = -1 implicitly_created: bool = False + replicated = 0 + replicates : "Uop | None" = None def dump(self, indent: str) -> None: print( @@ -271,15 +275,19 @@ def override_error( ) -def convert_stack_item(item: parser.StackEffect) -> StackItem: - return StackItem(item.name, item.type, item.cond, (item.size or "1")) +def convert_stack_item(item: parser.StackEffect, replace_op_arg_1: str | None) -> StackItem: + cond = item.cond + if replace_op_arg_1 and OPARG_AND_1.match(item.cond): + cond = replace_op_arg_1 + return StackItem( + item.name, item.type, cond, (item.size or "1") + ) - -def analyze_stack(op: parser.InstDef) -> StackEffect: +def analyze_stack(op: parser.InstDef, replace_op_arg_1: str | None = None) -> StackEffect: inputs: list[StackItem] = [ - convert_stack_item(i) for i in op.inputs if isinstance(i, parser.StackEffect) + convert_stack_item(i, replace_op_arg_1) for i in op.inputs if isinstance(i, parser.StackEffect) ] - outputs: list[StackItem] = [convert_stack_item(i) for i in op.outputs] + outputs: list[StackItem] = [convert_stack_item(i, replace_op_arg_1) for i in op.outputs] for input, output in zip(inputs, outputs): if input.name == output.name: input.peek = output.peek = True @@ -442,6 +450,22 @@ def stack_effect_only_peeks(instr: parser.InstDef) -> bool: for s, other in zip(stack_inputs, instr.outputs) ) +OPARG_AND_1 = re.compile("\\(*oparg *& *1") + +def effect_depends_on_oparg_1(op: parser.InstDef) -> bool: + for effect in op.inputs: + if isinstance(effect, parser.CacheEffect): + continue + if not effect.cond: + continue + if OPARG_AND_1.match(effect.cond): + return True + for effect in op.outputs: + if not effect.cond: + continue + if OPARG_AND_1.match(effect.cond): + return True + return False def compute_properties(op: parser.InstDef) -> Properties: has_free = ( @@ -485,8 +509,8 @@ def compute_properties(op: parser.InstDef) -> Properties: ) -def make_uop(name: str, op: parser.InstDef, inputs: list[parser.InputEffect]) -> Uop: - return Uop( +def make_uop(name: str, op: parser.InstDef, inputs: list[parser.InputEffect], uops: dict[str, Uop]) -> Uop: + result = Uop( name=name, context=op.context, annotations=op.annotations, @@ -495,6 +519,49 @@ def make_uop(name: str, op: parser.InstDef, inputs: list[parser.InputEffect]) -> body=op.block.tokens, properties=compute_properties(op), ) + if effect_depends_on_oparg_1(op) and "split" in op.annotations: + result.properties.oparg_and_1 = True + for bit in ("0", "1"): + name_x = name + "_" + bit + properties = compute_properties(op) + if properties.oparg: + # May not need oparg anymore + properties.oparg = any(token.text == "oparg" for token in op.block.tokens) + rep = Uop( + name=name_x, + context=op.context, + annotations=op.annotations, + stack=analyze_stack(op, bit), + caches=analyze_caches(inputs), + body=op.block.tokens, + properties=properties, + ) + rep.replicates = result + uops[name_x] = rep + for anno in op.annotations: + if anno.startswith("replicate"): + result.replicated = int(anno[10:-1]) + break + else: + return result + for oparg in range(result.replicated): + name_x = name + "_" + str(oparg) + properties = compute_properties(op) + properties.oparg = False + properties.const_oparg = oparg + rep = Uop( + name=name_x, + context=op.context, + annotations=op.annotations, + stack=analyze_stack(op), + caches=analyze_caches(inputs), + body=op.block.tokens, + properties=properties, + ) + rep.replicates = result + uops[name_x] = rep + + return result def add_op(op: parser.InstDef, uops: dict[str, Uop]) -> None: @@ -504,7 +571,7 @@ def add_op(op: parser.InstDef, uops: dict[str, Uop]) -> None: raise override_error( op.name, op.context, uops[op.name].context, op.tokens[0] ) - uops[op.name] = make_uop(op.name, op, op.inputs) + uops[op.name] = make_uop(op.name, op, op.inputs, uops) def add_instruction( @@ -531,7 +598,7 @@ def desugar_inst( uop_index = len(parts) # Place holder for the uop. parts.append(Skip(0)) - uop = make_uop("_" + inst.name, inst, op_inputs) + uop = make_uop("_" + inst.name, inst, op_inputs, uops) uop.implicitly_created = True uops[inst.name] = uop if uop_index < 0: diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 54ffea71b53..0b4b99c6076 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -119,7 +119,10 @@ def replace_decrefs( out.emit(f"Py_DECREF({var.name}[_i]);\n") out.emit("}\n") elif var.condition: - out.emit(f"Py_XDECREF({var.name});\n") + if var.condition == "1": + out.emit(f"Py_DECREF({var.name});\n") + elif var.condition != "0": + out.emit(f"Py_XDECREF({var.name});\n") else: out.emit(f"Py_DECREF({var.name});\n") @@ -216,6 +219,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_PURE_FLAG") if p.passthrough: flags.append("HAS_PASSTHROUGH_FLAG") + if p.oparg_and_1: + flags.append("HAS_OPARG_AND_1_FLAG") if flags: return " | ".join(flags) else: diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 4f8d01c5492..0077921e7d7 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -222,6 +222,8 @@ annotations = { "register", "replaced", "pure", + "split", + "replicate", } __all__ = [] diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 52dc09e1499..24fbea6cfcb 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -53,6 +53,7 @@ FLAGS = [ "EXIT", "PURE", "PASSTHROUGH", + "OPARG_AND_1", ] diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index a8961f28bab..0d54820e4e7 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -179,7 +179,13 @@ class Parser(PLexer): # | annotation* op(NAME, (inputs -- outputs)) annotations = [] while anno := self.expect(lx.ANNOTATION): - annotations.append(anno.text) + if anno.text == "replicate": + self.require(lx.LPAREN) + times = self.require(lx.NUMBER) + self.require(lx.RPAREN) + annotations.append(f"replicate({times.text})") + else: + annotations.append(anno.text) tkn = self.expect(lx.INST) if not tkn: tkn = self.expect(lx.OP) diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 97a301142d5..5aecac39aef 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -23,8 +23,12 @@ def maybe_parenthesize(sym: str) -> str: def var_size(var: StackItem) -> str: if var.condition: - # Special case simplification - if var.condition == "oparg & 1" and var.size == "1": + # Special case simplifications + if var.condition == "0": + return "0" + elif var.condition == "1": + return var.size + elif var.condition == "oparg & 1" and var.size == "1": return f"({var.condition})" else: return f"(({var.condition}) ? {var.size} : 0)" @@ -154,7 +158,12 @@ class Stack: f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}];" ) if var.condition: - return f"if ({var.condition}) {{ {assign} }}\n" + if var.condition == "1": + return f"{assign}\n" + elif var.condition == "0": + return "" + else: + return f"if ({var.condition}) {{ {assign} }}\n" return f"{assign}\n" def push(self, var: StackItem) -> str: @@ -175,7 +184,10 @@ class Stack: cast = f"({cast_type})" if var.type else "" if var.name not in UNUSED and not var.is_array(): if var.condition: - out.emit(f"if ({var.condition}) ") + if var.condition == "0": + continue + elif var.condition != "1": + out.emit(f"if ({var.condition}) ") out.emit( f"stack_pointer[{self.base_offset.to_c()}] = {cast}{var.name};\n" ) diff --git a/Tools/cases_generator/tier2_abstract_generator.py b/Tools/cases_generator/tier2_abstract_generator.py index cc29b1660d2..47a862643d9 100644 --- a/Tools/cases_generator/tier2_abstract_generator.py +++ b/Tools/cases_generator/tier2_abstract_generator.py @@ -178,6 +178,8 @@ def generate_abstract_interpreter( validate_uop(override, uop) if uop.properties.tier_one_only: continue + if uop.replicates: + continue if uop.is_super(): continue if not uop.is_viable(): diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 8b4d1646dd5..6fbe5c355c9 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -33,24 +33,29 @@ from stack import StackOffset, Stack, SizeMismatch DEFAULT_OUTPUT = ROOT / "Python/executor_cases.c.h" +def declare_variable( + var: StackItem, uop: Uop, variables: set[str], out: CWriter +) -> None: + if var.name in variables: + return + type = var.type if var.type else "PyObject *" + variables.add(var.name) + if var.condition: + out.emit(f"{type}{var.name} = NULL;\n") + if uop.replicates: + # Replicas may not use all their conditional variables + # So avoid a compiler warning with a fake use + out.emit(f"(void){var.name};\n") + else: + out.emit(f"{type}{var.name};\n") + + def declare_variables(uop: Uop, out: CWriter) -> None: variables = {"unused"} for var in reversed(uop.stack.inputs): - if var.name not in variables: - type = var.type if var.type else "PyObject *" - variables.add(var.name) - if var.condition: - out.emit(f"{type}{var.name} = NULL;\n") - else: - out.emit(f"{type}{var.name};\n") + declare_variable(var, uop, variables, out) for var in uop.stack.outputs: - if var.name not in variables: - variables.add(var.name) - type = var.type if var.type else "PyObject *" - if var.condition: - out.emit(f"{type}{var.name} = NULL;\n") - else: - out.emit(f"{type}{var.name};\n") + declare_variable(var, uop, variables, out) def tier2_replace_error( @@ -113,9 +118,31 @@ def tier2_replace_exit_if( out.emit(") goto side_exit;\n") +def tier2_replace_oparg( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, +) -> None: + if not uop.name.endswith("_0") and not uop.name.endswith("_1"): + out.emit(tkn) + return + amp = next(tkn_iter) + if amp.text != "&": + out.emit(tkn) + out.emit(amp) + return + one = next(tkn_iter) + assert one.text == "1" + out.emit_at(uop.name[-1], tkn) + + TIER2_REPLACEMENT_FUNCTIONS = REPLACEMENT_FUNCTIONS.copy() TIER2_REPLACEMENT_FUNCTIONS["ERROR_IF"] = tier2_replace_error TIER2_REPLACEMENT_FUNCTIONS["DEOPT_IF"] = tier2_replace_deopt +TIER2_REPLACEMENT_FUNCTIONS["oparg"] = tier2_replace_oparg TIER2_REPLACEMENT_FUNCTIONS["EXIT_IF"] = tier2_replace_exit_if @@ -124,6 +151,10 @@ def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None: out.start_line() if uop.properties.oparg: out.emit("oparg = CURRENT_OPARG();\n") + assert uop.properties.const_oparg < 0 + elif uop.properties.const_oparg >= 0: + out.emit(f"oparg = {uop.properties.const_oparg};\n") + out.emit(f"assert(oparg == CURRENT_OPARG());\n") for var in reversed(uop.stack.inputs): out.emit(stack.pop(var)) if not uop.properties.stores_sp: @@ -165,6 +196,9 @@ def generate_tier2( for name, uop in analysis.uops.items(): if uop.properties.tier_one_only: continue + if uop.properties.oparg_and_1: + out.emit(f"/* {uop.name} is split on (oparg & 1) */\n\n") + continue if uop.is_super(): continue if not uop.is_viable(): diff --git a/Tools/cases_generator/uop_id_generator.py b/Tools/cases_generator/uop_id_generator.py index 633249f1c6b..907158f2795 100644 --- a/Tools/cases_generator/uop_id_generator.py +++ b/Tools/cases_generator/uop_id_generator.py @@ -38,15 +38,17 @@ def generate_uop_ids( next_id += 1 PRE_DEFINED = {"_EXIT_TRACE", "_SET_IP"} - for uop in analysis.uops.values(): - if uop.name in PRE_DEFINED: + uops = [(uop.name, uop) for uop in analysis.uops.values()] + # Sort so that _BASE comes immediately before _BASE_0, etc. + for name, uop in sorted(uops): + if name in PRE_DEFINED: continue if uop.properties.tier_one_only: continue - if uop.implicitly_created and not distinct_namespace: - out.emit(f"#define {uop.name} {uop.name[1:]}\n") + if uop.implicitly_created and not distinct_namespace and not uop.replicated: + out.emit(f"#define {name} {name[1:]}\n") else: - out.emit(f"#define {uop.name} {next_id}\n") + out.emit(f"#define {name} {next_id}\n") next_id += 1 out.emit(f"#define MAX_UOP_ID {next_id-1}\n") diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py index 9083ecc48bd..f85f1c6ce9c 100644 --- a/Tools/cases_generator/uop_metadata_generator.py +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -24,6 +24,7 @@ DEFAULT_OUTPUT = ROOT / "Include/internal/pycore_uop_metadata.h" def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: out.emit("extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];\n") + out.emit("extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1];\n") out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];\n\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit("const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") @@ -31,6 +32,12 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: if uop.is_viable() and not uop.properties.tier_one_only: out.emit(f"[{uop.name}] = {cflags(uop.properties)},\n") + out.emit("};\n\n") + out.emit("const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = {\n") + for uop in analysis.uops.values(): + if uop.replicated: + out.emit(f"[{uop.name}] = {uop.replicated},\n") + out.emit("};\n\n") out.emit("const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {\n") for uop in sorted(analysis.uops.values(), key=lambda t: t.name):