From 3b3d30e8f78271a488965c9cd11136e1aa890757 Mon Sep 17 00:00:00 2001 From: Dennis Sweeney <36520290+sweeneyde@users.noreply.github.com> Date: Thu, 14 Oct 2021 10:56:33 -0400 Subject: [PATCH] bpo-45367: Specialize BINARY_MULTIPLY (GH-28727) --- Include/internal/pycore_code.h | 3 +- Include/internal/pycore_long.h | 1 + Include/opcode.h | 57 +++++++++-------- Lib/opcode.py | 3 + .../2021-10-05-03-49-07.bpo-45367._astoU.rst | 1 + Objects/longobject.c | 13 ++-- Python/ceval.c | 62 ++++++++++++++++++- Python/opcode_targets.h | 44 ++++++------- Python/specialize.c | 33 ++++++++++ 9 files changed, 162 insertions(+), 55 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-10-05-03-49-07.bpo-45367._astoU.rst diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 0b127ed2899..a91209b39f6 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -307,7 +307,8 @@ int _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *nam int _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache); int _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache); int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr); -int _Py_Specialize_BinaryAdd(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr); +int _Py_Specialize_BinaryAdd(PyObject *left, PyObject *right, _Py_CODEUNIT *instr); +int _Py_Specialize_BinaryMultiply(PyObject *left, PyObject *right, _Py_CODEUNIT *instr); #define PRINT_SPECIALIZATION_STATS 0 #define PRINT_SPECIALIZATION_STATS_DETAILED 0 diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 7336c317c3f..8edc9021854 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -35,6 +35,7 @@ static inline PyObject* _PyLong_GetOne(void) { return __PyLong_GetSmallInt_internal(1); } PyObject *_PyLong_Add(PyLongObject *left, PyLongObject *right); +PyObject *_PyLong_Multiply(PyLongObject *left, PyLongObject *right); #ifdef __cplusplus } diff --git a/Include/opcode.h b/Include/opcode.h index 15f722630dc..fabb8d123bb 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -141,33 +141,36 @@ extern "C" { #define BINARY_ADD_FLOAT 13 #define BINARY_ADD_UNICODE 14 #define BINARY_ADD_UNICODE_INPLACE_FAST 18 -#define BINARY_SUBSCR_ADAPTIVE 21 -#define BINARY_SUBSCR_LIST_INT 36 -#define BINARY_SUBSCR_TUPLE_INT 38 -#define BINARY_SUBSCR_DICT 39 -#define JUMP_ABSOLUTE_QUICK 40 -#define LOAD_ATTR_ADAPTIVE 41 -#define LOAD_ATTR_INSTANCE_VALUE 42 -#define LOAD_ATTR_WITH_HINT 43 -#define LOAD_ATTR_SLOT 44 -#define LOAD_ATTR_MODULE 45 -#define LOAD_GLOBAL_ADAPTIVE 46 -#define LOAD_GLOBAL_MODULE 47 -#define LOAD_GLOBAL_BUILTIN 48 -#define LOAD_METHOD_ADAPTIVE 58 -#define LOAD_METHOD_CACHED 80 -#define LOAD_METHOD_CLASS 81 -#define LOAD_METHOD_MODULE 87 -#define LOAD_METHOD_NO_DICT 88 -#define STORE_ATTR_ADAPTIVE 120 -#define STORE_ATTR_INSTANCE_VALUE 122 -#define STORE_ATTR_SLOT 123 -#define STORE_ATTR_WITH_HINT 127 -#define LOAD_FAST__LOAD_FAST 128 -#define STORE_FAST__LOAD_FAST 134 -#define LOAD_FAST__LOAD_CONST 140 -#define LOAD_CONST__LOAD_FAST 143 -#define STORE_FAST__STORE_FAST 149 +#define BINARY_MULTIPLY_ADAPTIVE 21 +#define BINARY_MULTIPLY_INT 36 +#define BINARY_MULTIPLY_FLOAT 38 +#define BINARY_SUBSCR_ADAPTIVE 39 +#define BINARY_SUBSCR_LIST_INT 40 +#define BINARY_SUBSCR_TUPLE_INT 41 +#define BINARY_SUBSCR_DICT 42 +#define JUMP_ABSOLUTE_QUICK 43 +#define LOAD_ATTR_ADAPTIVE 44 +#define LOAD_ATTR_INSTANCE_VALUE 45 +#define LOAD_ATTR_WITH_HINT 46 +#define LOAD_ATTR_SLOT 47 +#define LOAD_ATTR_MODULE 48 +#define LOAD_GLOBAL_ADAPTIVE 58 +#define LOAD_GLOBAL_MODULE 80 +#define LOAD_GLOBAL_BUILTIN 81 +#define LOAD_METHOD_ADAPTIVE 87 +#define LOAD_METHOD_CACHED 88 +#define LOAD_METHOD_CLASS 120 +#define LOAD_METHOD_MODULE 122 +#define LOAD_METHOD_NO_DICT 123 +#define STORE_ATTR_ADAPTIVE 127 +#define STORE_ATTR_INSTANCE_VALUE 128 +#define STORE_ATTR_SLOT 134 +#define STORE_ATTR_WITH_HINT 140 +#define LOAD_FAST__LOAD_FAST 143 +#define STORE_FAST__LOAD_FAST 149 +#define LOAD_FAST__LOAD_CONST 150 +#define LOAD_CONST__LOAD_FAST 151 +#define STORE_FAST__STORE_FAST 153 #define DO_TRACING 255 #ifdef NEED_OPCODE_JUMP_TABLES static uint32_t _PyOpcode_RelativeJump[8] = { diff --git a/Lib/opcode.py b/Lib/opcode.py index efd6aefccc5..9d9f35855e2 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -225,6 +225,9 @@ _specialized_instructions = [ "BINARY_ADD_FLOAT", "BINARY_ADD_UNICODE", "BINARY_ADD_UNICODE_INPLACE_FAST", + "BINARY_MULTIPLY_ADAPTIVE", + "BINARY_MULTIPLY_INT", + "BINARY_MULTIPLY_FLOAT", "BINARY_SUBSCR_ADAPTIVE", "BINARY_SUBSCR_LIST_INT", "BINARY_SUBSCR_TUPLE_INT", diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-05-03-49-07.bpo-45367._astoU.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-05-03-49-07.bpo-45367._astoU.rst new file mode 100644 index 00000000000..aae518da1e8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-05-03-49-07.bpo-45367._astoU.rst @@ -0,0 +1 @@ +Specialized the ``BINARY_MULTIPLY`` opcode to ``BINARY_MULTIPLY_INT`` and ``BINARY_MULTIPLY_FLOAT`` using the PEP 659 machinery. \ No newline at end of file diff --git a/Objects/longobject.c b/Objects/longobject.c index 111ef4566cd..66e164974a9 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3594,13 +3594,11 @@ k_lopsided_mul(PyLongObject *a, PyLongObject *b) return NULL; } -static PyObject * -long_mul(PyLongObject *a, PyLongObject *b) +PyObject * +_PyLong_Multiply(PyLongObject *a, PyLongObject *b) { PyLongObject *z; - CHECK_BINOP(a, b); - /* fast path for single-digit multiplication */ if (IS_MEDIUM_VALUE(a) && IS_MEDIUM_VALUE(b)) { stwodigits v = medium_value(a) * medium_value(b); @@ -3617,6 +3615,13 @@ long_mul(PyLongObject *a, PyLongObject *b) return (PyObject *)z; } +static PyObject * +long_mul(PyLongObject *a, PyLongObject *b) +{ + CHECK_BINOP(a, b); + return _PyLong_Multiply(a, b); +} + /* Fast modulo division for single-digit longs. */ static PyObject * fast_mod(PyLongObject *a, PyLongObject *b) diff --git a/Python/ceval.c b/Python/ceval.c index aef83b9c355..de71ae5da0f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1933,14 +1933,73 @@ check_eval_breaker: } TARGET(BINARY_MULTIPLY) { + PREDICTED(BINARY_MULTIPLY); + STAT_INC(BINARY_MULTIPLY, unquickened); PyObject *right = POP(); PyObject *left = TOP(); PyObject *res = PyNumber_Multiply(left, right); Py_DECREF(left); Py_DECREF(right); SET_TOP(res); - if (res == NULL) + if (res == NULL) { goto error; + } + DISPATCH(); + } + + TARGET(BINARY_MULTIPLY_ADAPTIVE) { + if (oparg == 0) { + PyObject *left = SECOND(); + PyObject *right = TOP(); + next_instr--; + if (_Py_Specialize_BinaryMultiply(left, right, next_instr) < 0) { + goto error; + } + DISPATCH(); + } + else { + STAT_INC(BINARY_MULTIPLY, deferred); + UPDATE_PREV_INSTR_OPARG(next_instr, oparg - 1); + STAT_DEC(BINARY_MULTIPLY, unquickened); + JUMP_TO_INSTRUCTION(BINARY_MULTIPLY); + } + } + + TARGET(BINARY_MULTIPLY_INT) { + PyObject *left = SECOND(); + PyObject *right = TOP(); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_MULTIPLY); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_MULTIPLY); + STAT_INC(BINARY_MULTIPLY, hit); + record_hit_inline(next_instr, oparg); + PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); + SET_SECOND(prod); + Py_DECREF(right); + Py_DECREF(left); + STACK_SHRINK(1); + if (prod == NULL) { + goto error; + } + DISPATCH(); + } + + TARGET(BINARY_MULTIPLY_FLOAT) { + PyObject *left = SECOND(); + PyObject *right = TOP(); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_MULTIPLY); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_MULTIPLY); + STAT_INC(BINARY_MULTIPLY, hit); + record_hit_inline(next_instr, oparg); + double dprod = ((PyFloatObject *)left)->ob_fval * + ((PyFloatObject *)right)->ob_fval; + PyObject *prod = PyFloat_FromDouble(dprod); + SET_SECOND(prod); + Py_DECREF(right); + Py_DECREF(left); + STACK_SHRINK(1); + if (prod == NULL) { + goto error; + } DISPATCH(); } @@ -4954,6 +5013,7 @@ MISS_WITH_CACHE(LOAD_GLOBAL) MISS_WITH_CACHE(LOAD_METHOD) MISS_WITH_OPARG_COUNTER(BINARY_SUBSCR) MISS_WITH_OPARG_COUNTER(BINARY_ADD) +MISS_WITH_OPARG_COUNTER(BINARY_MULTIPLY) binary_subscr_dict_error: { diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 773f9254cf6..7d701e81a9a 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -20,7 +20,7 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_ADD_UNICODE_INPLACE_FAST, &&TARGET_BINARY_POWER, &&TARGET_BINARY_MULTIPLY, - &&TARGET_BINARY_SUBSCR_ADAPTIVE, + &&TARGET_BINARY_MULTIPLY_ADAPTIVE, &&TARGET_BINARY_MODULO, &&TARGET_BINARY_ADD, &&TARGET_BINARY_SUBTRACT, @@ -35,8 +35,11 @@ static void *opcode_targets[256] = { &&TARGET_MATCH_KEYS, &&TARGET_COPY_DICT_WITHOUT_KEYS, &&TARGET_PUSH_EXC_INFO, - &&TARGET_BINARY_SUBSCR_LIST_INT, + &&TARGET_BINARY_MULTIPLY_INT, &&TARGET_POP_EXCEPT_AND_RERAISE, + &&TARGET_BINARY_MULTIPLY_FLOAT, + &&TARGET_BINARY_SUBSCR_ADAPTIVE, + &&TARGET_BINARY_SUBSCR_LIST_INT, &&TARGET_BINARY_SUBSCR_TUPLE_INT, &&TARGET_BINARY_SUBSCR_DICT, &&TARGET_JUMP_ABSOLUTE_QUICK, @@ -45,9 +48,6 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_MODULE, - &&TARGET_LOAD_GLOBAL_ADAPTIVE, - &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -57,7 +57,7 @@ static void *opcode_targets[256] = { &&TARGET_INPLACE_ADD, &&TARGET_INPLACE_SUBTRACT, &&TARGET_INPLACE_MULTIPLY, - &&TARGET_LOAD_METHOD_ADAPTIVE, + &&TARGET_LOAD_GLOBAL_ADAPTIVE, &&TARGET_INPLACE_MODULO, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, @@ -79,15 +79,15 @@ static void *opcode_targets[256] = { &&TARGET_INPLACE_AND, &&TARGET_INPLACE_XOR, &&TARGET_INPLACE_OR, - &&TARGET_LOAD_METHOD_CACHED, - &&TARGET_LOAD_METHOD_CLASS, + &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, &&TARGET_YIELD_VALUE, - &&TARGET_LOAD_METHOD_MODULE, - &&TARGET_LOAD_METHOD_NO_DICT, + &&TARGET_LOAD_METHOD_ADAPTIVE, + &&TARGET_LOAD_METHOD_CACHED, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, &&TARGET_DELETE_NAME, @@ -119,40 +119,40 @@ static void *opcode_targets[256] = { &&TARGET_IS_OP, &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, - &&TARGET_STORE_ATTR_ADAPTIVE, + &&TARGET_LOAD_METHOD_CLASS, &&TARGET_JUMP_IF_NOT_EXC_MATCH, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, - &&TARGET_STORE_ATTR_SLOT, + &&TARGET_LOAD_METHOD_MODULE, + &&TARGET_LOAD_METHOD_NO_DICT, &&TARGET_LOAD_FAST, &&TARGET_STORE_FAST, &&TARGET_DELETE_FAST, - &&TARGET_STORE_ATTR_WITH_HINT, - &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_STORE_ATTR_ADAPTIVE, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_GEN_START, &&TARGET_RAISE_VARARGS, &&TARGET_CALL_FUNCTION, &&TARGET_MAKE_FUNCTION, &&TARGET_BUILD_SLICE, - &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_STORE_ATTR_SLOT, &&TARGET_MAKE_CELL, &&TARGET_LOAD_CLOSURE, &&TARGET_LOAD_DEREF, &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, - &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_CALL_FUNCTION_KW, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, &&TARGET_MAP_ADD, &&TARGET_LOAD_CLASSDEREF, - &&TARGET_STORE_FAST__STORE_FAST, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_MATCH_CLASS, - &&_unknown_opcode, + &&TARGET_STORE_FAST__STORE_FAST, &&_unknown_opcode, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, diff --git a/Python/specialize.c b/Python/specialize.c index 6efee7643a4..529eabf6bc3 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -124,6 +124,7 @@ _Py_GetSpecializationStats(void) { err += add_stat_dict(stats, LOAD_GLOBAL, "load_global"); err += add_stat_dict(stats, LOAD_METHOD, "load_method"); err += add_stat_dict(stats, BINARY_ADD, "binary_add"); + err += add_stat_dict(stats, BINARY_MULTIPLY, "binary_multiply"); err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr"); err += add_stat_dict(stats, STORE_ATTR, "store_attr"); if (err < 0) { @@ -180,6 +181,7 @@ _Py_PrintSpecializationStats(void) print_stats(out, &_specialization_stats[LOAD_GLOBAL], "load_global"); print_stats(out, &_specialization_stats[LOAD_METHOD], "load_method"); print_stats(out, &_specialization_stats[BINARY_ADD], "binary_add"); + print_stats(out, &_specialization_stats[BINARY_MULTIPLY], "binary_multiply"); print_stats(out, &_specialization_stats[BINARY_SUBSCR], "binary_subscr"); print_stats(out, &_specialization_stats[STORE_ATTR], "store_attr"); if (out != stderr) { @@ -230,6 +232,7 @@ static uint8_t adaptive_opcodes[256] = { [LOAD_GLOBAL] = LOAD_GLOBAL_ADAPTIVE, [LOAD_METHOD] = LOAD_METHOD_ADAPTIVE, [BINARY_ADD] = BINARY_ADD_ADAPTIVE, + [BINARY_MULTIPLY] = BINARY_MULTIPLY_ADAPTIVE, [BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE, [STORE_ATTR] = STORE_ATTR_ADAPTIVE, }; @@ -240,6 +243,7 @@ static uint8_t cache_requirements[256] = { [LOAD_GLOBAL] = 2, /* _PyAdaptiveEntry and _PyLoadGlobalCache */ [LOAD_METHOD] = 3, /* _PyAdaptiveEntry, _PyAttrCache and _PyObjectCache */ [BINARY_ADD] = 0, + [BINARY_MULTIPLY] = 0, [BINARY_SUBSCR] = 0, [STORE_ATTR] = 2, /* _PyAdaptiveEntry and _PyAttrCache */ }; @@ -1188,3 +1192,32 @@ success: assert(!PyErr_Occurred()); return 0; } + +int +_Py_Specialize_BinaryMultiply(PyObject *left, PyObject *right, _Py_CODEUNIT *instr) +{ + if (!Py_IS_TYPE(left, Py_TYPE(right))) { + SPECIALIZATION_FAIL(BINARY_MULTIPLY, SPEC_FAIL_DIFFERENT_TYPES); + goto fail; + } + if (PyLong_CheckExact(left)) { + *instr = _Py_MAKECODEUNIT(BINARY_MULTIPLY_INT, saturating_start()); + goto success; + } + else if (PyFloat_CheckExact(left)) { + *instr = _Py_MAKECODEUNIT(BINARY_MULTIPLY_FLOAT, saturating_start()); + goto success; + } + else { + SPECIALIZATION_FAIL(BINARY_MULTIPLY, SPEC_FAIL_OTHER); + } +fail: + STAT_INC(BINARY_MULTIPLY, specialization_failure); + assert(!PyErr_Occurred()); + *instr = _Py_MAKECODEUNIT(_Py_OPCODE(*instr), ADAPTIVE_CACHE_BACKOFF); + return 0; +success: + STAT_INC(BINARY_MULTIPLY, specialization_success); + assert(!PyErr_Occurred()); + return 0; +}