Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

bpo-45367: Specialize BINARY_MULTIPLY (GH-28727)

This commit is contained in:
Dennis Sweeney 2021-10-14 10:56:33 -04:00 committed by GitHub
parent c96d1546b1
commit 3b3d30e8f7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 162 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Include/internal/pycore_code.h
View File

@ -307,7 +307,8 @@ int _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *nam
int _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache);
int _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache);
int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr);
int _Py_Specialize_BinaryAdd(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr);
int _Py_Specialize_BinaryAdd(PyObject *left, PyObject *right, _Py_CODEUNIT *instr);
int _Py_Specialize_BinaryMultiply(PyObject *left, PyObject *right, _Py_CODEUNIT *instr);
#define PRINT_SPECIALIZATION_STATS 0
#define PRINT_SPECIALIZATION_STATS_DETAILED 0

1
Include/internal/pycore_long.h
View File

@ -35,6 +35,7 @@ static inline PyObject* _PyLong_GetOne(void)
{ return __PyLong_GetSmallInt_internal(1); }
PyObject *_PyLong_Add(PyLongObject *left, PyLongObject *right);
PyObject *_PyLong_Multiply(PyLongObject *left, PyLongObject *right);
#ifdef __cplusplus
}

57
Include/opcode.h generated
View File

@ -141,33 +141,36 @@ extern "C" {
#define BINARY_ADD_FLOAT 13
#define BINARY_ADD_UNICODE 14
#define BINARY_ADD_UNICODE_INPLACE_FAST 18
#define BINARY_SUBSCR_ADAPTIVE 21
#define BINARY_SUBSCR_LIST_INT 36
#define BINARY_SUBSCR_TUPLE_INT 38
#define BINARY_SUBSCR_DICT 39
#define JUMP_ABSOLUTE_QUICK 40
#define LOAD_ATTR_ADAPTIVE 41
#define LOAD_ATTR_INSTANCE_VALUE 42
#define LOAD_ATTR_WITH_HINT 43
#define LOAD_ATTR_SLOT 44
#define LOAD_ATTR_MODULE 45
#define LOAD_GLOBAL_ADAPTIVE 46
#define LOAD_GLOBAL_MODULE 47
#define LOAD_GLOBAL_BUILTIN 48
#define LOAD_METHOD_ADAPTIVE 58
#define LOAD_METHOD_CACHED 80
#define LOAD_METHOD_CLASS 81
#define LOAD_METHOD_MODULE 87
#define LOAD_METHOD_NO_DICT 88
#define STORE_ATTR_ADAPTIVE 120
#define STORE_ATTR_INSTANCE_VALUE 122
#define STORE_ATTR_SLOT 123
#define STORE_ATTR_WITH_HINT 127
#define LOAD_FAST__LOAD_FAST 128
#define STORE_FAST__LOAD_FAST 134
#define LOAD_FAST__LOAD_CONST 140
#define LOAD_CONST__LOAD_FAST 143
#define STORE_FAST__STORE_FAST 149
#define BINARY_MULTIPLY_ADAPTIVE 21
#define BINARY_MULTIPLY_INT 36
#define BINARY_MULTIPLY_FLOAT 38
#define BINARY_SUBSCR_ADAPTIVE 39
#define BINARY_SUBSCR_LIST_INT 40
#define BINARY_SUBSCR_TUPLE_INT 41
#define BINARY_SUBSCR_DICT 42
#define JUMP_ABSOLUTE_QUICK 43
#define LOAD_ATTR_ADAPTIVE 44
#define LOAD_ATTR_INSTANCE_VALUE 45
#define LOAD_ATTR_WITH_HINT 46
#define LOAD_ATTR_SLOT 47
#define LOAD_ATTR_MODULE 48
#define LOAD_GLOBAL_ADAPTIVE 58
#define LOAD_GLOBAL_MODULE 80
#define LOAD_GLOBAL_BUILTIN 81
#define LOAD_METHOD_ADAPTIVE 87
#define LOAD_METHOD_CACHED 88
#define LOAD_METHOD_CLASS 120
#define LOAD_METHOD_MODULE 122
#define LOAD_METHOD_NO_DICT 123
#define STORE_ATTR_ADAPTIVE 127
#define STORE_ATTR_INSTANCE_VALUE 128
#define STORE_ATTR_SLOT 134
#define STORE_ATTR_WITH_HINT 140
#define LOAD_FAST__LOAD_FAST 143
#define STORE_FAST__LOAD_FAST 149
#define LOAD_FAST__LOAD_CONST 150
#define LOAD_CONST__LOAD_FAST 151
#define STORE_FAST__STORE_FAST 153
#define DO_TRACING 255
#ifdef NEED_OPCODE_JUMP_TABLES
static uint32_t _PyOpcode_RelativeJump[8] = {

3
Lib/opcode.py
View File

@ -225,6 +225,9 @@ _specialized_instructions = [
"BINARY_ADD_FLOAT",
"BINARY_ADD_UNICODE",
"BINARY_ADD_UNICODE_INPLACE_FAST",
"BINARY_MULTIPLY_ADAPTIVE",
"BINARY_MULTIPLY_INT",
"BINARY_MULTIPLY_FLOAT",
"BINARY_SUBSCR_ADAPTIVE",
"BINARY_SUBSCR_LIST_INT",
"BINARY_SUBSCR_TUPLE_INT",

1
Misc/NEWS.d/next/Core and Builtins/2021-10-05-03-49-07.bpo-45367._astoU.rst Normal file
View File

@ -0,0 +1 @@
Specialized the ``BINARY_MULTIPLY`` opcode to ``BINARY_MULTIPLY_INT`` and ``BINARY_MULTIPLY_FLOAT`` using the PEP 659 machinery.

13
Objects/longobject.c
View File

@ -3594,13 +3594,11 @@ k_lopsided_mul(PyLongObject *a, PyLongObject *b)
return NULL;
}
static PyObject *
long_mul(PyLongObject *a, PyLongObject *b)
PyObject *
_PyLong_Multiply(PyLongObject *a, PyLongObject *b)
{
PyLongObject *z;
CHECK_BINOP(a, b);
/* fast path for single-digit multiplication */
if (IS_MEDIUM_VALUE(a) && IS_MEDIUM_VALUE(b)) {
stwodigits v = medium_value(a) * medium_value(b);
@ -3617,6 +3615,13 @@ long_mul(PyLongObject *a, PyLongObject *b)
return (PyObject *)z;
}
static PyObject *
long_mul(PyLongObject *a, PyLongObject *b)
{
CHECK_BINOP(a, b);
return _PyLong_Multiply(a, b);
}
/* Fast modulo division for single-digit longs. */
static PyObject *
fast_mod(PyLongObject *a, PyLongObject *b)

62
Python/ceval.c
View File

@ -1933,14 +1933,73 @@ check_eval_breaker:
}
TARGET(BINARY_MULTIPLY) {
PREDICTED(BINARY_MULTIPLY);
STAT_INC(BINARY_MULTIPLY, unquickened);
PyObject *right = POP();
PyObject *left = TOP();
PyObject *res = PyNumber_Multiply(left, right);
Py_DECREF(left);
Py_DECREF(right);
SET_TOP(res);
if (res == NULL)
if (res == NULL) {
goto error;
}
DISPATCH();
}
TARGET(BINARY_MULTIPLY_ADAPTIVE) {
if (oparg == 0) {
PyObject *left = SECOND();
PyObject *right = TOP();
next_instr--;
if (_Py_Specialize_BinaryMultiply(left, right, next_instr) < 0) {
goto error;
}
DISPATCH();
}
else {
STAT_INC(BINARY_MULTIPLY, deferred);
UPDATE_PREV_INSTR_OPARG(next_instr, oparg - 1);
STAT_DEC(BINARY_MULTIPLY, unquickened);
JUMP_TO_INSTRUCTION(BINARY_MULTIPLY);
}
}
TARGET(BINARY_MULTIPLY_INT) {
PyObject *left = SECOND();
PyObject *right = TOP();
DEOPT_IF(!PyLong_CheckExact(left), BINARY_MULTIPLY);
DEOPT_IF(!PyLong_CheckExact(right), BINARY_MULTIPLY);
STAT_INC(BINARY_MULTIPLY, hit);
record_hit_inline(next_instr, oparg);
PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);
SET_SECOND(prod);
Py_DECREF(right);
Py_DECREF(left);
STACK_SHRINK(1);
if (prod == NULL) {
goto error;
}
DISPATCH();
}
TARGET(BINARY_MULTIPLY_FLOAT) {
PyObject *left = SECOND();
PyObject *right = TOP();
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_MULTIPLY);
DEOPT_IF(!PyFloat_CheckExact(right), BINARY_MULTIPLY);
STAT_INC(BINARY_MULTIPLY, hit);
record_hit_inline(next_instr, oparg);
double dprod = ((PyFloatObject *)left)->ob_fval *
((PyFloatObject *)right)->ob_fval;
PyObject *prod = PyFloat_FromDouble(dprod);
SET_SECOND(prod);
Py_DECREF(right);
Py_DECREF(left);
STACK_SHRINK(1);
if (prod == NULL) {
goto error;
}
DISPATCH();
}
@ -4954,6 +5013,7 @@ MISS_WITH_CACHE(LOAD_GLOBAL)
MISS_WITH_CACHE(LOAD_METHOD)
MISS_WITH_OPARG_COUNTER(BINARY_SUBSCR)
MISS_WITH_OPARG_COUNTER(BINARY_ADD)
MISS_WITH_OPARG_COUNTER(BINARY_MULTIPLY)
binary_subscr_dict_error:
{

44
Python/opcode_targets.h generated
View File

@ -20,7 +20,7 @@ static void *opcode_targets[256] = {
&&TARGET_BINARY_ADD_UNICODE_INPLACE_FAST,
&&TARGET_BINARY_POWER,
&&TARGET_BINARY_MULTIPLY,
&&TARGET_BINARY_SUBSCR_ADAPTIVE,
&&TARGET_BINARY_MULTIPLY_ADAPTIVE,
&&TARGET_BINARY_MODULO,
&&TARGET_BINARY_ADD,
&&TARGET_BINARY_SUBTRACT,
@ -35,8 +35,11 @@ static void *opcode_targets[256] = {
&&TARGET_MATCH_KEYS,
&&TARGET_COPY_DICT_WITHOUT_KEYS,
&&TARGET_PUSH_EXC_INFO,
&&TARGET_BINARY_SUBSCR_LIST_INT,
&&TARGET_BINARY_MULTIPLY_INT,
&&TARGET_POP_EXCEPT_AND_RERAISE,
&&TARGET_BINARY_MULTIPLY_FLOAT,
&&TARGET_BINARY_SUBSCR_ADAPTIVE,
&&TARGET_BINARY_SUBSCR_LIST_INT,
&&TARGET_BINARY_SUBSCR_TUPLE_INT,
&&TARGET_BINARY_SUBSCR_DICT,
&&TARGET_JUMP_ABSOLUTE_QUICK,
@ -45,9 +48,6 @@ static void *opcode_targets[256] = {
&&TARGET_LOAD_ATTR_WITH_HINT,
&&TARGET_LOAD_ATTR_SLOT,
&&TARGET_LOAD_ATTR_MODULE,
&&TARGET_LOAD_GLOBAL_ADAPTIVE,
&&TARGET_LOAD_GLOBAL_MODULE,
&&TARGET_LOAD_GLOBAL_BUILTIN,
&&TARGET_WITH_EXCEPT_START,
&&TARGET_GET_AITER,
&&TARGET_GET_ANEXT,
@ -57,7 +57,7 @@ static void *opcode_targets[256] = {
&&TARGET_INPLACE_ADD,
&&TARGET_INPLACE_SUBTRACT,
&&TARGET_INPLACE_MULTIPLY,
&&TARGET_LOAD_METHOD_ADAPTIVE,
&&TARGET_LOAD_GLOBAL_ADAPTIVE,
&&TARGET_INPLACE_MODULO,
&&TARGET_STORE_SUBSCR,
&&TARGET_DELETE_SUBSCR,
@ -79,15 +79,15 @@ static void *opcode_targets[256] = {
&&TARGET_INPLACE_AND,
&&TARGET_INPLACE_XOR,
&&TARGET_INPLACE_OR,
&&TARGET_LOAD_METHOD_CACHED,
&&TARGET_LOAD_METHOD_CLASS,
&&TARGET_LOAD_GLOBAL_MODULE,
&&TARGET_LOAD_GLOBAL_BUILTIN,
&&TARGET_LIST_TO_TUPLE,
&&TARGET_RETURN_VALUE,
&&TARGET_IMPORT_STAR,
&&TARGET_SETUP_ANNOTATIONS,
&&TARGET_YIELD_VALUE,
&&TARGET_LOAD_METHOD_MODULE,
&&TARGET_LOAD_METHOD_NO_DICT,
&&TARGET_LOAD_METHOD_ADAPTIVE,
&&TARGET_LOAD_METHOD_CACHED,
&&TARGET_POP_EXCEPT,
&&TARGET_STORE_NAME,
&&TARGET_DELETE_NAME,
@ -119,40 +119,40 @@ static void *opcode_targets[256] = {
&&TARGET_IS_OP,
&&TARGET_CONTAINS_OP,
&&TARGET_RERAISE,
&&TARGET_STORE_ATTR_ADAPTIVE,
&&TARGET_LOAD_METHOD_CLASS,
&&TARGET_JUMP_IF_NOT_EXC_MATCH,
&&TARGET_STORE_ATTR_INSTANCE_VALUE,
&&TARGET_STORE_ATTR_SLOT,
&&TARGET_LOAD_METHOD_MODULE,
&&TARGET_LOAD_METHOD_NO_DICT,
&&TARGET_LOAD_FAST,
&&TARGET_STORE_FAST,
&&TARGET_DELETE_FAST,
&&TARGET_STORE_ATTR_WITH_HINT,
&&TARGET_LOAD_FAST__LOAD_FAST,
&&TARGET_STORE_ATTR_ADAPTIVE,
&&TARGET_STORE_ATTR_INSTANCE_VALUE,
&&TARGET_GEN_START,
&&TARGET_RAISE_VARARGS,
&&TARGET_CALL_FUNCTION,
&&TARGET_MAKE_FUNCTION,
&&TARGET_BUILD_SLICE,
&&TARGET_STORE_FAST__LOAD_FAST,
&&TARGET_STORE_ATTR_SLOT,
&&TARGET_MAKE_CELL,
&&TARGET_LOAD_CLOSURE,
&&TARGET_LOAD_DEREF,
&&TARGET_STORE_DEREF,
&&TARGET_DELETE_DEREF,
&&TARGET_LOAD_FAST__LOAD_CONST,
&&TARGET_STORE_ATTR_WITH_HINT,
&&TARGET_CALL_FUNCTION_KW,
&&TARGET_CALL_FUNCTION_EX,
&&TARGET_LOAD_CONST__LOAD_FAST,
&&TARGET_LOAD_FAST__LOAD_FAST,
&&TARGET_EXTENDED_ARG,
&&TARGET_LIST_APPEND,
&&TARGET_SET_ADD,
&&TARGET_MAP_ADD,
&&TARGET_LOAD_CLASSDEREF,
&&TARGET_STORE_FAST__STORE_FAST,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_STORE_FAST__LOAD_FAST,
&&TARGET_LOAD_FAST__LOAD_CONST,
&&TARGET_LOAD_CONST__LOAD_FAST,
&&TARGET_MATCH_CLASS,
&&_unknown_opcode,
&&TARGET_STORE_FAST__STORE_FAST,
&&_unknown_opcode,
&&TARGET_FORMAT_VALUE,
&&TARGET_BUILD_CONST_KEY_MAP,

33
Python/specialize.c
View File

@ -124,6 +124,7 @@ _Py_GetSpecializationStats(void) {
err += add_stat_dict(stats, LOAD_GLOBAL, "load_global");
err += add_stat_dict(stats, LOAD_METHOD, "load_method");
err += add_stat_dict(stats, BINARY_ADD, "binary_add");
err += add_stat_dict(stats, BINARY_MULTIPLY, "binary_multiply");
err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr");
err += add_stat_dict(stats, STORE_ATTR, "store_attr");
if (err < 0) {
@ -180,6 +181,7 @@ _Py_PrintSpecializationStats(void)
print_stats(out, &_specialization_stats[LOAD_GLOBAL], "load_global");
print_stats(out, &_specialization_stats[LOAD_METHOD], "load_method");
print_stats(out, &_specialization_stats[BINARY_ADD], "binary_add");
print_stats(out, &_specialization_stats[BINARY_MULTIPLY], "binary_multiply");
print_stats(out, &_specialization_stats[BINARY_SUBSCR], "binary_subscr");
print_stats(out, &_specialization_stats[STORE_ATTR], "store_attr");
if (out != stderr) {
@ -230,6 +232,7 @@ static uint8_t adaptive_opcodes[256] = {
[LOAD_GLOBAL] = LOAD_GLOBAL_ADAPTIVE,
[LOAD_METHOD] = LOAD_METHOD_ADAPTIVE,
[BINARY_ADD] = BINARY_ADD_ADAPTIVE,
[BINARY_MULTIPLY] = BINARY_MULTIPLY_ADAPTIVE,
[BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE,
[STORE_ATTR] = STORE_ATTR_ADAPTIVE,
};
@ -240,6 +243,7 @@ static uint8_t cache_requirements[256] = {
[LOAD_GLOBAL] = 2, /* _PyAdaptiveEntry and _PyLoadGlobalCache */
[LOAD_METHOD] = 3, /* _PyAdaptiveEntry, _PyAttrCache and _PyObjectCache */
[BINARY_ADD] = 0,
[BINARY_MULTIPLY] = 0,
[BINARY_SUBSCR] = 0,
[STORE_ATTR] = 2, /* _PyAdaptiveEntry and _PyAttrCache */
};
@ -1188,3 +1192,32 @@ success:
assert(!PyErr_Occurred());
return 0;
}
int
_Py_Specialize_BinaryMultiply(PyObject *left, PyObject *right, _Py_CODEUNIT *instr)
{
if (!Py_IS_TYPE(left, Py_TYPE(right))) {
SPECIALIZATION_FAIL(BINARY_MULTIPLY, SPEC_FAIL_DIFFERENT_TYPES);
goto fail;
}
if (PyLong_CheckExact(left)) {
*instr = _Py_MAKECODEUNIT(BINARY_MULTIPLY_INT, saturating_start());
goto success;
}
else if (PyFloat_CheckExact(left)) {
*instr = _Py_MAKECODEUNIT(BINARY_MULTIPLY_FLOAT, saturating_start());
goto success;
}
else {
SPECIALIZATION_FAIL(BINARY_MULTIPLY, SPEC_FAIL_OTHER);
}
fail:
STAT_INC(BINARY_MULTIPLY, specialization_failure);
assert(!PyErr_Occurred());
*instr = _Py_MAKECODEUNIT(_Py_OPCODE(*instr), ADAPTIVE_CACHE_BACKOFF);
return 0;
success:
STAT_INC(BINARY_MULTIPLY, specialization_success);
assert(!PyErr_Occurred());
return 0;
}