2023-06-02 07:46:18 -03:00
|
|
|
#include "Python.h"
|
|
|
|
#include "opcode.h"
|
|
|
|
#include "pycore_interp.h"
|
|
|
|
#include "pycore_opcode.h"
|
2023-06-26 23:02:57 -03:00
|
|
|
#include "opcode_metadata.h"
|
2023-07-02 13:37:37 -03:00
|
|
|
#include "pycore_pystate.h" // _PyInterpreterState_GET()
|
2023-06-26 23:02:57 -03:00
|
|
|
#include "pycore_uops.h"
|
2023-06-02 07:46:18 -03:00
|
|
|
#include "cpython/optimizer.h"
|
|
|
|
#include <stdbool.h>
|
|
|
|
#include <stdint.h>
|
|
|
|
#include <stddef.h>
|
|
|
|
|
2023-06-19 06:32:20 -03:00
|
|
|
static bool
|
|
|
|
has_space_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr)
|
|
|
|
{
|
|
|
|
if (instr->op.code == ENTER_EXECUTOR) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
if (code->co_executors == NULL) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return code->co_executors->size < 256;
|
|
|
|
}
|
|
|
|
|
2023-06-02 07:46:18 -03:00
|
|
|
static int32_t
|
2023-06-19 06:32:20 -03:00
|
|
|
get_index_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr)
|
2023-06-02 07:46:18 -03:00
|
|
|
{
|
2023-06-19 06:32:20 -03:00
|
|
|
if (instr->op.code == ENTER_EXECUTOR) {
|
|
|
|
return instr->op.arg;
|
|
|
|
}
|
2023-06-02 07:46:18 -03:00
|
|
|
_PyExecutorArray *old = code->co_executors;
|
|
|
|
int size = 0;
|
|
|
|
int capacity = 0;
|
|
|
|
if (old != NULL) {
|
|
|
|
size = old->size;
|
|
|
|
capacity = old->capacity;
|
2023-06-19 06:32:20 -03:00
|
|
|
assert(size < 256);
|
2023-06-02 07:46:18 -03:00
|
|
|
}
|
|
|
|
assert(size <= capacity);
|
|
|
|
if (size == capacity) {
|
|
|
|
/* Array is full. Grow array */
|
|
|
|
int new_capacity = capacity ? capacity * 2 : 4;
|
|
|
|
_PyExecutorArray *new = PyMem_Realloc(
|
|
|
|
old,
|
|
|
|
offsetof(_PyExecutorArray, executors) +
|
|
|
|
new_capacity * sizeof(_PyExecutorObject *));
|
|
|
|
if (new == NULL) {
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
new->capacity = new_capacity;
|
|
|
|
new->size = size;
|
|
|
|
code->co_executors = new;
|
|
|
|
}
|
|
|
|
assert(size < code->co_executors->capacity);
|
|
|
|
return size;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorObject *executor)
|
|
|
|
{
|
2023-06-19 06:32:20 -03:00
|
|
|
Py_INCREF(executor);
|
2023-06-02 07:46:18 -03:00
|
|
|
if (instr->op.code == ENTER_EXECUTOR) {
|
|
|
|
assert(index == instr->op.arg);
|
|
|
|
_PyExecutorObject *old = code->co_executors->executors[index];
|
|
|
|
executor->vm_data.opcode = old->vm_data.opcode;
|
|
|
|
executor->vm_data.oparg = old->vm_data.oparg;
|
|
|
|
old->vm_data.opcode = 0;
|
|
|
|
code->co_executors->executors[index] = executor;
|
|
|
|
Py_DECREF(old);
|
|
|
|
}
|
|
|
|
else {
|
2023-06-19 06:32:20 -03:00
|
|
|
assert(code->co_executors->size == index);
|
|
|
|
assert(code->co_executors->capacity > index);
|
2023-06-02 07:46:18 -03:00
|
|
|
executor->vm_data.opcode = instr->op.code;
|
|
|
|
executor->vm_data.oparg = instr->op.arg;
|
|
|
|
code->co_executors->executors[index] = executor;
|
|
|
|
assert(index < 256);
|
|
|
|
instr->op.code = ENTER_EXECUTOR;
|
|
|
|
instr->op.arg = index;
|
2023-06-19 06:32:20 -03:00
|
|
|
code->co_executors->size++;
|
2023-06-02 07:46:18 -03:00
|
|
|
}
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
PyUnstable_Replace_Executor(PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject *new)
|
|
|
|
{
|
|
|
|
if (instr->op.code != ENTER_EXECUTOR) {
|
|
|
|
PyErr_Format(PyExc_ValueError, "No executor to replace");
|
|
|
|
return -1;
|
|
|
|
}
|
2023-06-19 06:32:20 -03:00
|
|
|
int index = instr->op.arg;
|
2023-06-02 07:46:18 -03:00
|
|
|
assert(index >= 0);
|
|
|
|
insert_executor(code, instr, index, new);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2023-06-05 05:44:23 -03:00
|
|
|
static int
|
2023-06-02 07:46:18 -03:00
|
|
|
error_optimize(
|
|
|
|
_PyOptimizerObject* self,
|
|
|
|
PyCodeObject *code,
|
2023-06-05 05:44:23 -03:00
|
|
|
_Py_CODEUNIT *instr,
|
|
|
|
_PyExecutorObject **exec)
|
2023-06-02 07:46:18 -03:00
|
|
|
{
|
|
|
|
PyErr_Format(PyExc_SystemError, "Should never call error_optimize");
|
2023-06-05 05:44:23 -03:00
|
|
|
return -1;
|
2023-06-02 07:46:18 -03:00
|
|
|
}
|
|
|
|
|
|
|
|
static PyTypeObject DefaultOptimizer_Type = {
|
|
|
|
PyVarObject_HEAD_INIT(&PyType_Type, 0)
|
|
|
|
.tp_name = "noop_optimizer",
|
|
|
|
.tp_basicsize = sizeof(_PyOptimizerObject),
|
|
|
|
.tp_itemsize = 0,
|
|
|
|
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION,
|
|
|
|
};
|
|
|
|
|
|
|
|
_PyOptimizerObject _PyOptimizer_Default = {
|
|
|
|
PyObject_HEAD_INIT(&DefaultOptimizer_Type)
|
|
|
|
.optimize = error_optimize,
|
|
|
|
.resume_threshold = UINT16_MAX,
|
|
|
|
.backedge_threshold = UINT16_MAX,
|
|
|
|
};
|
|
|
|
|
|
|
|
_PyOptimizerObject *
|
|
|
|
PyUnstable_GetOptimizer(void)
|
|
|
|
{
|
2023-07-02 13:37:37 -03:00
|
|
|
PyInterpreterState *interp = _PyInterpreterState_GET();
|
2023-06-02 07:46:18 -03:00
|
|
|
if (interp->optimizer == &_PyOptimizer_Default) {
|
|
|
|
return NULL;
|
|
|
|
}
|
2023-06-19 06:32:20 -03:00
|
|
|
assert(interp->optimizer_backedge_threshold == interp->optimizer->backedge_threshold);
|
|
|
|
assert(interp->optimizer_resume_threshold == interp->optimizer->resume_threshold);
|
2023-06-02 07:46:18 -03:00
|
|
|
Py_INCREF(interp->optimizer);
|
|
|
|
return interp->optimizer;
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
PyUnstable_SetOptimizer(_PyOptimizerObject *optimizer)
|
|
|
|
{
|
2023-07-02 13:37:37 -03:00
|
|
|
PyInterpreterState *interp = _PyInterpreterState_GET();
|
2023-06-02 07:46:18 -03:00
|
|
|
if (optimizer == NULL) {
|
|
|
|
optimizer = &_PyOptimizer_Default;
|
|
|
|
}
|
|
|
|
_PyOptimizerObject *old = interp->optimizer;
|
|
|
|
Py_INCREF(optimizer);
|
|
|
|
interp->optimizer = optimizer;
|
|
|
|
interp->optimizer_backedge_threshold = optimizer->backedge_threshold;
|
|
|
|
interp->optimizer_resume_threshold = optimizer->resume_threshold;
|
|
|
|
Py_DECREF(old);
|
|
|
|
}
|
|
|
|
|
|
|
|
_PyInterpreterFrame *
|
|
|
|
_PyOptimizer_BackEdge(_PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest, PyObject **stack_pointer)
|
|
|
|
{
|
2023-06-14 09:46:37 -03:00
|
|
|
PyCodeObject *code = (PyCodeObject *)frame->f_executable;
|
|
|
|
assert(PyCode_Check(code));
|
2023-07-02 13:37:37 -03:00
|
|
|
PyInterpreterState *interp = _PyInterpreterState_GET();
|
2023-06-19 06:32:20 -03:00
|
|
|
if (!has_space_for_executor(code, src)) {
|
|
|
|
goto jump_to_destination;
|
2023-06-02 07:46:18 -03:00
|
|
|
}
|
|
|
|
_PyOptimizerObject *opt = interp->optimizer;
|
2023-06-19 06:32:20 -03:00
|
|
|
_PyExecutorObject *executor = NULL;
|
2023-06-14 09:46:37 -03:00
|
|
|
int err = opt->optimize(opt, code, dest, &executor);
|
2023-06-05 05:44:23 -03:00
|
|
|
if (err <= 0) {
|
2023-06-19 06:32:20 -03:00
|
|
|
assert(executor == NULL);
|
2023-06-05 05:44:23 -03:00
|
|
|
if (err < 0) {
|
|
|
|
return NULL;
|
|
|
|
}
|
2023-06-19 06:32:20 -03:00
|
|
|
goto jump_to_destination;
|
|
|
|
}
|
|
|
|
int index = get_index_for_executor(code, src);
|
|
|
|
if (index < 0) {
|
|
|
|
/* Out of memory. Don't raise and assume that the
|
|
|
|
* error will show up elsewhere.
|
|
|
|
*
|
|
|
|
* If an optimizer has already produced an executor,
|
|
|
|
* it might get confused by the executor disappearing,
|
|
|
|
* but there is not much we can do about that here. */
|
|
|
|
Py_DECREF(executor);
|
|
|
|
goto jump_to_destination;
|
2023-06-02 07:46:18 -03:00
|
|
|
}
|
2023-06-14 09:46:37 -03:00
|
|
|
insert_executor(code, src, index, executor);
|
2023-06-19 06:32:20 -03:00
|
|
|
assert(frame->prev_instr == src);
|
2023-07-06 15:39:53 -03:00
|
|
|
frame->prev_instr = dest - 1;
|
2023-06-02 07:46:18 -03:00
|
|
|
return executor->execute(executor, frame, stack_pointer);
|
2023-06-19 06:32:20 -03:00
|
|
|
jump_to_destination:
|
|
|
|
frame->prev_instr = dest - 1;
|
|
|
|
_PyFrame_SetStackPointer(frame, stack_pointer);
|
|
|
|
return frame;
|
2023-06-02 07:46:18 -03:00
|
|
|
}
|
|
|
|
|
2023-07-04 13:23:00 -03:00
|
|
|
_PyExecutorObject *
|
|
|
|
PyUnstable_GetExecutor(PyCodeObject *code, int offset)
|
|
|
|
{
|
|
|
|
int code_len = (int)Py_SIZE(code);
|
|
|
|
for (int i = 0 ; i < code_len;) {
|
|
|
|
if (_PyCode_CODE(code)[i].op.code == ENTER_EXECUTOR && i*2 == offset) {
|
|
|
|
int oparg = _PyCode_CODE(code)[i].op.arg;
|
|
|
|
_PyExecutorObject *res = code->co_executors->executors[oparg];
|
|
|
|
Py_INCREF(res);
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
i += _PyInstruction_GetLength(code, i);
|
|
|
|
}
|
2023-07-06 15:39:53 -03:00
|
|
|
PyErr_SetString(PyExc_ValueError, "no executor at given byte offset");
|
2023-07-04 13:23:00 -03:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2023-06-02 07:46:18 -03:00
|
|
|
/** Test support **/
|
|
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
_PyOptimizerObject base;
|
|
|
|
int64_t count;
|
|
|
|
} _PyCounterOptimizerObject;
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
_PyExecutorObject executor;
|
|
|
|
_PyCounterOptimizerObject *optimizer;
|
|
|
|
_Py_CODEUNIT *next_instr;
|
|
|
|
} _PyCounterExecutorObject;
|
|
|
|
|
|
|
|
static void
|
|
|
|
counter_dealloc(_PyCounterExecutorObject *self) {
|
|
|
|
Py_DECREF(self->optimizer);
|
|
|
|
PyObject_Free(self);
|
|
|
|
}
|
|
|
|
|
|
|
|
static PyTypeObject CounterExecutor_Type = {
|
|
|
|
PyVarObject_HEAD_INIT(&PyType_Type, 0)
|
|
|
|
.tp_name = "counting_executor",
|
|
|
|
.tp_basicsize = sizeof(_PyCounterExecutorObject),
|
|
|
|
.tp_itemsize = 0,
|
|
|
|
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION,
|
|
|
|
.tp_dealloc = (destructor)counter_dealloc,
|
|
|
|
};
|
|
|
|
|
|
|
|
static _PyInterpreterFrame *
|
|
|
|
counter_execute(_PyExecutorObject *self, _PyInterpreterFrame *frame, PyObject **stack_pointer)
|
|
|
|
{
|
|
|
|
((_PyCounterExecutorObject *)self)->optimizer->count++;
|
|
|
|
_PyFrame_SetStackPointer(frame, stack_pointer);
|
|
|
|
frame->prev_instr = ((_PyCounterExecutorObject *)self)->next_instr - 1;
|
|
|
|
Py_DECREF(self);
|
|
|
|
return frame;
|
|
|
|
}
|
|
|
|
|
2023-06-05 05:44:23 -03:00
|
|
|
static int
|
2023-06-02 07:46:18 -03:00
|
|
|
counter_optimize(
|
|
|
|
_PyOptimizerObject* self,
|
|
|
|
PyCodeObject *code,
|
2023-06-05 05:44:23 -03:00
|
|
|
_Py_CODEUNIT *instr,
|
|
|
|
_PyExecutorObject **exec_ptr)
|
2023-06-02 07:46:18 -03:00
|
|
|
{
|
|
|
|
_PyCounterExecutorObject *executor = (_PyCounterExecutorObject *)_PyObject_New(&CounterExecutor_Type);
|
|
|
|
if (executor == NULL) {
|
2023-06-05 05:44:23 -03:00
|
|
|
return -1;
|
2023-06-02 07:46:18 -03:00
|
|
|
}
|
|
|
|
executor->executor.execute = counter_execute;
|
|
|
|
Py_INCREF(self);
|
|
|
|
executor->optimizer = (_PyCounterOptimizerObject *)self;
|
|
|
|
executor->next_instr = instr;
|
2023-06-05 05:44:23 -03:00
|
|
|
*exec_ptr = (_PyExecutorObject *)executor;
|
|
|
|
return 1;
|
2023-06-02 07:46:18 -03:00
|
|
|
}
|
|
|
|
|
|
|
|
static PyObject *
|
|
|
|
counter_get_counter(PyObject *self, PyObject *args)
|
|
|
|
{
|
|
|
|
return PyLong_FromLongLong(((_PyCounterOptimizerObject *)self)->count);
|
|
|
|
}
|
|
|
|
|
|
|
|
static PyMethodDef counter_methods[] = {
|
|
|
|
{ "get_count", counter_get_counter, METH_NOARGS, NULL },
|
|
|
|
{ NULL, NULL },
|
|
|
|
};
|
|
|
|
|
|
|
|
static PyTypeObject CounterOptimizer_Type = {
|
|
|
|
PyVarObject_HEAD_INIT(&PyType_Type, 0)
|
|
|
|
.tp_name = "Counter optimizer",
|
|
|
|
.tp_basicsize = sizeof(_PyCounterOptimizerObject),
|
|
|
|
.tp_itemsize = 0,
|
|
|
|
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION,
|
|
|
|
.tp_methods = counter_methods,
|
|
|
|
};
|
|
|
|
|
|
|
|
PyObject *
|
|
|
|
PyUnstable_Optimizer_NewCounter(void)
|
|
|
|
{
|
|
|
|
_PyCounterOptimizerObject *opt = (_PyCounterOptimizerObject *)_PyObject_New(&CounterOptimizer_Type);
|
|
|
|
if (opt == NULL) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
opt->base.optimize = counter_optimize;
|
|
|
|
opt->base.resume_threshold = UINT16_MAX;
|
|
|
|
opt->base.backedge_threshold = 0;
|
|
|
|
opt->count = 0;
|
|
|
|
return (PyObject *)opt;
|
|
|
|
}
|
2023-06-26 23:02:57 -03:00
|
|
|
|
|
|
|
///////////////////// Experimental UOp Optimizer /////////////////////
|
|
|
|
|
|
|
|
static void
|
|
|
|
uop_dealloc(_PyUOpExecutorObject *self) {
|
|
|
|
PyObject_Free(self);
|
|
|
|
}
|
|
|
|
|
2023-07-04 13:23:00 -03:00
|
|
|
static const char *
|
|
|
|
uop_name(int index) {
|
|
|
|
if (index < EXIT_TRACE) {
|
|
|
|
return _PyOpcode_OpName[index];
|
|
|
|
}
|
|
|
|
return _PyOpcode_uop_name[index];
|
|
|
|
}
|
|
|
|
|
|
|
|
static Py_ssize_t
|
|
|
|
uop_len(_PyUOpExecutorObject *self)
|
|
|
|
{
|
|
|
|
int count = 1;
|
|
|
|
for (; count < _Py_UOP_MAX_TRACE_LENGTH; count++) {
|
|
|
|
if (self->trace[count-1].opcode == EXIT_TRACE) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
|
|
|
|
static PyObject *
|
|
|
|
uop_item(_PyUOpExecutorObject *self, Py_ssize_t index)
|
|
|
|
{
|
|
|
|
for (int i = 0; i < _Py_UOP_MAX_TRACE_LENGTH; i++) {
|
|
|
|
if (self->trace[i].opcode == EXIT_TRACE) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (i != index) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
const char *name = uop_name(self->trace[i].opcode);
|
|
|
|
PyObject *oname = _PyUnicode_FromASCII(name, strlen(name));
|
|
|
|
if (oname == NULL) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
PyObject *operand = PyLong_FromUnsignedLongLong(self->trace[i].operand);
|
|
|
|
if (operand == NULL) {
|
|
|
|
Py_DECREF(oname);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
PyObject *args[2] = { oname, operand };
|
|
|
|
return _PyTuple_FromArraySteal(args, 2);
|
|
|
|
}
|
|
|
|
PyErr_SetNone(PyExc_IndexError);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
PySequenceMethods uop_as_sequence = {
|
|
|
|
.sq_length = (lenfunc)uop_len,
|
|
|
|
.sq_item = (ssizeargfunc)uop_item,
|
|
|
|
};
|
|
|
|
|
2023-06-26 23:02:57 -03:00
|
|
|
static PyTypeObject UOpExecutor_Type = {
|
|
|
|
PyVarObject_HEAD_INIT(&PyType_Type, 0)
|
|
|
|
.tp_name = "uop_executor",
|
|
|
|
.tp_basicsize = sizeof(_PyUOpExecutorObject),
|
|
|
|
.tp_itemsize = 0,
|
|
|
|
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION,
|
|
|
|
.tp_dealloc = (destructor)uop_dealloc,
|
2023-07-04 13:23:00 -03:00
|
|
|
.tp_as_sequence = &uop_as_sequence,
|
2023-06-26 23:02:57 -03:00
|
|
|
};
|
|
|
|
|
|
|
|
static int
|
|
|
|
translate_bytecode_to_trace(
|
|
|
|
PyCodeObject *code,
|
|
|
|
_Py_CODEUNIT *instr,
|
|
|
|
_PyUOpInstruction *trace,
|
|
|
|
int max_length)
|
|
|
|
{
|
2023-07-06 15:39:53 -03:00
|
|
|
#ifdef Py_DEBUG
|
|
|
|
_Py_CODEUNIT *initial_instr = instr;
|
|
|
|
#endif
|
2023-07-03 17:05:11 -03:00
|
|
|
int trace_length = 0;
|
|
|
|
|
|
|
|
#ifdef Py_DEBUG
|
2023-06-26 23:02:57 -03:00
|
|
|
char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG");
|
|
|
|
int lltrace = 0;
|
|
|
|
if (uop_debug != NULL && *uop_debug >= '0') {
|
|
|
|
lltrace = *uop_debug - '0'; // TODO: Parse an int and all that
|
|
|
|
}
|
2023-07-03 17:05:11 -03:00
|
|
|
#define DPRINTF(level, ...) \
|
|
|
|
if (lltrace >= (level)) { fprintf(stderr, __VA_ARGS__); }
|
2023-06-26 23:02:57 -03:00
|
|
|
#else
|
2023-07-03 17:05:11 -03:00
|
|
|
#define DPRINTF(level, ...)
|
2023-06-26 23:02:57 -03:00
|
|
|
#endif
|
|
|
|
|
2023-07-03 17:05:11 -03:00
|
|
|
#define ADD_TO_TRACE(OPCODE, OPERAND) \
|
|
|
|
DPRINTF(2, \
|
|
|
|
" ADD_TO_TRACE(%s, %" PRIu64 ")\n", \
|
|
|
|
(OPCODE) < 256 ? _PyOpcode_OpName[(OPCODE)] : _PyOpcode_uop_name[(OPCODE)], \
|
|
|
|
(uint64_t)(OPERAND)); \
|
|
|
|
assert(trace_length < max_length); \
|
|
|
|
trace[trace_length].opcode = (OPCODE); \
|
|
|
|
trace[trace_length].operand = (OPERAND); \
|
|
|
|
trace_length++;
|
|
|
|
|
|
|
|
DPRINTF(4,
|
2023-07-06 15:39:53 -03:00
|
|
|
"Optimizing %s (%s:%d) at byte offset %ld\n",
|
2023-07-03 17:05:11 -03:00
|
|
|
PyUnicode_AsUTF8(code->co_qualname),
|
|
|
|
PyUnicode_AsUTF8(code->co_filename),
|
|
|
|
code->co_firstlineno,
|
2023-07-06 15:39:53 -03:00
|
|
|
2 * (long)(initial_instr - (_Py_CODEUNIT *)code->co_code_adaptive));
|
2023-07-03 17:05:11 -03:00
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
ADD_TO_TRACE(SAVE_IP, (int)(instr - (_Py_CODEUNIT *)code->co_code_adaptive));
|
2023-06-26 23:02:57 -03:00
|
|
|
int opcode = instr->op.code;
|
|
|
|
uint64_t operand = instr->op.arg;
|
|
|
|
switch (opcode) {
|
|
|
|
case LOAD_FAST_LOAD_FAST:
|
2023-07-03 17:05:11 -03:00
|
|
|
case STORE_FAST_LOAD_FAST:
|
|
|
|
case STORE_FAST_STORE_FAST:
|
2023-06-26 23:02:57 -03:00
|
|
|
{
|
2023-07-03 17:05:11 -03:00
|
|
|
// Reserve space for two uops (+ SAVE_IP + EXIT_TRACE)
|
2023-06-26 23:02:57 -03:00
|
|
|
if (trace_length + 4 > max_length) {
|
2023-07-03 17:05:11 -03:00
|
|
|
DPRINTF(1, "Ran out of space for LOAD_FAST_LOAD_FAST\n");
|
2023-06-26 23:02:57 -03:00
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
uint64_t oparg1 = operand >> 4;
|
|
|
|
uint64_t oparg2 = operand & 15;
|
2023-07-03 17:05:11 -03:00
|
|
|
switch (opcode) {
|
|
|
|
case LOAD_FAST_LOAD_FAST:
|
|
|
|
ADD_TO_TRACE(LOAD_FAST, oparg1);
|
|
|
|
ADD_TO_TRACE(LOAD_FAST, oparg2);
|
|
|
|
break;
|
|
|
|
case STORE_FAST_LOAD_FAST:
|
|
|
|
ADD_TO_TRACE(STORE_FAST, oparg1);
|
|
|
|
ADD_TO_TRACE(LOAD_FAST, oparg2);
|
|
|
|
break;
|
|
|
|
case STORE_FAST_STORE_FAST:
|
|
|
|
ADD_TO_TRACE(STORE_FAST, oparg1);
|
|
|
|
ADD_TO_TRACE(STORE_FAST, oparg2);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
Py_FatalError("Missing case");
|
|
|
|
}
|
2023-06-26 23:02:57 -03:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
default:
|
|
|
|
{
|
|
|
|
const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode];
|
|
|
|
if (expansion->nuops > 0) {
|
2023-07-03 17:05:11 -03:00
|
|
|
// Reserve space for nuops (+ SAVE_IP + EXIT_TRACE)
|
2023-06-26 23:02:57 -03:00
|
|
|
int nuops = expansion->nuops;
|
|
|
|
if (trace_length + nuops + 2 > max_length) {
|
2023-07-03 17:05:11 -03:00
|
|
|
DPRINTF(1,
|
|
|
|
"Ran out of space for %s\n",
|
|
|
|
opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode]);
|
2023-06-26 23:02:57 -03:00
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
for (int i = 0; i < nuops; i++) {
|
|
|
|
int offset = expansion->uops[i].offset;
|
|
|
|
switch (expansion->uops[i].size) {
|
|
|
|
case 0:
|
|
|
|
break;
|
|
|
|
case 1:
|
|
|
|
operand = read_u16(&instr[offset].cache);
|
|
|
|
break;
|
|
|
|
case 2:
|
|
|
|
operand = read_u32(&instr[offset].cache);
|
|
|
|
break;
|
|
|
|
case 4:
|
|
|
|
operand = read_u64(&instr[offset].cache);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
fprintf(stderr,
|
|
|
|
"opcode=%d, operand=%" PRIu64 "; nuops=%d, i=%d; size=%d, offset=%d\n",
|
|
|
|
opcode, operand, nuops, i,
|
|
|
|
expansion->uops[i].size,
|
|
|
|
expansion->uops[i].offset);
|
|
|
|
Py_FatalError("garbled expansion");
|
|
|
|
}
|
|
|
|
ADD_TO_TRACE(expansion->uops[i].uop, operand);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
2023-07-03 17:05:11 -03:00
|
|
|
DPRINTF(2,
|
|
|
|
"Unsupported opcode %s\n",
|
|
|
|
opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode]);
|
|
|
|
goto done; // Break out of loop
|
2023-06-26 23:02:57 -03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
instr++;
|
|
|
|
// Add cache size for opcode
|
|
|
|
instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]];
|
|
|
|
}
|
2023-07-03 17:05:11 -03:00
|
|
|
|
2023-06-26 23:02:57 -03:00
|
|
|
done:
|
2023-07-03 17:05:11 -03:00
|
|
|
// Skip short traces like SAVE_IP, LOAD_FAST, SAVE_IP, EXIT_TRACE
|
|
|
|
if (trace_length > 3) {
|
2023-06-26 23:02:57 -03:00
|
|
|
ADD_TO_TRACE(EXIT_TRACE, 0);
|
2023-07-03 17:05:11 -03:00
|
|
|
DPRINTF(1,
|
2023-07-06 15:39:53 -03:00
|
|
|
"Created a trace for %s (%s:%d) at byte offset %ld -- length %d\n",
|
2023-07-03 17:05:11 -03:00
|
|
|
PyUnicode_AsUTF8(code->co_qualname),
|
|
|
|
PyUnicode_AsUTF8(code->co_filename),
|
|
|
|
code->co_firstlineno,
|
2023-07-06 15:39:53 -03:00
|
|
|
2 * (long)(initial_instr - (_Py_CODEUNIT *)code->co_code_adaptive),
|
2023-07-03 17:05:11 -03:00
|
|
|
trace_length);
|
|
|
|
return trace_length;
|
2023-06-26 23:02:57 -03:00
|
|
|
}
|
|
|
|
else {
|
2023-07-03 17:05:11 -03:00
|
|
|
DPRINTF(4,
|
2023-07-06 15:39:53 -03:00
|
|
|
"No trace for %s (%s:%d) at byte offset %ld\n",
|
2023-07-03 17:05:11 -03:00
|
|
|
PyUnicode_AsUTF8(code->co_qualname),
|
|
|
|
PyUnicode_AsUTF8(code->co_filename),
|
|
|
|
code->co_firstlineno,
|
2023-07-06 15:39:53 -03:00
|
|
|
2 * (long)(initial_instr - (_Py_CODEUNIT *)code->co_code_adaptive));
|
2023-06-26 23:02:57 -03:00
|
|
|
}
|
2023-07-03 17:05:11 -03:00
|
|
|
return 0;
|
2023-06-26 23:02:57 -03:00
|
|
|
|
|
|
|
#undef ADD_TO_TRACE
|
2023-07-03 17:05:11 -03:00
|
|
|
#undef DPRINTF
|
2023-06-26 23:02:57 -03:00
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
uop_optimize(
|
|
|
|
_PyOptimizerObject *self,
|
|
|
|
PyCodeObject *code,
|
|
|
|
_Py_CODEUNIT *instr,
|
|
|
|
_PyExecutorObject **exec_ptr)
|
|
|
|
{
|
|
|
|
_PyUOpInstruction trace[_Py_UOP_MAX_TRACE_LENGTH];
|
|
|
|
int trace_length = translate_bytecode_to_trace(code, instr, trace, _Py_UOP_MAX_TRACE_LENGTH);
|
|
|
|
if (trace_length <= 0) {
|
|
|
|
// Error or nothing translated
|
|
|
|
return trace_length;
|
|
|
|
}
|
|
|
|
OBJECT_STAT_INC(optimization_traces_created);
|
2023-07-06 19:45:56 -03:00
|
|
|
_PyUOpExecutorObject *executor = PyObject_New(_PyUOpExecutorObject, &UOpExecutor_Type);
|
2023-06-26 23:02:57 -03:00
|
|
|
if (executor == NULL) {
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
executor->base.execute = _PyUopExecute;
|
|
|
|
memcpy(executor->trace, trace, trace_length * sizeof(_PyUOpInstruction));
|
|
|
|
*exec_ptr = (_PyExecutorObject *)executor;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2023-07-06 19:45:56 -03:00
|
|
|
static void
|
|
|
|
uop_opt_dealloc(PyObject *self) {
|
|
|
|
PyObject_Free(self);
|
|
|
|
}
|
|
|
|
|
2023-06-26 23:02:57 -03:00
|
|
|
static PyTypeObject UOpOptimizer_Type = {
|
|
|
|
PyVarObject_HEAD_INIT(&PyType_Type, 0)
|
|
|
|
.tp_name = "uop_optimizer",
|
|
|
|
.tp_basicsize = sizeof(_PyOptimizerObject),
|
|
|
|
.tp_itemsize = 0,
|
|
|
|
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION,
|
2023-07-06 19:45:56 -03:00
|
|
|
.tp_dealloc = uop_opt_dealloc,
|
2023-06-26 23:02:57 -03:00
|
|
|
};
|
|
|
|
|
|
|
|
PyObject *
|
|
|
|
PyUnstable_Optimizer_NewUOpOptimizer(void)
|
|
|
|
{
|
2023-07-06 19:45:56 -03:00
|
|
|
_PyOptimizerObject *opt = PyObject_New(_PyOptimizerObject, &UOpOptimizer_Type);
|
2023-06-26 23:02:57 -03:00
|
|
|
if (opt == NULL) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
opt->optimize = uop_optimize;
|
|
|
|
opt->resume_threshold = UINT16_MAX;
|
2023-06-28 15:28:07 -03:00
|
|
|
// Need at least 3 iterations to settle specializations.
|
|
|
|
// A few lower bits of the counter are reserved for other flags.
|
|
|
|
opt->backedge_threshold = 3 << OPTIMIZER_BITS_IN_COUNTER;
|
2023-06-26 23:02:57 -03:00
|
|
|
return (PyObject *)opt;
|
|
|
|
}
|