Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-98831: Modernize CALL and family (#101508) 

Includes a slight improvement to `DECREF_INPUTS()`.
This commit is contained in:
Guido van Rossum 2023-02-08 11:40:10 -08:00 committed by GitHub
parent d9de079248
commit 616aec1ff1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 644 additions and 586 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

524
Python/bytecodes.c
View File

@ -2360,70 +2360,87 @@ dummy_func(
assert(oparg & 1);
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_BOUND_METHOD_EXACT_ARGS) {
DEOPT_IF(is_method(stack_pointer, oparg), CALL);
PyObject *function = PEEK(oparg + 1);
DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, CALL);
STAT_INC(CALL, hit);
PyObject *self = ((PyMethodObject *)function)->im_self;
PEEK(oparg + 1) = Py_NewRef(self);
PyObject *meth = ((PyMethodObject *)function)->im_func;
PEEK(oparg + 2) = Py_NewRef(meth);
Py_DECREF(function);
GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS);
}
inst(KW_NAMES, (--)) {
assert(kwnames == NULL);
assert(oparg < PyTuple_GET_SIZE(consts));
kwnames = GETITEM(consts, oparg);
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL) {
// Cache layout: counter/1, func_version/2, min_args/1
// Neither CALL_INTRINSIC_1 nor CALL_FUNCTION_EX are members!
family(call, INLINE_CACHE_ENTRIES_CALL) = {
CALL,
CALL_BOUND_METHOD_EXACT_ARGS,
CALL_PY_EXACT_ARGS,
CALL_PY_WITH_DEFAULTS,
CALL_NO_KW_TYPE_1,
CALL_NO_KW_STR_1,
CALL_NO_KW_TUPLE_1,
CALL_BUILTIN_CLASS,
CALL_NO_KW_BUILTIN_O,
CALL_NO_KW_BUILTIN_FAST,
CALL_BUILTIN_FAST_WITH_KEYWORDS,
CALL_NO_KW_LEN,
CALL_NO_KW_ISINSTANCE,
CALL_NO_KW_LIST_APPEND,
CALL_NO_KW_METHOD_DESCRIPTOR_O,
CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS,
CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
CALL_NO_KW_METHOD_DESCRIPTOR_FAST,
};
// On entry, the stack is either
// [NULL, callable, arg1, arg2, ...]
// or
// [method, self, arg1, arg2, ...]
// (Some args may be keywords, see KW_NAMES, which sets 'kwnames'.)
// On exit, the stack is [result].
// When calling Python, inline the call using DISPATCH_INLINED().
inst(CALL, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
#if ENABLE_SPECIALIZATION
_PyCallCache *cache = (_PyCallCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0);
int is_meth = is_method(stack_pointer, oparg);
int nargs = oparg + is_meth;
PyObject *callable = PEEK(nargs + 1);
next_instr--;
_Py_Specialize_Call(callable, next_instr, nargs, kwnames);
_Py_Specialize_Call(callable, next_instr, total_args, kwnames);
DISPATCH_SAME_OPARG();
}
STAT_INC(CALL, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
#endif /* ENABLE_SPECIALIZATION */
int total_args, is_meth;
is_meth = is_method(stack_pointer, oparg);
PyObject *function = PEEK(oparg + 1);
if (!is_meth && Py_TYPE(function) == &PyMethod_Type) {
PyObject *self = ((PyMethodObject *)function)->im_self;
PEEK(oparg+1) = Py_NewRef(self);
PyObject *meth = ((PyMethodObject *)function)->im_func;
PEEK(oparg+2) = Py_NewRef(meth);
Py_DECREF(function);
is_meth = 1;
if (!is_meth && Py_TYPE(callable) == &PyMethod_Type) {
is_meth = 1; // For consistenct; it's dead, though
args--;
total_args++;
PyObject *self = ((PyMethodObject *)callable)->im_self;
args[0] = Py_NewRef(self);
method = ((PyMethodObject *)callable)->im_func;
args[-1] = Py_NewRef(method);
Py_DECREF(callable);
callable = method;
}
total_args = oparg + is_meth;
function = PEEK(total_args + 1);
int positional_args = total_args - KWNAMES_LEN();
// Check if the call can be inlined or not
if (Py_TYPE(function) == &PyFunction_Type &&
if (Py_TYPE(callable) == &PyFunction_Type &&
tstate->interp->eval_frame == NULL &&
((PyFunctionObject *)function)->vectorcall == _PyFunction_Vectorcall)
((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall)
{
int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(function));
STACK_SHRINK(total_args);
int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)function, locals,
stack_pointer, positional_args, kwnames
tstate, (PyFunctionObject *)callable, locals,
args, positional_args, kwnames
);
kwnames = NULL;
STACK_SHRINK(2-is_meth);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 2);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
@ -2433,190 +2450,180 @@ dummy_func(
DISPATCH_INLINED(new_frame);
}
/* Callable is not a normal Python function */
PyObject *res;
if (cframe.use_tracing) {
res = trace_call_function(
tstate, function, stack_pointer-total_args,
tstate, callable, args,
positional_args, kwnames);
}
else {
res = PyObject_Vectorcall(
function, stack_pointer-total_args,
callable, args,
positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
kwnames);
}
kwnames = NULL;
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(function);
/* Clear the stack */
STACK_SHRINK(total_args);
Py_DECREF(callable);
for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]);
Py_DECREF(args[i]);
}
STACK_SHRINK(2-is_meth);
PUSH(res);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_PY_EXACT_ARGS) {
// Start out with [NULL, bound_method, arg1, arg2, ...]
// Transform to [callable, self, arg1, arg2, ...]
// Then fall through to CALL_PY_EXACT_ARGS
inst(CALL_BOUND_METHOD_EXACT_ARGS, (unused/1, unused/2, unused/1, method, callable, unused[oparg] -- unused)) {
DEOPT_IF(method != NULL, CALL);
DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL);
STAT_INC(CALL, hit);
PyObject *self = ((PyMethodObject *)callable)->im_self;
PEEK(oparg + 1) = Py_NewRef(self); // callable
PyObject *meth = ((PyMethodObject *)callable)->im_func;
PEEK(oparg + 2) = Py_NewRef(meth); // method
Py_DECREF(callable);
GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS);
}
inst(CALL_PY_EXACT_ARGS, (unused/1, func_version/2, unused/1, method, callable, args[oparg] -- unused)) {
assert(kwnames == NULL);
DEOPT_IF(tstate->interp->eval_frame, CALL);
_PyCallCache *cache = (_PyCallCache *)next_instr;
int is_meth = is_method(stack_pointer, oparg);
int argcount = oparg + is_meth;
PyObject *callable = PEEK(argcount + 1);
int is_meth = method != NULL;
int argcount = oparg;
if (is_meth) {
callable = method;
args--;
argcount++;
}
DEOPT_IF(!PyFunction_Check(callable), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL);
DEOPT_IF(func->func_version != func_version, CALL);
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(code->co_argcount != argcount, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
STAT_INC(CALL, hit);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount);
STACK_SHRINK(argcount);
for (int i = 0; i < argcount; i++) {
new_frame->localsplus[i] = stack_pointer[i];
new_frame->localsplus[i] = args[i];
}
STACK_SHRINK(2-is_meth);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 2);
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
DISPATCH_INLINED(new_frame);
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_PY_WITH_DEFAULTS) {
inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, min_args/1, method, callable, args[oparg] -- unused)) {
assert(kwnames == NULL);
DEOPT_IF(tstate->interp->eval_frame, CALL);
_PyCallCache *cache = (_PyCallCache *)next_instr;
int is_meth = is_method(stack_pointer, oparg);
int argcount = oparg + is_meth;
PyObject *callable = PEEK(argcount + 1);
int is_meth = method != NULL;
int argcount = oparg;
if (is_meth) {
callable = method;
args--;
argcount++;
}
DEOPT_IF(!PyFunction_Check(callable), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL);
DEOPT_IF(func->func_version != func_version, CALL);
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(argcount > code->co_argcount, CALL);
int minargs = cache->min_args;
DEOPT_IF(argcount < minargs, CALL);
DEOPT_IF(argcount < min_args, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
STAT_INC(CALL, hit);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount);
STACK_SHRINK(argcount);
for (int i = 0; i < argcount; i++) {
new_frame->localsplus[i] = stack_pointer[i];
new_frame->localsplus[i] = args[i];
}
for (int i = argcount; i < code->co_argcount; i++) {
PyObject *def = PyTuple_GET_ITEM(func->func_defaults,
i - minargs);
PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args);
new_frame->localsplus[i] = Py_NewRef(def);
}
STACK_SHRINK(2-is_meth);
// Manipulate stack and cache directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 2);
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
DISPATCH_INLINED(new_frame);
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_NO_KW_TYPE_1) {
inst(CALL_NO_KW_TYPE_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) {
assert(kwnames == NULL);
assert(cframe.use_tracing == 0);
assert(oparg == 1);
DEOPT_IF(is_method(stack_pointer, 1), CALL);
PyObject *obj = TOP();
PyObject *callable = SECOND();
DEOPT_IF(null != NULL, CALL);
PyObject *obj = args[0];
DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL);
STAT_INC(CALL, hit);
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
PyObject *res = Py_NewRef(Py_TYPE(obj));
Py_DECREF(callable);
res = Py_NewRef(Py_TYPE(obj));
Py_DECREF(obj);
STACK_SHRINK(2);
SET_TOP(res);
Py_DECREF(&PyType_Type); // I.e., callable
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_NO_KW_STR_1) {
inst(CALL_NO_KW_STR_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) {
assert(kwnames == NULL);
assert(cframe.use_tracing == 0);
assert(oparg == 1);
DEOPT_IF(is_method(stack_pointer, 1), CALL);
PyObject *callable = PEEK(2);
DEOPT_IF(null != NULL, CALL);
DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL);
STAT_INC(CALL, hit);
PyObject *arg = TOP();
PyObject *res = PyObject_Str(arg);
PyObject *arg = args[0];
res = PyObject_Str(arg);
Py_DECREF(arg);
Py_DECREF(&PyUnicode_Type);
STACK_SHRINK(2);
SET_TOP(res);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
Py_DECREF(&PyUnicode_Type); // I.e., callable
ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_NO_KW_TUPLE_1) {
inst(CALL_NO_KW_TUPLE_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) {
assert(kwnames == NULL);
assert(oparg == 1);
DEOPT_IF(is_method(stack_pointer, 1), CALL);
PyObject *callable = PEEK(2);
DEOPT_IF(null != NULL, CALL);
DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL);
STAT_INC(CALL, hit);
PyObject *arg = TOP();
PyObject *res = PySequence_Tuple(arg);
PyObject *arg = args[0];
res = PySequence_Tuple(arg);
Py_DECREF(arg);
Py_DECREF(&PyTuple_Type);
STACK_SHRINK(2);
SET_TOP(res);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
Py_DECREF(&PyTuple_Type); // I.e., tuple
ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_BUILTIN_CLASS) {
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
inst(CALL_BUILTIN_CLASS, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
int kwnames_len = KWNAMES_LEN();
PyObject *callable = PEEK(total_args + 1);
DEOPT_IF(!PyType_Check(callable), CALL);
PyTypeObject *tp = (PyTypeObject *)callable;
DEOPT_IF(tp->tp_vectorcall == NULL, CALL);
STAT_INC(CALL, hit);
STACK_SHRINK(total_args);
PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer,
res = tp->tp_vectorcall((PyObject *)tp, args,
total_args - kwnames_len, kwnames);
kwnames = NULL;
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]);
Py_DECREF(args[i]);
}
Py_DECREF(tp);
STACK_SHRINK(1-is_meth);
SET_TOP(res);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_NO_KW_BUILTIN_O) {
inst(CALL_NO_KW_BUILTIN_O, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
assert(cframe.use_tracing == 0);
/* Builtin METH_O functions */
assert(kwnames == NULL);
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL);
PyObject *callable = PEEK(total_args + 1);
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL);
STAT_INC(CALL, hit);
@ -2626,81 +2633,74 @@ dummy_func(
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
goto error;
}
PyObject *arg = TOP();
PyObject *res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
PyObject *arg = args[0];
res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(arg);
Py_DECREF(callable);
STACK_SHRINK(2-is_meth);
SET_TOP(res);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_NO_KW_BUILTIN_FAST) {
inst(CALL_NO_KW_BUILTIN_FAST, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
assert(cframe.use_tracing == 0);
/* Builtin METH_FASTCALL functions, without keywords */
assert(kwnames == NULL);
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
PyObject *callable = PEEK(total_args + 1);
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL,
CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
STACK_SHRINK(total_args);
/* res = func(self, args, nargs) */
PyObject *res = ((_PyCFunctionFast)(void(*)(void))cfunc)(
res = ((_PyCFunctionFast)(void(*)(void))cfunc)(
PyCFunction_GET_SELF(callable),
stack_pointer,
args,
total_args);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]);
Py_DECREF(args[i]);
}
STACK_SHRINK(2-is_meth);
PUSH(res);
Py_DECREF(callable);
if (res == NULL) {
ERROR_IF(res == NULL, error);
/* Not deopting because this doesn't mean our optimization was
wrong. `res` can be NULL for valid reasons. Eg. getattr(x,
'invalid'). In those cases an exception is set, so we must
handle it.
*/
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
CHECK_EVAL_BREAKER();
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_BUILTIN_FAST_WITH_KEYWORDS) {
inst(CALL_BUILTIN_FAST_WITH_KEYWORDS, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
assert(cframe.use_tracing == 0);
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
PyObject *callable = PEEK(total_args + 1);
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) !=
(METH_FASTCALL | METH_KEYWORDS), CALL);
STAT_INC(CALL, hit);
STACK_SHRINK(total_args);
/* res = func(self, args, nargs, kwnames) */
_PyCFunctionFastWithKeywords cfunc =
(_PyCFunctionFastWithKeywords)(void(*)(void))
PyCFunction_GET_FUNCTION(callable);
PyObject *res = cfunc(
res = cfunc(
PyCFunction_GET_SELF(callable),
stack_pointer,
args,
total_args - KWNAMES_LEN(),
kwnames
);
@ -2709,117 +2709,109 @@ dummy_func(
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]);
Py_DECREF(args[i]);
}
STACK_SHRINK(2-is_meth);
PUSH(res);
Py_DECREF(callable);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_NO_KW_LEN) {
inst(CALL_NO_KW_LEN, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
assert(cframe.use_tracing == 0);
assert(kwnames == NULL);
/* len(o) */
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL);
PyObject *callable = PEEK(total_args + 1);
PyInterpreterState *interp = _PyInterpreterState_GET();
DEOPT_IF(callable != interp->callable_cache.len, CALL);
STAT_INC(CALL, hit);
PyObject *arg = TOP();
PyObject *arg = args[0];
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
goto error;
}
PyObject *res = PyLong_FromSsize_t(len_i);
res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
STACK_SHRINK(2-is_meth);
SET_TOP(res);
Py_DECREF(callable);
Py_DECREF(arg);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
ERROR_IF(res == NULL, error);
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_NO_KW_ISINSTANCE) {
inst(CALL_NO_KW_ISINSTANCE, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
assert(cframe.use_tracing == 0);
assert(kwnames == NULL);
/* isinstance(o, o2) */
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
PyObject *callable = PEEK(total_args + 1);
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(total_args != 2, CALL);
PyInterpreterState *interp = _PyInterpreterState_GET();
DEOPT_IF(callable != interp->callable_cache.isinstance, CALL);
STAT_INC(CALL, hit);
PyObject *cls = POP();
PyObject *inst = TOP();
PyObject *cls = args[1];
PyObject *inst = args[0];
int retval = PyObject_IsInstance(inst, cls);
if (retval < 0) {
Py_DECREF(cls);
goto error;
}
PyObject *res = PyBool_FromLong(retval);
res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
STACK_SHRINK(2-is_meth);
SET_TOP(res);
Py_DECREF(inst);
Py_DECREF(cls);
Py_DECREF(callable);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
ERROR_IF(res == NULL, error);
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_NO_KW_LIST_APPEND) {
// This is secretly a super-instruction
inst(CALL_NO_KW_LIST_APPEND, (unused/1, unused/2, unused/1, method, self, args[oparg] -- unused)) {
assert(cframe.use_tracing == 0);
assert(kwnames == NULL);
assert(oparg == 1);
PyObject *callable = PEEK(3);
assert(method != NULL);
PyInterpreterState *interp = _PyInterpreterState_GET();
DEOPT_IF(callable != interp->callable_cache.list_append, CALL);
PyObject *list = SECOND();
DEOPT_IF(!PyList_Check(list), CALL);
DEOPT_IF(method != interp->callable_cache.list_append, CALL);
DEOPT_IF(!PyList_Check(self), CALL);
STAT_INC(CALL, hit);
PyObject *arg = POP();
if (_PyList_AppendTakeRef((PyListObject *)list, arg) < 0) {
goto error;
if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) {
goto pop_1_error; // Since arg is DECREF'ed already
}
STACK_SHRINK(2);
Py_DECREF(list);
Py_DECREF(callable);
Py_DECREF(self);
Py_DECREF(method);
STACK_SHRINK(3);
// CALL + POP_TOP
JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1);
assert(_Py_OPCODE(next_instr[-1]) == POP_TOP);
DISPATCH();
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_NO_KW_METHOD_DESCRIPTOR_O) {
inst(CALL_NO_KW_METHOD_DESCRIPTOR_O, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) {
assert(kwnames == NULL);
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
args--;
total_args++;
}
PyMethodDescrObject *callable =
(PyMethodDescrObject *)PEEK(total_args + 1);
DEOPT_IF(total_args != 2, CALL);
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method;
DEOPT_IF(meth->ml_flags != METH_O, CALL);
PyObject *arg = TOP();
PyObject *self = SECOND();
PyObject *arg = args[1];
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
@ -2828,69 +2820,62 @@ dummy_func(
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
goto error;
}
PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, arg);
res = _PyCFunction_TrampolineCall(cfunc, self, arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
Py_DECREF(arg);
STACK_SHRINK(oparg + 1);
SET_TOP(res);
Py_DECREF(callable);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) {
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) {
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
args--;
total_args++;
}
PyMethodDescrObject *callable =
(PyMethodDescrObject *)PEEK(total_args + 1);
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method;
DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL);
PyTypeObject *d_type = callable->d_common.d_type;
PyObject *self = PEEK(total_args);
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL);
STAT_INC(CALL, hit);
int nargs = total_args - 1;
STACK_SHRINK(nargs);
_PyCFunctionFastWithKeywords cfunc =
(_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
PyObject *res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(),
kwnames);
res = cfunc(self, args + 1, nargs - KWNAMES_LEN(), kwnames);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
kwnames = NULL;
/* Free the arguments. */
for (int i = 0; i < nargs; i++) {
Py_DECREF(stack_pointer[i]);
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
Py_DECREF(self);
STACK_SHRINK(2-is_meth);
SET_TOP(res);
Py_DECREF(callable);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) {
inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) {
assert(kwnames == NULL);
assert(oparg == 0 || oparg == 1);
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL);
PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND();
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method;
PyObject *self = TOP();
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL);
STAT_INC(CALL, hit);
@ -2900,52 +2885,43 @@ dummy_func(
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
goto error;
}
PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
STACK_SHRINK(oparg + 1);
SET_TOP(res);
Py_DECREF(callable);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) {
inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) {
assert(kwnames == NULL);
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
args--;
total_args++;
}
PyMethodDescrObject *callable =
(PyMethodDescrObject *)PEEK(total_args + 1);
/* Builtin METH_FASTCALL methods, without keywords */
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method;
DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL);
PyObject *self = PEEK(total_args);
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
STAT_INC(CALL, hit);
_PyCFunctionFast cfunc =
(_PyCFunctionFast)(void(*)(void))meth->ml_meth;
int nargs = total_args - 1;
STACK_SHRINK(nargs);
PyObject *res = cfunc(self, stack_pointer, nargs);
res = cfunc(self, args + 1, nargs);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Clear the stack of the arguments. */
for (int i = 0; i < nargs; i++) {
Py_DECREF(stack_pointer[i]);
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
Py_DECREF(self);
STACK_SHRINK(2-is_meth);
SET_TOP(res);
Py_DECREF(callable);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
@ -3153,12 +3129,4 @@ dummy_func(
// Future families go below this point //
family(call, INLINE_CACHE_ENTRIES_CALL) = {
CALL, CALL_PY_EXACT_ARGS,
CALL_PY_WITH_DEFAULTS, CALL_BOUND_METHOD_EXACT_ARGS, CALL_BUILTIN_CLASS,
CALL_BUILTIN_FAST_WITH_KEYWORDS, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, CALL_NO_KW_BUILTIN_FAST,
CALL_NO_KW_BUILTIN_O, CALL_NO_KW_ISINSTANCE, CALL_NO_KW_LEN,
CALL_NO_KW_LIST_APPEND, CALL_NO_KW_METHOD_DESCRIPTOR_FAST, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
CALL_NO_KW_METHOD_DESCRIPTOR_O, CALL_NO_KW_STR_1, CALL_NO_KW_TUPLE_1,
CALL_NO_KW_TYPE_1 };
family(store_fast) = { STORE_FAST, STORE_FAST__LOAD_FAST, STORE_FAST__STORE_FAST };

6
Python/ceval.c
View File

@ -688,12 +688,6 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) {
}
// GH-89279: Must be a macro to be sure it's inlined by MSVC.
#define is_method(stack_pointer, args) (PEEK((args)+2) != NULL)
#define KWNAMES_LEN() \
(kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(kwnames)))
/* Disable unused label warnings. They are handy for debugging, even
if computed gotos aren't used. */

3
Python/ceval_macros.h
View File

@ -347,3 +347,6 @@ GETITEM(PyObject *v, Py_ssize_t i) {
} while (0);
#define NAME_ERROR_MSG "name '%.200s' is not defined"
#define KWNAMES_LEN() \
(kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(kwnames)))

552
Python/generated_cases.c.h generated
View File

@ -2994,19 +2994,6 @@
DISPATCH();
}
TARGET(CALL_BOUND_METHOD_EXACT_ARGS) {
DEOPT_IF(is_method(stack_pointer, oparg), CALL);
PyObject *function = PEEK(oparg + 1);
DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, CALL);
STAT_INC(CALL, hit);
PyObject *self = ((PyMethodObject *)function)->im_self;
PEEK(oparg + 1) = Py_NewRef(self);
PyObject *meth = ((PyMethodObject *)function)->im_func;
PEEK(oparg + 2) = Py_NewRef(meth);
Py_DECREF(function);
GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS);
}
TARGET(KW_NAMES) {
assert(kwnames == NULL);
assert(oparg < PyTuple_GET_SIZE(consts));
@ -3016,48 +3003,55 @@
TARGET(CALL) {
PREDICTED(CALL);
static_assert(INLINE_CACHE_ENTRIES_CALL == 4, "incorrect cache size");
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
#if ENABLE_SPECIALIZATION
_PyCallCache *cache = (_PyCallCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0);
int is_meth = is_method(stack_pointer, oparg);
int nargs = oparg + is_meth;
PyObject *callable = PEEK(nargs + 1);
next_instr--;
_Py_Specialize_Call(callable, next_instr, nargs, kwnames);
_Py_Specialize_Call(callable, next_instr, total_args, kwnames);
DISPATCH_SAME_OPARG();
}
STAT_INC(CALL, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
#endif /* ENABLE_SPECIALIZATION */
int total_args, is_meth;
is_meth = is_method(stack_pointer, oparg);
PyObject *function = PEEK(oparg + 1);
if (!is_meth && Py_TYPE(function) == &PyMethod_Type) {
PyObject *self = ((PyMethodObject *)function)->im_self;
PEEK(oparg+1) = Py_NewRef(self);
PyObject *meth = ((PyMethodObject *)function)->im_func;
PEEK(oparg+2) = Py_NewRef(meth);
Py_DECREF(function);
is_meth = 1;
if (!is_meth && Py_TYPE(callable) == &PyMethod_Type) {
is_meth = 1; // For consistenct; it's dead, though
args--;
total_args++;
PyObject *self = ((PyMethodObject *)callable)->im_self;
args[0] = Py_NewRef(self);
method = ((PyMethodObject *)callable)->im_func;
args[-1] = Py_NewRef(method);
Py_DECREF(callable);
callable = method;
}
total_args = oparg + is_meth;
function = PEEK(total_args + 1);
int positional_args = total_args - KWNAMES_LEN();
// Check if the call can be inlined or not
if (Py_TYPE(function) == &PyFunction_Type &&
if (Py_TYPE(callable) == &PyFunction_Type &&
tstate->interp->eval_frame == NULL &&
((PyFunctionObject *)function)->vectorcall == _PyFunction_Vectorcall)
((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall)
{
int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(function));
STACK_SHRINK(total_args);
int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)function, locals,
stack_pointer, positional_args, kwnames
tstate, (PyFunctionObject *)callable, locals,
args, positional_args, kwnames
);
kwnames = NULL;
STACK_SHRINK(2-is_meth);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 2);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
@ -3067,189 +3061,234 @@
DISPATCH_INLINED(new_frame);
}
/* Callable is not a normal Python function */
PyObject *res;
if (cframe.use_tracing) {
res = trace_call_function(
tstate, function, stack_pointer-total_args,
tstate, callable, args,
positional_args, kwnames);
}
else {
res = PyObject_Vectorcall(
function, stack_pointer-total_args,
callable, args,
positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
kwnames);
}
kwnames = NULL;
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(function);
/* Clear the stack */
STACK_SHRINK(total_args);
Py_DECREF(callable);
for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]);
Py_DECREF(args[i]);
}
STACK_SHRINK(2-is_meth);
PUSH(res);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
POKE(1, res);
JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_BOUND_METHOD_EXACT_ARGS) {
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
DEOPT_IF(method != NULL, CALL);
DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL);
STAT_INC(CALL, hit);
PyObject *self = ((PyMethodObject *)callable)->im_self;
PEEK(oparg + 1) = Py_NewRef(self); // callable
PyObject *meth = ((PyMethodObject *)callable)->im_func;
PEEK(oparg + 2) = Py_NewRef(meth); // method
Py_DECREF(callable);
GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS);
}
TARGET(CALL_PY_EXACT_ARGS) {
PREDICTED(CALL_PY_EXACT_ARGS);
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
uint32_t func_version = read_u32(&next_instr[1].cache);
assert(kwnames == NULL);
DEOPT_IF(tstate->interp->eval_frame, CALL);
_PyCallCache *cache = (_PyCallCache *)next_instr;
int is_meth = is_method(stack_pointer, oparg);
int argcount = oparg + is_meth;
PyObject *callable = PEEK(argcount + 1);
int is_meth = method != NULL;
int argcount = oparg;
if (is_meth) {
callable = method;
args--;
argcount++;
}
DEOPT_IF(!PyFunction_Check(callable), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL);
DEOPT_IF(func->func_version != func_version, CALL);
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(code->co_argcount != argcount, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
STAT_INC(CALL, hit);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount);
STACK_SHRINK(argcount);
for (int i = 0; i < argcount; i++) {
new_frame->localsplus[i] = stack_pointer[i];
new_frame->localsplus[i] = args[i];
}
STACK_SHRINK(2-is_meth);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 2);
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
DISPATCH_INLINED(new_frame);
}
TARGET(CALL_PY_WITH_DEFAULTS) {
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
uint32_t func_version = read_u32(&next_instr[1].cache);
uint16_t min_args = read_u16(&next_instr[3].cache);
assert(kwnames == NULL);
DEOPT_IF(tstate->interp->eval_frame, CALL);
_PyCallCache *cache = (_PyCallCache *)next_instr;
int is_meth = is_method(stack_pointer, oparg);
int argcount = oparg + is_meth;
PyObject *callable = PEEK(argcount + 1);
int is_meth = method != NULL;
int argcount = oparg;
if (is_meth) {
callable = method;
args--;
argcount++;
}
DEOPT_IF(!PyFunction_Check(callable), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL);
DEOPT_IF(func->func_version != func_version, CALL);
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(argcount > code->co_argcount, CALL);
int minargs = cache->min_args;
DEOPT_IF(argcount < minargs, CALL);
DEOPT_IF(argcount < min_args, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
STAT_INC(CALL, hit);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount);
STACK_SHRINK(argcount);
for (int i = 0; i < argcount; i++) {
new_frame->localsplus[i] = stack_pointer[i];
new_frame->localsplus[i] = args[i];
}
for (int i = argcount; i < code->co_argcount; i++) {
PyObject *def = PyTuple_GET_ITEM(func->func_defaults,
i - minargs);
PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args);
new_frame->localsplus[i] = Py_NewRef(def);
}
STACK_SHRINK(2-is_meth);
// Manipulate stack and cache directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 2);
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
DISPATCH_INLINED(new_frame);
}
TARGET(CALL_NO_KW_TYPE_1) {
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *null = PEEK(2 + oparg);
PyObject *res;
assert(kwnames == NULL);
assert(cframe.use_tracing == 0);
assert(oparg == 1);
DEOPT_IF(is_method(stack_pointer, 1), CALL);
PyObject *obj = TOP();
PyObject *callable = SECOND();
DEOPT_IF(null != NULL, CALL);
PyObject *obj = args[0];
DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL);
STAT_INC(CALL, hit);
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
PyObject *res = Py_NewRef(Py_TYPE(obj));
Py_DECREF(callable);
res = Py_NewRef(Py_TYPE(obj));
Py_DECREF(obj);
STACK_SHRINK(2);
SET_TOP(res);
Py_DECREF(&PyType_Type); // I.e., callable
STACK_SHRINK(oparg);
STACK_SHRINK(1);
POKE(1, res);
JUMPBY(4);
DISPATCH();
}
TARGET(CALL_NO_KW_STR_1) {
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *null = PEEK(2 + oparg);
PyObject *res;
assert(kwnames == NULL);
assert(cframe.use_tracing == 0);
assert(oparg == 1);
DEOPT_IF(is_method(stack_pointer, 1), CALL);
PyObject *callable = PEEK(2);
DEOPT_IF(null != NULL, CALL);
DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL);
STAT_INC(CALL, hit);
PyObject *arg = TOP();
PyObject *res = PyObject_Str(arg);
PyObject *arg = args[0];
res = PyObject_Str(arg);
Py_DECREF(arg);
Py_DECREF(&PyUnicode_Type);
STACK_SHRINK(2);
SET_TOP(res);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
Py_DECREF(&PyUnicode_Type); // I.e., callable
if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
POKE(1, res);
JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_NO_KW_TUPLE_1) {
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *null = PEEK(2 + oparg);
PyObject *res;
assert(kwnames == NULL);
assert(oparg == 1);
DEOPT_IF(is_method(stack_pointer, 1), CALL);
PyObject *callable = PEEK(2);
DEOPT_IF(null != NULL, CALL);
DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL);
STAT_INC(CALL, hit);
PyObject *arg = TOP();
PyObject *res = PySequence_Tuple(arg);
PyObject *arg = args[0];
res = PySequence_Tuple(arg);
Py_DECREF(arg);
Py_DECREF(&PyTuple_Type);
STACK_SHRINK(2);
SET_TOP(res);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
Py_DECREF(&PyTuple_Type); // I.e., tuple
if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
POKE(1, res);
JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_BUILTIN_CLASS) {
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
int kwnames_len = KWNAMES_LEN();
PyObject *callable = PEEK(total_args + 1);
DEOPT_IF(!PyType_Check(callable), CALL);
PyTypeObject *tp = (PyTypeObject *)callable;
DEOPT_IF(tp->tp_vectorcall == NULL, CALL);
STAT_INC(CALL, hit);
STACK_SHRINK(total_args);
PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer,
res = tp->tp_vectorcall((PyObject *)tp, args,
total_args - kwnames_len, kwnames);
kwnames = NULL;
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]);
Py_DECREF(args[i]);
}
Py_DECREF(tp);
STACK_SHRINK(1-is_meth);
SET_TOP(res);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
POKE(1, res);
JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_NO_KW_BUILTIN_O) {
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
assert(cframe.use_tracing == 0);
/* Builtin METH_O functions */
assert(kwnames == NULL);
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL);
PyObject *callable = PEEK(total_args + 1);
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL);
STAT_INC(CALL, hit);
@ -3259,81 +3298,92 @@
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
goto error;
}
PyObject *arg = TOP();
PyObject *res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
PyObject *arg = args[0];
res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(arg);
Py_DECREF(callable);
STACK_SHRINK(2-is_meth);
SET_TOP(res);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
POKE(1, res);
JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_NO_KW_BUILTIN_FAST) {
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
assert(cframe.use_tracing == 0);
/* Builtin METH_FASTCALL functions, without keywords */
assert(kwnames == NULL);
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
PyObject *callable = PEEK(total_args + 1);
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL,
CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
STACK_SHRINK(total_args);
/* res = func(self, args, nargs) */
PyObject *res = ((_PyCFunctionFast)(void(*)(void))cfunc)(
res = ((_PyCFunctionFast)(void(*)(void))cfunc)(
PyCFunction_GET_SELF(callable),
stack_pointer,
args,
total_args);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]);
Py_DECREF(args[i]);
}
STACK_SHRINK(2-is_meth);
PUSH(res);
Py_DECREF(callable);
if (res == NULL) {
if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
/* Not deopting because this doesn't mean our optimization was
wrong. `res` can be NULL for valid reasons. Eg. getattr(x,
'invalid'). In those cases an exception is set, so we must
handle it.
*/
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
STACK_SHRINK(oparg);
STACK_SHRINK(1);
POKE(1, res);
JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) {
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
assert(cframe.use_tracing == 0);
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
PyObject *callable = PEEK(total_args + 1);
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) !=
(METH_FASTCALL | METH_KEYWORDS), CALL);
STAT_INC(CALL, hit);
STACK_SHRINK(total_args);
/* res = func(self, args, nargs, kwnames) */
_PyCFunctionFastWithKeywords cfunc =
(_PyCFunctionFastWithKeywords)(void(*)(void))
PyCFunction_GET_FUNCTION(callable);
PyObject *res = cfunc(
res = cfunc(
PyCFunction_GET_SELF(callable),
stack_pointer,
args,
total_args - KWNAMES_LEN(),
kwnames
);
@ -3342,99 +3392,112 @@
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]);
Py_DECREF(args[i]);
}
STACK_SHRINK(2-is_meth);
PUSH(res);
Py_DECREF(callable);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
POKE(1, res);
JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_NO_KW_LEN) {
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
assert(cframe.use_tracing == 0);
assert(kwnames == NULL);
/* len(o) */
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL);
PyObject *callable = PEEK(total_args + 1);
PyInterpreterState *interp = _PyInterpreterState_GET();
DEOPT_IF(callable != interp->callable_cache.len, CALL);
STAT_INC(CALL, hit);
PyObject *arg = TOP();
PyObject *arg = args[0];
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
goto error;
}
PyObject *res = PyLong_FromSsize_t(len_i);
res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
STACK_SHRINK(2-is_meth);
SET_TOP(res);
Py_DECREF(callable);
Py_DECREF(arg);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
POKE(1, res);
JUMPBY(4);
DISPATCH();
}
TARGET(CALL_NO_KW_ISINSTANCE) {
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
assert(cframe.use_tracing == 0);
assert(kwnames == NULL);
/* isinstance(o, o2) */
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
PyObject *callable = PEEK(total_args + 1);
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(total_args != 2, CALL);
PyInterpreterState *interp = _PyInterpreterState_GET();
DEOPT_IF(callable != interp->callable_cache.isinstance, CALL);
STAT_INC(CALL, hit);
PyObject *cls = POP();
PyObject *inst = TOP();
PyObject *cls = args[1];
PyObject *inst = args[0];
int retval = PyObject_IsInstance(inst, cls);
if (retval < 0) {
Py_DECREF(cls);
goto error;
}
PyObject *res = PyBool_FromLong(retval);
res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
STACK_SHRINK(2-is_meth);
SET_TOP(res);
Py_DECREF(inst);
Py_DECREF(cls);
Py_DECREF(callable);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
POKE(1, res);
JUMPBY(4);
DISPATCH();
}
TARGET(CALL_NO_KW_LIST_APPEND) {
PyObject **args = &PEEK(oparg);
PyObject *self = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
assert(cframe.use_tracing == 0);
assert(kwnames == NULL);
assert(oparg == 1);
PyObject *callable = PEEK(3);
assert(method != NULL);
PyInterpreterState *interp = _PyInterpreterState_GET();
DEOPT_IF(callable != interp->callable_cache.list_append, CALL);
PyObject *list = SECOND();
DEOPT_IF(!PyList_Check(list), CALL);
DEOPT_IF(method != interp->callable_cache.list_append, CALL);
DEOPT_IF(!PyList_Check(self), CALL);
STAT_INC(CALL, hit);
PyObject *arg = POP();
if (_PyList_AppendTakeRef((PyListObject *)list, arg) < 0) {
goto error;
if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) {
goto pop_1_error; // Since arg is DECREF'ed already
}
STACK_SHRINK(2);
Py_DECREF(list);
Py_DECREF(callable);
Py_DECREF(self);
Py_DECREF(method);
STACK_SHRINK(3);
// CALL + POP_TOP
JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1);
assert(_Py_OPCODE(next_instr[-1]) == POP_TOP);
@ -3442,17 +3505,24 @@
}
TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) {
PyObject **args = &PEEK(oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
assert(kwnames == NULL);
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
args--;
total_args++;
}
PyMethodDescrObject *callable =
(PyMethodDescrObject *)PEEK(total_args + 1);
DEOPT_IF(total_args != 2, CALL);
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method;
DEOPT_IF(meth->ml_flags != METH_O, CALL);
PyObject *arg = TOP();
PyObject *self = SECOND();
PyObject *arg = args[1];
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
@ -3461,69 +3531,78 @@
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
goto error;
}
PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, arg);
res = _PyCFunction_TrampolineCall(cfunc, self, arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
Py_DECREF(arg);
STACK_SHRINK(oparg + 1);
SET_TOP(res);
Py_DECREF(callable);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
POKE(1, res);
JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) {
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
PyObject **args = &PEEK(oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
args--;
total_args++;
}
PyMethodDescrObject *callable =
(PyMethodDescrObject *)PEEK(total_args + 1);
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method;
DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL);
PyTypeObject *d_type = callable->d_common.d_type;
PyObject *self = PEEK(total_args);
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL);
STAT_INC(CALL, hit);
int nargs = total_args - 1;
STACK_SHRINK(nargs);
_PyCFunctionFastWithKeywords cfunc =
(_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
PyObject *res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(),
kwnames);
res = cfunc(self, args + 1, nargs - KWNAMES_LEN(), kwnames);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
kwnames = NULL;
/* Free the arguments. */
for (int i = 0; i < nargs; i++) {
Py_DECREF(stack_pointer[i]);
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
Py_DECREF(self);
STACK_SHRINK(2-is_meth);
SET_TOP(res);
Py_DECREF(callable);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
POKE(1, res);
JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) {
PyObject **args = &PEEK(oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
assert(kwnames == NULL);
assert(oparg == 0 || oparg == 1);
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL);
PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND();
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method;
PyObject *self = TOP();
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL);
STAT_INC(CALL, hit);
@ -3533,52 +3612,55 @@
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
goto error;
}
PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
STACK_SHRINK(oparg + 1);
SET_TOP(res);
Py_DECREF(callable);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
POKE(1, res);
JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) {
PyObject **args = &PEEK(oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
assert(kwnames == NULL);
int is_meth = is_method(stack_pointer, oparg);
int total_args = oparg + is_meth;
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
args--;
total_args++;
}
PyMethodDescrObject *callable =
(PyMethodDescrObject *)PEEK(total_args + 1);
/* Builtin METH_FASTCALL methods, without keywords */
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method;
DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL);
PyObject *self = PEEK(total_args);
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
STAT_INC(CALL, hit);
_PyCFunctionFast cfunc =
(_PyCFunctionFast)(void(*)(void))meth->ml_meth;
int nargs = total_args - 1;
STACK_SHRINK(nargs);
PyObject *res = cfunc(self, stack_pointer, nargs);
res = cfunc(self, args + 1, nargs);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Clear the stack of the arguments. */
for (int i = 0; i < nargs; i++) {
Py_DECREF(stack_pointer[i]);
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
Py_DECREF(self);
STACK_SHRINK(2-is_meth);
SET_TOP(res);
Py_DECREF(callable);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
POKE(1, res);
JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}

112
Python/opcode_metadata.h
View File

@ -286,44 +286,44 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) {
return 1;
case LOAD_ATTR_METHOD_LAZY_DICT:
return 1;
case CALL_BOUND_METHOD_EXACT_ARGS:
return -1;
case KW_NAMES:
return 0;
case CALL:
return -1;
return oparg + 2;
case CALL_BOUND_METHOD_EXACT_ARGS:
return oparg + 2;
case CALL_PY_EXACT_ARGS:
return -1;
return oparg + 2;
case CALL_PY_WITH_DEFAULTS:
return -1;
return oparg + 2;
case CALL_NO_KW_TYPE_1:
return -1;
return oparg + 2;
case CALL_NO_KW_STR_1:
return -1;
return oparg + 2;
case CALL_NO_KW_TUPLE_1:
return -1;
return oparg + 2;
case CALL_BUILTIN_CLASS:
return -1;
return oparg + 2;
case CALL_NO_KW_BUILTIN_O:
return -1;
return oparg + 2;
case CALL_NO_KW_BUILTIN_FAST:
return -1;
return oparg + 2;
case CALL_BUILTIN_FAST_WITH_KEYWORDS:
return -1;
return oparg + 2;
case CALL_NO_KW_LEN:
return -1;
return oparg + 2;
case CALL_NO_KW_ISINSTANCE:
return -1;
return oparg + 2;
case CALL_NO_KW_LIST_APPEND:
return -1;
return oparg + 2;
case CALL_NO_KW_METHOD_DESCRIPTOR_O:
return -1;
return oparg + 2;
case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS:
return -1;
return oparg + 2;
case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS:
return -1;
return oparg + 2;
case CALL_NO_KW_METHOD_DESCRIPTOR_FAST:
return -1;
return oparg + 2;
case CALL_FUNCTION_EX:
return ((oparg & 1) ? 1 : 0) + 3;
case MAKE_FUNCTION:
@ -634,44 +634,44 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) {
return ((oparg & 1) ? 1 : 0) + 1;
case LOAD_ATTR_METHOD_LAZY_DICT:
return ((oparg & 1) ? 1 : 0) + 1;
case CALL_BOUND_METHOD_EXACT_ARGS:
return -1;
case KW_NAMES:
return 0;
case CALL:
return -1;
return 1;
case CALL_BOUND_METHOD_EXACT_ARGS:
return 1;
case CALL_PY_EXACT_ARGS:
return -1;
return 1;
case CALL_PY_WITH_DEFAULTS:
return -1;
return 1;
case CALL_NO_KW_TYPE_1:
return -1;
return 1;
case CALL_NO_KW_STR_1:
return -1;
return 1;
case CALL_NO_KW_TUPLE_1:
return -1;
return 1;
case CALL_BUILTIN_CLASS:
return -1;
return 1;
case CALL_NO_KW_BUILTIN_O:
return -1;
return 1;
case CALL_NO_KW_BUILTIN_FAST:
return -1;
return 1;
case CALL_BUILTIN_FAST_WITH_KEYWORDS:
return -1;
return 1;
case CALL_NO_KW_LEN:
return -1;
return 1;
case CALL_NO_KW_ISINSTANCE:
return -1;
return 1;
case CALL_NO_KW_LIST_APPEND:
return -1;
return 1;
case CALL_NO_KW_METHOD_DESCRIPTOR_O:
return -1;
return 1;
case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS:
return -1;
return 1;
case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS:
return -1;
return 1;
case CALL_NO_KW_METHOD_DESCRIPTOR_FAST:
return -1;
return 1;
case CALL_FUNCTION_EX:
return 1;
case MAKE_FUNCTION:
@ -846,25 +846,25 @@ struct opcode_metadata {
[LOAD_ATTR_METHOD_WITH_VALUES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 },
[LOAD_ATTR_METHOD_NO_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 },
[LOAD_ATTR_METHOD_LAZY_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 },
[CALL_BOUND_METHOD_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[KW_NAMES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL_PY_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL_PY_WITH_DEFAULTS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL_NO_KW_TYPE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL_NO_KW_STR_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL_NO_KW_TUPLE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL_BUILTIN_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL_NO_KW_BUILTIN_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL_NO_KW_BUILTIN_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL_BUILTIN_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL_NO_KW_LEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL_NO_KW_ISINSTANCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL_NO_KW_LIST_APPEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL_NO_KW_METHOD_DESCRIPTOR_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_BOUND_METHOD_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_PY_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_PY_WITH_DEFAULTS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_TYPE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_STR_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_TUPLE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_BUILTIN_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_BUILTIN_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_BUILTIN_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_BUILTIN_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_LEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_ISINSTANCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_LIST_APPEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_METHOD_DESCRIPTOR_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_FUNCTION_EX] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[MAKE_FUNCTION] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[RETURN_GENERATOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX },

21
Tools/cases_generator/generate_cases.py
View File

@ -391,9 +391,11 @@ class Instruction:
# Write the body, substituting a goto for ERROR_IF() and other stuff
assert dedent <= 0
extra = " " * -dedent
names_to_skip = self.unmoved_names | frozenset({UNUSED, "null"})
for line in self.block_text:
if m := re.match(r"(\s*)ERROR_IF\((.+), (\w+)\);\s*(?://.*)?$", line):
space, cond, label = m.groups()
space = extra + space
# ERROR_IF() must pop the inputs from the stack.
# The code block is responsible for DECREF()ing them.
# NOTE: If the label doesn't exist, just add it to ceval.c.
@ -412,16 +414,25 @@ class Instruction:
symbolic = ""
if symbolic:
out.write_raw(
f"{extra}{space}if ({cond}) {{ STACK_SHRINK({symbolic}); goto {label}; }}\n"
f"{space}if ({cond}) {{ STACK_SHRINK({symbolic}); goto {label}; }}\n"
)
else:
out.write_raw(f"{extra}{space}if ({cond}) goto {label};\n")
out.write_raw(f"{space}if ({cond}) goto {label};\n")
elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*(?://.*)?$", line):
if not self.register:
space = m.group(1)
space = extra + m.group(1)
for ieff in self.input_effects:
if ieff.name not in self.unmoved_names:
out.write_raw(f"{extra}{space}Py_DECREF({ieff.name});\n")
if ieff.name in names_to_skip:
continue
if ieff.size:
out.write_raw(
f"{space}for (int _i = {ieff.size}; --_i >= 0;) {{\n"
)
out.write_raw(f"{space} Py_DECREF({ieff.name}[_i]);\n")
out.write_raw(f"{space}}}\n")
else:
decref = "XDECREF" if ieff.cond else "DECREF"
out.write_raw(f"{space}Py_{decref}({ieff.name});\n")
else:
out.write_raw(extra + line)