diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 336088e0819..bb1deaf3fbe 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -436,7 +436,7 @@ dummy_func( PREDICT(JUMP_BACKWARD); } - family(store_subscr) = { + family(store_subscr, INLINE_CACHE_ENTRIES_STORE_SUBSCR) = { STORE_SUBSCR, STORE_SUBSCR_DICT, STORE_SUBSCR_LIST_INT, @@ -950,7 +950,7 @@ dummy_func( Py_DECREF(seq); } - family(store_attr) = { + family(store_attr, INLINE_CACHE_ENTRIES_STORE_ATTR) = { STORE_ATTR, STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_SLOT, @@ -1436,6 +1436,20 @@ dummy_func( PREDICT(JUMP_BACKWARD); } + family(load_attr, INLINE_CACHE_ENTRIES_LOAD_ATTR) = { + LOAD_ATTR, + LOAD_ATTR_INSTANCE_VALUE, + LOAD_ATTR_MODULE, + LOAD_ATTR_WITH_HINT, + LOAD_ATTR_SLOT, + LOAD_ATTR_CLASS, + LOAD_ATTR_PROPERTY, + LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + LOAD_ATTR_METHOD_WITH_VALUES, + LOAD_ATTR_METHOD_NO_DICT, + LOAD_ATTR_METHOD_LAZY_DICT, + }; + inst(LOAD_ATTR, (unused/9, owner -- res2 if (oparg & 1), res)) { #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -1485,64 +1499,43 @@ dummy_func( } } - // error: LOAD_ATTR has irregular stack effect - inst(LOAD_ATTR_INSTANCE_VALUE) { + inst(LOAD_ATTR_INSTANCE_VALUE, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); - PyObject *res; PyTypeObject *tp = Py_TYPE(owner); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - uint32_t type_version = read_u32(cache->version); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); assert(tp->tp_dictoffset < 0); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - res = _PyDictOrValues_GetValues(dorv)->values[cache->index]; + res = _PyDictOrValues_GetValues(dorv)->values[index]; DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); - SET_TOP(NULL); - STACK_GROW((oparg & 1)); - SET_TOP(res); + res2 = NULL; Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); } - // error: LOAD_ATTR has irregular stack effect - inst(LOAD_ATTR_MODULE) { + inst(LOAD_ATTR_MODULE, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); - PyObject *res; - _PyAttrCache *cache = (_PyAttrCache *)next_instr; DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != read_u32(cache->version), - LOAD_ATTR); + DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(cache->index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + cache->index; + assert(index < dict->ma_keys->dk_nentries); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; res = ep->me_value; DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); - SET_TOP(NULL); - STACK_GROW((oparg & 1)); - SET_TOP(res); + res2 = NULL; Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); } - // error: LOAD_ATTR has irregular stack effect - inst(LOAD_ATTR_WITH_HINT) { + inst(LOAD_ATTR_WITH_HINT, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); - PyObject *res; PyTypeObject *tp = Py_TYPE(owner); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - uint32_t type_version = read_u32(cache->version); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); @@ -1552,7 +1545,7 @@ dummy_func( DEOPT_IF(dict == NULL, LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(names, oparg>>1); - uint16_t hint = cache->index; + uint16_t hint = index; DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; @@ -1567,73 +1560,49 @@ dummy_func( DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); - SET_TOP(NULL); - STACK_GROW((oparg & 1)); - SET_TOP(res); + res2 = NULL; Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); } - // error: LOAD_ATTR has irregular stack effect - inst(LOAD_ATTR_SLOT) { + inst(LOAD_ATTR_SLOT, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); - PyObject *res; PyTypeObject *tp = Py_TYPE(owner); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - uint32_t type_version = read_u32(cache->version); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - char *addr = (char *)owner + cache->index; + char *addr = (char *)owner + index; res = *(PyObject **)addr; DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); - SET_TOP(NULL); - STACK_GROW((oparg & 1)); - SET_TOP(res); + res2 = NULL; Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); } - // error: LOAD_ATTR has irregular stack effect - inst(LOAD_ATTR_CLASS) { + inst(LOAD_ATTR_CLASS, (unused/1, type_version/2, unused/2, descr/4, cls -- res2 if (oparg & 1), res)) { assert(cframe.use_tracing == 0); - _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; - PyObject *cls = TOP(); DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); - uint32_t type_version = read_u32(cache->type_version); DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); STAT_INC(LOAD_ATTR, hit); - PyObject *res = read_obj(cache->descr); + res2 = NULL; + res = descr; assert(res != NULL); Py_INCREF(res); - SET_TOP(NULL); - STACK_GROW((oparg & 1)); - SET_TOP(res); Py_DECREF(cls); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); } - // error: LOAD_ATTR has irregular stack effect - inst(LOAD_ATTR_PROPERTY) { + inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused if (oparg & 1), unused)) { assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; - PyObject *owner = TOP(); PyTypeObject *cls = Py_TYPE(owner); - uint32_t type_version = read_u32(cache->type_version); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); - PyObject *fget = read_obj(cache->descr); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; - uint32_t func_version = read_u32(cache->keys_version); assert(func_version != 0); DEOPT_IF(f->func_version != func_version, LOAD_ATTR); PyCodeObject *code = (PyCodeObject *)f->func_code; @@ -1642,6 +1611,7 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); Py_INCREF(fget); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). SET_TOP(NULL); int shrink_stack = !(oparg & 1); STACK_SHRINK(shrink_stack); @@ -1650,20 +1620,14 @@ dummy_func( DISPATCH_INLINED(new_frame); } - // error: LOAD_ATTR has irregular stack effect - inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused if (oparg & 1), unused)) { assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; - PyObject *owner = TOP(); PyTypeObject *cls = Py_TYPE(owner); - uint32_t type_version = read_u32(cache->type_version); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); - PyObject *getattribute = read_obj(cache->descr); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; - uint32_t func_version = read_u32(cache->keys_version); assert(func_version != 0); DEOPT_IF(f->func_version != func_version, LOAD_ATTR); PyCodeObject *code = (PyCodeObject *)f->func_code; @@ -1674,6 +1638,7 @@ dummy_func( PyObject *name = GETITEM(names, oparg >> 1); Py_INCREF(f); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). SET_TOP(NULL); int shrink_stack = !(oparg & 1); STACK_SHRINK(shrink_stack); @@ -1768,6 +1733,7 @@ dummy_func( ERROR_IF(res == NULL, error); } + // No cache size here, since this is a family of super-instructions. family(compare_and_branch) = { COMPARE_AND_BRANCH, COMPARE_AND_BRANCH_FLOAT, @@ -2373,14 +2339,10 @@ dummy_func( } - // error: LOAD_ATTR has irregular stack effect - inst(LOAD_ATTR_METHOD_WITH_VALUES) { + inst(LOAD_ATTR_METHOD_WITH_VALUES, (unused/1, type_version/2, keys_version/2, descr/4, self -- res2 if (oparg & 1), res)) { /* Cached method object */ assert(cframe.use_tracing == 0); - PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); - _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; - uint32_t type_version = read_u32(cache->type_version); assert(type_version != 0); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); @@ -2388,41 +2350,31 @@ dummy_func( DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); PyHeapTypeObject *self_heap_type = (PyHeapTypeObject *)self_cls; DEOPT_IF(self_heap_type->ht_cached_keys->dk_version != - read_u32(cache->keys_version), LOAD_ATTR); + keys_version, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - PyObject *res = read_obj(cache->descr); - assert(res != NULL); - assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); - SET_TOP(Py_NewRef(res)); - PUSH(self); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + assert(descr != NULL); + res2 = Py_NewRef(descr); + assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); + res = self; + assert(oparg & 1); } - // error: LOAD_ATTR has irregular stack effect - inst(LOAD_ATTR_METHOD_NO_DICT) { + inst(LOAD_ATTR_METHOD_NO_DICT, (unused/1, type_version/2, unused/2, descr/4, self -- res2 if (oparg & 1), res)) { assert(cframe.use_tracing == 0); - PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); - _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; - uint32_t type_version = read_u32(cache->type_version); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); - PyObject *res = read_obj(cache->descr); - assert(res != NULL); - assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); - SET_TOP(Py_NewRef(res)); - PUSH(self); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + res2 = Py_NewRef(descr); + res = self; + assert(oparg & 1); } - // error: LOAD_ATTR has irregular stack effect - inst(LOAD_ATTR_METHOD_LAZY_DICT) { + inst(LOAD_ATTR_METHOD_LAZY_DICT, (unused/1, type_version/2, unused/2, descr/4, self -- res2 if (oparg & 1), res)) { assert(cframe.use_tracing == 0); - PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); - _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; - uint32_t type_version = read_u32(cache->type_version); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; assert(dictoffset > 0); @@ -2430,12 +2382,11 @@ dummy_func( /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - PyObject *res = read_obj(cache->descr); - assert(res != NULL); - assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); - SET_TOP(Py_NewRef(res)); - PUSH(self); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + res2 = Py_NewRef(descr); + res = self; + assert(oparg & 1); } // stack effect: (__0, __array[oparg] -- ) @@ -3265,7 +3216,7 @@ dummy_func( // Future families go below this point // -family(call) = { +family(call, INLINE_CACHE_ENTRIES_CALL) = { CALL, CALL_PY_EXACT_ARGS, CALL_PY_WITH_DEFAULTS, CALL_BOUND_METHOD_EXACT_ARGS, CALL_BUILTIN_CLASS, CALL_BUILTIN_FAST_WITH_KEYWORDS, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, CALL_NO_KW_BUILTIN_FAST, @@ -3273,19 +3224,13 @@ family(call) = { CALL_NO_KW_LIST_APPEND, CALL_NO_KW_METHOD_DESCRIPTOR_FAST, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, CALL_NO_KW_METHOD_DESCRIPTOR_O, CALL_NO_KW_STR_1, CALL_NO_KW_TUPLE_1, CALL_NO_KW_TYPE_1 }; -family(for_iter) = { +family(for_iter, INLINE_CACHE_ENTRIES_FOR_ITER) = { FOR_ITER, FOR_ITER_LIST, FOR_ITER_RANGE }; -family(load_attr) = { - LOAD_ATTR, LOAD_ATTR_CLASS, - LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_MODULE, - LOAD_ATTR_PROPERTY, LOAD_ATTR_SLOT, LOAD_ATTR_WITH_HINT, - LOAD_ATTR_METHOD_LAZY_DICT, LOAD_ATTR_METHOD_NO_DICT, - LOAD_ATTR_METHOD_WITH_VALUES }; -family(load_global) = { +family(load_global, INLINE_CACHE_ENTRIES_LOAD_GLOBAL) = { LOAD_GLOBAL, LOAD_GLOBAL_BUILTIN, LOAD_GLOBAL_MODULE }; family(store_fast) = { STORE_FAST, STORE_FAST__LOAD_FAST, STORE_FAST__STORE_FAST }; -family(unpack_sequence) = { +family(unpack_sequence, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE) = { UNPACK_SEQUENCE, UNPACK_SEQUENCE_LIST, UNPACK_SEQUENCE_TUPLE, UNPACK_SEQUENCE_TWO_TUPLE }; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d70d64ecbdd..e5c5c7e557a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -596,6 +596,7 @@ TARGET(STORE_SUBSCR) { PREDICTED(STORE_SUBSCR); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); PyObject *sub = PEEK(1); PyObject *container = PEEK(2); PyObject *v = PEEK(3); @@ -1180,6 +1181,7 @@ TARGET(STORE_ATTR) { PREDICTED(STORE_ATTR); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner = PEEK(1); PyObject *v = PEEK(2); uint16_t counter = read_u16(&next_instr[0].cache); @@ -1748,6 +1750,7 @@ TARGET(LOAD_ATTR) { PREDICTED(LOAD_ATTR); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner = PEEK(1); PyObject *res2 = NULL; PyObject *res; @@ -1805,62 +1808,67 @@ } TARGET(LOAD_ATTR_INSTANCE_VALUE) { - assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); + PyObject *owner = PEEK(1); + PyObject *res2 = NULL; PyObject *res; + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t index = read_u16(&next_instr[3].cache); + assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - uint32_t type_version = read_u32(cache->version); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); assert(tp->tp_dictoffset < 0); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - res = _PyDictOrValues_GetValues(dorv)->values[cache->index]; + res = _PyDictOrValues_GetValues(dorv)->values[index]; DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); - SET_TOP(NULL); - STACK_GROW((oparg & 1)); - SET_TOP(res); + res2 = NULL; Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + STACK_GROW(((oparg & 1) ? 1 : 0)); + POKE(1, res); + if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), res2); } + JUMPBY(9); DISPATCH(); } TARGET(LOAD_ATTR_MODULE) { - assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); + PyObject *owner = PEEK(1); + PyObject *res2 = NULL; PyObject *res; - _PyAttrCache *cache = (_PyAttrCache *)next_instr; + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t index = read_u16(&next_instr[3].cache); + assert(cframe.use_tracing == 0); DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != read_u32(cache->version), - LOAD_ATTR); + DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(cache->index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + cache->index; + assert(index < dict->ma_keys->dk_nentries); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; res = ep->me_value; DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); - SET_TOP(NULL); - STACK_GROW((oparg & 1)); - SET_TOP(res); + res2 = NULL; Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + STACK_GROW(((oparg & 1) ? 1 : 0)); + POKE(1, res); + if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), res2); } + JUMPBY(9); DISPATCH(); } TARGET(LOAD_ATTR_WITH_HINT) { - assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); + PyObject *owner = PEEK(1); + PyObject *res2 = NULL; PyObject *res; + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t index = read_u16(&next_instr[3].cache); + assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - uint32_t type_version = read_u32(cache->version); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); @@ -1870,7 +1878,7 @@ DEOPT_IF(dict == NULL, LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(names, oparg>>1); - uint16_t hint = cache->index; + uint16_t hint = index; DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; @@ -1885,73 +1893,78 @@ DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); - SET_TOP(NULL); - STACK_GROW((oparg & 1)); - SET_TOP(res); + res2 = NULL; Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + STACK_GROW(((oparg & 1) ? 1 : 0)); + POKE(1, res); + if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), res2); } + JUMPBY(9); DISPATCH(); } TARGET(LOAD_ATTR_SLOT) { - assert(cframe.use_tracing == 0); - PyObject *owner = TOP(); + PyObject *owner = PEEK(1); + PyObject *res2 = NULL; PyObject *res; + uint32_t type_version = read_u32(&next_instr[1].cache); + uint16_t index = read_u16(&next_instr[3].cache); + assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - uint32_t type_version = read_u32(cache->version); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - char *addr = (char *)owner + cache->index; + char *addr = (char *)owner + index; res = *(PyObject **)addr; DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); - SET_TOP(NULL); - STACK_GROW((oparg & 1)); - SET_TOP(res); + res2 = NULL; Py_DECREF(owner); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + STACK_GROW(((oparg & 1) ? 1 : 0)); + POKE(1, res); + if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), res2); } + JUMPBY(9); DISPATCH(); } TARGET(LOAD_ATTR_CLASS) { + PyObject *cls = PEEK(1); + PyObject *res2 = NULL; + PyObject *res; + uint32_t type_version = read_u32(&next_instr[1].cache); + PyObject *descr = read_obj(&next_instr[5].cache); assert(cframe.use_tracing == 0); - _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; - PyObject *cls = TOP(); DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); - uint32_t type_version = read_u32(cache->type_version); DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); STAT_INC(LOAD_ATTR, hit); - PyObject *res = read_obj(cache->descr); + res2 = NULL; + res = descr; assert(res != NULL); Py_INCREF(res); - SET_TOP(NULL); - STACK_GROW((oparg & 1)); - SET_TOP(res); Py_DECREF(cls); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + STACK_GROW(((oparg & 1) ? 1 : 0)); + POKE(1, res); + if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), res2); } + JUMPBY(9); DISPATCH(); } TARGET(LOAD_ATTR_PROPERTY) { + PyObject *owner = PEEK(1); + uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t func_version = read_u32(&next_instr[3].cache); + PyObject *fget = read_obj(&next_instr[5].cache); assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; - PyObject *owner = TOP(); PyTypeObject *cls = Py_TYPE(owner); - uint32_t type_version = read_u32(cache->type_version); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); - PyObject *fget = read_obj(cache->descr); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; - uint32_t func_version = read_u32(cache->keys_version); assert(func_version != 0); DEOPT_IF(f->func_version != func_version, LOAD_ATTR); PyCodeObject *code = (PyCodeObject *)f->func_code; @@ -1960,6 +1973,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(fget); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). SET_TOP(NULL); int shrink_stack = !(oparg & 1); STACK_SHRINK(shrink_stack); @@ -1969,18 +1983,17 @@ } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + PyObject *owner = PEEK(1); + uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t func_version = read_u32(&next_instr[3].cache); + PyObject *getattribute = read_obj(&next_instr[5].cache); assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; - PyObject *owner = TOP(); PyTypeObject *cls = Py_TYPE(owner); - uint32_t type_version = read_u32(cache->type_version); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); - PyObject *getattribute = read_obj(cache->descr); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; - uint32_t func_version = read_u32(cache->keys_version); assert(func_version != 0); DEOPT_IF(f->func_version != func_version, LOAD_ATTR); PyCodeObject *code = (PyCodeObject *)f->func_code; @@ -1991,6 +2004,7 @@ PyObject *name = GETITEM(names, oparg >> 1); Py_INCREF(f); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). SET_TOP(NULL); int shrink_stack = !(oparg & 1); STACK_SHRINK(shrink_stack); @@ -2831,12 +2845,15 @@ } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + PyObject *self = PEEK(1); + PyObject *res2 = NULL; + PyObject *res; + uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t keys_version = read_u32(&next_instr[3].cache); + PyObject *descr = read_obj(&next_instr[5].cache); /* Cached method object */ assert(cframe.use_tracing == 0); - PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); - _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; - uint32_t type_version = read_u32(cache->type_version); assert(type_version != 0); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); @@ -2844,41 +2861,51 @@ DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); PyHeapTypeObject *self_heap_type = (PyHeapTypeObject *)self_cls; DEOPT_IF(self_heap_type->ht_cached_keys->dk_version != - read_u32(cache->keys_version), LOAD_ATTR); + keys_version, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - PyObject *res = read_obj(cache->descr); - assert(res != NULL); - assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); - SET_TOP(Py_NewRef(res)); - PUSH(self); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + assert(descr != NULL); + res2 = Py_NewRef(descr); + assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); + res = self; + assert(oparg & 1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + POKE(1, res); + if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), res2); } + JUMPBY(9); DISPATCH(); } TARGET(LOAD_ATTR_METHOD_NO_DICT) { + PyObject *self = PEEK(1); + PyObject *res2 = NULL; + PyObject *res; + uint32_t type_version = read_u32(&next_instr[1].cache); + PyObject *descr = read_obj(&next_instr[5].cache); assert(cframe.use_tracing == 0); - PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); - _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; - uint32_t type_version = read_u32(cache->type_version); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); - PyObject *res = read_obj(cache->descr); - assert(res != NULL); - assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); - SET_TOP(Py_NewRef(res)); - PUSH(self); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + res2 = Py_NewRef(descr); + res = self; + assert(oparg & 1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + POKE(1, res); + if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), res2); } + JUMPBY(9); DISPATCH(); } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + PyObject *self = PEEK(1); + PyObject *res2 = NULL; + PyObject *res; + uint32_t type_version = read_u32(&next_instr[1].cache); + PyObject *descr = read_obj(&next_instr[5].cache); assert(cframe.use_tracing == 0); - PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); - _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; - uint32_t type_version = read_u32(cache->type_version); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; assert(dictoffset > 0); @@ -2886,12 +2913,15 @@ /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - PyObject *res = read_obj(cache->descr); - assert(res != NULL); - assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); - SET_TOP(Py_NewRef(res)); - PUSH(self); - JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + res2 = Py_NewRef(descr); + res = self; + assert(oparg & 1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + POKE(1, res); + if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), res2); } + JUMPBY(9); DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 171fed363d3..96e57be8cf5 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -187,19 +187,19 @@ _PyOpcode_num_popped(int opcode, int oparg) { case LOAD_ATTR: return 1; case LOAD_ATTR_INSTANCE_VALUE: - return -1; + return 1; case LOAD_ATTR_MODULE: - return -1; + return 1; case LOAD_ATTR_WITH_HINT: - return -1; + return 1; case LOAD_ATTR_SLOT: - return -1; + return 1; case LOAD_ATTR_CLASS: - return -1; + return 1; case LOAD_ATTR_PROPERTY: - return -1; + return 1; case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: - return -1; + return 1; case STORE_ATTR_INSTANCE_VALUE: return 2; case STORE_ATTR_WITH_HINT: @@ -279,11 +279,11 @@ _PyOpcode_num_popped(int opcode, int oparg) { case PUSH_EXC_INFO: return -1; case LOAD_ATTR_METHOD_WITH_VALUES: - return -1; + return 1; case LOAD_ATTR_METHOD_NO_DICT: - return -1; + return 1; case LOAD_ATTR_METHOD_LAZY_DICT: - return -1; + return 1; case CALL_BOUND_METHOD_EXACT_ARGS: return -1; case KW_NAMES: @@ -533,19 +533,19 @@ _PyOpcode_num_pushed(int opcode, int oparg) { case LOAD_ATTR: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_INSTANCE_VALUE: - return -1; + return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_MODULE: - return -1; + return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_WITH_HINT: - return -1; + return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_SLOT: - return -1; + return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_CLASS: - return -1; + return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_PROPERTY: - return -1; + return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: - return -1; + return ((oparg & 1) ? 1 : 0) + 1; case STORE_ATTR_INSTANCE_VALUE: return 0; case STORE_ATTR_WITH_HINT: @@ -625,11 +625,11 @@ _PyOpcode_num_pushed(int opcode, int oparg) { case PUSH_EXC_INFO: return -1; case LOAD_ATTR_METHOD_WITH_VALUES: - return -1; + return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_METHOD_NO_DICT: - return -1; + return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_METHOD_LAZY_DICT: - return -1; + return ((oparg & 1) ? 1 : 0) + 1; case CALL_BOUND_METHOD_EXACT_ARGS: return -1; case KW_NAMES: @@ -792,13 +792,13 @@ struct opcode_metadata { [DICT_MERGE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [MAP_ADD] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [LOAD_ATTR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_INSTANCE_VALUE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_ATTR_MODULE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_ATTR_WITH_HINT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_ATTR_SLOT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_ATTR_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_ATTR_PROPERTY] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [LOAD_ATTR_INSTANCE_VALUE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR_MODULE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR_WITH_HINT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR_SLOT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR_PROPERTY] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, [STORE_ATTR_INSTANCE_VALUE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC000 }, [STORE_ATTR_WITH_HINT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [STORE_ATTR_SLOT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC000 }, @@ -838,9 +838,9 @@ struct opcode_metadata { [BEFORE_WITH] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [WITH_EXCEPT_START] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [PUSH_EXC_INFO] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [LOAD_ATTR_METHOD_NO_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR_METHOD_NO_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, [CALL_BOUND_METHOD_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [KW_NAMES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },