From 8319114feedd2a5b77378bba24eb9fb2689c5033 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 7 Dec 2021 16:02:53 +0000 Subject: [PATCH] bpo-45947: Place dict and values pointer at fixed (negative) offset just before GC header. (GH-29879) * Place __dict__ immediately before GC header for plain Python objects. * Fix up lazy dict creation logic to use managed dict pointers. * Manage values pointer, placing them directly before managed dict pointers. * Convert hint-based load/store attr specialization target managed dict classes. * Specialize LOAD_METHOD for managed dict objects. * Remove unsafe _PyObject_GC_Calloc function. * Remove unsafe _PyObject_GC_Malloc() function. * Add comment explaning use of Py_TPFLAGS_MANAGED_DICT. --- Include/cpython/object.h | 1 - Include/cpython/objimpl.h | 3 - Include/internal/pycore_object.h | 23 +++- Include/object.h | 6 + Lib/test/test_stable_abi_ctypes.py | 1 - Lib/test/test_sys.py | 4 +- .../2021-12-01-14-06-36.bpo-45947.1XPPm_.rst | 3 + Misc/stable_abi.txt | 3 - Modules/_testcapimodule.c | 17 ++- Modules/gcmodule.c | 68 +++++------ Objects/dictobject.c | 86 ++++++++----- Objects/exceptions.c | 5 +- Objects/object.c | 71 ++++------- Objects/typeobject.c | 113 ++++++++---------- PC/python3dll.c | 1 - Python/ceval.c | 24 ++-- Python/specialize.c | 100 ++++++---------- Python/sysmodule.c | 5 +- Tools/gdb/libpython.py | 11 +- 19 files changed, 260 insertions(+), 285 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-12-01-14-06-36.bpo-45947.1XPPm_.rst diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 3a8a256e3b9..0c3957aff4b 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -270,7 +270,6 @@ struct _typeobject { destructor tp_finalize; vectorcallfunc tp_vectorcall; - Py_ssize_t tp_inline_values_offset; }; /* The *real* layout of a type object when allocated on the heap */ diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h index d83700e2a46..4a905c25cc8 100644 --- a/Include/cpython/objimpl.h +++ b/Include/cpython/objimpl.h @@ -90,9 +90,6 @@ PyAPI_FUNC(int) PyObject_IS_GC(PyObject *obj); # define _PyGC_FINALIZED(o) PyObject_GC_IsFinalized(o) #endif -PyAPI_FUNC(PyObject *) _PyObject_GC_Malloc(size_t size); -PyAPI_FUNC(PyObject *) _PyObject_GC_Calloc(size_t size); - /* Test if a type supports weak references */ #define PyType_SUPPORTS_WEAKREFS(t) ((t)->tp_weaklistoffset > 0) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 3c126aaef11..9041a4dc8a3 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -168,6 +168,15 @@ _PyObject_IS_GC(PyObject *obj) // Fast inlined version of PyType_IS_GC() #define _PyType_IS_GC(t) _PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) +static inline size_t +_PyType_PreHeaderSize(PyTypeObject *tp) +{ + return _PyType_IS_GC(tp) * sizeof(PyGC_Head) + + _PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT) * 2 * sizeof(PyObject *); +} + +void _PyObject_GC_Link(PyObject *op); + // Usage: assert(_Py_CheckSlotResult(obj, "__getitem__", result != NULL)); extern int _Py_CheckSlotResult( PyObject *obj, @@ -185,7 +194,19 @@ extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value); PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name); -PyDictValues ** _PyObject_ValuesPointer(PyObject *); + +static inline PyDictValues **_PyObject_ValuesPointer(PyObject *obj) +{ + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + return ((PyDictValues **)obj)-4; +} + +static inline PyObject **_PyObject_ManagedDictPointer(PyObject *obj) +{ + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + return ((PyObject **)obj)-3; +} + PyObject ** _PyObject_DictPointer(PyObject *); int _PyObject_VisitInstanceAttributes(PyObject *self, visitproc visit, void *arg); void _PyObject_ClearInstanceAttributes(PyObject *self); diff --git a/Include/object.h b/Include/object.h index 33df303a44e..e5544e8b588 100644 --- a/Include/object.h +++ b/Include/object.h @@ -334,6 +334,12 @@ given type object has a specified feature. #ifndef Py_LIMITED_API +/* Placement of dict (and values) pointers are managed by the VM, not by the type. + * The VM will automatically set tp_dictoffset. Should not be used for variable sized + * classes, such as classes that extend tuple. + */ +#define Py_TPFLAGS_MANAGED_DICT (1 << 4) + /* Set if instances of the type object are treated as sequences for pattern matching */ #define Py_TPFLAGS_SEQUENCE (1 << 5) /* Set if instances of the type object are treated as mappings for pattern matching */ diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 1e27bcaf889..d0cd5c20dd5 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -817,7 +817,6 @@ SYMBOL_NAMES = ( "_PyErr_BadInternalCall", "_PyObject_CallFunction_SizeT", "_PyObject_CallMethod_SizeT", - "_PyObject_GC_Malloc", "_PyObject_GC_New", "_PyObject_GC_NewVar", "_PyObject_GC_Resize", diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 81802893a8e..41ac03b920e 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1421,8 +1421,8 @@ class SizeofTest(unittest.TestCase): check((1,2,3), vsize('') + 3*self.P) # type # static type: PyTypeObject - fmt = 'P2nPI13Pl4Pn9Pn12PIPP' - s = vsize(fmt) + fmt = 'P2nPI13Pl4Pn9Pn12PIP' + s = vsize('2P' + fmt) check(int, s) # class s = vsize(fmt + # PyTypeObject diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-12-01-14-06-36.bpo-45947.1XPPm_.rst b/Misc/NEWS.d/next/Core and Builtins/2021-12-01-14-06-36.bpo-45947.1XPPm_.rst new file mode 100644 index 00000000000..5156bd35369 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-12-01-14-06-36.bpo-45947.1XPPm_.rst @@ -0,0 +1,3 @@ +Place pointers to dict and values immediately before GC header. This reduces +number of dependent memory loads to access either dict or values from 3 to +1. diff --git a/Misc/stable_abi.txt b/Misc/stable_abi.txt index 9f5a85bdec4..de6caa8c807 100644 --- a/Misc/stable_abi.txt +++ b/Misc/stable_abi.txt @@ -1577,9 +1577,6 @@ function _PyObject_CallFunction_SizeT function _PyObject_CallMethod_SizeT added 3.2 abi_only -function _PyObject_GC_Malloc - added 3.2 - abi_only function _PyObject_GC_New added 3.2 abi_only diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 0216c985415..56d394985eb 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5861,6 +5861,7 @@ test_tstate_capi(PyObject *self, PyObject *Py_UNUSED(args)) } +static PyObject *negative_dictoffset(PyObject *, PyObject *); static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); static PyObject *getargs_s_hash_int(PyObject *, PyObject *, PyObject*); @@ -5929,14 +5930,15 @@ static PyMethodDef TestMethods[] = { #if (defined(__linux__) || defined(__FreeBSD__)) && defined(__GNUC__) {"test_pep3118_obsolete_write_locks", (PyCFunction)test_pep3118_obsolete_write_locks, METH_NOARGS}, #endif - {"getbuffer_with_null_view", getbuffer_with_null_view, METH_O}, - {"PyBuffer_SizeFromFormat", test_PyBuffer_SizeFromFormat, METH_VARARGS}, - {"test_buildvalue_N", test_buildvalue_N, METH_NOARGS}, + {"getbuffer_with_null_view", getbuffer_with_null_view, METH_O}, + {"PyBuffer_SizeFromFormat", test_PyBuffer_SizeFromFormat, METH_VARARGS}, + {"test_buildvalue_N", test_buildvalue_N, METH_NOARGS}, + {"negative_dictoffset", negative_dictoffset, METH_NOARGS}, {"test_buildvalue_issue38913", test_buildvalue_issue38913, METH_NOARGS}, - {"get_args", get_args, METH_VARARGS}, + {"get_args", get_args, METH_VARARGS}, {"test_get_statictype_slots", test_get_statictype_slots, METH_NOARGS}, {"test_get_type_name", test_get_type_name, METH_NOARGS}, - {"test_get_type_qualname", test_get_type_qualname, METH_NOARGS}, + {"test_get_type_qualname", test_get_type_qualname, METH_NOARGS}, {"test_type_from_ephemeral_spec", test_type_from_ephemeral_spec, METH_NOARGS}, {"get_kwargs", (PyCFunction)(void(*)(void))get_kwargs, METH_VARARGS|METH_KEYWORDS}, @@ -7629,6 +7631,11 @@ PyInit__testcapi(void) return m; } +static PyObject * +negative_dictoffset(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyType_FromSpec(&HeapCTypeWithNegativeDict_spec); +} /* Test the C API exposed when PY_SSIZE_T_CLEAN is not defined */ diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index b505676636d..1808057a650 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -69,10 +69,10 @@ module gc #define NEXT_MASK_UNREACHABLE (1) /* Get an object's GC head */ -#define AS_GC(o) ((PyGC_Head *)(o)-1) +#define AS_GC(o) ((PyGC_Head *)(((char *)(o))-sizeof(PyGC_Head))) /* Get the object given the GC head */ -#define FROM_GC(g) ((PyObject *)(((PyGC_Head *)g)+1)) +#define FROM_GC(g) ((PyObject *)(((char *)(g))+sizeof(PyGC_Head))) static inline int gc_is_collecting(PyGC_Head *g) @@ -2231,28 +2231,14 @@ PyObject_IS_GC(PyObject *obj) return _PyObject_IS_GC(obj); } -static PyObject * -_PyObject_GC_Alloc(int use_calloc, size_t basicsize) +void +_PyObject_GC_Link(PyObject *op) { + PyGC_Head *g = AS_GC(op); + assert(((uintptr_t)g & (sizeof(uintptr_t)-1)) == 0); // g must be correctly aligned + PyThreadState *tstate = _PyThreadState_GET(); GCState *gcstate = &tstate->interp->gc; - if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head)) { - return _PyErr_NoMemory(tstate); - } - size_t size = sizeof(PyGC_Head) + basicsize; - - PyGC_Head *g; - if (use_calloc) { - g = (PyGC_Head *)PyObject_Calloc(1, size); - } - else { - g = (PyGC_Head *)PyObject_Malloc(size); - } - if (g == NULL) { - return _PyErr_NoMemory(tstate); - } - assert(((uintptr_t)g & 3) == 0); // g must be aligned 4bytes boundary - g->_gc_next = 0; g->_gc_prev = 0; gcstate->generations[0].count++; /* number of allocated GC objects */ @@ -2266,26 +2252,32 @@ _PyObject_GC_Alloc(int use_calloc, size_t basicsize) gc_collect_generations(tstate); gcstate->collecting = 0; } - PyObject *op = FROM_GC(g); +} + +static PyObject * +gc_alloc(size_t basicsize, size_t presize) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (basicsize > PY_SSIZE_T_MAX - presize) { + return _PyErr_NoMemory(tstate); + } + size_t size = presize + basicsize; + char *mem = PyObject_Malloc(size); + if (mem == NULL) { + return _PyErr_NoMemory(tstate); + } + ((PyObject **)mem)[0] = NULL; + ((PyObject **)mem)[1] = NULL; + PyObject *op = (PyObject *)(mem + presize); + _PyObject_GC_Link(op); return op; } -PyObject * -_PyObject_GC_Malloc(size_t basicsize) -{ - return _PyObject_GC_Alloc(0, basicsize); -} - -PyObject * -_PyObject_GC_Calloc(size_t basicsize) -{ - return _PyObject_GC_Alloc(1, basicsize); -} - PyObject * _PyObject_GC_New(PyTypeObject *tp) { - PyObject *op = _PyObject_GC_Malloc(_PyObject_SIZE(tp)); + size_t presize = _PyType_PreHeaderSize(tp); + PyObject *op = gc_alloc(_PyObject_SIZE(tp), presize); if (op == NULL) { return NULL; } @@ -2303,8 +2295,9 @@ _PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems) PyErr_BadInternalCall(); return NULL; } + size_t presize = _PyType_PreHeaderSize(tp); size = _PyObject_VAR_SIZE(tp, nitems); - op = (PyVarObject *) _PyObject_GC_Malloc(size); + op = (PyVarObject *)gc_alloc(size, presize); if (op == NULL) { return NULL; } @@ -2333,6 +2326,7 @@ _PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) void PyObject_GC_Del(void *op) { + size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); PyGC_Head *g = AS_GC(op); if (_PyObject_GC_IS_TRACKED(op)) { gc_list_remove(g); @@ -2341,7 +2335,7 @@ PyObject_GC_Del(void *op) if (gcstate->generations[0].count > 0) { gcstate->generations[0].count--; } - PyObject_Free(g); + PyObject_Free(((char *)op)-presize); } int diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 475d92d3298..7ce4b9069f7 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4961,8 +4961,8 @@ static int init_inline_values(PyObject *obj, PyTypeObject *tp) { assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); - assert(tp->tp_dictoffset > 0); - assert(tp->tp_inline_values_offset > 0); + // assert(type->tp_dictoffset > 0); -- TO DO Update this assert. + assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictKeysObject *keys = CACHED_KEYS(tp); assert(keys != NULL); if (keys->dk_usable > 1) { @@ -4979,7 +4979,7 @@ init_inline_values(PyObject *obj, PyTypeObject *tp) for (int i = 0; i < size; i++) { values->values[i] = NULL; } - *((PyDictValues **)((char *)obj + tp->tp_inline_values_offset)) = values; + *_PyObject_ValuesPointer(obj) = values; return 0; } @@ -4990,7 +4990,7 @@ _PyObject_InitializeDict(PyObject *obj) if (tp->tp_dictoffset == 0) { return 0; } - if (tp->tp_inline_values_offset) { + if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { return init_inline_values(obj, tp); } PyObject *dict; @@ -5032,7 +5032,7 @@ make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values) PyObject * _PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) { - assert(Py_TYPE(obj)->tp_inline_values_offset != 0); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); return make_dict_from_instance_attributes(keys, values); } @@ -5042,10 +5042,10 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value) { assert(PyUnicode_CheckExact(name)); - PyTypeObject *tp = Py_TYPE(obj); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); assert(keys != NULL); assert(values != NULL); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); int ix = insert_into_dictkeys(keys, name); if (ix == DKIX_EMPTY) { if (value == NULL) { @@ -5056,8 +5056,8 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, if (dict == NULL) { return -1; } - *((PyDictValues **)((char *)obj + tp->tp_inline_values_offset)) = NULL; - *((PyObject **) ((char *)obj + tp->tp_dictoffset)) = dict; + *_PyObject_ValuesPointer(obj) = NULL; + *_PyObject_ManagedDictPointer(obj) = dict; return PyDict_SetItem(dict, name, value); } PyObject *old_value = values->values[ix]; @@ -5102,17 +5102,23 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj) if (tp->tp_dictoffset == 0) { return 1; } - PyDictValues **values_ptr = _PyObject_ValuesPointer(obj); - if (values_ptr && *values_ptr) { - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - if ((*values_ptr)->values[i] != NULL) { - return 0; + PyObject **dictptr; + if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + PyDictValues *values = *_PyObject_ValuesPointer(obj); + if (values) { + PyDictKeysObject *keys = CACHED_KEYS(tp); + for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { + if (values->values[i] != NULL) { + return 0; + } } + return 1; } - return 1; + dictptr = _PyObject_ManagedDictPointer(obj); + } + else { + dictptr = _PyObject_DictPointer(obj); } - PyObject **dictptr = _PyObject_DictPointer(obj); PyObject *dict = *dictptr; if (dict == NULL) { return 1; @@ -5125,7 +5131,7 @@ int _PyObject_VisitInstanceAttributes(PyObject *self, visitproc visit, void *arg) { PyTypeObject *tp = Py_TYPE(self); - assert(tp->tp_inline_values_offset); + assert(Py_TYPE(self)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictValues **values_ptr = _PyObject_ValuesPointer(self); if (*values_ptr == NULL) { return 0; @@ -5141,7 +5147,7 @@ void _PyObject_ClearInstanceAttributes(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); - assert(tp->tp_inline_values_offset); + assert(Py_TYPE(self)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictValues **values_ptr = _PyObject_ValuesPointer(self); if (*values_ptr == NULL) { return; @@ -5156,7 +5162,7 @@ void _PyObject_FreeInstanceAttributes(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); - assert(tp->tp_inline_values_offset); + assert(Py_TYPE(self)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictValues **values_ptr = _PyObject_ValuesPointer(self); if (*values_ptr == NULL) { return; @@ -5171,28 +5177,42 @@ _PyObject_FreeInstanceAttributes(PyObject *self) PyObject * PyObject_GenericGetDict(PyObject *obj, void *context) { - PyObject **dictptr = _PyObject_DictPointer(obj); - if (dictptr == NULL) { - PyErr_SetString(PyExc_AttributeError, - "This object has no __dict__"); - return NULL; - } - PyObject *dict = *dictptr; - if (dict == NULL) { - PyTypeObject *tp = Py_TYPE(obj); + PyObject *dict; + PyTypeObject *tp = Py_TYPE(obj); + if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { PyDictValues **values_ptr = _PyObject_ValuesPointer(obj); - if (values_ptr && *values_ptr) { + PyObject **dictptr = _PyObject_ManagedDictPointer(obj); + if (*values_ptr) { + assert(*dictptr == NULL); *dictptr = dict = make_dict_from_instance_attributes(CACHED_KEYS(tp), *values_ptr); if (dict != NULL) { *values_ptr = NULL; } } - else if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { - dictkeys_incref(CACHED_KEYS(tp)); - *dictptr = dict = new_dict_with_shared_keys(CACHED_KEYS(tp)); + else if (*dictptr == NULL) { + *dictptr = dict = PyDict_New(); } else { - *dictptr = dict = PyDict_New(); + dict = *dictptr; + } + } + else { + PyObject **dictptr = _PyObject_DictPointer(obj); + if (dictptr == NULL) { + PyErr_SetString(PyExc_AttributeError, + "This object has no __dict__"); + return NULL; + } + dict = *dictptr; + if (dict == NULL) { + PyTypeObject *tp = Py_TYPE(obj); + if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { + dictkeys_incref(CACHED_KEYS(tp)); + *dictptr = dict = new_dict_with_shared_keys(CACHED_KEYS(tp)); + } + else { + *dictptr = dict = PyDict_New(); + } } } Py_XINCREF(dict); diff --git a/Objects/exceptions.c b/Objects/exceptions.c index c99f17a30f1..e1a8c1363ef 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3484,7 +3484,6 @@ _PyErr_TrySetFromCause(const char *format, ...) PyObject* msg_prefix; PyObject *exc, *val, *tb; PyTypeObject *caught_type; - PyObject **dictptr; PyObject *instance_args; Py_ssize_t num_args, caught_type_size, base_exc_size; PyObject *new_exc, *new_val, *new_tb; @@ -3530,9 +3529,7 @@ _PyErr_TrySetFromCause(const char *format, ...) } /* Ensure the instance dict is also empty */ - dictptr = _PyObject_GetDictPtr(val); - if (dictptr != NULL && *dictptr != NULL && - PyDict_GET_SIZE(*dictptr) > 0) { + if (!_PyObject_IsInstanceDictEmpty(val)) { /* While we could potentially copy a non-empty instance dictionary * to the replacement exception, for now we take the more * conservative path of leaving exceptions with attributes set diff --git a/Objects/object.c b/Objects/object.c index 25f5a2133d5..a1c2e16b6fa 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1073,6 +1073,9 @@ _PyObject_DictPointer(PyObject *obj) Py_ssize_t dictoffset; PyTypeObject *tp = Py_TYPE(obj); + if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + return _PyObject_ManagedDictPointer(obj); + } dictoffset = tp->tp_dictoffset; if (dictoffset == 0) return NULL; @@ -1096,24 +1099,20 @@ _PyObject_DictPointer(PyObject *obj) PyObject ** _PyObject_GetDictPtr(PyObject *obj) { - PyObject **dict_ptr = _PyObject_DictPointer(obj); - if (dict_ptr == NULL) { - return NULL; - } - if (*dict_ptr != NULL) { - return dict_ptr; + if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { + return _PyObject_DictPointer(obj); } + PyObject **dict_ptr = _PyObject_ManagedDictPointer(obj); PyDictValues **values_ptr = _PyObject_ValuesPointer(obj); - if (values_ptr == NULL || *values_ptr == NULL) { + if (*values_ptr == NULL) { return dict_ptr; } + assert(*dict_ptr == NULL); PyObject *dict = _PyObject_MakeDictFromInstanceAttributes(obj, *values_ptr); if (dict == NULL) { PyErr_Clear(); return NULL; } - assert(*dict_ptr == NULL); - assert(*values_ptr != NULL); *values_ptr = NULL; *dict_ptr = dict; return dict_ptr; @@ -1185,10 +1184,12 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) } } } - PyDictValues **values_ptr = _PyObject_ValuesPointer(obj); - if (values_ptr && *values_ptr) { + PyDictValues *values; + if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) && + (values = *_PyObject_ValuesPointer(obj))) + { assert(*_PyObject_DictPointer(obj) == NULL); - PyObject *attr = _PyObject_GetInstanceAttribute(obj, *values_ptr, name); + PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name); if (attr != NULL) { *method = attr; Py_XDECREF(descr); @@ -1240,17 +1241,6 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) return 0; } -PyDictValues ** -_PyObject_ValuesPointer(PyObject *obj) -{ - PyTypeObject *tp = Py_TYPE(obj); - Py_ssize_t offset = tp->tp_inline_values_offset; - if (offset == 0) { - return NULL; - } - return (PyDictValues **) ((char *)obj + offset); -} - /* Generic GetAttr functions - put these in your tp_[gs]etattro slot. */ PyObject * @@ -1267,7 +1257,6 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, PyObject *descr = NULL; PyObject *res = NULL; descrgetfunc f; - Py_ssize_t dictoffset; PyObject **dictptr; if (!PyUnicode_Check(name)){ @@ -1299,8 +1288,10 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, } } if (dict == NULL) { - PyDictValues **values_ptr = _PyObject_ValuesPointer(obj); - if (values_ptr && *values_ptr) { + if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) && + *_PyObject_ValuesPointer(obj)) + { + PyDictValues **values_ptr = _PyObject_ValuesPointer(obj); if (PyUnicode_CheckExact(name)) { assert(*_PyObject_DictPointer(obj) == NULL); res = _PyObject_GetInstanceAttribute(obj, *values_ptr, name); @@ -1320,22 +1311,8 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, } } else { - /* Inline _PyObject_DictPointer */ - dictoffset = tp->tp_dictoffset; - if (dictoffset != 0) { - if (dictoffset < 0) { - Py_ssize_t tsize = Py_SIZE(obj); - if (tsize < 0) { - tsize = -tsize; - } - size_t size = _PyObject_VAR_SIZE(tp, tsize); - _PyObject_ASSERT(obj, size <= PY_SSIZE_T_MAX); - - dictoffset += (Py_ssize_t)size; - _PyObject_ASSERT(obj, dictoffset > 0); - _PyObject_ASSERT(obj, dictoffset % SIZEOF_VOID_P == 0); - } - dictptr = (PyObject **) ((char *)obj + dictoffset); + dictptr = _PyObject_DictPointer(obj); + if (dictptr) { dict = *dictptr; } } @@ -1426,9 +1403,8 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, } if (dict == NULL) { - PyDictValues **values_ptr = _PyObject_ValuesPointer(obj); - if (values_ptr && *values_ptr) { - res = _PyObject_StoreInstanceAttribute(obj, *values_ptr, name, value); + if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) && *_PyObject_ValuesPointer(obj)) { + res = _PyObject_StoreInstanceAttribute(obj, *_PyObject_ValuesPointer(obj), name, value); } else { PyObject **dictptr = _PyObject_DictPointer(obj); @@ -1478,8 +1454,9 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) { PyObject **dictptr = _PyObject_GetDictPtr(obj); if (dictptr == NULL) { - PyDictValues** values_ptr = _PyObject_ValuesPointer(obj); - if (values_ptr != NULL && *values_ptr != NULL) { + if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_MANAGED_DICT) && + *_PyObject_ValuesPointer(obj) != NULL) + { /* Was unable to convert to dict */ PyErr_NoMemory(); } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 22e509be26f..2fd93b61c0b 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1146,17 +1146,17 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) const size_t size = _PyObject_VAR_SIZE(type, nitems+1); /* note that we need to add one, for the sentinel */ - if (_PyType_IS_GC(type)) { - obj = _PyObject_GC_Malloc(size); - } - else { - obj = (PyObject *)PyObject_Malloc(size); - } - - if (obj == NULL) { + const size_t presize = _PyType_PreHeaderSize(type); + char *alloc = PyObject_Malloc(size + presize); + if (alloc == NULL) { return PyErr_NoMemory(); } - + obj = (PyObject *)(alloc + presize); + if (presize) { + ((PyObject **)alloc)[0] = NULL; + ((PyObject **)alloc)[1] = NULL; + _PyObject_GC_Link(obj); + } memset(obj, '\0', size); if (type->tp_itemsize == 0) { @@ -1232,7 +1232,7 @@ subtype_traverse(PyObject *self, visitproc visit, void *arg) assert(base); } - if (type->tp_inline_values_offset) { + if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) { assert(type->tp_dictoffset); int err = _PyObject_VisitInstanceAttributes(self, visit, arg); if (err) { @@ -1301,7 +1301,7 @@ subtype_clear(PyObject *self) /* Clear the instance dict (if any), to break cycles involving only __dict__ slots (as in the case 'self.__dict__ is self'). */ - if (type->tp_inline_values_offset) { + if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) { _PyObject_ClearInstanceAttributes(self); } if (type->tp_dictoffset != base->tp_dictoffset) { @@ -1360,6 +1360,8 @@ subtype_dealloc(PyObject *self) int type_needs_decref = (type->tp_flags & Py_TPFLAGS_HEAPTYPE && !(base->tp_flags & Py_TPFLAGS_HEAPTYPE)); + assert((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0); + /* Call the base tp_dealloc() */ assert(basedealloc); basedealloc(self); @@ -1445,10 +1447,18 @@ subtype_dealloc(PyObject *self) } /* If we added a dict, DECREF it, or free inline values. */ - if (type->tp_inline_values_offset) { - _PyObject_FreeInstanceAttributes(self); + if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + PyObject **dictptr = _PyObject_ManagedDictPointer(self); + if (*dictptr != NULL) { + assert(*_PyObject_ValuesPointer(self) == NULL); + Py_DECREF(*dictptr); + *dictptr = NULL; + } + else { + _PyObject_FreeInstanceAttributes(self); + } } - if (type->tp_dictoffset && !base->tp_dictoffset) { + else if (type->tp_dictoffset && !base->tp_dictoffset) { PyObject **dictptr = _PyObject_DictPointer(self); if (dictptr != NULL) { PyObject *dict = *dictptr; @@ -2243,18 +2253,10 @@ extra_ivars(PyTypeObject *type, PyTypeObject *base) return t_size != b_size || type->tp_itemsize != base->tp_itemsize; } - if (type->tp_inline_values_offset && base->tp_inline_values_offset == 0 && - type->tp_inline_values_offset + sizeof(PyDictValues *) == t_size && - type->tp_flags & Py_TPFLAGS_HEAPTYPE) - t_size -= sizeof(PyDictValues *); if (type->tp_weaklistoffset && base->tp_weaklistoffset == 0 && type->tp_weaklistoffset + sizeof(PyObject *) == t_size && type->tp_flags & Py_TPFLAGS_HEAPTYPE) t_size -= sizeof(PyObject *); - if (type->tp_dictoffset && base->tp_dictoffset == 0 && - type->tp_dictoffset + sizeof(PyObject *) == t_size && - type->tp_flags & Py_TPFLAGS_HEAPTYPE) - t_size -= sizeof(PyObject *); return t_size != b_size; } @@ -2982,13 +2984,8 @@ type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type) } } - if (ctx->add_dict) { - if (ctx->base->tp_itemsize) { - type->tp_dictoffset = -(long)sizeof(PyObject *); - } - else { - type->tp_dictoffset = slotoffset; - } + if (ctx->add_dict && ctx->base->tp_itemsize) { + type->tp_dictoffset = -(long)sizeof(PyObject *); slotoffset += sizeof(PyObject *); } @@ -2997,12 +2994,10 @@ type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type) type->tp_weaklistoffset = slotoffset; slotoffset += sizeof(PyObject *); } - if (type->tp_dictoffset > 0) { - type->tp_inline_values_offset = slotoffset; - slotoffset += sizeof(PyDictValues *); - } - else { - type->tp_inline_values_offset = 0; + if (ctx->add_dict && ctx->base->tp_itemsize == 0) { + assert((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0); + type->tp_flags |= Py_TPFLAGS_MANAGED_DICT; + type->tp_dictoffset = -slotoffset - sizeof(PyObject *)*3; } type->tp_basicsize = slotoffset; @@ -3206,8 +3201,7 @@ type_new_impl(type_new_ctx *ctx) // Put the proper slots in place fixup_slot_dispatchers(type); - if (type->tp_inline_values_offset) { - assert(type->tp_dictoffset > 0); + if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) { PyHeapTypeObject *et = (PyHeapTypeObject*)type; et->ht_cached_keys = _PyDict_NewKeysForClass(); } @@ -3594,8 +3588,7 @@ PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) if (PyType_Ready(type) < 0) goto fail; - if (type->tp_inline_values_offset) { - assert(type->tp_dictoffset > 0); + if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) { res->ht_cached_keys = _PyDict_NewKeysForClass(); } @@ -4658,7 +4651,6 @@ compatible_with_tp_base(PyTypeObject *child) child->tp_itemsize == parent->tp_itemsize && child->tp_dictoffset == parent->tp_dictoffset && child->tp_weaklistoffset == parent->tp_weaklistoffset && - child->tp_inline_values_offset == parent->tp_inline_values_offset && ((child->tp_flags & Py_TPFLAGS_HAVE_GC) == (parent->tp_flags & Py_TPFLAGS_HAVE_GC)) && (child->tp_dealloc == subtype_dealloc || @@ -4678,8 +4670,6 @@ same_slots_added(PyTypeObject *a, PyTypeObject *b) size += sizeof(PyObject *); if (a->tp_weaklistoffset == size && b->tp_weaklistoffset == size) size += sizeof(PyObject *); - if (a->tp_inline_values_offset == size && b->tp_inline_values_offset == size) - size += sizeof(PyObject *); /* Check slots compliance */ if (!(a->tp_flags & Py_TPFLAGS_HEAPTYPE) || @@ -4729,16 +4719,22 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* if (newbase != oldbase && (newbase->tp_base != oldbase->tp_base || !same_slots_added(newbase, oldbase))) { - PyErr_Format(PyExc_TypeError, - "%s assignment: " - "'%s' object layout differs from '%s'", - attr, - newto->tp_name, - oldto->tp_name); - return 0; + goto differs; } - - return 1; + /* The above does not check for managed __dicts__ */ + if ((oldto->tp_flags & Py_TPFLAGS_MANAGED_DICT) == + ((newto->tp_flags & Py_TPFLAGS_MANAGED_DICT))) + { + return 1; + } +differs: + PyErr_Format(PyExc_TypeError, + "%s assignment: " + "'%s' object layout differs from '%s'", + attr, + newto->tp_name, + oldto->tp_name); + return 0; } static int @@ -4826,15 +4822,13 @@ object_set_class(PyObject *self, PyObject *value, void *closure) if (compatible_for_assignment(oldto, newto, "__class__")) { /* Changing the class will change the implicit dict keys, * so we must materialize the dictionary first. */ - assert(oldto->tp_inline_values_offset == newto->tp_inline_values_offset); + assert((oldto->tp_flags & Py_TPFLAGS_MANAGED_DICT) == (newto->tp_flags & Py_TPFLAGS_MANAGED_DICT)); _PyObject_GetDictPtr(self); - PyDictValues** values_ptr = _PyObject_ValuesPointer(self); - if (values_ptr != NULL && *values_ptr != NULL) { + if (oldto->tp_flags & Py_TPFLAGS_MANAGED_DICT && *_PyObject_ValuesPointer(self)) { /* Was unable to convert to dict */ PyErr_NoMemory(); return -1; } - assert(_PyObject_ValuesPointer(self) == NULL || *_PyObject_ValuesPointer(self) == NULL); if (newto->tp_flags & Py_TPFLAGS_HEAPTYPE) { Py_INCREF(newto); } @@ -4980,15 +4974,14 @@ _PyObject_GetState(PyObject *obj, int required) assert(slotnames == Py_None || PyList_Check(slotnames)); if (required) { Py_ssize_t basicsize = PyBaseObject_Type.tp_basicsize; - if (Py_TYPE(obj)->tp_dictoffset) { + if (Py_TYPE(obj)->tp_dictoffset && + (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) + { basicsize += sizeof(PyObject *); } if (Py_TYPE(obj)->tp_weaklistoffset) { basicsize += sizeof(PyObject *); } - if (Py_TYPE(obj)->tp_inline_values_offset) { - basicsize += sizeof(PyDictValues *); - } if (slotnames != Py_None) { basicsize += sizeof(PyObject *) * PyList_GET_SIZE(slotnames); } @@ -5749,6 +5742,7 @@ inherit_special(PyTypeObject *type, PyTypeObject *base) if (type->tp_clear == NULL) type->tp_clear = base->tp_clear; } + type->tp_flags |= (base->tp_flags & Py_TPFLAGS_MANAGED_DICT); if (type->tp_basicsize == 0) type->tp_basicsize = base->tp_basicsize; @@ -5761,7 +5755,6 @@ inherit_special(PyTypeObject *type, PyTypeObject *base) COPYVAL(tp_itemsize); COPYVAL(tp_weaklistoffset); COPYVAL(tp_dictoffset); - COPYVAL(tp_inline_values_offset); #undef COPYVAL /* Setup fast subclass flags */ diff --git a/PC/python3dll.c b/PC/python3dll.c index d2a87070de5..6e469357ede 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -28,7 +28,6 @@ EXPORT_FUNC(_PyArg_VaParseTupleAndKeywords_SizeT) EXPORT_FUNC(_PyErr_BadInternalCall) EXPORT_FUNC(_PyObject_CallFunction_SizeT) EXPORT_FUNC(_PyObject_CallMethod_SizeT) -EXPORT_FUNC(_PyObject_GC_Malloc) EXPORT_FUNC(_PyObject_GC_New) EXPORT_FUNC(_PyObject_GC_NewVar) EXPORT_FUNC(_PyObject_GC_Resize) diff --git a/Python/ceval.c b/Python/ceval.c index a8bbad33552..446772dd4f7 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3599,9 +3599,9 @@ check_eval_breaker: _PyAttrCache *cache1 = &caches[-1].attr; assert(cache1->tp_version != 0); DEOPT_IF(tp->tp_version_tag != cache1->tp_version, LOAD_ATTR); - assert(tp->tp_dictoffset > 0); - assert(tp->tp_inline_values_offset > 0); - PyDictValues *values = *(PyDictValues **)(((char *)owner) + tp->tp_inline_values_offset); + assert(tp->tp_dictoffset < 0); + assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictValues *values = *_PyObject_ValuesPointer(owner); DEOPT_IF(values == NULL, LOAD_ATTR); res = values->values[cache0->index]; DEOPT_IF(res == NULL, LOAD_ATTR); @@ -3633,8 +3633,8 @@ check_eval_breaker: _PyAttrCache *cache1 = &caches[-1].attr; assert(cache1->tp_version != 0); DEOPT_IF(tp->tp_version_tag != cache1->tp_version, LOAD_ATTR); - assert(tp->tp_dictoffset > 0); - PyDictObject *dict = *(PyDictObject **)(((char *)owner) + tp->tp_dictoffset); + assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = *(PyDictObject **)_PyObject_ManagedDictPointer(owner); DEOPT_IF(dict == NULL, LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(names, cache0->original_oparg); @@ -3701,9 +3701,8 @@ check_eval_breaker: _PyAttrCache *cache1 = &caches[-1].attr; assert(cache1->tp_version != 0); DEOPT_IF(tp->tp_version_tag != cache1->tp_version, STORE_ATTR); - assert(tp->tp_dictoffset > 0); - assert(tp->tp_inline_values_offset > 0); - PyDictValues *values = *(PyDictValues **)(((char *)owner) + tp->tp_inline_values_offset); + assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictValues *values = *_PyObject_ValuesPointer(owner); DEOPT_IF(values == NULL, STORE_ATTR); STAT_INC(STORE_ATTR, hit); int index = cache0->index; @@ -3731,8 +3730,8 @@ check_eval_breaker: _PyAttrCache *cache1 = &caches[-1].attr; assert(cache1->tp_version != 0); DEOPT_IF(tp->tp_version_tag != cache1->tp_version, STORE_ATTR); - assert(tp->tp_dictoffset > 0); - PyDictObject *dict = *(PyDictObject **)(((char *)owner) + tp->tp_dictoffset); + assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = *(PyDictObject **)_PyObject_ManagedDictPointer(owner); DEOPT_IF(dict == NULL, STORE_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(names, cache0->original_oparg); @@ -4506,9 +4505,8 @@ check_eval_breaker: _PyObjectCache *cache2 = &caches[-2].obj; DEOPT_IF(self_cls->tp_version_tag != cache1->tp_version, LOAD_METHOD); - assert(self_cls->tp_dictoffset > 0); - assert(self_cls->tp_inline_values_offset > 0); - PyDictObject *dict = *(PyDictObject **)(((char *)self) + self_cls->tp_dictoffset); + assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = *(PyDictObject**)_PyObject_ManagedDictPointer(self); DEOPT_IF(dict != NULL, LOAD_METHOD); DEOPT_IF(((PyHeapTypeObject *)self_cls)->ht_cached_keys->dk_version != cache1->dk_version_or_hint, LOAD_METHOD); STAT_INC(LOAD_METHOD, hit); diff --git a/Python/specialize.c b/Python/specialize.c index b384675560b..cdc535396fa 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -451,6 +451,7 @@ initial_counter_value(void) { #define SPEC_FAIL_NON_OBJECT_SLOT 14 #define SPEC_FAIL_READ_ONLY 15 #define SPEC_FAIL_AUDITED_SLOT 16 +#define SPEC_FAIL_NOT_MANAGED_DICT 17 /* Methods */ @@ -506,7 +507,7 @@ specialize_module_load_attr( PyObject *value = NULL; PyObject *getattr; _Py_IDENTIFIER(__getattr__); - assert(owner->ob_type->tp_inline_values_offset == 0); + assert((owner->ob_type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0); PyDictObject *dict = (PyDictObject *)m->md_dict; if (dict == NULL) { SPECIALIZATION_FAIL(opcode, SPEC_FAIL_NO_DICT); @@ -634,66 +635,44 @@ specialize_dict_access( assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT || kind == BUILTIN_CLASSMETHOD || kind == PYTHON_CLASSMETHOD); // No descriptor, or non overriding. - if (type->tp_dictoffset < 0) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); + if ((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NOT_MANAGED_DICT); return 0; } - if (type->tp_dictoffset > 0) { - PyObject **dictptr = (PyObject **) ((char *)owner + type->tp_dictoffset); - PyDictObject *dict = (PyDictObject *)*dictptr; - if (type->tp_inline_values_offset && dict == NULL) { - // Virtual dictionary - PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; - assert(type->tp_inline_values_offset > 0); - assert(PyUnicode_CheckExact(name)); - Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); - assert (index != DKIX_ERROR); - if (index != (uint16_t)index) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); - return 0; - } - cache1->tp_version = type->tp_version_tag; - cache0->index = (uint16_t)index; - *instr = _Py_MAKECODEUNIT(values_op, _Py_OPARG(*instr)); + PyObject **dictptr = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = (PyDictObject *)*dictptr; + if (dict == NULL) { + // Virtual dictionary + PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; + assert(PyUnicode_CheckExact(name)); + Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); + assert (index != DKIX_ERROR); + if (index != (uint16_t)index) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); return 0; } - else { - if (dict == NULL || !PyDict_CheckExact(dict)) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT); - return 0; - } - // We found an instance with a __dict__. - PyObject *value = NULL; - Py_ssize_t hint = - _PyDict_GetItemHint(dict, name, -1, &value); - if (hint != (uint32_t)hint) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); - return 0; - } - cache1->dk_version_or_hint = (uint32_t)hint; - cache1->tp_version = type->tp_version_tag; - *instr = _Py_MAKECODEUNIT(hint_op, _Py_OPARG(*instr)); - return 1; + cache1->tp_version = type->tp_version_tag; + cache0->index = (uint16_t)index; + *instr = _Py_MAKECODEUNIT(values_op, _Py_OPARG(*instr)); + } + else { + if (!PyDict_CheckExact(dict)) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT); + return 0; } + // We found an instance with a __dict__. + PyObject *value = NULL; + Py_ssize_t hint = + _PyDict_GetItemHint(dict, name, -1, &value); + if (hint != (uint32_t)hint) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); + return 0; + } + cache1->dk_version_or_hint = (uint32_t)hint; + cache1->tp_version = type->tp_version_tag; + *instr = _Py_MAKECODEUNIT(hint_op, _Py_OPARG(*instr)); } - assert(type->tp_dictoffset == 0); - /* No attribute in instance dictionary */ - switch(kind) { - case NON_OVERRIDING: - case BUILTIN_CLASSMETHOD: - case PYTHON_CLASSMETHOD: - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NON_OVERRIDING_DESCRIPTOR); - return 0; - case NON_DESCRIPTOR: - /* To do -- Optimize this case */ - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NOT_DESCRIPTOR); - return 0; - case ABSENT: - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_EXPECTED_ERROR); - return 0; - default: - Py_UNREACHABLE(); - } + return 1; } int @@ -965,12 +944,6 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, } goto success; } - // Technically this is fine for bound method calls, but it's uncommon and - // slightly slower at runtime to get dict. - if (owner_cls->tp_dictoffset < 0) { - SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OUT_OF_RANGE); - goto fail; - } PyObject *descr = NULL; DesciptorClassification kind = 0; @@ -980,9 +953,8 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SPECIALIZATION_FAIL(LOAD_METHOD, load_method_fail_kind(kind)); goto fail; } - if (owner_cls->tp_inline_values_offset) { - PyObject **owner_dictptr = _PyObject_DictPointer(owner); - assert(owner_dictptr); + if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + PyObject **owner_dictptr = _PyObject_ManagedDictPointer(owner); if (*owner_dictptr) { SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_IS_ATTR); goto fail; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 13fae797b29..af4f926f0e4 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1681,10 +1681,7 @@ _PySys_GetSizeOf(PyObject *o) return (size_t)-1; } - /* add gc_head size */ - if (_PyObject_IS_GC(o)) - return ((size_t)size) + sizeof(PyGC_Head); - return (size_t)size; + return (size_t)size + _PyType_PreHeaderSize(Py_TYPE(o)); } static PyObject * diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index a105e58b4af..a0a95e3fc63 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -83,7 +83,8 @@ def _sizeof_void_p(): # value computed later, see PyUnicodeObjectPtr.proxy() _is_pep393 = None -Py_TPFLAGS_HEAPTYPE = (1 << 9) +Py_TPFLAGS_MANAGED_DICT = (1 << 4) +Py_TPFLAGS_HEAPTYPE = (1 << 9) Py_TPFLAGS_LONG_SUBCLASS = (1 << 24) Py_TPFLAGS_LIST_SUBCLASS = (1 << 25) Py_TPFLAGS_TUPLE_SUBCLASS = (1 << 26) @@ -507,7 +508,6 @@ class HeapTypeObjectPtr(PyObjectPtr): tsize = -tsize size = _PyObject_VAR_SIZE(typeobj, tsize) dictoffset += size - assert dictoffset > 0 assert dictoffset % _sizeof_void_p() == 0 dictptr = self._gdbval.cast(_type_char_ptr()) + dictoffset @@ -523,12 +523,11 @@ class HeapTypeObjectPtr(PyObjectPtr): def get_keys_values(self): typeobj = self.type() - values_offset = int_from_int(typeobj.field('tp_inline_values_offset')) - if values_offset == 0: + has_values = int_from_int(typeobj.field('tp_flags')) & Py_TPFLAGS_MANAGED_DICT + if not has_values: return None - charptr = self._gdbval.cast(_type_char_ptr()) + values_offset PyDictValuesPtrPtr = gdb.lookup_type("PyDictValues").pointer().pointer() - valuesptr = charptr.cast(PyDictValuesPtrPtr) + valuesptr = self._gdbval.cast(PyDictValuesPtrPtr) - 4 values = valuesptr.dereference() if long(values) == 0: return None