From 5a8c15819c27c516e5b75b7c9d89eacdb16b77c3 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 16 Aug 2022 13:57:18 +0100 Subject: [PATCH] GH-95245: Move weakreflist into the pre-header. (GH-95996) --- Include/internal/pycore_object.h | 6 +-- Include/object.h | 10 ++++- Lib/test/test_capi.py | 31 +++++++++++++ ...2-08-15-12-41-14.gh-issue-95245.N4gOUV.rst | 3 ++ Modules/_testcapi/heaptype.c | 32 ++++++++++++++ Objects/typeobject.c | 43 ++++++++++++------- 6 files changed, 105 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-08-15-12-41-14.gh-issue-95245.N4gOUV.rst diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 5a328f04bae..8b78f79e950 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -279,7 +279,7 @@ static inline size_t _PyType_PreHeaderSize(PyTypeObject *tp) { return _PyType_IS_GC(tp) * sizeof(PyGC_Head) + - _PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT) * 2 * sizeof(PyObject *); + _PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER) * 2 * sizeof(PyObject *); } void _PyObject_GC_Link(PyObject *op); @@ -296,7 +296,7 @@ extern int _Py_CheckSlotResult( // Test if a type supports weak references static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { - return (type->tp_weaklistoffset > 0); + return (type->tp_weaklistoffset != 0); } extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); @@ -346,7 +346,7 @@ _PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values) ptr->values = ((char *)values) - 1; } -#define MANAGED_DICT_OFFSET (((int)sizeof(PyObject *))*-3) +#define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-4) extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); extern void _PyObject_FreeInstanceAttributes(PyObject *obj); diff --git a/Include/object.h b/Include/object.h index 7d499d8b306..21d3a75834e 100644 --- a/Include/object.h +++ b/Include/object.h @@ -360,12 +360,18 @@ given type object has a specified feature. /* Track types initialized using _PyStaticType_InitBuiltin(). */ #define _Py_TPFLAGS_STATIC_BUILTIN (1 << 1) +/* Placement of weakref pointers are managed by the VM, not by the type. + * The VM will automatically set tp_weaklistoffset. + */ +#define Py_TPFLAGS_MANAGED_WEAKREF (1 << 3) + /* Placement of dict (and values) pointers are managed by the VM, not by the type. - * The VM will automatically set tp_dictoffset. Should not be used for variable sized - * classes, such as classes that extend tuple. + * The VM will automatically set tp_dictoffset. */ #define Py_TPFLAGS_MANAGED_DICT (1 << 4) +#define Py_TPFLAGS_PREHEADER (Py_TPFLAGS_MANAGED_WEAKREF | Py_TPFLAGS_MANAGED_DICT) + /* Set if instances of the type object are treated as sequences for pattern matching */ #define Py_TPFLAGS_SEQUENCE (1 << 5) /* Set if instances of the type object are treated as mappings for pattern matching */ diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index e4d20355d43..e6516b33aec 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -579,6 +579,37 @@ class CAPITest(unittest.TestCase): self.assertEqual(ref(), inst) self.assertEqual(inst.weakreflist, ref) + def test_heaptype_with_managed_weakref(self): + inst = _testcapi.HeapCTypeWithManagedWeakref() + ref = weakref.ref(inst) + self.assertEqual(ref(), inst) + + def test_sublclassing_managed_weakref(self): + + class C(_testcapi.HeapCTypeWithManagedWeakref): + pass + + inst = C() + ref = weakref.ref(inst) + self.assertEqual(ref(), inst) + + def test_sublclassing_managed_both(self): + + class C1(_testcapi.HeapCTypeWithManagedWeakref, _testcapi.HeapCTypeWithManagedDict): + pass + + class C2(_testcapi.HeapCTypeWithManagedDict, _testcapi.HeapCTypeWithManagedWeakref): + pass + + for cls in (C1, C2): + inst = cls() + ref = weakref.ref(inst) + self.assertEqual(ref(), inst) + inst.spam = inst + del inst + ref = weakref.ref(cls()) + self.assertIs(ref(), None) + def test_heaptype_with_buffer(self): inst = _testcapi.HeapCTypeWithBuffer() b = bytes(inst) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-08-15-12-41-14.gh-issue-95245.N4gOUV.rst b/Misc/NEWS.d/next/Core and Builtins/2022-08-15-12-41-14.gh-issue-95245.N4gOUV.rst new file mode 100644 index 00000000000..4449ddd8ded --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-08-15-12-41-14.gh-issue-95245.N4gOUV.rst @@ -0,0 +1,3 @@ +Reduces the size of a "simple" Python object from 8 to 6 words by moving the +weakreflist pointer into the pre-header directly before the object's +dict/values pointer. diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c index eca95450df9..bf80fd64d80 100644 --- a/Modules/_testcapi/heaptype.c +++ b/Modules/_testcapi/heaptype.c @@ -801,6 +801,32 @@ static PyType_Spec HeapCTypeWithManagedDict_spec = { HeapCTypeWithManagedDict_slots }; +static void +heapctypewithmanagedweakref_dealloc(PyObject* self) +{ + + PyTypeObject *tp = Py_TYPE(self); + PyObject_ClearWeakRefs(self); + PyObject_GC_UnTrack(self); + PyObject_GC_Del(self); + Py_DECREF(tp); +} + +static PyType_Slot HeapCTypeWithManagedWeakref_slots[] = { + {Py_tp_traverse, heapgcctype_traverse}, + {Py_tp_getset, heapctypewithdict_getsetlist}, + {Py_tp_dealloc, heapctypewithmanagedweakref_dealloc}, + {0, 0}, +}; + +static PyType_Spec HeapCTypeWithManagedWeakref_spec = { + "_testcapi.HeapCTypeWithManagedWeakref", + sizeof(PyObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MANAGED_WEAKREF, + HeapCTypeWithManagedWeakref_slots +}; + static struct PyMemberDef heapctypewithnegativedict_members[] = { {"dictobj", T_OBJECT, offsetof(HeapCTypeWithDictObject, dict)}, {"__dictoffset__", T_PYSSIZET, -(Py_ssize_t)sizeof(void*), READONLY}, @@ -1009,6 +1035,12 @@ _PyTestCapi_Init_Heaptype(PyObject *m) { } PyModule_AddObject(m, "HeapCTypeWithManagedDict", HeapCTypeWithManagedDict); + PyObject *HeapCTypeWithManagedWeakref = PyType_FromSpec(&HeapCTypeWithManagedWeakref_spec); + if (HeapCTypeWithManagedWeakref == NULL) { + return -1; + } + PyModule_AddObject(m, "HeapCTypeWithManagedWeakref", HeapCTypeWithManagedWeakref); + PyObject *HeapCTypeWithWeakref = PyType_FromSpec(&HeapCTypeWithWeakref_spec); if (HeapCTypeWithWeakref == NULL) { return -1; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 67dfc6f8483..c881aeb63d7 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2497,10 +2497,11 @@ subtype_getweakref(PyObject *obj, void *context) return NULL; } _PyObject_ASSERT((PyObject *)type, - type->tp_weaklistoffset > 0); + type->tp_weaklistoffset > 0 || + type->tp_weaklistoffset == MANAGED_WEAKREF_OFFSET); _PyObject_ASSERT((PyObject *)type, - ((type->tp_weaklistoffset + sizeof(PyObject *)) - <= (size_t)(type->tp_basicsize))); + ((type->tp_weaklistoffset + (Py_ssize_t)sizeof(PyObject *)) + <= type->tp_basicsize)); weaklistptr = (PyObject **)((char *)obj + type->tp_weaklistoffset); if (*weaklistptr == NULL) result = Py_None; @@ -3093,9 +3094,9 @@ type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type) } if (ctx->add_weak) { - assert(!ctx->base->tp_itemsize); - type->tp_weaklistoffset = slotoffset; - slotoffset += sizeof(PyObject *); + assert((type->tp_flags & Py_TPFLAGS_MANAGED_WEAKREF) == 0); + type->tp_flags |= Py_TPFLAGS_MANAGED_WEAKREF; + type->tp_weaklistoffset = MANAGED_WEAKREF_OFFSET; } if (ctx->add_dict) { assert((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0); @@ -5116,9 +5117,9 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* !same_slots_added(newbase, oldbase))) { goto differs; } - /* The above does not check for managed __dicts__ */ - if ((oldto->tp_flags & Py_TPFLAGS_MANAGED_DICT) == - ((newto->tp_flags & Py_TPFLAGS_MANAGED_DICT))) + /* The above does not check for the preheader */ + if ((oldto->tp_flags & Py_TPFLAGS_PREHEADER) == + ((newto->tp_flags & Py_TPFLAGS_PREHEADER))) { return 1; } @@ -5217,7 +5218,7 @@ object_set_class(PyObject *self, PyObject *value, void *closure) if (compatible_for_assignment(oldto, newto, "__class__")) { /* Changing the class will change the implicit dict keys, * so we must materialize the dictionary first. */ - assert((oldto->tp_flags & Py_TPFLAGS_MANAGED_DICT) == (newto->tp_flags & Py_TPFLAGS_MANAGED_DICT)); + assert((oldto->tp_flags & Py_TPFLAGS_PREHEADER) == (newto->tp_flags & Py_TPFLAGS_PREHEADER)); _PyObject_GetDictPtr(self); if (oldto->tp_flags & Py_TPFLAGS_MANAGED_DICT && _PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(self))) @@ -5360,7 +5361,7 @@ object_getstate_default(PyObject *obj, int required) { basicsize += sizeof(PyObject *); } - if (Py_TYPE(obj)->tp_weaklistoffset) { + if (Py_TYPE(obj)->tp_weaklistoffset > 0) { basicsize += sizeof(PyObject *); } if (slotnames != Py_None) { @@ -6150,7 +6151,7 @@ inherit_special(PyTypeObject *type, PyTypeObject *base) if (type->tp_clear == NULL) type->tp_clear = base->tp_clear; } - type->tp_flags |= (base->tp_flags & Py_TPFLAGS_MANAGED_DICT); + type->tp_flags |= (base->tp_flags & Py_TPFLAGS_PREHEADER); if (type->tp_basicsize == 0) type->tp_basicsize = base->tp_basicsize; @@ -6571,7 +6572,7 @@ type_ready_fill_dict(PyTypeObject *type) } static int -type_ready_dict_offset(PyTypeObject *type) +type_ready_preheader(PyTypeObject *type) { if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) { if (type->tp_dictoffset > 0 || type->tp_dictoffset < -1) { @@ -6583,6 +6584,18 @@ type_ready_dict_offset(PyTypeObject *type) } type->tp_dictoffset = -1; } + if (type->tp_flags & Py_TPFLAGS_MANAGED_WEAKREF) { + if (type->tp_weaklistoffset != 0 && + type->tp_weaklistoffset != MANAGED_WEAKREF_OFFSET) + { + PyErr_Format(PyExc_TypeError, + "type %s has the Py_TPFLAGS_MANAGED_WEAKREF flag " + "but tp_weaklistoffset is set", + type->tp_name); + return -1; + } + type->tp_weaklistoffset = MANAGED_WEAKREF_OFFSET; + } return 0; } @@ -6802,7 +6815,7 @@ type_ready_post_checks(PyTypeObject *type) return -1; } } - else if (type->tp_dictoffset < sizeof(PyObject)) { + else if (type->tp_dictoffset < (Py_ssize_t)sizeof(PyObject)) { if (type->tp_dictoffset + type->tp_basicsize <= 0) { PyErr_Format(PyExc_SystemError, "type %s has a tp_dictoffset that is too small"); @@ -6847,7 +6860,7 @@ type_ready(PyTypeObject *type) if (type_ready_inherit(type) < 0) { return -1; } - if (type_ready_dict_offset(type) < 0) { + if (type_ready_preheader(type) < 0) { return -1; } if (type_ready_set_hash(type) < 0) {