From 87154d8dd890af0e847e91b06c71c741c08510d0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 4 Aug 2022 19:26:59 -0600 Subject: [PATCH] gh-94673: Add Per-Interpreter tp_subclasses for Static Builtin Types (gh-95301) --- Doc/c-api/typeobj.rst | 14 +++- Doc/whatsnew/3.12.rst | 10 +++ Include/cpython/object.h | 5 +- Include/internal/pycore_object.h | 1 + Include/internal/pycore_typeobject.h | 1 + Lib/test/test_sys.py | 2 +- Objects/structseq.c | 2 +- Objects/typeobject.c | 116 +++++++++++++++++++++------ 8 files changed, 120 insertions(+), 31 deletions(-) diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 44884ac2890..3af48f408ea 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -135,7 +135,7 @@ Quick Reference +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | [:c:member:`~PyTypeObject.tp_cache`] | :c:type:`PyObject` * | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_subclasses`] | :c:type:`PyObject` * | __subclasses__ | | | | + | [:c:member:`~PyTypeObject.tp_subclasses`] | void * | __subclasses__ | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | [:c:member:`~PyTypeObject.tp_weaklist`] | :c:type:`PyObject` * | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ @@ -1934,9 +1934,17 @@ and :c:type:`PyType_Type` effectively act as defaults.) This field is not inherited. -.. c:member:: PyObject* PyTypeObject.tp_subclasses +.. c:member:: void* PyTypeObject.tp_subclasses - List of weak references to subclasses. Internal use only. + A collection of subclasses. Internal use only. May be an invalid pointer. + + To get a list of subclasses, call the Python method + :py:meth:`~class.__subclasses__`. + + .. versionchanged:: 3.12 + + For some types, this field does not hold a valid :c:expr:`PyObject*`. + The type was changed to :c:expr:`void*` to indicate this. **Inheritance:** diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 97a52e27fec..9ea4d0369e6 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -440,6 +440,16 @@ Porting to Python 3.12 using the existing public C-API instead, or, if necessary, the (internal-only) ``_PyObject_GET_WEAKREFS_LISTPTR()`` macro. +* This internal-only :c:member:`PyTypeObject.tp_subclasses` may now not be + a valid object pointer. Its type was changed to :c:expr:`void *` to + reflect this. We mention this in case someone happens to be accessing the + internal-only field directly. + + To get a list of subclasses, call the Python method + :py:meth:`~class.__subclasses__` (using :c:func:`PyObject_CallMethod`, + for example). + + Deprecated ---------- diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 60c7c3e2aa6..a26fc7f6aad 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -217,8 +217,8 @@ struct _typeobject { inquiry tp_is_gc; /* For PyObject_IS_GC */ PyObject *tp_bases; PyObject *tp_mro; /* method resolution order */ - PyObject *tp_cache; - PyObject *tp_subclasses; + PyObject *tp_cache; /* no longer used */ + void *tp_subclasses; /* for static builtin types this is an index */ PyObject *tp_weaklist; /* not used for static builtin types */ destructor tp_del; @@ -227,7 +227,6 @@ struct _typeobject { destructor tp_finalize; vectorcallfunc tp_vectorcall; - size_t tp_static_builtin_index; /* 0 means "not initialized" */ }; /* This struct is used by the specializer diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index b3b0b846480..5a328f04bae 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -351,6 +351,7 @@ _PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values) extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); +extern int _PyType_HasSubclasses(PyTypeObject *); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); // Access macro to the members which are floating "behind" the object diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 4f9d6b1c4d4..4eb0efcedb5 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -45,6 +45,7 @@ struct type_cache { typedef struct { PyTypeObject *type; + PyObject *tp_subclasses; /* We never clean up weakrefs for static builtin types since they will effectively never get triggered. However, there are also some diagnostic uses for the list of weakrefs, diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index c4798afc920..05fbcc19b6b 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1507,7 +1507,7 @@ class SizeofTest(unittest.TestCase): check((1,2,3), vsize('') + 3*self.P) # type # static type: PyTypeObject - fmt = 'P2nPI13Pl4Pn9Pn12PIPI' + fmt = 'P2nPI13Pl4Pn9Pn12PIP' s = vsize('2P' + fmt) check(int, s) # class diff --git a/Objects/structseq.c b/Objects/structseq.c index 24cd0e70596..9a7013372e6 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -580,7 +580,7 @@ _PyStructSequence_FiniType(PyTypeObject *type) assert(type->tp_base == &PyTuple_Type); // Cannot delete a type if it still has subclasses - if (type->tp_subclasses != NULL) { + if (_PyType_HasSubclasses(type)) { return; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 1f56a5866e3..b2df9e7fad3 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -73,7 +73,7 @@ static inline PyTypeObject * subclass_from_ref(PyObject *ref); static inline int static_builtin_index_is_set(PyTypeObject *self) { - return self->tp_static_builtin_index > 0; + return self->tp_subclasses != NULL; } static inline size_t @@ -81,7 +81,7 @@ static_builtin_index_get(PyTypeObject *self) { assert(static_builtin_index_is_set(self)); /* We store a 1-based index so 0 can mean "not initialized". */ - return self->tp_static_builtin_index - 1; + return (size_t)self->tp_subclasses - 1; } static inline void @@ -89,13 +89,13 @@ static_builtin_index_set(PyTypeObject *self, size_t index) { assert(index < _Py_MAX_STATIC_BUILTIN_TYPES); /* We store a 1-based index so 0 can mean "not initialized". */ - self->tp_static_builtin_index = index + 1; + self->tp_subclasses = (PyObject *)(index + 1); } static inline void static_builtin_index_clear(PyTypeObject *self) { - self->tp_static_builtin_index = 0; + self->tp_subclasses = NULL; } static inline static_builtin_state * @@ -127,6 +127,7 @@ static_builtin_state_init(PyTypeObject *self) static_builtin_state *state = static_builtin_state_get(interp, self); state->type = self; + /* state->tp_subclasses is left NULL until init_subclasses() sets it. */ /* state->tp_weaklist is left NULL until insert_head() or insert_after() (in weakrefobject.c) sets it. */ } @@ -373,6 +374,8 @@ _PyTypes_Fini(PyInterpreterState *interp) } +static PyObject * lookup_subclasses(PyTypeObject *); + void PyType_Modified(PyTypeObject *type) { @@ -395,7 +398,7 @@ PyType_Modified(PyTypeObject *type) return; } - PyObject *subclasses = type->tp_subclasses; + PyObject *subclasses = lookup_subclasses(type); if (subclasses != NULL) { assert(PyDict_CheckExact(subclasses)); @@ -783,7 +786,7 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) Py_XDECREF(old_mro); // Avoid creating an empty list if there is no subclass - if (type->tp_subclasses != NULL) { + if (_PyType_HasSubclasses(type)) { /* Obtain a copy of subclasses list to iterate over. Otherwise type->tp_subclasses might be altered @@ -4345,10 +4348,13 @@ type_dealloc_common(PyTypeObject *type) } +static void clear_subclasses(PyTypeObject *self); + static void clear_static_tp_subclasses(PyTypeObject *type) { - if (type->tp_subclasses == NULL) { + PyObject *subclasses = lookup_subclasses(type); + if (subclasses == NULL) { return; } @@ -4372,9 +4378,19 @@ clear_static_tp_subclasses(PyTypeObject *type) going to leak. This mostly only affects embedding scenarios. */ - // For now we just clear tp_subclasses. + // For now we just do a sanity check and then clear tp_subclasses. + Py_ssize_t i = 0; + PyObject *key, *ref; // borrowed ref + while (PyDict_Next(subclasses, &i, &key, &ref)) { + PyTypeObject *subclass = subclass_from_ref(ref); // borrowed + if (subclass == NULL) { + continue; + } + // All static builtin subtypes should have been finalized already. + assert(!(subclass->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); + } - Py_CLEAR(type->tp_subclasses); + clear_subclasses(type); } void @@ -4424,7 +4440,7 @@ type_dealloc(PyTypeObject *type) Py_XDECREF(type->tp_bases); Py_XDECREF(type->tp_mro); Py_XDECREF(type->tp_cache); - Py_XDECREF(type->tp_subclasses); + clear_subclasses(type); /* A type's tp_doc is heap allocated, unlike the tp_doc slots * of most other objects. It's okay to cast it to char *. @@ -4444,6 +4460,30 @@ type_dealloc(PyTypeObject *type) } +static PyObject * +lookup_subclasses(PyTypeObject *self) +{ + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + static_builtin_state *state = _PyStaticType_GetState(self); + assert(state != NULL); + return state->tp_subclasses; + } + return (PyObject *)self->tp_subclasses; +} + +int +_PyType_HasSubclasses(PyTypeObject *self) +{ + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN && + _PyStaticType_GetState(self) == NULL) { + return 0; + } + if (lookup_subclasses(self) == NULL) { + return 0; + } + return 1; +} + PyObject* _PyType_GetSubclasses(PyTypeObject *self) { @@ -4452,7 +4492,7 @@ _PyType_GetSubclasses(PyTypeObject *self) return NULL; } - PyObject *subclasses = self->tp_subclasses; // borrowed ref + PyObject *subclasses = lookup_subclasses(self); // borrowed ref if (subclasses == NULL) { return list; } @@ -6830,6 +6870,36 @@ _PyStaticType_InitBuiltin(PyTypeObject *self) } +static PyObject * +init_subclasses(PyTypeObject *self) +{ + PyObject *subclasses = PyDict_New(); + if (subclasses == NULL) { + return NULL; + } + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + static_builtin_state *state = _PyStaticType_GetState(self); + state->tp_subclasses = subclasses; + return subclasses; + } + self->tp_subclasses = (void *)subclasses; + return subclasses; +} + +static void +clear_subclasses(PyTypeObject *self) +{ + /* Delete the dictionary to save memory. _PyStaticType_Dealloc() + callers also test if tp_subclasses is NULL to check if a static type + has no subclass. */ + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + static_builtin_state *state = _PyStaticType_GetState(self); + Py_CLEAR(state->tp_subclasses); + return; + } + Py_CLEAR(self->tp_subclasses); +} + static int add_subclass(PyTypeObject *base, PyTypeObject *type) { @@ -6846,9 +6916,9 @@ add_subclass(PyTypeObject *base, PyTypeObject *type) // Only get tp_subclasses after creating the key and value. // PyWeakref_NewRef() can trigger a garbage collection which can execute // arbitrary Python code and so modify base->tp_subclasses. - PyObject *subclasses = base->tp_subclasses; + PyObject *subclasses = lookup_subclasses(base); if (subclasses == NULL) { - base->tp_subclasses = subclasses = PyDict_New(); + subclasses = init_subclasses(base); if (subclasses == NULL) { Py_DECREF(key); Py_DECREF(ref); @@ -6905,10 +6975,13 @@ get_subclasses_key(PyTypeObject *type, PyTypeObject *base) We fall back to manually traversing the values. */ Py_ssize_t i = 0; PyObject *ref; // borrowed ref - while (PyDict_Next((PyObject *)base->tp_subclasses, &i, &key, &ref)) { - PyTypeObject *subclass = subclass_from_ref(ref); // borrowed - if (subclass == type) { - return Py_NewRef(key); + PyObject *subclasses = lookup_subclasses(base); + if (subclasses != NULL) { + while (PyDict_Next(subclasses, &i, &key, &ref)) { + PyTypeObject *subclass = subclass_from_ref(ref); // borrowed + if (subclass == type) { + return Py_NewRef(key); + } } } /* It wasn't found. */ @@ -6918,7 +6991,7 @@ get_subclasses_key(PyTypeObject *type, PyTypeObject *base) static void remove_subclass(PyTypeObject *base, PyTypeObject *type) { - PyObject *subclasses = base->tp_subclasses; // borrowed ref + PyObject *subclasses = lookup_subclasses(base); // borrowed ref if (subclasses == NULL) { return; } @@ -6934,10 +7007,7 @@ remove_subclass(PyTypeObject *base, PyTypeObject *type) Py_XDECREF(key); if (PyDict_Size(subclasses) == 0) { - // Delete the dictionary to save memory. _PyStaticType_Dealloc() - // callers also test if tp_subclasses is NULL to check if a static type - // has no subclass. - Py_CLEAR(base->tp_subclasses); + clear_subclasses(base); } } @@ -9022,7 +9092,7 @@ recurse_down_subclasses(PyTypeObject *type, PyObject *attr_name, // It is safe to use a borrowed reference because update_subclasses() is // only used with update_slots_callback() which doesn't modify // tp_subclasses. - PyObject *subclasses = type->tp_subclasses; // borrowed ref + PyObject *subclasses = lookup_subclasses(type); // borrowed ref if (subclasses == NULL) { return 0; }