Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-124218: Refactor per-thread reference counting (#124844) 

Currently, we only use per-thread reference counting for heap type objects and
the naming reflects that. We will extend it to a few additional types in an
upcoming change to avoid scaling bottlenecks when creating nested functions.

Rename some of the files and functions in preparation for this change.
This commit is contained in:
Sam Gross 2024-10-01 13:05:42 -04:00 committed by GitHub
parent 5aa91c56bf
commit b482538523
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 168 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Include/internal/pycore_interp.h
View File

@ -35,7 +35,7 @@ extern "C" {
#include "pycore_qsbr.h" // struct _qsbr_state #include "pycore_qsbr.h" // struct _qsbr_state
#include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_tstate.h" // _PyThreadStateImpl
#include "pycore_tuple.h" // struct _Py_tuple_state #include "pycore_tuple.h" // struct _Py_tuple_state
#include "pycore_typeid.h" // struct _Py_type_id_pool #include "pycore_uniqueid.h" // struct _Py_unique_id_pool
#include "pycore_typeobject.h" // struct types_state #include "pycore_typeobject.h" // struct types_state
#include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_unicodeobject.h" // struct _Py_unicode_state
#include "pycore_warnings.h" // struct _warnings_runtime_state #include "pycore_warnings.h" // struct _warnings_runtime_state
@ -221,7 +221,7 @@ struct _is {
#if defined(Py_GIL_DISABLED) #if defined(Py_GIL_DISABLED)
struct _mimalloc_interp_state mimalloc; struct _mimalloc_interp_state mimalloc;
struct _brc_state brc; // biased reference counting state struct _brc_state brc; // biased reference counting state
struct _Py_type_id_pool type_ids; struct _Py_unique_id_pool unique_ids; // object ids for per-thread refcounts
PyMutex weakref_locks[NUM_WEAKREF_LIST_LOCKS]; PyMutex weakref_locks[NUM_WEAKREF_LIST_LOCKS];
#endif #endif

10
Include/internal/pycore_object.h
View File

@ -14,7 +14,7 @@ extern "C" {
#include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_interp.h" // PyInterpreterState.gc
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED
#include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "pycore_typeid.h" // _PyType_IncrefSlow #include "pycore_uniqueid.h" // _PyType_IncrefSlow
#define _Py_IMMORTAL_REFCNT_LOOSE ((_Py_IMMORTAL_REFCNT >> 1) + 1) #define _Py_IMMORTAL_REFCNT_LOOSE ((_Py_IMMORTAL_REFCNT >> 1) + 1)
@ -335,12 +335,12 @@ _Py_INCREF_TYPE(PyTypeObject *type)
// Unsigned comparison so that `unique_id=-1`, which indicates that // Unsigned comparison so that `unique_id=-1`, which indicates that
// per-thread refcounting has been disabled on this type, is handled by // per-thread refcounting has been disabled on this type, is handled by
// the "else". // the "else".
if ((size_t)ht->unique_id < (size_t)tstate->types.size) { if ((size_t)ht->unique_id < (size_t)tstate->refcounts.size) {
# ifdef Py_REF_DEBUG # ifdef Py_REF_DEBUG
_Py_INCREF_IncRefTotal(); _Py_INCREF_IncRefTotal();
# endif # endif
_Py_INCREF_STAT_INC(); _Py_INCREF_STAT_INC();
tstate->types.refcounts[ht->unique_id]++; tstate->refcounts.values[ht->unique_id]++;
} }
else { else {
// The slow path resizes the thread-local refcount array if necessary. // The slow path resizes the thread-local refcount array if necessary.
@ -368,12 +368,12 @@ _Py_DECREF_TYPE(PyTypeObject *type)
// Unsigned comparison so that `unique_id=-1`, which indicates that // Unsigned comparison so that `unique_id=-1`, which indicates that
// per-thread refcounting has been disabled on this type, is handled by // per-thread refcounting has been disabled on this type, is handled by
// the "else". // the "else".
if ((size_t)ht->unique_id < (size_t)tstate->types.size) { if ((size_t)ht->unique_id < (size_t)tstate->refcounts.size) {
# ifdef Py_REF_DEBUG # ifdef Py_REF_DEBUG
_Py_DECREF_DecRefTotal(); _Py_DECREF_DecRefTotal();
# endif # endif
_Py_DECREF_STAT_INC(); _Py_DECREF_STAT_INC();
tstate->types.refcounts[ht->unique_id]--; tstate->refcounts.values[ht->unique_id]--;
} }
else { else {
// Directly decref the type if the type id is not assigned or if // Directly decref the type if the type id is not assigned or if

8
Include/internal/pycore_tstate.h
View File

@ -32,15 +32,15 @@ typedef struct _PyThreadStateImpl {
struct _Py_freelists freelists; struct _Py_freelists freelists;
struct _brc_thread_state brc; struct _brc_thread_state brc;
struct { struct {
// The thread-local refcounts for heap type objects // The per-thread refcounts
Py_ssize_t *refcounts; Py_ssize_t *values;
// Size of the refcounts array. // Size of the refcounts array.
Py_ssize_t size; Py_ssize_t size;
// If set, don't use thread-local refcounts // If set, don't use per-thread refcounts
int is_finalized; int is_finalized;
} types; } refcounts;
#endif #endif
#if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED)

75
Include/internal/pycore_typeid.h
View File

@ -1,75 +0,0 @@
#ifndef Py_INTERNAL_TYPEID_H
#define Py_INTERNAL_TYPEID_H
#ifdef __cplusplus
extern "C" {
#endif
#ifndef Py_BUILD_CORE
# error "this header requires Py_BUILD_CORE define"
#endif
#ifdef Py_GIL_DISABLED
// This contains code for allocating unique ids to heap type objects
// and re-using those ids when the type is deallocated.
//
// The type ids are used to implement per-thread reference counts of
// heap type objects to avoid contention on the reference count fields
// of heap type objects. Static type objects are immortal, so contention
// is not an issue for those types.
//
// Type id of -1 is used to indicate a type doesn't use thread-local
// refcounting. This value is used when a type object is finalized by the GC
// and during interpreter shutdown to allow the type object to be
// deallocated promptly when the object's refcount reaches zero.
//
// Each entry implicitly represents a type id based on it's offset in the
// table. Non-allocated entries form a free-list via the 'next' pointer.
// Allocated entries store the corresponding PyTypeObject.
typedef union _Py_type_id_entry {
// Points to the next free type id, when part of the freelist
union _Py_type_id_entry *next;
// Stores the type object when the id is assigned
PyHeapTypeObject *type;
} _Py_type_id_entry;
struct _Py_type_id_pool {
PyMutex mutex;
// combined table of types with allocated type ids and unallocated
// type ids.
_Py_type_id_entry *table;
// Next entry to allocate inside 'table' or NULL
_Py_type_id_entry *freelist;
// size of 'table'
Py_ssize_t size;
};
// Assigns the next id from the pool of type ids.
extern void _PyType_AssignId(PyHeapTypeObject *type);
// Releases the allocated type id back to the pool.
extern void _PyType_ReleaseId(PyHeapTypeObject *type);
// Merges the thread-local reference counts into the corresponding types.
extern void _PyType_MergeThreadLocalRefcounts(_PyThreadStateImpl *tstate);
// Like _PyType_MergeThreadLocalRefcounts, but also frees the thread-local
// array of refcounts.
extern void _PyType_FinalizeThreadLocalRefcounts(_PyThreadStateImpl *tstate);
// Frees the interpreter's pool of type ids.
extern void _PyType_FinalizeIdPool(PyInterpreterState *interp);
// Increfs the type, resizing the thread-local refcount array if necessary.
PyAPI_FUNC(void) _PyType_IncrefSlow(PyHeapTypeObject *type);
#endif /* Py_GIL_DISABLED */
#ifdef __cplusplus
}
#endif
#endif /* !Py_INTERNAL_TYPEID_H */

72
Include/internal/pycore_uniqueid.h Normal file
View File

@ -0,0 +1,72 @@
#ifndef Py_INTERNAL_UNIQUEID_H
#define Py_INTERNAL_UNIQUEID_H
#ifdef __cplusplus
extern "C" {
#endif
#ifndef Py_BUILD_CORE
# error "this header requires Py_BUILD_CORE define"
#endif
#ifdef Py_GIL_DISABLED
// This contains code for allocating unique ids to objects for per-thread
// reference counting.
//
// Per-thread reference counting is used along with deferred reference
// counting to avoid scaling bottlenecks due to reference count contention.
//
// An id of -1 is used to indicate that an object doesn't use per-thread
// refcounting. This value is used when the object is finalized by the GC
// and during interpreter shutdown to allow the object to be
// deallocated promptly when the object's refcount reaches zero.
//
// Each entry implicitly represents a unique id based on its offset in the
// table. Non-allocated entries form a free-list via the 'next' pointer.
// Allocated entries store the corresponding PyObject.
typedef union _Py_unique_id_entry {
// Points to the next free type id, when part of the freelist
union _Py_unique_id_entry *next;
// Stores the object when the id is assigned
PyObject *obj;
} _Py_unique_id_entry;
struct _Py_unique_id_pool {
PyMutex mutex;
// combined table of object with allocated unique ids and unallocated ids.
_Py_unique_id_entry *table;
// Next entry to allocate inside 'table' or NULL
_Py_unique_id_entry *freelist;
// size of 'table'
Py_ssize_t size;
};
// Assigns the next id from the pool of ids.
extern Py_ssize_t _PyObject_AssignUniqueId(PyObject *obj);
// Releases the allocated id back to the pool.
extern void _PyObject_ReleaseUniqueId(Py_ssize_t unique_id);
// Merges the per-thread reference counts into the corresponding objects.
extern void _PyObject_MergePerThreadRefcounts(_PyThreadStateImpl *tstate);
// Like _PyObject_MergePerThreadRefcounts, but also frees the per-thread
// array of refcounts.
extern void _PyObject_FinalizePerThreadRefcounts(_PyThreadStateImpl *tstate);
// Frees the interpreter's pool of type ids.
extern void _PyObject_FinalizeUniqueIdPool(PyInterpreterState *interp);
// Increfs the type, resizing the per-thread refcount array if necessary.
PyAPI_FUNC(void) _PyType_IncrefSlow(PyHeapTypeObject *type);
#endif /* Py_GIL_DISABLED */
#ifdef __cplusplus
}
#endif
#endif /* !Py_INTERNAL_UNIQUEID_H */

4
Makefile.pre.in
View File

@ -490,7 +490,7 @@ PYTHON_OBJS= \
Python/thread.o \ Python/thread.o \
Python/traceback.o \ Python/traceback.o \
Python/tracemalloc.o \ Python/tracemalloc.o \
Python/typeid.o \ Python/uniqueid.o \
Python/getopt.o \ Python/getopt.o \
Python/pystrcmp.o \ Python/pystrcmp.o \
Python/pystrtod.o \ Python/pystrtod.o \
@ -1279,7 +1279,7 @@ PYTHON_HEADERS= \
$(srcdir)/Include/internal/pycore_tracemalloc.h \ $(srcdir)/Include/internal/pycore_tracemalloc.h \
$(srcdir)/Include/internal/pycore_tstate.h \ $(srcdir)/Include/internal/pycore_tstate.h \
$(srcdir)/Include/internal/pycore_tuple.h \ $(srcdir)/Include/internal/pycore_tuple.h \
$(srcdir)/Include/internal/pycore_typeid.h \ $(srcdir)/Include/internal/pycore_uniqueid.h \
$(srcdir)/Include/internal/pycore_typeobject.h \ $(srcdir)/Include/internal/pycore_typeobject.h \
$(srcdir)/Include/internal/pycore_typevarobject.h \ $(srcdir)/Include/internal/pycore_typevarobject.h \
$(srcdir)/Include/internal/pycore_ucnhash.h \ $(srcdir)/Include/internal/pycore_ucnhash.h \

6
Objects/typeobject.c
View File

@ -3932,7 +3932,7 @@ type_new_alloc(type_new_ctx *ctx)
et->ht_token = NULL; et->ht_token = NULL;
#ifdef Py_GIL_DISABLED #ifdef Py_GIL_DISABLED
_PyType_AssignId(et); et->unique_id = _PyObject_AssignUniqueId((PyObject *)et);
#endif #endif
return type; return type;
@ -5026,7 +5026,7 @@ PyType_FromMetaclass(
#ifdef Py_GIL_DISABLED #ifdef Py_GIL_DISABLED
// Assign a type id to enable thread-local refcounting // Assign a type id to enable thread-local refcounting
_PyType_AssignId(res); res->unique_id = _PyObject_AssignUniqueId((PyObject *)res);
#endif #endif
/* Ready the type (which includes inheritance). /* Ready the type (which includes inheritance).
@ -6080,7 +6080,7 @@ type_dealloc(PyObject *self)
Py_XDECREF(et->ht_module); Py_XDECREF(et->ht_module);
PyMem_Free(et->_ht_tpname); PyMem_Free(et->_ht_tpname);
#ifdef Py_GIL_DISABLED #ifdef Py_GIL_DISABLED
_PyType_ReleaseId(et); _PyObject_ReleaseUniqueId(et->unique_id);
#endif #endif
et->ht_token = NULL; et->ht_token = NULL;
Py_TYPE(type)->tp_free((PyObject *)type); Py_TYPE(type)->tp_free((PyObject *)type);

2
PCbuild/_freeze_module.vcxproj
View File

@ -268,7 +268,7 @@
<ClCompile Include="..\Python\thread.c" /> <ClCompile Include="..\Python\thread.c" />
<ClCompile Include="..\Python\traceback.c" /> <ClCompile Include="..\Python\traceback.c" />
<ClCompile Include="..\Python\tracemalloc.c" /> <ClCompile Include="..\Python\tracemalloc.c" />
<ClCompile Include="..\Python\typeid.c" /> <ClCompile Include="..\Python\uniqueid.c" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ClInclude Include="..\PC\pyconfig.h.in" /> <ClInclude Include="..\PC\pyconfig.h.in" />

2
PCbuild/_freeze_module.vcxproj.filters
View File

@ -467,7 +467,7 @@
<ClCompile Include="..\Python\tracemalloc.c"> <ClCompile Include="..\Python\tracemalloc.c">
<Filter>Source Files</Filter> <Filter>Source Files</Filter>
</ClCompile> </ClCompile>
<ClCompile Include="..\Python\typeid.c"> <ClCompile Include="..\Python\uniqueid.c">
<Filter>Source Files</Filter> <Filter>Source Files</Filter>
</ClCompile> </ClCompile>
<ClCompile Include="..\Objects\tupleobject.c"> <ClCompile Include="..\Objects\tupleobject.c">

4
PCbuild/pythoncore.vcxproj
View File

@ -304,13 +304,13 @@
<ClInclude Include="..\Include\internal\pycore_tracemalloc.h" /> <ClInclude Include="..\Include\internal\pycore_tracemalloc.h" />
<ClInclude Include="..\Include\internal\pycore_tstate.h" /> <ClInclude Include="..\Include\internal\pycore_tstate.h" />
<ClInclude Include="..\Include\internal\pycore_tuple.h" /> <ClInclude Include="..\Include\internal\pycore_tuple.h" />
<ClInclude Include="..\Include\internal\pycore_typeid.h" />
<ClInclude Include="..\Include\internal\pycore_typeobject.h" /> <ClInclude Include="..\Include\internal\pycore_typeobject.h" />
<ClInclude Include="..\Include\internal\pycore_typevarobject.h" /> <ClInclude Include="..\Include\internal\pycore_typevarobject.h" />
<ClInclude Include="..\Include\internal\pycore_ucnhash.h" /> <ClInclude Include="..\Include\internal\pycore_ucnhash.h" />
<ClInclude Include="..\Include\internal\pycore_unionobject.h" /> <ClInclude Include="..\Include\internal\pycore_unionobject.h" />
<ClInclude Include="..\Include\internal\pycore_unicodeobject.h" /> <ClInclude Include="..\Include\internal\pycore_unicodeobject.h" />
<ClInclude Include="..\Include\internal\pycore_unicodeobject_generated.h" /> <ClInclude Include="..\Include\internal\pycore_unicodeobject_generated.h" />
<ClInclude Include="..\Include\internal\pycore_uniqueid.h" />
<ClInclude Include="..\Include\internal\pycore_warnings.h" /> <ClInclude Include="..\Include\internal\pycore_warnings.h" />
<ClInclude Include="..\Include\internal\pycore_weakref.h" /> <ClInclude Include="..\Include\internal\pycore_weakref.h" />
<ClInclude Include="..\Include\intrcheck.h" /> <ClInclude Include="..\Include\intrcheck.h" />
@ -657,7 +657,7 @@
<ClCompile Include="..\Python\thread.c" /> <ClCompile Include="..\Python\thread.c" />
<ClCompile Include="..\Python\traceback.c" /> <ClCompile Include="..\Python\traceback.c" />
<ClCompile Include="..\Python\tracemalloc.c" /> <ClCompile Include="..\Python\tracemalloc.c" />
<ClCompile Include="..\Python\typeid.c" /> <ClCompile Include="..\Python\uniqueid.c" />
</ItemGroup> </ItemGroup>
<ItemGroup Condition="$(IncludeExternals)"> <ItemGroup Condition="$(IncludeExternals)">
<ClCompile Include="..\Modules\zlibmodule.c" /> <ClCompile Include="..\Modules\zlibmodule.c" />

8
PCbuild/pythoncore.vcxproj.filters
View File

@ -831,9 +831,6 @@
<ClInclude Include="..\Include\internal\pycore_tuple.h"> <ClInclude Include="..\Include\internal\pycore_tuple.h">
<Filter>Include\internal</Filter> <Filter>Include\internal</Filter>
</ClInclude> </ClInclude>
<ClInclude Include="..\Include\internal\pycore_typeid.h">
<Filter>Include\internal</Filter>
</ClInclude>
<ClInclude Include="..\Include\internal\pycore_typeobject.h"> <ClInclude Include="..\Include\internal\pycore_typeobject.h">
<Filter>Include\internal</Filter> <Filter>Include\internal</Filter>
</ClInclude> </ClInclude>
@ -846,6 +843,9 @@
<ClInclude Include="..\Include\internal\pycore_unionobject.h"> <ClInclude Include="..\Include\internal\pycore_unionobject.h">
<Filter>Include\internal</Filter> <Filter>Include\internal</Filter>
</ClInclude> </ClInclude>
<ClInclude Include="..\Include\internal\pycore_uniqueid.h">
<Filter>Include\internal</Filter>
</ClInclude>
<ClInclude Include="..\Include\internal\mimalloc\mimalloc.h"> <ClInclude Include="..\Include\internal\mimalloc\mimalloc.h">
<Filter>Include\internal\mimalloc</Filter> <Filter>Include\internal\mimalloc</Filter>
</ClInclude> </ClInclude>
@ -1499,7 +1499,7 @@
<ClCompile Include="..\Python\tracemalloc.c"> <ClCompile Include="..\Python\tracemalloc.c">
<Filter>Python</Filter> <Filter>Python</Filter>
</ClCompile> </ClCompile>
<ClCompile Include="..\Python\typeid.c"> <ClCompile Include="..\Python\uniqueid.c">
<Filter>Python</Filter> <Filter>Python</Filter>
</ClCompile> </ClCompile>
<ClCompile Include="..\Python\bootstrap_hash.c"> <ClCompile Include="..\Python\bootstrap_hash.c">

14
Python/gc_free_threading.c
View File

@ -15,7 +15,7 @@
#include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_tstate.h" // _PyThreadStateImpl
#include "pycore_weakref.h" // _PyWeakref_ClearRef() #include "pycore_weakref.h" // _PyWeakref_ClearRef()
#include "pydtrace.h" #include "pydtrace.h"
#include "pycore_typeid.h" // _PyType_MergeThreadLocalRefcounts #include "pycore_uniqueid.h" // _PyType_MergeThreadLocalRefcounts
#ifdef Py_GIL_DISABLED #ifdef Py_GIL_DISABLED
@ -217,12 +217,12 @@ disable_deferred_refcounting(PyObject *op)
merge_refcount(op, 0); merge_refcount(op, 0);
} }
// Heap types also use thread-local refcounting -- disable it here. // Heap types also use per-thread refcounting -- disable it here.
if (PyType_Check(op)) { if (PyType_Check(op)) {
// Disable thread-local refcounting for heap types if (PyType_HasFeature((PyTypeObject *)op, Py_TPFLAGS_HEAPTYPE)) {
PyTypeObject *type = (PyTypeObject *)op; PyHeapTypeObject *ht = (PyHeapTypeObject *)op;
if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { _PyObject_ReleaseUniqueId(ht->unique_id);
_PyType_ReleaseId((PyHeapTypeObject *)op); ht->unique_id = -1;
} }
} }
@ -1221,7 +1221,7 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state,
_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)p; _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)p;
// merge per-thread refcount for types into the type's actual refcount // merge per-thread refcount for types into the type's actual refcount
_PyType_MergeThreadLocalRefcounts(tstate); _PyObject_MergePerThreadRefcounts(tstate);
// merge refcounts for all queued objects // merge refcounts for all queued objects
merge_queued_objects(tstate, state); merge_queued_objects(tstate, state);

4
Python/pylifecycle.c
View File

@ -28,7 +28,7 @@
#include "pycore_sliceobject.h" // _PySlice_Fini() #include "pycore_sliceobject.h" // _PySlice_Fini()
#include "pycore_sysmodule.h" // _PySys_ClearAuditHooks() #include "pycore_sysmodule.h" // _PySys_ClearAuditHooks()
#include "pycore_traceback.h" // _Py_DumpTracebackThreads() #include "pycore_traceback.h" // _Py_DumpTracebackThreads()
#include "pycore_typeid.h" // _PyType_FinalizeIdPool() #include "pycore_uniqueid.h" // _PyType_FinalizeIdPool()
#include "pycore_typeobject.h" // _PyTypes_InitTypes() #include "pycore_typeobject.h" // _PyTypes_InitTypes()
#include "pycore_typevarobject.h" // _Py_clear_generic_types() #include "pycore_typevarobject.h" // _Py_clear_generic_types()
#include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes()
@ -1834,7 +1834,7 @@ finalize_interp_types(PyInterpreterState *interp)
_PyTypes_Fini(interp); _PyTypes_Fini(interp);
#ifdef Py_GIL_DISABLED #ifdef Py_GIL_DISABLED
_PyType_FinalizeIdPool(interp); _PyObject_FinalizeUniqueIdPool(interp);
#endif #endif
_PyCode_Fini(interp); _PyCode_Fini(interp);

6
Python/pystate.c
View File

@ -20,7 +20,7 @@
#include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_runtime_init.h" // _PyRuntimeState_INIT
#include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_sysmodule.h" // _PySys_Audit()
#include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap() #include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap()
#include "pycore_typeid.h" // _PyType_FinalizeThreadLocalRefcounts() #include "pycore_uniqueid.h" // _PyType_FinalizeThreadLocalRefcounts()
/* -------------------------------------------------------------------------- /* --------------------------------------------------------------------------
CAUTION CAUTION
@ -1745,7 +1745,7 @@ PyThreadState_Clear(PyThreadState *tstate)
// Merge our thread-local refcounts into the type's own refcount and // Merge our thread-local refcounts into the type's own refcount and
// free our local refcount array. // free our local refcount array.
_PyType_FinalizeThreadLocalRefcounts((_PyThreadStateImpl *)tstate); _PyObject_FinalizePerThreadRefcounts((_PyThreadStateImpl *)tstate);
// Remove ourself from the biased reference counting table of threads. // Remove ourself from the biased reference counting table of threads.
_Py_brc_remove_thread(tstate); _Py_brc_remove_thread(tstate);
@ -1805,7 +1805,7 @@ tstate_delete_common(PyThreadState *tstate, int release_gil)
_PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;
tstate->interp->object_state.reftotal += tstate_impl->reftotal; tstate->interp->object_state.reftotal += tstate_impl->reftotal;
tstate_impl->reftotal = 0; tstate_impl->reftotal = 0;
assert(tstate_impl->types.refcounts == NULL); assert(tstate_impl->refcounts.values == NULL);
#endif #endif
HEAD_UNLOCK(runtime); HEAD_UNLOCK(runtime);

114
Python/typeid.c → Python/uniqueid.c
View File

@ -3,12 +3,14 @@
#include "pycore_lock.h" // PyMutex_LockFlags() #include "pycore_lock.h" // PyMutex_LockFlags()
#include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_pystate.h" // _PyThreadState_GET()
#include "pycore_object.h" // _Py_IncRefTotal #include "pycore_object.h" // _Py_IncRefTotal
#include "pycore_typeid.h" #include "pycore_uniqueid.h"
// This contains code for allocating unique ids to heap type objects // This contains code for allocating unique ids for per-thread reference
// and re-using those ids when the type is deallocated. // counting and re-using those ids when an object is deallocated.
// //
// See Include/internal/pycore_typeid.h for more details. // Currently, per-thread reference counting is only used for heap types.
//
// See Include/internal/pycore_uniqueid.h for more details.
#ifdef Py_GIL_DISABLED #ifdef Py_GIL_DISABLED
@ -18,7 +20,7 @@
#define UNLOCK_POOL(pool) PyMutex_Unlock(&pool->mutex) #define UNLOCK_POOL(pool) PyMutex_Unlock(&pool->mutex)
static int static int
resize_interp_type_id_pool(struct _Py_type_id_pool *pool) resize_interp_type_id_pool(struct _Py_unique_id_pool *pool)
{ {
if ((size_t)pool->size > PY_SSIZE_T_MAX / (2 * sizeof(*pool->table))) { if ((size_t)pool->size > PY_SSIZE_T_MAX / (2 * sizeof(*pool->table))) {
return -1; return -1;
@ -29,7 +31,7 @@ resize_interp_type_id_pool(struct _Py_type_id_pool *pool)
new_size = POOL_MIN_SIZE; new_size = POOL_MIN_SIZE;
} }
_Py_type_id_entry *table = PyMem_Realloc(pool->table, _Py_unique_id_entry *table = PyMem_Realloc(pool->table,
new_size * sizeof(*pool->table)); new_size * sizeof(*pool->table));
if (table == NULL) { if (table == NULL) {
return -1; return -1;
@ -50,70 +52,67 @@ resize_interp_type_id_pool(struct _Py_type_id_pool *pool)
static int static int
resize_local_refcounts(_PyThreadStateImpl *tstate) resize_local_refcounts(_PyThreadStateImpl *tstate)
{ {
if (tstate->types.is_finalized) { if (tstate->refcounts.is_finalized) {
return -1; return -1;
} }
struct _Py_type_id_pool *pool = &tstate->base.interp->type_ids; struct _Py_unique_id_pool *pool = &tstate->base.interp->unique_ids;
Py_ssize_t size = _Py_atomic_load_ssize(&pool->size); Py_ssize_t size = _Py_atomic_load_ssize(&pool->size);
Py_ssize_t *refcnts = PyMem_Realloc(tstate->types.refcounts, Py_ssize_t *refcnts = PyMem_Realloc(tstate->refcounts.values,
size * sizeof(Py_ssize_t)); size * sizeof(Py_ssize_t));
if (refcnts == NULL) { if (refcnts == NULL) {
return -1; return -1;
} }
Py_ssize_t old_size = tstate->types.size; Py_ssize_t old_size = tstate->refcounts.size;
if (old_size < size) { if (old_size < size) {
memset(refcnts + old_size, 0, (size - old_size) * sizeof(Py_ssize_t)); memset(refcnts + old_size, 0, (size - old_size) * sizeof(Py_ssize_t));
} }
tstate->types.refcounts = refcnts; tstate->refcounts.values = refcnts;
tstate->types.size = size; tstate->refcounts.size = size;
return 0; return 0;
} }
void Py_ssize_t
_PyType_AssignId(PyHeapTypeObject *type) _PyObject_AssignUniqueId(PyObject *obj)
{ {
PyInterpreterState *interp = _PyInterpreterState_GET(); PyInterpreterState *interp = _PyInterpreterState_GET();
struct _Py_type_id_pool *pool = &interp->type_ids; struct _Py_unique_id_pool *pool = &interp->unique_ids;
LOCK_POOL(pool); LOCK_POOL(pool);
if (pool->freelist == NULL) { if (pool->freelist == NULL) {
if (resize_interp_type_id_pool(pool) < 0) { if (resize_interp_type_id_pool(pool) < 0) {
type->unique_id = -1;
UNLOCK_POOL(pool); UNLOCK_POOL(pool);
return; return -1;
} }
} }
_Py_type_id_entry *entry = pool->freelist; _Py_unique_id_entry *entry = pool->freelist;
pool->freelist = entry->next; pool->freelist = entry->next;
entry->type = type; entry->obj = obj;
_PyObject_SetDeferredRefcount((PyObject *)type); _PyObject_SetDeferredRefcount(obj);
type->unique_id = (entry - pool->table); Py_ssize_t unique_id = (entry - pool->table);
UNLOCK_POOL(pool); UNLOCK_POOL(pool);
return unique_id;
} }
void void
_PyType_ReleaseId(PyHeapTypeObject *type) _PyObject_ReleaseUniqueId(Py_ssize_t unique_id)
{ {
PyInterpreterState *interp = _PyInterpreterState_GET(); PyInterpreterState *interp = _PyInterpreterState_GET();
struct _Py_type_id_pool *pool = &interp->type_ids; struct _Py_unique_id_pool *pool = &interp->unique_ids;
if (type->unique_id < 0) { if (unique_id < 0) {
// The type doesn't have an id assigned. // The id is not assigned
return; return;
} }
LOCK_POOL(pool); LOCK_POOL(pool);
_Py_type_id_entry *entry = &pool->table[type->unique_id]; _Py_unique_id_entry *entry = &pool->table[unique_id];
assert(entry->type == type);
entry->next = pool->freelist; entry->next = pool->freelist;
pool->freelist = entry; pool->freelist = entry;
type->unique_id = -1;
UNLOCK_POOL(pool); UNLOCK_POOL(pool);
} }
@ -127,8 +126,8 @@ _PyType_IncrefSlow(PyHeapTypeObject *type)
return; return;
} }
assert(type->unique_id < tstate->types.size); assert(type->unique_id < tstate->refcounts.size);
tstate->types.refcounts[type->unique_id]++; tstate->refcounts.values[type->unique_id]++;
#ifdef Py_REF_DEBUG #ifdef Py_REF_DEBUG
_Py_IncRefTotal((PyThreadState *)tstate); _Py_IncRefTotal((PyThreadState *)tstate);
#endif #endif
@ -136,59 +135,64 @@ _PyType_IncrefSlow(PyHeapTypeObject *type)
} }
void void
_PyType_MergeThreadLocalRefcounts(_PyThreadStateImpl *tstate) _PyObject_MergePerThreadRefcounts(_PyThreadStateImpl *tstate)
{ {
if (tstate->types.refcounts == NULL) { if (tstate->refcounts.values == NULL) {
return; return;
} }
struct _Py_type_id_pool *pool = &tstate->base.interp->type_ids; struct _Py_unique_id_pool *pool = &tstate->base.interp->unique_ids;
LOCK_POOL(pool); LOCK_POOL(pool);
for (Py_ssize_t i = 0, n = tstate->types.size; i < n; i++) { for (Py_ssize_t i = 0, n = tstate->refcounts.size; i < n; i++) {
Py_ssize_t refcnt = tstate->types.refcounts[i]; Py_ssize_t refcnt = tstate->refcounts.values[i];
if (refcnt != 0) { if (refcnt != 0) {
PyObject *type = (PyObject *)pool->table[i].type; PyObject *obj = pool->table[i].obj;
assert(PyType_Check(type)); _Py_atomic_add_ssize(&obj->ob_ref_shared,
_Py_atomic_add_ssize(&type->ob_ref_shared,
refcnt << _Py_REF_SHARED_SHIFT); refcnt << _Py_REF_SHARED_SHIFT);
tstate->types.refcounts[i] = 0; tstate->refcounts.values[i] = 0;
} }
} }
UNLOCK_POOL(pool); UNLOCK_POOL(pool);
} }
void void
_PyType_FinalizeThreadLocalRefcounts(_PyThreadStateImpl *tstate) _PyObject_FinalizePerThreadRefcounts(_PyThreadStateImpl *tstate)
{ {
_PyType_MergeThreadLocalRefcounts(tstate); _PyObject_MergePerThreadRefcounts(tstate);
PyMem_Free(tstate->types.refcounts); PyMem_Free(tstate->refcounts.values);
tstate->types.refcounts = NULL; tstate->refcounts.values = NULL;
tstate->types.size = 0; tstate->refcounts.size = 0;
tstate->types.is_finalized = 1; tstate->refcounts.is_finalized = 1;
} }
void void
_PyType_FinalizeIdPool(PyInterpreterState *interp) _PyObject_FinalizeUniqueIdPool(PyInterpreterState *interp)
{ {
struct _Py_type_id_pool *pool = &interp->type_ids; struct _Py_unique_id_pool *pool = &interp->unique_ids;
// First, set the free-list to NULL values // First, set the free-list to NULL values
while (pool->freelist) { while (pool->freelist) {
_Py_type_id_entry *next = pool->freelist->next; _Py_unique_id_entry *next = pool->freelist->next;
pool->freelist->type = NULL; pool->freelist->obj = NULL;
pool->freelist = next; pool->freelist = next;
} }
// Now everything non-NULL is a type. Set the type's id to -1 in case it // Now everything non-NULL is a type. Set the type's id to -1 in case it
// outlives the interpreter. // outlives the interpreter.
for (Py_ssize_t i = 0; i < pool->size; i++) { for (Py_ssize_t i = 0; i < pool->size; i++) {
PyHeapTypeObject *ht = pool->table[i].type; PyObject *obj = pool->table[i].obj;
if (ht) { pool->table[i].obj = NULL;
ht->unique_id = -1; if (obj == NULL) {
pool->table[i].type = NULL; continue;
}
if (PyType_Check(obj)) {
assert(PyType_HasFeature((PyTypeObject *)obj, Py_TPFLAGS_HEAPTYPE));
((PyHeapTypeObject *)obj)->unique_id = -1;
}
else {
Py_UNREACHABLE();
} }
} }
PyMem_Free(pool->table); PyMem_Free(pool->table);