diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index 10c1f1e52be..2ecb40ba584 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -5,7 +5,6 @@ # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState #include "pycore_hamt.h" // PyHamtObject diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 56cc49432cc..a4bdf0d7ad8 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -8,7 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState #include "pycore_object.h" // PyManagedDictPointer #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_ACQUIRE diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index f984df69569..be1c6cc9772 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -8,7 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState #include "pycore_unicodeobject.h" // _PyUnicodeWriter /* runtime lifecycle */ diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h index e684e084b8b..1ac0fed584e 100644 --- a/Include/internal/pycore_freelist.h +++ b/Include/internal/pycore_freelist.h @@ -8,144 +8,109 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -// PyTuple_MAXSAVESIZE - largest tuple to save on free list -// PyTuple_MAXFREELIST - maximum number of tuples of each size to save +#include "pycore_freelist_state.h" // struct _Py_freelists +#include "pycore_object.h" // _PyObject_IS_GC +#include "pycore_pystate.h" // _PyThreadState_GET +#include "pycore_code.h" // OBJECT_STAT_INC -#ifdef WITH_FREELISTS -// with freelists -# define PyTuple_MAXSAVESIZE 20 -# define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE -# define PyTuple_MAXFREELIST 2000 -# define PyList_MAXFREELIST 80 -# define PyDict_MAXFREELIST 80 -# define PyFloat_MAXFREELIST 100 -# define PyContext_MAXFREELIST 255 -# define _PyAsyncGen_MAXFREELIST 80 -# define _PyObjectStackChunk_MAXFREELIST 4 +static inline struct _Py_freelists * +_Py_freelists_GET(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); +#ifdef Py_DEBUG + _Py_EnsureTstateNotNULL(tstate); +#endif + +#ifdef Py_GIL_DISABLED + return &((_PyThreadStateImpl*)tstate)->freelists; #else -# define PyTuple_NFREELISTS 0 -# define PyTuple_MAXFREELIST 0 -# define PyList_MAXFREELIST 0 -# define PyDict_MAXFREELIST 0 -# define PyFloat_MAXFREELIST 0 -# define PyContext_MAXFREELIST 0 -# define _PyAsyncGen_MAXFREELIST 0 -# define _PyObjectStackChunk_MAXFREELIST 0 + return &tstate->interp->object_state.freelists; #endif +} -struct _Py_list_freelist { -#ifdef WITH_FREELISTS - PyListObject *items[PyList_MAXFREELIST]; - int numfree; -#endif -}; - -struct _Py_tuple_freelist { -#if WITH_FREELISTS - /* There is one freelist for each size from 1 to PyTuple_MAXSAVESIZE. - The empty tuple is handled separately. - - Each tuple stored in the array is the head of the linked list - (and the next available tuple) for that size. The actual tuple - object is used as the linked list node, with its first item - (ob_item[0]) pointing to the next node (i.e. the previous head). - Each linked list is initially NULL. */ - PyTupleObject *items[PyTuple_NFREELISTS]; - int numfree[PyTuple_NFREELISTS]; +#ifndef WITH_FREELISTS +#define _Py_FREELIST_FREE(NAME, op, freefunc) freefunc(op) +#define _Py_FREELIST_PUSH(NAME, op, limit) (0) +#define _Py_FREELIST_POP(TYPE, NAME) (NULL) +#define _Py_FREELIST_POP_MEM(NAME) (NULL) +#define _Py_FREELIST_SIZE(NAME) (0) #else - char _unused; // Empty structs are not allowed. +// Pushes `op` to the freelist, calls `freefunc` if the freelist is full +#define _Py_FREELIST_FREE(NAME, op, freefunc) \ + _PyFreeList_Free(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), \ + Py_ ## NAME ## _MAXFREELIST, freefunc) +// Pushes `op` to the freelist, returns 1 if successful, 0 if the freelist is full +#define _Py_FREELIST_PUSH(NAME, op, limit) \ + _PyFreeList_Push(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), limit) + +// Pops a PyObject from the freelist, returns NULL if the freelist is empty. +#define _Py_FREELIST_POP(TYPE, NAME) \ + _Py_CAST(TYPE*, _PyFreeList_Pop(&_Py_freelists_GET()->NAME)) + +// Pops a non-PyObject data structure from the freelist, returns NULL if the +// freelist is empty. +#define _Py_FREELIST_POP_MEM(NAME) \ + _PyFreeList_PopMem(&_Py_freelists_GET()->NAME) + +#define _Py_FREELIST_SIZE(NAME) (int)((_Py_freelists_GET()->NAME).size) + +static inline int +_PyFreeList_Push(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize) +{ + if (fl->size < maxsize && fl->size >= 0) { + *(void **)obj = fl->freelist; + fl->freelist = obj; + fl->size++; + OBJECT_STAT_INC(to_freelist); + return 1; + } + return 0; +} + +static inline void +_PyFreeList_Free(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize, + freefunc dofree) +{ + if (!_PyFreeList_Push(fl, obj, maxsize)) { + dofree(obj); + } +} + +static inline void * +_PyFreeList_PopNoStats(struct _Py_freelist *fl) +{ + void *obj = fl->freelist; + if (obj != NULL) { + assert(fl->size > 0); + fl->freelist = *(void **)obj; + fl->size--; + } + return obj; +} + +static inline PyObject * +_PyFreeList_Pop(struct _Py_freelist *fl) +{ + PyObject *op = _PyFreeList_PopNoStats(fl); + if (op != NULL) { + OBJECT_STAT_INC(from_freelist); + _Py_NewReference(op); + } + return op; +} + +static inline void * +_PyFreeList_PopMem(struct _Py_freelist *fl) +{ + void *op = _PyFreeList_PopNoStats(fl); + if (op != NULL) { + OBJECT_STAT_INC(from_freelist); + } + return op; +} #endif -}; -struct _Py_float_freelist { -#ifdef WITH_FREELISTS - /* Special free list - free_list is a singly-linked list of available PyFloatObjects, - linked via abuse of their ob_type members. */ - int numfree; - PyFloatObject *items; -#endif -}; - -struct _Py_dict_freelist { -#ifdef WITH_FREELISTS - /* Dictionary reuse scheme to save calls to malloc and free */ - PyDictObject *items[PyDict_MAXFREELIST]; - int numfree; -#endif -}; - -struct _Py_dictkeys_freelist { -#ifdef WITH_FREELISTS - /* Dictionary keys reuse scheme to save calls to malloc and free */ - PyDictKeysObject *items[PyDict_MAXFREELIST]; - int numfree; -#endif -}; - -struct _Py_slice_freelist { -#ifdef WITH_FREELISTS - /* Using a cache is very effective since typically only a single slice is - created and then deleted again. */ - PySliceObject *slice_cache; -#endif -}; - -struct _Py_context_freelist { -#ifdef WITH_FREELISTS - // List of free PyContext objects - PyContext *items; - int numfree; -#endif -}; - -struct _Py_async_gen_freelist { -#ifdef WITH_FREELISTS - /* Freelists boost performance 6-10%; they also reduce memory - fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend - are short-living objects that are instantiated for every - __anext__() call. */ - struct _PyAsyncGenWrappedValue* items[_PyAsyncGen_MAXFREELIST]; - int numfree; -#endif -}; - -struct _Py_async_gen_asend_freelist { -#ifdef WITH_FREELISTS - struct PyAsyncGenASend* items[_PyAsyncGen_MAXFREELIST]; - int numfree; -#endif -}; - -struct _PyObjectStackChunk; - -struct _Py_object_stack_freelist { - struct _PyObjectStackChunk *items; - Py_ssize_t numfree; -}; - -struct _Py_object_freelists { - struct _Py_float_freelist floats; - struct _Py_tuple_freelist tuples; - struct _Py_list_freelist lists; - struct _Py_dict_freelist dicts; - struct _Py_dictkeys_freelist dictkeys; - struct _Py_slice_freelist slices; - struct _Py_context_freelist contexts; - struct _Py_async_gen_freelist async_gens; - struct _Py_async_gen_asend_freelist async_gen_asends; - struct _Py_object_stack_freelist object_stacks; -}; - -extern void _PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization); #ifdef __cplusplus } diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h new file mode 100644 index 00000000000..d4c3d600b61 --- /dev/null +++ b/Include/internal/pycore_freelist_state.h @@ -0,0 +1,59 @@ +#ifndef Py_INTERNAL_FREELIST_STATE_H +#define Py_INTERNAL_FREELIST_STATE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef WITH_FREELISTS +// with freelists +# define PyTuple_MAXSAVESIZE 20 // Largest tuple to save on freelist +# define Py_tuple_MAXFREELIST 2000 // Maximum number of tuples of each size to save +# define Py_lists_MAXFREELIST 80 +# define Py_dicts_MAXFREELIST 80 +# define Py_dictkeys_MAXFREELIST 80 +# define Py_floats_MAXFREELIST 100 +# define Py_slices_MAXFREELIST 1 +# define Py_contexts_MAXFREELIST 255 +# define Py_async_gens_MAXFREELIST 80 +# define Py_async_gen_asends_MAXFREELIST 80 +# define Py_object_stack_chunks_MAXFREELIST 4 +#else +# define PyTuple_MAXSAVESIZE 0 +#endif + +// A generic freelist of either PyObjects or other data structures. +struct _Py_freelist { + // Entries are linked together using the first word of the the object. + // For PyObjects, this overlaps with the `ob_refcnt` field or the `ob_tid` + // field. + void *freelist; + + // The number of items in the free list or -1 if the free list is disabled + Py_ssize_t size; +}; + +struct _Py_freelists { +#ifdef WITH_FREELISTS + struct _Py_freelist floats; + struct _Py_freelist tuples[PyTuple_MAXSAVESIZE]; + struct _Py_freelist lists; + struct _Py_freelist dicts; + struct _Py_freelist dictkeys; + struct _Py_freelist slices; + struct _Py_freelist contexts; + struct _Py_freelist async_gens; + struct _Py_freelist async_gen_asends; + struct _Py_freelist object_stack_chunks; +#else + char _unused; // Empty structs are not allowed. +#endif +}; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_FREELIST_STATE_H */ diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 28e34d38096..b4bf36e82e3 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -8,8 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState - /* GC information is stored BEFORE the object structure. */ typedef struct { // Pointer to next object in the list. diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 73695d10e0c..56ddb545d37 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -8,8 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState - PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); diff --git a/Include/internal/pycore_object_stack.h b/Include/internal/pycore_object_stack.h index fc130b1e992..c607ea8bc52 100644 --- a/Include/internal/pycore_object_stack.h +++ b/Include/internal/pycore_object_stack.h @@ -1,8 +1,6 @@ #ifndef Py_INTERNAL_OBJECT_STACK_H #define Py_INTERNAL_OBJECT_STACK_H -#include "pycore_freelist.h" // _PyFreeListState - #ifdef __cplusplus extern "C" { #endif diff --git a/Include/internal/pycore_object_state.h b/Include/internal/pycore_object_state.h index cd7c9335b3e..e7fa7c1f10d 100644 --- a/Include/internal/pycore_object_state.h +++ b/Include/internal/pycore_object_state.h @@ -8,8 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyObject_freelists -#include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_freelist_state.h" // _Py_freelists +#include "pycore_hashtable.h" // _Py_hashtable_t struct _py_object_runtime_state { #ifdef Py_REF_DEBUG @@ -20,7 +20,7 @@ struct _py_object_runtime_state { struct _py_object_state { #if !defined(Py_GIL_DISABLED) - struct _Py_object_freelists freelists; + struct _Py_freelists freelists; #endif #ifdef Py_REF_DEBUG Py_ssize_t reftotal; diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index b0e72523f58..fade55945b7 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -8,11 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState #include "pycore_runtime.h" // _PyRuntime #include "pycore_tstate.h" // _PyThreadStateImpl - // Values for PyThreadState.state. A thread must be in the "attached" state // before calling most Python APIs. If the GIL is enabled, then "attached" // implies that the thread holds the GIL and "detached" implies that the @@ -279,20 +277,6 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); // See also PyInterpreterState_Get() and _PyInterpreterState_GET(). extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void); -static inline struct _Py_object_freelists* _Py_object_freelists_GET(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); -#ifdef Py_DEBUG - _Py_EnsureTstateNotNULL(tstate); -#endif - -#ifdef Py_GIL_DISABLED - return &((_PyThreadStateImpl*)tstate)->freelists; -#else - return &tstate->interp->object_state.freelists; -#endif -} - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 1ed5b1d826a..18c972bd367 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -8,10 +8,10 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_brc.h" // struct _brc_thread_state -#include "pycore_freelist.h" // struct _Py_freelist_state -#include "pycore_mimalloc.h" // struct _mimalloc_thread_state -#include "pycore_qsbr.h" // struct qsbr +#include "pycore_brc.h" // struct _brc_thread_state +#include "pycore_freelist_state.h" // struct _Py_freelists +#include "pycore_mimalloc.h" // struct _mimalloc_thread_state +#include "pycore_qsbr.h" // struct qsbr // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The @@ -29,7 +29,7 @@ typedef struct _PyThreadStateImpl { #ifdef Py_GIL_DISABLED struct _gc_thread_state gc; struct _mimalloc_thread_state mimalloc; - struct _Py_object_freelists freelists; + struct _Py_freelists freelists; struct _brc_thread_state brc; #endif diff --git a/Makefile.pre.in b/Makefile.pre.in index 1a8f4d3ec7e..9ea7bc49be3 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1183,6 +1183,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_format.h \ $(srcdir)/Include/internal/pycore_frame.h \ $(srcdir)/Include/internal/pycore_freelist.h \ + $(srcdir)/Include/internal/pycore_freelist_state.h \ $(srcdir)/Include/internal/pycore_function.h \ $(srcdir)/Include/internal/pycore_gc.h \ $(srcdir)/Include/internal/pycore_genobject.h \ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index ca91da7771b..7310c3c8e13 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -395,44 +395,6 @@ static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj); #include "clinic/dictobject.c.h" -#ifdef WITH_FREELISTS -static struct _Py_dict_freelist * -get_dict_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - return &freelists->dicts; -} - -static struct _Py_dictkeys_freelist * -get_dictkeys_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - return &freelists->dictkeys; -} -#endif - - -void -_PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_dict_freelist *freelist = &freelists->dicts; - while (freelist->numfree > 0) { - PyDictObject *op = freelist->items[--freelist->numfree]; - assert(PyDict_CheckExact(op)); - PyObject_GC_Del(op); - } - struct _Py_dictkeys_freelist *keys_freelist = &freelists->dictkeys; - while (keys_freelist->numfree > 0) { - PyMem_Free(keys_freelist->items[--keys_freelist->numfree]); - } - if (is_finalization) { - freelist->numfree = -1; - keys_freelist->numfree = -1; - } -#endif -} - static inline Py_hash_t unicode_get_hash(PyObject *o) { @@ -445,12 +407,12 @@ void _PyDict_DebugMallocStats(FILE *out) { #ifdef WITH_FREELISTS - struct _Py_dict_freelist *dict_freelist = get_dict_freelist(); _PyDebugAllocatorStats(out, "free PyDictObject", - dict_freelist->numfree, sizeof(PyDictObject)); - struct _Py_dictkeys_freelist *dictkeys_freelist = get_dictkeys_freelist(); + _Py_FREELIST_SIZE(dicts), + sizeof(PyDictObject)); _PyDebugAllocatorStats(out, "free PyDictKeysObject", - dictkeys_freelist->numfree, sizeof(PyDictKeysObject)); + _Py_FREELIST_SIZE(dictkeys), + sizeof(PyDictKeysObject)); #endif } @@ -785,7 +747,6 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) static PyDictKeysObject* new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) { - PyDictKeysObject *dk; Py_ssize_t usable; int log2_bytes; size_t entry_size = unicode ? sizeof(PyDictUnicodeEntry) : sizeof(PyDictKeyEntry); @@ -808,15 +769,11 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) log2_bytes = log2_size + 2; } -#ifdef WITH_FREELISTS - struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist(); - if (log2_size == PyDict_LOG_MINSIZE && unicode && freelist->numfree > 0) { - dk = freelist->items[--freelist->numfree]; - OBJECT_STAT_INC(from_freelist); + PyDictKeysObject *dk = NULL; + if (log2_size == PyDict_LOG_MINSIZE && unicode) { + dk = _Py_FREELIST_POP_MEM(dictkeys); } - else -#endif - { + if (dk == NULL) { dk = PyMem_Malloc(sizeof(PyDictKeysObject) + ((size_t)1 << log2_bytes) + entry_size * usable); @@ -852,18 +809,12 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr) return; } #endif -#ifdef WITH_FREELISTS - struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist(); - if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE - && freelist->numfree < PyDict_MAXFREELIST - && freelist->numfree >= 0 - && DK_IS_UNICODE(keys)) { - freelist->items[freelist->numfree++] = keys; - OBJECT_STAT_INC(to_freelist); - return; + if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE && keys->dk_kind == DICT_KEYS_UNICODE) { + _Py_FREELIST_FREE(dictkeys, keys, PyMem_Free); + } + else { + PyMem_Free(keys); } -#endif - PyMem_Free(keys); } static size_t @@ -912,20 +863,9 @@ new_dict(PyInterpreterState *interp, PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free_values_on_failure) { - PyDictObject *mp; assert(keys != NULL); -#ifdef WITH_FREELISTS - struct _Py_dict_freelist *freelist = get_dict_freelist(); - if (freelist->numfree > 0) { - mp = freelist->items[--freelist->numfree]; - assert (mp != NULL); - assert (Py_IS_TYPE(mp, &PyDict_Type)); - OBJECT_STAT_INC(from_freelist); - _Py_NewReference((PyObject *)mp); - } - else -#endif - { + PyDictObject *mp = _Py_FREELIST_POP(PyDictObject, dicts); + if (mp == NULL) { mp = PyObject_GC_New(PyDictObject, &PyDict_Type); if (mp == NULL) { dictkeys_decref(interp, keys, false); @@ -935,6 +875,7 @@ new_dict(PyInterpreterState *interp, return NULL; } } + assert(Py_IS_TYPE(mp, &PyDict_Type)); mp->ma_keys = keys; mp->ma_values = values; mp->ma_used = used; @@ -3153,16 +3094,10 @@ dict_dealloc(PyObject *self) assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); dictkeys_decref(interp, keys, false); } -#ifdef WITH_FREELISTS - struct _Py_dict_freelist *freelist = get_dict_freelist(); - if (freelist->numfree < PyDict_MAXFREELIST && freelist->numfree >=0 && - Py_IS_TYPE(mp, &PyDict_Type)) { - freelist->items[freelist->numfree++] = mp; - OBJECT_STAT_INC(to_freelist); + if (Py_IS_TYPE(mp, &PyDict_Type)) { + _Py_FREELIST_FREE(dicts, mp, Py_TYPE(mp)->tp_free); } - else -#endif - { + else { Py_TYPE(mp)->tp_free((PyObject *)mp); } Py_TRASHCAN_END diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 08d8d854c38..82f39de421f 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -7,8 +7,8 @@ #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_dtoa.h" // _Py_dg_dtoa() #include "pycore_floatobject.h" // _PyFloat_FormatAdvancedWriter() +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_initconfig.h" // _PyStatus_OK() -#include "pycore_interp.h" // _Py_float_freelist #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_Init(), _PyDebugAllocatorStats() @@ -26,16 +26,6 @@ class float "PyObject *" "&PyFloat_Type" #include "clinic/floatobject.c.h" -#ifdef WITH_FREELISTS -static struct _Py_float_freelist * -get_float_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - assert(freelists != NULL); - return &freelists->floats; -} -#endif - double PyFloat_GetMax(void) @@ -132,24 +122,14 @@ PyFloat_GetInfo(void) PyObject * PyFloat_FromDouble(double fval) { - PyFloatObject *op; -#ifdef WITH_FREELISTS - struct _Py_float_freelist *float_freelist = get_float_freelist(); - op = float_freelist->items; - if (op != NULL) { - float_freelist->items = (PyFloatObject *) Py_TYPE(op); - float_freelist->numfree--; - OBJECT_STAT_INC(from_freelist); - } - else -#endif - { + PyFloatObject *op = _Py_FREELIST_POP(PyFloatObject, floats); + if (op == NULL) { op = PyObject_Malloc(sizeof(PyFloatObject)); if (!op) { return PyErr_NoMemory(); } + _PyObject_Init((PyObject*)op, &PyFloat_Type); } - _PyObject_Init((PyObject*)op, &PyFloat_Type); op->ob_fval = fval; return (PyObject *) op; } @@ -248,20 +228,7 @@ void _PyFloat_ExactDealloc(PyObject *obj) { assert(PyFloat_CheckExact(obj)); - PyFloatObject *op = (PyFloatObject *)obj; -#ifdef WITH_FREELISTS - struct _Py_float_freelist *float_freelist = get_float_freelist(); - if (float_freelist->numfree >= PyFloat_MAXFREELIST || float_freelist->numfree < 0) { - PyObject_Free(op); - return; - } - float_freelist->numfree++; - Py_SET_TYPE(op, (PyTypeObject *)float_freelist->items); - float_freelist->items = op; - OBJECT_STAT_INC(to_freelist); -#else - PyObject_Free(op); -#endif + _Py_FREELIST_FREE(floats, obj, PyObject_Free); } static void @@ -1994,27 +1961,6 @@ _PyFloat_InitTypes(PyInterpreterState *interp) return _PyStatus_OK(); } -void -_PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_float_freelist *state = &freelists->floats; - PyFloatObject *f = state->items; - while (f != NULL) { - PyFloatObject *next = (PyFloatObject*) Py_TYPE(f); - PyObject_Free(f); - f = next; - } - state->items = NULL; - if (is_finalization) { - state->numfree = -1; - } - else { - state->numfree = 0; - } -#endif -} - void _PyFloat_FiniType(PyInterpreterState *interp) { @@ -2026,10 +1972,10 @@ void _PyFloat_DebugMallocStats(FILE *out) { #ifdef WITH_FREELISTS - struct _Py_float_freelist *float_freelist = get_float_freelist(); _PyDebugAllocatorStats(out, "free PyFloatObject", - float_freelist->numfree, sizeof(PyFloatObject)); + _Py_FREELIST_SIZE(floats), + sizeof(PyFloatObject)); #endif } diff --git a/Objects/genobject.c b/Objects/genobject.c index 37b40530589..c204ac04b48 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -6,7 +6,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_EvalFrame() #include "pycore_frame.h" // _PyInterpreterFrame -#include "pycore_freelist.h" // struct _Py_async_gen_freelist +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_gc.h" // _PyGC_CLEAR_FINALIZED() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() @@ -1629,23 +1629,6 @@ PyTypeObject PyAsyncGen_Type = { }; -#ifdef WITH_FREELISTS -static struct _Py_async_gen_freelist * -get_async_gen_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - return &freelists->async_gens; -} - -static struct _Py_async_gen_asend_freelist * -get_async_gen_asend_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - return &freelists->async_gen_asends; -} -#endif - - PyObject * PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname) { @@ -1662,36 +1645,6 @@ PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname) return (PyObject*)o; } - -void -_PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelist_state, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_async_gen_freelist *freelist = &freelist_state->async_gens; - - while (freelist->numfree > 0) { - _PyAsyncGenWrappedValue *o; - o = freelist->items[--freelist->numfree]; - assert(_PyAsyncGenWrappedValue_CheckExact(o)); - PyObject_GC_Del(o); - } - - struct _Py_async_gen_asend_freelist *asend_freelist = &freelist_state->async_gen_asends; - - while (asend_freelist->numfree > 0) { - PyAsyncGenASend *o; - o = asend_freelist->items[--asend_freelist->numfree]; - assert(Py_IS_TYPE(o, &_PyAsyncGenASend_Type)); - PyObject_GC_Del(o); - } - - if (is_finalization) { - freelist->numfree = -1; - asend_freelist->numfree = -1; - } -#endif -} - static PyObject * async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result) { @@ -1735,18 +1688,11 @@ async_gen_asend_dealloc(PyAsyncGenASend *o) _PyObject_GC_UNTRACK((PyObject *)o); Py_CLEAR(o->ags_gen); Py_CLEAR(o->ags_sendval); -#ifdef WITH_FREELISTS - struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist(); - if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) { - assert(PyAsyncGenASend_CheckExact(o)); - _PyGC_CLEAR_FINALIZED((PyObject *)o); - freelist->items[freelist->numfree++] = o; - } - else -#endif - { - PyObject_GC_Del(o); - } + + assert(PyAsyncGenASend_CheckExact(o)); + _PyGC_CLEAR_FINALIZED((PyObject *)o); + + _Py_FREELIST_FREE(async_gen_asends, o, PyObject_GC_Del); } static int @@ -1936,17 +1882,8 @@ PyTypeObject _PyAsyncGenASend_Type = { static PyObject * async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval) { - PyAsyncGenASend *o; -#ifdef WITH_FREELISTS - struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist(); - if (freelist->numfree > 0) { - freelist->numfree--; - o = freelist->items[freelist->numfree]; - _Py_NewReference((PyObject *)o); - } - else -#endif - { + PyAsyncGenASend *o = _Py_FREELIST_POP(PyAsyncGenASend, async_gen_asends); + if (o == NULL) { o = PyObject_GC_New(PyAsyncGenASend, &_PyAsyncGenASend_Type); if (o == NULL) { return NULL; @@ -1972,18 +1909,7 @@ async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o) { _PyObject_GC_UNTRACK((PyObject *)o); Py_CLEAR(o->agw_val); -#ifdef WITH_FREELISTS - struct _Py_async_gen_freelist *freelist = get_async_gen_freelist(); - if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) { - assert(_PyAsyncGenWrappedValue_CheckExact(o)); - freelist->items[freelist->numfree++] = o; - OBJECT_STAT_INC(to_freelist); - } - else -#endif - { - PyObject_GC_Del(o); - } + _Py_FREELIST_FREE(async_gens, o, PyObject_GC_Del); } @@ -2042,27 +1968,17 @@ PyTypeObject _PyAsyncGenWrappedValue_Type = { PyObject * _PyAsyncGenValueWrapperNew(PyThreadState *tstate, PyObject *val) { - _PyAsyncGenWrappedValue *o; assert(val); -#ifdef WITH_FREELISTS - struct _Py_async_gen_freelist *freelist = get_async_gen_freelist(); - if (freelist->numfree > 0) { - freelist->numfree--; - o = freelist->items[freelist->numfree]; - OBJECT_STAT_INC(from_freelist); - assert(_PyAsyncGenWrappedValue_CheckExact(o)); - _Py_NewReference((PyObject*)o); - } - else -#endif - { + _PyAsyncGenWrappedValue *o = _Py_FREELIST_POP(_PyAsyncGenWrappedValue, async_gens); + if (o == NULL) { o = PyObject_GC_New(_PyAsyncGenWrappedValue, &_PyAsyncGenWrappedValue_Type); if (o == NULL) { return NULL; } } + assert(_PyAsyncGenWrappedValue_CheckExact(o)); o->agw_val = Py_NewRef(val); _PyObject_GC_TRACK((PyObject*)o); return (PyObject*)o; diff --git a/Objects/listobject.c b/Objects/listobject.c index f29f58dc25b..4d654c2ed33 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -4,6 +4,7 @@ #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_dict.h" // _PyDictViewObject +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_interp.h" // PyInterpreterState.list #include "pycore_list.h" // struct _Py_list_freelist, _PyListIterObject @@ -23,16 +24,6 @@ class list "PyListObject *" "&PyList_Type" _Py_DECLARE_STR(list_err, "list index out of range"); -#ifdef WITH_FREELISTS -static struct _Py_list_freelist * -get_list_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - assert(freelists != NULL); - return &freelists->lists; -} -#endif - #ifdef Py_GIL_DISABLED typedef struct { Py_ssize_t allocated; @@ -205,55 +196,28 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size) return 0; } -void -_PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_list_freelist *state = &freelists->lists; - while (state->numfree > 0) { - PyListObject *op = state->items[--state->numfree]; - assert(PyList_CheckExact(op)); - PyObject_GC_Del(op); - } - if (is_finalization) { - state->numfree = -1; - } -#endif -} - /* Print summary info about the state of the optimized allocator */ void _PyList_DebugMallocStats(FILE *out) { #ifdef WITH_FREELISTS - struct _Py_list_freelist *list_freelist = get_list_freelist(); _PyDebugAllocatorStats(out, "free PyListObject", - list_freelist->numfree, sizeof(PyListObject)); + _Py_FREELIST_SIZE(lists), + sizeof(PyListObject)); #endif } PyObject * PyList_New(Py_ssize_t size) { - PyListObject *op; - if (size < 0) { PyErr_BadInternalCall(); return NULL; } -#ifdef WITH_FREELISTS - struct _Py_list_freelist *list_freelist = get_list_freelist(); - if (PyList_MAXFREELIST && list_freelist->numfree > 0) { - list_freelist->numfree--; - op = list_freelist->items[list_freelist->numfree]; - OBJECT_STAT_INC(from_freelist); - _Py_NewReference((PyObject *)op); - } - else -#endif - { + PyListObject *op = _Py_FREELIST_POP(PyListObject, lists); + if (op == NULL) { op = PyObject_GC_New(PyListObject, &PyList_Type); if (op == NULL) { return NULL; @@ -548,16 +512,11 @@ list_dealloc(PyObject *self) } free_list_items(op->ob_item, false); } -#ifdef WITH_FREELISTS - struct _Py_list_freelist *list_freelist = get_list_freelist(); - if (list_freelist->numfree < PyList_MAXFREELIST && list_freelist->numfree >= 0 && PyList_CheckExact(op)) { - list_freelist->items[list_freelist->numfree++] = op; - OBJECT_STAT_INC(to_freelist); + if (PyList_CheckExact(op)) { + _Py_FREELIST_FREE(lists, op, PyObject_GC_Del); } - else -#endif - { - Py_TYPE(op)->tp_free((PyObject *)op); + else { + PyObject_GC_Del(op); } Py_TRASHCAN_END } diff --git a/Objects/object.c b/Objects/object.c index e2f96af5477..6d6bb87433c 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -10,6 +10,7 @@ #include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes() #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() +#include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() #include "pycore_instruction_sequence.h" // _PyInstructionSequence_Type #include "pycore_hashtable.h" // _Py_hashtable_new() @@ -808,20 +809,54 @@ PyObject_Bytes(PyObject *v) return PyBytes_FromObject(v); } -void -_PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization) +#ifdef WITH_FREELISTS +static void +clear_freelist(struct _Py_freelist *freelist, int is_finalization, + freefunc dofree) { + void *ptr; + while ((ptr = _PyFreeList_PopNoStats(freelist)) != NULL) { + dofree(ptr); + } + assert(freelist->size == 0 || freelist->size == -1); + assert(freelist->freelist == NULL); + if (is_finalization) { + freelist->size = -1; + } +} + +static void +free_object(void *obj) +{ + PyObject *op = (PyObject *)obj; + Py_TYPE(op)->tp_free(op); +} + +#endif + +void +_PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) +{ +#ifdef WITH_FREELISTS // In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear() // In the default build, freelists are per-interpreter and cleared in finalize_interp_types() - _PyFloat_ClearFreeList(freelists, is_finalization); - _PyTuple_ClearFreeList(freelists, is_finalization); - _PyList_ClearFreeList(freelists, is_finalization); - _PyDict_ClearFreeList(freelists, is_finalization); - _PyContext_ClearFreeList(freelists, is_finalization); - _PyAsyncGen_ClearFreeLists(freelists, is_finalization); - // Only be cleared if is_finalization is true. - _PyObjectStackChunk_ClearFreeList(freelists, is_finalization); - _PySlice_ClearFreeList(freelists, is_finalization); + clear_freelist(&freelists->floats, is_finalization, free_object); + for (Py_ssize_t i = 0; i < PyTuple_MAXSAVESIZE; i++) { + clear_freelist(&freelists->tuples[i], is_finalization, free_object); + } + clear_freelist(&freelists->lists, is_finalization, free_object); + clear_freelist(&freelists->dicts, is_finalization, free_object); + clear_freelist(&freelists->dictkeys, is_finalization, PyMem_Free); + clear_freelist(&freelists->slices, is_finalization, free_object); + clear_freelist(&freelists->contexts, is_finalization, free_object); + clear_freelist(&freelists->async_gens, is_finalization, free_object); + clear_freelist(&freelists->async_gen_asends, is_finalization, free_object); + if (is_finalization) { + // Only clear object stack chunks during finalization. We use object + // stacks during GC, so emptying the free-list is counterproductive. + clear_freelist(&freelists->object_stack_chunks, 1, PyMem_RawFree); + } +#endif } /* diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 245bea98d58..1b6d35998c2 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -15,6 +15,7 @@ this type and there is exactly one in existence. #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" // _PyObject_GC_TRACK() @@ -108,20 +109,6 @@ PyObject _Py_EllipsisObject = _PyObject_HEAD_INIT(&PyEllipsis_Type); /* Slice object implementation */ -void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ - if (!is_finalization) { - return; - } -#ifdef WITH_FREELISTS - PySliceObject *obj = freelists->slices.slice_cache; - if (obj != NULL) { - freelists->slices.slice_cache = NULL; - PyObject_GC_Del(obj); - } -#endif -} - /* start, stop, and step are python objects with None indicating no index is present. */ @@ -130,17 +117,8 @@ static PySliceObject * _PyBuildSlice_Consume2(PyObject *start, PyObject *stop, PyObject *step) { assert(start != NULL && stop != NULL && step != NULL); - PySliceObject *obj; -#ifdef WITH_FREELISTS - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - if (freelists->slices.slice_cache != NULL) { - obj = freelists->slices.slice_cache; - freelists->slices.slice_cache = NULL; - _Py_NewReference((PyObject *)obj); - } - else -#endif - { + PySliceObject *obj = _Py_FREELIST_POP(PySliceObject, slices); + if (obj == NULL) { obj = PyObject_GC_New(PySliceObject, &PySlice_Type); if (obj == NULL) { goto error; @@ -369,16 +347,7 @@ slice_dealloc(PySliceObject *r) Py_DECREF(r->step); Py_DECREF(r->start); Py_DECREF(r->stop); -#ifdef WITH_FREELISTS - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - if (freelists->slices.slice_cache == NULL) { - freelists->slices.slice_cache = r; - } - else -#endif - { - PyObject_GC_Del(r); - } + _Py_FREELIST_FREE(slices, r, PyObject_GC_Del); } static PyObject * diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 3704d095a97..bd6e5681911 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_freelist.h" // _Py_FREELIST_PUSH(), _Py_FREELIST_POP() #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_modsupport.h" // _PyArg_NoKwnames() @@ -17,7 +18,6 @@ class tuple "PyTupleObject *" "&PyTuple_Type" #include "clinic/tupleobject.c.h" -static inline PyTupleObject * maybe_freelist_pop(Py_ssize_t); static inline int maybe_freelist_push(PyTupleObject *); @@ -38,22 +38,20 @@ tuple_alloc(Py_ssize_t size) PyErr_BadInternalCall(); return NULL; } -#ifdef Py_DEBUG assert(size != 0); // The empty tuple is statically allocated. -#endif - - PyTupleObject *op = maybe_freelist_pop(size); - if (op == NULL) { - /* Check for overflow */ - if ((size_t)size > ((size_t)PY_SSIZE_T_MAX - (sizeof(PyTupleObject) - - sizeof(PyObject *))) / sizeof(PyObject *)) { - return (PyTupleObject *)PyErr_NoMemory(); + Py_ssize_t index = size - 1; + if (index < PyTuple_MAXSAVESIZE) { + PyTupleObject *op = _Py_FREELIST_POP(PyTupleObject, tuples[index]); + if (op != NULL) { + return op; } - op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size); - if (op == NULL) - return NULL; } - return op; + /* Check for overflow */ + if ((size_t)size > ((size_t)PY_SSIZE_T_MAX - (sizeof(PyTupleObject) - + sizeof(PyObject *))) / sizeof(PyObject *)) { + return (PyTupleObject *)PyErr_NoMemory(); + } + return PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size); } // The empty tuple singleton is not tracked by the GC. @@ -982,16 +980,6 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) return 0; } - -static void maybe_freelist_clear(struct _Py_object_freelists *, int); - - -void -_PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ - maybe_freelist_clear(freelists, is_finalization); -} - /*********************** Tuple Iterator **************************/ @@ -1141,102 +1129,31 @@ tuple_iter(PyObject *seq) * freelists * *************/ -#define TUPLE_FREELIST (freelists->tuples) -#define FREELIST_FINALIZED (TUPLE_FREELIST.numfree[0] < 0) - -static inline PyTupleObject * -maybe_freelist_pop(Py_ssize_t size) -{ -#ifdef WITH_FREELISTS - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - if (size == 0) { - return NULL; - } - assert(size > 0); - if (size <= PyTuple_MAXSAVESIZE) { - Py_ssize_t index = size - 1; - PyTupleObject *op = TUPLE_FREELIST.items[index]; - if (op != NULL) { - /* op is the head of a linked list, with the first item - pointing to the next node. Here we pop off the old head. */ - TUPLE_FREELIST.items[index] = (PyTupleObject *) op->ob_item[0]; - TUPLE_FREELIST.numfree[index]--; - /* Inlined _PyObject_InitVar() without _PyType_HasFeature() test */ -#ifdef Py_TRACE_REFS - /* maybe_freelist_push() ensures these were already set. */ - // XXX Can we drop these? See commit 68055ce6fe01 (GvR, Dec 1998). - Py_SET_SIZE(op, size); - Py_SET_TYPE(op, &PyTuple_Type); -#endif - _Py_NewReference((PyObject *)op); - /* END inlined _PyObject_InitVar() */ - OBJECT_STAT_INC(from_freelist); - return op; - } - } -#endif - return NULL; -} - static inline int maybe_freelist_push(PyTupleObject *op) { -#ifdef WITH_FREELISTS - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - if (Py_SIZE(op) == 0) { + if (!Py_IS_TYPE(op, &PyTuple_Type)) { return 0; } Py_ssize_t index = Py_SIZE(op) - 1; - if (index < PyTuple_NFREELISTS - && TUPLE_FREELIST.numfree[index] < PyTuple_MAXFREELIST - && TUPLE_FREELIST.numfree[index] >= 0 - && Py_IS_TYPE(op, &PyTuple_Type)) - { - /* op is the head of a linked list, with the first item - pointing to the next node. Here we set op as the new head. */ - op->ob_item[0] = (PyObject *) TUPLE_FREELIST.items[index]; - TUPLE_FREELIST.items[index] = op; - TUPLE_FREELIST.numfree[index]++; - OBJECT_STAT_INC(to_freelist); - return 1; + if (index < PyTuple_MAXSAVESIZE) { + return _Py_FREELIST_PUSH(tuples[index], op, Py_tuple_MAXFREELIST); } -#endif return 0; } -static void -maybe_freelist_clear(struct _Py_object_freelists *freelists, int fini) -{ -#ifdef WITH_FREELISTS - for (Py_ssize_t i = 0; i < PyTuple_NFREELISTS; i++) { - PyTupleObject *p = TUPLE_FREELIST.items[i]; - TUPLE_FREELIST.items[i] = NULL; - TUPLE_FREELIST.numfree[i] = fini ? -1 : 0; - while (p) { - PyTupleObject *q = p; - p = (PyTupleObject *)(p->ob_item[0]); - PyObject_GC_Del(q); - } - } -#endif -} - /* Print summary info about the state of the optimized allocator */ void _PyTuple_DebugMallocStats(FILE *out) { #ifdef WITH_FREELISTS - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - for (int i = 0; i < PyTuple_NFREELISTS; i++) { + for (int i = 0; i < PyTuple_MAXSAVESIZE; i++) { int len = i + 1; char buf[128]; PyOS_snprintf(buf, sizeof(buf), "free %d-sized PyTupleObject", len); - _PyDebugAllocatorStats(out, buf, TUPLE_FREELIST.numfree[i], + _PyDebugAllocatorStats(out, buf, _Py_FREELIST_SIZE(tuples[i]), _PyObject_VAR_SIZE(&PyTuple_Type, len)); } #endif } - -#undef STATE -#undef FREELIST_FINALIZED diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index f36fcb8caec..9e3af689f4a 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -239,6 +239,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index a1b43addf9e..31f7971bda8 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -639,6 +639,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/Python/context.c b/Python/context.c index c32c15f5562..5cafde4dab9 100644 --- a/Python/context.c +++ b/Python/context.c @@ -1,6 +1,7 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_VectorcallTstate() #include "pycore_context.h" +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_gc.h" // _PyObject_GC_MAY_BE_TRACKED() #include "pycore_hamt.h" #include "pycore_initconfig.h" // _PyStatus_OK() @@ -64,16 +65,6 @@ static int contextvar_del(PyContextVar *var); -#ifdef WITH_FREELISTS -static struct _Py_context_freelist * -get_context_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - return &freelists->contexts; -} -#endif - - PyObject * _PyContext_NewHamtForTests(void) { @@ -343,20 +334,8 @@ class _contextvars.Context "PyContext *" "&PyContext_Type" static inline PyContext * _context_alloc(void) { - PyContext *ctx; -#ifdef WITH_FREELISTS - struct _Py_context_freelist *context_freelist = get_context_freelist(); - if (context_freelist->numfree > 0) { - context_freelist->numfree--; - ctx = context_freelist->items; - context_freelist->items = (PyContext *)ctx->ctx_weakreflist; - OBJECT_STAT_INC(from_freelist); - ctx->ctx_weakreflist = NULL; - _Py_NewReference((PyObject *)ctx); - } - else -#endif - { + PyContext *ctx = _Py_FREELIST_POP(PyContext, contexts); + if (ctx == NULL) { ctx = PyObject_GC_New(PyContext, &PyContext_Type); if (ctx == NULL) { return NULL; @@ -471,19 +450,7 @@ context_tp_dealloc(PyContext *self) } (void)context_tp_clear(self); -#ifdef WITH_FREELISTS - struct _Py_context_freelist *context_freelist = get_context_freelist(); - if (context_freelist->numfree >= 0 && context_freelist->numfree < PyContext_MAXFREELIST) { - context_freelist->numfree++; - self->ctx_weakreflist = (PyObject *)context_freelist->items; - context_freelist->items = self; - OBJECT_STAT_INC(to_freelist); - } - else -#endif - { - Py_TYPE(self)->tp_free(self); - } + _Py_FREELIST_FREE(contexts, self, Py_TYPE(self)->tp_free); } static PyObject * @@ -1264,24 +1231,6 @@ get_token_missing(void) /////////////////////////// -void -_PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_context_freelist *state = &freelists->contexts; - for (; state->numfree > 0; state->numfree--) { - PyContext *ctx = state->items; - state->items = (PyContext *)ctx->ctx_weakreflist; - ctx->ctx_weakreflist = NULL; - PyObject_GC_Del(ctx); - } - if (is_finalization) { - state->numfree = -1; - } -#endif -} - - PyStatus _PyContext_Init(PyInterpreterState *interp) { diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index d1d5664ab96..53f04160c38 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -4,6 +4,7 @@ #include "pycore_ceval.h" // _Py_set_eval_breaker_bit() #include "pycore_context.h" #include "pycore_dict.h" // _PyDict_MaybeUntrack() +#include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" #include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_object.h" diff --git a/Python/gc_gil.c b/Python/gc_gil.c index 48646c7af86..7d62a9ed0b1 100644 --- a/Python/gc_gil.c +++ b/Python/gc_gil.c @@ -1,5 +1,5 @@ #include "Python.h" -#include "pycore_pystate.h" // _Py_ClearFreeLists() +#include "pycore_freelist.h" // _PyObject_ClearFreeLists() #ifndef Py_GIL_DISABLED diff --git a/Python/object_stack.c b/Python/object_stack.c index bd9696822c8..bda3190f45e 100644 --- a/Python/object_stack.c +++ b/Python/object_stack.c @@ -8,24 +8,11 @@ extern _PyObjectStackChunk *_PyObjectStackChunk_New(void); extern void _PyObjectStackChunk_Free(_PyObjectStackChunk *); -static struct _Py_object_stack_freelist * -get_object_stack_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - return &freelists->object_stacks; -} - _PyObjectStackChunk * _PyObjectStackChunk_New(void) { - _PyObjectStackChunk *buf; - struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist(); - if (obj_stack_freelist->numfree > 0) { - buf = obj_stack_freelist->items; - obj_stack_freelist->items = buf->prev; - obj_stack_freelist->numfree--; - } - else { + _PyObjectStackChunk *buf = _Py_FREELIST_POP_MEM(object_stack_chunks); + if (buf == NULL) { // NOTE: we use PyMem_RawMalloc() here because this is used by the GC // during mimalloc heap traversal. In that context, it is not safe to // allocate mimalloc memory, such as via PyMem_Malloc(). @@ -43,17 +30,7 @@ void _PyObjectStackChunk_Free(_PyObjectStackChunk *buf) { assert(buf->n == 0); - struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist(); - if (obj_stack_freelist->numfree >= 0 && - obj_stack_freelist->numfree < _PyObjectStackChunk_MAXFREELIST) - { - buf->prev = obj_stack_freelist->items; - obj_stack_freelist->items = buf; - obj_stack_freelist->numfree++; - } - else { - PyMem_RawFree(buf); - } + _Py_FREELIST_FREE(object_stack_chunks, buf, PyMem_RawFree); } void @@ -87,22 +64,3 @@ _PyObjectStack_Merge(_PyObjectStack *dst, _PyObjectStack *src) dst->head = src->head; src->head = NULL; } - -void -_PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ - if (!is_finalization) { - // Ignore requests to clear the free list during GC. We use object - // stacks during GC, so emptying the free-list is counterproductive. - return; - } - - struct _Py_object_stack_freelist *freelist = &freelists->object_stacks; - while (freelist->numfree > 0) { - _PyObjectStackChunk *buf = freelist->items; - freelist->items = buf->prev; - freelist->numfree--; - PyMem_RawFree(buf); - } - freelist->numfree = -1; -} diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 39eaa86098b..6b641c0775f 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -9,6 +9,7 @@ #include "pycore_dict.h" // _PyDict_Fini() #include "pycore_exceptions.h" // _PyExc_InitTypes() #include "pycore_fileutils.h" // _Py_ResetForceASCII() +#include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_floatobject.h" // _PyFloat_InitTypes() #include "pycore_global_objects_fini_generated.h" // "_PyStaticObjects_CheckRefcnt() #include "pycore_import.h" // _PyImport_BootstrapImp() @@ -1843,7 +1844,7 @@ finalize_interp_types(PyInterpreterState *interp) #ifndef Py_GIL_DISABLED // With Py_GIL_DISABLED: // the freelists for the current thread state have already been cleared. - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + struct _Py_freelists *freelists = _Py_freelists_GET(); _PyObject_ClearFreeLists(freelists, 1); #endif diff --git a/Python/pystate.c b/Python/pystate.c index f0452aa3cce..6fbd17f7eae 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -9,9 +9,9 @@ #include "pycore_dtoa.h" // _dtoa_state_INIT() #include "pycore_emscripten_trampoline.h" // _Py_EmscriptenTrampoline_Init() #include "pycore_frame.h" +#include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_object.h" // _PyType_InitCache() -#include "pycore_object_stack.h" // _PyObjectStackChunk_ClearFreeList() #include "pycore_parking_lot.h" // _PyParkingLot_AfterFork() #include "pycore_pyerrors.h" // _PyErr_Clear() #include "pycore_pylifecycle.h" // _PyAST_Fini() @@ -1738,7 +1738,7 @@ PyThreadState_Clear(PyThreadState *tstate) #ifdef Py_GIL_DISABLED // Each thread should clear own freelists in free-threading builds. - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + struct _Py_freelists *freelists = _Py_freelists_GET(); _PyObject_ClearFreeLists(freelists, 1); // Remove ourself from the biased reference counting table of threads.