gh-100240: Use a consistent implementation for freelists (#121934)

This combines and updates our freelist handling to use a consistent
implementation. Objects in the freelist are linked together using the
first word of memory block.

If configured with freelists disabled, these operations are essentially
no-ops.

Include/internal/pycore_context.h

@@ -5,7 +5,6 @@
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_freelist.h"      // _PyFreeListState
 #include "pycore_hamt.h"          // PyHamtObject
 
 

Include/internal/pycore_dict.h

@@ -8,7 +8,6 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_freelist.h"             // _PyFreeListState
 #include "pycore_object.h"               // PyManagedDictPointer
 #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_ACQUIRE
 

Include/internal/pycore_floatobject.h

@@ -8,7 +8,6 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_freelist.h"      // _PyFreeListState
 #include "pycore_unicodeobject.h" // _PyUnicodeWriter
 
 /* runtime lifecycle */

Include/internal/pycore_freelist.h

@@ -8,144 +8,109 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-// PyTuple_MAXSAVESIZE - largest tuple to save on free list
+#include "pycore_freelist_state.h"      // struct _Py_freelists
-// PyTuple_MAXFREELIST - maximum number of tuples of each size to save
+#include "pycore_object.h"              // _PyObject_IS_GC
+#include "pycore_pystate.h"             // _PyThreadState_GET
+#include "pycore_code.h"                // OBJECT_STAT_INC
 
-#ifdef WITH_FREELISTS
+static inline struct _Py_freelists *
-// with freelists
+_Py_freelists_GET(void)
-#  define PyTuple_MAXSAVESIZE 20
+{
-#  define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE
+    PyThreadState *tstate = _PyThreadState_GET();
-#  define PyTuple_MAXFREELIST 2000
+#ifdef Py_DEBUG
-#  define PyList_MAXFREELIST 80
+    _Py_EnsureTstateNotNULL(tstate);
-#  define PyDict_MAXFREELIST 80
+#endif
-#  define PyFloat_MAXFREELIST 100
+
-#  define PyContext_MAXFREELIST 255
+#ifdef Py_GIL_DISABLED
-# define _PyAsyncGen_MAXFREELIST 80
+    return &((_PyThreadStateImpl*)tstate)->freelists;
-#  define _PyObjectStackChunk_MAXFREELIST 4
 #else
-#  define PyTuple_NFREELISTS 0
+    return &tstate->interp->object_state.freelists;
-#  define PyTuple_MAXFREELIST 0
-#  define PyList_MAXFREELIST 0
-#  define PyDict_MAXFREELIST 0
-#  define PyFloat_MAXFREELIST 0
-#  define PyContext_MAXFREELIST 0
-#  define _PyAsyncGen_MAXFREELIST 0
-#  define _PyObjectStackChunk_MAXFREELIST 0
 #endif
+}
 
-struct _Py_list_freelist {
+#ifndef WITH_FREELISTS
-#ifdef WITH_FREELISTS
+#define _Py_FREELIST_FREE(NAME, op, freefunc) freefunc(op)
-    PyListObject *items[PyList_MAXFREELIST];
+#define _Py_FREELIST_PUSH(NAME, op, limit) (0)
-    int numfree;
+#define _Py_FREELIST_POP(TYPE, NAME) (NULL)
-#endif
+#define _Py_FREELIST_POP_MEM(NAME) (NULL)
-};
+#define _Py_FREELIST_SIZE(NAME) (0)
-
-struct _Py_tuple_freelist {
-#if WITH_FREELISTS
-    /* There is one freelist for each size from 1 to PyTuple_MAXSAVESIZE.
-       The empty tuple is handled separately.
-
-       Each tuple stored in the array is the head of the linked list
-       (and the next available tuple) for that size.  The actual tuple
-       object is used as the linked list node, with its first item
-       (ob_item[0]) pointing to the next node (i.e. the previous head).
-       Each linked list is initially NULL. */
-    PyTupleObject *items[PyTuple_NFREELISTS];
-    int numfree[PyTuple_NFREELISTS];
 #else
-    char _unused;  // Empty structs are not allowed.
+// Pushes `op` to the freelist, calls `freefunc` if the freelist is full
+#define _Py_FREELIST_FREE(NAME, op, freefunc) \
+    _PyFreeList_Free(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), \
+                     Py_ ## NAME ## _MAXFREELIST, freefunc)
+// Pushes `op` to the freelist, returns 1 if successful, 0 if the freelist is full
+#define _Py_FREELIST_PUSH(NAME, op, limit) \
+    _PyFreeList_Push(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), limit)
+
+// Pops a PyObject from the freelist, returns NULL if the freelist is empty.
+#define _Py_FREELIST_POP(TYPE, NAME) \
+    _Py_CAST(TYPE*, _PyFreeList_Pop(&_Py_freelists_GET()->NAME))
+
+// Pops a non-PyObject data structure from the freelist, returns NULL if the
+// freelist is empty.
+#define _Py_FREELIST_POP_MEM(NAME) \
+    _PyFreeList_PopMem(&_Py_freelists_GET()->NAME)
+
+#define _Py_FREELIST_SIZE(NAME) (int)((_Py_freelists_GET()->NAME).size)
+
+static inline int
+_PyFreeList_Push(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize)
+{
+    if (fl->size < maxsize && fl->size >= 0) {
+        *(void **)obj = fl->freelist;
+        fl->freelist = obj;
+        fl->size++;
+        OBJECT_STAT_INC(to_freelist);
+        return 1;
+    }
+    return 0;
+}
+
+static inline void
+_PyFreeList_Free(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize,
+                 freefunc dofree)
+{
+    if (!_PyFreeList_Push(fl, obj, maxsize)) {
+        dofree(obj);
+    }
+}
+
+static inline void *
+_PyFreeList_PopNoStats(struct _Py_freelist *fl)
+{
+    void *obj = fl->freelist;
+    if (obj != NULL) {
+        assert(fl->size > 0);
+        fl->freelist = *(void **)obj;
+        fl->size--;
+    }
+    return obj;
+}
+
+static inline PyObject *
+_PyFreeList_Pop(struct _Py_freelist *fl)
+{
+    PyObject *op = _PyFreeList_PopNoStats(fl);
+    if (op != NULL) {
+        OBJECT_STAT_INC(from_freelist);
+        _Py_NewReference(op);
+    }
+    return op;
+}
+
+static inline void *
+_PyFreeList_PopMem(struct _Py_freelist *fl)
+{
+    void *op = _PyFreeList_PopNoStats(fl);
+    if (op != NULL) {
+        OBJECT_STAT_INC(from_freelist);
+    }
+    return op;
+}
 #endif
-};
 
-struct _Py_float_freelist {
+extern void _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization);
-#ifdef WITH_FREELISTS
-    /* Special free list
-       free_list is a singly-linked list of available PyFloatObjects,
-       linked via abuse of their ob_type members. */
-    int numfree;
-    PyFloatObject *items;
-#endif
-};
-
-struct _Py_dict_freelist {
-#ifdef WITH_FREELISTS
-    /* Dictionary reuse scheme to save calls to malloc and free */
-    PyDictObject *items[PyDict_MAXFREELIST];
-    int numfree;
-#endif
-};
-
-struct _Py_dictkeys_freelist {
-#ifdef WITH_FREELISTS
-    /* Dictionary keys reuse scheme to save calls to malloc and free */
-    PyDictKeysObject *items[PyDict_MAXFREELIST];
-    int numfree;
-#endif
-};
-
-struct _Py_slice_freelist {
-#ifdef WITH_FREELISTS
-    /* Using a cache is very effective since typically only a single slice is
-       created and then deleted again. */
-    PySliceObject *slice_cache;
-#endif
-};
-
-struct _Py_context_freelist {
-#ifdef WITH_FREELISTS
-    // List of free PyContext objects
-    PyContext *items;
-    int numfree;
-#endif
-};
-
-struct _Py_async_gen_freelist {
-#ifdef WITH_FREELISTS
-    /* Freelists boost performance 6-10%; they also reduce memory
-       fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend
-       are short-living objects that are instantiated for every
-       __anext__() call. */
-    struct _PyAsyncGenWrappedValue* items[_PyAsyncGen_MAXFREELIST];
-    int numfree;
-#endif
-};
-
-struct _Py_async_gen_asend_freelist {
-#ifdef WITH_FREELISTS
-    struct PyAsyncGenASend* items[_PyAsyncGen_MAXFREELIST];
-    int numfree;
-#endif
-};
-
-struct _PyObjectStackChunk;
-
-struct _Py_object_stack_freelist {
-    struct _PyObjectStackChunk *items;
-    Py_ssize_t numfree;
-};
-
-struct _Py_object_freelists {
-    struct _Py_float_freelist floats;
-    struct _Py_tuple_freelist tuples;
-    struct _Py_list_freelist lists;
-    struct _Py_dict_freelist dicts;
-    struct _Py_dictkeys_freelist dictkeys;
-    struct _Py_slice_freelist slices;
-    struct _Py_context_freelist contexts;
-    struct _Py_async_gen_freelist async_gens;
-    struct _Py_async_gen_asend_freelist async_gen_asends;
-    struct _Py_object_stack_freelist object_stacks;
-};
-
-extern void _PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization);
-extern void _PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
-extern void _PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
-extern void _PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
-extern void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
-extern void _PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
-extern void _PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization);
-extern void _PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
-extern void _PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
 
 #ifdef __cplusplus
 }

Include/internal/pycore_freelist_state.h

@@ -0,0 +1,59 @@
+#ifndef Py_INTERNAL_FREELIST_STATE_H
+#define Py_INTERNAL_FREELIST_STATE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+#ifdef WITH_FREELISTS
+// with freelists
+#  define PyTuple_MAXSAVESIZE 20     // Largest tuple to save on freelist
+#  define Py_tuple_MAXFREELIST 2000  // Maximum number of tuples of each size to save
+#  define Py_lists_MAXFREELIST 80
+#  define Py_dicts_MAXFREELIST 80
+#  define Py_dictkeys_MAXFREELIST 80
+#  define Py_floats_MAXFREELIST 100
+#  define Py_slices_MAXFREELIST 1
+#  define Py_contexts_MAXFREELIST 255
+#  define Py_async_gens_MAXFREELIST 80
+#  define Py_async_gen_asends_MAXFREELIST 80
+#  define Py_object_stack_chunks_MAXFREELIST 4
+#else
+#  define PyTuple_MAXSAVESIZE 0
+#endif
+
+// A generic freelist of either PyObjects or other data structures.
+struct _Py_freelist {
+    // Entries are linked together using the first word of the the object.
+    // For PyObjects, this overlaps with the `ob_refcnt` field or the `ob_tid`
+    // field.
+    void *freelist;
+
+    // The number of items in the free list or -1 if the free list is disabled
+    Py_ssize_t size;
+};
+
+struct _Py_freelists {
+#ifdef WITH_FREELISTS
+    struct _Py_freelist floats;
+    struct _Py_freelist tuples[PyTuple_MAXSAVESIZE];
+    struct _Py_freelist lists;
+    struct _Py_freelist dicts;
+    struct _Py_freelist dictkeys;
+    struct _Py_freelist slices;
+    struct _Py_freelist contexts;
+    struct _Py_freelist async_gens;
+    struct _Py_freelist async_gen_asends;
+    struct _Py_freelist object_stack_chunks;
+#else
+    char _unused;  // Empty structs are not allowed.
+#endif
+};
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_FREELIST_STATE_H */

Include/internal/pycore_gc.h

@@ -8,8 +8,6 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_freelist.h"   // _PyFreeListState
-
 /* GC information is stored BEFORE the object structure. */
 typedef struct {
     // Pointer to next object in the list.

Include/internal/pycore_list.h

@@ -8,8 +8,6 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_freelist.h"  // _PyFreeListState
-
 PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *);
 extern void _PyList_DebugMallocStats(FILE *out);
 

Include/internal/pycore_object_stack.h

@@ -1,8 +1,6 @@
 #ifndef Py_INTERNAL_OBJECT_STACK_H
 #define Py_INTERNAL_OBJECT_STACK_H
 
-#include "pycore_freelist.h"        // _PyFreeListState
-
 #ifdef __cplusplus
 extern "C" {
 #endif

Include/internal/pycore_object_state.h

@@ -8,7 +8,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_freelist.h"      // _PyObject_freelists
+#include "pycore_freelist_state.h"  // _Py_freelists
 #include "pycore_hashtable.h"       // _Py_hashtable_t
 
 struct _py_object_runtime_state {
@@ -20,7 +20,7 @@ struct _py_object_runtime_state {
 
 struct _py_object_state {
 #if !defined(Py_GIL_DISABLED)
-    struct _Py_object_freelists freelists;
+    struct _Py_freelists freelists;
 #endif
 #ifdef Py_REF_DEBUG
     Py_ssize_t reftotal;

Include/internal/pycore_pystate.h

@@ -8,11 +8,9 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_freelist.h"      // _PyFreeListState
 #include "pycore_runtime.h"       // _PyRuntime
 #include "pycore_tstate.h"        // _PyThreadStateImpl
 
-
 // Values for PyThreadState.state. A thread must be in the "attached" state
 // before calling most Python APIs. If the GIL is enabled, then "attached"
 // implies that the thread holds the GIL and "detached" implies that the
@@ -279,20 +277,6 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void);
 // See also PyInterpreterState_Get() and _PyInterpreterState_GET().
 extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void);
 
-static inline struct _Py_object_freelists* _Py_object_freelists_GET(void)
-{
-    PyThreadState *tstate = _PyThreadState_GET();
-#ifdef Py_DEBUG
-    _Py_EnsureTstateNotNULL(tstate);
-#endif
-
-#ifdef Py_GIL_DISABLED
-    return &((_PyThreadStateImpl*)tstate)->freelists;
-#else
-    return &tstate->interp->object_state.freelists;
-#endif
-}
-
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_tstate.h

@@ -9,7 +9,7 @@ extern "C" {
 #endif
 
 #include "pycore_brc.h"             // struct _brc_thread_state
-#include "pycore_freelist.h"      // struct _Py_freelist_state
+#include "pycore_freelist_state.h"  // struct _Py_freelists
 #include "pycore_mimalloc.h"        // struct _mimalloc_thread_state
 #include "pycore_qsbr.h"            // struct qsbr
 
@@ -29,7 +29,7 @@ typedef struct _PyThreadStateImpl {
 #ifdef Py_GIL_DISABLED
     struct _gc_thread_state gc;
     struct _mimalloc_thread_state mimalloc;
-    struct _Py_object_freelists freelists;
+    struct _Py_freelists freelists;
     struct _brc_thread_state brc;
 #endif
 

Makefile.pre.in

@@ -1183,6 +1183,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_format.h \
 		$(srcdir)/Include/internal/pycore_frame.h \
 		$(srcdir)/Include/internal/pycore_freelist.h \
+		$(srcdir)/Include/internal/pycore_freelist_state.h \
 		$(srcdir)/Include/internal/pycore_function.h \
 		$(srcdir)/Include/internal/pycore_gc.h \
 		$(srcdir)/Include/internal/pycore_genobject.h \

Objects/dictobject.c

@@ -395,44 +395,6 @@ static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj);
 #include "clinic/dictobject.c.h"
 
 
-#ifdef WITH_FREELISTS
-static struct _Py_dict_freelist *
-get_dict_freelist(void)
-{
-    struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
-    return &freelists->dicts;
-}
-
-static struct _Py_dictkeys_freelist *
-get_dictkeys_freelist(void)
-{
-    struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
-    return &freelists->dictkeys;
-}
-#endif
-
-
-void
-_PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
-{
-#ifdef WITH_FREELISTS
-    struct _Py_dict_freelist *freelist = &freelists->dicts;
-    while (freelist->numfree > 0) {
-        PyDictObject *op = freelist->items[--freelist->numfree];
-        assert(PyDict_CheckExact(op));
-        PyObject_GC_Del(op);
-    }
-    struct _Py_dictkeys_freelist *keys_freelist = &freelists->dictkeys;
-    while (keys_freelist->numfree > 0) {
-        PyMem_Free(keys_freelist->items[--keys_freelist->numfree]);
-    }
-    if (is_finalization) {
-        freelist->numfree = -1;
-        keys_freelist->numfree = -1;
-    }
-#endif
-}
-
 static inline Py_hash_t
 unicode_get_hash(PyObject *o)
 {
@@ -445,12 +407,12 @@ void
 _PyDict_DebugMallocStats(FILE *out)
 {
 #ifdef WITH_FREELISTS
-    struct _Py_dict_freelist *dict_freelist = get_dict_freelist();
     _PyDebugAllocatorStats(out, "free PyDictObject",
-                           dict_freelist->numfree, sizeof(PyDictObject));
+                           _Py_FREELIST_SIZE(dicts),
-    struct _Py_dictkeys_freelist *dictkeys_freelist = get_dictkeys_freelist();
+                           sizeof(PyDictObject));
     _PyDebugAllocatorStats(out, "free PyDictKeysObject",
-                           dictkeys_freelist->numfree, sizeof(PyDictKeysObject));
+                           _Py_FREELIST_SIZE(dictkeys),
+                           sizeof(PyDictKeysObject));
 #endif
 }
 
@@ -785,7 +747,6 @@ _PyDict_CheckConsistency(PyObject *op, int check_content)
 static PyDictKeysObject*
 new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode)
 {
-    PyDictKeysObject *dk;
     Py_ssize_t usable;
     int log2_bytes;
     size_t entry_size = unicode ? sizeof(PyDictUnicodeEntry) : sizeof(PyDictKeyEntry);
@@ -808,15 +769,11 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode)
         log2_bytes = log2_size + 2;
     }
 
-#ifdef WITH_FREELISTS
+    PyDictKeysObject *dk = NULL;
-    struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist();
+    if (log2_size == PyDict_LOG_MINSIZE && unicode) {
-    if (log2_size == PyDict_LOG_MINSIZE && unicode && freelist->numfree > 0) {
+        dk = _Py_FREELIST_POP_MEM(dictkeys);
-        dk = freelist->items[--freelist->numfree];
-        OBJECT_STAT_INC(from_freelist);
     }
-    else
+    if (dk == NULL) {
-#endif
-    {
         dk = PyMem_Malloc(sizeof(PyDictKeysObject)
                           + ((size_t)1 << log2_bytes)
                           + entry_size * usable);
@@ -852,19 +809,13 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr)
         return;
     }
 #endif
-#ifdef WITH_FREELISTS
+    if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE && keys->dk_kind == DICT_KEYS_UNICODE) {
-    struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist();
+        _Py_FREELIST_FREE(dictkeys, keys, PyMem_Free);
-    if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE
-            && freelist->numfree < PyDict_MAXFREELIST
-            && freelist->numfree >= 0
-            && DK_IS_UNICODE(keys)) {
-        freelist->items[freelist->numfree++] = keys;
-        OBJECT_STAT_INC(to_freelist);
-        return;
     }
-#endif
+    else {
         PyMem_Free(keys);
     }
+}
 
 static size_t
 values_size_from_count(size_t count)
@@ -912,20 +863,9 @@ new_dict(PyInterpreterState *interp,
          PyDictKeysObject *keys, PyDictValues *values,
          Py_ssize_t used, int free_values_on_failure)
 {
-    PyDictObject *mp;
     assert(keys != NULL);
-#ifdef WITH_FREELISTS
+    PyDictObject *mp = _Py_FREELIST_POP(PyDictObject, dicts);
-    struct _Py_dict_freelist *freelist = get_dict_freelist();
+    if (mp == NULL) {
-    if (freelist->numfree > 0) {
-        mp = freelist->items[--freelist->numfree];
-        assert (mp != NULL);
-        assert (Py_IS_TYPE(mp, &PyDict_Type));
-        OBJECT_STAT_INC(from_freelist);
-        _Py_NewReference((PyObject *)mp);
-    }
-    else
-#endif
-    {
         mp = PyObject_GC_New(PyDictObject, &PyDict_Type);
         if (mp == NULL) {
             dictkeys_decref(interp, keys, false);
@@ -935,6 +875,7 @@ new_dict(PyInterpreterState *interp,
             return NULL;
         }
     }
+    assert(Py_IS_TYPE(mp, &PyDict_Type));
     mp->ma_keys = keys;
     mp->ma_values = values;
     mp->ma_used = used;
@@ -3153,16 +3094,10 @@ dict_dealloc(PyObject *self)
         assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS);
         dictkeys_decref(interp, keys, false);
     }
-#ifdef WITH_FREELISTS
+    if (Py_IS_TYPE(mp, &PyDict_Type)) {
-    struct _Py_dict_freelist *freelist = get_dict_freelist();
+        _Py_FREELIST_FREE(dicts, mp, Py_TYPE(mp)->tp_free);
-    if (freelist->numfree < PyDict_MAXFREELIST && freelist->numfree >=0 &&
-        Py_IS_TYPE(mp, &PyDict_Type)) {
-        freelist->items[freelist->numfree++] = mp;
-        OBJECT_STAT_INC(to_freelist);
     }
-    else
+    else {
-#endif
-    {
         Py_TYPE(mp)->tp_free((PyObject *)mp);
     }
     Py_TRASHCAN_END

Objects/floatobject.c

@@ -7,8 +7,8 @@
 #include "pycore_abstract.h"      // _PyNumber_Index()
 #include "pycore_dtoa.h"          // _Py_dg_dtoa()
 #include "pycore_floatobject.h"   // _PyFloat_FormatAdvancedWriter()
+#include "pycore_freelist.h"      // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
 #include "pycore_initconfig.h"    // _PyStatus_OK()
-#include "pycore_interp.h"        // _Py_float_freelist
 #include "pycore_long.h"          // _PyLong_GetOne()
 #include "pycore_modsupport.h"    // _PyArg_NoKwnames()
 #include "pycore_object.h"        // _PyObject_Init(), _PyDebugAllocatorStats()
@@ -26,16 +26,6 @@ class float "PyObject *" "&PyFloat_Type"
 
 #include "clinic/floatobject.c.h"
 
-#ifdef WITH_FREELISTS
-static struct _Py_float_freelist *
-get_float_freelist(void)
-{
-    struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
-    assert(freelists != NULL);
-    return &freelists->floats;
-}
-#endif
-
 
 double
 PyFloat_GetMax(void)
@@ -132,24 +122,14 @@ PyFloat_GetInfo(void)
 PyObject *
 PyFloat_FromDouble(double fval)
 {
-    PyFloatObject *op;
+    PyFloatObject *op = _Py_FREELIST_POP(PyFloatObject, floats);
-#ifdef WITH_FREELISTS
+    if (op == NULL) {
-    struct _Py_float_freelist *float_freelist = get_float_freelist();
-    op = float_freelist->items;
-    if (op != NULL) {
-        float_freelist->items = (PyFloatObject *) Py_TYPE(op);
-        float_freelist->numfree--;
-        OBJECT_STAT_INC(from_freelist);
-    }
-    else
-#endif
-    {
         op = PyObject_Malloc(sizeof(PyFloatObject));
         if (!op) {
             return PyErr_NoMemory();
         }
-    }
         _PyObject_Init((PyObject*)op, &PyFloat_Type);
+    }
     op->ob_fval = fval;
     return (PyObject *) op;
 }
@@ -248,20 +228,7 @@ void
 _PyFloat_ExactDealloc(PyObject *obj)
 {
     assert(PyFloat_CheckExact(obj));
-    PyFloatObject *op = (PyFloatObject *)obj;
+    _Py_FREELIST_FREE(floats, obj, PyObject_Free);
-#ifdef WITH_FREELISTS
-    struct _Py_float_freelist *float_freelist = get_float_freelist();
-    if (float_freelist->numfree >= PyFloat_MAXFREELIST || float_freelist->numfree < 0) {
-        PyObject_Free(op);
-        return;
-    }
-    float_freelist->numfree++;
-    Py_SET_TYPE(op, (PyTypeObject *)float_freelist->items);
-    float_freelist->items = op;
-    OBJECT_STAT_INC(to_freelist);
-#else
-    PyObject_Free(op);
-#endif
 }
 
 static void
@@ -1994,27 +1961,6 @@ _PyFloat_InitTypes(PyInterpreterState *interp)
     return _PyStatus_OK();
 }
 
-void
-_PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
-{
-#ifdef WITH_FREELISTS
-    struct _Py_float_freelist *state = &freelists->floats;
-    PyFloatObject *f = state->items;
-    while (f != NULL) {
-        PyFloatObject *next = (PyFloatObject*) Py_TYPE(f);
-        PyObject_Free(f);
-        f = next;
-    }
-    state->items = NULL;
-    if (is_finalization) {
-        state->numfree = -1;
-    }
-    else {
-        state->numfree = 0;
-    }
-#endif
-}
-
 void
 _PyFloat_FiniType(PyInterpreterState *interp)
 {
@@ -2026,10 +1972,10 @@ void
 _PyFloat_DebugMallocStats(FILE *out)
 {
 #ifdef WITH_FREELISTS
-    struct _Py_float_freelist *float_freelist = get_float_freelist();
     _PyDebugAllocatorStats(out,
                            "free PyFloatObject",
-                           float_freelist->numfree, sizeof(PyFloatObject));
+                           _Py_FREELIST_SIZE(floats),
+                           sizeof(PyFloatObject));
 #endif
 }
 

Objects/genobject.c

@@ -6,7 +6,7 @@
 #include "pycore_call.h"          // _PyObject_CallNoArgs()
 #include "pycore_ceval.h"         // _PyEval_EvalFrame()
 #include "pycore_frame.h"         // _PyInterpreterFrame
-#include "pycore_freelist.h"      // struct _Py_async_gen_freelist
+#include "pycore_freelist.h"      // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
 #include "pycore_gc.h"            // _PyGC_CLEAR_FINALIZED()
 #include "pycore_modsupport.h"    // _PyArg_CheckPositional()
 #include "pycore_object.h"        // _PyObject_GC_UNTRACK()
@@ -1629,23 +1629,6 @@ PyTypeObject PyAsyncGen_Type = {
 };
 
 
-#ifdef WITH_FREELISTS
-static struct _Py_async_gen_freelist *
-get_async_gen_freelist(void)
-{
-    struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
-    return &freelists->async_gens;
-}
-
-static struct _Py_async_gen_asend_freelist *
-get_async_gen_asend_freelist(void)
-{
-    struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
-    return &freelists->async_gen_asends;
-}
-#endif
-
-
 PyObject *
 PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
 {
@@ -1662,36 +1645,6 @@ PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
     return (PyObject*)o;
 }
 
-
-void
-_PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelist_state, int is_finalization)
-{
-#ifdef WITH_FREELISTS
-    struct _Py_async_gen_freelist *freelist = &freelist_state->async_gens;
-
-    while (freelist->numfree > 0) {
-        _PyAsyncGenWrappedValue *o;
-        o = freelist->items[--freelist->numfree];
-        assert(_PyAsyncGenWrappedValue_CheckExact(o));
-        PyObject_GC_Del(o);
-    }
-
-    struct _Py_async_gen_asend_freelist *asend_freelist = &freelist_state->async_gen_asends;
-
-    while (asend_freelist->numfree > 0) {
-        PyAsyncGenASend *o;
-        o = asend_freelist->items[--asend_freelist->numfree];
-        assert(Py_IS_TYPE(o, &_PyAsyncGenASend_Type));
-        PyObject_GC_Del(o);
-    }
-
-    if (is_finalization) {
-        freelist->numfree = -1;
-        asend_freelist->numfree = -1;
-    }
-#endif
-}
-
 static PyObject *
 async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result)
 {
@@ -1735,18 +1688,11 @@ async_gen_asend_dealloc(PyAsyncGenASend *o)
     _PyObject_GC_UNTRACK((PyObject *)o);
     Py_CLEAR(o->ags_gen);
     Py_CLEAR(o->ags_sendval);
-#ifdef WITH_FREELISTS
+
-    struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist();
-    if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) {
     assert(PyAsyncGenASend_CheckExact(o));
     _PyGC_CLEAR_FINALIZED((PyObject *)o);
-        freelist->items[freelist->numfree++] = o;
+
-    }
+    _Py_FREELIST_FREE(async_gen_asends, o, PyObject_GC_Del);
-    else
-#endif
-    {
-        PyObject_GC_Del(o);
-    }
 }
 
 static int
@@ -1936,17 +1882,8 @@ PyTypeObject _PyAsyncGenASend_Type = {
 static PyObject *
 async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval)
 {
-    PyAsyncGenASend *o;
+    PyAsyncGenASend *o = _Py_FREELIST_POP(PyAsyncGenASend, async_gen_asends);
-#ifdef WITH_FREELISTS
+    if (o == NULL) {
-    struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist();
-    if (freelist->numfree > 0) {
-        freelist->numfree--;
-        o = freelist->items[freelist->numfree];
-        _Py_NewReference((PyObject *)o);
-    }
-    else
-#endif
-    {
         o = PyObject_GC_New(PyAsyncGenASend, &_PyAsyncGenASend_Type);
         if (o == NULL) {
             return NULL;
@@ -1972,18 +1909,7 @@ async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o)
 {
     _PyObject_GC_UNTRACK((PyObject *)o);
     Py_CLEAR(o->agw_val);
-#ifdef WITH_FREELISTS
+    _Py_FREELIST_FREE(async_gens, o, PyObject_GC_Del);
-    struct _Py_async_gen_freelist *freelist = get_async_gen_freelist();
-    if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) {
-        assert(_PyAsyncGenWrappedValue_CheckExact(o));
-        freelist->items[freelist->numfree++] = o;
-        OBJECT_STAT_INC(to_freelist);
-    }
-    else
-#endif
-    {
-        PyObject_GC_Del(o);
-    }
 }
 
 
@@ -2042,27 +1968,17 @@ PyTypeObject _PyAsyncGenWrappedValue_Type = {
 PyObject *
 _PyAsyncGenValueWrapperNew(PyThreadState *tstate, PyObject *val)
 {
-    _PyAsyncGenWrappedValue *o;
     assert(val);
 
-#ifdef WITH_FREELISTS
+    _PyAsyncGenWrappedValue *o = _Py_FREELIST_POP(_PyAsyncGenWrappedValue, async_gens);
-    struct _Py_async_gen_freelist *freelist = get_async_gen_freelist();
+    if (o == NULL) {
-    if (freelist->numfree > 0) {
-        freelist->numfree--;
-        o = freelist->items[freelist->numfree];
-        OBJECT_STAT_INC(from_freelist);
-        assert(_PyAsyncGenWrappedValue_CheckExact(o));
-        _Py_NewReference((PyObject*)o);
-    }
-    else
-#endif
-    {
         o = PyObject_GC_New(_PyAsyncGenWrappedValue,
                             &_PyAsyncGenWrappedValue_Type);
         if (o == NULL) {
             return NULL;
         }
     }
+    assert(_PyAsyncGenWrappedValue_CheckExact(o));
     o->agw_val = Py_NewRef(val);
     _PyObject_GC_TRACK((PyObject*)o);
     return (PyObject*)o;

Objects/listobject.c

@@ -4,6 +4,7 @@
 #include "pycore_abstract.h"      // _PyIndex_Check()
 #include "pycore_ceval.h"         // _PyEval_GetBuiltin()
 #include "pycore_dict.h"          // _PyDictViewObject
+#include "pycore_freelist.h"      // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
 #include "pycore_pyatomic_ft_wrappers.h"
 #include "pycore_interp.h"        // PyInterpreterState.list
 #include "pycore_list.h"          // struct _Py_list_freelist, _PyListIterObject
@@ -23,16 +24,6 @@ class list "PyListObject *" "&PyList_Type"
 
 _Py_DECLARE_STR(list_err, "list index out of range");
 
-#ifdef WITH_FREELISTS
-static struct _Py_list_freelist *
-get_list_freelist(void)
-{
-    struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
-    assert(freelists != NULL);
-    return &freelists->lists;
-}
-#endif
-
 #ifdef Py_GIL_DISABLED
 typedef struct {
     Py_ssize_t allocated;
@@ -205,55 +196,28 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size)
     return 0;
 }
 
-void
-_PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
-{
-#ifdef WITH_FREELISTS
-    struct _Py_list_freelist *state = &freelists->lists;
-    while (state->numfree > 0) {
-        PyListObject *op = state->items[--state->numfree];
-        assert(PyList_CheckExact(op));
-        PyObject_GC_Del(op);
-    }
-    if (is_finalization) {
-        state->numfree = -1;
-    }
-#endif
-}
-
 /* Print summary info about the state of the optimized allocator */
 void
 _PyList_DebugMallocStats(FILE *out)
 {
 #ifdef WITH_FREELISTS
-    struct _Py_list_freelist *list_freelist = get_list_freelist();
     _PyDebugAllocatorStats(out,
                            "free PyListObject",
-                           list_freelist->numfree, sizeof(PyListObject));
+                            _Py_FREELIST_SIZE(lists),
+                           sizeof(PyListObject));
 #endif
 }
 
 PyObject *
 PyList_New(Py_ssize_t size)
 {
-    PyListObject *op;
-
     if (size < 0) {
         PyErr_BadInternalCall();
         return NULL;
     }
 
-#ifdef WITH_FREELISTS
+    PyListObject *op = _Py_FREELIST_POP(PyListObject, lists);
-    struct _Py_list_freelist *list_freelist = get_list_freelist();
+    if (op == NULL) {
-    if (PyList_MAXFREELIST && list_freelist->numfree > 0) {
-        list_freelist->numfree--;
-        op = list_freelist->items[list_freelist->numfree];
-        OBJECT_STAT_INC(from_freelist);
-        _Py_NewReference((PyObject *)op);
-    }
-    else
-#endif
-    {
         op = PyObject_GC_New(PyListObject, &PyList_Type);
         if (op == NULL) {
             return NULL;
@@ -548,16 +512,11 @@ list_dealloc(PyObject *self)
         }
         free_list_items(op->ob_item, false);
     }
-#ifdef WITH_FREELISTS
+    if (PyList_CheckExact(op)) {
-    struct _Py_list_freelist *list_freelist = get_list_freelist();
+        _Py_FREELIST_FREE(lists, op, PyObject_GC_Del);
-    if (list_freelist->numfree < PyList_MAXFREELIST && list_freelist->numfree >= 0 && PyList_CheckExact(op)) {
-        list_freelist->items[list_freelist->numfree++] = op;
-        OBJECT_STAT_INC(to_freelist);
     }
-    else
+    else {
-#endif
+        PyObject_GC_Del(op);
-    {
-        Py_TYPE(op)->tp_free((PyObject *)op);
     }
     Py_TRASHCAN_END
 }

Objects/object.c

@@ -10,6 +10,7 @@
 #include "pycore_descrobject.h"   // _PyMethodWrapper_Type
 #include "pycore_dict.h"          // _PyObject_MakeDictFromInstanceAttributes()
 #include "pycore_floatobject.h"   // _PyFloat_DebugMallocStats()
+#include "pycore_freelist.h"      // _PyObject_ClearFreeLists()
 #include "pycore_initconfig.h"    // _PyStatus_EXCEPTION()
 #include "pycore_instruction_sequence.h" // _PyInstructionSequence_Type
 #include "pycore_hashtable.h"     // _Py_hashtable_new()
@@ -808,20 +809,54 @@ PyObject_Bytes(PyObject *v)
     return PyBytes_FromObject(v);
 }
 
-void
+#ifdef WITH_FREELISTS
-_PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization)
+static void
+clear_freelist(struct _Py_freelist *freelist, int is_finalization,
+               freefunc dofree)
 {
+    void *ptr;
+    while ((ptr = _PyFreeList_PopNoStats(freelist)) != NULL) {
+        dofree(ptr);
+    }
+    assert(freelist->size == 0 || freelist->size == -1);
+    assert(freelist->freelist == NULL);
+    if (is_finalization) {
+        freelist->size = -1;
+    }
+}
+
+static void
+free_object(void *obj)
+{
+    PyObject *op = (PyObject *)obj;
+    Py_TYPE(op)->tp_free(op);
+}
+
+#endif
+
+void
+_PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization)
+{
+#ifdef WITH_FREELISTS
     // In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear()
     // In the default build, freelists are per-interpreter and cleared in finalize_interp_types()
-    _PyFloat_ClearFreeList(freelists, is_finalization);
+    clear_freelist(&freelists->floats, is_finalization, free_object);
-    _PyTuple_ClearFreeList(freelists, is_finalization);
+    for (Py_ssize_t i = 0; i < PyTuple_MAXSAVESIZE; i++) {
-    _PyList_ClearFreeList(freelists, is_finalization);
+        clear_freelist(&freelists->tuples[i], is_finalization, free_object);
-    _PyDict_ClearFreeList(freelists, is_finalization);
+    }
-    _PyContext_ClearFreeList(freelists, is_finalization);
+    clear_freelist(&freelists->lists, is_finalization, free_object);
-    _PyAsyncGen_ClearFreeLists(freelists, is_finalization);
+    clear_freelist(&freelists->dicts, is_finalization, free_object);
-    // Only be cleared if is_finalization is true.
+    clear_freelist(&freelists->dictkeys, is_finalization, PyMem_Free);
-    _PyObjectStackChunk_ClearFreeList(freelists, is_finalization);
+    clear_freelist(&freelists->slices, is_finalization, free_object);
-    _PySlice_ClearFreeList(freelists, is_finalization);
+    clear_freelist(&freelists->contexts, is_finalization, free_object);
+    clear_freelist(&freelists->async_gens, is_finalization, free_object);
+    clear_freelist(&freelists->async_gen_asends, is_finalization, free_object);
+    if (is_finalization) {
+        // Only clear object stack chunks during finalization. We use object
+        // stacks during GC, so emptying the free-list is counterproductive.
+        clear_freelist(&freelists->object_stack_chunks, 1, PyMem_RawFree);
+    }
+#endif
 }
 
 /*

Objects/sliceobject.c

@@ -15,6 +15,7 @@ this type and there is exactly one in existence.
 
 #include "Python.h"
 #include "pycore_abstract.h"      // _PyIndex_Check()
+#include "pycore_freelist.h"      // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
 #include "pycore_long.h"          // _PyLong_GetZero()
 #include "pycore_modsupport.h"    // _PyArg_NoKeywords()
 #include "pycore_object.h"        // _PyObject_GC_TRACK()
@@ -108,20 +109,6 @@ PyObject _Py_EllipsisObject = _PyObject_HEAD_INIT(&PyEllipsis_Type);
 
 /* Slice object implementation */
 
-void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
-{
-    if (!is_finalization) {
-        return;
-    }
-#ifdef WITH_FREELISTS
-    PySliceObject *obj = freelists->slices.slice_cache;
-    if (obj != NULL) {
-        freelists->slices.slice_cache = NULL;
-        PyObject_GC_Del(obj);
-    }
-#endif
-}
-
 /* start, stop, and step are python objects with None indicating no
    index is present.
 */
@@ -130,17 +117,8 @@ static PySliceObject *
 _PyBuildSlice_Consume2(PyObject *start, PyObject *stop, PyObject *step)
 {
     assert(start != NULL && stop != NULL && step != NULL);
-    PySliceObject *obj;
+    PySliceObject *obj = _Py_FREELIST_POP(PySliceObject, slices);
-#ifdef WITH_FREELISTS
+    if (obj == NULL) {
-    struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
-    if (freelists->slices.slice_cache != NULL) {
-        obj = freelists->slices.slice_cache;
-        freelists->slices.slice_cache = NULL;
-        _Py_NewReference((PyObject *)obj);
-    }
-    else
-#endif
-    {
         obj = PyObject_GC_New(PySliceObject, &PySlice_Type);
         if (obj == NULL) {
             goto error;
@@ -369,16 +347,7 @@ slice_dealloc(PySliceObject *r)
     Py_DECREF(r->step);
     Py_DECREF(r->start);
     Py_DECREF(r->stop);
-#ifdef WITH_FREELISTS
+    _Py_FREELIST_FREE(slices, r, PyObject_GC_Del);
-    struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
-    if (freelists->slices.slice_cache == NULL) {
-        freelists->slices.slice_cache = r;
-    }
-    else
-#endif
-    {
-        PyObject_GC_Del(r);
-    }
 }
 
 static PyObject *

Objects/tupleobject.c

@@ -4,6 +4,7 @@
 #include "Python.h"
 #include "pycore_abstract.h"      // _PyIndex_Check()
 #include "pycore_ceval.h"         // _PyEval_GetBuiltin()
+#include "pycore_freelist.h"      // _Py_FREELIST_PUSH(), _Py_FREELIST_POP()
 #include "pycore_gc.h"            // _PyObject_GC_IS_TRACKED()
 #include "pycore_initconfig.h"    // _PyStatus_OK()
 #include "pycore_modsupport.h"    // _PyArg_NoKwnames()
@@ -17,7 +18,6 @@ class tuple "PyTupleObject *" "&PyTuple_Type"
 #include "clinic/tupleobject.c.h"
 
 
-static inline PyTupleObject * maybe_freelist_pop(Py_ssize_t);
 static inline int maybe_freelist_push(PyTupleObject *);
 
 
@@ -38,22 +38,20 @@ tuple_alloc(Py_ssize_t size)
         PyErr_BadInternalCall();
         return NULL;
     }
-#ifdef Py_DEBUG
     assert(size != 0);    // The empty tuple is statically allocated.
-#endif
+    Py_ssize_t index = size - 1;
-
+    if (index < PyTuple_MAXSAVESIZE) {
-    PyTupleObject *op = maybe_freelist_pop(size);
+        PyTupleObject *op = _Py_FREELIST_POP(PyTupleObject, tuples[index]);
-    if (op == NULL) {
+        if (op != NULL) {
+            return op;
+        }
+    }
     /* Check for overflow */
     if ((size_t)size > ((size_t)PY_SSIZE_T_MAX - (sizeof(PyTupleObject) -
                 sizeof(PyObject *))) / sizeof(PyObject *)) {
         return (PyTupleObject *)PyErr_NoMemory();
     }
-        op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);
+    return PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);
-        if (op == NULL)
-            return NULL;
-    }
-    return op;
 }
 
 // The empty tuple singleton is not tracked by the GC.
@@ -982,16 +980,6 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize)
     return 0;
 }
 
-
-static void maybe_freelist_clear(struct _Py_object_freelists *, int);
-
-
-void
-_PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
-{
-    maybe_freelist_clear(freelists, is_finalization);
-}
-
 /*********************** Tuple Iterator **************************/
 
 
@@ -1141,102 +1129,31 @@ tuple_iter(PyObject *seq)
  * freelists *
  *************/
 
-#define TUPLE_FREELIST (freelists->tuples)
-#define FREELIST_FINALIZED (TUPLE_FREELIST.numfree[0] < 0)
-
-static inline PyTupleObject *
-maybe_freelist_pop(Py_ssize_t size)
-{
-#ifdef WITH_FREELISTS
-    struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
-    if (size == 0) {
-        return NULL;
-    }
-    assert(size > 0);
-    if (size <= PyTuple_MAXSAVESIZE) {
-        Py_ssize_t index = size - 1;
-        PyTupleObject *op = TUPLE_FREELIST.items[index];
-        if (op != NULL) {
-            /* op is the head of a linked list, with the first item
-               pointing to the next node.  Here we pop off the old head. */
-            TUPLE_FREELIST.items[index] = (PyTupleObject *) op->ob_item[0];
-            TUPLE_FREELIST.numfree[index]--;
-            /* Inlined _PyObject_InitVar() without _PyType_HasFeature() test */
-#ifdef Py_TRACE_REFS
-            /* maybe_freelist_push() ensures these were already set. */
-            // XXX Can we drop these?  See commit 68055ce6fe01 (GvR, Dec 1998).
-            Py_SET_SIZE(op, size);
-            Py_SET_TYPE(op, &PyTuple_Type);
-#endif
-            _Py_NewReference((PyObject *)op);
-            /* END inlined _PyObject_InitVar() */
-            OBJECT_STAT_INC(from_freelist);
-            return op;
-        }
-    }
-#endif
-    return NULL;
-}
-
 static inline int
 maybe_freelist_push(PyTupleObject *op)
 {
-#ifdef WITH_FREELISTS
+    if (!Py_IS_TYPE(op, &PyTuple_Type)) {
-    struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
-    if (Py_SIZE(op) == 0) {
         return 0;
     }
     Py_ssize_t index = Py_SIZE(op) - 1;
-    if (index < PyTuple_NFREELISTS
+    if (index < PyTuple_MAXSAVESIZE) {
-        && TUPLE_FREELIST.numfree[index] < PyTuple_MAXFREELIST
+        return _Py_FREELIST_PUSH(tuples[index], op, Py_tuple_MAXFREELIST);
-        && TUPLE_FREELIST.numfree[index] >= 0
-        && Py_IS_TYPE(op, &PyTuple_Type))
-    {
-        /* op is the head of a linked list, with the first item
-           pointing to the next node.  Here we set op as the new head. */
-        op->ob_item[0] = (PyObject *) TUPLE_FREELIST.items[index];
-        TUPLE_FREELIST.items[index] = op;
-        TUPLE_FREELIST.numfree[index]++;
-        OBJECT_STAT_INC(to_freelist);
-        return 1;
     }
-#endif
     return 0;
 }
 
-static void
-maybe_freelist_clear(struct _Py_object_freelists *freelists, int fini)
-{
-#ifdef WITH_FREELISTS
-    for (Py_ssize_t i = 0; i < PyTuple_NFREELISTS; i++) {
-        PyTupleObject *p = TUPLE_FREELIST.items[i];
-        TUPLE_FREELIST.items[i] = NULL;
-        TUPLE_FREELIST.numfree[i] = fini ? -1 : 0;
-        while (p) {
-            PyTupleObject *q = p;
-            p = (PyTupleObject *)(p->ob_item[0]);
-            PyObject_GC_Del(q);
-        }
-    }
-#endif
-}
-
 /* Print summary info about the state of the optimized allocator */
 void
 _PyTuple_DebugMallocStats(FILE *out)
 {
 #ifdef WITH_FREELISTS
-    struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
+    for (int i = 0; i < PyTuple_MAXSAVESIZE; i++) {
-    for (int i = 0; i < PyTuple_NFREELISTS; i++) {
         int len = i + 1;
         char buf[128];
         PyOS_snprintf(buf, sizeof(buf),
                       "free %d-sized PyTupleObject", len);
-        _PyDebugAllocatorStats(out, buf, TUPLE_FREELIST.numfree[i],
+        _PyDebugAllocatorStats(out, buf, _Py_FREELIST_SIZE(tuples[i]),
                                _PyObject_VAR_SIZE(&PyTuple_Type, len));
     }
 #endif
 }
-
-#undef STATE
-#undef FREELIST_FINALIZED

PCbuild/pythoncore.vcxproj

@@ -239,6 +239,7 @@
     <ClInclude Include="..\Include\internal\pycore_format.h" />
     <ClInclude Include="..\Include\internal\pycore_frame.h" />
     <ClInclude Include="..\Include\internal\pycore_freelist.h" />
+    <ClInclude Include="..\Include\internal\pycore_freelist_state.h" />
     <ClInclude Include="..\Include\internal\pycore_function.h" />
     <ClInclude Include="..\Include\internal\pycore_gc.h" />
     <ClInclude Include="..\Include\internal\pycore_genobject.h" />

PCbuild/pythoncore.vcxproj.filters

@@ -639,6 +639,9 @@
     <ClInclude Include="..\Include\internal\pycore_freelist.h">
       <Filter>Include\internal</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\internal\pycore_freelist_state.h">
+      <Filter>Include\internal</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\internal\pycore_function.h">
       <Filter>Include\internal</Filter>
     </ClInclude>

Python/context.c

@@ -1,6 +1,7 @@
 #include "Python.h"
 #include "pycore_call.h"          // _PyObject_VectorcallTstate()
 #include "pycore_context.h"
+#include "pycore_freelist.h"      // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
 #include "pycore_gc.h"            // _PyObject_GC_MAY_BE_TRACKED()
 #include "pycore_hamt.h"
 #include "pycore_initconfig.h"    // _PyStatus_OK()
@@ -64,16 +65,6 @@ static int
 contextvar_del(PyContextVar *var);
 
 
-#ifdef WITH_FREELISTS
-static struct _Py_context_freelist *
-get_context_freelist(void)
-{
-    struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
-    return &freelists->contexts;
-}
-#endif
-
-
 PyObject *
 _PyContext_NewHamtForTests(void)
 {
@@ -343,20 +334,8 @@ class _contextvars.Context "PyContext *" "&PyContext_Type"
 static inline PyContext *
 _context_alloc(void)
 {
-    PyContext *ctx;
+    PyContext *ctx = _Py_FREELIST_POP(PyContext, contexts);
-#ifdef WITH_FREELISTS
+    if (ctx == NULL) {
-    struct _Py_context_freelist *context_freelist = get_context_freelist();
-    if (context_freelist->numfree > 0) {
-        context_freelist->numfree--;
-        ctx = context_freelist->items;
-        context_freelist->items = (PyContext *)ctx->ctx_weakreflist;
-        OBJECT_STAT_INC(from_freelist);
-        ctx->ctx_weakreflist = NULL;
-        _Py_NewReference((PyObject *)ctx);
-    }
-    else
-#endif
-    {
         ctx = PyObject_GC_New(PyContext, &PyContext_Type);
         if (ctx == NULL) {
             return NULL;
@@ -471,19 +450,7 @@ context_tp_dealloc(PyContext *self)
     }
     (void)context_tp_clear(self);
 
-#ifdef WITH_FREELISTS
+    _Py_FREELIST_FREE(contexts, self, Py_TYPE(self)->tp_free);
-    struct _Py_context_freelist *context_freelist = get_context_freelist();
-    if (context_freelist->numfree >= 0 && context_freelist->numfree < PyContext_MAXFREELIST) {
-        context_freelist->numfree++;
-        self->ctx_weakreflist = (PyObject *)context_freelist->items;
-        context_freelist->items = self;
-        OBJECT_STAT_INC(to_freelist);
-    }
-    else
-#endif
-    {
-        Py_TYPE(self)->tp_free(self);
-    }
 }
 
 static PyObject *
@@ -1264,24 +1231,6 @@ get_token_missing(void)
 ///////////////////////////
 
 
-void
-_PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
-{
-#ifdef WITH_FREELISTS
-    struct _Py_context_freelist *state = &freelists->contexts;
-    for (; state->numfree > 0; state->numfree--) {
-        PyContext *ctx = state->items;
-        state->items = (PyContext *)ctx->ctx_weakreflist;
-        ctx->ctx_weakreflist = NULL;
-        PyObject_GC_Del(ctx);
-    }
-    if (is_finalization) {
-        state->numfree = -1;
-    }
-#endif
-}
-
-
 PyStatus
 _PyContext_Init(PyInterpreterState *interp)
 {

Python/gc_free_threading.c

@@ -4,6 +4,7 @@
 #include "pycore_ceval.h"         // _Py_set_eval_breaker_bit()
 #include "pycore_context.h"
 #include "pycore_dict.h"          // _PyDict_MaybeUntrack()
+#include "pycore_freelist.h"      // _PyObject_ClearFreeLists()
 #include "pycore_initconfig.h"
 #include "pycore_interp.h"        // PyInterpreterState.gc
 #include "pycore_object.h"

Python/gc_gil.c

@@ -1,5 +1,5 @@
 #include "Python.h"
-#include "pycore_pystate.h"   // _Py_ClearFreeLists()
+#include "pycore_freelist.h"   // _PyObject_ClearFreeLists()
 
 #ifndef Py_GIL_DISABLED
 

Python/object_stack.c

@@ -8,24 +8,11 @@
 extern _PyObjectStackChunk *_PyObjectStackChunk_New(void);
 extern void _PyObjectStackChunk_Free(_PyObjectStackChunk *);
 
-static struct _Py_object_stack_freelist *
-get_object_stack_freelist(void)
-{
-    struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
-    return &freelists->object_stacks;
-}
-
 _PyObjectStackChunk *
 _PyObjectStackChunk_New(void)
 {
-    _PyObjectStackChunk *buf;
+    _PyObjectStackChunk *buf = _Py_FREELIST_POP_MEM(object_stack_chunks);
-    struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist();
+    if (buf == NULL) {
-    if (obj_stack_freelist->numfree > 0) {
-        buf = obj_stack_freelist->items;
-        obj_stack_freelist->items = buf->prev;
-        obj_stack_freelist->numfree--;
-    }
-    else {
         // NOTE: we use PyMem_RawMalloc() here because this is used by the GC
         // during mimalloc heap traversal. In that context, it is not safe to
         // allocate mimalloc memory, such as via PyMem_Malloc().
@@ -43,17 +30,7 @@ void
 _PyObjectStackChunk_Free(_PyObjectStackChunk *buf)
 {
     assert(buf->n == 0);
-    struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist();
+    _Py_FREELIST_FREE(object_stack_chunks, buf, PyMem_RawFree);
-    if (obj_stack_freelist->numfree >= 0 &&
-        obj_stack_freelist->numfree < _PyObjectStackChunk_MAXFREELIST)
-    {
-        buf->prev = obj_stack_freelist->items;
-        obj_stack_freelist->items = buf;
-        obj_stack_freelist->numfree++;
-    }
-    else {
-        PyMem_RawFree(buf);
-    }
 }
 
 void
@@ -87,22 +64,3 @@ _PyObjectStack_Merge(_PyObjectStack *dst, _PyObjectStack *src)
     dst->head = src->head;
     src->head = NULL;
 }
-
-void
-_PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
-{
-    if (!is_finalization) {
-        // Ignore requests to clear the free list during GC. We use object
-        // stacks during GC, so emptying the free-list is counterproductive.
-        return;
-    }
-
-    struct _Py_object_stack_freelist *freelist = &freelists->object_stacks;
-    while (freelist->numfree > 0) {
-        _PyObjectStackChunk *buf = freelist->items;
-        freelist->items = buf->prev;
-        freelist->numfree--;
-        PyMem_RawFree(buf);
-    }
-    freelist->numfree = -1;
-}

Python/pylifecycle.c

@@ -9,6 +9,7 @@
 #include "pycore_dict.h"          // _PyDict_Fini()
 #include "pycore_exceptions.h"    // _PyExc_InitTypes()
 #include "pycore_fileutils.h"     // _Py_ResetForceASCII()
+#include "pycore_freelist.h"      // _PyObject_ClearFreeLists()
 #include "pycore_floatobject.h"   // _PyFloat_InitTypes()
 #include "pycore_global_objects_fini_generated.h"  // "_PyStaticObjects_CheckRefcnt()
 #include "pycore_import.h"        // _PyImport_BootstrapImp()
@@ -1843,7 +1844,7 @@ finalize_interp_types(PyInterpreterState *interp)
 #ifndef Py_GIL_DISABLED
     // With Py_GIL_DISABLED:
     // the freelists for the current thread state have already been cleared.
-    struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
+    struct _Py_freelists *freelists = _Py_freelists_GET();
     _PyObject_ClearFreeLists(freelists, 1);
 #endif
 

Python/pystate.c

@@ -9,9 +9,9 @@
 #include "pycore_dtoa.h"          // _dtoa_state_INIT()
 #include "pycore_emscripten_trampoline.h"  // _Py_EmscriptenTrampoline_Init()
 #include "pycore_frame.h"
+#include "pycore_freelist.h"      // _PyObject_ClearFreeLists()
 #include "pycore_initconfig.h"    // _PyStatus_OK()
 #include "pycore_object.h"        // _PyType_InitCache()
-#include "pycore_object_stack.h"  // _PyObjectStackChunk_ClearFreeList()
 #include "pycore_parking_lot.h"   // _PyParkingLot_AfterFork()
 #include "pycore_pyerrors.h"      // _PyErr_Clear()
 #include "pycore_pylifecycle.h"   // _PyAST_Fini()
@@ -1738,7 +1738,7 @@ PyThreadState_Clear(PyThreadState *tstate)
 
 #ifdef Py_GIL_DISABLED
     // Each thread should clear own freelists in free-threading builds.
-    struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
+    struct _Py_freelists *freelists = _Py_freelists_GET();
     _PyObject_ClearFreeLists(freelists, 1);
 
     // Remove ourself from the biased reference counting table of threads.