diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h
index 10c1f1e52be..2ecb40ba584 100644
--- a/Include/internal/pycore_context.h
+++ b/Include/internal/pycore_context.h
@@ -5,7 +5,6 @@
# error "this header requires Py_BUILD_CORE define"
#endif
-#include "pycore_freelist.h" // _PyFreeListState
#include "pycore_hamt.h" // PyHamtObject
diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h
index 56cc49432cc..a4bdf0d7ad8 100644
--- a/Include/internal/pycore_dict.h
+++ b/Include/internal/pycore_dict.h
@@ -8,7 +8,6 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
-#include "pycore_freelist.h" // _PyFreeListState
#include "pycore_object.h" // PyManagedDictPointer
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_ACQUIRE
diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h
index f984df69569..be1c6cc9772 100644
--- a/Include/internal/pycore_floatobject.h
+++ b/Include/internal/pycore_floatobject.h
@@ -8,7 +8,6 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
-#include "pycore_freelist.h" // _PyFreeListState
#include "pycore_unicodeobject.h" // _PyUnicodeWriter
/* runtime lifecycle */
diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h
index e684e084b8b..1ac0fed584e 100644
--- a/Include/internal/pycore_freelist.h
+++ b/Include/internal/pycore_freelist.h
@@ -8,144 +8,109 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
-// PyTuple_MAXSAVESIZE - largest tuple to save on free list
-// PyTuple_MAXFREELIST - maximum number of tuples of each size to save
+#include "pycore_freelist_state.h" // struct _Py_freelists
+#include "pycore_object.h" // _PyObject_IS_GC
+#include "pycore_pystate.h" // _PyThreadState_GET
+#include "pycore_code.h" // OBJECT_STAT_INC
-#ifdef WITH_FREELISTS
-// with freelists
-# define PyTuple_MAXSAVESIZE 20
-# define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE
-# define PyTuple_MAXFREELIST 2000
-# define PyList_MAXFREELIST 80
-# define PyDict_MAXFREELIST 80
-# define PyFloat_MAXFREELIST 100
-# define PyContext_MAXFREELIST 255
-# define _PyAsyncGen_MAXFREELIST 80
-# define _PyObjectStackChunk_MAXFREELIST 4
+static inline struct _Py_freelists *
+_Py_freelists_GET(void)
+{
+ PyThreadState *tstate = _PyThreadState_GET();
+#ifdef Py_DEBUG
+ _Py_EnsureTstateNotNULL(tstate);
+#endif
+
+#ifdef Py_GIL_DISABLED
+ return &((_PyThreadStateImpl*)tstate)->freelists;
#else
-# define PyTuple_NFREELISTS 0
-# define PyTuple_MAXFREELIST 0
-# define PyList_MAXFREELIST 0
-# define PyDict_MAXFREELIST 0
-# define PyFloat_MAXFREELIST 0
-# define PyContext_MAXFREELIST 0
-# define _PyAsyncGen_MAXFREELIST 0
-# define _PyObjectStackChunk_MAXFREELIST 0
+ return &tstate->interp->object_state.freelists;
#endif
+}
-struct _Py_list_freelist {
-#ifdef WITH_FREELISTS
- PyListObject *items[PyList_MAXFREELIST];
- int numfree;
-#endif
-};
-
-struct _Py_tuple_freelist {
-#if WITH_FREELISTS
- /* There is one freelist for each size from 1 to PyTuple_MAXSAVESIZE.
- The empty tuple is handled separately.
-
- Each tuple stored in the array is the head of the linked list
- (and the next available tuple) for that size. The actual tuple
- object is used as the linked list node, with its first item
- (ob_item[0]) pointing to the next node (i.e. the previous head).
- Each linked list is initially NULL. */
- PyTupleObject *items[PyTuple_NFREELISTS];
- int numfree[PyTuple_NFREELISTS];
+#ifndef WITH_FREELISTS
+#define _Py_FREELIST_FREE(NAME, op, freefunc) freefunc(op)
+#define _Py_FREELIST_PUSH(NAME, op, limit) (0)
+#define _Py_FREELIST_POP(TYPE, NAME) (NULL)
+#define _Py_FREELIST_POP_MEM(NAME) (NULL)
+#define _Py_FREELIST_SIZE(NAME) (0)
#else
- char _unused; // Empty structs are not allowed.
+// Pushes `op` to the freelist, calls `freefunc` if the freelist is full
+#define _Py_FREELIST_FREE(NAME, op, freefunc) \
+ _PyFreeList_Free(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), \
+ Py_ ## NAME ## _MAXFREELIST, freefunc)
+// Pushes `op` to the freelist, returns 1 if successful, 0 if the freelist is full
+#define _Py_FREELIST_PUSH(NAME, op, limit) \
+ _PyFreeList_Push(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), limit)
+
+// Pops a PyObject from the freelist, returns NULL if the freelist is empty.
+#define _Py_FREELIST_POP(TYPE, NAME) \
+ _Py_CAST(TYPE*, _PyFreeList_Pop(&_Py_freelists_GET()->NAME))
+
+// Pops a non-PyObject data structure from the freelist, returns NULL if the
+// freelist is empty.
+#define _Py_FREELIST_POP_MEM(NAME) \
+ _PyFreeList_PopMem(&_Py_freelists_GET()->NAME)
+
+#define _Py_FREELIST_SIZE(NAME) (int)((_Py_freelists_GET()->NAME).size)
+
+static inline int
+_PyFreeList_Push(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize)
+{
+ if (fl->size < maxsize && fl->size >= 0) {
+ *(void **)obj = fl->freelist;
+ fl->freelist = obj;
+ fl->size++;
+ OBJECT_STAT_INC(to_freelist);
+ return 1;
+ }
+ return 0;
+}
+
+static inline void
+_PyFreeList_Free(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize,
+ freefunc dofree)
+{
+ if (!_PyFreeList_Push(fl, obj, maxsize)) {
+ dofree(obj);
+ }
+}
+
+static inline void *
+_PyFreeList_PopNoStats(struct _Py_freelist *fl)
+{
+ void *obj = fl->freelist;
+ if (obj != NULL) {
+ assert(fl->size > 0);
+ fl->freelist = *(void **)obj;
+ fl->size--;
+ }
+ return obj;
+}
+
+static inline PyObject *
+_PyFreeList_Pop(struct _Py_freelist *fl)
+{
+ PyObject *op = _PyFreeList_PopNoStats(fl);
+ if (op != NULL) {
+ OBJECT_STAT_INC(from_freelist);
+ _Py_NewReference(op);
+ }
+ return op;
+}
+
+static inline void *
+_PyFreeList_PopMem(struct _Py_freelist *fl)
+{
+ void *op = _PyFreeList_PopNoStats(fl);
+ if (op != NULL) {
+ OBJECT_STAT_INC(from_freelist);
+ }
+ return op;
+}
#endif
-};
-struct _Py_float_freelist {
-#ifdef WITH_FREELISTS
- /* Special free list
- free_list is a singly-linked list of available PyFloatObjects,
- linked via abuse of their ob_type members. */
- int numfree;
- PyFloatObject *items;
-#endif
-};
-
-struct _Py_dict_freelist {
-#ifdef WITH_FREELISTS
- /* Dictionary reuse scheme to save calls to malloc and free */
- PyDictObject *items[PyDict_MAXFREELIST];
- int numfree;
-#endif
-};
-
-struct _Py_dictkeys_freelist {
-#ifdef WITH_FREELISTS
- /* Dictionary keys reuse scheme to save calls to malloc and free */
- PyDictKeysObject *items[PyDict_MAXFREELIST];
- int numfree;
-#endif
-};
-
-struct _Py_slice_freelist {
-#ifdef WITH_FREELISTS
- /* Using a cache is very effective since typically only a single slice is
- created and then deleted again. */
- PySliceObject *slice_cache;
-#endif
-};
-
-struct _Py_context_freelist {
-#ifdef WITH_FREELISTS
- // List of free PyContext objects
- PyContext *items;
- int numfree;
-#endif
-};
-
-struct _Py_async_gen_freelist {
-#ifdef WITH_FREELISTS
- /* Freelists boost performance 6-10%; they also reduce memory
- fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend
- are short-living objects that are instantiated for every
- __anext__() call. */
- struct _PyAsyncGenWrappedValue* items[_PyAsyncGen_MAXFREELIST];
- int numfree;
-#endif
-};
-
-struct _Py_async_gen_asend_freelist {
-#ifdef WITH_FREELISTS
- struct PyAsyncGenASend* items[_PyAsyncGen_MAXFREELIST];
- int numfree;
-#endif
-};
-
-struct _PyObjectStackChunk;
-
-struct _Py_object_stack_freelist {
- struct _PyObjectStackChunk *items;
- Py_ssize_t numfree;
-};
-
-struct _Py_object_freelists {
- struct _Py_float_freelist floats;
- struct _Py_tuple_freelist tuples;
- struct _Py_list_freelist lists;
- struct _Py_dict_freelist dicts;
- struct _Py_dictkeys_freelist dictkeys;
- struct _Py_slice_freelist slices;
- struct _Py_context_freelist contexts;
- struct _Py_async_gen_freelist async_gens;
- struct _Py_async_gen_asend_freelist async_gen_asends;
- struct _Py_object_stack_freelist object_stacks;
-};
-
-extern void _PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization);
-extern void _PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
-extern void _PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
-extern void _PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
-extern void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
-extern void _PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
-extern void _PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization);
-extern void _PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
-extern void _PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
+extern void _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization);
#ifdef __cplusplus
}
diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h
new file mode 100644
index 00000000000..d4c3d600b61
--- /dev/null
+++ b/Include/internal/pycore_freelist_state.h
@@ -0,0 +1,59 @@
+#ifndef Py_INTERNAL_FREELIST_STATE_H
+#define Py_INTERNAL_FREELIST_STATE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+# error "this header requires Py_BUILD_CORE define"
+#endif
+
+#ifdef WITH_FREELISTS
+// with freelists
+# define PyTuple_MAXSAVESIZE 20 // Largest tuple to save on freelist
+# define Py_tuple_MAXFREELIST 2000 // Maximum number of tuples of each size to save
+# define Py_lists_MAXFREELIST 80
+# define Py_dicts_MAXFREELIST 80
+# define Py_dictkeys_MAXFREELIST 80
+# define Py_floats_MAXFREELIST 100
+# define Py_slices_MAXFREELIST 1
+# define Py_contexts_MAXFREELIST 255
+# define Py_async_gens_MAXFREELIST 80
+# define Py_async_gen_asends_MAXFREELIST 80
+# define Py_object_stack_chunks_MAXFREELIST 4
+#else
+# define PyTuple_MAXSAVESIZE 0
+#endif
+
+// A generic freelist of either PyObjects or other data structures.
+struct _Py_freelist {
+ // Entries are linked together using the first word of the the object.
+ // For PyObjects, this overlaps with the `ob_refcnt` field or the `ob_tid`
+ // field.
+ void *freelist;
+
+ // The number of items in the free list or -1 if the free list is disabled
+ Py_ssize_t size;
+};
+
+struct _Py_freelists {
+#ifdef WITH_FREELISTS
+ struct _Py_freelist floats;
+ struct _Py_freelist tuples[PyTuple_MAXSAVESIZE];
+ struct _Py_freelist lists;
+ struct _Py_freelist dicts;
+ struct _Py_freelist dictkeys;
+ struct _Py_freelist slices;
+ struct _Py_freelist contexts;
+ struct _Py_freelist async_gens;
+ struct _Py_freelist async_gen_asends;
+ struct _Py_freelist object_stack_chunks;
+#else
+ char _unused; // Empty structs are not allowed.
+#endif
+};
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_FREELIST_STATE_H */
diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h
index 28e34d38096..b4bf36e82e3 100644
--- a/Include/internal/pycore_gc.h
+++ b/Include/internal/pycore_gc.h
@@ -8,8 +8,6 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
-#include "pycore_freelist.h" // _PyFreeListState
-
/* GC information is stored BEFORE the object structure. */
typedef struct {
// Pointer to next object in the list.
diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h
index 73695d10e0c..56ddb545d37 100644
--- a/Include/internal/pycore_list.h
+++ b/Include/internal/pycore_list.h
@@ -8,8 +8,6 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
-#include "pycore_freelist.h" // _PyFreeListState
-
PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *);
extern void _PyList_DebugMallocStats(FILE *out);
diff --git a/Include/internal/pycore_object_stack.h b/Include/internal/pycore_object_stack.h
index fc130b1e992..c607ea8bc52 100644
--- a/Include/internal/pycore_object_stack.h
+++ b/Include/internal/pycore_object_stack.h
@@ -1,8 +1,6 @@
#ifndef Py_INTERNAL_OBJECT_STACK_H
#define Py_INTERNAL_OBJECT_STACK_H
-#include "pycore_freelist.h" // _PyFreeListState
-
#ifdef __cplusplus
extern "C" {
#endif
diff --git a/Include/internal/pycore_object_state.h b/Include/internal/pycore_object_state.h
index cd7c9335b3e..e7fa7c1f10d 100644
--- a/Include/internal/pycore_object_state.h
+++ b/Include/internal/pycore_object_state.h
@@ -8,8 +8,8 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
-#include "pycore_freelist.h" // _PyObject_freelists
-#include "pycore_hashtable.h" // _Py_hashtable_t
+#include "pycore_freelist_state.h" // _Py_freelists
+#include "pycore_hashtable.h" // _Py_hashtable_t
struct _py_object_runtime_state {
#ifdef Py_REF_DEBUG
@@ -20,7 +20,7 @@ struct _py_object_runtime_state {
struct _py_object_state {
#if !defined(Py_GIL_DISABLED)
- struct _Py_object_freelists freelists;
+ struct _Py_freelists freelists;
#endif
#ifdef Py_REF_DEBUG
Py_ssize_t reftotal;
diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h
index b0e72523f58..fade55945b7 100644
--- a/Include/internal/pycore_pystate.h
+++ b/Include/internal/pycore_pystate.h
@@ -8,11 +8,9 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
-#include "pycore_freelist.h" // _PyFreeListState
#include "pycore_runtime.h" // _PyRuntime
#include "pycore_tstate.h" // _PyThreadStateImpl
-
// Values for PyThreadState.state. A thread must be in the "attached" state
// before calling most Python APIs. If the GIL is enabled, then "attached"
// implies that the thread holds the GIL and "detached" implies that the
@@ -279,20 +277,6 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void);
// See also PyInterpreterState_Get() and _PyInterpreterState_GET().
extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void);
-static inline struct _Py_object_freelists* _Py_object_freelists_GET(void)
-{
- PyThreadState *tstate = _PyThreadState_GET();
-#ifdef Py_DEBUG
- _Py_EnsureTstateNotNULL(tstate);
-#endif
-
-#ifdef Py_GIL_DISABLED
- return &((_PyThreadStateImpl*)tstate)->freelists;
-#else
- return &tstate->interp->object_state.freelists;
-#endif
-}
-
#ifdef __cplusplus
}
#endif
diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h
index 1ed5b1d826a..18c972bd367 100644
--- a/Include/internal/pycore_tstate.h
+++ b/Include/internal/pycore_tstate.h
@@ -8,10 +8,10 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
-#include "pycore_brc.h" // struct _brc_thread_state
-#include "pycore_freelist.h" // struct _Py_freelist_state
-#include "pycore_mimalloc.h" // struct _mimalloc_thread_state
-#include "pycore_qsbr.h" // struct qsbr
+#include "pycore_brc.h" // struct _brc_thread_state
+#include "pycore_freelist_state.h" // struct _Py_freelists
+#include "pycore_mimalloc.h" // struct _mimalloc_thread_state
+#include "pycore_qsbr.h" // struct qsbr
// Every PyThreadState is actually allocated as a _PyThreadStateImpl. The
@@ -29,7 +29,7 @@ typedef struct _PyThreadStateImpl {
#ifdef Py_GIL_DISABLED
struct _gc_thread_state gc;
struct _mimalloc_thread_state mimalloc;
- struct _Py_object_freelists freelists;
+ struct _Py_freelists freelists;
struct _brc_thread_state brc;
#endif
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 1a8f4d3ec7e..9ea7bc49be3 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1183,6 +1183,7 @@ PYTHON_HEADERS= \
$(srcdir)/Include/internal/pycore_format.h \
$(srcdir)/Include/internal/pycore_frame.h \
$(srcdir)/Include/internal/pycore_freelist.h \
+ $(srcdir)/Include/internal/pycore_freelist_state.h \
$(srcdir)/Include/internal/pycore_function.h \
$(srcdir)/Include/internal/pycore_gc.h \
$(srcdir)/Include/internal/pycore_genobject.h \
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index ca91da7771b..7310c3c8e13 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -395,44 +395,6 @@ static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj);
#include "clinic/dictobject.c.h"
-#ifdef WITH_FREELISTS
-static struct _Py_dict_freelist *
-get_dict_freelist(void)
-{
- struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
- return &freelists->dicts;
-}
-
-static struct _Py_dictkeys_freelist *
-get_dictkeys_freelist(void)
-{
- struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
- return &freelists->dictkeys;
-}
-#endif
-
-
-void
-_PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
-{
-#ifdef WITH_FREELISTS
- struct _Py_dict_freelist *freelist = &freelists->dicts;
- while (freelist->numfree > 0) {
- PyDictObject *op = freelist->items[--freelist->numfree];
- assert(PyDict_CheckExact(op));
- PyObject_GC_Del(op);
- }
- struct _Py_dictkeys_freelist *keys_freelist = &freelists->dictkeys;
- while (keys_freelist->numfree > 0) {
- PyMem_Free(keys_freelist->items[--keys_freelist->numfree]);
- }
- if (is_finalization) {
- freelist->numfree = -1;
- keys_freelist->numfree = -1;
- }
-#endif
-}
-
static inline Py_hash_t
unicode_get_hash(PyObject *o)
{
@@ -445,12 +407,12 @@ void
_PyDict_DebugMallocStats(FILE *out)
{
#ifdef WITH_FREELISTS
- struct _Py_dict_freelist *dict_freelist = get_dict_freelist();
_PyDebugAllocatorStats(out, "free PyDictObject",
- dict_freelist->numfree, sizeof(PyDictObject));
- struct _Py_dictkeys_freelist *dictkeys_freelist = get_dictkeys_freelist();
+ _Py_FREELIST_SIZE(dicts),
+ sizeof(PyDictObject));
_PyDebugAllocatorStats(out, "free PyDictKeysObject",
- dictkeys_freelist->numfree, sizeof(PyDictKeysObject));
+ _Py_FREELIST_SIZE(dictkeys),
+ sizeof(PyDictKeysObject));
#endif
}
@@ -785,7 +747,6 @@ _PyDict_CheckConsistency(PyObject *op, int check_content)
static PyDictKeysObject*
new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode)
{
- PyDictKeysObject *dk;
Py_ssize_t usable;
int log2_bytes;
size_t entry_size = unicode ? sizeof(PyDictUnicodeEntry) : sizeof(PyDictKeyEntry);
@@ -808,15 +769,11 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode)
log2_bytes = log2_size + 2;
}
-#ifdef WITH_FREELISTS
- struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist();
- if (log2_size == PyDict_LOG_MINSIZE && unicode && freelist->numfree > 0) {
- dk = freelist->items[--freelist->numfree];
- OBJECT_STAT_INC(from_freelist);
+ PyDictKeysObject *dk = NULL;
+ if (log2_size == PyDict_LOG_MINSIZE && unicode) {
+ dk = _Py_FREELIST_POP_MEM(dictkeys);
}
- else
-#endif
- {
+ if (dk == NULL) {
dk = PyMem_Malloc(sizeof(PyDictKeysObject)
+ ((size_t)1 << log2_bytes)
+ entry_size * usable);
@@ -852,18 +809,12 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr)
return;
}
#endif
-#ifdef WITH_FREELISTS
- struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist();
- if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE
- && freelist->numfree < PyDict_MAXFREELIST
- && freelist->numfree >= 0
- && DK_IS_UNICODE(keys)) {
- freelist->items[freelist->numfree++] = keys;
- OBJECT_STAT_INC(to_freelist);
- return;
+ if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE && keys->dk_kind == DICT_KEYS_UNICODE) {
+ _Py_FREELIST_FREE(dictkeys, keys, PyMem_Free);
+ }
+ else {
+ PyMem_Free(keys);
}
-#endif
- PyMem_Free(keys);
}
static size_t
@@ -912,20 +863,9 @@ new_dict(PyInterpreterState *interp,
PyDictKeysObject *keys, PyDictValues *values,
Py_ssize_t used, int free_values_on_failure)
{
- PyDictObject *mp;
assert(keys != NULL);
-#ifdef WITH_FREELISTS
- struct _Py_dict_freelist *freelist = get_dict_freelist();
- if (freelist->numfree > 0) {
- mp = freelist->items[--freelist->numfree];
- assert (mp != NULL);
- assert (Py_IS_TYPE(mp, &PyDict_Type));
- OBJECT_STAT_INC(from_freelist);
- _Py_NewReference((PyObject *)mp);
- }
- else
-#endif
- {
+ PyDictObject *mp = _Py_FREELIST_POP(PyDictObject, dicts);
+ if (mp == NULL) {
mp = PyObject_GC_New(PyDictObject, &PyDict_Type);
if (mp == NULL) {
dictkeys_decref(interp, keys, false);
@@ -935,6 +875,7 @@ new_dict(PyInterpreterState *interp,
return NULL;
}
}
+ assert(Py_IS_TYPE(mp, &PyDict_Type));
mp->ma_keys = keys;
mp->ma_values = values;
mp->ma_used = used;
@@ -3153,16 +3094,10 @@ dict_dealloc(PyObject *self)
assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS);
dictkeys_decref(interp, keys, false);
}
-#ifdef WITH_FREELISTS
- struct _Py_dict_freelist *freelist = get_dict_freelist();
- if (freelist->numfree < PyDict_MAXFREELIST && freelist->numfree >=0 &&
- Py_IS_TYPE(mp, &PyDict_Type)) {
- freelist->items[freelist->numfree++] = mp;
- OBJECT_STAT_INC(to_freelist);
+ if (Py_IS_TYPE(mp, &PyDict_Type)) {
+ _Py_FREELIST_FREE(dicts, mp, Py_TYPE(mp)->tp_free);
}
- else
-#endif
- {
+ else {
Py_TYPE(mp)->tp_free((PyObject *)mp);
}
Py_TRASHCAN_END
diff --git a/Objects/floatobject.c b/Objects/floatobject.c
index 08d8d854c38..82f39de421f 100644
--- a/Objects/floatobject.c
+++ b/Objects/floatobject.c
@@ -7,8 +7,8 @@
#include "pycore_abstract.h" // _PyNumber_Index()
#include "pycore_dtoa.h" // _Py_dg_dtoa()
#include "pycore_floatobject.h" // _PyFloat_FormatAdvancedWriter()
+#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
#include "pycore_initconfig.h" // _PyStatus_OK()
-#include "pycore_interp.h" // _Py_float_freelist
#include "pycore_long.h" // _PyLong_GetOne()
#include "pycore_modsupport.h" // _PyArg_NoKwnames()
#include "pycore_object.h" // _PyObject_Init(), _PyDebugAllocatorStats()
@@ -26,16 +26,6 @@ class float "PyObject *" "&PyFloat_Type"
#include "clinic/floatobject.c.h"
-#ifdef WITH_FREELISTS
-static struct _Py_float_freelist *
-get_float_freelist(void)
-{
- struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
- assert(freelists != NULL);
- return &freelists->floats;
-}
-#endif
-
double
PyFloat_GetMax(void)
@@ -132,24 +122,14 @@ PyFloat_GetInfo(void)
PyObject *
PyFloat_FromDouble(double fval)
{
- PyFloatObject *op;
-#ifdef WITH_FREELISTS
- struct _Py_float_freelist *float_freelist = get_float_freelist();
- op = float_freelist->items;
- if (op != NULL) {
- float_freelist->items = (PyFloatObject *) Py_TYPE(op);
- float_freelist->numfree--;
- OBJECT_STAT_INC(from_freelist);
- }
- else
-#endif
- {
+ PyFloatObject *op = _Py_FREELIST_POP(PyFloatObject, floats);
+ if (op == NULL) {
op = PyObject_Malloc(sizeof(PyFloatObject));
if (!op) {
return PyErr_NoMemory();
}
+ _PyObject_Init((PyObject*)op, &PyFloat_Type);
}
- _PyObject_Init((PyObject*)op, &PyFloat_Type);
op->ob_fval = fval;
return (PyObject *) op;
}
@@ -248,20 +228,7 @@ void
_PyFloat_ExactDealloc(PyObject *obj)
{
assert(PyFloat_CheckExact(obj));
- PyFloatObject *op = (PyFloatObject *)obj;
-#ifdef WITH_FREELISTS
- struct _Py_float_freelist *float_freelist = get_float_freelist();
- if (float_freelist->numfree >= PyFloat_MAXFREELIST || float_freelist->numfree < 0) {
- PyObject_Free(op);
- return;
- }
- float_freelist->numfree++;
- Py_SET_TYPE(op, (PyTypeObject *)float_freelist->items);
- float_freelist->items = op;
- OBJECT_STAT_INC(to_freelist);
-#else
- PyObject_Free(op);
-#endif
+ _Py_FREELIST_FREE(floats, obj, PyObject_Free);
}
static void
@@ -1994,27 +1961,6 @@ _PyFloat_InitTypes(PyInterpreterState *interp)
return _PyStatus_OK();
}
-void
-_PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
-{
-#ifdef WITH_FREELISTS
- struct _Py_float_freelist *state = &freelists->floats;
- PyFloatObject *f = state->items;
- while (f != NULL) {
- PyFloatObject *next = (PyFloatObject*) Py_TYPE(f);
- PyObject_Free(f);
- f = next;
- }
- state->items = NULL;
- if (is_finalization) {
- state->numfree = -1;
- }
- else {
- state->numfree = 0;
- }
-#endif
-}
-
void
_PyFloat_FiniType(PyInterpreterState *interp)
{
@@ -2026,10 +1972,10 @@ void
_PyFloat_DebugMallocStats(FILE *out)
{
#ifdef WITH_FREELISTS
- struct _Py_float_freelist *float_freelist = get_float_freelist();
_PyDebugAllocatorStats(out,
"free PyFloatObject",
- float_freelist->numfree, sizeof(PyFloatObject));
+ _Py_FREELIST_SIZE(floats),
+ sizeof(PyFloatObject));
#endif
}
diff --git a/Objects/genobject.c b/Objects/genobject.c
index 37b40530589..c204ac04b48 100644
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -6,7 +6,7 @@
#include "pycore_call.h" // _PyObject_CallNoArgs()
#include "pycore_ceval.h" // _PyEval_EvalFrame()
#include "pycore_frame.h" // _PyInterpreterFrame
-#include "pycore_freelist.h" // struct _Py_async_gen_freelist
+#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
#include "pycore_gc.h" // _PyGC_CLEAR_FINALIZED()
#include "pycore_modsupport.h" // _PyArg_CheckPositional()
#include "pycore_object.h" // _PyObject_GC_UNTRACK()
@@ -1629,23 +1629,6 @@ PyTypeObject PyAsyncGen_Type = {
};
-#ifdef WITH_FREELISTS
-static struct _Py_async_gen_freelist *
-get_async_gen_freelist(void)
-{
- struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
- return &freelists->async_gens;
-}
-
-static struct _Py_async_gen_asend_freelist *
-get_async_gen_asend_freelist(void)
-{
- struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
- return &freelists->async_gen_asends;
-}
-#endif
-
-
PyObject *
PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
{
@@ -1662,36 +1645,6 @@ PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
return (PyObject*)o;
}
-
-void
-_PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelist_state, int is_finalization)
-{
-#ifdef WITH_FREELISTS
- struct _Py_async_gen_freelist *freelist = &freelist_state->async_gens;
-
- while (freelist->numfree > 0) {
- _PyAsyncGenWrappedValue *o;
- o = freelist->items[--freelist->numfree];
- assert(_PyAsyncGenWrappedValue_CheckExact(o));
- PyObject_GC_Del(o);
- }
-
- struct _Py_async_gen_asend_freelist *asend_freelist = &freelist_state->async_gen_asends;
-
- while (asend_freelist->numfree > 0) {
- PyAsyncGenASend *o;
- o = asend_freelist->items[--asend_freelist->numfree];
- assert(Py_IS_TYPE(o, &_PyAsyncGenASend_Type));
- PyObject_GC_Del(o);
- }
-
- if (is_finalization) {
- freelist->numfree = -1;
- asend_freelist->numfree = -1;
- }
-#endif
-}
-
static PyObject *
async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result)
{
@@ -1735,18 +1688,11 @@ async_gen_asend_dealloc(PyAsyncGenASend *o)
_PyObject_GC_UNTRACK((PyObject *)o);
Py_CLEAR(o->ags_gen);
Py_CLEAR(o->ags_sendval);
-#ifdef WITH_FREELISTS
- struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist();
- if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) {
- assert(PyAsyncGenASend_CheckExact(o));
- _PyGC_CLEAR_FINALIZED((PyObject *)o);
- freelist->items[freelist->numfree++] = o;
- }
- else
-#endif
- {
- PyObject_GC_Del(o);
- }
+
+ assert(PyAsyncGenASend_CheckExact(o));
+ _PyGC_CLEAR_FINALIZED((PyObject *)o);
+
+ _Py_FREELIST_FREE(async_gen_asends, o, PyObject_GC_Del);
}
static int
@@ -1936,17 +1882,8 @@ PyTypeObject _PyAsyncGenASend_Type = {
static PyObject *
async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval)
{
- PyAsyncGenASend *o;
-#ifdef WITH_FREELISTS
- struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist();
- if (freelist->numfree > 0) {
- freelist->numfree--;
- o = freelist->items[freelist->numfree];
- _Py_NewReference((PyObject *)o);
- }
- else
-#endif
- {
+ PyAsyncGenASend *o = _Py_FREELIST_POP(PyAsyncGenASend, async_gen_asends);
+ if (o == NULL) {
o = PyObject_GC_New(PyAsyncGenASend, &_PyAsyncGenASend_Type);
if (o == NULL) {
return NULL;
@@ -1972,18 +1909,7 @@ async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o)
{
_PyObject_GC_UNTRACK((PyObject *)o);
Py_CLEAR(o->agw_val);
-#ifdef WITH_FREELISTS
- struct _Py_async_gen_freelist *freelist = get_async_gen_freelist();
- if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) {
- assert(_PyAsyncGenWrappedValue_CheckExact(o));
- freelist->items[freelist->numfree++] = o;
- OBJECT_STAT_INC(to_freelist);
- }
- else
-#endif
- {
- PyObject_GC_Del(o);
- }
+ _Py_FREELIST_FREE(async_gens, o, PyObject_GC_Del);
}
@@ -2042,27 +1968,17 @@ PyTypeObject _PyAsyncGenWrappedValue_Type = {
PyObject *
_PyAsyncGenValueWrapperNew(PyThreadState *tstate, PyObject *val)
{
- _PyAsyncGenWrappedValue *o;
assert(val);
-#ifdef WITH_FREELISTS
- struct _Py_async_gen_freelist *freelist = get_async_gen_freelist();
- if (freelist->numfree > 0) {
- freelist->numfree--;
- o = freelist->items[freelist->numfree];
- OBJECT_STAT_INC(from_freelist);
- assert(_PyAsyncGenWrappedValue_CheckExact(o));
- _Py_NewReference((PyObject*)o);
- }
- else
-#endif
- {
+ _PyAsyncGenWrappedValue *o = _Py_FREELIST_POP(_PyAsyncGenWrappedValue, async_gens);
+ if (o == NULL) {
o = PyObject_GC_New(_PyAsyncGenWrappedValue,
&_PyAsyncGenWrappedValue_Type);
if (o == NULL) {
return NULL;
}
}
+ assert(_PyAsyncGenWrappedValue_CheckExact(o));
o->agw_val = Py_NewRef(val);
_PyObject_GC_TRACK((PyObject*)o);
return (PyObject*)o;
diff --git a/Objects/listobject.c b/Objects/listobject.c
index f29f58dc25b..4d654c2ed33 100644
--- a/Objects/listobject.c
+++ b/Objects/listobject.c
@@ -4,6 +4,7 @@
#include "pycore_abstract.h" // _PyIndex_Check()
#include "pycore_ceval.h" // _PyEval_GetBuiltin()
#include "pycore_dict.h" // _PyDictViewObject
+#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
#include "pycore_pyatomic_ft_wrappers.h"
#include "pycore_interp.h" // PyInterpreterState.list
#include "pycore_list.h" // struct _Py_list_freelist, _PyListIterObject
@@ -23,16 +24,6 @@ class list "PyListObject *" "&PyList_Type"
_Py_DECLARE_STR(list_err, "list index out of range");
-#ifdef WITH_FREELISTS
-static struct _Py_list_freelist *
-get_list_freelist(void)
-{
- struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
- assert(freelists != NULL);
- return &freelists->lists;
-}
-#endif
-
#ifdef Py_GIL_DISABLED
typedef struct {
Py_ssize_t allocated;
@@ -205,55 +196,28 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size)
return 0;
}
-void
-_PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
-{
-#ifdef WITH_FREELISTS
- struct _Py_list_freelist *state = &freelists->lists;
- while (state->numfree > 0) {
- PyListObject *op = state->items[--state->numfree];
- assert(PyList_CheckExact(op));
- PyObject_GC_Del(op);
- }
- if (is_finalization) {
- state->numfree = -1;
- }
-#endif
-}
-
/* Print summary info about the state of the optimized allocator */
void
_PyList_DebugMallocStats(FILE *out)
{
#ifdef WITH_FREELISTS
- struct _Py_list_freelist *list_freelist = get_list_freelist();
_PyDebugAllocatorStats(out,
"free PyListObject",
- list_freelist->numfree, sizeof(PyListObject));
+ _Py_FREELIST_SIZE(lists),
+ sizeof(PyListObject));
#endif
}
PyObject *
PyList_New(Py_ssize_t size)
{
- PyListObject *op;
-
if (size < 0) {
PyErr_BadInternalCall();
return NULL;
}
-#ifdef WITH_FREELISTS
- struct _Py_list_freelist *list_freelist = get_list_freelist();
- if (PyList_MAXFREELIST && list_freelist->numfree > 0) {
- list_freelist->numfree--;
- op = list_freelist->items[list_freelist->numfree];
- OBJECT_STAT_INC(from_freelist);
- _Py_NewReference((PyObject *)op);
- }
- else
-#endif
- {
+ PyListObject *op = _Py_FREELIST_POP(PyListObject, lists);
+ if (op == NULL) {
op = PyObject_GC_New(PyListObject, &PyList_Type);
if (op == NULL) {
return NULL;
@@ -548,16 +512,11 @@ list_dealloc(PyObject *self)
}
free_list_items(op->ob_item, false);
}
-#ifdef WITH_FREELISTS
- struct _Py_list_freelist *list_freelist = get_list_freelist();
- if (list_freelist->numfree < PyList_MAXFREELIST && list_freelist->numfree >= 0 && PyList_CheckExact(op)) {
- list_freelist->items[list_freelist->numfree++] = op;
- OBJECT_STAT_INC(to_freelist);
+ if (PyList_CheckExact(op)) {
+ _Py_FREELIST_FREE(lists, op, PyObject_GC_Del);
}
- else
-#endif
- {
- Py_TYPE(op)->tp_free((PyObject *)op);
+ else {
+ PyObject_GC_Del(op);
}
Py_TRASHCAN_END
}
diff --git a/Objects/object.c b/Objects/object.c
index e2f96af5477..6d6bb87433c 100644
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -10,6 +10,7 @@
#include "pycore_descrobject.h" // _PyMethodWrapper_Type
#include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes()
#include "pycore_floatobject.h" // _PyFloat_DebugMallocStats()
+#include "pycore_freelist.h" // _PyObject_ClearFreeLists()
#include "pycore_initconfig.h" // _PyStatus_EXCEPTION()
#include "pycore_instruction_sequence.h" // _PyInstructionSequence_Type
#include "pycore_hashtable.h" // _Py_hashtable_new()
@@ -808,20 +809,54 @@ PyObject_Bytes(PyObject *v)
return PyBytes_FromObject(v);
}
-void
-_PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization)
+#ifdef WITH_FREELISTS
+static void
+clear_freelist(struct _Py_freelist *freelist, int is_finalization,
+ freefunc dofree)
{
+ void *ptr;
+ while ((ptr = _PyFreeList_PopNoStats(freelist)) != NULL) {
+ dofree(ptr);
+ }
+ assert(freelist->size == 0 || freelist->size == -1);
+ assert(freelist->freelist == NULL);
+ if (is_finalization) {
+ freelist->size = -1;
+ }
+}
+
+static void
+free_object(void *obj)
+{
+ PyObject *op = (PyObject *)obj;
+ Py_TYPE(op)->tp_free(op);
+}
+
+#endif
+
+void
+_PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization)
+{
+#ifdef WITH_FREELISTS
// In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear()
// In the default build, freelists are per-interpreter and cleared in finalize_interp_types()
- _PyFloat_ClearFreeList(freelists, is_finalization);
- _PyTuple_ClearFreeList(freelists, is_finalization);
- _PyList_ClearFreeList(freelists, is_finalization);
- _PyDict_ClearFreeList(freelists, is_finalization);
- _PyContext_ClearFreeList(freelists, is_finalization);
- _PyAsyncGen_ClearFreeLists(freelists, is_finalization);
- // Only be cleared if is_finalization is true.
- _PyObjectStackChunk_ClearFreeList(freelists, is_finalization);
- _PySlice_ClearFreeList(freelists, is_finalization);
+ clear_freelist(&freelists->floats, is_finalization, free_object);
+ for (Py_ssize_t i = 0; i < PyTuple_MAXSAVESIZE; i++) {
+ clear_freelist(&freelists->tuples[i], is_finalization, free_object);
+ }
+ clear_freelist(&freelists->lists, is_finalization, free_object);
+ clear_freelist(&freelists->dicts, is_finalization, free_object);
+ clear_freelist(&freelists->dictkeys, is_finalization, PyMem_Free);
+ clear_freelist(&freelists->slices, is_finalization, free_object);
+ clear_freelist(&freelists->contexts, is_finalization, free_object);
+ clear_freelist(&freelists->async_gens, is_finalization, free_object);
+ clear_freelist(&freelists->async_gen_asends, is_finalization, free_object);
+ if (is_finalization) {
+ // Only clear object stack chunks during finalization. We use object
+ // stacks during GC, so emptying the free-list is counterproductive.
+ clear_freelist(&freelists->object_stack_chunks, 1, PyMem_RawFree);
+ }
+#endif
}
/*
diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c
index 245bea98d58..1b6d35998c2 100644
--- a/Objects/sliceobject.c
+++ b/Objects/sliceobject.c
@@ -15,6 +15,7 @@ this type and there is exactly one in existence.
#include "Python.h"
#include "pycore_abstract.h" // _PyIndex_Check()
+#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
#include "pycore_long.h" // _PyLong_GetZero()
#include "pycore_modsupport.h" // _PyArg_NoKeywords()
#include "pycore_object.h" // _PyObject_GC_TRACK()
@@ -108,20 +109,6 @@ PyObject _Py_EllipsisObject = _PyObject_HEAD_INIT(&PyEllipsis_Type);
/* Slice object implementation */
-void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
-{
- if (!is_finalization) {
- return;
- }
-#ifdef WITH_FREELISTS
- PySliceObject *obj = freelists->slices.slice_cache;
- if (obj != NULL) {
- freelists->slices.slice_cache = NULL;
- PyObject_GC_Del(obj);
- }
-#endif
-}
-
/* start, stop, and step are python objects with None indicating no
index is present.
*/
@@ -130,17 +117,8 @@ static PySliceObject *
_PyBuildSlice_Consume2(PyObject *start, PyObject *stop, PyObject *step)
{
assert(start != NULL && stop != NULL && step != NULL);
- PySliceObject *obj;
-#ifdef WITH_FREELISTS
- struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
- if (freelists->slices.slice_cache != NULL) {
- obj = freelists->slices.slice_cache;
- freelists->slices.slice_cache = NULL;
- _Py_NewReference((PyObject *)obj);
- }
- else
-#endif
- {
+ PySliceObject *obj = _Py_FREELIST_POP(PySliceObject, slices);
+ if (obj == NULL) {
obj = PyObject_GC_New(PySliceObject, &PySlice_Type);
if (obj == NULL) {
goto error;
@@ -369,16 +347,7 @@ slice_dealloc(PySliceObject *r)
Py_DECREF(r->step);
Py_DECREF(r->start);
Py_DECREF(r->stop);
-#ifdef WITH_FREELISTS
- struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
- if (freelists->slices.slice_cache == NULL) {
- freelists->slices.slice_cache = r;
- }
- else
-#endif
- {
- PyObject_GC_Del(r);
- }
+ _Py_FREELIST_FREE(slices, r, PyObject_GC_Del);
}
static PyObject *
diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c
index 3704d095a97..bd6e5681911 100644
--- a/Objects/tupleobject.c
+++ b/Objects/tupleobject.c
@@ -4,6 +4,7 @@
#include "Python.h"
#include "pycore_abstract.h" // _PyIndex_Check()
#include "pycore_ceval.h" // _PyEval_GetBuiltin()
+#include "pycore_freelist.h" // _Py_FREELIST_PUSH(), _Py_FREELIST_POP()
#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED()
#include "pycore_initconfig.h" // _PyStatus_OK()
#include "pycore_modsupport.h" // _PyArg_NoKwnames()
@@ -17,7 +18,6 @@ class tuple "PyTupleObject *" "&PyTuple_Type"
#include "clinic/tupleobject.c.h"
-static inline PyTupleObject * maybe_freelist_pop(Py_ssize_t);
static inline int maybe_freelist_push(PyTupleObject *);
@@ -38,22 +38,20 @@ tuple_alloc(Py_ssize_t size)
PyErr_BadInternalCall();
return NULL;
}
-#ifdef Py_DEBUG
assert(size != 0); // The empty tuple is statically allocated.
-#endif
-
- PyTupleObject *op = maybe_freelist_pop(size);
- if (op == NULL) {
- /* Check for overflow */
- if ((size_t)size > ((size_t)PY_SSIZE_T_MAX - (sizeof(PyTupleObject) -
- sizeof(PyObject *))) / sizeof(PyObject *)) {
- return (PyTupleObject *)PyErr_NoMemory();
+ Py_ssize_t index = size - 1;
+ if (index < PyTuple_MAXSAVESIZE) {
+ PyTupleObject *op = _Py_FREELIST_POP(PyTupleObject, tuples[index]);
+ if (op != NULL) {
+ return op;
}
- op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);
- if (op == NULL)
- return NULL;
}
- return op;
+ /* Check for overflow */
+ if ((size_t)size > ((size_t)PY_SSIZE_T_MAX - (sizeof(PyTupleObject) -
+ sizeof(PyObject *))) / sizeof(PyObject *)) {
+ return (PyTupleObject *)PyErr_NoMemory();
+ }
+ return PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);
}
// The empty tuple singleton is not tracked by the GC.
@@ -982,16 +980,6 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize)
return 0;
}
-
-static void maybe_freelist_clear(struct _Py_object_freelists *, int);
-
-
-void
-_PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
-{
- maybe_freelist_clear(freelists, is_finalization);
-}
-
/*********************** Tuple Iterator **************************/
@@ -1141,102 +1129,31 @@ tuple_iter(PyObject *seq)
* freelists *
*************/
-#define TUPLE_FREELIST (freelists->tuples)
-#define FREELIST_FINALIZED (TUPLE_FREELIST.numfree[0] < 0)
-
-static inline PyTupleObject *
-maybe_freelist_pop(Py_ssize_t size)
-{
-#ifdef WITH_FREELISTS
- struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
- if (size == 0) {
- return NULL;
- }
- assert(size > 0);
- if (size <= PyTuple_MAXSAVESIZE) {
- Py_ssize_t index = size - 1;
- PyTupleObject *op = TUPLE_FREELIST.items[index];
- if (op != NULL) {
- /* op is the head of a linked list, with the first item
- pointing to the next node. Here we pop off the old head. */
- TUPLE_FREELIST.items[index] = (PyTupleObject *) op->ob_item[0];
- TUPLE_FREELIST.numfree[index]--;
- /* Inlined _PyObject_InitVar() without _PyType_HasFeature() test */
-#ifdef Py_TRACE_REFS
- /* maybe_freelist_push() ensures these were already set. */
- // XXX Can we drop these? See commit 68055ce6fe01 (GvR, Dec 1998).
- Py_SET_SIZE(op, size);
- Py_SET_TYPE(op, &PyTuple_Type);
-#endif
- _Py_NewReference((PyObject *)op);
- /* END inlined _PyObject_InitVar() */
- OBJECT_STAT_INC(from_freelist);
- return op;
- }
- }
-#endif
- return NULL;
-}
-
static inline int
maybe_freelist_push(PyTupleObject *op)
{
-#ifdef WITH_FREELISTS
- struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
- if (Py_SIZE(op) == 0) {
+ if (!Py_IS_TYPE(op, &PyTuple_Type)) {
return 0;
}
Py_ssize_t index = Py_SIZE(op) - 1;
- if (index < PyTuple_NFREELISTS
- && TUPLE_FREELIST.numfree[index] < PyTuple_MAXFREELIST
- && TUPLE_FREELIST.numfree[index] >= 0
- && Py_IS_TYPE(op, &PyTuple_Type))
- {
- /* op is the head of a linked list, with the first item
- pointing to the next node. Here we set op as the new head. */
- op->ob_item[0] = (PyObject *) TUPLE_FREELIST.items[index];
- TUPLE_FREELIST.items[index] = op;
- TUPLE_FREELIST.numfree[index]++;
- OBJECT_STAT_INC(to_freelist);
- return 1;
+ if (index < PyTuple_MAXSAVESIZE) {
+ return _Py_FREELIST_PUSH(tuples[index], op, Py_tuple_MAXFREELIST);
}
-#endif
return 0;
}
-static void
-maybe_freelist_clear(struct _Py_object_freelists *freelists, int fini)
-{
-#ifdef WITH_FREELISTS
- for (Py_ssize_t i = 0; i < PyTuple_NFREELISTS; i++) {
- PyTupleObject *p = TUPLE_FREELIST.items[i];
- TUPLE_FREELIST.items[i] = NULL;
- TUPLE_FREELIST.numfree[i] = fini ? -1 : 0;
- while (p) {
- PyTupleObject *q = p;
- p = (PyTupleObject *)(p->ob_item[0]);
- PyObject_GC_Del(q);
- }
- }
-#endif
-}
-
/* Print summary info about the state of the optimized allocator */
void
_PyTuple_DebugMallocStats(FILE *out)
{
#ifdef WITH_FREELISTS
- struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
- for (int i = 0; i < PyTuple_NFREELISTS; i++) {
+ for (int i = 0; i < PyTuple_MAXSAVESIZE; i++) {
int len = i + 1;
char buf[128];
PyOS_snprintf(buf, sizeof(buf),
"free %d-sized PyTupleObject", len);
- _PyDebugAllocatorStats(out, buf, TUPLE_FREELIST.numfree[i],
+ _PyDebugAllocatorStats(out, buf, _Py_FREELIST_SIZE(tuples[i]),
_PyObject_VAR_SIZE(&PyTuple_Type, len));
}
#endif
}
-
-#undef STATE
-#undef FREELIST_FINALIZED
diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
index f36fcb8caec..9e3af689f4a 100644
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -239,6 +239,7 @@
+
diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters
index a1b43addf9e..31f7971bda8 100644
--- a/PCbuild/pythoncore.vcxproj.filters
+++ b/PCbuild/pythoncore.vcxproj.filters
@@ -639,6 +639,9 @@
Include\internal
+
+ Include\internal
+
Include\internal
diff --git a/Python/context.c b/Python/context.c
index c32c15f5562..5cafde4dab9 100644
--- a/Python/context.c
+++ b/Python/context.c
@@ -1,6 +1,7 @@
#include "Python.h"
#include "pycore_call.h" // _PyObject_VectorcallTstate()
#include "pycore_context.h"
+#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
#include "pycore_gc.h" // _PyObject_GC_MAY_BE_TRACKED()
#include "pycore_hamt.h"
#include "pycore_initconfig.h" // _PyStatus_OK()
@@ -64,16 +65,6 @@ static int
contextvar_del(PyContextVar *var);
-#ifdef WITH_FREELISTS
-static struct _Py_context_freelist *
-get_context_freelist(void)
-{
- struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
- return &freelists->contexts;
-}
-#endif
-
-
PyObject *
_PyContext_NewHamtForTests(void)
{
@@ -343,20 +334,8 @@ class _contextvars.Context "PyContext *" "&PyContext_Type"
static inline PyContext *
_context_alloc(void)
{
- PyContext *ctx;
-#ifdef WITH_FREELISTS
- struct _Py_context_freelist *context_freelist = get_context_freelist();
- if (context_freelist->numfree > 0) {
- context_freelist->numfree--;
- ctx = context_freelist->items;
- context_freelist->items = (PyContext *)ctx->ctx_weakreflist;
- OBJECT_STAT_INC(from_freelist);
- ctx->ctx_weakreflist = NULL;
- _Py_NewReference((PyObject *)ctx);
- }
- else
-#endif
- {
+ PyContext *ctx = _Py_FREELIST_POP(PyContext, contexts);
+ if (ctx == NULL) {
ctx = PyObject_GC_New(PyContext, &PyContext_Type);
if (ctx == NULL) {
return NULL;
@@ -471,19 +450,7 @@ context_tp_dealloc(PyContext *self)
}
(void)context_tp_clear(self);
-#ifdef WITH_FREELISTS
- struct _Py_context_freelist *context_freelist = get_context_freelist();
- if (context_freelist->numfree >= 0 && context_freelist->numfree < PyContext_MAXFREELIST) {
- context_freelist->numfree++;
- self->ctx_weakreflist = (PyObject *)context_freelist->items;
- context_freelist->items = self;
- OBJECT_STAT_INC(to_freelist);
- }
- else
-#endif
- {
- Py_TYPE(self)->tp_free(self);
- }
+ _Py_FREELIST_FREE(contexts, self, Py_TYPE(self)->tp_free);
}
static PyObject *
@@ -1264,24 +1231,6 @@ get_token_missing(void)
///////////////////////////
-void
-_PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
-{
-#ifdef WITH_FREELISTS
- struct _Py_context_freelist *state = &freelists->contexts;
- for (; state->numfree > 0; state->numfree--) {
- PyContext *ctx = state->items;
- state->items = (PyContext *)ctx->ctx_weakreflist;
- ctx->ctx_weakreflist = NULL;
- PyObject_GC_Del(ctx);
- }
- if (is_finalization) {
- state->numfree = -1;
- }
-#endif
-}
-
-
PyStatus
_PyContext_Init(PyInterpreterState *interp)
{
diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c
index d1d5664ab96..53f04160c38 100644
--- a/Python/gc_free_threading.c
+++ b/Python/gc_free_threading.c
@@ -4,6 +4,7 @@
#include "pycore_ceval.h" // _Py_set_eval_breaker_bit()
#include "pycore_context.h"
#include "pycore_dict.h" // _PyDict_MaybeUntrack()
+#include "pycore_freelist.h" // _PyObject_ClearFreeLists()
#include "pycore_initconfig.h"
#include "pycore_interp.h" // PyInterpreterState.gc
#include "pycore_object.h"
diff --git a/Python/gc_gil.c b/Python/gc_gil.c
index 48646c7af86..7d62a9ed0b1 100644
--- a/Python/gc_gil.c
+++ b/Python/gc_gil.c
@@ -1,5 +1,5 @@
#include "Python.h"
-#include "pycore_pystate.h" // _Py_ClearFreeLists()
+#include "pycore_freelist.h" // _PyObject_ClearFreeLists()
#ifndef Py_GIL_DISABLED
diff --git a/Python/object_stack.c b/Python/object_stack.c
index bd9696822c8..bda3190f45e 100644
--- a/Python/object_stack.c
+++ b/Python/object_stack.c
@@ -8,24 +8,11 @@
extern _PyObjectStackChunk *_PyObjectStackChunk_New(void);
extern void _PyObjectStackChunk_Free(_PyObjectStackChunk *);
-static struct _Py_object_stack_freelist *
-get_object_stack_freelist(void)
-{
- struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
- return &freelists->object_stacks;
-}
-
_PyObjectStackChunk *
_PyObjectStackChunk_New(void)
{
- _PyObjectStackChunk *buf;
- struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist();
- if (obj_stack_freelist->numfree > 0) {
- buf = obj_stack_freelist->items;
- obj_stack_freelist->items = buf->prev;
- obj_stack_freelist->numfree--;
- }
- else {
+ _PyObjectStackChunk *buf = _Py_FREELIST_POP_MEM(object_stack_chunks);
+ if (buf == NULL) {
// NOTE: we use PyMem_RawMalloc() here because this is used by the GC
// during mimalloc heap traversal. In that context, it is not safe to
// allocate mimalloc memory, such as via PyMem_Malloc().
@@ -43,17 +30,7 @@ void
_PyObjectStackChunk_Free(_PyObjectStackChunk *buf)
{
assert(buf->n == 0);
- struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist();
- if (obj_stack_freelist->numfree >= 0 &&
- obj_stack_freelist->numfree < _PyObjectStackChunk_MAXFREELIST)
- {
- buf->prev = obj_stack_freelist->items;
- obj_stack_freelist->items = buf;
- obj_stack_freelist->numfree++;
- }
- else {
- PyMem_RawFree(buf);
- }
+ _Py_FREELIST_FREE(object_stack_chunks, buf, PyMem_RawFree);
}
void
@@ -87,22 +64,3 @@ _PyObjectStack_Merge(_PyObjectStack *dst, _PyObjectStack *src)
dst->head = src->head;
src->head = NULL;
}
-
-void
-_PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
-{
- if (!is_finalization) {
- // Ignore requests to clear the free list during GC. We use object
- // stacks during GC, so emptying the free-list is counterproductive.
- return;
- }
-
- struct _Py_object_stack_freelist *freelist = &freelists->object_stacks;
- while (freelist->numfree > 0) {
- _PyObjectStackChunk *buf = freelist->items;
- freelist->items = buf->prev;
- freelist->numfree--;
- PyMem_RawFree(buf);
- }
- freelist->numfree = -1;
-}
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 39eaa86098b..6b641c0775f 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -9,6 +9,7 @@
#include "pycore_dict.h" // _PyDict_Fini()
#include "pycore_exceptions.h" // _PyExc_InitTypes()
#include "pycore_fileutils.h" // _Py_ResetForceASCII()
+#include "pycore_freelist.h" // _PyObject_ClearFreeLists()
#include "pycore_floatobject.h" // _PyFloat_InitTypes()
#include "pycore_global_objects_fini_generated.h" // "_PyStaticObjects_CheckRefcnt()
#include "pycore_import.h" // _PyImport_BootstrapImp()
@@ -1843,7 +1844,7 @@ finalize_interp_types(PyInterpreterState *interp)
#ifndef Py_GIL_DISABLED
// With Py_GIL_DISABLED:
// the freelists for the current thread state have already been cleared.
- struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
+ struct _Py_freelists *freelists = _Py_freelists_GET();
_PyObject_ClearFreeLists(freelists, 1);
#endif
diff --git a/Python/pystate.c b/Python/pystate.c
index f0452aa3cce..6fbd17f7eae 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -9,9 +9,9 @@
#include "pycore_dtoa.h" // _dtoa_state_INIT()
#include "pycore_emscripten_trampoline.h" // _Py_EmscriptenTrampoline_Init()
#include "pycore_frame.h"
+#include "pycore_freelist.h" // _PyObject_ClearFreeLists()
#include "pycore_initconfig.h" // _PyStatus_OK()
#include "pycore_object.h" // _PyType_InitCache()
-#include "pycore_object_stack.h" // _PyObjectStackChunk_ClearFreeList()
#include "pycore_parking_lot.h" // _PyParkingLot_AfterFork()
#include "pycore_pyerrors.h" // _PyErr_Clear()
#include "pycore_pylifecycle.h" // _PyAST_Fini()
@@ -1738,7 +1738,7 @@ PyThreadState_Clear(PyThreadState *tstate)
#ifdef Py_GIL_DISABLED
// Each thread should clear own freelists in free-threading builds.
- struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
+ struct _Py_freelists *freelists = _Py_freelists_GET();
_PyObject_ClearFreeLists(freelists, 1);
// Remove ourself from the biased reference counting table of threads.