gh-100240: Use a consistent implementation for freelists (#121934)

This combines and updates our freelist handling to use a consistent
implementation. Objects in the freelist are linked together using the
first word of memory block.

If configured with freelists disabled, these operations are essentially
no-ops.
This commit is contained in:
Sam Gross 2024-07-22 12:08:27 -04:00 committed by GitHub
parent 2408a8a22b
commit 5716cc3529
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
27 changed files with 295 additions and 705 deletions

View File

@ -5,7 +5,6 @@
# error "this header requires Py_BUILD_CORE define" # error "this header requires Py_BUILD_CORE define"
#endif #endif
#include "pycore_freelist.h" // _PyFreeListState
#include "pycore_hamt.h" // PyHamtObject #include "pycore_hamt.h" // PyHamtObject

View File

@ -8,7 +8,6 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define" # error "this header requires Py_BUILD_CORE define"
#endif #endif
#include "pycore_freelist.h" // _PyFreeListState
#include "pycore_object.h" // PyManagedDictPointer #include "pycore_object.h" // PyManagedDictPointer
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_ACQUIRE #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_ACQUIRE

View File

@ -8,7 +8,6 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define" # error "this header requires Py_BUILD_CORE define"
#endif #endif
#include "pycore_freelist.h" // _PyFreeListState
#include "pycore_unicodeobject.h" // _PyUnicodeWriter #include "pycore_unicodeobject.h" // _PyUnicodeWriter
/* runtime lifecycle */ /* runtime lifecycle */

View File

@ -8,144 +8,109 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define" # error "this header requires Py_BUILD_CORE define"
#endif #endif
// PyTuple_MAXSAVESIZE - largest tuple to save on free list #include "pycore_freelist_state.h" // struct _Py_freelists
// PyTuple_MAXFREELIST - maximum number of tuples of each size to save #include "pycore_object.h" // _PyObject_IS_GC
#include "pycore_pystate.h" // _PyThreadState_GET
#include "pycore_code.h" // OBJECT_STAT_INC
#ifdef WITH_FREELISTS static inline struct _Py_freelists *
// with freelists _Py_freelists_GET(void)
# define PyTuple_MAXSAVESIZE 20 {
# define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE PyThreadState *tstate = _PyThreadState_GET();
# define PyTuple_MAXFREELIST 2000 #ifdef Py_DEBUG
# define PyList_MAXFREELIST 80 _Py_EnsureTstateNotNULL(tstate);
# define PyDict_MAXFREELIST 80 #endif
# define PyFloat_MAXFREELIST 100
# define PyContext_MAXFREELIST 255 #ifdef Py_GIL_DISABLED
# define _PyAsyncGen_MAXFREELIST 80 return &((_PyThreadStateImpl*)tstate)->freelists;
# define _PyObjectStackChunk_MAXFREELIST 4
#else #else
# define PyTuple_NFREELISTS 0 return &tstate->interp->object_state.freelists;
# define PyTuple_MAXFREELIST 0
# define PyList_MAXFREELIST 0
# define PyDict_MAXFREELIST 0
# define PyFloat_MAXFREELIST 0
# define PyContext_MAXFREELIST 0
# define _PyAsyncGen_MAXFREELIST 0
# define _PyObjectStackChunk_MAXFREELIST 0
#endif #endif
}
struct _Py_list_freelist { #ifndef WITH_FREELISTS
#ifdef WITH_FREELISTS #define _Py_FREELIST_FREE(NAME, op, freefunc) freefunc(op)
PyListObject *items[PyList_MAXFREELIST]; #define _Py_FREELIST_PUSH(NAME, op, limit) (0)
int numfree; #define _Py_FREELIST_POP(TYPE, NAME) (NULL)
#endif #define _Py_FREELIST_POP_MEM(NAME) (NULL)
}; #define _Py_FREELIST_SIZE(NAME) (0)
struct _Py_tuple_freelist {
#if WITH_FREELISTS
/* There is one freelist for each size from 1 to PyTuple_MAXSAVESIZE.
The empty tuple is handled separately.
Each tuple stored in the array is the head of the linked list
(and the next available tuple) for that size. The actual tuple
object is used as the linked list node, with its first item
(ob_item[0]) pointing to the next node (i.e. the previous head).
Each linked list is initially NULL. */
PyTupleObject *items[PyTuple_NFREELISTS];
int numfree[PyTuple_NFREELISTS];
#else #else
char _unused; // Empty structs are not allowed. // Pushes `op` to the freelist, calls `freefunc` if the freelist is full
#define _Py_FREELIST_FREE(NAME, op, freefunc) \
_PyFreeList_Free(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), \
Py_ ## NAME ## _MAXFREELIST, freefunc)
// Pushes `op` to the freelist, returns 1 if successful, 0 if the freelist is full
#define _Py_FREELIST_PUSH(NAME, op, limit) \
_PyFreeList_Push(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), limit)
// Pops a PyObject from the freelist, returns NULL if the freelist is empty.
#define _Py_FREELIST_POP(TYPE, NAME) \
_Py_CAST(TYPE*, _PyFreeList_Pop(&_Py_freelists_GET()->NAME))
// Pops a non-PyObject data structure from the freelist, returns NULL if the
// freelist is empty.
#define _Py_FREELIST_POP_MEM(NAME) \
_PyFreeList_PopMem(&_Py_freelists_GET()->NAME)
#define _Py_FREELIST_SIZE(NAME) (int)((_Py_freelists_GET()->NAME).size)
static inline int
_PyFreeList_Push(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize)
{
if (fl->size < maxsize && fl->size >= 0) {
*(void **)obj = fl->freelist;
fl->freelist = obj;
fl->size++;
OBJECT_STAT_INC(to_freelist);
return 1;
}
return 0;
}
static inline void
_PyFreeList_Free(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize,
freefunc dofree)
{
if (!_PyFreeList_Push(fl, obj, maxsize)) {
dofree(obj);
}
}
static inline void *
_PyFreeList_PopNoStats(struct _Py_freelist *fl)
{
void *obj = fl->freelist;
if (obj != NULL) {
assert(fl->size > 0);
fl->freelist = *(void **)obj;
fl->size--;
}
return obj;
}
static inline PyObject *
_PyFreeList_Pop(struct _Py_freelist *fl)
{
PyObject *op = _PyFreeList_PopNoStats(fl);
if (op != NULL) {
OBJECT_STAT_INC(from_freelist);
_Py_NewReference(op);
}
return op;
}
static inline void *
_PyFreeList_PopMem(struct _Py_freelist *fl)
{
void *op = _PyFreeList_PopNoStats(fl);
if (op != NULL) {
OBJECT_STAT_INC(from_freelist);
}
return op;
}
#endif #endif
};
struct _Py_float_freelist { extern void _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization);
#ifdef WITH_FREELISTS
/* Special free list
free_list is a singly-linked list of available PyFloatObjects,
linked via abuse of their ob_type members. */
int numfree;
PyFloatObject *items;
#endif
};
struct _Py_dict_freelist {
#ifdef WITH_FREELISTS
/* Dictionary reuse scheme to save calls to malloc and free */
PyDictObject *items[PyDict_MAXFREELIST];
int numfree;
#endif
};
struct _Py_dictkeys_freelist {
#ifdef WITH_FREELISTS
/* Dictionary keys reuse scheme to save calls to malloc and free */
PyDictKeysObject *items[PyDict_MAXFREELIST];
int numfree;
#endif
};
struct _Py_slice_freelist {
#ifdef WITH_FREELISTS
/* Using a cache is very effective since typically only a single slice is
created and then deleted again. */
PySliceObject *slice_cache;
#endif
};
struct _Py_context_freelist {
#ifdef WITH_FREELISTS
// List of free PyContext objects
PyContext *items;
int numfree;
#endif
};
struct _Py_async_gen_freelist {
#ifdef WITH_FREELISTS
/* Freelists boost performance 6-10%; they also reduce memory
fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend
are short-living objects that are instantiated for every
__anext__() call. */
struct _PyAsyncGenWrappedValue* items[_PyAsyncGen_MAXFREELIST];
int numfree;
#endif
};
struct _Py_async_gen_asend_freelist {
#ifdef WITH_FREELISTS
struct PyAsyncGenASend* items[_PyAsyncGen_MAXFREELIST];
int numfree;
#endif
};
struct _PyObjectStackChunk;
struct _Py_object_stack_freelist {
struct _PyObjectStackChunk *items;
Py_ssize_t numfree;
};
struct _Py_object_freelists {
struct _Py_float_freelist floats;
struct _Py_tuple_freelist tuples;
struct _Py_list_freelist lists;
struct _Py_dict_freelist dicts;
struct _Py_dictkeys_freelist dictkeys;
struct _Py_slice_freelist slices;
struct _Py_context_freelist contexts;
struct _Py_async_gen_freelist async_gens;
struct _Py_async_gen_asend_freelist async_gen_asends;
struct _Py_object_stack_freelist object_stacks;
};
extern void _PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization);
extern void _PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
extern void _PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
extern void _PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
extern void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
extern void _PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
extern void _PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization);
extern void _PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
extern void _PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -0,0 +1,59 @@
#ifndef Py_INTERNAL_FREELIST_STATE_H
#define Py_INTERNAL_FREELIST_STATE_H
#ifdef __cplusplus
extern "C" {
#endif
#ifndef Py_BUILD_CORE
# error "this header requires Py_BUILD_CORE define"
#endif
#ifdef WITH_FREELISTS
// with freelists
# define PyTuple_MAXSAVESIZE 20 // Largest tuple to save on freelist
# define Py_tuple_MAXFREELIST 2000 // Maximum number of tuples of each size to save
# define Py_lists_MAXFREELIST 80
# define Py_dicts_MAXFREELIST 80
# define Py_dictkeys_MAXFREELIST 80
# define Py_floats_MAXFREELIST 100
# define Py_slices_MAXFREELIST 1
# define Py_contexts_MAXFREELIST 255
# define Py_async_gens_MAXFREELIST 80
# define Py_async_gen_asends_MAXFREELIST 80
# define Py_object_stack_chunks_MAXFREELIST 4
#else
# define PyTuple_MAXSAVESIZE 0
#endif
// A generic freelist of either PyObjects or other data structures.
struct _Py_freelist {
// Entries are linked together using the first word of the the object.
// For PyObjects, this overlaps with the `ob_refcnt` field or the `ob_tid`
// field.
void *freelist;
// The number of items in the free list or -1 if the free list is disabled
Py_ssize_t size;
};
struct _Py_freelists {
#ifdef WITH_FREELISTS
struct _Py_freelist floats;
struct _Py_freelist tuples[PyTuple_MAXSAVESIZE];
struct _Py_freelist lists;
struct _Py_freelist dicts;
struct _Py_freelist dictkeys;
struct _Py_freelist slices;
struct _Py_freelist contexts;
struct _Py_freelist async_gens;
struct _Py_freelist async_gen_asends;
struct _Py_freelist object_stack_chunks;
#else
char _unused; // Empty structs are not allowed.
#endif
};
#ifdef __cplusplus
}
#endif
#endif /* !Py_INTERNAL_FREELIST_STATE_H */

View File

@ -8,8 +8,6 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define" # error "this header requires Py_BUILD_CORE define"
#endif #endif
#include "pycore_freelist.h" // _PyFreeListState
/* GC information is stored BEFORE the object structure. */ /* GC information is stored BEFORE the object structure. */
typedef struct { typedef struct {
// Pointer to next object in the list. // Pointer to next object in the list.

View File

@ -8,8 +8,6 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define" # error "this header requires Py_BUILD_CORE define"
#endif #endif
#include "pycore_freelist.h" // _PyFreeListState
PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *); PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *);
extern void _PyList_DebugMallocStats(FILE *out); extern void _PyList_DebugMallocStats(FILE *out);

View File

@ -1,8 +1,6 @@
#ifndef Py_INTERNAL_OBJECT_STACK_H #ifndef Py_INTERNAL_OBJECT_STACK_H
#define Py_INTERNAL_OBJECT_STACK_H #define Py_INTERNAL_OBJECT_STACK_H
#include "pycore_freelist.h" // _PyFreeListState
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif

View File

@ -8,8 +8,8 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define" # error "this header requires Py_BUILD_CORE define"
#endif #endif
#include "pycore_freelist.h" // _PyObject_freelists #include "pycore_freelist_state.h" // _Py_freelists
#include "pycore_hashtable.h" // _Py_hashtable_t #include "pycore_hashtable.h" // _Py_hashtable_t
struct _py_object_runtime_state { struct _py_object_runtime_state {
#ifdef Py_REF_DEBUG #ifdef Py_REF_DEBUG
@ -20,7 +20,7 @@ struct _py_object_runtime_state {
struct _py_object_state { struct _py_object_state {
#if !defined(Py_GIL_DISABLED) #if !defined(Py_GIL_DISABLED)
struct _Py_object_freelists freelists; struct _Py_freelists freelists;
#endif #endif
#ifdef Py_REF_DEBUG #ifdef Py_REF_DEBUG
Py_ssize_t reftotal; Py_ssize_t reftotal;

View File

@ -8,11 +8,9 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define" # error "this header requires Py_BUILD_CORE define"
#endif #endif
#include "pycore_freelist.h" // _PyFreeListState
#include "pycore_runtime.h" // _PyRuntime #include "pycore_runtime.h" // _PyRuntime
#include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_tstate.h" // _PyThreadStateImpl
// Values for PyThreadState.state. A thread must be in the "attached" state // Values for PyThreadState.state. A thread must be in the "attached" state
// before calling most Python APIs. If the GIL is enabled, then "attached" // before calling most Python APIs. If the GIL is enabled, then "attached"
// implies that the thread holds the GIL and "detached" implies that the // implies that the thread holds the GIL and "detached" implies that the
@ -279,20 +277,6 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void);
// See also PyInterpreterState_Get() and _PyInterpreterState_GET(). // See also PyInterpreterState_Get() and _PyInterpreterState_GET().
extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void); extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void);
static inline struct _Py_object_freelists* _Py_object_freelists_GET(void)
{
PyThreadState *tstate = _PyThreadState_GET();
#ifdef Py_DEBUG
_Py_EnsureTstateNotNULL(tstate);
#endif
#ifdef Py_GIL_DISABLED
return &((_PyThreadStateImpl*)tstate)->freelists;
#else
return &tstate->interp->object_state.freelists;
#endif
}
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -8,10 +8,10 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define" # error "this header requires Py_BUILD_CORE define"
#endif #endif
#include "pycore_brc.h" // struct _brc_thread_state #include "pycore_brc.h" // struct _brc_thread_state
#include "pycore_freelist.h" // struct _Py_freelist_state #include "pycore_freelist_state.h" // struct _Py_freelists
#include "pycore_mimalloc.h" // struct _mimalloc_thread_state #include "pycore_mimalloc.h" // struct _mimalloc_thread_state
#include "pycore_qsbr.h" // struct qsbr #include "pycore_qsbr.h" // struct qsbr
// Every PyThreadState is actually allocated as a _PyThreadStateImpl. The // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The
@ -29,7 +29,7 @@ typedef struct _PyThreadStateImpl {
#ifdef Py_GIL_DISABLED #ifdef Py_GIL_DISABLED
struct _gc_thread_state gc; struct _gc_thread_state gc;
struct _mimalloc_thread_state mimalloc; struct _mimalloc_thread_state mimalloc;
struct _Py_object_freelists freelists; struct _Py_freelists freelists;
struct _brc_thread_state brc; struct _brc_thread_state brc;
#endif #endif

View File

@ -1183,6 +1183,7 @@ PYTHON_HEADERS= \
$(srcdir)/Include/internal/pycore_format.h \ $(srcdir)/Include/internal/pycore_format.h \
$(srcdir)/Include/internal/pycore_frame.h \ $(srcdir)/Include/internal/pycore_frame.h \
$(srcdir)/Include/internal/pycore_freelist.h \ $(srcdir)/Include/internal/pycore_freelist.h \
$(srcdir)/Include/internal/pycore_freelist_state.h \
$(srcdir)/Include/internal/pycore_function.h \ $(srcdir)/Include/internal/pycore_function.h \
$(srcdir)/Include/internal/pycore_gc.h \ $(srcdir)/Include/internal/pycore_gc.h \
$(srcdir)/Include/internal/pycore_genobject.h \ $(srcdir)/Include/internal/pycore_genobject.h \

View File

@ -395,44 +395,6 @@ static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj);
#include "clinic/dictobject.c.h" #include "clinic/dictobject.c.h"
#ifdef WITH_FREELISTS
static struct _Py_dict_freelist *
get_dict_freelist(void)
{
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
return &freelists->dicts;
}
static struct _Py_dictkeys_freelist *
get_dictkeys_freelist(void)
{
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
return &freelists->dictkeys;
}
#endif
void
_PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
{
#ifdef WITH_FREELISTS
struct _Py_dict_freelist *freelist = &freelists->dicts;
while (freelist->numfree > 0) {
PyDictObject *op = freelist->items[--freelist->numfree];
assert(PyDict_CheckExact(op));
PyObject_GC_Del(op);
}
struct _Py_dictkeys_freelist *keys_freelist = &freelists->dictkeys;
while (keys_freelist->numfree > 0) {
PyMem_Free(keys_freelist->items[--keys_freelist->numfree]);
}
if (is_finalization) {
freelist->numfree = -1;
keys_freelist->numfree = -1;
}
#endif
}
static inline Py_hash_t static inline Py_hash_t
unicode_get_hash(PyObject *o) unicode_get_hash(PyObject *o)
{ {
@ -445,12 +407,12 @@ void
_PyDict_DebugMallocStats(FILE *out) _PyDict_DebugMallocStats(FILE *out)
{ {
#ifdef WITH_FREELISTS #ifdef WITH_FREELISTS
struct _Py_dict_freelist *dict_freelist = get_dict_freelist();
_PyDebugAllocatorStats(out, "free PyDictObject", _PyDebugAllocatorStats(out, "free PyDictObject",
dict_freelist->numfree, sizeof(PyDictObject)); _Py_FREELIST_SIZE(dicts),
struct _Py_dictkeys_freelist *dictkeys_freelist = get_dictkeys_freelist(); sizeof(PyDictObject));
_PyDebugAllocatorStats(out, "free PyDictKeysObject", _PyDebugAllocatorStats(out, "free PyDictKeysObject",
dictkeys_freelist->numfree, sizeof(PyDictKeysObject)); _Py_FREELIST_SIZE(dictkeys),
sizeof(PyDictKeysObject));
#endif #endif
} }
@ -785,7 +747,6 @@ _PyDict_CheckConsistency(PyObject *op, int check_content)
static PyDictKeysObject* static PyDictKeysObject*
new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode)
{ {
PyDictKeysObject *dk;
Py_ssize_t usable; Py_ssize_t usable;
int log2_bytes; int log2_bytes;
size_t entry_size = unicode ? sizeof(PyDictUnicodeEntry) : sizeof(PyDictKeyEntry); size_t entry_size = unicode ? sizeof(PyDictUnicodeEntry) : sizeof(PyDictKeyEntry);
@ -808,15 +769,11 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode)
log2_bytes = log2_size + 2; log2_bytes = log2_size + 2;
} }
#ifdef WITH_FREELISTS PyDictKeysObject *dk = NULL;
struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist(); if (log2_size == PyDict_LOG_MINSIZE && unicode) {
if (log2_size == PyDict_LOG_MINSIZE && unicode && freelist->numfree > 0) { dk = _Py_FREELIST_POP_MEM(dictkeys);
dk = freelist->items[--freelist->numfree];
OBJECT_STAT_INC(from_freelist);
} }
else if (dk == NULL) {
#endif
{
dk = PyMem_Malloc(sizeof(PyDictKeysObject) dk = PyMem_Malloc(sizeof(PyDictKeysObject)
+ ((size_t)1 << log2_bytes) + ((size_t)1 << log2_bytes)
+ entry_size * usable); + entry_size * usable);
@ -852,18 +809,12 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr)
return; return;
} }
#endif #endif
#ifdef WITH_FREELISTS if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE && keys->dk_kind == DICT_KEYS_UNICODE) {
struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist(); _Py_FREELIST_FREE(dictkeys, keys, PyMem_Free);
if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE }
&& freelist->numfree < PyDict_MAXFREELIST else {
&& freelist->numfree >= 0 PyMem_Free(keys);
&& DK_IS_UNICODE(keys)) {
freelist->items[freelist->numfree++] = keys;
OBJECT_STAT_INC(to_freelist);
return;
} }
#endif
PyMem_Free(keys);
} }
static size_t static size_t
@ -912,20 +863,9 @@ new_dict(PyInterpreterState *interp,
PyDictKeysObject *keys, PyDictValues *values, PyDictKeysObject *keys, PyDictValues *values,
Py_ssize_t used, int free_values_on_failure) Py_ssize_t used, int free_values_on_failure)
{ {
PyDictObject *mp;
assert(keys != NULL); assert(keys != NULL);
#ifdef WITH_FREELISTS PyDictObject *mp = _Py_FREELIST_POP(PyDictObject, dicts);
struct _Py_dict_freelist *freelist = get_dict_freelist(); if (mp == NULL) {
if (freelist->numfree > 0) {
mp = freelist->items[--freelist->numfree];
assert (mp != NULL);
assert (Py_IS_TYPE(mp, &PyDict_Type));
OBJECT_STAT_INC(from_freelist);
_Py_NewReference((PyObject *)mp);
}
else
#endif
{
mp = PyObject_GC_New(PyDictObject, &PyDict_Type); mp = PyObject_GC_New(PyDictObject, &PyDict_Type);
if (mp == NULL) { if (mp == NULL) {
dictkeys_decref(interp, keys, false); dictkeys_decref(interp, keys, false);
@ -935,6 +875,7 @@ new_dict(PyInterpreterState *interp,
return NULL; return NULL;
} }
} }
assert(Py_IS_TYPE(mp, &PyDict_Type));
mp->ma_keys = keys; mp->ma_keys = keys;
mp->ma_values = values; mp->ma_values = values;
mp->ma_used = used; mp->ma_used = used;
@ -3153,16 +3094,10 @@ dict_dealloc(PyObject *self)
assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS);
dictkeys_decref(interp, keys, false); dictkeys_decref(interp, keys, false);
} }
#ifdef WITH_FREELISTS if (Py_IS_TYPE(mp, &PyDict_Type)) {
struct _Py_dict_freelist *freelist = get_dict_freelist(); _Py_FREELIST_FREE(dicts, mp, Py_TYPE(mp)->tp_free);
if (freelist->numfree < PyDict_MAXFREELIST && freelist->numfree >=0 &&
Py_IS_TYPE(mp, &PyDict_Type)) {
freelist->items[freelist->numfree++] = mp;
OBJECT_STAT_INC(to_freelist);
} }
else else {
#endif
{
Py_TYPE(mp)->tp_free((PyObject *)mp); Py_TYPE(mp)->tp_free((PyObject *)mp);
} }
Py_TRASHCAN_END Py_TRASHCAN_END

View File

@ -7,8 +7,8 @@
#include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_abstract.h" // _PyNumber_Index()
#include "pycore_dtoa.h" // _Py_dg_dtoa() #include "pycore_dtoa.h" // _Py_dg_dtoa()
#include "pycore_floatobject.h" // _PyFloat_FormatAdvancedWriter() #include "pycore_floatobject.h" // _PyFloat_FormatAdvancedWriter()
#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
#include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_initconfig.h" // _PyStatus_OK()
#include "pycore_interp.h" // _Py_float_freelist
#include "pycore_long.h" // _PyLong_GetOne() #include "pycore_long.h" // _PyLong_GetOne()
#include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_modsupport.h" // _PyArg_NoKwnames()
#include "pycore_object.h" // _PyObject_Init(), _PyDebugAllocatorStats() #include "pycore_object.h" // _PyObject_Init(), _PyDebugAllocatorStats()
@ -26,16 +26,6 @@ class float "PyObject *" "&PyFloat_Type"
#include "clinic/floatobject.c.h" #include "clinic/floatobject.c.h"
#ifdef WITH_FREELISTS
static struct _Py_float_freelist *
get_float_freelist(void)
{
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
assert(freelists != NULL);
return &freelists->floats;
}
#endif
double double
PyFloat_GetMax(void) PyFloat_GetMax(void)
@ -132,24 +122,14 @@ PyFloat_GetInfo(void)
PyObject * PyObject *
PyFloat_FromDouble(double fval) PyFloat_FromDouble(double fval)
{ {
PyFloatObject *op; PyFloatObject *op = _Py_FREELIST_POP(PyFloatObject, floats);
#ifdef WITH_FREELISTS if (op == NULL) {
struct _Py_float_freelist *float_freelist = get_float_freelist();
op = float_freelist->items;
if (op != NULL) {
float_freelist->items = (PyFloatObject *) Py_TYPE(op);
float_freelist->numfree--;
OBJECT_STAT_INC(from_freelist);
}
else
#endif
{
op = PyObject_Malloc(sizeof(PyFloatObject)); op = PyObject_Malloc(sizeof(PyFloatObject));
if (!op) { if (!op) {
return PyErr_NoMemory(); return PyErr_NoMemory();
} }
_PyObject_Init((PyObject*)op, &PyFloat_Type);
} }
_PyObject_Init((PyObject*)op, &PyFloat_Type);
op->ob_fval = fval; op->ob_fval = fval;
return (PyObject *) op; return (PyObject *) op;
} }
@ -248,20 +228,7 @@ void
_PyFloat_ExactDealloc(PyObject *obj) _PyFloat_ExactDealloc(PyObject *obj)
{ {
assert(PyFloat_CheckExact(obj)); assert(PyFloat_CheckExact(obj));
PyFloatObject *op = (PyFloatObject *)obj; _Py_FREELIST_FREE(floats, obj, PyObject_Free);
#ifdef WITH_FREELISTS
struct _Py_float_freelist *float_freelist = get_float_freelist();
if (float_freelist->numfree >= PyFloat_MAXFREELIST || float_freelist->numfree < 0) {
PyObject_Free(op);
return;
}
float_freelist->numfree++;
Py_SET_TYPE(op, (PyTypeObject *)float_freelist->items);
float_freelist->items = op;
OBJECT_STAT_INC(to_freelist);
#else
PyObject_Free(op);
#endif
} }
static void static void
@ -1994,27 +1961,6 @@ _PyFloat_InitTypes(PyInterpreterState *interp)
return _PyStatus_OK(); return _PyStatus_OK();
} }
void
_PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
{
#ifdef WITH_FREELISTS
struct _Py_float_freelist *state = &freelists->floats;
PyFloatObject *f = state->items;
while (f != NULL) {
PyFloatObject *next = (PyFloatObject*) Py_TYPE(f);
PyObject_Free(f);
f = next;
}
state->items = NULL;
if (is_finalization) {
state->numfree = -1;
}
else {
state->numfree = 0;
}
#endif
}
void void
_PyFloat_FiniType(PyInterpreterState *interp) _PyFloat_FiniType(PyInterpreterState *interp)
{ {
@ -2026,10 +1972,10 @@ void
_PyFloat_DebugMallocStats(FILE *out) _PyFloat_DebugMallocStats(FILE *out)
{ {
#ifdef WITH_FREELISTS #ifdef WITH_FREELISTS
struct _Py_float_freelist *float_freelist = get_float_freelist();
_PyDebugAllocatorStats(out, _PyDebugAllocatorStats(out,
"free PyFloatObject", "free PyFloatObject",
float_freelist->numfree, sizeof(PyFloatObject)); _Py_FREELIST_SIZE(floats),
sizeof(PyFloatObject));
#endif #endif
} }

View File

@ -6,7 +6,7 @@
#include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_call.h" // _PyObject_CallNoArgs()
#include "pycore_ceval.h" // _PyEval_EvalFrame() #include "pycore_ceval.h" // _PyEval_EvalFrame()
#include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_frame.h" // _PyInterpreterFrame
#include "pycore_freelist.h" // struct _Py_async_gen_freelist #include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
#include "pycore_gc.h" // _PyGC_CLEAR_FINALIZED() #include "pycore_gc.h" // _PyGC_CLEAR_FINALIZED()
#include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_modsupport.h" // _PyArg_CheckPositional()
#include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_object.h" // _PyObject_GC_UNTRACK()
@ -1629,23 +1629,6 @@ PyTypeObject PyAsyncGen_Type = {
}; };
#ifdef WITH_FREELISTS
static struct _Py_async_gen_freelist *
get_async_gen_freelist(void)
{
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
return &freelists->async_gens;
}
static struct _Py_async_gen_asend_freelist *
get_async_gen_asend_freelist(void)
{
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
return &freelists->async_gen_asends;
}
#endif
PyObject * PyObject *
PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname) PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
{ {
@ -1662,36 +1645,6 @@ PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
return (PyObject*)o; return (PyObject*)o;
} }
void
_PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelist_state, int is_finalization)
{
#ifdef WITH_FREELISTS
struct _Py_async_gen_freelist *freelist = &freelist_state->async_gens;
while (freelist->numfree > 0) {
_PyAsyncGenWrappedValue *o;
o = freelist->items[--freelist->numfree];
assert(_PyAsyncGenWrappedValue_CheckExact(o));
PyObject_GC_Del(o);
}
struct _Py_async_gen_asend_freelist *asend_freelist = &freelist_state->async_gen_asends;
while (asend_freelist->numfree > 0) {
PyAsyncGenASend *o;
o = asend_freelist->items[--asend_freelist->numfree];
assert(Py_IS_TYPE(o, &_PyAsyncGenASend_Type));
PyObject_GC_Del(o);
}
if (is_finalization) {
freelist->numfree = -1;
asend_freelist->numfree = -1;
}
#endif
}
static PyObject * static PyObject *
async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result) async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result)
{ {
@ -1735,18 +1688,11 @@ async_gen_asend_dealloc(PyAsyncGenASend *o)
_PyObject_GC_UNTRACK((PyObject *)o); _PyObject_GC_UNTRACK((PyObject *)o);
Py_CLEAR(o->ags_gen); Py_CLEAR(o->ags_gen);
Py_CLEAR(o->ags_sendval); Py_CLEAR(o->ags_sendval);
#ifdef WITH_FREELISTS
struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist(); assert(PyAsyncGenASend_CheckExact(o));
if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) { _PyGC_CLEAR_FINALIZED((PyObject *)o);
assert(PyAsyncGenASend_CheckExact(o));
_PyGC_CLEAR_FINALIZED((PyObject *)o); _Py_FREELIST_FREE(async_gen_asends, o, PyObject_GC_Del);
freelist->items[freelist->numfree++] = o;
}
else
#endif
{
PyObject_GC_Del(o);
}
} }
static int static int
@ -1936,17 +1882,8 @@ PyTypeObject _PyAsyncGenASend_Type = {
static PyObject * static PyObject *
async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval) async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval)
{ {
PyAsyncGenASend *o; PyAsyncGenASend *o = _Py_FREELIST_POP(PyAsyncGenASend, async_gen_asends);
#ifdef WITH_FREELISTS if (o == NULL) {
struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist();
if (freelist->numfree > 0) {
freelist->numfree--;
o = freelist->items[freelist->numfree];
_Py_NewReference((PyObject *)o);
}
else
#endif
{
o = PyObject_GC_New(PyAsyncGenASend, &_PyAsyncGenASend_Type); o = PyObject_GC_New(PyAsyncGenASend, &_PyAsyncGenASend_Type);
if (o == NULL) { if (o == NULL) {
return NULL; return NULL;
@ -1972,18 +1909,7 @@ async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o)
{ {
_PyObject_GC_UNTRACK((PyObject *)o); _PyObject_GC_UNTRACK((PyObject *)o);
Py_CLEAR(o->agw_val); Py_CLEAR(o->agw_val);
#ifdef WITH_FREELISTS _Py_FREELIST_FREE(async_gens, o, PyObject_GC_Del);
struct _Py_async_gen_freelist *freelist = get_async_gen_freelist();
if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) {
assert(_PyAsyncGenWrappedValue_CheckExact(o));
freelist->items[freelist->numfree++] = o;
OBJECT_STAT_INC(to_freelist);
}
else
#endif
{
PyObject_GC_Del(o);
}
} }
@ -2042,27 +1968,17 @@ PyTypeObject _PyAsyncGenWrappedValue_Type = {
PyObject * PyObject *
_PyAsyncGenValueWrapperNew(PyThreadState *tstate, PyObject *val) _PyAsyncGenValueWrapperNew(PyThreadState *tstate, PyObject *val)
{ {
_PyAsyncGenWrappedValue *o;
assert(val); assert(val);
#ifdef WITH_FREELISTS _PyAsyncGenWrappedValue *o = _Py_FREELIST_POP(_PyAsyncGenWrappedValue, async_gens);
struct _Py_async_gen_freelist *freelist = get_async_gen_freelist(); if (o == NULL) {
if (freelist->numfree > 0) {
freelist->numfree--;
o = freelist->items[freelist->numfree];
OBJECT_STAT_INC(from_freelist);
assert(_PyAsyncGenWrappedValue_CheckExact(o));
_Py_NewReference((PyObject*)o);
}
else
#endif
{
o = PyObject_GC_New(_PyAsyncGenWrappedValue, o = PyObject_GC_New(_PyAsyncGenWrappedValue,
&_PyAsyncGenWrappedValue_Type); &_PyAsyncGenWrappedValue_Type);
if (o == NULL) { if (o == NULL) {
return NULL; return NULL;
} }
} }
assert(_PyAsyncGenWrappedValue_CheckExact(o));
o->agw_val = Py_NewRef(val); o->agw_val = Py_NewRef(val);
_PyObject_GC_TRACK((PyObject*)o); _PyObject_GC_TRACK((PyObject*)o);
return (PyObject*)o; return (PyObject*)o;

View File

@ -4,6 +4,7 @@
#include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_abstract.h" // _PyIndex_Check()
#include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_ceval.h" // _PyEval_GetBuiltin()
#include "pycore_dict.h" // _PyDictViewObject #include "pycore_dict.h" // _PyDictViewObject
#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
#include "pycore_pyatomic_ft_wrappers.h" #include "pycore_pyatomic_ft_wrappers.h"
#include "pycore_interp.h" // PyInterpreterState.list #include "pycore_interp.h" // PyInterpreterState.list
#include "pycore_list.h" // struct _Py_list_freelist, _PyListIterObject #include "pycore_list.h" // struct _Py_list_freelist, _PyListIterObject
@ -23,16 +24,6 @@ class list "PyListObject *" "&PyList_Type"
_Py_DECLARE_STR(list_err, "list index out of range"); _Py_DECLARE_STR(list_err, "list index out of range");
#ifdef WITH_FREELISTS
static struct _Py_list_freelist *
get_list_freelist(void)
{
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
assert(freelists != NULL);
return &freelists->lists;
}
#endif
#ifdef Py_GIL_DISABLED #ifdef Py_GIL_DISABLED
typedef struct { typedef struct {
Py_ssize_t allocated; Py_ssize_t allocated;
@ -205,55 +196,28 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size)
return 0; return 0;
} }
void
_PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
{
#ifdef WITH_FREELISTS
struct _Py_list_freelist *state = &freelists->lists;
while (state->numfree > 0) {
PyListObject *op = state->items[--state->numfree];
assert(PyList_CheckExact(op));
PyObject_GC_Del(op);
}
if (is_finalization) {
state->numfree = -1;
}
#endif
}
/* Print summary info about the state of the optimized allocator */ /* Print summary info about the state of the optimized allocator */
void void
_PyList_DebugMallocStats(FILE *out) _PyList_DebugMallocStats(FILE *out)
{ {
#ifdef WITH_FREELISTS #ifdef WITH_FREELISTS
struct _Py_list_freelist *list_freelist = get_list_freelist();
_PyDebugAllocatorStats(out, _PyDebugAllocatorStats(out,
"free PyListObject", "free PyListObject",
list_freelist->numfree, sizeof(PyListObject)); _Py_FREELIST_SIZE(lists),
sizeof(PyListObject));
#endif #endif
} }
PyObject * PyObject *
PyList_New(Py_ssize_t size) PyList_New(Py_ssize_t size)
{ {
PyListObject *op;
if (size < 0) { if (size < 0) {
PyErr_BadInternalCall(); PyErr_BadInternalCall();
return NULL; return NULL;
} }
#ifdef WITH_FREELISTS PyListObject *op = _Py_FREELIST_POP(PyListObject, lists);
struct _Py_list_freelist *list_freelist = get_list_freelist(); if (op == NULL) {
if (PyList_MAXFREELIST && list_freelist->numfree > 0) {
list_freelist->numfree--;
op = list_freelist->items[list_freelist->numfree];
OBJECT_STAT_INC(from_freelist);
_Py_NewReference((PyObject *)op);
}
else
#endif
{
op = PyObject_GC_New(PyListObject, &PyList_Type); op = PyObject_GC_New(PyListObject, &PyList_Type);
if (op == NULL) { if (op == NULL) {
return NULL; return NULL;
@ -548,16 +512,11 @@ list_dealloc(PyObject *self)
} }
free_list_items(op->ob_item, false); free_list_items(op->ob_item, false);
} }
#ifdef WITH_FREELISTS if (PyList_CheckExact(op)) {
struct _Py_list_freelist *list_freelist = get_list_freelist(); _Py_FREELIST_FREE(lists, op, PyObject_GC_Del);
if (list_freelist->numfree < PyList_MAXFREELIST && list_freelist->numfree >= 0 && PyList_CheckExact(op)) {
list_freelist->items[list_freelist->numfree++] = op;
OBJECT_STAT_INC(to_freelist);
} }
else else {
#endif PyObject_GC_Del(op);
{
Py_TYPE(op)->tp_free((PyObject *)op);
} }
Py_TRASHCAN_END Py_TRASHCAN_END
} }

View File

@ -10,6 +10,7 @@
#include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_descrobject.h" // _PyMethodWrapper_Type
#include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes() #include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes()
#include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats()
#include "pycore_freelist.h" // _PyObject_ClearFreeLists()
#include "pycore_initconfig.h" // _PyStatus_EXCEPTION() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION()
#include "pycore_instruction_sequence.h" // _PyInstructionSequence_Type #include "pycore_instruction_sequence.h" // _PyInstructionSequence_Type
#include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_hashtable.h" // _Py_hashtable_new()
@ -808,20 +809,54 @@ PyObject_Bytes(PyObject *v)
return PyBytes_FromObject(v); return PyBytes_FromObject(v);
} }
void #ifdef WITH_FREELISTS
_PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization) static void
clear_freelist(struct _Py_freelist *freelist, int is_finalization,
freefunc dofree)
{ {
void *ptr;
while ((ptr = _PyFreeList_PopNoStats(freelist)) != NULL) {
dofree(ptr);
}
assert(freelist->size == 0 || freelist->size == -1);
assert(freelist->freelist == NULL);
if (is_finalization) {
freelist->size = -1;
}
}
static void
free_object(void *obj)
{
PyObject *op = (PyObject *)obj;
Py_TYPE(op)->tp_free(op);
}
#endif
void
_PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization)
{
#ifdef WITH_FREELISTS
// In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear() // In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear()
// In the default build, freelists are per-interpreter and cleared in finalize_interp_types() // In the default build, freelists are per-interpreter and cleared in finalize_interp_types()
_PyFloat_ClearFreeList(freelists, is_finalization); clear_freelist(&freelists->floats, is_finalization, free_object);
_PyTuple_ClearFreeList(freelists, is_finalization); for (Py_ssize_t i = 0; i < PyTuple_MAXSAVESIZE; i++) {
_PyList_ClearFreeList(freelists, is_finalization); clear_freelist(&freelists->tuples[i], is_finalization, free_object);
_PyDict_ClearFreeList(freelists, is_finalization); }
_PyContext_ClearFreeList(freelists, is_finalization); clear_freelist(&freelists->lists, is_finalization, free_object);
_PyAsyncGen_ClearFreeLists(freelists, is_finalization); clear_freelist(&freelists->dicts, is_finalization, free_object);
// Only be cleared if is_finalization is true. clear_freelist(&freelists->dictkeys, is_finalization, PyMem_Free);
_PyObjectStackChunk_ClearFreeList(freelists, is_finalization); clear_freelist(&freelists->slices, is_finalization, free_object);
_PySlice_ClearFreeList(freelists, is_finalization); clear_freelist(&freelists->contexts, is_finalization, free_object);
clear_freelist(&freelists->async_gens, is_finalization, free_object);
clear_freelist(&freelists->async_gen_asends, is_finalization, free_object);
if (is_finalization) {
// Only clear object stack chunks during finalization. We use object
// stacks during GC, so emptying the free-list is counterproductive.
clear_freelist(&freelists->object_stack_chunks, 1, PyMem_RawFree);
}
#endif
} }
/* /*

View File

@ -15,6 +15,7 @@ this type and there is exactly one in existence.
#include "Python.h" #include "Python.h"
#include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_abstract.h" // _PyIndex_Check()
#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
#include "pycore_long.h" // _PyLong_GetZero() #include "pycore_long.h" // _PyLong_GetZero()
#include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_modsupport.h" // _PyArg_NoKeywords()
#include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_object.h" // _PyObject_GC_TRACK()
@ -108,20 +109,6 @@ PyObject _Py_EllipsisObject = _PyObject_HEAD_INIT(&PyEllipsis_Type);
/* Slice object implementation */ /* Slice object implementation */
void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
{
if (!is_finalization) {
return;
}
#ifdef WITH_FREELISTS
PySliceObject *obj = freelists->slices.slice_cache;
if (obj != NULL) {
freelists->slices.slice_cache = NULL;
PyObject_GC_Del(obj);
}
#endif
}
/* start, stop, and step are python objects with None indicating no /* start, stop, and step are python objects with None indicating no
index is present. index is present.
*/ */
@ -130,17 +117,8 @@ static PySliceObject *
_PyBuildSlice_Consume2(PyObject *start, PyObject *stop, PyObject *step) _PyBuildSlice_Consume2(PyObject *start, PyObject *stop, PyObject *step)
{ {
assert(start != NULL && stop != NULL && step != NULL); assert(start != NULL && stop != NULL && step != NULL);
PySliceObject *obj; PySliceObject *obj = _Py_FREELIST_POP(PySliceObject, slices);
#ifdef WITH_FREELISTS if (obj == NULL) {
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
if (freelists->slices.slice_cache != NULL) {
obj = freelists->slices.slice_cache;
freelists->slices.slice_cache = NULL;
_Py_NewReference((PyObject *)obj);
}
else
#endif
{
obj = PyObject_GC_New(PySliceObject, &PySlice_Type); obj = PyObject_GC_New(PySliceObject, &PySlice_Type);
if (obj == NULL) { if (obj == NULL) {
goto error; goto error;
@ -369,16 +347,7 @@ slice_dealloc(PySliceObject *r)
Py_DECREF(r->step); Py_DECREF(r->step);
Py_DECREF(r->start); Py_DECREF(r->start);
Py_DECREF(r->stop); Py_DECREF(r->stop);
#ifdef WITH_FREELISTS _Py_FREELIST_FREE(slices, r, PyObject_GC_Del);
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
if (freelists->slices.slice_cache == NULL) {
freelists->slices.slice_cache = r;
}
else
#endif
{
PyObject_GC_Del(r);
}
} }
static PyObject * static PyObject *

View File

@ -4,6 +4,7 @@
#include "Python.h" #include "Python.h"
#include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_abstract.h" // _PyIndex_Check()
#include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_ceval.h" // _PyEval_GetBuiltin()
#include "pycore_freelist.h" // _Py_FREELIST_PUSH(), _Py_FREELIST_POP()
#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED()
#include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_initconfig.h" // _PyStatus_OK()
#include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_modsupport.h" // _PyArg_NoKwnames()
@ -17,7 +18,6 @@ class tuple "PyTupleObject *" "&PyTuple_Type"
#include "clinic/tupleobject.c.h" #include "clinic/tupleobject.c.h"
static inline PyTupleObject * maybe_freelist_pop(Py_ssize_t);
static inline int maybe_freelist_push(PyTupleObject *); static inline int maybe_freelist_push(PyTupleObject *);
@ -38,22 +38,20 @@ tuple_alloc(Py_ssize_t size)
PyErr_BadInternalCall(); PyErr_BadInternalCall();
return NULL; return NULL;
} }
#ifdef Py_DEBUG
assert(size != 0); // The empty tuple is statically allocated. assert(size != 0); // The empty tuple is statically allocated.
#endif Py_ssize_t index = size - 1;
if (index < PyTuple_MAXSAVESIZE) {
PyTupleObject *op = maybe_freelist_pop(size); PyTupleObject *op = _Py_FREELIST_POP(PyTupleObject, tuples[index]);
if (op == NULL) { if (op != NULL) {
/* Check for overflow */ return op;
if ((size_t)size > ((size_t)PY_SSIZE_T_MAX - (sizeof(PyTupleObject) -
sizeof(PyObject *))) / sizeof(PyObject *)) {
return (PyTupleObject *)PyErr_NoMemory();
} }
op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);
if (op == NULL)
return NULL;
} }
return op; /* Check for overflow */
if ((size_t)size > ((size_t)PY_SSIZE_T_MAX - (sizeof(PyTupleObject) -
sizeof(PyObject *))) / sizeof(PyObject *)) {
return (PyTupleObject *)PyErr_NoMemory();
}
return PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);
} }
// The empty tuple singleton is not tracked by the GC. // The empty tuple singleton is not tracked by the GC.
@ -982,16 +980,6 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize)
return 0; return 0;
} }
static void maybe_freelist_clear(struct _Py_object_freelists *, int);
void
_PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
{
maybe_freelist_clear(freelists, is_finalization);
}
/*********************** Tuple Iterator **************************/ /*********************** Tuple Iterator **************************/
@ -1141,102 +1129,31 @@ tuple_iter(PyObject *seq)
* freelists * * freelists *
*************/ *************/
#define TUPLE_FREELIST (freelists->tuples)
#define FREELIST_FINALIZED (TUPLE_FREELIST.numfree[0] < 0)
static inline PyTupleObject *
maybe_freelist_pop(Py_ssize_t size)
{
#ifdef WITH_FREELISTS
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
if (size == 0) {
return NULL;
}
assert(size > 0);
if (size <= PyTuple_MAXSAVESIZE) {
Py_ssize_t index = size - 1;
PyTupleObject *op = TUPLE_FREELIST.items[index];
if (op != NULL) {
/* op is the head of a linked list, with the first item
pointing to the next node. Here we pop off the old head. */
TUPLE_FREELIST.items[index] = (PyTupleObject *) op->ob_item[0];
TUPLE_FREELIST.numfree[index]--;
/* Inlined _PyObject_InitVar() without _PyType_HasFeature() test */
#ifdef Py_TRACE_REFS
/* maybe_freelist_push() ensures these were already set. */
// XXX Can we drop these? See commit 68055ce6fe01 (GvR, Dec 1998).
Py_SET_SIZE(op, size);
Py_SET_TYPE(op, &PyTuple_Type);
#endif
_Py_NewReference((PyObject *)op);
/* END inlined _PyObject_InitVar() */
OBJECT_STAT_INC(from_freelist);
return op;
}
}
#endif
return NULL;
}
static inline int static inline int
maybe_freelist_push(PyTupleObject *op) maybe_freelist_push(PyTupleObject *op)
{ {
#ifdef WITH_FREELISTS if (!Py_IS_TYPE(op, &PyTuple_Type)) {
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
if (Py_SIZE(op) == 0) {
return 0; return 0;
} }
Py_ssize_t index = Py_SIZE(op) - 1; Py_ssize_t index = Py_SIZE(op) - 1;
if (index < PyTuple_NFREELISTS if (index < PyTuple_MAXSAVESIZE) {
&& TUPLE_FREELIST.numfree[index] < PyTuple_MAXFREELIST return _Py_FREELIST_PUSH(tuples[index], op, Py_tuple_MAXFREELIST);
&& TUPLE_FREELIST.numfree[index] >= 0
&& Py_IS_TYPE(op, &PyTuple_Type))
{
/* op is the head of a linked list, with the first item
pointing to the next node. Here we set op as the new head. */
op->ob_item[0] = (PyObject *) TUPLE_FREELIST.items[index];
TUPLE_FREELIST.items[index] = op;
TUPLE_FREELIST.numfree[index]++;
OBJECT_STAT_INC(to_freelist);
return 1;
} }
#endif
return 0; return 0;
} }
static void
maybe_freelist_clear(struct _Py_object_freelists *freelists, int fini)
{
#ifdef WITH_FREELISTS
for (Py_ssize_t i = 0; i < PyTuple_NFREELISTS; i++) {
PyTupleObject *p = TUPLE_FREELIST.items[i];
TUPLE_FREELIST.items[i] = NULL;
TUPLE_FREELIST.numfree[i] = fini ? -1 : 0;
while (p) {
PyTupleObject *q = p;
p = (PyTupleObject *)(p->ob_item[0]);
PyObject_GC_Del(q);
}
}
#endif
}
/* Print summary info about the state of the optimized allocator */ /* Print summary info about the state of the optimized allocator */
void void
_PyTuple_DebugMallocStats(FILE *out) _PyTuple_DebugMallocStats(FILE *out)
{ {
#ifdef WITH_FREELISTS #ifdef WITH_FREELISTS
struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); for (int i = 0; i < PyTuple_MAXSAVESIZE; i++) {
for (int i = 0; i < PyTuple_NFREELISTS; i++) {
int len = i + 1; int len = i + 1;
char buf[128]; char buf[128];
PyOS_snprintf(buf, sizeof(buf), PyOS_snprintf(buf, sizeof(buf),
"free %d-sized PyTupleObject", len); "free %d-sized PyTupleObject", len);
_PyDebugAllocatorStats(out, buf, TUPLE_FREELIST.numfree[i], _PyDebugAllocatorStats(out, buf, _Py_FREELIST_SIZE(tuples[i]),
_PyObject_VAR_SIZE(&PyTuple_Type, len)); _PyObject_VAR_SIZE(&PyTuple_Type, len));
} }
#endif #endif
} }
#undef STATE
#undef FREELIST_FINALIZED

View File

@ -239,6 +239,7 @@
<ClInclude Include="..\Include\internal\pycore_format.h" /> <ClInclude Include="..\Include\internal\pycore_format.h" />
<ClInclude Include="..\Include\internal\pycore_frame.h" /> <ClInclude Include="..\Include\internal\pycore_frame.h" />
<ClInclude Include="..\Include\internal\pycore_freelist.h" /> <ClInclude Include="..\Include\internal\pycore_freelist.h" />
<ClInclude Include="..\Include\internal\pycore_freelist_state.h" />
<ClInclude Include="..\Include\internal\pycore_function.h" /> <ClInclude Include="..\Include\internal\pycore_function.h" />
<ClInclude Include="..\Include\internal\pycore_gc.h" /> <ClInclude Include="..\Include\internal\pycore_gc.h" />
<ClInclude Include="..\Include\internal\pycore_genobject.h" /> <ClInclude Include="..\Include\internal\pycore_genobject.h" />

View File

@ -639,6 +639,9 @@
<ClInclude Include="..\Include\internal\pycore_freelist.h"> <ClInclude Include="..\Include\internal\pycore_freelist.h">
<Filter>Include\internal</Filter> <Filter>Include\internal</Filter>
</ClInclude> </ClInclude>
<ClInclude Include="..\Include\internal\pycore_freelist_state.h">
<Filter>Include\internal</Filter>
</ClInclude>
<ClInclude Include="..\Include\internal\pycore_function.h"> <ClInclude Include="..\Include\internal\pycore_function.h">
<Filter>Include\internal</Filter> <Filter>Include\internal</Filter>
</ClInclude> </ClInclude>

View File

@ -1,6 +1,7 @@
#include "Python.h" #include "Python.h"
#include "pycore_call.h" // _PyObject_VectorcallTstate() #include "pycore_call.h" // _PyObject_VectorcallTstate()
#include "pycore_context.h" #include "pycore_context.h"
#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
#include "pycore_gc.h" // _PyObject_GC_MAY_BE_TRACKED() #include "pycore_gc.h" // _PyObject_GC_MAY_BE_TRACKED()
#include "pycore_hamt.h" #include "pycore_hamt.h"
#include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_initconfig.h" // _PyStatus_OK()
@ -64,16 +65,6 @@ static int
contextvar_del(PyContextVar *var); contextvar_del(PyContextVar *var);
#ifdef WITH_FREELISTS
static struct _Py_context_freelist *
get_context_freelist(void)
{
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
return &freelists->contexts;
}
#endif
PyObject * PyObject *
_PyContext_NewHamtForTests(void) _PyContext_NewHamtForTests(void)
{ {
@ -343,20 +334,8 @@ class _contextvars.Context "PyContext *" "&PyContext_Type"
static inline PyContext * static inline PyContext *
_context_alloc(void) _context_alloc(void)
{ {
PyContext *ctx; PyContext *ctx = _Py_FREELIST_POP(PyContext, contexts);
#ifdef WITH_FREELISTS if (ctx == NULL) {
struct _Py_context_freelist *context_freelist = get_context_freelist();
if (context_freelist->numfree > 0) {
context_freelist->numfree--;
ctx = context_freelist->items;
context_freelist->items = (PyContext *)ctx->ctx_weakreflist;
OBJECT_STAT_INC(from_freelist);
ctx->ctx_weakreflist = NULL;
_Py_NewReference((PyObject *)ctx);
}
else
#endif
{
ctx = PyObject_GC_New(PyContext, &PyContext_Type); ctx = PyObject_GC_New(PyContext, &PyContext_Type);
if (ctx == NULL) { if (ctx == NULL) {
return NULL; return NULL;
@ -471,19 +450,7 @@ context_tp_dealloc(PyContext *self)
} }
(void)context_tp_clear(self); (void)context_tp_clear(self);
#ifdef WITH_FREELISTS _Py_FREELIST_FREE(contexts, self, Py_TYPE(self)->tp_free);
struct _Py_context_freelist *context_freelist = get_context_freelist();
if (context_freelist->numfree >= 0 && context_freelist->numfree < PyContext_MAXFREELIST) {
context_freelist->numfree++;
self->ctx_weakreflist = (PyObject *)context_freelist->items;
context_freelist->items = self;
OBJECT_STAT_INC(to_freelist);
}
else
#endif
{
Py_TYPE(self)->tp_free(self);
}
} }
static PyObject * static PyObject *
@ -1264,24 +1231,6 @@ get_token_missing(void)
/////////////////////////// ///////////////////////////
void
_PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
{
#ifdef WITH_FREELISTS
struct _Py_context_freelist *state = &freelists->contexts;
for (; state->numfree > 0; state->numfree--) {
PyContext *ctx = state->items;
state->items = (PyContext *)ctx->ctx_weakreflist;
ctx->ctx_weakreflist = NULL;
PyObject_GC_Del(ctx);
}
if (is_finalization) {
state->numfree = -1;
}
#endif
}
PyStatus PyStatus
_PyContext_Init(PyInterpreterState *interp) _PyContext_Init(PyInterpreterState *interp)
{ {

View File

@ -4,6 +4,7 @@
#include "pycore_ceval.h" // _Py_set_eval_breaker_bit() #include "pycore_ceval.h" // _Py_set_eval_breaker_bit()
#include "pycore_context.h" #include "pycore_context.h"
#include "pycore_dict.h" // _PyDict_MaybeUntrack() #include "pycore_dict.h" // _PyDict_MaybeUntrack()
#include "pycore_freelist.h" // _PyObject_ClearFreeLists()
#include "pycore_initconfig.h" #include "pycore_initconfig.h"
#include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_interp.h" // PyInterpreterState.gc
#include "pycore_object.h" #include "pycore_object.h"

View File

@ -1,5 +1,5 @@
#include "Python.h" #include "Python.h"
#include "pycore_pystate.h" // _Py_ClearFreeLists() #include "pycore_freelist.h" // _PyObject_ClearFreeLists()
#ifndef Py_GIL_DISABLED #ifndef Py_GIL_DISABLED

View File

@ -8,24 +8,11 @@
extern _PyObjectStackChunk *_PyObjectStackChunk_New(void); extern _PyObjectStackChunk *_PyObjectStackChunk_New(void);
extern void _PyObjectStackChunk_Free(_PyObjectStackChunk *); extern void _PyObjectStackChunk_Free(_PyObjectStackChunk *);
static struct _Py_object_stack_freelist *
get_object_stack_freelist(void)
{
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
return &freelists->object_stacks;
}
_PyObjectStackChunk * _PyObjectStackChunk *
_PyObjectStackChunk_New(void) _PyObjectStackChunk_New(void)
{ {
_PyObjectStackChunk *buf; _PyObjectStackChunk *buf = _Py_FREELIST_POP_MEM(object_stack_chunks);
struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist(); if (buf == NULL) {
if (obj_stack_freelist->numfree > 0) {
buf = obj_stack_freelist->items;
obj_stack_freelist->items = buf->prev;
obj_stack_freelist->numfree--;
}
else {
// NOTE: we use PyMem_RawMalloc() here because this is used by the GC // NOTE: we use PyMem_RawMalloc() here because this is used by the GC
// during mimalloc heap traversal. In that context, it is not safe to // during mimalloc heap traversal. In that context, it is not safe to
// allocate mimalloc memory, such as via PyMem_Malloc(). // allocate mimalloc memory, such as via PyMem_Malloc().
@ -43,17 +30,7 @@ void
_PyObjectStackChunk_Free(_PyObjectStackChunk *buf) _PyObjectStackChunk_Free(_PyObjectStackChunk *buf)
{ {
assert(buf->n == 0); assert(buf->n == 0);
struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist(); _Py_FREELIST_FREE(object_stack_chunks, buf, PyMem_RawFree);
if (obj_stack_freelist->numfree >= 0 &&
obj_stack_freelist->numfree < _PyObjectStackChunk_MAXFREELIST)
{
buf->prev = obj_stack_freelist->items;
obj_stack_freelist->items = buf;
obj_stack_freelist->numfree++;
}
else {
PyMem_RawFree(buf);
}
} }
void void
@ -87,22 +64,3 @@ _PyObjectStack_Merge(_PyObjectStack *dst, _PyObjectStack *src)
dst->head = src->head; dst->head = src->head;
src->head = NULL; src->head = NULL;
} }
void
_PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
{
if (!is_finalization) {
// Ignore requests to clear the free list during GC. We use object
// stacks during GC, so emptying the free-list is counterproductive.
return;
}
struct _Py_object_stack_freelist *freelist = &freelists->object_stacks;
while (freelist->numfree > 0) {
_PyObjectStackChunk *buf = freelist->items;
freelist->items = buf->prev;
freelist->numfree--;
PyMem_RawFree(buf);
}
freelist->numfree = -1;
}

View File

@ -9,6 +9,7 @@
#include "pycore_dict.h" // _PyDict_Fini() #include "pycore_dict.h" // _PyDict_Fini()
#include "pycore_exceptions.h" // _PyExc_InitTypes() #include "pycore_exceptions.h" // _PyExc_InitTypes()
#include "pycore_fileutils.h" // _Py_ResetForceASCII() #include "pycore_fileutils.h" // _Py_ResetForceASCII()
#include "pycore_freelist.h" // _PyObject_ClearFreeLists()
#include "pycore_floatobject.h" // _PyFloat_InitTypes() #include "pycore_floatobject.h" // _PyFloat_InitTypes()
#include "pycore_global_objects_fini_generated.h" // "_PyStaticObjects_CheckRefcnt() #include "pycore_global_objects_fini_generated.h" // "_PyStaticObjects_CheckRefcnt()
#include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_import.h" // _PyImport_BootstrapImp()
@ -1843,7 +1844,7 @@ finalize_interp_types(PyInterpreterState *interp)
#ifndef Py_GIL_DISABLED #ifndef Py_GIL_DISABLED
// With Py_GIL_DISABLED: // With Py_GIL_DISABLED:
// the freelists for the current thread state have already been cleared. // the freelists for the current thread state have already been cleared.
struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); struct _Py_freelists *freelists = _Py_freelists_GET();
_PyObject_ClearFreeLists(freelists, 1); _PyObject_ClearFreeLists(freelists, 1);
#endif #endif

View File

@ -9,9 +9,9 @@
#include "pycore_dtoa.h" // _dtoa_state_INIT() #include "pycore_dtoa.h" // _dtoa_state_INIT()
#include "pycore_emscripten_trampoline.h" // _Py_EmscriptenTrampoline_Init() #include "pycore_emscripten_trampoline.h" // _Py_EmscriptenTrampoline_Init()
#include "pycore_frame.h" #include "pycore_frame.h"
#include "pycore_freelist.h" // _PyObject_ClearFreeLists()
#include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_initconfig.h" // _PyStatus_OK()
#include "pycore_object.h" // _PyType_InitCache() #include "pycore_object.h" // _PyType_InitCache()
#include "pycore_object_stack.h" // _PyObjectStackChunk_ClearFreeList()
#include "pycore_parking_lot.h" // _PyParkingLot_AfterFork() #include "pycore_parking_lot.h" // _PyParkingLot_AfterFork()
#include "pycore_pyerrors.h" // _PyErr_Clear() #include "pycore_pyerrors.h" // _PyErr_Clear()
#include "pycore_pylifecycle.h" // _PyAST_Fini() #include "pycore_pylifecycle.h" // _PyAST_Fini()
@ -1738,7 +1738,7 @@ PyThreadState_Clear(PyThreadState *tstate)
#ifdef Py_GIL_DISABLED #ifdef Py_GIL_DISABLED
// Each thread should clear own freelists in free-threading builds. // Each thread should clear own freelists in free-threading builds.
struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); struct _Py_freelists *freelists = _Py_freelists_GET();
_PyObject_ClearFreeLists(freelists, 1); _PyObject_ClearFreeLists(freelists, 1);
// Remove ourself from the biased reference counting table of threads. // Remove ourself from the biased reference counting table of threads.