Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-100240: Use a consistent implementation for freelists (#121934) 

This combines and updates our freelist handling to use a consistent
implementation. Objects in the freelist are linked together using the
first word of memory block.

If configured with freelists disabled, these operations are essentially
no-ops.
This commit is contained in:
Sam Gross 2024-07-22 12:08:27 -04:00 committed by GitHub
parent 2408a8a22b
commit 5716cc3529
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
27 changed files with 295 additions and 705 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Include/internal/pycore_context.h
View File

@ -5,7 +5,6 @@
# error "this header requires Py_BUILD_CORE define"
#endif
#include "pycore_freelist.h" // _PyFreeListState
#include "pycore_hamt.h" // PyHamtObject

1
Include/internal/pycore_dict.h
View File

@ -8,7 +8,6 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
#include "pycore_freelist.h" // _PyFreeListState
#include "pycore_object.h" // PyManagedDictPointer
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_ACQUIRE

1
Include/internal/pycore_floatobject.h
View File

@ -8,7 +8,6 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
#include "pycore_freelist.h" // _PyFreeListState
#include "pycore_unicodeobject.h" // _PyUnicodeWriter
/* runtime lifecycle */

227
Include/internal/pycore_freelist.h
View File

@ -8,144 +8,109 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
// PyTuple_MAXSAVESIZE - largest tuple to save on free list
// PyTuple_MAXFREELIST - maximum number of tuples of each size to save
#include "pycore_freelist_state.h" // struct _Py_freelists
#include "pycore_object.h" // _PyObject_IS_GC
#include "pycore_pystate.h" // _PyThreadState_GET
#include "pycore_code.h" // OBJECT_STAT_INC
#ifdef WITH_FREELISTS
// with freelists
# define PyTuple_MAXSAVESIZE 20
# define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE
# define PyTuple_MAXFREELIST 2000
# define PyList_MAXFREELIST 80
# define PyDict_MAXFREELIST 80
# define PyFloat_MAXFREELIST 100
# define PyContext_MAXFREELIST 255
# define _PyAsyncGen_MAXFREELIST 80
# define _PyObjectStackChunk_MAXFREELIST 4
static inline struct _Py_freelists *
_Py_freelists_GET(void)
{
PyThreadState *tstate = _PyThreadState_GET();
#ifdef Py_DEBUG
_Py_EnsureTstateNotNULL(tstate);
#endif
#ifdef Py_GIL_DISABLED
return &((_PyThreadStateImpl*)tstate)->freelists;
#else
# define PyTuple_NFREELISTS 0
# define PyTuple_MAXFREELIST 0
# define PyList_MAXFREELIST 0
# define PyDict_MAXFREELIST 0
# define PyFloat_MAXFREELIST 0
# define PyContext_MAXFREELIST 0
# define _PyAsyncGen_MAXFREELIST 0
# define _PyObjectStackChunk_MAXFREELIST 0
return &tstate->interp->object_state.freelists;
#endif
}
struct _Py_list_freelist {
#ifdef WITH_FREELISTS
PyListObject *items[PyList_MAXFREELIST];
int numfree;
#endif
};
struct _Py_tuple_freelist {
#if WITH_FREELISTS
/* There is one freelist for each size from 1 to PyTuple_MAXSAVESIZE.
The empty tuple is handled separately.
Each tuple stored in the array is the head of the linked list
(and the next available tuple) for that size. The actual tuple
object is used as the linked list node, with its first item
(ob_item[0]) pointing to the next node (i.e. the previous head).
Each linked list is initially NULL. */
PyTupleObject *items[PyTuple_NFREELISTS];
int numfree[PyTuple_NFREELISTS];
#ifndef WITH_FREELISTS
#define _Py_FREELIST_FREE(NAME, op, freefunc) freefunc(op)
#define _Py_FREELIST_PUSH(NAME, op, limit) (0)
#define _Py_FREELIST_POP(TYPE, NAME) (NULL)
#define _Py_FREELIST_POP_MEM(NAME) (NULL)
#define _Py_FREELIST_SIZE(NAME) (0)
#else
char _unused; // Empty structs are not allowed.
// Pushes `op` to the freelist, calls `freefunc` if the freelist is full
#define _Py_FREELIST_FREE(NAME, op, freefunc) \
_PyFreeList_Free(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), \
Py_ ## NAME ## _MAXFREELIST, freefunc)
// Pushes `op` to the freelist, returns 1 if successful, 0 if the freelist is full
#define _Py_FREELIST_PUSH(NAME, op, limit) \
_PyFreeList_Push(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), limit)
// Pops a PyObject from the freelist, returns NULL if the freelist is empty.
#define _Py_FREELIST_POP(TYPE, NAME) \
_Py_CAST(TYPE*, _PyFreeList_Pop(&_Py_freelists_GET()->NAME))
// Pops a non-PyObject data structure from the freelist, returns NULL if the
// freelist is empty.
#define _Py_FREELIST_POP_MEM(NAME) \
_PyFreeList_PopMem(&_Py_freelists_GET()->NAME)
#define _Py_FREELIST_SIZE(NAME) (int)((_Py_freelists_GET()->NAME).size)
static inline int
_PyFreeList_Push(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize)
{
if (fl->size < maxsize && fl->size >= 0) {
*(void **)obj = fl->freelist;
fl->freelist = obj;
fl->size++;
OBJECT_STAT_INC(to_freelist);
return 1;
}
return 0;
}
static inline void
_PyFreeList_Free(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize,
freefunc dofree)
{
if (!_PyFreeList_Push(fl, obj, maxsize)) {
dofree(obj);
}
}
static inline void *
_PyFreeList_PopNoStats(struct _Py_freelist *fl)
{
void *obj = fl->freelist;
if (obj != NULL) {
assert(fl->size > 0);
fl->freelist = *(void **)obj;
fl->size--;
}
return obj;
}
static inline PyObject *
_PyFreeList_Pop(struct _Py_freelist *fl)
{
PyObject *op = _PyFreeList_PopNoStats(fl);
if (op != NULL) {
OBJECT_STAT_INC(from_freelist);
_Py_NewReference(op);
}
return op;
}
static inline void *
_PyFreeList_PopMem(struct _Py_freelist *fl)
{
void *op = _PyFreeList_PopNoStats(fl);
if (op != NULL) {
OBJECT_STAT_INC(from_freelist);
}
return op;
}
#endif
};
struct _Py_float_freelist {
#ifdef WITH_FREELISTS
/* Special free list
free_list is a singly-linked list of available PyFloatObjects,
linked via abuse of their ob_type members. */
int numfree;
PyFloatObject *items;
#endif
};
struct _Py_dict_freelist {
#ifdef WITH_FREELISTS
/* Dictionary reuse scheme to save calls to malloc and free */
PyDictObject *items[PyDict_MAXFREELIST];
int numfree;
#endif
};
struct _Py_dictkeys_freelist {
#ifdef WITH_FREELISTS
/* Dictionary keys reuse scheme to save calls to malloc and free */
PyDictKeysObject *items[PyDict_MAXFREELIST];
int numfree;
#endif
};
struct _Py_slice_freelist {
#ifdef WITH_FREELISTS
/* Using a cache is very effective since typically only a single slice is
created and then deleted again. */
PySliceObject *slice_cache;
#endif
};
struct _Py_context_freelist {
#ifdef WITH_FREELISTS
// List of free PyContext objects
PyContext *items;
int numfree;
#endif
};
struct _Py_async_gen_freelist {
#ifdef WITH_FREELISTS
/* Freelists boost performance 6-10%; they also reduce memory
fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend
are short-living objects that are instantiated for every
__anext__() call. */
struct _PyAsyncGenWrappedValue* items[_PyAsyncGen_MAXFREELIST];
int numfree;
#endif
};
struct _Py_async_gen_asend_freelist {
#ifdef WITH_FREELISTS
struct PyAsyncGenASend* items[_PyAsyncGen_MAXFREELIST];
int numfree;
#endif
};
struct _PyObjectStackChunk;
struct _Py_object_stack_freelist {
struct _PyObjectStackChunk *items;
Py_ssize_t numfree;
};
struct _Py_object_freelists {
struct _Py_float_freelist floats;
struct _Py_tuple_freelist tuples;
struct _Py_list_freelist lists;
struct _Py_dict_freelist dicts;
struct _Py_dictkeys_freelist dictkeys;
struct _Py_slice_freelist slices;
struct _Py_context_freelist contexts;
struct _Py_async_gen_freelist async_gens;
struct _Py_async_gen_asend_freelist async_gen_asends;
struct _Py_object_stack_freelist object_stacks;
};
extern void _PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization);
extern void _PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
extern void _PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
extern void _PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
extern void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
extern void _PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
extern void _PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization);
extern void _PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
extern void _PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
extern void _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization);
#ifdef __cplusplus
}

59
Include/internal/pycore_freelist_state.h Normal file
View File

@ -0,0 +1,59 @@
#ifndef Py_INTERNAL_FREELIST_STATE_H
#define Py_INTERNAL_FREELIST_STATE_H
#ifdef __cplusplus
extern "C" {
#endif
#ifndef Py_BUILD_CORE
# error "this header requires Py_BUILD_CORE define"
#endif
#ifdef WITH_FREELISTS
// with freelists
# define PyTuple_MAXSAVESIZE 20 // Largest tuple to save on freelist
# define Py_tuple_MAXFREELIST 2000 // Maximum number of tuples of each size to save
# define Py_lists_MAXFREELIST 80
# define Py_dicts_MAXFREELIST 80
# define Py_dictkeys_MAXFREELIST 80
# define Py_floats_MAXFREELIST 100
# define Py_slices_MAXFREELIST 1
# define Py_contexts_MAXFREELIST 255
# define Py_async_gens_MAXFREELIST 80
# define Py_async_gen_asends_MAXFREELIST 80
# define Py_object_stack_chunks_MAXFREELIST 4
#else
# define PyTuple_MAXSAVESIZE 0
#endif
// A generic freelist of either PyObjects or other data structures.
struct _Py_freelist {
// Entries are linked together using the first word of the the object.
// For PyObjects, this overlaps with the `ob_refcnt` field or the `ob_tid`
// field.
void *freelist;
// The number of items in the free list or -1 if the free list is disabled
Py_ssize_t size;
};
struct _Py_freelists {
#ifdef WITH_FREELISTS
struct _Py_freelist floats;
struct _Py_freelist tuples[PyTuple_MAXSAVESIZE];
struct _Py_freelist lists;
struct _Py_freelist dicts;
struct _Py_freelist dictkeys;
struct _Py_freelist slices;
struct _Py_freelist contexts;
struct _Py_freelist async_gens;
struct _Py_freelist async_gen_asends;
struct _Py_freelist object_stack_chunks;
#else
char _unused; // Empty structs are not allowed.
#endif
};
#ifdef __cplusplus
}
#endif
#endif /* !Py_INTERNAL_FREELIST_STATE_H */

2
Include/internal/pycore_gc.h
View File

@ -8,8 +8,6 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
#include "pycore_freelist.h" // _PyFreeListState
/* GC information is stored BEFORE the object structure. */
typedef struct {
// Pointer to next object in the list.

2
Include/internal/pycore_list.h
View File

@ -8,8 +8,6 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
#include "pycore_freelist.h" // _PyFreeListState
PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *);
extern void _PyList_DebugMallocStats(FILE *out);

2
Include/internal/pycore_object_stack.h
View File

@ -1,8 +1,6 @@
#ifndef Py_INTERNAL_OBJECT_STACK_H
#define Py_INTERNAL_OBJECT_STACK_H
#include "pycore_freelist.h" // _PyFreeListState
#ifdef __cplusplus
extern "C" {
#endif

6
Include/internal/pycore_object_state.h
View File

@ -8,8 +8,8 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
#include "pycore_freelist.h" // _PyObject_freelists
#include "pycore_hashtable.h" // _Py_hashtable_t
#include "pycore_freelist_state.h" // _Py_freelists
#include "pycore_hashtable.h" // _Py_hashtable_t
struct _py_object_runtime_state {
#ifdef Py_REF_DEBUG
@ -20,7 +20,7 @@ struct _py_object_runtime_state {
struct _py_object_state {
#if !defined(Py_GIL_DISABLED)
struct _Py_object_freelists freelists;
struct _Py_freelists freelists;
#endif
#ifdef Py_REF_DEBUG
Py_ssize_t reftotal;

16
Include/internal/pycore_pystate.h
View File

@ -8,11 +8,9 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
#include "pycore_freelist.h" // _PyFreeListState
#include "pycore_runtime.h" // _PyRuntime
#include "pycore_tstate.h" // _PyThreadStateImpl
// Values for PyThreadState.state. A thread must be in the "attached" state
// before calling most Python APIs. If the GIL is enabled, then "attached"
// implies that the thread holds the GIL and "detached" implies that the
@ -279,20 +277,6 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void);
// See also PyInterpreterState_Get() and _PyInterpreterState_GET().
extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void);
static inline struct _Py_object_freelists* _Py_object_freelists_GET(void)
{
PyThreadState *tstate = _PyThreadState_GET();
#ifdef Py_DEBUG
_Py_EnsureTstateNotNULL(tstate);
#endif
#ifdef Py_GIL_DISABLED
return &((_PyThreadStateImpl*)tstate)->freelists;
#else
return &tstate->interp->object_state.freelists;
#endif
}
#ifdef __cplusplus
}
#endif

10
Include/internal/pycore_tstate.h
View File

@ -8,10 +8,10 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
#include "pycore_brc.h" // struct _brc_thread_state
#include "pycore_freelist.h" // struct _Py_freelist_state
#include "pycore_mimalloc.h" // struct _mimalloc_thread_state
#include "pycore_qsbr.h" // struct qsbr
#include "pycore_brc.h" // struct _brc_thread_state
#include "pycore_freelist_state.h" // struct _Py_freelists
#include "pycore_mimalloc.h" // struct _mimalloc_thread_state
#include "pycore_qsbr.h" // struct qsbr
// Every PyThreadState is actually allocated as a _PyThreadStateImpl. The
@ -29,7 +29,7 @@ typedef struct _PyThreadStateImpl {
#ifdef Py_GIL_DISABLED
struct _gc_thread_state gc;
struct _mimalloc_thread_state mimalloc;
struct _Py_object_freelists freelists;
struct _Py_freelists freelists;
struct _brc_thread_state brc;
#endif

1
Makefile.pre.in
View File

@ -1183,6 +1183,7 @@ PYTHON_HEADERS= \
$(srcdir)/Include/internal/pycore_format.h \
$(srcdir)/Include/internal/pycore_frame.h \
$(srcdir)/Include/internal/pycore_freelist.h \
$(srcdir)/Include/internal/pycore_freelist_state.h \
$(srcdir)/Include/internal/pycore_function.h \
$(srcdir)/Include/internal/pycore_gc.h \
$(srcdir)/Include/internal/pycore_genobject.h \

103
Objects/dictobject.c
View File

@ -395,44 +395,6 @@ static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj);
#include "clinic/dictobject.c.h"
#ifdef WITH_FREELISTS
static struct _Py_dict_freelist *
get_dict_freelist(void)
{
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
return &freelists->dicts;
}
static struct _Py_dictkeys_freelist *
get_dictkeys_freelist(void)
{
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
return &freelists->dictkeys;
}
#endif
void
_PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
{
#ifdef WITH_FREELISTS
struct _Py_dict_freelist *freelist = &freelists->dicts;
while (freelist->numfree > 0) {
PyDictObject *op = freelist->items[--freelist->numfree];
assert(PyDict_CheckExact(op));
PyObject_GC_Del(op);
}
struct _Py_dictkeys_freelist *keys_freelist = &freelists->dictkeys;
while (keys_freelist->numfree > 0) {
PyMem_Free(keys_freelist->items[--keys_freelist->numfree]);
}
if (is_finalization) {
freelist->numfree = -1;
keys_freelist->numfree = -1;
}
#endif
}
static inline Py_hash_t
unicode_get_hash(PyObject *o)
{
@ -445,12 +407,12 @@ void
_PyDict_DebugMallocStats(FILE *out)
{
#ifdef WITH_FREELISTS
struct _Py_dict_freelist *dict_freelist = get_dict_freelist();
_PyDebugAllocatorStats(out, "free PyDictObject",
dict_freelist->numfree, sizeof(PyDictObject));
struct _Py_dictkeys_freelist *dictkeys_freelist = get_dictkeys_freelist();
_Py_FREELIST_SIZE(dicts),
sizeof(PyDictObject));
_PyDebugAllocatorStats(out, "free PyDictKeysObject",
dictkeys_freelist->numfree, sizeof(PyDictKeysObject));
_Py_FREELIST_SIZE(dictkeys),
sizeof(PyDictKeysObject));
#endif
}
@ -785,7 +747,6 @@ _PyDict_CheckConsistency(PyObject *op, int check_content)
static PyDictKeysObject*
new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode)
{
PyDictKeysObject *dk;
Py_ssize_t usable;
int log2_bytes;
size_t entry_size = unicode ? sizeof(PyDictUnicodeEntry) : sizeof(PyDictKeyEntry);
@ -808,15 +769,11 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode)
log2_bytes = log2_size + 2;
}
#ifdef WITH_FREELISTS
struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist();
if (log2_size == PyDict_LOG_MINSIZE && unicode && freelist->numfree > 0) {
dk = freelist->items[--freelist->numfree];
OBJECT_STAT_INC(from_freelist);
PyDictKeysObject *dk = NULL;
if (log2_size == PyDict_LOG_MINSIZE && unicode) {
dk = _Py_FREELIST_POP_MEM(dictkeys);
}
else
#endif
{
if (dk == NULL) {
dk = PyMem_Malloc(sizeof(PyDictKeysObject)
+ ((size_t)1 << log2_bytes)
+ entry_size * usable);
@ -852,18 +809,12 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr)
return;
}
#endif
#ifdef WITH_FREELISTS
struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist();
if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE
&& freelist->numfree < PyDict_MAXFREELIST
&& freelist->numfree >= 0
&& DK_IS_UNICODE(keys)) {
freelist->items[freelist->numfree++] = keys;
OBJECT_STAT_INC(to_freelist);
return;
if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE && keys->dk_kind == DICT_KEYS_UNICODE) {
_Py_FREELIST_FREE(dictkeys, keys, PyMem_Free);
}
else {
PyMem_Free(keys);
}
#endif
PyMem_Free(keys);
}
static size_t
@ -912,20 +863,9 @@ new_dict(PyInterpreterState *interp,
PyDictKeysObject *keys, PyDictValues *values,
Py_ssize_t used, int free_values_on_failure)
{
PyDictObject *mp;
assert(keys != NULL);
#ifdef WITH_FREELISTS
struct _Py_dict_freelist *freelist = get_dict_freelist();
if (freelist->numfree > 0) {
mp = freelist->items[--freelist->numfree];
assert (mp != NULL);
assert (Py_IS_TYPE(mp, &PyDict_Type));
OBJECT_STAT_INC(from_freelist);
_Py_NewReference((PyObject *)mp);
}
else
#endif
{
PyDictObject *mp = _Py_FREELIST_POP(PyDictObject, dicts);
if (mp == NULL) {
mp = PyObject_GC_New(PyDictObject, &PyDict_Type);
if (mp == NULL) {
dictkeys_decref(interp, keys, false);
@ -935,6 +875,7 @@ new_dict(PyInterpreterState *interp,
return NULL;
}
}
assert(Py_IS_TYPE(mp, &PyDict_Type));
mp->ma_keys = keys;
mp->ma_values = values;
mp->ma_used = used;
@ -3153,16 +3094,10 @@ dict_dealloc(PyObject *self)
assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS);
dictkeys_decref(interp, keys, false);
}
#ifdef WITH_FREELISTS
struct _Py_dict_freelist *freelist = get_dict_freelist();
if (freelist->numfree < PyDict_MAXFREELIST && freelist->numfree >=0 &&
Py_IS_TYPE(mp, &PyDict_Type)) {
freelist->items[freelist->numfree++] = mp;
OBJECT_STAT_INC(to_freelist);
if (Py_IS_TYPE(mp, &PyDict_Type)) {
_Py_FREELIST_FREE(dicts, mp, Py_TYPE(mp)->tp_free);
}
else
#endif
{
else {
Py_TYPE(mp)->tp_free((PyObject *)mp);
}
Py_TRASHCAN_END

68
Objects/floatobject.c
View File

@ -7,8 +7,8 @@
#include "pycore_abstract.h" // _PyNumber_Index()
#include "pycore_dtoa.h" // _Py_dg_dtoa()
#include "pycore_floatobject.h" // _PyFloat_FormatAdvancedWriter()
#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
#include "pycore_initconfig.h" // _PyStatus_OK()
#include "pycore_interp.h" // _Py_float_freelist
#include "pycore_long.h" // _PyLong_GetOne()
#include "pycore_modsupport.h" // _PyArg_NoKwnames()
#include "pycore_object.h" // _PyObject_Init(), _PyDebugAllocatorStats()
@ -26,16 +26,6 @@ class float "PyObject *" "&PyFloat_Type"
#include "clinic/floatobject.c.h"
#ifdef WITH_FREELISTS
static struct _Py_float_freelist *
get_float_freelist(void)
{
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
assert(freelists != NULL);
return &freelists->floats;
}
#endif
double
PyFloat_GetMax(void)
@ -132,24 +122,14 @@ PyFloat_GetInfo(void)
PyObject *
PyFloat_FromDouble(double fval)
{
PyFloatObject *op;
#ifdef WITH_FREELISTS
struct _Py_float_freelist *float_freelist = get_float_freelist();
op = float_freelist->items;
if (op != NULL) {
float_freelist->items = (PyFloatObject *) Py_TYPE(op);
float_freelist->numfree--;
OBJECT_STAT_INC(from_freelist);
}
else
#endif
{
PyFloatObject *op = _Py_FREELIST_POP(PyFloatObject, floats);
if (op == NULL) {
op = PyObject_Malloc(sizeof(PyFloatObject));
if (!op) {
return PyErr_NoMemory();
}
_PyObject_Init((PyObject*)op, &PyFloat_Type);
}
_PyObject_Init((PyObject*)op, &PyFloat_Type);
op->ob_fval = fval;
return (PyObject *) op;
}
@ -248,20 +228,7 @@ void
_PyFloat_ExactDealloc(PyObject *obj)
{
assert(PyFloat_CheckExact(obj));
PyFloatObject *op = (PyFloatObject *)obj;
#ifdef WITH_FREELISTS
struct _Py_float_freelist *float_freelist = get_float_freelist();
if (float_freelist->numfree >= PyFloat_MAXFREELIST || float_freelist->numfree < 0) {
PyObject_Free(op);
return;
}
float_freelist->numfree++;
Py_SET_TYPE(op, (PyTypeObject *)float_freelist->items);
float_freelist->items = op;
OBJECT_STAT_INC(to_freelist);
#else
PyObject_Free(op);
#endif
_Py_FREELIST_FREE(floats, obj, PyObject_Free);
}
static void
@ -1994,27 +1961,6 @@ _PyFloat_InitTypes(PyInterpreterState *interp)
return _PyStatus_OK();
}
void
_PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
{
#ifdef WITH_FREELISTS
struct _Py_float_freelist *state = &freelists->floats;
PyFloatObject *f = state->items;
while (f != NULL) {
PyFloatObject *next = (PyFloatObject*) Py_TYPE(f);
PyObject_Free(f);
f = next;
}
state->items = NULL;
if (is_finalization) {
state->numfree = -1;
}
else {
state->numfree = 0;
}
#endif
}
void
_PyFloat_FiniType(PyInterpreterState *interp)
{
@ -2026,10 +1972,10 @@ void
_PyFloat_DebugMallocStats(FILE *out)
{
#ifdef WITH_FREELISTS
struct _Py_float_freelist *float_freelist = get_float_freelist();
_PyDebugAllocatorStats(out,
"free PyFloatObject",
float_freelist->numfree, sizeof(PyFloatObject));
_Py_FREELIST_SIZE(floats),
sizeof(PyFloatObject));
#endif
}

108
Objects/genobject.c
View File

@ -6,7 +6,7 @@
#include "pycore_call.h" // _PyObject_CallNoArgs()
#include "pycore_ceval.h" // _PyEval_EvalFrame()
#include "pycore_frame.h" // _PyInterpreterFrame
#include "pycore_freelist.h" // struct _Py_async_gen_freelist
#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
#include "pycore_gc.h" // _PyGC_CLEAR_FINALIZED()
#include "pycore_modsupport.h" // _PyArg_CheckPositional()
#include "pycore_object.h" // _PyObject_GC_UNTRACK()
@ -1629,23 +1629,6 @@ PyTypeObject PyAsyncGen_Type = {
};
#ifdef WITH_FREELISTS
static struct _Py_async_gen_freelist *
get_async_gen_freelist(void)
{
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
return &freelists->async_gens;
}
static struct _Py_async_gen_asend_freelist *
get_async_gen_asend_freelist(void)
{
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
return &freelists->async_gen_asends;
}
#endif
PyObject *
PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
{
@ -1662,36 +1645,6 @@ PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
return (PyObject*)o;
}
void
_PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelist_state, int is_finalization)
{
#ifdef WITH_FREELISTS
struct _Py_async_gen_freelist *freelist = &freelist_state->async_gens;
while (freelist->numfree > 0) {
_PyAsyncGenWrappedValue *o;
o = freelist->items[--freelist->numfree];
assert(_PyAsyncGenWrappedValue_CheckExact(o));
PyObject_GC_Del(o);
}
struct _Py_async_gen_asend_freelist *asend_freelist = &freelist_state->async_gen_asends;
while (asend_freelist->numfree > 0) {
PyAsyncGenASend *o;
o = asend_freelist->items[--asend_freelist->numfree];
assert(Py_IS_TYPE(o, &_PyAsyncGenASend_Type));
PyObject_GC_Del(o);
}
if (is_finalization) {
freelist->numfree = -1;
asend_freelist->numfree = -1;
}
#endif
}
static PyObject *
async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result)
{
@ -1735,18 +1688,11 @@ async_gen_asend_dealloc(PyAsyncGenASend *o)
_PyObject_GC_UNTRACK((PyObject *)o);
Py_CLEAR(o->ags_gen);
Py_CLEAR(o->ags_sendval);
#ifdef WITH_FREELISTS
struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist();
if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) {
assert(PyAsyncGenASend_CheckExact(o));
_PyGC_CLEAR_FINALIZED((PyObject *)o);
freelist->items[freelist->numfree++] = o;
}
else
#endif
{
PyObject_GC_Del(o);
}
assert(PyAsyncGenASend_CheckExact(o));
_PyGC_CLEAR_FINALIZED((PyObject *)o);
_Py_FREELIST_FREE(async_gen_asends, o, PyObject_GC_Del);
}
static int
@ -1936,17 +1882,8 @@ PyTypeObject _PyAsyncGenASend_Type = {
static PyObject *
async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval)
{
PyAsyncGenASend *o;
#ifdef WITH_FREELISTS
struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist();
if (freelist->numfree > 0) {
freelist->numfree--;
o = freelist->items[freelist->numfree];
_Py_NewReference((PyObject *)o);
}
else
#endif
{
PyAsyncGenASend *o = _Py_FREELIST_POP(PyAsyncGenASend, async_gen_asends);
if (o == NULL) {
o = PyObject_GC_New(PyAsyncGenASend, &_PyAsyncGenASend_Type);
if (o == NULL) {
return NULL;
@ -1972,18 +1909,7 @@ async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o)
{
_PyObject_GC_UNTRACK((PyObject *)o);
Py_CLEAR(o->agw_val);
#ifdef WITH_FREELISTS
struct _Py_async_gen_freelist *freelist = get_async_gen_freelist();
if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) {
assert(_PyAsyncGenWrappedValue_CheckExact(o));
freelist->items[freelist->numfree++] = o;
OBJECT_STAT_INC(to_freelist);
}
else
#endif
{
PyObject_GC_Del(o);
}
_Py_FREELIST_FREE(async_gens, o, PyObject_GC_Del);
}
@ -2042,27 +1968,17 @@ PyTypeObject _PyAsyncGenWrappedValue_Type = {
PyObject *
_PyAsyncGenValueWrapperNew(PyThreadState *tstate, PyObject *val)
{
_PyAsyncGenWrappedValue *o;
assert(val);
#ifdef WITH_FREELISTS
struct _Py_async_gen_freelist *freelist = get_async_gen_freelist();
if (freelist->numfree > 0) {
freelist->numfree--;
o = freelist->items[freelist->numfree];
OBJECT_STAT_INC(from_freelist);
assert(_PyAsyncGenWrappedValue_CheckExact(o));
_Py_NewReference((PyObject*)o);
}
else
#endif
{
_PyAsyncGenWrappedValue *o = _Py_FREELIST_POP(_PyAsyncGenWrappedValue, async_gens);
if (o == NULL) {
o = PyObject_GC_New(_PyAsyncGenWrappedValue,
&_PyAsyncGenWrappedValue_Type);
if (o == NULL) {
return NULL;
}
}
assert(_PyAsyncGenWrappedValue_CheckExact(o));
o->agw_val = Py_NewRef(val);
_PyObject_GC_TRACK((PyObject*)o);
return (PyObject*)o;

59
Objects/listobject.c
View File

@ -4,6 +4,7 @@
#include "pycore_abstract.h" // _PyIndex_Check()
#include "pycore_ceval.h" // _PyEval_GetBuiltin()
#include "pycore_dict.h" // _PyDictViewObject
#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
#include "pycore_pyatomic_ft_wrappers.h"
#include "pycore_interp.h" // PyInterpreterState.list
#include "pycore_list.h" // struct _Py_list_freelist, _PyListIterObject
@ -23,16 +24,6 @@ class list "PyListObject *" "&PyList_Type"
_Py_DECLARE_STR(list_err, "list index out of range");
#ifdef WITH_FREELISTS
static struct _Py_list_freelist *
get_list_freelist(void)
{
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
assert(freelists != NULL);
return &freelists->lists;
}
#endif
#ifdef Py_GIL_DISABLED
typedef struct {
Py_ssize_t allocated;
@ -205,55 +196,28 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size)
return 0;
}
void
_PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
{
#ifdef WITH_FREELISTS
struct _Py_list_freelist *state = &freelists->lists;
while (state->numfree > 0) {
PyListObject *op = state->items[--state->numfree];
assert(PyList_CheckExact(op));
PyObject_GC_Del(op);
}
if (is_finalization) {
state->numfree = -1;
}
#endif
}
/* Print summary info about the state of the optimized allocator */
void
_PyList_DebugMallocStats(FILE *out)
{
#ifdef WITH_FREELISTS
struct _Py_list_freelist *list_freelist = get_list_freelist();
_PyDebugAllocatorStats(out,
"free PyListObject",
list_freelist->numfree, sizeof(PyListObject));
_Py_FREELIST_SIZE(lists),
sizeof(PyListObject));
#endif
}
PyObject *
PyList_New(Py_ssize_t size)
{
PyListObject *op;
if (size < 0) {
PyErr_BadInternalCall();
return NULL;
}
#ifdef WITH_FREELISTS
struct _Py_list_freelist *list_freelist = get_list_freelist();
if (PyList_MAXFREELIST && list_freelist->numfree > 0) {
list_freelist->numfree--;
op = list_freelist->items[list_freelist->numfree];
OBJECT_STAT_INC(from_freelist);
_Py_NewReference((PyObject *)op);
}
else
#endif
{
PyListObject *op = _Py_FREELIST_POP(PyListObject, lists);
if (op == NULL) {
op = PyObject_GC_New(PyListObject, &PyList_Type);
if (op == NULL) {
return NULL;
@ -548,16 +512,11 @@ list_dealloc(PyObject *self)
}
free_list_items(op->ob_item, false);
}
#ifdef WITH_FREELISTS
struct _Py_list_freelist *list_freelist = get_list_freelist();
if (list_freelist->numfree < PyList_MAXFREELIST && list_freelist->numfree >= 0 && PyList_CheckExact(op)) {
list_freelist->items[list_freelist->numfree++] = op;
OBJECT_STAT_INC(to_freelist);
if (PyList_CheckExact(op)) {
_Py_FREELIST_FREE(lists, op, PyObject_GC_Del);
}
else
#endif
{
Py_TYPE(op)->tp_free((PyObject *)op);
else {
PyObject_GC_Del(op);
}
Py_TRASHCAN_END
}

57
Objects/object.c
View File

@ -10,6 +10,7 @@
#include "pycore_descrobject.h" // _PyMethodWrapper_Type
#include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes()
#include "pycore_floatobject.h" // _PyFloat_DebugMallocStats()
#include "pycore_freelist.h" // _PyObject_ClearFreeLists()
#include "pycore_initconfig.h" // _PyStatus_EXCEPTION()
#include "pycore_instruction_sequence.h" // _PyInstructionSequence_Type
#include "pycore_hashtable.h" // _Py_hashtable_new()
@ -808,20 +809,54 @@ PyObject_Bytes(PyObject *v)
return PyBytes_FromObject(v);
}
void
_PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization)
#ifdef WITH_FREELISTS
static void
clear_freelist(struct _Py_freelist *freelist, int is_finalization,
freefunc dofree)
{
void *ptr;
while ((ptr = _PyFreeList_PopNoStats(freelist)) != NULL) {
dofree(ptr);
}
assert(freelist->size == 0 || freelist->size == -1);
assert(freelist->freelist == NULL);
if (is_finalization) {
freelist->size = -1;
}
}
static void
free_object(void *obj)
{
PyObject *op = (PyObject *)obj;
Py_TYPE(op)->tp_free(op);
}
#endif
void
_PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization)
{
#ifdef WITH_FREELISTS
// In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear()
// In the default build, freelists are per-interpreter and cleared in finalize_interp_types()
_PyFloat_ClearFreeList(freelists, is_finalization);
_PyTuple_ClearFreeList(freelists, is_finalization);
_PyList_ClearFreeList(freelists, is_finalization);
_PyDict_ClearFreeList(freelists, is_finalization);
_PyContext_ClearFreeList(freelists, is_finalization);
_PyAsyncGen_ClearFreeLists(freelists, is_finalization);
// Only be cleared if is_finalization is true.
_PyObjectStackChunk_ClearFreeList(freelists, is_finalization);
_PySlice_ClearFreeList(freelists, is_finalization);
clear_freelist(&freelists->floats, is_finalization, free_object);
for (Py_ssize_t i = 0; i < PyTuple_MAXSAVESIZE; i++) {
clear_freelist(&freelists->tuples[i], is_finalization, free_object);
}
clear_freelist(&freelists->lists, is_finalization, free_object);
clear_freelist(&freelists->dicts, is_finalization, free_object);
clear_freelist(&freelists->dictkeys, is_finalization, PyMem_Free);
clear_freelist(&freelists->slices, is_finalization, free_object);
clear_freelist(&freelists->contexts, is_finalization, free_object);
clear_freelist(&freelists->async_gens, is_finalization, free_object);
clear_freelist(&freelists->async_gen_asends, is_finalization, free_object);
if (is_finalization) {
// Only clear object stack chunks during finalization. We use object
// stacks during GC, so emptying the free-list is counterproductive.
clear_freelist(&freelists->object_stack_chunks, 1, PyMem_RawFree);
}
#endif
}
/*

39
Objects/sliceobject.c
View File

@ -15,6 +15,7 @@ this type and there is exactly one in existence.
#include "Python.h"
#include "pycore_abstract.h" // _PyIndex_Check()
#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
#include "pycore_long.h" // _PyLong_GetZero()
#include "pycore_modsupport.h" // _PyArg_NoKeywords()
#include "pycore_object.h" // _PyObject_GC_TRACK()
@ -108,20 +109,6 @@ PyObject _Py_EllipsisObject = _PyObject_HEAD_INIT(&PyEllipsis_Type);
/* Slice object implementation */
void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
{
if (!is_finalization) {
return;
}
#ifdef WITH_FREELISTS
PySliceObject *obj = freelists->slices.slice_cache;
if (obj != NULL) {
freelists->slices.slice_cache = NULL;
PyObject_GC_Del(obj);
}
#endif
}
/* start, stop, and step are python objects with None indicating no
index is present.
*/
@ -130,17 +117,8 @@ static PySliceObject *
_PyBuildSlice_Consume2(PyObject *start, PyObject *stop, PyObject *step)
{
assert(start != NULL && stop != NULL && step != NULL);
PySliceObject *obj;
#ifdef WITH_FREELISTS
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
if (freelists->slices.slice_cache != NULL) {
obj = freelists->slices.slice_cache;
freelists->slices.slice_cache = NULL;
_Py_NewReference((PyObject *)obj);
}
else
#endif
{
PySliceObject *obj = _Py_FREELIST_POP(PySliceObject, slices);
if (obj == NULL) {
obj = PyObject_GC_New(PySliceObject, &PySlice_Type);
if (obj == NULL) {
goto error;
@ -369,16 +347,7 @@ slice_dealloc(PySliceObject *r)
Py_DECREF(r->step);
Py_DECREF(r->start);
Py_DECREF(r->stop);
#ifdef WITH_FREELISTS
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
if (freelists->slices.slice_cache == NULL) {
freelists->slices.slice_cache = r;
}
else
#endif
{
PyObject_GC_Del(r);
}
_Py_FREELIST_FREE(slices, r, PyObject_GC_Del);
}
static PyObject *

117
Objects/tupleobject.c
View File

@ -4,6 +4,7 @@
#include "Python.h"
#include "pycore_abstract.h" // _PyIndex_Check()
#include "pycore_ceval.h" // _PyEval_GetBuiltin()
#include "pycore_freelist.h" // _Py_FREELIST_PUSH(), _Py_FREELIST_POP()
#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED()
#include "pycore_initconfig.h" // _PyStatus_OK()
#include "pycore_modsupport.h" // _PyArg_NoKwnames()
@ -17,7 +18,6 @@ class tuple "PyTupleObject *" "&PyTuple_Type"
#include "clinic/tupleobject.c.h"
static inline PyTupleObject * maybe_freelist_pop(Py_ssize_t);
static inline int maybe_freelist_push(PyTupleObject *);
@ -38,22 +38,20 @@ tuple_alloc(Py_ssize_t size)
PyErr_BadInternalCall();
return NULL;
}
#ifdef Py_DEBUG
assert(size != 0); // The empty tuple is statically allocated.
#endif
PyTupleObject *op = maybe_freelist_pop(size);
if (op == NULL) {
/* Check for overflow */
if ((size_t)size > ((size_t)PY_SSIZE_T_MAX - (sizeof(PyTupleObject) -
sizeof(PyObject *))) / sizeof(PyObject *)) {
return (PyTupleObject *)PyErr_NoMemory();
Py_ssize_t index = size - 1;
if (index < PyTuple_MAXSAVESIZE) {
PyTupleObject *op = _Py_FREELIST_POP(PyTupleObject, tuples[index]);
if (op != NULL) {
return op;
}
op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);
if (op == NULL)
return NULL;
}
return op;
/* Check for overflow */
if ((size_t)size > ((size_t)PY_SSIZE_T_MAX - (sizeof(PyTupleObject) -
sizeof(PyObject *))) / sizeof(PyObject *)) {
return (PyTupleObject *)PyErr_NoMemory();
}
return PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);
}
// The empty tuple singleton is not tracked by the GC.
@ -982,16 +980,6 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize)
return 0;
}
static void maybe_freelist_clear(struct _Py_object_freelists *, int);
void
_PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
{
maybe_freelist_clear(freelists, is_finalization);
}
/*********************** Tuple Iterator **************************/
@ -1141,102 +1129,31 @@ tuple_iter(PyObject *seq)
* freelists *
*************/
#define TUPLE_FREELIST (freelists->tuples)
#define FREELIST_FINALIZED (TUPLE_FREELIST.numfree[0] < 0)
static inline PyTupleObject *
maybe_freelist_pop(Py_ssize_t size)
{
#ifdef WITH_FREELISTS
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
if (size == 0) {
return NULL;
}
assert(size > 0);
if (size <= PyTuple_MAXSAVESIZE) {
Py_ssize_t index = size - 1;
PyTupleObject *op = TUPLE_FREELIST.items[index];
if (op != NULL) {
/* op is the head of a linked list, with the first item
pointing to the next node. Here we pop off the old head. */
TUPLE_FREELIST.items[index] = (PyTupleObject *) op->ob_item[0];
TUPLE_FREELIST.numfree[index]--;
/* Inlined _PyObject_InitVar() without _PyType_HasFeature() test */
#ifdef Py_TRACE_REFS
/* maybe_freelist_push() ensures these were already set. */
// XXX Can we drop these? See commit 68055ce6fe01 (GvR, Dec 1998).
Py_SET_SIZE(op, size);
Py_SET_TYPE(op, &PyTuple_Type);
#endif
_Py_NewReference((PyObject *)op);
/* END inlined _PyObject_InitVar() */
OBJECT_STAT_INC(from_freelist);
return op;
}
}
#endif
return NULL;
}
static inline int
maybe_freelist_push(PyTupleObject *op)
{
#ifdef WITH_FREELISTS
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
if (Py_SIZE(op) == 0) {
if (!Py_IS_TYPE(op, &PyTuple_Type)) {
return 0;
}
Py_ssize_t index = Py_SIZE(op) - 1;
if (index < PyTuple_NFREELISTS
&& TUPLE_FREELIST.numfree[index] < PyTuple_MAXFREELIST
&& TUPLE_FREELIST.numfree[index] >= 0
&& Py_IS_TYPE(op, &PyTuple_Type))
{
/* op is the head of a linked list, with the first item
pointing to the next node. Here we set op as the new head. */
op->ob_item[0] = (PyObject *) TUPLE_FREELIST.items[index];
TUPLE_FREELIST.items[index] = op;
TUPLE_FREELIST.numfree[index]++;
OBJECT_STAT_INC(to_freelist);
return 1;
if (index < PyTuple_MAXSAVESIZE) {
return _Py_FREELIST_PUSH(tuples[index], op, Py_tuple_MAXFREELIST);
}
#endif
return 0;
}
static void
maybe_freelist_clear(struct _Py_object_freelists *freelists, int fini)
{
#ifdef WITH_FREELISTS
for (Py_ssize_t i = 0; i < PyTuple_NFREELISTS; i++) {
PyTupleObject *p = TUPLE_FREELIST.items[i];
TUPLE_FREELIST.items[i] = NULL;
TUPLE_FREELIST.numfree[i] = fini ? -1 : 0;
while (p) {
PyTupleObject *q = p;
p = (PyTupleObject *)(p->ob_item[0]);
PyObject_GC_Del(q);
}
}
#endif
}
/* Print summary info about the state of the optimized allocator */
void
_PyTuple_DebugMallocStats(FILE *out)
{
#ifdef WITH_FREELISTS
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
for (int i = 0; i < PyTuple_NFREELISTS; i++) {
for (int i = 0; i < PyTuple_MAXSAVESIZE; i++) {
int len = i + 1;
char buf[128];
PyOS_snprintf(buf, sizeof(buf),
"free %d-sized PyTupleObject", len);
_PyDebugAllocatorStats(out, buf, TUPLE_FREELIST.numfree[i],
_PyDebugAllocatorStats(out, buf, _Py_FREELIST_SIZE(tuples[i]),
_PyObject_VAR_SIZE(&PyTuple_Type, len));
}
#endif
}
#undef STATE
#undef FREELIST_FINALIZED

1
PCbuild/pythoncore.vcxproj
View File

@ -239,6 +239,7 @@
<ClInclude Include="..\Include\internal\pycore_format.h" />
<ClInclude Include="..\Include\internal\pycore_frame.h" />
<ClInclude Include="..\Include\internal\pycore_freelist.h" />
<ClInclude Include="..\Include\internal\pycore_freelist_state.h" />
<ClInclude Include="..\Include\internal\pycore_function.h" />
<ClInclude Include="..\Include\internal\pycore_gc.h" />
<ClInclude Include="..\Include\internal\pycore_genobject.h" />

3
PCbuild/pythoncore.vcxproj.filters
View File

@ -639,6 +639,9 @@
<ClInclude Include="..\Include\internal\pycore_freelist.h">
<Filter>Include\internal</Filter>
</ClInclude>
<ClInclude Include="..\Include\internal\pycore_freelist_state.h">
<Filter>Include\internal</Filter>
</ClInclude>
<ClInclude Include="..\Include\internal\pycore_function.h">
<Filter>Include\internal</Filter>
</ClInclude>

59
Python/context.c
View File

@ -1,6 +1,7 @@
#include "Python.h"
#include "pycore_call.h" // _PyObject_VectorcallTstate()
#include "pycore_context.h"
#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP()
#include "pycore_gc.h" // _PyObject_GC_MAY_BE_TRACKED()
#include "pycore_hamt.h"
#include "pycore_initconfig.h" // _PyStatus_OK()
@ -64,16 +65,6 @@ static int
contextvar_del(PyContextVar *var);
#ifdef WITH_FREELISTS
static struct _Py_context_freelist *
get_context_freelist(void)
{
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
return &freelists->contexts;
}
#endif
PyObject *
_PyContext_NewHamtForTests(void)
{
@ -343,20 +334,8 @@ class _contextvars.Context "PyContext *" "&PyContext_Type"
static inline PyContext *
_context_alloc(void)
{
PyContext *ctx;
#ifdef WITH_FREELISTS
struct _Py_context_freelist *context_freelist = get_context_freelist();
if (context_freelist->numfree > 0) {
context_freelist->numfree--;
ctx = context_freelist->items;
context_freelist->items = (PyContext *)ctx->ctx_weakreflist;
OBJECT_STAT_INC(from_freelist);
ctx->ctx_weakreflist = NULL;
_Py_NewReference((PyObject *)ctx);
}
else
#endif
{
PyContext *ctx = _Py_FREELIST_POP(PyContext, contexts);
if (ctx == NULL) {
ctx = PyObject_GC_New(PyContext, &PyContext_Type);
if (ctx == NULL) {
return NULL;
@ -471,19 +450,7 @@ context_tp_dealloc(PyContext *self)
}
(void)context_tp_clear(self);
#ifdef WITH_FREELISTS
struct _Py_context_freelist *context_freelist = get_context_freelist();
if (context_freelist->numfree >= 0 && context_freelist->numfree < PyContext_MAXFREELIST) {
context_freelist->numfree++;
self->ctx_weakreflist = (PyObject *)context_freelist->items;
context_freelist->items = self;
OBJECT_STAT_INC(to_freelist);
}
else
#endif
{
Py_TYPE(self)->tp_free(self);
}
_Py_FREELIST_FREE(contexts, self, Py_TYPE(self)->tp_free);
}
static PyObject *
@ -1264,24 +1231,6 @@ get_token_missing(void)
///////////////////////////
void
_PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
{
#ifdef WITH_FREELISTS
struct _Py_context_freelist *state = &freelists->contexts;
for (; state->numfree > 0; state->numfree--) {
PyContext *ctx = state->items;
state->items = (PyContext *)ctx->ctx_weakreflist;
ctx->ctx_weakreflist = NULL;
PyObject_GC_Del(ctx);
}
if (is_finalization) {
state->numfree = -1;
}
#endif
}
PyStatus
_PyContext_Init(PyInterpreterState *interp)
{

1
Python/gc_free_threading.c
View File

@ -4,6 +4,7 @@
#include "pycore_ceval.h" // _Py_set_eval_breaker_bit()
#include "pycore_context.h"
#include "pycore_dict.h" // _PyDict_MaybeUntrack()
#include "pycore_freelist.h" // _PyObject_ClearFreeLists()
#include "pycore_initconfig.h"
#include "pycore_interp.h" // PyInterpreterState.gc
#include "pycore_object.h"

2
Python/gc_gil.c
View File

@ -1,5 +1,5 @@
#include "Python.h"
#include "pycore_pystate.h" // _Py_ClearFreeLists()
#include "pycore_freelist.h" // _PyObject_ClearFreeLists()
#ifndef Py_GIL_DISABLED

48
Python/object_stack.c
View File

@ -8,24 +8,11 @@
extern _PyObjectStackChunk *_PyObjectStackChunk_New(void);
extern void _PyObjectStackChunk_Free(_PyObjectStackChunk *);
static struct _Py_object_stack_freelist *
get_object_stack_freelist(void)
{
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
return &freelists->object_stacks;
}
_PyObjectStackChunk *
_PyObjectStackChunk_New(void)
{
_PyObjectStackChunk *buf;
struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist();
if (obj_stack_freelist->numfree > 0) {
buf = obj_stack_freelist->items;
obj_stack_freelist->items = buf->prev;
obj_stack_freelist->numfree--;
}
else {
_PyObjectStackChunk *buf = _Py_FREELIST_POP_MEM(object_stack_chunks);
if (buf == NULL) {
// NOTE: we use PyMem_RawMalloc() here because this is used by the GC
// during mimalloc heap traversal. In that context, it is not safe to
// allocate mimalloc memory, such as via PyMem_Malloc().
@ -43,17 +30,7 @@ void
_PyObjectStackChunk_Free(_PyObjectStackChunk *buf)
{
assert(buf->n == 0);
struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist();
if (obj_stack_freelist->numfree >= 0 &&
obj_stack_freelist->numfree < _PyObjectStackChunk_MAXFREELIST)
{
buf->prev = obj_stack_freelist->items;
obj_stack_freelist->items = buf;
obj_stack_freelist->numfree++;
}
else {
PyMem_RawFree(buf);
}
_Py_FREELIST_FREE(object_stack_chunks, buf, PyMem_RawFree);
}
void
@ -87,22 +64,3 @@ _PyObjectStack_Merge(_PyObjectStack *dst, _PyObjectStack *src)
dst->head = src->head;
src->head = NULL;
}
void
_PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization)
{
if (!is_finalization) {
// Ignore requests to clear the free list during GC. We use object
// stacks during GC, so emptying the free-list is counterproductive.
return;
}
struct _Py_object_stack_freelist *freelist = &freelists->object_stacks;
while (freelist->numfree > 0) {
_PyObjectStackChunk *buf = freelist->items;
freelist->items = buf->prev;
freelist->numfree--;
PyMem_RawFree(buf);
}
freelist->numfree = -1;
}

3
Python/pylifecycle.c
View File

@ -9,6 +9,7 @@
#include "pycore_dict.h" // _PyDict_Fini()
#include "pycore_exceptions.h" // _PyExc_InitTypes()
#include "pycore_fileutils.h" // _Py_ResetForceASCII()
#include "pycore_freelist.h" // _PyObject_ClearFreeLists()
#include "pycore_floatobject.h" // _PyFloat_InitTypes()
#include "pycore_global_objects_fini_generated.h" // "_PyStaticObjects_CheckRefcnt()
#include "pycore_import.h" // _PyImport_BootstrapImp()
@ -1843,7 +1844,7 @@ finalize_interp_types(PyInterpreterState *interp)
#ifndef Py_GIL_DISABLED
// With Py_GIL_DISABLED:
// the freelists for the current thread state have already been cleared.
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
struct _Py_freelists *freelists = _Py_freelists_GET();
_PyObject_ClearFreeLists(freelists, 1);
#endif

4
Python/pystate.c
View File

@ -9,9 +9,9 @@
#include "pycore_dtoa.h" // _dtoa_state_INIT()
#include "pycore_emscripten_trampoline.h" // _Py_EmscriptenTrampoline_Init()
#include "pycore_frame.h"
#include "pycore_freelist.h" // _PyObject_ClearFreeLists()
#include "pycore_initconfig.h" // _PyStatus_OK()
#include "pycore_object.h" // _PyType_InitCache()
#include "pycore_object_stack.h" // _PyObjectStackChunk_ClearFreeList()
#include "pycore_parking_lot.h" // _PyParkingLot_AfterFork()
#include "pycore_pyerrors.h" // _PyErr_Clear()
#include "pycore_pylifecycle.h" // _PyAST_Fini()
@ -1738,7 +1738,7 @@ PyThreadState_Clear(PyThreadState *tstate)
#ifdef Py_GIL_DISABLED
// Each thread should clear own freelists in free-threading builds.
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
struct _Py_freelists *freelists = _Py_freelists_GET();
_PyObject_ClearFreeLists(freelists, 1);
// Remove ourself from the biased reference counting table of threads.