mirror of https://github.com/python/cpython
gh-112075: Support freeing object memory via QSBR (#116344)
Free objects with qsbr if shared
This commit is contained in:
parent
61831585b6
commit
7db871e4fa
|
@ -44,6 +44,7 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) {
|
|||
# define _PyGC_BITS_UNREACHABLE (4)
|
||||
# define _PyGC_BITS_FROZEN (8)
|
||||
# define _PyGC_BITS_SHARED (16)
|
||||
# define _PyGC_BITS_SHARED_INLINE (32)
|
||||
#endif
|
||||
|
||||
/* True if the object is currently tracked by the GC. */
|
||||
|
@ -71,9 +72,12 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj) {
|
|||
|
||||
#ifdef Py_GIL_DISABLED
|
||||
|
||||
/* True if an object is shared between multiple threads and
|
||||
* needs special purpose when freeing to do the possibility
|
||||
* of in-flight lock-free reads occurring */
|
||||
/* True if memory the object references is shared between
|
||||
* multiple threads and needs special purpose when freeing
|
||||
* those references due to the possibility of in-flight
|
||||
* lock-free reads occurring. The object is responsible
|
||||
* for calling _PyMem_FreeDelayed on the referenced
|
||||
* memory. */
|
||||
static inline int _PyObject_GC_IS_SHARED(PyObject *op) {
|
||||
return (op->ob_gc_bits & _PyGC_BITS_SHARED) != 0;
|
||||
}
|
||||
|
@ -84,6 +88,23 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) {
|
|||
}
|
||||
#define _PyObject_GC_SET_SHARED(op) _PyObject_GC_SET_SHARED(_Py_CAST(PyObject*, op))
|
||||
|
||||
/* True if the memory of the object is shared between multiple
|
||||
* threads and needs special purpose when freeing due to
|
||||
* the possibility of in-flight lock-free reads occurring.
|
||||
* Objects with this bit that are GC objects will automatically
|
||||
* delay-freed by PyObject_GC_Del. */
|
||||
static inline int _PyObject_GC_IS_SHARED_INLINE(PyObject *op) {
|
||||
return (op->ob_gc_bits & _PyGC_BITS_SHARED_INLINE) != 0;
|
||||
}
|
||||
#define _PyObject_GC_IS_SHARED_INLINE(op) \
|
||||
_PyObject_GC_IS_SHARED_INLINE(_Py_CAST(PyObject*, op))
|
||||
|
||||
static inline void _PyObject_GC_SET_SHARED_INLINE(PyObject *op) {
|
||||
op->ob_gc_bits |= _PyGC_BITS_SHARED_INLINE;
|
||||
}
|
||||
#define _PyObject_GC_SET_SHARED_INLINE(op) \
|
||||
_PyObject_GC_SET_SHARED_INLINE(_Py_CAST(PyObject*, op))
|
||||
|
||||
#endif
|
||||
|
||||
/* Bit flags for _gc_prev */
|
||||
|
|
|
@ -119,6 +119,9 @@ extern int _PyMem_DebugEnabled(void);
|
|||
// Enqueue a pointer to be freed possibly after some delay.
|
||||
extern void _PyMem_FreeDelayed(void *ptr);
|
||||
|
||||
// Enqueue an object to be freed possibly after some delay
|
||||
extern void _PyObject_FreeDelayed(void *ptr);
|
||||
|
||||
// Periodically process delayed free requests.
|
||||
extern void _PyMem_ProcessDelayed(PyThreadState *tstate);
|
||||
|
||||
|
|
|
@ -1070,7 +1070,7 @@ _PyMem_Strdup(const char *str)
|
|||
|
||||
// A pointer to be freed once the QSBR read sequence reaches qsbr_goal.
|
||||
struct _mem_work_item {
|
||||
void *ptr;
|
||||
uintptr_t ptr; // lowest bit tagged 1 for objects freed with PyObject_Free
|
||||
uint64_t qsbr_goal;
|
||||
};
|
||||
|
||||
|
@ -1084,16 +1084,27 @@ struct _mem_work_chunk {
|
|||
struct _mem_work_item array[WORK_ITEMS_PER_CHUNK];
|
||||
};
|
||||
|
||||
void
|
||||
_PyMem_FreeDelayed(void *ptr)
|
||||
static void
|
||||
free_work_item(uintptr_t ptr)
|
||||
{
|
||||
if (ptr & 0x01) {
|
||||
PyObject_Free((char *)(ptr - 1));
|
||||
}
|
||||
else {
|
||||
PyMem_Free((void *)ptr);
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
free_delayed(uintptr_t ptr)
|
||||
{
|
||||
#ifndef Py_GIL_DISABLED
|
||||
PyMem_Free(ptr);
|
||||
free_work_item(ptr);
|
||||
#else
|
||||
if (_PyRuntime.stoptheworld.world_stopped) {
|
||||
// Free immediately if the world is stopped, including during
|
||||
// interpreter shutdown.
|
||||
PyMem_Free(ptr);
|
||||
free_work_item(ptr);
|
||||
return;
|
||||
}
|
||||
|
||||
|
@ -1120,7 +1131,7 @@ _PyMem_FreeDelayed(void *ptr)
|
|||
if (buf == NULL) {
|
||||
// failed to allocate a buffer, free immediately
|
||||
_PyEval_StopTheWorld(tstate->base.interp);
|
||||
PyMem_Free(ptr);
|
||||
free_work_item(ptr);
|
||||
_PyEval_StartTheWorld(tstate->base.interp);
|
||||
return;
|
||||
}
|
||||
|
@ -1137,6 +1148,20 @@ _PyMem_FreeDelayed(void *ptr)
|
|||
#endif
|
||||
}
|
||||
|
||||
void
|
||||
_PyMem_FreeDelayed(void *ptr)
|
||||
{
|
||||
assert(!((uintptr_t)ptr & 0x01));
|
||||
free_delayed((uintptr_t)ptr);
|
||||
}
|
||||
|
||||
void
|
||||
_PyObject_FreeDelayed(void *ptr)
|
||||
{
|
||||
assert(!((uintptr_t)ptr & 0x01));
|
||||
free_delayed(((uintptr_t)ptr)|0x01);
|
||||
}
|
||||
|
||||
static struct _mem_work_chunk *
|
||||
work_queue_first(struct llist_node *head)
|
||||
{
|
||||
|
@ -1156,7 +1181,7 @@ process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr,
|
|||
return;
|
||||
}
|
||||
|
||||
PyMem_Free(item->ptr);
|
||||
free_work_item(item->ptr);
|
||||
buf->rd_idx++;
|
||||
}
|
||||
|
||||
|
@ -1243,7 +1268,7 @@ _PyMem_FiniDelayed(PyInterpreterState *interp)
|
|||
// Free the remaining items immediately. There should be no other
|
||||
// threads accessing the memory at this point during shutdown.
|
||||
struct _mem_work_item *item = &buf->array[buf->rd_idx];
|
||||
PyMem_Free(item->ptr);
|
||||
free_work_item(item->ptr);
|
||||
buf->rd_idx++;
|
||||
}
|
||||
|
||||
|
|
|
@ -1695,6 +1695,7 @@ PyObject_GC_Del(void *op)
|
|||
{
|
||||
size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type);
|
||||
if (_PyObject_GC_IS_TRACKED(op)) {
|
||||
_PyObject_GC_UNTRACK(op);
|
||||
#ifdef Py_DEBUG
|
||||
PyObject *exc = PyErr_GetRaisedException();
|
||||
if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0,
|
||||
|
@ -1707,8 +1708,13 @@ PyObject_GC_Del(void *op)
|
|||
}
|
||||
|
||||
record_deallocation(_PyThreadState_GET());
|
||||
|
||||
PyObject_Free(((char *)op)-presize);
|
||||
PyObject *self = (PyObject *)op;
|
||||
if (_PyObject_GC_IS_SHARED_INLINE(self)) {
|
||||
_PyObject_FreeDelayed(((char *)op)-presize);
|
||||
}
|
||||
else {
|
||||
PyObject_Free(((char *)op)-presize);
|
||||
}
|
||||
}
|
||||
|
||||
int
|
||||
|
|
Loading…
Reference in New Issue