gh-112075: Support freeing object memory via QSBR (#116344)

Free objects with qsbr if shared
This commit is contained in:
Dino Viehland 2024-03-08 09:56:36 -08:00 committed by GitHub
parent 61831585b6
commit 7db871e4fa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 68 additions and 13 deletions

View File

@ -44,6 +44,7 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) {
# define _PyGC_BITS_UNREACHABLE (4)
# define _PyGC_BITS_FROZEN (8)
# define _PyGC_BITS_SHARED (16)
# define _PyGC_BITS_SHARED_INLINE (32)
#endif
/* True if the object is currently tracked by the GC. */
@ -71,9 +72,12 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj) {
#ifdef Py_GIL_DISABLED
/* True if an object is shared between multiple threads and
* needs special purpose when freeing to do the possibility
* of in-flight lock-free reads occurring */
/* True if memory the object references is shared between
* multiple threads and needs special purpose when freeing
* those references due to the possibility of in-flight
* lock-free reads occurring. The object is responsible
* for calling _PyMem_FreeDelayed on the referenced
* memory. */
static inline int _PyObject_GC_IS_SHARED(PyObject *op) {
return (op->ob_gc_bits & _PyGC_BITS_SHARED) != 0;
}
@ -84,6 +88,23 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) {
}
#define _PyObject_GC_SET_SHARED(op) _PyObject_GC_SET_SHARED(_Py_CAST(PyObject*, op))
/* True if the memory of the object is shared between multiple
* threads and needs special purpose when freeing due to
* the possibility of in-flight lock-free reads occurring.
* Objects with this bit that are GC objects will automatically
* delay-freed by PyObject_GC_Del. */
static inline int _PyObject_GC_IS_SHARED_INLINE(PyObject *op) {
return (op->ob_gc_bits & _PyGC_BITS_SHARED_INLINE) != 0;
}
#define _PyObject_GC_IS_SHARED_INLINE(op) \
_PyObject_GC_IS_SHARED_INLINE(_Py_CAST(PyObject*, op))
static inline void _PyObject_GC_SET_SHARED_INLINE(PyObject *op) {
op->ob_gc_bits |= _PyGC_BITS_SHARED_INLINE;
}
#define _PyObject_GC_SET_SHARED_INLINE(op) \
_PyObject_GC_SET_SHARED_INLINE(_Py_CAST(PyObject*, op))
#endif
/* Bit flags for _gc_prev */

View File

@ -119,6 +119,9 @@ extern int _PyMem_DebugEnabled(void);
// Enqueue a pointer to be freed possibly after some delay.
extern void _PyMem_FreeDelayed(void *ptr);
// Enqueue an object to be freed possibly after some delay
extern void _PyObject_FreeDelayed(void *ptr);
// Periodically process delayed free requests.
extern void _PyMem_ProcessDelayed(PyThreadState *tstate);

View File

@ -1070,7 +1070,7 @@ _PyMem_Strdup(const char *str)
// A pointer to be freed once the QSBR read sequence reaches qsbr_goal.
struct _mem_work_item {
void *ptr;
uintptr_t ptr; // lowest bit tagged 1 for objects freed with PyObject_Free
uint64_t qsbr_goal;
};
@ -1084,16 +1084,27 @@ struct _mem_work_chunk {
struct _mem_work_item array[WORK_ITEMS_PER_CHUNK];
};
void
_PyMem_FreeDelayed(void *ptr)
static void
free_work_item(uintptr_t ptr)
{
if (ptr & 0x01) {
PyObject_Free((char *)(ptr - 1));
}
else {
PyMem_Free((void *)ptr);
}
}
static void
free_delayed(uintptr_t ptr)
{
#ifndef Py_GIL_DISABLED
PyMem_Free(ptr);
free_work_item(ptr);
#else
if (_PyRuntime.stoptheworld.world_stopped) {
// Free immediately if the world is stopped, including during
// interpreter shutdown.
PyMem_Free(ptr);
free_work_item(ptr);
return;
}
@ -1120,7 +1131,7 @@ _PyMem_FreeDelayed(void *ptr)
if (buf == NULL) {
// failed to allocate a buffer, free immediately
_PyEval_StopTheWorld(tstate->base.interp);
PyMem_Free(ptr);
free_work_item(ptr);
_PyEval_StartTheWorld(tstate->base.interp);
return;
}
@ -1137,6 +1148,20 @@ _PyMem_FreeDelayed(void *ptr)
#endif
}
void
_PyMem_FreeDelayed(void *ptr)
{
assert(!((uintptr_t)ptr & 0x01));
free_delayed((uintptr_t)ptr);
}
void
_PyObject_FreeDelayed(void *ptr)
{
assert(!((uintptr_t)ptr & 0x01));
free_delayed(((uintptr_t)ptr)|0x01);
}
static struct _mem_work_chunk *
work_queue_first(struct llist_node *head)
{
@ -1156,7 +1181,7 @@ process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr,
return;
}
PyMem_Free(item->ptr);
free_work_item(item->ptr);
buf->rd_idx++;
}
@ -1243,7 +1268,7 @@ _PyMem_FiniDelayed(PyInterpreterState *interp)
// Free the remaining items immediately. There should be no other
// threads accessing the memory at this point during shutdown.
struct _mem_work_item *item = &buf->array[buf->rd_idx];
PyMem_Free(item->ptr);
free_work_item(item->ptr);
buf->rd_idx++;
}

View File

@ -1695,6 +1695,7 @@ PyObject_GC_Del(void *op)
{
size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type);
if (_PyObject_GC_IS_TRACKED(op)) {
_PyObject_GC_UNTRACK(op);
#ifdef Py_DEBUG
PyObject *exc = PyErr_GetRaisedException();
if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0,
@ -1707,8 +1708,13 @@ PyObject_GC_Del(void *op)
}
record_deallocation(_PyThreadState_GET());
PyObject_Free(((char *)op)-presize);
PyObject *self = (PyObject *)op;
if (_PyObject_GC_IS_SHARED_INLINE(self)) {
_PyObject_FreeDelayed(((char *)op)-presize);
}
else {
PyObject_Free(((char *)op)-presize);
}
}
int