bpo-40521: Make async gen free lists per-interpreter (GH-20643)

Each interpreter now has its own asynchronous generator free lists:

* Move async gen free lists into PyInterpreterState.
* Move _PyAsyncGen_MAXFREELIST define to pycore_interp.h
* Add _Py_async_gen_state structure.
* Add tstate parameter to _PyAsyncGen_ClearFreeLists
  and _PyAsyncGen_Fini().

Include/internal/pycore_gc.h

@@ -170,7 +170,7 @@ extern void _PyTuple_ClearFreeList(PyThreadState *tstate);
 extern void _PyFloat_ClearFreeList(PyThreadState *tstate);
 extern void _PyList_ClearFreeList(PyThreadState *tstate);
 extern void _PyDict_ClearFreeList(void);
-extern void _PyAsyncGen_ClearFreeLists(void);
+extern void _PyAsyncGen_ClearFreeLists(PyThreadState *tstate);
 extern void _PyContext_ClearFreeList(void);
 
 #ifdef __cplusplus

Include/internal/pycore_interp.h

@@ -108,6 +108,23 @@ struct _Py_frame_state {
     int numfree;
 };
 
+#ifndef _PyAsyncGen_MAXFREELIST
+#  define _PyAsyncGen_MAXFREELIST 80
+#endif
+
+struct _Py_async_gen_state {
+    /* Freelists boost performance 6-10%; they also reduce memory
+       fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend
+       are short-living objects that are instantiated for every
+       __anext__() call. */
+    struct _PyAsyncGenWrappedValue* value_freelist[_PyAsyncGen_MAXFREELIST];
+    int value_numfree;
+
+    struct PyAsyncGenASend* asend_freelist[_PyAsyncGen_MAXFREELIST];
+    int asend_numfree;
+};
+
+
 
 /* interpreter state */
 
@@ -205,6 +222,7 @@ struct _is {
     struct _Py_list_state list;
     struct _Py_float_state float_state;
     struct _Py_frame_state frame;
+    struct _Py_async_gen_state async_gen;
 
     /* Using a cache is very effective since typically only a single slice is
        created and then deleted again. */

Include/internal/pycore_pylifecycle.h

@@ -66,7 +66,7 @@ extern void _PySet_Fini(void);
 extern void _PyBytes_Fini(void);
 extern void _PyFloat_Fini(PyThreadState *tstate);
 extern void _PySlice_Fini(PyThreadState *tstate);
-extern void _PyAsyncGen_Fini(void);
+extern void _PyAsyncGen_Fini(PyThreadState *tstate);
 
 extern void PyOS_FiniInterrupts(void);
 

Misc/NEWS.d/next/Core and Builtins/2020-05-20-01-17-34.bpo-40521.wvAehI.rst

@@ -1,3 +1,4 @@
 The tuple free lists, the empty tuple singleton, the list free list, the float
-free list, the slice cache, and the frame free list are no longer shared by all
+free list, the slice cache, the frame free list, the asynchronous generator
-interpreters: each interpreter now its has own free lists and caches.
+free lists are no longer shared by all interpreters: each interpreter now its
+has own free lists and caches.

Modules/gcmodule.c

@@ -1031,7 +1031,7 @@ clear_freelists(void)
     _PyFloat_ClearFreeList(tstate);
     _PyList_ClearFreeList(tstate);
     _PyDict_ClearFreeList();
-    _PyAsyncGen_ClearFreeLists();
+    _PyAsyncGen_ClearFreeLists(tstate);
     _PyContext_ClearFreeList();
 }
 

Objects/genobject.c

@@ -1162,7 +1162,7 @@ typedef enum {
 } AwaitableState;
 
 
-typedef struct {
+typedef struct PyAsyncGenASend {
     PyObject_HEAD
     PyAsyncGenObject *ags_gen;
 
@@ -1174,7 +1174,7 @@ typedef struct {
 } PyAsyncGenASend;
 
 
-typedef struct {
+typedef struct PyAsyncGenAThrow {
     PyObject_HEAD
     PyAsyncGenObject *agt_gen;
 
@@ -1186,28 +1186,12 @@ typedef struct {
 } PyAsyncGenAThrow;
 
 
-typedef struct {
+typedef struct _PyAsyncGenWrappedValue {
     PyObject_HEAD
     PyObject *agw_val;
 } _PyAsyncGenWrappedValue;
 
 
-#ifndef _PyAsyncGen_MAXFREELIST
-#define _PyAsyncGen_MAXFREELIST 80
-#endif
-
-/* Freelists boost performance 6-10%; they also reduce memory
-   fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend
-   are short-living objects that are instantiated for every
-   __anext__ call.
-*/
-
-static _PyAsyncGenWrappedValue *ag_value_freelist[_PyAsyncGen_MAXFREELIST];
-static int ag_value_freelist_free = 0;
-
-static PyAsyncGenASend *ag_asend_freelist[_PyAsyncGen_MAXFREELIST];
-static int ag_asend_freelist_free = 0;
-
 #define _PyAsyncGenWrappedValue_CheckExact(o) \
                     Py_IS_TYPE(o, &_PyAsyncGenWrappedValue_Type)
 
@@ -1423,27 +1407,29 @@ PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
 
 
 void
-_PyAsyncGen_ClearFreeLists(void)
+_PyAsyncGen_ClearFreeLists(PyThreadState *tstate)
 {
-    while (ag_value_freelist_free) {
+    struct _Py_async_gen_state *state = &tstate->interp->async_gen;
+
+    while (state->value_numfree) {
         _PyAsyncGenWrappedValue *o;
-        o = ag_value_freelist[--ag_value_freelist_free];
+        o = state->value_freelist[--state->value_numfree];
         assert(_PyAsyncGenWrappedValue_CheckExact(o));
         PyObject_GC_Del(o);
     }
 
-    while (ag_asend_freelist_free) {
+    while (state->asend_numfree) {
         PyAsyncGenASend *o;
-        o = ag_asend_freelist[--ag_asend_freelist_free];
+        o = state->asend_freelist[--state->asend_numfree];
         assert(Py_IS_TYPE(o, &_PyAsyncGenASend_Type));
         PyObject_GC_Del(o);
     }
 }
 
 void
-_PyAsyncGen_Fini(void)
+_PyAsyncGen_Fini(PyThreadState *tstate)
 {
-    _PyAsyncGen_ClearFreeLists();
+    _PyAsyncGen_ClearFreeLists(tstate);
 }
 
 
@@ -1486,10 +1472,13 @@ async_gen_asend_dealloc(PyAsyncGenASend *o)
     _PyObject_GC_UNTRACK((PyObject *)o);
     Py_CLEAR(o->ags_gen);
     Py_CLEAR(o->ags_sendval);
-    if (ag_asend_freelist_free < _PyAsyncGen_MAXFREELIST) {
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    struct _Py_async_gen_state *state = &interp->async_gen;
+    if (state->asend_numfree < _PyAsyncGen_MAXFREELIST) {
         assert(PyAsyncGenASend_CheckExact(o));
-        ag_asend_freelist[ag_asend_freelist_free++] = o;
+        state->asend_freelist[state->asend_numfree++] = o;
-    } else {
+    }
+    else {
         PyObject_GC_Del(o);
     }
 }
@@ -1641,11 +1630,14 @@ static PyObject *
 async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval)
 {
     PyAsyncGenASend *o;
-    if (ag_asend_freelist_free) {
+    PyInterpreterState *interp = _PyInterpreterState_GET();
-        ag_asend_freelist_free--;
+    struct _Py_async_gen_state *state = &interp->async_gen;
-        o = ag_asend_freelist[ag_asend_freelist_free];
+    if (state->asend_numfree) {
+        state->asend_numfree--;
+        o = state->asend_freelist[state->asend_numfree];
         _Py_NewReference((PyObject *)o);
-    } else {
+    }
+    else {
         o = PyObject_GC_New(PyAsyncGenASend, &_PyAsyncGenASend_Type);
         if (o == NULL) {
             return NULL;
@@ -1673,10 +1665,13 @@ async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o)
 {
     _PyObject_GC_UNTRACK((PyObject *)o);
     Py_CLEAR(o->agw_val);
-    if (ag_value_freelist_free < _PyAsyncGen_MAXFREELIST) {
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    struct _Py_async_gen_state *state = &interp->async_gen;
+    if (state->value_numfree < _PyAsyncGen_MAXFREELIST) {
         assert(_PyAsyncGenWrappedValue_CheckExact(o));
-        ag_value_freelist[ag_value_freelist_free++] = o;
+        state->value_freelist[state->value_numfree++] = o;
-    } else {
+    }
+    else {
         PyObject_GC_Del(o);
     }
 }
@@ -1740,12 +1735,15 @@ _PyAsyncGenValueWrapperNew(PyObject *val)
     _PyAsyncGenWrappedValue *o;
     assert(val);
 
-    if (ag_value_freelist_free) {
+    PyInterpreterState *interp = _PyInterpreterState_GET();
-        ag_value_freelist_free--;
+    struct _Py_async_gen_state *state = &interp->async_gen;
-        o = ag_value_freelist[ag_value_freelist_free];
+    if (state->value_numfree) {
+        state->value_numfree--;
+        o = state->value_freelist[state->value_numfree];
         assert(_PyAsyncGenWrappedValue_CheckExact(o));
         _Py_NewReference((PyObject*)o);
-    } else {
+    }
+    else {
         o = PyObject_GC_New(_PyAsyncGenWrappedValue,
                             &_PyAsyncGenWrappedValue_Type);
         if (o == NULL) {

Python/pylifecycle.c

@@ -1270,7 +1270,11 @@ finalize_interp_types(PyThreadState *tstate, int is_main_interp)
     if (is_main_interp) {
         _Py_HashRandomization_Fini();
         _PyArg_Fini();
-        _PyAsyncGen_Fini();
+    }
+
+    _PyAsyncGen_Fini(tstate);
+
+    if (is_main_interp) {
         _PyContext_Fini();
     }