bpo-32030: Add _PyCoreConfig_Copy() (#4874)

Each interpreter now has its core_config and main_config copy:

* Add _PyCoreConfig_Copy() and _PyMainInterpreterConfig_Copy()
* Move _PyCoreConfig_Read(), _PyCoreConfig_Clear() and
  _PyMainInterpreterConfig_Clear() from Python/pylifecycle.c to
  Modules/main.c
* Fix _Py_InitializeEx_Private(): call _PyCoreConfig_ReadEnv() before
  _Py_InitializeCore()

Include/pylifecycle.h

@@ -57,9 +57,15 @@ PyAPI_FUNC(int) _Py_IsCoreInitialized(void);
 PyAPI_FUNC(_PyInitError) _PyCoreConfig_ReadEnv(_PyCoreConfig *);
 PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *);
 PyAPI_FUNC(void) _PyCoreConfig_Clear(_PyCoreConfig *);
+PyAPI_FUNC(int) _PyCoreConfig_Copy(
+    _PyCoreConfig *config,
+    const _PyCoreConfig *config2);
 
 PyAPI_FUNC(_PyInitError) _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *, _PyCoreConfig *);
 PyAPI_FUNC(void) _PyMainInterpreterConfig_Clear(_PyMainInterpreterConfig *);
+PyAPI_FUNC(int) _PyMainInterpreterConfig_Copy(
+    _PyMainInterpreterConfig *config,
+    const _PyMainInterpreterConfig *config2);
 
 PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *);
 #endif

Modules/main.c

@@ -1914,6 +1914,154 @@ pymain_parse_cmdline_envvars(_PyMain *pymain)
 }
 
 
+/* Read configuration settings from standard locations
+ *
+ * This function doesn't make any changes to the interpreter state - it
+ * merely populates any missing configuration settings. This allows an
+ * embedding application to completely override a config option by
+ * setting it before calling this function, or else modify the default
+ * setting before passing the fully populated config to Py_EndInitialization.
+ *
+ * More advanced selective initialization tricks are possible by calling
+ * this function multiple times with various preconfigured settings.
+ */
+
+_PyInitError
+_PyCoreConfig_Read(_PyCoreConfig *config)
+{
+    if (config->program_name == NULL) {
+#ifdef MS_WINDOWS
+        const wchar_t *program_name = L"python";
+#else
+        const wchar_t *program_name = L"python3";
+#endif
+        config->program_name = _PyMem_RawWcsdup(program_name);
+        if (config->program_name == NULL) {
+            return _Py_INIT_NO_MEMORY();
+        }
+    }
+
+    return _Py_INIT_OK();
+}
+
+
+void
+_PyCoreConfig_Clear(_PyCoreConfig *config)
+{
+#define CLEAR(ATTR) \
+    do { \
+        PyMem_RawFree(ATTR); \
+        ATTR = NULL; \
+    } while (0)
+
+    CLEAR(config->module_search_path_env);
+    CLEAR(config->home);
+    CLEAR(config->program_name);
+#undef CLEAR
+}
+
+
+int
+_PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2)
+{
+    _PyCoreConfig_Clear(config);
+
+#define COPY_ATTR(ATTR) config->ATTR = config2->ATTR
+    COPY_ATTR(ignore_environment);
+    COPY_ATTR(use_hash_seed);
+    COPY_ATTR(hash_seed);
+    COPY_ATTR(_disable_importlib);
+    COPY_ATTR(allocator);
+    COPY_ATTR(dev_mode);
+    COPY_ATTR(faulthandler);
+    COPY_ATTR(tracemalloc);
+    COPY_ATTR(import_time);
+    COPY_ATTR(show_ref_count);
+    COPY_ATTR(show_alloc_count);
+    COPY_ATTR(dump_refs);
+    COPY_ATTR(malloc_stats);
+    COPY_ATTR(utf8_mode);
+#undef COPY_ATTR
+
+#define COPY_STR_ATTR(ATTR) \
+    do { \
+        if (config2->ATTR != NULL) { \
+            config->ATTR = _PyMem_RawWcsdup(config2->ATTR); \
+            if (config->ATTR == NULL) { \
+                return -1; \
+            } \
+        } \
+    } while (0)
+
+    COPY_STR_ATTR(module_search_path_env);
+    COPY_STR_ATTR(home);
+    COPY_STR_ATTR(program_name);
+#undef COPY_STR_ATTR
+    return 0;
+}
+
+
+void
+_PyMainInterpreterConfig_Clear(_PyMainInterpreterConfig *config)
+{
+    Py_CLEAR(config->argv);
+    Py_CLEAR(config->module_search_path);
+    Py_CLEAR(config->warnoptions);
+    Py_CLEAR(config->xoptions);
+}
+
+
+static PyObject*
+config_copy_attr(PyObject *obj)
+{
+    if (PyUnicode_Check(obj)) {
+        Py_INCREF(obj);
+        return obj;
+    }
+    else if (PyList_Check(obj)) {
+        return PyList_GetSlice(obj, 0, Py_SIZE(obj));
+    }
+    else if (PyDict_Check(obj)) {
+        /* The dict type is used for xoptions. Make the assumption that keys
+           and values are immutables */
+        return PyDict_Copy(obj);
+    }
+    else {
+        PyErr_Format(PyExc_TypeError,
+                     "cannot copy config attribute of type %.200s",
+                     Py_TYPE(obj)->tp_name);
+        return NULL;
+    }
+}
+
+
+int
+_PyMainInterpreterConfig_Copy(_PyMainInterpreterConfig *config,
+                              const _PyMainInterpreterConfig *config2)
+{
+    _PyMainInterpreterConfig_Clear(config);
+
+#define COPY_ATTR(ATTR) \
+    do { \
+        if (config2->ATTR != NULL) { \
+            config->ATTR = config_copy_attr(config2->ATTR); \
+            if (config->ATTR == NULL) { \
+                return -1; \
+            } \
+        } \
+    } while (0)
+
+    COPY_ATTR(argv);
+    COPY_ATTR(module_search_path);
+    COPY_ATTR(warnoptions);
+    COPY_ATTR(xoptions);
+#undef COPY_ATTR
+    return 0;
+}
+
+
+
+
 static PyObject *
 config_create_path_list(const wchar_t *path, wchar_t delim)
 {

Python/pylifecycle.c

@@ -607,13 +607,13 @@ _Py_SetLocaleFromEnv(int category)
  */
 
 _PyInitError
-_Py_InitializeCore(const _PyCoreConfig *config)
+_Py_InitializeCore(const _PyCoreConfig *core_config)
 {
+    assert(core_config != NULL);
+
     PyInterpreterState *interp;
     PyThreadState *tstate;
     PyObject *bimod, *sysmod, *pstderr;
-    _PyCoreConfig core_config = _PyCoreConfig_INIT;
-    _PyMainInterpreterConfig preinit_config = _PyMainInterpreterConfig_INIT;
     _PyInitError err;
 
     err = _PyRuntime_Initialize();
@@ -621,11 +621,7 @@ _Py_InitializeCore(const _PyCoreConfig *config)
         return err;
     }
 
-    if (config != NULL) {
+    if (_PyMem_SetupAllocators(core_config->allocator) < 0) {
-        core_config = *config;
-    }
-
-    if (_PyMem_SetupAllocators(core_config.allocator) < 0) {
         return _Py_INIT_USER_ERR("Unknown PYTHONMALLOC allocator");
     }
 
@@ -655,12 +651,12 @@ _Py_InitializeCore(const _PyCoreConfig *config)
     _emit_stderr_warning_for_legacy_locale();
 #endif
 
-    err = _Py_HashRandomization_Init(&core_config);
+    err = _Py_HashRandomization_Init(core_config);
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
 
-    if (!core_config.use_hash_seed || core_config.hash_seed) {
+    if (!core_config->use_hash_seed || core_config->hash_seed) {
         /* Random or non-zero hash seed */
         Py_HashRandomizationFlag = 1;
     }
@@ -671,10 +667,13 @@ _Py_InitializeCore(const _PyCoreConfig *config)
     }
 
     interp = PyInterpreterState_New();
-    if (interp == NULL)
+    if (interp == NULL) {
         return _Py_INIT_ERR("can't make main interpreter");
-    interp->core_config = core_config;
+    }
-    interp->config = preinit_config;
+
+    if (_PyCoreConfig_Copy(&interp->core_config, core_config) < 0) {
+        return _Py_INIT_ERR("failed to copy core config");
+    }
 
     tstate = PyThreadState_New(interp);
     if (tstate == NULL)
@@ -779,62 +778,6 @@ _Py_InitializeCore(const _PyCoreConfig *config)
     return _Py_INIT_OK();
 }
 
-/* Read configuration settings from standard locations
- *
- * This function doesn't make any changes to the interpreter state - it
- * merely populates any missing configuration settings. This allows an
- * embedding application to completely override a config option by
- * setting it before calling this function, or else modify the default
- * setting before passing the fully populated config to Py_EndInitialization.
- *
- * More advanced selective initialization tricks are possible by calling
- * this function multiple times with various preconfigured settings.
- */
-
-_PyInitError
-_PyCoreConfig_Read(_PyCoreConfig *config)
-{
-    if (config->program_name == NULL) {
-#ifdef MS_WINDOWS
-        const wchar_t *program_name = L"python";
-#else
-        const wchar_t *program_name = L"python3";
-#endif
-        config->program_name = _PyMem_RawWcsdup(program_name);
-        if (config->program_name == NULL) {
-            return _Py_INIT_NO_MEMORY();
-        }
-    }
-
-    return _Py_INIT_OK();
-}
-
-void
-_PyCoreConfig_Clear(_PyCoreConfig *config)
-{
-#define CLEAR(ATTR) \
-    do { \
-        PyMem_RawFree(ATTR); \
-        ATTR = NULL; \
-    } while (0)
-
-    CLEAR(config->module_search_path_env);
-    CLEAR(config->home);
-    CLEAR(config->program_name);
-#undef CLEAR
-}
-
-
-void
-_PyMainInterpreterConfig_Clear(_PyMainInterpreterConfig *config)
-{
-    Py_CLEAR(config->argv);
-    Py_CLEAR(config->module_search_path);
-    Py_CLEAR(config->warnoptions);
-    Py_CLEAR(config->xoptions);
-}
-
-
 /* Update interpreter state based on supplied configuration settings
  *
  * After calling this function, most of the restrictions on the interpreter
@@ -869,7 +812,9 @@ _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config)
         return _Py_INIT_ERR("failed to get interpreter");
 
     /* Now finish configuring the main interpreter */
-    interp->config = *config;
+    if (_PyMainInterpreterConfig_Copy(&interp->config, config) < 0) {
+        return _Py_INIT_ERR("failed to copy main interpreter config");
+    }
 
     if (interp->core_config._disable_importlib) {
         /* Special mode for freeze_importlib: run with no import system
@@ -906,8 +851,9 @@ _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config)
         }
     }
 
-    if (_PySys_EndInit(interp->sysdict) < 0)
+    if (_PySys_EndInit(interp->sysdict) < 0) {
         return _Py_INIT_ERR("can't finish initializing sys");
+    }
 
     err = initexternalimport(interp);
     if (_Py_INIT_FAILED(err)) {
@@ -979,12 +925,12 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
     core_config._disable_importlib = !install_importlib;
     config.install_signal_handlers = install_sigs;
 
-    err = _Py_InitializeCore(&core_config);
+    err = _PyCoreConfig_ReadEnv(&core_config);
     if (_Py_INIT_FAILED(err)) {
         goto done;
     }
 
-    err = _PyCoreConfig_ReadEnv(&core_config);
+    err = _Py_InitializeCore(&core_config);
     if (_Py_INIT_FAILED(err)) {
         goto done;
     }
@@ -1118,9 +1064,11 @@ Py_FinalizeEx(void)
     tstate = PyThreadState_GET();
     interp = tstate->interp;
 
-    /* Copy the core config to be able to use it even
+    /* Copy the core config, PyInterpreterState_Delete() free
-       after PyInterpreterState_Delete() */
+       the core config memory */
-    _PyCoreConfig core_config = interp->core_config;
+    int show_ref_count = interp->core_config.show_ref_count;
+    int dump_refs = interp->core_config.dump_refs;
+    int malloc_stats = interp->core_config.malloc_stats;
 
     /* Remaining threads (e.g. daemon threads) will automatically exit
        after taking the GIL (in PyEval_RestoreThread()). */
@@ -1205,7 +1153,7 @@ Py_FinalizeEx(void)
     _PyHash_Fini();
 
 #ifdef Py_REF_DEBUG
-    if (core_config.show_ref_count) {
+    if (show_ref_count) {
         _PyDebug_PrintTotalRefs();
     }
 #endif
@@ -1216,7 +1164,7 @@ Py_FinalizeEx(void)
      * Alas, a lot of stuff may still be alive now that will be cleaned
      * up later.
      */
-    if (core_config.dump_refs) {
+    if (dump_refs) {
         _Py_PrintReferences(stderr);
     }
 #endif /* Py_TRACE_REFS */
@@ -1280,12 +1228,12 @@ Py_FinalizeEx(void)
      * An address can be used to find the repr of the object, printed
      * above by _Py_PrintReferences.
      */
-    if (core_config.dump_refs) {
+    if (dump_refs) {
         _Py_PrintReferenceAddresses(stderr);
     }
 #endif /* Py_TRACE_REFS */
 #ifdef WITH_PYMALLOC
-    if (core_config.malloc_stats) {
+    if (malloc_stats) {
         _PyObject_DebugMallocStats(stderr);
     }
 #endif
@@ -1347,14 +1295,23 @@ new_interpreter(PyThreadState **tstate_p)
     save_tstate = PyThreadState_Swap(tstate);
 
     /* Copy the current interpreter config into the new interpreter */
+    _PyCoreConfig *core_config;
+    _PyMainInterpreterConfig *config;
     if (save_tstate != NULL) {
-        interp->core_config = save_tstate->interp->core_config;
+        core_config = &save_tstate->interp->core_config;
-        interp->config = save_tstate->interp->config;
+        config = &save_tstate->interp->config;
     } else {
         /* No current thread state, copy from the main interpreter */
         PyInterpreterState *main_interp = PyInterpreterState_Main();
-        interp->core_config = main_interp->core_config;
+        core_config = &main_interp->core_config;
-        interp->config = main_interp->config;
+        config = &main_interp->config;
+    }
+
+    if (_PyCoreConfig_Copy(&interp->core_config, core_config) < 0) {
+        return _Py_INIT_ERR("failed to copy core config");
+    }
+    if (_PyMainInterpreterConfig_Copy(&interp->config, config) < 0) {
+        return _Py_INIT_ERR("failed to copy main interpreter config");
     }
 
     err = _PyPathConfig_Init(&interp->core_config);

Python/pystate.c

@@ -183,6 +183,8 @@ PyInterpreterState_Clear(PyInterpreterState *interp)
     for (p = interp->tstate_head; p != NULL; p = p->next)
         PyThreadState_Clear(p);
     HEAD_UNLOCK();
+    _PyCoreConfig_Clear(&interp->core_config);
+    _PyMainInterpreterConfig_Clear(&interp->config);
     Py_CLEAR(interp->codec_search_path);
     Py_CLEAR(interp->codec_search_cache);
     Py_CLEAR(interp->codec_error_registry);