bpo-36142: Add _PyPreConfig.allocator (GH-12181)

* Move 'allocator' and 'dev_mode' fields from _PyCoreConfig
  to _PyPreConfig.
* Fix InitConfigTests of test_embed: dev_mode sets allocator to
  "debug", add a new tests for env vars with dev mode enabled.

Include/cpython/coreconfig.h

@@ -80,6 +80,9 @@ typedef struct {
        Set by -X utf8 command line option and PYTHONUTF8 environment variable.
        If set to -1 (default), inherit Py_UTF8Mode value. */
     int utf8_mode;
+
+    int dev_mode;           /* Development mode. PYTHONDEVMODE, -X dev */
+    char *allocator;        /* Memory allocator: PYTHONMALLOC */
 } _PyPreConfig;
 
 #ifdef MS_WINDOWS
@@ -109,9 +112,6 @@ typedef struct {
     int use_hash_seed;      /* PYTHONHASHSEED=x */
     unsigned long hash_seed;
 
-    const char *allocator;  /* Memory allocator: PYTHONMALLOC */
-    int dev_mode;           /* PYTHONDEVMODE, -X dev */
-
     /* Enable faulthandler?
        Set to 1 by -X faulthandler and PYTHONFAULTHANDLER. -1 means unset. */
     int faulthandler;

Lib/test/test_embed.py

@@ -561,30 +561,36 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         }
         self.check_config("init_from_config", config)
 
+    INIT_ENV_CONFIG = {
+        'use_hash_seed': 1,
+        'hash_seed': 42,
+        'allocator': 'malloc_debug',
+        'tracemalloc': 2,
+        'import_time': 1,
+        'malloc_stats': 1,
+        'utf8_mode': 1,
+        'filesystem_encoding': 'utf-8',
+        'filesystem_errors': UTF8_MODE_ERRORS,
+        'inspect': 1,
+        'optimization_level': 2,
+        'pycache_prefix': 'env_pycache_prefix',
+        'write_bytecode': 0,
+        'verbose': 1,
+        'buffered_stdio': 0,
+        'stdio_encoding': 'iso8859-1',
+        'stdio_errors': 'replace',
+        'user_site_directory': 0,
+        'faulthandler': 1,
+    }
+
     def test_init_env(self):
-        config = {
+        self.check_config("init_env", self.INIT_ENV_CONFIG)
-            'use_hash_seed': 1,
+
-            'hash_seed': 42,
+    def test_init_env_dev_mode(self):
-            'allocator': 'malloc_debug',
+        config = dict(self.INIT_ENV_CONFIG,
-            'tracemalloc': 2,
+                      allocator='debug',
-            'import_time': 1,
+                      dev_mode=1)
-            'malloc_stats': 1,
+        self.check_config("init_env_dev_mode", config)
-            'utf8_mode': 1,
-            'filesystem_encoding': 'utf-8',
-            'filesystem_errors': self.UTF8_MODE_ERRORS,
-            'inspect': 1,
-            'optimization_level': 2,
-            'pycache_prefix': 'env_pycache_prefix',
-            'write_bytecode': 0,
-            'verbose': 1,
-            'buffered_stdio': 0,
-            'stdio_encoding': 'iso8859-1',
-            'stdio_errors': 'replace',
-            'user_site_directory': 0,
-            'faulthandler': 1,
-            'dev_mode': 1,
-        }
-        self.check_config("init_env", config)
 
     def test_init_dev_mode(self):
         config = {

Programs/_testembed.c

@@ -437,7 +437,7 @@ static int test_init_from_config(void)
     config.hash_seed = 123;
 
     putenv("PYTHONMALLOC=malloc");
-    config.allocator = "malloc_debug";
+    config.preconfig.allocator = "malloc_debug";
 
     /* dev_mode=1 is tested in test_init_dev_mode() */
 
@@ -577,7 +577,6 @@ static void test_init_env_putenvs(void)
     putenv("PYTHONPYCACHEPREFIX=env_pycache_prefix");
     putenv("PYTHONNOUSERSITE=1");
     putenv("PYTHONFAULTHANDLER=1");
-    putenv("PYTHONDEVMODE=1");
     putenv("PYTHONIOENCODING=iso8859-1:replace");
     /* FIXME: test PYTHONWARNINGS */
     /* FIXME: test PYTHONEXECUTABLE */
@@ -589,6 +588,15 @@ static void test_init_env_putenvs(void)
 }
 
 
+static void test_init_env_dev_mode_putenvs(void)
+{
+    test_init_env_putenvs();
+    putenv("PYTHONMALLOC=malloc");
+    putenv("PYTHONFAULTHANDLER=");
+    putenv("PYTHONDEVMODE=1");
+}
+
+
 static int test_init_env(void)
 {
     /* Test initialization from environment variables */
@@ -601,6 +609,18 @@ static int test_init_env(void)
 }
 
 
+static int test_init_env_dev_mode(void)
+{
+    /* Test initialization from environment variables */
+    Py_IgnoreEnvironmentFlag = 0;
+    test_init_env_dev_mode_putenvs();
+    _testembed_Py_Initialize();
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
 static int test_init_isolated(void)
 {
     /* Test _PyCoreConfig.isolated=1 */
@@ -615,7 +635,7 @@ static int test_init_isolated(void)
     /* Use path starting with "./" avoids a search along the PATH */
     config.program_name = L"./_testembed";
 
-    test_init_env_putenvs();
+    test_init_env_dev_mode_putenvs();
     _PyInitError err = _Py_InitializeFromConfig(&config);
     if (_Py_INIT_FAILED(err)) {
         _Py_ExitInitError(err);
@@ -631,7 +651,7 @@ static int test_init_dev_mode(void)
     _PyCoreConfig config = _PyCoreConfig_INIT;
     putenv("PYTHONFAULTHANDLER=");
     putenv("PYTHONMALLOC=");
-    config.dev_mode = 1;
+    config.preconfig.dev_mode = 1;
     config.program_name = L"./_testembed";
     _PyInitError err = _Py_InitializeFromConfig(&config);
     if (_Py_INIT_FAILED(err)) {
@@ -673,6 +693,7 @@ static struct TestCase TestCases[] = {
     { "init_global_config", test_init_global_config },
     { "init_from_config", test_init_from_config },
     { "init_env", test_init_env },
+    { "init_env_dev_mode", test_init_env_dev_mode },
     { "init_dev_mode", test_init_dev_mode },
     { "init_isolated", test_init_isolated },
     { NULL, NULL }

Python/coreconfig.c

@@ -521,8 +521,6 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2)
     COPY_ATTR(use_hash_seed);
     COPY_ATTR(hash_seed);
     COPY_ATTR(_install_importlib);
-    COPY_ATTR(allocator);
-    COPY_ATTR(dev_mode);
     COPY_ATTR(faulthandler);
     COPY_ATTR(tracemalloc);
     COPY_ATTR(import_time);
@@ -931,10 +929,6 @@ config_read_env_vars(_PyCoreConfig *config)
                  "PYTHONLEGACYWINDOWSSTDIO");
 #endif
 
-    if (config->allocator == NULL) {
-        config->allocator = _PyCoreConfig_GetEnv(config, "PYTHONMALLOC");
-    }
-
     if (_PyCoreConfig_GetEnv(config, "PYTHONDUMPREFS")) {
         config->dump_refs = 1;
     }
@@ -1059,11 +1053,6 @@ config_read_complex_options(_PyCoreConfig *config)
        || config_get_xoption(config, L"importtime")) {
         config->import_time = 1;
     }
-    if (config_get_xoption(config, L"dev" ) ||
-        _PyCoreConfig_GetEnv(config, "PYTHONDEVMODE"))
-    {
-        config->dev_mode = 1;
-    }
 
     _PyInitError err;
     if (config->tracemalloc < 0) {
@@ -1427,13 +1416,10 @@ _PyCoreConfig_Read(_PyCoreConfig *config, const _PyPreConfig *preconfig)
     }
 
     /* default values */
-    if (config->dev_mode) {
+    if (config->preconfig.dev_mode) {
         if (config->faulthandler < 0) {
             config->faulthandler = 1;
         }
-        if (config->allocator == NULL) {
-            config->allocator = "debug";
-        }
     }
     if (config->use_hash_seed < 0) {
         config->use_hash_seed = 0;
@@ -1572,8 +1558,6 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config)
     SET_ITEM_INT(install_signal_handlers);
     SET_ITEM_INT(use_hash_seed);
     SET_ITEM_UINT(hash_seed);
-    SET_ITEM_STR(allocator);
-    SET_ITEM_INT(dev_mode);
     SET_ITEM_INT(faulthandler);
     SET_ITEM_INT(tracemalloc);
     SET_ITEM_INT(import_time);
@@ -1950,7 +1934,7 @@ config_init_warnoptions(_PyCoreConfig *config, const _PyCmdline *cmdline)
      * the lowest precedence entries first so that later entries override them.
      */
 
-    if (config->dev_mode) {
+    if (config->preconfig.dev_mode) {
         err = _Py_wstrlist_append(&config->nwarnoption,
                                   &config->warnoptions,
                                   L"default");

Python/preconfig.c

@@ -125,6 +125,15 @@ precmdline_clear(_PyPreCmdline *cmdline)
 void
 _PyPreConfig_Clear(_PyPreConfig *config)
 {
+#define CLEAR(ATTR) \
+    do { \
+        PyMem_RawFree(ATTR); \
+        ATTR = NULL; \
+    } while (0)
+
+    CLEAR(config->allocator);
+
+#undef CLEAR
 }
 
 
@@ -134,6 +143,15 @@ _PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2)
     _PyPreConfig_Clear(config);
 
 #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR
+#define COPY_STR_ATTR(ATTR) \
+    do { \
+        if (config2->ATTR != NULL) { \
+            config->ATTR = _PyMem_RawStrdup(config2->ATTR); \
+            if (config->ATTR == NULL) { \
+                return -1; \
+            } \
+        } \
+    } while (0)
 
     COPY_ATTR(isolated);
     COPY_ATTR(use_environment);
@@ -143,8 +161,11 @@ _PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2)
     COPY_ATTR(legacy_windows_fs_encoding);
 #endif
     COPY_ATTR(utf8_mode);
+    COPY_ATTR(dev_mode);
+    COPY_STR_ATTR(allocator);
 
 #undef COPY_ATTR
+#undef COPY_STR_ATTR
     return 0;
 }
 
@@ -345,6 +366,7 @@ preconfig_read(_PyPreConfig *config, const _PyPreCmdline *cmdline)
 {
     _PyPreConfig_GetGlobalConfig(config);
 
+    /* isolated and use_environment */
     if (config->isolated > 0) {
         config->use_environment = 0;
     }
@@ -354,6 +376,7 @@ preconfig_read(_PyPreConfig *config, const _PyPreCmdline *cmdline)
         config->use_environment = 0;
     }
 
+    /* legacy_windows_fs_encoding, utf8_mode, coerce_c_locale */
     if (config->use_environment) {
 #ifdef MS_WINDOWS
         _Py_get_env_flag(config, &config->legacy_windows_fs_encoding,
@@ -414,11 +437,43 @@ preconfig_read(_PyPreConfig *config, const _PyPreCmdline *cmdline)
     if (config->utf8_mode < 0) {
         config->utf8_mode = 0;
     }
+    if (config->coerce_c_locale < 0) {
+        config->coerce_c_locale = 0;
+    }
+
+    /* dev_mode */
+    if ((cmdline && _Py_get_xoption(cmdline->nxoption, cmdline->xoptions, L"dev"))
+        || _PyPreConfig_GetEnv(config, "PYTHONDEVMODE"))
+    {
+        config->dev_mode = 1;
+    }
+    if (config->dev_mode < 0) {
+        config->dev_mode = 0;
+    }
+
+    /* allocator */
+    if (config->dev_mode && config->allocator == NULL) {
+        config->allocator = _PyMem_RawStrdup("debug");
+        if (config->allocator == NULL) {
+            return _Py_INIT_NO_MEMORY();
+        }
+    }
+
+    if (config->allocator == NULL) {
+        const char *allocator = _PyPreConfig_GetEnv(config, "PYTHONMALLOC");
+        if (allocator) {
+            config->allocator = _PyMem_RawStrdup(allocator);
+            if (config->allocator == NULL) {
+                return _Py_INIT_NO_MEMORY();
+            }
+        }
+    }
 
     assert(config->coerce_c_locale >= 0);
     assert(config->utf8_mode >= 0);
     assert(config->isolated >= 0);
     assert(config->use_environment >= 0);
+    assert(config->dev_mode >= 0);
 
     return _Py_INIT_OK();
 }
@@ -448,6 +503,12 @@ _PyPreConfig_AsDict(const _PyPreConfig *config, PyObject *dict)
         } while (0)
 #define SET_ITEM_INT(ATTR) \
     SET_ITEM(#ATTR, PyLong_FromLong(config->ATTR))
+#define FROM_STRING(STR) \
+    ((STR != NULL) ? \
+        PyUnicode_FromString(STR) \
+        : (Py_INCREF(Py_None), Py_None))
+#define SET_ITEM_STR(ATTR) \
+    SET_ITEM(#ATTR, FROM_STRING(config->ATTR))
 
     SET_ITEM_INT(isolated);
     SET_ITEM_INT(use_environment);
@@ -457,13 +518,17 @@ _PyPreConfig_AsDict(const _PyPreConfig *config, PyObject *dict)
 #ifdef MS_WINDOWS
     SET_ITEM_INT(legacy_windows_fs_encoding);
 #endif
+    SET_ITEM_INT(dev_mode);
+    SET_ITEM_STR(allocator);
     return 0;
 
 fail:
     return -1;
 
+#undef FROM_STRING
 #undef SET_ITEM
 #undef SET_ITEM_INT
+#undef SET_ITEM_STR
 }
 
 

Python/pylifecycle.c

@@ -482,9 +482,9 @@ _Py_Initialize_ReconfigureCore(PyInterpreterState **interp_p,
 
     /* bpo-34008: For backward compatibility reasons, calling Py_Main() after
        Py_Initialize() ignores the new configuration. */
-    if (core_config->allocator != NULL) {
+    if (core_config->preconfig.allocator != NULL) {
         const char *allocator = _PyMem_GetAllocatorsName();
-        if (allocator == NULL || strcmp(core_config->allocator, allocator) != 0) {
+        if (allocator == NULL || strcmp(core_config->preconfig.allocator, allocator) != 0) {
             return _Py_INIT_USER_ERR("cannot modify memory allocator "
                                      "after first Py_Initialize()");
         }
@@ -521,8 +521,8 @@ pycore_init_runtime(const _PyCoreConfig *core_config)
         return err;
     }
 
-    if (core_config->allocator != NULL) {
+    if (core_config->preconfig.allocator != NULL) {
-        if (_PyMem_SetupAllocators(core_config->allocator) < 0) {
+        if (_PyMem_SetupAllocators(core_config->preconfig.allocator) < 0) {
             return _Py_INIT_USER_ERR("Unknown PYTHONMALLOC allocator");
         }
     }

Python/sysmodule.c

@@ -2180,7 +2180,7 @@ make_flags(void)
     SetFlag(config->quiet);
     SetFlag(config->use_hash_seed == 0 || config->hash_seed != 0);
     SetFlag(config->preconfig.isolated);
-    PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->dev_mode));
+    PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->preconfig.dev_mode));
     SetFlag(config->preconfig.utf8_mode);
 #undef SetFlag