bpo-36301: Add _Py_GetConfigsAsDict() function (GH-12540)

* Add _Py_GetConfigsAsDict() function to get all configurations as a
  dict.
* dump_config() of _testembed.c now dumps preconfig as a separated
  key: call _Py_GetConfigsAsDict().
* Make _PyMainInterpreterConfig_AsDict() private.

Include/cpython/coreconfig.h

@@ -400,8 +400,7 @@ typedef struct {
 
 /* --- Function used for testing ---------------------------------- */
 
-PyAPI_FUNC(PyObject *) _Py_GetGlobalVariablesAsDict(void);
+PyAPI_FUNC(PyObject*) _Py_GetConfigsAsDict(void);
-PyAPI_FUNC(PyObject *) _PyCoreConfig_AsDict(const _PyCoreConfig *config);
 
 #ifdef __cplusplus
 }

Include/cpython/pylifecycle.h

@@ -33,9 +33,6 @@ PyAPI_FUNC(void) _PyMainInterpreterConfig_Clear(_PyMainInterpreterConfig *);
 PyAPI_FUNC(int) _PyMainInterpreterConfig_Copy(
     _PyMainInterpreterConfig *config,
     const _PyMainInterpreterConfig *config2);
-/* Used by _testcapi.get_main_config() */
-PyAPI_FUNC(PyObject*) _PyMainInterpreterConfig_AsDict(
-    const _PyMainInterpreterConfig *config);
 
 PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter(
     PyInterpreterState *interp,

Include/internal/pycore_coreconfig.h

@@ -90,8 +90,7 @@ PyAPI_FUNC(void) _Py_get_env_flag(_PyPreConfig *config,
 PyAPI_FUNC(_PyInitError) _PyPreConfig_Read(_PyPreConfig *config,
     const _PyArgv *args,
     const _PyCoreConfig *coreconfig);
-PyAPI_FUNC(int) _PyPreConfig_AsDict(const _PyPreConfig *config,
+PyAPI_FUNC(PyObject*) _PyPreConfig_AsDict(const _PyPreConfig *config);
-    PyObject *dict);
 PyAPI_FUNC(_PyInitError) _PyPreConfig_ReadFromArgv(_PyPreConfig *config,
     const _PyArgv *args);
 PyAPI_FUNC(_PyInitError) _PyPreConfig_Write(_PyPreConfig *config);
@@ -121,6 +120,11 @@ PyAPI_FUNC(_PyInitError) _PyCoreConfig_ReadFromArgv(_PyCoreConfig *config,
     const _PyArgv *args);
 PyAPI_FUNC(_PyInitError) _PyCoreConfig_Write(const _PyCoreConfig *config);
 
+/* --- _PyMainInterpreterConfig ----------------------------------- */
+
+PyAPI_FUNC(PyObject*) _PyMainInterpreterConfig_AsDict(
+    const _PyMainInterpreterConfig *config);
+
 #ifdef __cplusplus
 }
 #endif

Lib/test/pythoninfo.py

@@ -598,18 +598,15 @@ def collect_get_config(info_add):
     # Dump global configuration variables, _PyCoreConfig
     # and _PyMainInterpreterConfig
     try:
-        from _testcapi import get_global_config, get_core_config, get_main_config
+        from _testcapi import get_configs
     except ImportError:
         return
 
-    for prefix, get_config_func in (
+    all_configs = get_configs()
-        ('global_config', get_global_config),
+    for config_type in sorted(all_configs):
-        ('core_config', get_core_config),
+        config = all_configs[config_type]
-        ('main_config', get_main_config),
-    ):
-        config = get_config_func()
         for key in sorted(config):
-            info_add('%s[%s]' % (prefix, key), repr(config[key]))
+            info_add('%s[%s]' % (config_type, key), repr(config[key]))
 
 
 def collect_subprocess(info_add):

Lib/test/test_embed.py

@@ -268,13 +268,19 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
     )
     # Mark config which should be get by get_default_config()
     GET_DEFAULT_CONFIG = object()
+    DEFAULT_PRE_CONFIG = {
+        'allocator': None,
+        'coerce_c_locale': 0,
+        'coerce_c_locale_warn': 0,
+        'dev_mode': 0,
+        'isolated': 0,
+        'use_environment': 1,
+        'utf8_mode': 0,
+    }
     DEFAULT_CORE_CONFIG = {
         'install_signal_handlers': 1,
-        'use_environment': 1,
         'use_hash_seed': 0,
         'hash_seed': 0,
-        'allocator': None,
-        'dev_mode': 0,
         'faulthandler': 0,
         'tracemalloc': 0,
         'import_time': 0,
@@ -286,10 +292,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'filesystem_encoding': GET_DEFAULT_CONFIG,
         'filesystem_errors': GET_DEFAULT_CONFIG,
 
-        'utf8_mode': 0,
-        'coerce_c_locale': 0,
-        'coerce_c_locale_warn': 0,
-
         'pycache_prefix': None,
         'program_name': './_testembed',
         'argv': [""],
@@ -306,7 +308,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'exec_prefix': GET_DEFAULT_CONFIG,
         'base_exec_prefix': GET_DEFAULT_CONFIG,
 
-        'isolated': 0,
         'site_import': 1,
         'bytes_warning': 0,
         'inspect': 0,
@@ -332,8 +333,10 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         '_frozen': 0,
     }
     if MS_WINDOWS:
-        DEFAULT_CORE_CONFIG.update({
+        DEFAULT_PRE_CONFIG.update({
             'legacy_windows_fs_encoding': 0,
+        })
+        DEFAULT_CORE_CONFIG.update({
             'legacy_windows_stdio': 0,
         })
 
@@ -359,6 +362,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'Py_HashRandomizationFlag': 1,
         '_Py_HasFileSystemDefaultEncodeErrors': 0,
     }
+    COPY_GLOBAL_PRE_CONFIG = [
+        ('Py_IgnoreEnvironmentFlag', 'use_environment', True),
+        ('Py_IsolatedFlag', 'isolated'),
+        ('Py_UTF8Mode', 'utf8_mode'),
+    ]
     COPY_GLOBAL_CONFIG = [
         # Copy core config to global config for expected values
         # True means that the core config value is inverted (0 => 1 and 1 => 0)
@@ -368,21 +376,20 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         ('Py_FileSystemDefaultEncodeErrors', 'filesystem_errors'),
         ('Py_FileSystemDefaultEncoding', 'filesystem_encoding'),
         ('Py_FrozenFlag', '_frozen'),
-        ('Py_IgnoreEnvironmentFlag', 'use_environment', True),
         ('Py_InspectFlag', 'inspect'),
         ('Py_InteractiveFlag', 'interactive'),
-        ('Py_IsolatedFlag', 'isolated'),
         ('Py_NoSiteFlag', 'site_import', True),
         ('Py_NoUserSiteDirectory', 'user_site_directory', True),
         ('Py_OptimizeFlag', 'optimization_level'),
         ('Py_QuietFlag', 'quiet'),
-        ('Py_UTF8Mode', 'utf8_mode'),
         ('Py_UnbufferedStdioFlag', 'buffered_stdio', True),
         ('Py_VerboseFlag', 'verbose'),
     ]
     if MS_WINDOWS:
-        COPY_GLOBAL_CONFIG.extend((
+        COPY_GLOBAL_PRE_CONFIG.extend((
             ('Py_LegacyWindowsFSEncodingFlag', 'legacy_windows_fs_encoding'),
+        ))
+        COPY_GLOBAL_CONFIG.extend((
             ('Py_LegacyWindowsStdioFlag', 'legacy_windows_stdio'),
         ))
 
@@ -408,7 +415,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         expected['xoptions'] = self.main_xoptions(core_config['xoptions'])
         self.assertEqual(main_config, expected)
 
-    def get_expected_config(self, expected, env):
+    def get_expected_config(self, expected, expected_preconfig, env):
         expected = dict(self.DEFAULT_CORE_CONFIG, **expected)
 
         code = textwrap.dedent('''
@@ -436,7 +443,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         # when test_embed is run from a venv (bpo-35313)
         args = (sys.executable, '-S', '-c', code)
         env = dict(env)
-        if not expected['isolated']:
+        if not expected_preconfig['isolated']:
             env['PYTHONCOERCECLOCALE'] = '0'
             env['PYTHONUTF8'] = '0'
         proc = subprocess.run(args, env=env,
@@ -453,6 +460,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                 expected[key] = config[key]
         return expected
 
+    def check_pre_config(self, config, expected):
+        pre_config = dict(config['pre_config'])
+        core_config = dict(config['core_config'])
+        self.assertEqual(pre_config, expected)
+
     def check_core_config(self, config, expected):
         core_config = dict(config['core_config'])
         for key in self.UNTESTED_CORE_CONFIG:
@@ -460,6 +472,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         self.assertEqual(core_config, expected)
 
     def check_global_config(self, config):
+        pre_config = config['pre_config']
         core_config = config['core_config']
 
         expected = dict(self.DEFAULT_GLOBAL_CONFIG)
@@ -470,10 +483,17 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             else:
                 global_key, core_key = item
                 expected[global_key] = core_config[core_key]
+        for item in self.COPY_GLOBAL_PRE_CONFIG:
+            if len(item) == 3:
+                global_key, core_key, opposite = item
+                expected[global_key] = 0 if pre_config[core_key] else 1
+            else:
+                global_key, core_key = item
+                expected[global_key] = pre_config[core_key]
 
         self.assertEqual(config['global_config'], expected)
 
-    def check_config(self, testname, expected):
+    def check_config(self, testname, expected_config, expected_preconfig):
         env = dict(os.environ)
         # Remove PYTHON* environment variables to get deterministic environment
         for key in list(env):
@@ -488,15 +508,21 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         # Ignore err
         config = json.loads(out)
 
-        expected = self.get_expected_config(expected, env)
+        expected_preconfig = dict(self.DEFAULT_PRE_CONFIG, **expected_preconfig)
-        self.check_core_config(config, expected)
+        expected_config = self.get_expected_config(expected_config, expected_preconfig, env)
+
+        self.check_core_config(config, expected_config)
+        self.check_pre_config(config, expected_preconfig)
         self.check_main_config(config)
         self.check_global_config(config)
 
     def test_init_default_config(self):
-        self.check_config("init_default_config", {})
+        self.check_config("init_default_config", {}, {})
 
     def test_init_global_config(self):
+        preconfig = {
+            'utf8_mode': 1,
+        }
         config = {
             'program_name': './globalvar',
             'site_import': 0,
@@ -509,7 +535,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'quiet': 1,
             'buffered_stdio': 0,
 
-            'utf8_mode': 1,
             'stdio_encoding': 'utf-8',
             'stdio_errors': 'surrogateescape',
             'filesystem_encoding': 'utf-8',
@@ -517,21 +542,23 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'user_site_directory': 0,
             '_frozen': 1,
         }
-        self.check_config("init_global_config", config)
+        self.check_config("init_global_config", config, preconfig)
 
     def test_init_from_config(self):
+        preconfig = {
+            'allocator': 'malloc',
+            'utf8_mode': 1,
+        }
         config = {
             'install_signal_handlers': 0,
             'use_hash_seed': 1,
             'hash_seed': 123,
-            'allocator': 'malloc',
             'tracemalloc': 2,
             'import_time': 1,
             'show_ref_count': 1,
             'show_alloc_count': 1,
             'malloc_stats': 1,
 
-            'utf8_mode': 1,
             'stdio_encoding': 'iso8859-1',
             'stdio_errors': 'replace',
             'filesystem_encoding': 'utf-8',
@@ -559,16 +586,18 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             '_check_hash_pycs_mode': 'always',
             '_frozen': 1,
         }
-        self.check_config("init_from_config", config)
+        self.check_config("init_from_config", config, preconfig)
 
+    INIT_ENV_PRECONFIG = {
+        'allocator': 'malloc',
+        'utf8_mode': 1,
+    }
     INIT_ENV_CONFIG = {
         'use_hash_seed': 1,
         'hash_seed': 42,
-        'allocator': 'malloc',
         'tracemalloc': 2,
         'import_time': 1,
         'malloc_stats': 1,
-        'utf8_mode': 1,
         'filesystem_encoding': 'utf-8',
         'filesystem_errors': UTF8_MODE_ERRORS,
         'inspect': 1,
@@ -584,35 +613,42 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
     }
 
     def test_init_env(self):
-        self.check_config("init_env", self.INIT_ENV_CONFIG)
+        self.check_config("init_env", self.INIT_ENV_CONFIG, self.INIT_ENV_PRECONFIG)
 
     def test_init_env_dev_mode(self):
-        config = dict(self.INIT_ENV_CONFIG,
+        preconfig = dict(self.INIT_ENV_PRECONFIG,
                       allocator='debug',
                       dev_mode=1)
-        self.check_config("init_env_dev_mode", config)
+        config = dict(self.INIT_ENV_CONFIG,
+                      dev_mode=1)
+        self.check_config("init_env_dev_mode", config, preconfig)
 
     def test_init_env_dev_mode(self):
-        config = dict(self.INIT_ENV_CONFIG,
+        preconfig = dict(self.INIT_ENV_PRECONFIG,
-                      allocator='malloc',
+                         allocator='malloc',
-                      dev_mode=1)
+                         dev_mode=1)
-        self.check_config("init_env_dev_mode_alloc", config)
+        config = dict(self.INIT_ENV_CONFIG)
+        self.check_config("init_env_dev_mode_alloc", config, preconfig)
 
     def test_init_dev_mode(self):
-        config = {
+        preconfig = {
-            'dev_mode': 1,
-            'faulthandler': 1,
             'allocator': 'debug',
+            'dev_mode': 1,
         }
-        self.check_config("init_dev_mode", config)
+        config = {
+            'faulthandler': 1,
+        }
+        self.check_config("init_dev_mode", config, preconfig)
 
     def test_init_isolated(self):
-        config = {
+        preconfig = {
             'isolated': 1,
             'use_environment': 0,
+        }
+        config = {
             'user_site_directory': 0,
         }
-        self.check_config("init_isolated", config)
+        self.check_config("init_isolated", config, preconfig)
 
 
 if __name__ == "__main__":

Modules/_testcapimodule.c

@@ -4675,27 +4675,9 @@ decode_locale_ex(PyObject *self, PyObject *args)
 
 
 static PyObject *
-get_global_config(PyObject *self, PyObject *Py_UNUSED(args))
+get_configs(PyObject *self, PyObject *Py_UNUSED(args))
 {
-    return _Py_GetGlobalVariablesAsDict();
+    return _Py_GetConfigsAsDict();
-}
-
-
-static PyObject *
-get_core_config(PyObject *self, PyObject *Py_UNUSED(args))
-{
-    PyInterpreterState *interp = _PyInterpreterState_Get();
-    const _PyCoreConfig *config = _PyInterpreterState_GetCoreConfig(interp);
-    return _PyCoreConfig_AsDict(config);
-}
-
-
-static PyObject *
-get_main_config(PyObject *self, PyObject *Py_UNUSED(args))
-{
-    PyInterpreterState *interp = _PyInterpreterState_Get();
-    const _PyMainInterpreterConfig *config = _PyInterpreterState_GetMainConfig(interp);
-    return _PyMainInterpreterConfig_AsDict(config);
 }
 
 
@@ -4942,9 +4924,7 @@ static PyMethodDef TestMethods[] = {
     {"bad_get", (PyCFunction)(void(*)(void))bad_get, METH_FASTCALL},
     {"EncodeLocaleEx", encode_locale_ex, METH_VARARGS},
     {"DecodeLocaleEx", decode_locale_ex, METH_VARARGS},
-    {"get_global_config", get_global_config, METH_NOARGS},
+    {"get_configs", get_configs, METH_NOARGS},
-    {"get_core_config", get_core_config, METH_NOARGS},
-    {"get_main_config", get_main_config, METH_NOARGS},
 #ifdef Py_REF_DEBUG
     {"negative_refcount", negative_refcount, METH_NOARGS},
 #endif

Programs/_testembed.c

@@ -301,64 +301,29 @@ static int test_initialize_pymain(void)
 static int
 dump_config_impl(void)
 {
-    PyObject *config = NULL;
+    PyObject *config = _Py_GetConfigsAsDict();
-    PyObject *dict = NULL;
-
-    config = PyDict_New();
     if (config == NULL) {
-        goto error;
+        return -1;
     }
 
-    /* global config */
+    PyObject *res;
-    dict = _Py_GetGlobalVariablesAsDict();
-    if (dict == NULL) {
-        goto error;
-    }
-    if (PyDict_SetItemString(config, "global_config", dict) < 0) {
-        goto error;
-    }
-    Py_CLEAR(dict);
-
-    /* core config */
-    PyInterpreterState *interp = _PyInterpreterState_Get();
-    const _PyCoreConfig *core_config = _PyInterpreterState_GetCoreConfig(interp);
-    dict = _PyCoreConfig_AsDict(core_config);
-    if (dict == NULL) {
-        goto error;
-    }
-    if (PyDict_SetItemString(config, "core_config", dict) < 0) {
-        goto error;
-    }
-    Py_CLEAR(dict);
-
-    /* main config */
-    const _PyMainInterpreterConfig *main_config = _PyInterpreterState_GetMainConfig(interp);
-    dict = _PyMainInterpreterConfig_AsDict(main_config);
-    if (dict == NULL) {
-        goto error;
-    }
-    if (PyDict_SetItemString(config, "main_config", dict) < 0) {
-        goto error;
-    }
-    Py_CLEAR(dict);
-
     PyObject *json = PyImport_ImportModule("json");
-    PyObject *res = PyObject_CallMethod(json, "dumps", "O", config);
+    if (json) {
-    Py_DECREF(json);
+        res = PyObject_CallMethod(json, "dumps", "O", config);
+        Py_DECREF(json);
+    }
+    else {
+        res = NULL;
+    }
     Py_CLEAR(config);
     if (res == NULL) {
-        goto error;
+        return -1;
     }
 
     PySys_FormatStdout("%S\n", res);
     Py_DECREF(res);
 
     return 0;
-
-error:
-    Py_XDECREF(config);
-    Py_XDECREF(dict);
-    return -1;
 }
 
 

Python/coreconfig.c

@@ -131,7 +131,7 @@ int Py_LegacyWindowsStdioFlag = 0; /* Uses FileIO instead of WindowsConsoleIO */
 #endif
 
 
-PyObject *
+static PyObject *
 _Py_GetGlobalVariablesAsDict(void)
 {
     PyObject *dict, *obj;
@@ -1563,7 +1563,7 @@ _PyCoreConfig_Write(const _PyCoreConfig *config)
 }
 
 
-PyObject *
+static PyObject *
 _PyCoreConfig_AsDict(const _PyCoreConfig *config)
 {
     PyObject *dict;
@@ -1573,11 +1573,6 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config)
         return NULL;
     }
 
-    if (_PyPreConfig_AsDict(&config->preconfig, dict) < 0) {
-        Py_DECREF(dict);
-        return NULL;
-    }
-
 #define SET_ITEM(KEY, EXPR) \
         do { \
             PyObject *obj = (EXPR); \
@@ -2158,3 +2153,67 @@ done:
     cmdline_clear(&cmdline);
     return err;
 }
+
+
+PyObject*
+_Py_GetConfigsAsDict(void)
+{
+    PyObject *config = NULL;
+    PyObject *dict = NULL;
+
+    config = PyDict_New();
+    if (config == NULL) {
+        goto error;
+    }
+
+    /* global config */
+    dict = _Py_GetGlobalVariablesAsDict();
+    if (dict == NULL) {
+        goto error;
+    }
+    if (PyDict_SetItemString(config, "global_config", dict) < 0) {
+        goto error;
+    }
+    Py_CLEAR(dict);
+
+    /* pre config */
+    PyInterpreterState *interp = _PyInterpreterState_Get();
+    const _PyCoreConfig *core_config = _PyInterpreterState_GetCoreConfig(interp);
+    const _PyPreConfig *pre_config = &core_config->preconfig;
+    dict = _PyPreConfig_AsDict(pre_config);
+    if (dict == NULL) {
+        goto error;
+    }
+    if (PyDict_SetItemString(config, "pre_config", dict) < 0) {
+        goto error;
+    }
+    Py_CLEAR(dict);
+
+    /* core config */
+    dict = _PyCoreConfig_AsDict(core_config);
+    if (dict == NULL) {
+        goto error;
+    }
+    if (PyDict_SetItemString(config, "core_config", dict) < 0) {
+        goto error;
+    }
+    Py_CLEAR(dict);
+
+    /* main config */
+    const _PyMainInterpreterConfig *main_config = _PyInterpreterState_GetMainConfig(interp);
+    dict = _PyMainInterpreterConfig_AsDict(main_config);
+    if (dict == NULL) {
+        goto error;
+    }
+    if (PyDict_SetItemString(config, "main_config", dict) < 0) {
+        goto error;
+    }
+    Py_CLEAR(dict);
+
+    return config;
+
+error:
+    Py_XDECREF(config);
+    Py_XDECREF(dict);
+    return NULL;
+}

Python/preconfig.c

@@ -574,9 +574,16 @@ _PyPreCmdline_SetPreConfig(const _PyPreCmdline *cmdline, _PyPreConfig *config)
 }
 
 
-int
+PyObject*
-_PyPreConfig_AsDict(const _PyPreConfig *config, PyObject *dict)
+_PyPreConfig_AsDict(const _PyPreConfig *config)
 {
+    PyObject *dict;
+
+    dict = PyDict_New();
+    if (dict == NULL) {
+        return NULL;
+    }
+
 #define SET_ITEM(KEY, EXPR) \
         do { \
             PyObject *obj = (EXPR); \
@@ -608,10 +615,11 @@ _PyPreConfig_AsDict(const _PyPreConfig *config, PyObject *dict)
 #endif
     SET_ITEM_INT(dev_mode);
     SET_ITEM_STR(allocator);
-    return 0;
+    return dict;
 
 fail:
-    return -1;
+    Py_DECREF(dict);
+    return NULL;
 
 #undef FROM_STRING
 #undef SET_ITEM