bpo-40910: Export Py_GetArgcArgv() function (GH-20721)

Export explicitly the Py_GetArgcArgv() function to the C API and
document the function. Previously, it was exported implicitly which
no longer works since Python is built with -fvisibility=hidden.

* Add PyConfig._orig_argv member.
* Py_InitializeFromConfig() no longer calls _PyConfig_Write() twice.
* PyConfig_Read() no longer initializes Py_GetArgcArgv(): it is now
  _PyConfig_Write() responsibility.
* _PyConfig_Write() result type becomes PyStatus instead of void.
* Write an unit test on Py_GetArgcArgv().

Doc/c-api/init_config.rst

@@ -43,6 +43,7 @@ Functions:
 * :c:func:`Py_PreInitializeFromArgs`
 * :c:func:`Py_PreInitializeFromBytesArgs`
 * :c:func:`Py_RunMain`
+* :c:func:`Py_GetArgcArgv`
 
 The preconfiguration (``PyPreConfig`` type) is stored in
 ``_PyRuntime.preconfig`` and the configuration (``PyConfig`` type) is stored in
@@ -984,6 +985,14 @@ customized Python always running in isolated mode using
 :c:func:`Py_RunMain`.
 
 
+Py_GetArgcArgv()
+----------------
+
+.. c:function:: void Py_GetArgcArgv(int *argc, wchar_t ***argv)
+
+   Get the original command line arguments, before Python modified them.
+
+
 Multi-Phase Initialization Private Provisional API
 --------------------------------------------------
 

Include/cpython/initconfig.h

@@ -411,6 +411,14 @@ typedef struct {
     /* If non-zero, disallow threads, subprocesses, and fork.
        Default: 0. */
     int _isolated_interpreter;
+
+    /* Original command line arguments. If _orig_argv is empty and _argv is
+       not equal to [''], PyConfig_Read() copies the configuration 'argv' list
+       into '_orig_argv' list before modifying 'argv' list (if parse_argv
+       is non-zero).
+
+       _PyConfig_Write() initializes Py_GetArgcArgv() to this list. */
+    PyWideStringList _orig_argv;
 } PyConfig;
 
 PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config);
@@ -436,5 +444,13 @@ PyAPI_FUNC(PyStatus) PyConfig_SetWideStringList(PyConfig *config,
     PyWideStringList *list,
     Py_ssize_t length, wchar_t **items);
 
+
+/* --- Helper functions --------------------------------------- */
+
+/* Get the original command line arguments, before Python modified them.
+
+   See also PyConfig._orig_argv. */
+PyAPI_FUNC(void) Py_GetArgcArgv(int *argc, wchar_t ***argv);
+
 #endif /* !Py_LIMITED_API */
 #endif /* !Py_PYCORECONFIG_H */

Include/internal/pycore_initconfig.h

@@ -150,7 +150,7 @@ extern PyStatus _PyConfig_Copy(
     PyConfig *config,
     const PyConfig *config2);
 extern PyStatus _PyConfig_InitPathConfig(PyConfig *config);
-extern void _PyConfig_Write(const PyConfig *config,
+extern PyStatus _PyConfig_Write(const PyConfig *config,
     struct pyruntimestate *runtime);
 extern PyStatus _PyConfig_SetPyArgv(
     PyConfig *config,

Lib/test/test_embed.py

@@ -366,6 +366,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'program_name': GET_DEFAULT_CONFIG,
         'parse_argv': 0,
         'argv': [""],
+        '_orig_argv': [],
 
         'xoptions': [],
         'warnoptions': [],
@@ -739,7 +740,12 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
 
             'pycache_prefix': 'conf_pycache_prefix',
             'program_name': './conf_program_name',
-            'argv': ['-c', 'arg2', ],
+            'argv': ['-c', 'arg2'],
+            '_orig_argv': ['python3',
+                           '-W', 'cmdline_warnoption',
+                           '-X', 'cmdline_xoption',
+                           '-c', 'pass',
+                           'arg2'],
             'parse_argv': 1,
             'xoptions': [
                 'config_xoption1=3',
@@ -872,6 +878,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         }
         config = {
             'argv': ['script.py'],
+            '_orig_argv': ['python3', '-X', 'dev', 'script.py'],
             'run_filename': os.path.abspath('script.py'),
             'dev_mode': 1,
             'faulthandler': 1,
@@ -886,9 +893,14 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         preconfig = {
             'isolated': 0,
         }
+        argv = ["python3",
+               "-E", "-I",
+               "-X", "dev",
+               "-X", "utf8",
+               "script.py"]
         config = {
-            'argv': ["python3", "-E", "-I",
+            'argv': argv,
-                     "-X", "dev", "-X", "utf8", "script.py"],
+            '_orig_argv': argv,
             'isolated': 0,
         }
         self.check_all_configs("test_preinit_dont_parse_argv", config, preconfig,
@@ -967,6 +979,9 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                 'ignore:::sysadd_warnoption',
                 'ignore:::config_warnoption',
             ],
+            '_orig_argv': ['python3',
+                           '-W', 'ignore:::cmdline_warnoption',
+                           '-X', 'cmdline_xoption'],
         }
         self.check_all_configs("test_init_sys_add", config, api=API_PYTHON)
 
@@ -975,6 +990,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                 'print(json.dumps(_testinternalcapi.get_configs()))')
         config = {
             'argv': ['-c', 'arg2'],
+            '_orig_argv': ['python3', '-c', code, 'arg2'],
             'program_name': './python3',
             'run_command': code + '\n',
             'parse_argv': 1,
@@ -986,6 +1002,9 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                 'print(json.dumps(_testinternalcapi.get_configs()))')
         config = {
             'argv': ['-c', 'arg2'],
+            '_orig_argv': ['python3',
+                           '-c', code,
+                           'arg2'],
             'program_name': './python3',
             'run_command': code + '\n',
             'parse_argv': 1,
@@ -999,6 +1018,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         config = {
             'parse_argv': 1,
             'argv': ['-c', 'arg1', '-v', 'arg3'],
+            '_orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'],
             'program_name': './argv0',
             'run_command': 'pass\n',
             'use_environment': 0,
@@ -1012,6 +1032,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         config = {
             'parse_argv': 0,
             'argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'],
+            '_orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'],
             'program_name': './argv0',
         }
         self.check_all_configs("test_init_dont_parse_argv", config, pre_config,
@@ -1299,10 +1320,17 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'faulthandler': 1,
             'bytes_warning': 1,
             'warnoptions': warnoptions,
+            '_orig_argv': ['python3',
+                           '-Wignore:::cmdline1',
+                           '-Wignore:::cmdline2'],
         }
         self.check_all_configs("test_init_warnoptions", config, preconfig,
                                api=API_PYTHON)
 
+    def test_get_argc_argv(self):
+        self.run_embedded_interpreter("test_get_argc_argv")
+        # ignore output
+
 
 class AuditingTests(EmbeddingTestsMixin, unittest.TestCase):
     def test_open_code_hook(self):

Misc/NEWS.d/next/C API/2020-06-08-15-59-06.bpo-40910.L56oI0.rst

@@ -0,0 +1,3 @@
+Export explicitly the :c:func:`Py_GetArgcArgv` function to the C API and
+document the function. Previously, it was exported implicitly which no
+longer works since Python is built with ``-fvisibility=hidden``.

PC/python3.def

@@ -734,6 +734,7 @@ EXPORTS
   Py_FinalizeEx=python310.Py_FinalizeEx
   Py_GenericAlias=python310.Py_GenericAlias
   Py_GenericAliasType=python310.Py_GenericAliasType
+  Py_GetArgcArgv=python310.Py_GetArgcArgv
   Py_GetBuildInfo=python310.Py_GetBuildInfo
   Py_GetCompiler=python310.Py_GetCompiler
   Py_GetCopyright=python310.Py_GetCopyright

Programs/_testembed.c

@@ -1334,6 +1334,7 @@ static int test_init_read_set(void)
     return 0;
 
 fail:
+    PyConfig_Clear(&config);
     Py_ExitStatusException(status);
 }
 
@@ -1592,6 +1593,46 @@ static int test_run_main(void)
 }
 
 
+static int test_get_argc_argv(void)
+{
+    PyConfig config;
+    PyConfig_InitPythonConfig(&config);
+
+    wchar_t *argv[] = {L"python3", L"-c",
+                       (L"import sys; "
+                        L"print(f'Py_RunMain(): sys.argv={sys.argv}')"),
+                       L"arg2"};
+    config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv);
+    config_set_string(&config, &config.program_name, L"./python3");
+
+    // Calling PyConfig_Read() twice must not change Py_GetArgcArgv() result.
+    // The second call is done by Py_InitializeFromConfig().
+    PyStatus status = PyConfig_Read(&config);
+    if (PyStatus_Exception(status)) {
+        PyConfig_Clear(&config);
+        Py_ExitStatusException(status);
+    }
+
+    init_from_config_clear(&config);
+
+    int get_argc;
+    wchar_t **get_argv;
+    Py_GetArgcArgv(&get_argc, &get_argv);
+    printf("argc: %i\n", get_argc);
+    assert(get_argc == Py_ARRAY_LENGTH(argv));
+    for (int i=0; i < get_argc; i++) {
+        printf("argv[%i]: %ls\n", i, get_argv[i]);
+        assert(wcscmp(get_argv[i], argv[i]) == 0);
+    }
+
+    Py_Finalize();
+
+    printf("\n");
+    printf("test ok\n");
+    return 0;
+}
+
+
 /* *********************************************************
  * List of test cases and the function that implements it.
  *
@@ -1649,6 +1690,7 @@ static struct TestCase TestCases[] = {
     {"test_init_setpythonhome", test_init_setpythonhome},
     {"test_init_warnoptions", test_init_warnoptions},
     {"test_run_main", test_run_main},
+    {"test_get_argc_argv", test_get_argc_argv},
 
     {"test_open_code_hook", test_open_code_hook},
     {"test_audit", test_audit},

Python/bootstrap_hash.c

@@ -580,7 +580,7 @@ _Py_HashRandomization_Init(const PyConfig *config)
         res = pyurandom(secret, secret_size, 0, 0);
         if (res < 0) {
             return _PyStatus_ERR("failed to get random numbers "
-                                "to initialize Python");
+                                 "to initialize Python");
         }
     }
     return _PyStatus_OK();

Python/initconfig.c

@@ -548,8 +548,6 @@ _Py_SetArgcArgv(Py_ssize_t argc, wchar_t * const *argv)
 }
 
 
-/* Make the *original* argc/argv available to other modules.
-   This is rare, but it is needed by the secureware extension. */
 void
 Py_GetArgcArgv(int *argc, wchar_t ***argv)
 {
@@ -859,6 +857,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
     COPY_ATTR(pathconfig_warnings);
     COPY_ATTR(_init_main);
     COPY_ATTR(_isolated_interpreter);
+    COPY_WSTRLIST(_orig_argv);
 
 #undef COPY_ATTR
 #undef COPY_WSTR_ATTR
@@ -960,6 +959,7 @@ config_as_dict(const PyConfig *config)
     SET_ITEM_INT(pathconfig_warnings);
     SET_ITEM_INT(_init_main);
     SET_ITEM_INT(_isolated_interpreter);
+    SET_ITEM_WSTRLIST(_orig_argv);
 
     return dict;
 
@@ -1856,7 +1856,7 @@ config_init_stdio(const PyConfig *config)
 
    - set Py_xxx global configuration variables
    - initialize C standard streams (stdin, stdout, stderr) */
-void
+PyStatus
 _PyConfig_Write(const PyConfig *config, _PyRuntimeState *runtime)
 {
     config_set_global_vars(config);
@@ -1870,6 +1870,13 @@ _PyConfig_Write(const PyConfig *config, _PyRuntimeState *runtime)
     preconfig->isolated = config->isolated;
     preconfig->use_environment = config->use_environment;
     preconfig->dev_mode = config->dev_mode;
+
+    if (_Py_SetArgcArgv(config->_orig_argv.length,
+                        config->_orig_argv.items) < 0)
+    {
+        return _PyStatus_NO_MEMORY();
+    }
+    return _PyStatus_OK();
 }
 
 
@@ -2493,7 +2500,6 @@ PyStatus
 PyConfig_Read(PyConfig *config)
 {
     PyStatus status;
-    PyWideStringList orig_argv = _PyWideStringList_INIT;
 
     status = _Py_PreInitializeFromConfig(config, NULL);
     if (_PyStatus_EXCEPTION(status)) {
@@ -2502,8 +2508,13 @@ PyConfig_Read(PyConfig *config)
 
     config_get_global_vars(config);
 
-    if (_PyWideStringList_Copy(&orig_argv, &config->argv) < 0) {
+    if (config->_orig_argv.length == 0
-        return _PyStatus_NO_MEMORY();
+        && !(config->argv.length == 1
+             && wcscmp(config->argv.items[0], L"") == 0))
+    {
+        if (_PyWideStringList_Copy(&config->_orig_argv, &config->argv) < 0) {
+            return _PyStatus_NO_MEMORY();
+        }
     }
 
     _PyPreCmdline precmdline = _PyPreCmdline_INIT;
@@ -2534,11 +2545,6 @@ PyConfig_Read(PyConfig *config)
         goto done;
     }
 
-    if (_Py_SetArgcArgv(orig_argv.length, orig_argv.items) < 0) {
-        status = _PyStatus_NO_MEMORY();
-        goto done;
-    }
-
     /* Check config consistency */
     assert(config->isolated >= 0);
     assert(config->use_environment >= 0);
@@ -2591,11 +2597,11 @@ PyConfig_Read(PyConfig *config)
     assert(config->check_hash_pycs_mode != NULL);
     assert(config->_install_importlib >= 0);
     assert(config->pathconfig_warnings >= 0);
+    assert(_PyWideStringList_CheckConsistency(&config->_orig_argv));
 
     status = _PyStatus_OK();
 
 done:
-    _PyWideStringList_Clear(&orig_argv);
     _PyPreCmdline_Clear(&precmdline);
     return status;
 }

Python/pylifecycle.c

@@ -460,7 +460,10 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime,
         return _PyStatus_ERR("can't make main interpreter");
     }
 
-    _PyConfig_Write(config, runtime);
+    status = _PyConfig_Write(config, runtime);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
 
     status = _PyInterpreterState_SetConfig(interp, config);
     if (_PyStatus_EXCEPTION(status)) {
@@ -486,7 +489,10 @@ pycore_init_runtime(_PyRuntimeState *runtime,
         return _PyStatus_ERR("main interpreter already initialized");
     }
 
-    _PyConfig_Write(config, runtime);
+    PyStatus status = _PyConfig_Write(config, runtime);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
 
     /* Py_Finalize leaves _Py_Finalizing set in order to help daemon
      * threads behave a little more gracefully at interpreter shutdown.
@@ -499,7 +505,7 @@ pycore_init_runtime(_PyRuntimeState *runtime,
      */
     _PyRuntimeState_SetFinalizing(runtime, NULL);
 
-    PyStatus status = _Py_HashRandomization_Init(config);
+    status = _Py_HashRandomization_Init(config);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
@@ -746,8 +752,6 @@ pyinit_config(_PyRuntimeState *runtime,
               PyThreadState **tstate_p,
               const PyConfig *config)
 {
-    _PyConfig_Write(config, runtime);
-
     PyStatus status = pycore_init_runtime(runtime, config);
     if (_PyStatus_EXCEPTION(status)) {
         return status;