bpo-38236: Dump path config at first import error (GH-16300) (GH-16332)

Python now dumps path configuration if it fails to import the Python
codecs of the filesystem and stdio encodings.

(cherry picked from commit fcdb027234)

Doc/c-api/init_config.rst

@@ -425,6 +425,11 @@ PyConfig
 
       :data:`sys.base_exec_prefix`.
 
+   .. c:member:: wchar_t* base_executable
+
+      :data:`sys._base_executable`: ``__PYVENV_LAUNCHER__`` environment
+      variable value, or copy of :c:member:`PyConfig.executable`.
+
    .. c:member:: wchar_t* base_prefix
 
       :data:`sys.base_prefix`.
@@ -862,11 +867,13 @@ Path Configuration
 * Path configuration input fields:
 
   * :c:member:`PyConfig.home`
-  * :c:member:`PyConfig.pythonpath_env`
   * :c:member:`PyConfig.pathconfig_warnings`
+  * :c:member:`PyConfig.program_name`
+  * :c:member:`PyConfig.pythonpath_env`
 
 * Path configuration output fields:
 
+  * :c:member:`PyConfig.base_executable`
   * :c:member:`PyConfig.exec_prefix`
   * :c:member:`PyConfig.executable`
   * :c:member:`PyConfig.prefix`
@@ -918,6 +925,9 @@ The following configuration files are used by the path configuration:
 * ``python._pth`` (Windows only)
 * ``pybuilddir.txt`` (Unix only)
 
+The ``__PYVENV_LAUNCHER__`` environment variable is used to set
+:c:member:`PyConfig.base_executable`
+
 
 Py_RunMain()
 ------------

Include/internal/pycore_pathconfig.h

@@ -60,6 +60,7 @@ extern wchar_t* _Py_GetDLLPath(void);
 #endif
 
 extern PyStatus _PyPathConfig_Init(void);
+extern void _Py_DumpPathConfig(PyThreadState *tstate);
 
 #ifdef __cplusplus
 }

Include/internal/pycore_pylifecycle.h

@@ -19,7 +19,7 @@ extern int _Py_SetFileSystemEncoding(
     const char *encoding,
     const char *errors);
 extern void _Py_ClearFileSystemEncoding(void);
-extern PyStatus _PyUnicode_InitEncodings(PyInterpreterState *interp);
+extern PyStatus _PyUnicode_InitEncodings(PyThreadState *tstate);
 #ifdef MS_WINDOWS
 extern int _PyUnicode_EnableLegacyWindowsFSEncoding(void);
 #endif

Misc/NEWS.d/next/Core and Builtins/2019-09-20-19-06-23.bpo-38236.eQ0Tmj.rst

@@ -0,0 +1,2 @@
+Python now dumps path configuration if it fails to import the Python codecs
+of the filesystem and stdio encodings.

Objects/unicodeobject.c

@@ -15727,13 +15727,13 @@ error:
 
 
 static PyStatus
-init_stdio_encoding(PyInterpreterState *interp)
+init_stdio_encoding(PyThreadState *tstate)
 {
     /* Update the stdio encoding to the normalized Python codec name. */
-    PyConfig *config = &interp->config;
+    PyConfig *config = &tstate->interp->config;
     if (config_get_codec_name(&config->stdio_encoding) < 0) {
         return _PyStatus_ERR("failed to get the Python codec name "
-                            "of the stdio encoding");
+                             "of the stdio encoding");
     }
     return _PyStatus_OK();
 }
@@ -15787,15 +15787,18 @@ init_fs_codec(PyInterpreterState *interp)
 
 
 static PyStatus
-init_fs_encoding(PyInterpreterState *interp)
+init_fs_encoding(PyThreadState *tstate)
 {
+    PyInterpreterState *interp = tstate->interp;
+
     /* Update the filesystem encoding to the normalized Python codec name.
        For example, replace "ANSI_X3.4-1968" (locale encoding) with "ascii"
        (Python codec name). */
     PyConfig *config = &interp->config;
     if (config_get_codec_name(&config->filesystem_encoding) < 0) {
+        _Py_DumpPathConfig(tstate);
         return _PyStatus_ERR("failed to get the Python codec "
-                            "of the filesystem encoding");
+                             "of the filesystem encoding");
     }
 
     if (init_fs_codec(interp) < 0) {
@@ -15806,14 +15809,14 @@ init_fs_encoding(PyInterpreterState *interp)
 
 
 PyStatus
-_PyUnicode_InitEncodings(PyInterpreterState *interp)
+_PyUnicode_InitEncodings(PyThreadState *tstate)
 {
-    PyStatus status = init_fs_encoding(interp);
+    PyStatus status = init_fs_encoding(tstate);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
-    return init_stdio_encoding(interp);
+    return init_stdio_encoding(tstate);
 }
 
 

PC/getpathp.c

@@ -532,8 +532,7 @@ _Py_GetDLLPath(void)
 
 
 static PyStatus
-get_program_full_path(const PyConfig *config,
-                      PyCalculatePath *calculate, _PyPathConfig *pathconfig)
+get_program_full_path(_PyPathConfig *pathconfig)
 {
     const wchar_t *pyvenv_launcher;
     wchar_t program_full_path[MAXPATHLEN+1];
@@ -977,7 +976,7 @@ calculate_path_impl(const PyConfig *config,
 {
     PyStatus status;
 
-    status = get_program_full_path(config, calculate, pathconfig);
+    status = get_program_full_path(pathconfig);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }

Python/initconfig.c

@@ -1,12 +1,13 @@
 #include "Python.h"
 #include "osdefs.h"       /* DELIM */
-#include "pycore_initconfig.h"
 #include "pycore_fileutils.h"
 #include "pycore_getopt.h"
+#include "pycore_initconfig.h"
+#include "pycore_pathconfig.h"
+#include "pycore_pyerrors.h"
 #include "pycore_pylifecycle.h"
 #include "pycore_pymem.h"
 #include "pycore_pystate.h"   /* _PyRuntime */
-#include "pycore_pathconfig.h"
 #include <locale.h>       /* setlocale() */
 #ifdef HAVE_LANGINFO_H
 #  include <langinfo.h>   /* nl_langinfo(CODESET) */
@@ -2492,3 +2493,98 @@ error:
     Py_XDECREF(dict);
     return NULL;
 }
+
+
+static void
+init_dump_ascii_wstr(const wchar_t *str)
+{
+    if (str == NULL) {
+        PySys_WriteStderr("(not set)");
+        return;
+    }
+
+    PySys_WriteStderr("'");
+    for (; *str != L'\0'; str++) {
+        wchar_t ch = *str;
+        if (ch == L'\'') {
+            PySys_WriteStderr("\\'");
+        } else if (0x20 <= ch && ch < 0x7f) {
+            PySys_WriteStderr("%lc", ch);
+        }
+        else if (ch <= 0xff) {
+            PySys_WriteStderr("\\x%02x", ch);
+        }
+#if SIZEOF_WCHAR_T > 2
+        else if (ch > 0xffff) {
+            PySys_WriteStderr("\\U%08x", ch);
+        }
+#endif
+        else {
+            PySys_WriteStderr("\\u%04x", ch);
+        }
+    }
+    PySys_WriteStderr("'");
+}
+
+
+/* Dump the Python path configuration into sys.stderr */
+void
+_Py_DumpPathConfig(PyThreadState *tstate)
+{
+    PyObject *exc_type, *exc_value, *exc_tb;
+    _PyErr_Fetch(tstate, &exc_type, &exc_value, &exc_tb);
+
+    PySys_WriteStderr("Python path configuration:\n");
+
+#define DUMP_CONFIG(NAME, FIELD) \
+        do { \
+            PySys_WriteStderr("  " NAME " = "); \
+            init_dump_ascii_wstr(config->FIELD); \
+            PySys_WriteStderr("\n"); \
+        } while (0)
+
+    PyConfig *config = &tstate->interp->config;
+    DUMP_CONFIG("PYTHONHOME", home);
+    DUMP_CONFIG("PYTHONPATH", pythonpath_env);
+    DUMP_CONFIG("program name", program_name);
+    PySys_WriteStderr("  isolated = %i\n", config->isolated);
+    PySys_WriteStderr("  environment = %i\n", config->use_environment);
+    PySys_WriteStderr("  user site = %i\n", config->user_site_directory);
+    PySys_WriteStderr("  import site = %i\n", config->site_import);
+#undef DUMP_CONFIG
+
+#define DUMP_SYS(NAME) \
+        do { \
+            obj = PySys_GetObject(#NAME); \
+            PySys_FormatStderr("  sys.%s = ", #NAME); \
+            if (obj != NULL) { \
+                PySys_FormatStderr("%A", obj); \
+            } \
+            else { \
+                PySys_WriteStderr("(not set)"); \
+            } \
+            PySys_FormatStderr("\n"); \
+        } while (0)
+
+    PyObject *obj;
+    DUMP_SYS(_base_executable);
+    DUMP_SYS(base_prefix);
+    DUMP_SYS(base_exec_prefix);
+    DUMP_SYS(executable);
+    DUMP_SYS(prefix);
+    DUMP_SYS(exec_prefix);
+#undef DUMP_SYS
+
+    PyObject *sys_path = PySys_GetObject("path");  /* borrowed reference */
+    if (sys_path != NULL && PyList_Check(sys_path)) {
+        PySys_WriteStderr("  sys.path = [\n");
+        Py_ssize_t len = PyList_GET_SIZE(sys_path);
+        for (Py_ssize_t i=0; i < len; i++) {
+            PyObject *path = PyList_GET_ITEM(sys_path, i);
+            PySys_FormatStderr("    %A,\n", path);
+        }
+        PySys_WriteStderr("  ]\n");
+    }
+
+    _PyErr_Restore(tstate, exc_type, exc_value, exc_tb);
+}

Python/pylifecycle.c

@@ -941,7 +941,8 @@ pyinit_main(_PyRuntimeState *runtime, PyInterpreterState *interp)
         return status;
     }
 
-    status = _PyUnicode_InitEncodings(interp);
+    PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
+    status = _PyUnicode_InitEncodings(tstate);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
@@ -1508,7 +1509,7 @@ new_interpreter(PyThreadState **tstate_p)
             return status;
         }
 
-        status = _PyUnicode_InitEncodings(interp);
+        status = _PyUnicode_InitEncodings(tstate);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }