bpo-45582: Port getpath[p].c to Python (GH-29041)

The getpath.py file is frozen at build time and executed as code over a namespace. It is never imported, nor is it meant to be importable or reusable. However, it should be easier to read, modify, and patch than the previous code.

This commit attempts to preserve every previously tested quirk, but these may be changed in the future to better align platforms.

.gitignore

@@ -74,6 +74,7 @@ Mac/pythonw
 Misc/python.pc
 Misc/python-embed.pc
 Misc/python-config.sh
+Modules/getpath.h
 Modules/Setup.config
 Modules/Setup.local
 Modules/Setup.stdlib

Doc/c-api/init_config.rst

@@ -479,6 +479,9 @@ PyConfig
 
       Fields which are already initialized are left unchanged.
 
+      Fields for :ref:`path configuration <init-path-config>` are no longer
+      calculated or modified when calling this function, as of Python 3.11.
+
       The :c:func:`PyConfig_Read` function only parses
       :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv`
       is set to ``2`` after arguments are parsed. Since Python arguments are
@@ -493,6 +496,12 @@ PyConfig
          parsed, and arguments are only parsed if
          :c:member:`PyConfig.parse_argv` equals ``1``.
 
+      .. versionchanged:: 3.11
+         :c:func:`PyConfig_Read` no longer calculates all paths, and so fields
+         listed under :ref:`Python Path Configuration <init-path-config>` may
+         no longer be updated until :c:func:`Py_InitializeFromConfig` is
+         called.
+
    .. c:function:: void PyConfig_Clear(PyConfig *config)
 
       Release configuration memory.
@@ -848,12 +857,19 @@ PyConfig
 
       Default: value of the ``PLATLIBDIR`` macro which is set by the
       :option:`configure --with-platlibdir option <--with-platlibdir>`
-      (default: ``"lib"``).
+      (default: ``"lib"``, or ``"DLLs"`` on Windows).
 
       Part of the :ref:`Python Path Configuration <init-path-config>` input.
 
       .. versionadded:: 3.9
 
+      .. versionchanged:: 3.11
+         This macro is now used on Windows to locate the standard
+         library extension modules, typically under ``DLLs``. However,
+         for compatibility, note that this value is ignored for any
+         non-standard layouts, including in-tree builds and virtual
+         environments.
+
    .. c:member:: wchar_t* pythonpath_env
 
       Module search paths (:data:`sys.path`) as a string separated by ``DELIM``
@@ -870,9 +886,9 @@ PyConfig
 
       Module search paths: :data:`sys.path`.
 
-      If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, the
+      If :c:member:`~PyConfig.module_search_paths_set` is equal to 0,
-      function calculating the :ref:`Python Path Configuration <init-path-config>`
+      :c:func:`Py_InitializeFromConfig` will replace
-      overrides the :c:member:`~PyConfig.module_search_paths` and sets
+      :c:member:`~PyConfig.module_search_paths` and sets
       :c:member:`~PyConfig.module_search_paths_set` to ``1``.
 
       Default: empty list (``module_search_paths``) and ``0``
@@ -944,16 +960,16 @@ PyConfig
 
    .. c:member:: int pathconfig_warnings
 
-      On Unix, if non-zero, calculating the :ref:`Python Path Configuration
+      If non-zero, calculation of path configuration is allowed to log
-      <init-path-config>` can log warnings into ``stderr``. If equals to 0,
+      warnings into ``stderr``. If equals to 0, suppress these warnings.
-      suppress these warnings.
-
-      It has no effect on Windows.
 
       Default: ``1`` in Python mode, ``0`` in isolated mode.
 
       Part of the :ref:`Python Path Configuration <init-path-config>` input.
 
+      .. versionchanged:: 3.11
+         Now also applies on Windows.
+
    .. c:member:: wchar_t* prefix
 
       The site-specific directory prefix where the platform independent Python
@@ -1305,10 +1321,9 @@ variables, command line arguments (:c:member:`PyConfig.argv` is not parsed)
 and user site directory. The C standard streams (ex: ``stdout``) and the
 LC_CTYPE locale are left unchanged. Signal handlers are not installed.
 
-Configuration files are still used with this configuration. Set the
+Configuration files are still used with this configuration to determine
-:ref:`Python Path Configuration <init-path-config>` ("output fields") to ignore these
+paths that are unspecified. Ensure :c:member:`PyConfig.home` is specified
-configuration files and avoid the function computing the default path
+to avoid computing the default path configuration.
-configuration.
 
 
 .. _init-python-config:

Include/cpython/fileutils.h

@@ -135,12 +135,6 @@ PyAPI_FUNC(wchar_t*) _Py_wrealpath(
     size_t resolved_path_len);
 #endif
 
-#ifndef MS_WINDOWS
-PyAPI_FUNC(int) _Py_isabs(const wchar_t *path);
-#endif
-
-PyAPI_FUNC(int) _Py_abspath(const wchar_t *path, wchar_t **abspath_p);
-
 PyAPI_FUNC(wchar_t*) _Py_wgetcwd(
     wchar_t *buf,
     /* Number of characters of 'buf' buffer

Include/cpython/initconfig.h

@@ -213,6 +213,9 @@ typedef struct PyConfig {
     // If non-zero, disallow threads, subprocesses, and fork.
     // Default: 0.
     int _isolated_interpreter;
+
+    // If non-zero, we believe we're running from a source tree.
+    int _is_python_build;
 } PyConfig;
 
 PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config);

Include/internal/pycore_fileutils.h

@@ -74,14 +74,15 @@ extern int _Py_EncodeNonUnicodeWchar_InPlace(
     Py_ssize_t size);
 #endif
 
+extern int _Py_isabs(const wchar_t *path);
+extern int _Py_abspath(const wchar_t *path, wchar_t **abspath_p);
 extern wchar_t * _Py_join_relfile(const wchar_t *dirname,
                                   const wchar_t *relfile);
 extern int _Py_add_relfile(wchar_t *dirname,
                            const wchar_t *relfile,
                            size_t bufsize);
 extern size_t _Py_find_basename(const wchar_t *filename);
-PyAPI_FUNC(int) _Py_normalize_path(const wchar_t *path,
+PyAPI_FUNC(wchar_t *) _Py_normpath(wchar_t *path, Py_ssize_t size);
-                                   wchar_t *buf, const size_t buf_len);
 
 
 // Macros to protect CRT calls against instant termination when passed an

Include/internal/pycore_initconfig.h

@@ -166,6 +166,10 @@ extern PyStatus _PyConfig_SetPyArgv(
 PyAPI_FUNC(PyObject*) _PyConfig_AsDict(const PyConfig *config);
 PyAPI_FUNC(int) _PyConfig_FromDict(PyConfig *config, PyObject *dict);
 
+extern void _Py_DumpPathConfig(PyThreadState *tstate);
+
+PyAPI_FUNC(PyObject*) _Py_Get_Getpath_CodeObject();
+
 
 /* --- Function used for testing ---------------------------------- */
 

Include/internal/pycore_pathconfig.h

@@ -8,65 +8,15 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-typedef struct _PyPathConfig {
+PyAPI_FUNC(void) _PyPathConfig_ClearGlobal(void);
-    /* Full path to the Python program */
+extern PyStatus _PyPathConfig_ReadGlobal(PyConfig *config);
-    wchar_t *program_full_path;
+extern PyStatus _PyPathConfig_UpdateGlobal(const PyConfig *config);
-    wchar_t *prefix;
+extern const wchar_t * _PyPathConfig_GetGlobalModuleSearchPath(void);
-    wchar_t *exec_prefix;
-    wchar_t *stdlib_dir;
-    /* Set by Py_SetPath(), or computed by _PyConfig_InitPathConfig() */
-    wchar_t *module_search_path;
-    /* Python program name */
-    wchar_t *program_name;
-    /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */
-    wchar_t *home;
-#ifdef MS_WINDOWS
-    /* isolated and site_import are used to set Py_IsolatedFlag and
-       Py_NoSiteFlag flags on Windows in read_pth_file(). These fields
-       are ignored when their value are equal to -1 (unset). */
-    int isolated;
-    int site_import;
-    /* Set when a venv is detected */
-    wchar_t *base_executable;
-#endif
-} _PyPathConfig;
 
-#ifdef MS_WINDOWS
-#  define _PyPathConfig_INIT \
-      {.module_search_path = NULL, \
-       .isolated = -1, \
-       .site_import = -1}
-#else
-#  define _PyPathConfig_INIT \
-      {.module_search_path = NULL}
-#endif
-/* Note: _PyPathConfig_INIT sets other fields to 0/NULL */
-
-PyAPI_DATA(_PyPathConfig) _Py_path_config;
-#ifdef MS_WINDOWS
-PyAPI_DATA(wchar_t*) _Py_dll_path;
-#endif
-
-extern void _PyPathConfig_ClearGlobal(void);
-
-extern PyStatus _PyPathConfig_Calculate(
-    _PyPathConfig *pathconfig,
-    const PyConfig *config);
 extern int _PyPathConfig_ComputeSysPath0(
     const PyWideStringList *argv,
     PyObject **path0);
-extern PyStatus _Py_FindEnvConfigValue(
-    FILE *env_file,
-    const wchar_t *key,
-    wchar_t **value_p);
 
-#ifdef MS_WINDOWS
-extern wchar_t* _Py_GetDLLPath(void);
-#endif
-
-extern PyStatus _PyConfig_WritePathConfig(const PyConfig *config);
-extern void _Py_DumpPathConfig(PyThreadState *tstate);
-extern PyObject* _PyPathConfig_AsDict(void);
 
 #ifdef __cplusplus
 }

Lib/ntpath.py

@@ -70,7 +70,7 @@ def isabs(s):
         if s.replace('/', '\\').startswith('\\\\?\\'):
             return True
     s = splitdrive(s)[1]
-    return len(s) > 0 and s[0] in _get_bothseps(s)
+    return len(s) > 0 and s[0] and s[0] in _get_bothseps(s)
 
 
 # Join two (or more) paths.
@@ -268,11 +268,13 @@ def ismount(path):
     root, rest = splitdrive(path)
     if root and root[0] in seps:
         return (not rest) or (rest in seps)
-    if rest in seps:
+    if rest and rest in seps:
         return True
 
     if _getvolumepathname:
-        return path.rstrip(seps) == _getvolumepathname(path).rstrip(seps)
+        x = path.rstrip(seps)
+        y =_getvolumepathname(path).rstrip(seps)
+        return x.casefold() == y.casefold()
     else:
         return False
 
@@ -459,56 +461,68 @@ def expandvars(path):
 # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B.
 # Previously, this function also truncated pathnames to 8+3 format,
 # but as this module is called "ntpath", that's obviously wrong!
+try:
+    from nt import _path_normpath
 
-def normpath(path):
+except ImportError:
-    """Normalize path, eliminating double slashes, etc."""
+    def normpath(path):
-    path = os.fspath(path)
+        """Normalize path, eliminating double slashes, etc."""
-    if isinstance(path, bytes):
+        path = os.fspath(path)
-        sep = b'\\'
+        if isinstance(path, bytes):
-        altsep = b'/'
+            sep = b'\\'
-        curdir = b'.'
+            altsep = b'/'
-        pardir = b'..'
+            curdir = b'.'
-        special_prefixes = (b'\\\\.\\', b'\\\\?\\')
+            pardir = b'..'
-    else:
+            special_prefixes = (b'\\\\.\\', b'\\\\?\\')
-        sep = '\\'
+        else:
-        altsep = '/'
+            sep = '\\'
-        curdir = '.'
+            altsep = '/'
-        pardir = '..'
+            curdir = '.'
-        special_prefixes = ('\\\\.\\', '\\\\?\\')
+            pardir = '..'
-    if path.startswith(special_prefixes):
+            special_prefixes = ('\\\\.\\', '\\\\?\\')
-        # in the case of paths with these prefixes:
+        if path.startswith(special_prefixes):
-        # \\.\ -> device names
+            # in the case of paths with these prefixes:
-        # \\?\ -> literal paths
+            # \\.\ -> device names
-        # do not do any normalization, but return the path
+            # \\?\ -> literal paths
-        # unchanged apart from the call to os.fspath()
+            # do not do any normalization, but return the path
-        return path
+            # unchanged apart from the call to os.fspath()
-    path = path.replace(altsep, sep)
+            return path
-    prefix, path = splitdrive(path)
+        path = path.replace(altsep, sep)
+        prefix, path = splitdrive(path)
 
-    # collapse initial backslashes
+        # collapse initial backslashes
-    if path.startswith(sep):
+        if path.startswith(sep):
-        prefix += sep
+            prefix += sep
-        path = path.lstrip(sep)
+            path = path.lstrip(sep)
 
-    comps = path.split(sep)
+        comps = path.split(sep)
-    i = 0
+        i = 0
-    while i < len(comps):
+        while i < len(comps):
-        if not comps[i] or comps[i] == curdir:
+            if not comps[i] or comps[i] == curdir:
-            del comps[i]
-        elif comps[i] == pardir:
-            if i > 0 and comps[i-1] != pardir:
-                del comps[i-1:i+1]
-                i -= 1
-            elif i == 0 and prefix.endswith(sep):
                 del comps[i]
+            elif comps[i] == pardir:
+                if i > 0 and comps[i-1] != pardir:
+                    del comps[i-1:i+1]
+                    i -= 1
+                elif i == 0 and prefix.endswith(sep):
+                    del comps[i]
+                else:
+                    i += 1
             else:
                 i += 1
-        else:
+        # If the path is now empty, substitute '.'
-            i += 1
+        if not prefix and not comps:
-    # If the path is now empty, substitute '.'
+            comps.append(curdir)
-    if not prefix and not comps:
+        return prefix + sep.join(comps)
-        comps.append(curdir)
+
-    return prefix + sep.join(comps)
+else:
+    def normpath(path):
+        """Normalize path, eliminating double slashes, etc."""
+        path = os.fspath(path)
+        if isinstance(path, bytes):
+            return os.fsencode(_path_normpath(os.fsdecode(path))) or b"."
+        return _path_normpath(path) or "."
+
 
 def _abspath_fallback(path):
     """Return the absolute version of a path as a fallback function in case

Lib/posixpath.py

@@ -334,43 +334,55 @@ def expandvars(path):
 # It should be understood that this may change the meaning of the path
 # if it contains symbolic links!
 
-def normpath(path):
+try:
-    """Normalize path, eliminating double slashes, etc."""
+    from posix import _path_normpath
-    path = os.fspath(path)
+
-    if isinstance(path, bytes):
+except ImportError:
-        sep = b'/'
+    def normpath(path):
-        empty = b''
+        """Normalize path, eliminating double slashes, etc."""
-        dot = b'.'
+        path = os.fspath(path)
-        dotdot = b'..'
+        if isinstance(path, bytes):
-    else:
+            sep = b'/'
-        sep = '/'
+            empty = b''
-        empty = ''
+            dot = b'.'
-        dot = '.'
+            dotdot = b'..'
-        dotdot = '..'
+        else:
-    if path == empty:
+            sep = '/'
-        return dot
+            empty = ''
-    initial_slashes = path.startswith(sep)
+            dot = '.'
-    # POSIX allows one or two initial slashes, but treats three or more
+            dotdot = '..'
-    # as single slash.
+        if path == empty:
-    # (see http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13)
+            return dot
-    if (initial_slashes and
+        initial_slashes = path.startswith(sep)
-        path.startswith(sep*2) and not path.startswith(sep*3)):
+        # POSIX allows one or two initial slashes, but treats three or more
-        initial_slashes = 2
+        # as single slash.
-    comps = path.split(sep)
+        # (see http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13)
-    new_comps = []
+        if (initial_slashes and
-    for comp in comps:
+            path.startswith(sep*2) and not path.startswith(sep*3)):
-        if comp in (empty, dot):
+            initial_slashes = 2
-            continue
+        comps = path.split(sep)
-        if (comp != dotdot or (not initial_slashes and not new_comps) or
+        new_comps = []
-             (new_comps and new_comps[-1] == dotdot)):
+        for comp in comps:
-            new_comps.append(comp)
+            if comp in (empty, dot):
-        elif new_comps:
+                continue
-            new_comps.pop()
+            if (comp != dotdot or (not initial_slashes and not new_comps) or
-    comps = new_comps
+                 (new_comps and new_comps[-1] == dotdot)):
-    path = sep.join(comps)
+                new_comps.append(comp)
-    if initial_slashes:
+            elif new_comps:
-        path = sep*initial_slashes + path
+                new_comps.pop()
-    return path or dot
+        comps = new_comps
+        path = sep.join(comps)
+        if initial_slashes:
+            path = sep*initial_slashes + path
+        return path or dot
+
+else:
+    def normpath(path):
+        """Normalize path, eliminating double slashes, etc."""
+        path = os.fspath(path)
+        if isinstance(path, bytes):
+            return os.fsencode(_path_normpath(os.fsdecode(path))) or b"."
+        return _path_normpath(path) or "."
 
 
 def abspath(path):

Lib/site.py

@@ -361,11 +361,11 @@ def getsitepackages(prefixes=None):
             continue
         seen.add(prefix)
 
-        libdirs = [sys.platlibdir]
-        if sys.platlibdir != "lib":
-            libdirs.append("lib")
-
         if os.sep == '/':
+            libdirs = [sys.platlibdir]
+            if sys.platlibdir != "lib":
+                libdirs.append("lib")
+
             for libdir in libdirs:
                 path = os.path.join(prefix, libdir,
                                     "python%d.%d" % sys.version_info[:2],
@@ -373,10 +373,7 @@ def getsitepackages(prefixes=None):
                 sitepackages.append(path)
         else:
             sitepackages.append(prefix)
-
+            sitepackages.append(os.path.join(prefix, "Lib", "site-packages"))
-            for libdir in libdirs:
-                path = os.path.join(prefix, libdir, "site-packages")
-                sitepackages.append(path)
     return sitepackages
 
 def addsitepackages(known_paths, prefixes=None):

Lib/sysconfig.py

@@ -216,6 +216,11 @@ def _expand_vars(scheme, vars):
     if vars is None:
         vars = {}
     _extend_dict(vars, get_config_vars())
+    if os.name == 'nt':
+        # On Windows we want to substitute 'lib' for schemes rather
+        # than the native value (without modifying vars, in case it
+        # was passed in)
+        vars = vars | {'platlibdir': 'lib'}
 
     for key, value in _INSTALL_SCHEMES[scheme].items():
         if os.name in ('posix', 'nt'):

Lib/test/_test_embed_set_config.py

@@ -20,6 +20,7 @@ class SetConfigTests(unittest.TestCase):
         self.sys_copy = dict(sys.__dict__)
 
     def tearDown(self):
+        _testinternalcapi.reset_path_config()
         _testinternalcapi.set_config(self.old_config)
         sys.__dict__.clear()
         sys.__dict__.update(self.sys_copy)

Lib/test/test_embed.py

@@ -13,6 +13,7 @@ import re
 import shutil
 import subprocess
 import sys
+import sysconfig
 import tempfile
 import textwrap
 
@@ -445,6 +446,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         '_init_main': 1,
         '_isolated_interpreter': 0,
         'use_frozen_modules': 1,
+        '_is_python_build': IGNORE_CONFIG,
     }
     if MS_WINDOWS:
         CONFIG_COMPAT.update({
@@ -508,32 +510,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             ('Py_LegacyWindowsStdioFlag', 'legacy_windows_stdio'),
         ))
 
-    # path config
-    if MS_WINDOWS:
-        PATH_CONFIG = {
-            'isolated': -1,
-            'site_import': -1,
-            'python3_dll': GET_DEFAULT_CONFIG,
-        }
-    else:
-        PATH_CONFIG = {}
-    # other keys are copied by COPY_PATH_CONFIG
-
-    COPY_PATH_CONFIG = [
-        # Copy core config to global config for expected values
-        'prefix',
-        'exec_prefix',
-        'program_name',
-        'home',
-        'stdlib_dir',
-        # program_full_path and module_search_path are copied indirectly from
-        # the core configuration in check_path_config().
-    ]
-    if MS_WINDOWS:
-        COPY_PATH_CONFIG.extend((
-            'base_executable',
-        ))
-
     EXPECTED_CONFIG = None
 
     @classmethod
@@ -599,8 +575,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         return configs
 
     def get_expected_config(self, expected_preconfig, expected,
-                            expected_pathconfig, env, api,
+                            env, api, modify_path_cb=None):
-                            modify_path_cb=None):
         configs = self._get_expected_config()
 
         pre_config = configs['pre_config']
@@ -608,11 +583,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             if value is self.GET_DEFAULT_CONFIG:
                 expected_preconfig[key] = pre_config[key]
 
-        path_config = configs['path_config']
-        for key, value in expected_pathconfig.items():
-            if value is self.GET_DEFAULT_CONFIG:
-                expected_pathconfig[key] = path_config[key]
-
         if not expected_preconfig['configure_locale'] or api == API_COMPAT:
             # there is no easy way to get the locale encoding before
             # setlocale(LC_CTYPE, "") is called: don't test encodings
@@ -705,18 +675,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
 
         self.assertEqual(configs['global_config'], expected)
 
-    def check_path_config(self, configs, expected):
-        config = configs['config']
-
-        for key in self.COPY_PATH_CONFIG:
-            expected[key] = config[key]
-        expected['module_search_path'] = os.path.pathsep.join(config['module_search_paths'])
-        expected['program_full_path'] = config['executable']
-
-        self.assertEqual(configs['path_config'], expected)
-
     def check_all_configs(self, testname, expected_config=None,
-                          expected_preconfig=None, expected_pathconfig=None,
+                          expected_preconfig=None,
                           modify_path_cb=None,
                           stderr=None, *, api, preconfig_api=None,
                           env=None, ignore_stderr=False, cwd=None):
@@ -740,10 +700,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         if expected_config is None:
             expected_config = {}
 
-        if expected_pathconfig is None:
-            expected_pathconfig = {}
-        expected_pathconfig = dict(self.PATH_CONFIG, **expected_pathconfig)
-
         if api == API_PYTHON:
             default_config = self.CONFIG_PYTHON
         elif api == API_ISOLATED:
@@ -754,7 +710,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
 
         self.get_expected_config(expected_preconfig,
                                  expected_config,
-                                 expected_pathconfig,
                                  env,
                                  api, modify_path_cb)
 
@@ -772,7 +727,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         self.check_pre_config(configs, expected_preconfig)
         self.check_config(configs, expected_config)
         self.check_global_config(configs)
-        self.check_path_config(configs, expected_pathconfig)
         return configs
 
     def test_init_default_config(self):
@@ -1039,10 +993,13 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         self.check_all_configs("test_init_dont_configure_locale", {}, preconfig,
                                api=API_PYTHON)
 
+    @unittest.skip('as of 3.11 this test no longer works because '
+                   'path calculations do not occur on read')
     def test_init_read_set(self):
         config = {
             'program_name': './init_read_set',
             'executable': 'my_executable',
+            'base_executable': 'my_executable',
         }
         def modify_path(path):
             path.insert(1, "test_path_insert1")
@@ -1156,7 +1113,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
              # The current getpath.c doesn't determine the stdlib dir
              # in this case.
             'stdlib_dir': '',
-            'use_frozen_modules': -1,
         }
         self.default_program_name(config)
         env = {'TESTPATH': os.path.pathsep.join(paths)}
@@ -1180,7 +1136,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
              # The current getpath.c doesn't determine the stdlib dir
              # in this case.
             'stdlib_dir': '',
-            'use_frozen_modules': -1,
+            'use_frozen_modules': 1,
             # overridden by PyConfig
             'program_name': 'conf_program_name',
             'base_executable': 'conf_executable',
@@ -1210,13 +1166,16 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             ]
 
     @contextlib.contextmanager
-    def tmpdir_with_python(self):
+    def tmpdir_with_python(self, subdir=None):
         # Temporary directory with a copy of the Python program
         with tempfile.TemporaryDirectory() as tmpdir:
             # bpo-38234: On macOS and FreeBSD, the temporary directory
             # can be symbolic link. For example, /tmp can be a symbolic link
             # to /var/tmp. Call realpath() to resolve all symbolic links.
             tmpdir = os.path.realpath(tmpdir)
+            if subdir:
+                tmpdir = os.path.normpath(os.path.join(tmpdir, subdir))
+                os.makedirs(tmpdir)
 
             if MS_WINDOWS:
                 # Copy pythonXY.dll (or pythonXY_d.dll)
@@ -1260,11 +1219,14 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
 
         prefix = exec_prefix = home
         if MS_WINDOWS:
-            stdlib = os.path.join(home, sys.platlibdir)
+            stdlib = os.path.join(home, "Lib")
+            # Because we are specifying 'home', module search paths
+            # are fairly static
+            expected_paths = [paths[0], stdlib, os.path.join(home, 'DLLs')]
         else:
             version = f'{sys.version_info.major}.{sys.version_info.minor}'
             stdlib = os.path.join(home, sys.platlibdir, f'python{version}')
-        expected_paths = self.module_search_paths(prefix=home, exec_prefix=home)
+            expected_paths = self.module_search_paths(prefix=home, exec_prefix=home)
 
         config = {
             'home': home,
@@ -1291,7 +1253,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         env = {'PYTHONPATH': paths_str}
         return env
 
-    @unittest.skipIf(MS_WINDOWS, 'Windows does not use pybuilddir.txt')
+    @unittest.skipIf(MS_WINDOWS, 'See test_init_pybuilddir_win32')
     def test_init_pybuilddir(self):
         # Test path configuration with pybuilddir.txt configuration file
 
@@ -1300,6 +1262,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             # directory (tmpdir)
             subdir = 'libdir'
             libdir = os.path.join(tmpdir, subdir)
+            # The stdlib dir is dirname(executable) + VPATH + 'Lib'
+            stdlibdir = os.path.join(tmpdir, 'Lib')
             os.mkdir(libdir)
 
             filename = os.path.join(tmpdir, 'pybuilddir.txt')
@@ -1307,21 +1271,58 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                 fp.write(subdir)
 
             module_search_paths = self.module_search_paths()
+            module_search_paths[-2] = stdlibdir
             module_search_paths[-1] = libdir
 
             executable = self.test_exe
             config = {
+                'base_exec_prefix': sysconfig.get_config_var("exec_prefix"),
+                'base_prefix': sysconfig.get_config_var("prefix"),
                 'base_executable': executable,
                 'executable': executable,
                 'module_search_paths': module_search_paths,
-                'stdlib_dir': STDLIB_INSTALL,
+                'stdlib_dir': stdlibdir,
-                'use_frozen_modules': 1 if STDLIB_INSTALL else -1,
             }
             env = self.copy_paths_by_env(config)
             self.check_all_configs("test_init_compat_config", config,
                                    api=API_COMPAT, env=env,
                                    ignore_stderr=True, cwd=tmpdir)
 
+    @unittest.skipUnless(MS_WINDOWS, 'See test_init_pybuilddir')
+    def test_init_pybuilddir_win32(self):
+        # Test path configuration with pybuilddir.txt configuration file
+
+        with self.tmpdir_with_python(r'PCbuild\arch') as tmpdir:
+            # The prefix is dirname(executable) + VPATH
+            prefix = os.path.normpath(os.path.join(tmpdir, r'..\..'))
+            # The stdlib dir is dirname(executable) + VPATH + 'Lib'
+            stdlibdir = os.path.normpath(os.path.join(tmpdir, r'..\..\Lib'))
+
+            filename = os.path.join(tmpdir, 'pybuilddir.txt')
+            with open(filename, "w", encoding="utf8") as fp:
+                fp.write(tmpdir)
+
+            module_search_paths = self.module_search_paths()
+            module_search_paths[-3] = os.path.join(tmpdir, os.path.basename(module_search_paths[-3]))
+            module_search_paths[-2] = stdlibdir
+            module_search_paths[-1] = tmpdir
+
+            executable = self.test_exe
+            config = {
+                'base_exec_prefix': prefix,
+                'base_prefix': prefix,
+                'base_executable': executable,
+                'executable': executable,
+                'prefix': prefix,
+                'exec_prefix': prefix,
+                'module_search_paths': module_search_paths,
+                'stdlib_dir': stdlibdir,
+            }
+            env = self.copy_paths_by_env(config)
+            self.check_all_configs("test_init_compat_config", config,
+                                   api=API_COMPAT, env=env,
+                                   ignore_stderr=False, cwd=tmpdir)
+
     def test_init_pyvenv_cfg(self):
         # Test path configuration with pyvenv.cfg configuration file
 
@@ -1336,12 +1337,12 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                                            'lib-dynload')
                 os.makedirs(lib_dynload)
             else:
-                lib_dynload = os.path.join(pyvenv_home, 'lib')
+                lib_folder = os.path.join(pyvenv_home, 'Lib')
-                os.makedirs(lib_dynload)
+                os.makedirs(lib_folder)
-                # getpathp.c uses Lib\os.py as the LANDMARK
+                # getpath.py uses Lib\os.py as the LANDMARK
                 shutil.copyfile(
                     os.path.join(support.STDLIB_DIR, 'os.py'),
-                    os.path.join(lib_dynload, 'os.py'),
+                    os.path.join(lib_folder, 'os.py'),
                 )
 
             filename = os.path.join(tmpdir, 'pyvenv.cfg')
@@ -1355,43 +1356,38 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             else:
                 for index, path in enumerate(paths):
                     if index == 0:
+                        # Because we copy the DLLs into tmpdir as well, the zip file
+                        # entry in sys.path will be there. For a regular venv, it will
+                        # usually be in the home directory.
                         paths[index] = os.path.join(tmpdir, os.path.basename(path))
                     else:
                         paths[index] = os.path.join(pyvenv_home, os.path.basename(path))
                 paths[-1] = pyvenv_home
 
             executable = self.test_exe
+            base_executable = os.path.join(pyvenv_home, os.path.basename(executable))
             exec_prefix = pyvenv_home
             config = {
+                'base_prefix': sysconfig.get_config_var("prefix"),
                 'base_exec_prefix': exec_prefix,
                 'exec_prefix': exec_prefix,
-                'base_executable': executable,
+                'base_executable': base_executable,
                 'executable': executable,
                 'module_search_paths': paths,
             }
-            path_config = {}
             if MS_WINDOWS:
                 config['base_prefix'] = pyvenv_home
                 config['prefix'] = pyvenv_home
-                config['stdlib_dir'] = os.path.join(pyvenv_home, 'lib')
+                config['stdlib_dir'] = os.path.join(pyvenv_home, 'Lib')
                 config['use_frozen_modules'] = 1
-
-                ver = sys.version_info
-                dll = f'python{ver.major}'
-                if debug_build(executable):
-                    dll += '_d'
-                dll += '.DLL'
-                dll = os.path.join(os.path.dirname(executable), dll)
-                path_config['python3_dll'] = dll
             else:
-                # The current getpath.c doesn't determine the stdlib dir
+                # cannot reliably assume stdlib_dir here because it
-                # in this case.
+                # depends too much on our build. But it ought to be found
-                config['stdlib_dir'] = STDLIB_INSTALL
+                config['stdlib_dir'] = self.IGNORE_CONFIG
-                config['use_frozen_modules'] = 1 if STDLIB_INSTALL else -1
+                config['use_frozen_modules'] = 1
 
             env = self.copy_paths_by_env(config)
             self.check_all_configs("test_init_compat_config", config,
-                                   expected_pathconfig=path_config,
                                    api=API_COMPAT, env=env,
                                    ignore_stderr=True, cwd=tmpdir)
 
@@ -1502,10 +1498,14 @@ class SetConfigTests(unittest.TestCase):
     def test_set_config(self):
         # bpo-42260: Test _PyInterpreterState_SetConfig()
         import_helper.import_module('_testcapi')
-        cmd = [sys.executable, '-I', '-m', 'test._test_embed_set_config']
+        cmd = [sys.executable, '-X', 'utf8', '-I', '-m', 'test._test_embed_set_config']
         proc = subprocess.run(cmd,
                               stdout=subprocess.PIPE,
-                              stderr=subprocess.PIPE)
+                              stderr=subprocess.PIPE,
+                              encoding='utf-8', errors='backslashreplace')
+        if proc.returncode and support.verbose:
+            print(proc.stdout)
+            print(proc.stderr)
         self.assertEqual(proc.returncode, 0,
                          (proc.returncode, proc.stdout, proc.stderr))
 

Lib/test/test_getpath.py

@@ -0,0 +1,879 @@
+import copy
+import ntpath
+import pathlib
+import posixpath
+import sys
+import unittest
+
+from test.support import verbose
+
+try:
+    # If we are in a source tree, use the original source file for tests
+    SOURCE = (pathlib.Path(__file__).absolute().parent.parent.parent / "Modules/getpath.py").read_bytes()
+except FileNotFoundError:
+    # Try from _testcapimodule instead
+    from _testinternalcapi import get_getpath_codeobject
+    SOURCE = get_getpath_codeobject()
+
+
+class MockGetPathTests(unittest.TestCase):
+    def __init__(self, *a, **kw):
+        super().__init__(*a, **kw)
+        self.maxDiff = None
+
+    def test_normal_win32(self):
+        "Test a 'standard' install layout on Windows."
+        ns = MockNTNamespace(
+            argv0=r"C:\Python\python.exe",
+            real_executable=r"C:\Python\python.exe",
+        )
+        ns.add_known_xfile(r"C:\Python\python.exe")
+        ns.add_known_file(r"C:\Python\Lib\os.py")
+        ns.add_known_dir(r"C:\Python\DLLs")
+        expected = dict(
+            executable=r"C:\Python\python.exe",
+            base_executable=r"C:\Python\python.exe",
+            prefix=r"C:\Python",
+            exec_prefix=r"C:\Python",
+            module_search_paths_set=1,
+            module_search_paths=[
+                r"C:\Python\python98.zip",
+                r"C:\Python\Lib",
+                r"C:\Python\DLLs",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_buildtree_win32(self):
+        "Test an in-build-tree layout on Windows."
+        ns = MockNTNamespace(
+            argv0=r"C:\CPython\PCbuild\amd64\python.exe",
+            real_executable=r"C:\CPython\PCbuild\amd64\python.exe",
+        )
+        ns.add_known_xfile(r"C:\CPython\PCbuild\amd64\python.exe")
+        ns.add_known_file(r"C:\CPython\Lib\os.py")
+        ns.add_known_file(r"C:\CPython\PCbuild\amd64\pybuilddir.txt", [""])
+        expected = dict(
+            executable=r"C:\CPython\PCbuild\amd64\python.exe",
+            base_executable=r"C:\CPython\PCbuild\amd64\python.exe",
+            prefix=r"C:\CPython",
+            exec_prefix=r"C:\CPython",
+            build_prefix=r"C:\CPython",
+            _is_python_build=1,
+            module_search_paths_set=1,
+            module_search_paths=[
+                r"C:\CPython\PCbuild\amd64\python98.zip",
+                r"C:\CPython\Lib",
+                r"C:\CPython\PCbuild\amd64",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_venv_win32(self):
+        """Test a venv layout on Windows.
+
+        This layout is discovered by the presence of %__PYVENV_LAUNCHER__%,
+        specifying the original launcher executable. site.py is responsible
+        for updating prefix and exec_prefix.
+        """
+        ns = MockNTNamespace(
+            argv0=r"C:\Python\python.exe",
+            ENV___PYVENV_LAUNCHER__=r"C:\venv\Scripts\python.exe",
+            real_executable=r"C:\Python\python.exe",
+        )
+        ns.add_known_xfile(r"C:\Python\python.exe")
+        ns.add_known_xfile(r"C:\venv\Scripts\python.exe")
+        ns.add_known_file(r"C:\Python\Lib\os.py")
+        ns.add_known_dir(r"C:\Python\DLLs")
+        ns.add_known_file(r"C:\venv\pyvenv.cfg", [
+            r"home = C:\Python"
+        ])
+        expected = dict(
+            executable=r"C:\venv\Scripts\python.exe",
+            prefix=r"C:\Python",
+            exec_prefix=r"C:\Python",
+            base_executable=r"C:\Python\python.exe",
+            base_prefix=r"C:\Python",
+            base_exec_prefix=r"C:\Python",
+            module_search_paths_set=1,
+            module_search_paths=[
+                r"C:\Python\python98.zip",
+                r"C:\Python\Lib",
+                r"C:\Python",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_registry_win32(self):
+        """Test registry lookup on Windows.
+
+        On Windows there are registry entries that are intended for other
+        applications to register search paths.
+        """
+        hkey = rf"HKLM\Software\Python\PythonCore\9.8-XY\PythonPath"
+        winreg = MockWinreg({
+            hkey: None,
+            f"{hkey}\\Path1": "path1-dir",
+            f"{hkey}\\Path1\\Subdir": "not-subdirs",
+        })
+        ns = MockNTNamespace(
+            argv0=r"C:\Python\python.exe",
+            real_executable=r"C:\Python\python.exe",
+            winreg=winreg,
+        )
+        ns.add_known_xfile(r"C:\Python\python.exe")
+        ns.add_known_file(r"C:\Python\Lib\os.py")
+        ns.add_known_dir(r"C:\Python\DLLs")
+        expected = dict(
+            module_search_paths_set=1,
+            module_search_paths=[
+                r"C:\Python\python98.zip",
+                "path1-dir",
+                # should not contain not-subdirs
+                r"C:\Python\Lib",
+                r"C:\Python\DLLs",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+        ns["config"]["use_environment"] = 0
+        ns["config"]["module_search_paths_set"] = 0
+        ns["config"]["module_search_paths"] = None
+        expected = dict(
+            module_search_paths_set=1,
+            module_search_paths=[
+                r"C:\Python\python98.zip",
+                r"C:\Python\Lib",
+                r"C:\Python\DLLs",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_symlink_normal_win32(self):
+        "Test a 'standard' install layout via symlink on Windows."
+        ns = MockNTNamespace(
+            argv0=r"C:\LinkedFrom\python.exe",
+            real_executable=r"C:\Python\python.exe",
+        )
+        ns.add_known_xfile(r"C:\LinkedFrom\python.exe")
+        ns.add_known_xfile(r"C:\Python\python.exe")
+        ns.add_known_link(r"C:\LinkedFrom\python.exe", r"C:\Python\python.exe")
+        ns.add_known_file(r"C:\Python\Lib\os.py")
+        ns.add_known_dir(r"C:\Python\DLLs")
+        expected = dict(
+            executable=r"C:\LinkedFrom\python.exe",
+            base_executable=r"C:\LinkedFrom\python.exe",
+            prefix=r"C:\Python",
+            exec_prefix=r"C:\Python",
+            module_search_paths_set=1,
+            module_search_paths=[
+                r"C:\Python\python98.zip",
+                r"C:\Python\Lib",
+                r"C:\Python\DLLs",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_symlink_buildtree_win32(self):
+        "Test an in-build-tree layout via symlink on Windows."
+        ns = MockNTNamespace(
+            argv0=r"C:\LinkedFrom\python.exe",
+            real_executable=r"C:\CPython\PCbuild\amd64\python.exe",
+        )
+        ns.add_known_xfile(r"C:\LinkedFrom\python.exe")
+        ns.add_known_xfile(r"C:\CPython\PCbuild\amd64\python.exe")
+        ns.add_known_link(r"C:\LinkedFrom\python.exe", r"C:\CPython\PCbuild\amd64\python.exe")
+        ns.add_known_file(r"C:\CPython\Lib\os.py")
+        ns.add_known_file(r"C:\CPython\PCbuild\amd64\pybuilddir.txt", [""])
+        expected = dict(
+            executable=r"C:\LinkedFrom\python.exe",
+            base_executable=r"C:\LinkedFrom\python.exe",
+            prefix=r"C:\CPython",
+            exec_prefix=r"C:\CPython",
+            build_prefix=r"C:\CPython",
+            _is_python_build=1,
+            module_search_paths_set=1,
+            module_search_paths=[
+                r"C:\CPython\PCbuild\amd64\python98.zip",
+                r"C:\CPython\Lib",
+                r"C:\CPython\PCbuild\amd64",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_normal_posix(self):
+        "Test a 'standard' install layout on *nix"
+        ns = MockPosixNamespace(
+            PREFIX="/usr",
+            argv0="python",
+            ENV_PATH="/usr/bin",
+        )
+        ns.add_known_xfile("/usr/bin/python")
+        ns.add_known_file("/usr/lib/python9.8/os.py")
+        ns.add_known_dir("/usr/lib/python9.8/lib-dynload")
+        expected = dict(
+            executable="/usr/bin/python",
+            base_executable="/usr/bin/python",
+            prefix="/usr",
+            exec_prefix="/usr",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/usr/lib/python98.zip",
+                "/usr/lib/python9.8",
+                "/usr/lib/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_buildpath_posix(self):
+        """Test an in-build-tree layout on POSIX.
+
+        This layout is discovered from the presence of pybuilddir.txt, which
+        contains the relative path from the executable's directory to the
+        platstdlib path.
+        """
+        ns = MockPosixNamespace(
+            argv0=r"/home/cpython/python",
+            PREFIX="/usr/local",
+        )
+        ns.add_known_xfile("/home/cpython/python")
+        ns.add_known_xfile("/usr/local/bin/python")
+        ns.add_known_file("/home/cpython/pybuilddir.txt", ["build/lib.linux-x86_64-9.8"])
+        ns.add_known_file("/home/cpython/Lib/os.py")
+        ns.add_known_dir("/home/cpython/lib-dynload")
+        expected = dict(
+            executable="/home/cpython/python",
+            prefix="/usr/local",
+            exec_prefix="/usr/local",
+            base_executable="/home/cpython/python",
+            build_prefix="/home/cpython",
+            _is_python_build=1,
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/usr/local/lib/python98.zip",
+                "/home/cpython/Lib",
+                "/home/cpython/build/lib.linux-x86_64-9.8",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_venv_posix(self):
+        "Test a venv layout on *nix."
+        ns = MockPosixNamespace(
+            argv0="python",
+            PREFIX="/usr",
+            ENV_PATH="/venv/bin:/usr/bin",
+        )
+        ns.add_known_xfile("/usr/bin/python")
+        ns.add_known_xfile("/venv/bin/python")
+        ns.add_known_file("/usr/lib/python9.8/os.py")
+        ns.add_known_dir("/usr/lib/python9.8/lib-dynload")
+        ns.add_known_file("/venv/pyvenv.cfg", [
+            r"home = /usr/bin"
+        ])
+        expected = dict(
+            executable="/venv/bin/python",
+            prefix="/usr",
+            exec_prefix="/usr",
+            base_executable="/usr/bin/python",
+            base_prefix="/usr",
+            base_exec_prefix="/usr",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/usr/lib/python98.zip",
+                "/usr/lib/python9.8",
+                "/usr/lib/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_symlink_normal_posix(self):
+        "Test a 'standard' install layout via symlink on *nix"
+        ns = MockPosixNamespace(
+            PREFIX="/usr",
+            argv0="/linkfrom/python",
+        )
+        ns.add_known_xfile("/linkfrom/python")
+        ns.add_known_xfile("/usr/bin/python")
+        ns.add_known_link("/linkfrom/python", "/usr/bin/python")
+        ns.add_known_file("/usr/lib/python9.8/os.py")
+        ns.add_known_dir("/usr/lib/python9.8/lib-dynload")
+        expected = dict(
+            executable="/linkfrom/python",
+            base_executable="/linkfrom/python",
+            prefix="/usr",
+            exec_prefix="/usr",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/usr/lib/python98.zip",
+                "/usr/lib/python9.8",
+                "/usr/lib/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_symlink_buildpath_posix(self):
+        """Test an in-build-tree layout on POSIX.
+
+        This layout is discovered from the presence of pybuilddir.txt, which
+        contains the relative path from the executable's directory to the
+        platstdlib path.
+        """
+        ns = MockPosixNamespace(
+            argv0=r"/linkfrom/python",
+            PREFIX="/usr/local",
+        )
+        ns.add_known_xfile("/linkfrom/python")
+        ns.add_known_xfile("/home/cpython/python")
+        ns.add_known_link("/linkfrom/python", "/home/cpython/python")
+        ns.add_known_xfile("/usr/local/bin/python")
+        ns.add_known_file("/home/cpython/pybuilddir.txt", ["build/lib.linux-x86_64-9.8"])
+        ns.add_known_file("/home/cpython/Lib/os.py")
+        ns.add_known_dir("/home/cpython/lib-dynload")
+        expected = dict(
+            executable="/linkfrom/python",
+            prefix="/usr/local",
+            exec_prefix="/usr/local",
+            base_executable="/linkfrom/python",
+            build_prefix="/home/cpython",
+            _is_python_build=1,
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/usr/local/lib/python98.zip",
+                "/home/cpython/Lib",
+                "/home/cpython/build/lib.linux-x86_64-9.8",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_custom_platlibdir_posix(self):
+        "Test an install with custom platlibdir on *nix"
+        ns = MockPosixNamespace(
+            PREFIX="/usr",
+            argv0="/linkfrom/python",
+            PLATLIBDIR="lib64",
+        )
+        ns.add_known_xfile("/usr/bin/python")
+        ns.add_known_file("/usr/lib64/python9.8/os.py")
+        ns.add_known_dir("/usr/lib64/python9.8/lib-dynload")
+        expected = dict(
+            executable="/linkfrom/python",
+            base_executable="/linkfrom/python",
+            prefix="/usr",
+            exec_prefix="/usr",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/usr/lib64/python98.zip",
+                "/usr/lib64/python9.8",
+                "/usr/lib64/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_venv_macos(self):
+        """Test a venv layout on macOS.
+
+        This layout is discovered when 'executable' and 'real_executable' match,
+        but $__PYVENV_LAUNCHER__ has been set to the original process.
+        """
+        ns = MockPosixNamespace(
+            os_name="darwin",
+            argv0="/usr/bin/python",
+            PREFIX="/usr",
+            ENV___PYVENV_LAUNCHER__="/framework/Python9.8/python",
+            real_executable="/usr/bin/python",
+        )
+        ns.add_known_xfile("/usr/bin/python")
+        ns.add_known_xfile("/framework/Python9.8/python")
+        ns.add_known_file("/usr/lib/python9.8/os.py")
+        ns.add_known_dir("/usr/lib/python9.8/lib-dynload")
+        ns.add_known_file("/framework/Python9.8/pyvenv.cfg", [
+            "home = /usr/bin"
+        ])
+        expected = dict(
+            executable="/framework/Python9.8/python",
+            prefix="/usr",
+            exec_prefix="/usr",
+            base_executable="/usr/bin/python",
+            base_prefix="/usr",
+            base_exec_prefix="/usr",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/usr/lib/python98.zip",
+                "/usr/lib/python9.8",
+                "/usr/lib/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_symlink_normal_macos(self):
+        "Test a 'standard' install layout via symlink on macOS"
+        ns = MockPosixNamespace(
+            os_name="darwin",
+            PREFIX="/usr",
+            argv0="python",
+            ENV_PATH="/linkfrom:/usr/bin",
+            # real_executable on macOS matches the invocation path
+            real_executable="/linkfrom/python",
+        )
+        ns.add_known_xfile("/linkfrom/python")
+        ns.add_known_xfile("/usr/bin/python")
+        ns.add_known_link("/linkfrom/python", "/usr/bin/python")
+        ns.add_known_file("/usr/lib/python9.8/os.py")
+        ns.add_known_dir("/usr/lib/python9.8/lib-dynload")
+        expected = dict(
+            executable="/linkfrom/python",
+            base_executable="/linkfrom/python",
+            prefix="/usr",
+            exec_prefix="/usr",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/usr/lib/python98.zip",
+                "/usr/lib/python9.8",
+                "/usr/lib/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_symlink_buildpath_macos(self):
+        """Test an in-build-tree layout via symlink on macOS.
+
+        This layout is discovered from the presence of pybuilddir.txt, which
+        contains the relative path from the executable's directory to the
+        platstdlib path.
+        """
+        ns = MockPosixNamespace(
+            os_name="darwin",
+            argv0=r"python",
+            ENV_PATH="/linkfrom:/usr/bin",
+            PREFIX="/usr/local",
+            # real_executable on macOS matches the invocation path
+            real_executable="/linkfrom/python",
+        )
+        ns.add_known_xfile("/linkfrom/python")
+        ns.add_known_xfile("/home/cpython/python")
+        ns.add_known_link("/linkfrom/python", "/home/cpython/python")
+        ns.add_known_xfile("/usr/local/bin/python")
+        ns.add_known_file("/home/cpython/pybuilddir.txt", ["build/lib.macos-9.8"])
+        ns.add_known_file("/home/cpython/Lib/os.py")
+        ns.add_known_dir("/home/cpython/lib-dynload")
+        expected = dict(
+            executable="/linkfrom/python",
+            prefix="/usr/local",
+            exec_prefix="/usr/local",
+            base_executable="/linkfrom/python",
+            build_prefix="/home/cpython",
+            _is_python_build=1,
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/usr/local/lib/python98.zip",
+                "/home/cpython/Lib",
+                "/home/cpython/build/lib.macos-9.8",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+
+# ******************************************************************************
+
+DEFAULT_NAMESPACE = dict(
+    PREFIX="",
+    EXEC_PREFIX="",
+    PYTHONPATH="",
+    VPATH="",
+    PLATLIBDIR="",
+    PYDEBUGEXT="",
+    VERSION_MAJOR=9,    # fixed version number for ease
+    VERSION_MINOR=8,    # of testing
+    PYWINVER=None,
+    EXE_SUFFIX=None,
+    
+    ENV_PATH="",
+    ENV_PYTHONHOME="",
+    ENV_PYTHONEXECUTABLE="",
+    ENV___PYVENV_LAUNCHER__="",
+    argv0="",
+    py_setpath="",
+    real_executable="",
+    executable_dir="",
+    library="",
+    winreg=None,
+    build_prefix=None,
+    venv_prefix=None,
+)
+
+DEFAULT_CONFIG = dict(
+    home=None,
+    platlibdir=None,
+    pythonpath=None,
+    program_name=None,
+    prefix=None,
+    exec_prefix=None,
+    base_prefix=None,
+    base_exec_prefix=None,
+    executable=None,
+    base_executable="",
+    stdlib_dir=None,
+    platstdlib_dir=None,
+    module_search_paths=None,
+    module_search_paths_set=0,
+    pythonpath_env=None,
+    argv=None,
+    orig_argv=None,
+
+    isolated=0,
+    use_environment=1,
+    use_site=1,
+)
+
+class MockNTNamespace(dict):
+    def __init__(self, *a, argv0=None, config=None, **kw):
+        self.update(DEFAULT_NAMESPACE)
+        self["config"] = DEFAULT_CONFIG.copy()
+        self["os_name"] = "nt"
+        self["PLATLIBDIR"] = "DLLs"
+        self["PYWINVER"] = "9.8-XY"
+        self["VPATH"] = r"..\.."
+        super().__init__(*a, **kw)
+        if argv0:
+            self["config"]["orig_argv"] = [argv0]
+        if config:
+            self["config"].update(config)
+        self._files = {}
+        self._links = {}
+        self._dirs = set()
+        self._warnings = []
+
+    def add_known_file(self, path, lines=None):
+        self._files[path.casefold()] = list(lines or ())
+        self.add_known_dir(path.rpartition("\\")[0])
+
+    def add_known_xfile(self, path):
+        self.add_known_file(path)
+
+    def add_known_link(self, path, target):
+        self._links[path.casefold()] = target
+
+    def add_known_dir(self, path):
+        p = path.rstrip("\\").casefold()
+        while p:
+            self._dirs.add(p)
+            p = p.rpartition("\\")[0]
+
+    def __missing__(self, key):
+        try:
+            return getattr(self, key)
+        except AttributeError:
+            raise KeyError(key) from None
+
+    def abspath(self, path):
+        if self.isabs(path):
+            return path
+        return self.joinpath("C:\\Absolute", path)
+
+    def basename(self, path):
+        return path.rpartition("\\")[2]
+
+    def dirname(self, path):
+        name = path.rstrip("\\").rpartition("\\")[0]
+        if name[1:] == ":":
+            return name + "\\"
+        return name
+
+    def hassuffix(self, path, suffix):
+        return path.casefold().endswith(suffix.casefold())
+
+    def isabs(self, path):
+        return path[1:3] == ":\\"
+
+    def isdir(self, path):
+        if verbose:
+            print("Check if", path, "is a dir")
+        return path.casefold() in self._dirs
+
+    def isfile(self, path):
+        if verbose:
+            print("Check if", path, "is a file")
+        return path.casefold() in self._files
+
+    def ismodule(self, path):
+        if verbose:
+            print("Check if", path, "is a module")
+        path = path.casefold()
+        return path in self._files and path.rpartition(".")[2] == "py".casefold()
+
+    def isxfile(self, path):
+        if verbose:
+            print("Check if", path, "is a executable")
+        path = path.casefold()
+        return path in self._files and path.rpartition(".")[2] == "exe".casefold()
+
+    def joinpath(self, *path):
+        return ntpath.normpath(ntpath.join(*path))
+
+    def readlines(self, path):
+        try:
+            return self._files[path.casefold()]
+        except KeyError:
+            raise FileNotFoundError(path) from None
+
+    def realpath(self, path, _trail=None):
+        if verbose:
+            print("Read link from", path)
+        try:
+            link = self._links[path.casefold()]
+        except KeyError:
+            return path
+        if _trail is None:
+            _trail = set()
+        elif link.casefold() in _trail:
+            raise OSError("circular link")
+        _trail.add(link.casefold())
+        return self.realpath(link, _trail)
+
+    def warn(self, message):
+        self._warnings.append(message)
+        if verbose:
+            print(message)
+
+
+class MockWinreg:
+    HKEY_LOCAL_MACHINE = "HKLM"
+    HKEY_CURRENT_USER = "HKCU"
+
+    def __init__(self, keys):
+        self.keys = {k.casefold(): v for k, v in keys.items()}
+        self.open = {}
+
+    def __repr__(self):
+        return "<MockWinreg>"
+
+    def __eq__(self, other):
+        return isinstance(other, type(self))
+
+    def open_keys(self):
+        return list(self.open)
+
+    def OpenKeyEx(self, hkey, subkey):
+        if verbose:
+            print(f"OpenKeyEx({hkey}, {subkey})")
+        key = f"{hkey}\\{subkey}".casefold()
+        if key in self.keys:
+            self.open[key] = self.open.get(key, 0) + 1
+            return key
+        raise FileNotFoundError()
+
+    def CloseKey(self, hkey):
+        if verbose:
+            print(f"CloseKey({hkey})")
+        hkey = hkey.casefold()
+        if hkey not in self.open:
+            raise RuntimeError("key is not open")
+        self.open[hkey] -= 1
+        if not self.open[hkey]:
+            del self.open[hkey]
+
+    def EnumKey(self, hkey, i):
+        if verbose:
+            print(f"EnumKey({hkey}, {i})")
+        hkey = hkey.casefold()
+        if hkey not in self.open:
+            raise RuntimeError("key is not open")
+        prefix = f'{hkey}\\'
+        subkeys = [k[len(prefix):] for k in sorted(self.keys) if k.startswith(prefix)]
+        subkeys[:] = [k for k in subkeys if '\\' not in k]
+        for j, n in enumerate(subkeys):
+            if j == i:
+                return n.removeprefix(prefix)
+        raise OSError("end of enumeration")
+
+    def QueryValue(self, hkey):
+        if verbose:
+            print(f"QueryValue({hkey})")
+        hkey = hkey.casefold()
+        if hkey not in self.open:
+            raise RuntimeError("key is not open")
+        try:
+            return self.keys[hkey]
+        except KeyError:
+            raise OSError()
+
+
+class MockPosixNamespace(dict):
+    def __init__(self, *a, argv0=None, config=None, **kw):
+        self.update(DEFAULT_NAMESPACE)
+        self["config"] = DEFAULT_CONFIG.copy()
+        self["os_name"] = "posix"
+        self["PLATLIBDIR"] = "lib"
+        super().__init__(*a, **kw)
+        if argv0:
+            self["config"]["orig_argv"] = [argv0]
+        if config:
+            self["config"].update(config)
+        self._files = {}
+        self._xfiles = set()
+        self._links = {}
+        self._dirs = set()
+        self._warnings = []
+
+    def add_known_file(self, path, lines=None):
+        self._files[path] = list(lines or ())
+        self.add_known_dir(path.rpartition("/")[0])
+
+    def add_known_xfile(self, path):
+        self.add_known_file(path)
+        self._xfiles.add(path)
+
+    def add_known_link(self, path, target):
+        self._links[path] = target
+
+    def add_known_dir(self, path):
+        p = path.rstrip("/")
+        while p:
+            self._dirs.add(p)
+            p = p.rpartition("/")[0]
+
+    def __missing__(self, key):
+        try:
+            return getattr(self, key)
+        except AttributeError:
+            raise KeyError(key) from None
+
+    def abspath(self, path):
+        if self.isabs(path):
+            return path
+        return self.joinpath("/Absolute", path)
+
+    def basename(self, path):
+        return path.rpartition("/")[2]
+
+    def dirname(self, path):
+        return path.rstrip("/").rpartition("/")[0]
+
+    def hassuffix(self, path, suffix):
+        return path.endswith(suffix)
+
+    def isabs(self, path):
+        return path[0:1] == "/"
+
+    def isdir(self, path):
+        if verbose:
+            print("Check if", path, "is a dir")
+        return path in self._dirs
+
+    def isfile(self, path):
+        if verbose:
+            print("Check if", path, "is a file")
+        return path in self._files
+
+    def ismodule(self, path):
+        if verbose:
+            print("Check if", path, "is a module")
+        return path in self._files and path.rpartition(".")[2] == "py"
+
+    def isxfile(self, path):
+        if verbose:
+            print("Check if", path, "is an xfile")
+        return path in self._xfiles
+
+    def joinpath(self, *path):
+        return posixpath.normpath(posixpath.join(*path))
+
+    def readlines(self, path):
+        try:
+            return self._files[path]
+        except KeyError:
+            raise FileNotFoundError(path) from None
+
+    def realpath(self, path, _trail=None):
+        if verbose:
+            print("Read link from", path)
+        try:
+            link = self._links[path]
+        except KeyError:
+            return path
+        if _trail is None:
+            _trail = set()
+        elif link in _trail:
+            raise OSError("circular link")
+        _trail.add(link)
+        return self.realpath(link, _trail)
+
+    def warn(self, message):
+        self._warnings.append(message)
+        if verbose:
+            print(message)
+
+
+def diff_dict(before, after, prefix="global"):
+    diff = []
+    for k in sorted(before):
+        if k[:2] == "__":
+            continue
+        if k == "config":
+            diff_dict(before[k], after[k], prefix="config")
+            continue
+        if k in after and after[k] != before[k]:
+            diff.append((k, before[k], after[k]))
+    if not diff:
+        return
+    max_k = max(len(k) for k, _, _ in diff)
+    indent = " " * (len(prefix) + 1 + max_k)
+    if verbose:
+        for k, b, a in diff:
+            if b:
+                print("{}.{} -{!r}\n{} +{!r}".format(prefix, k.ljust(max_k), b, indent, a))
+            else:
+                print("{}.{} +{!r}".format(prefix, k.ljust(max_k), a))
+
+
+def dump_dict(before, after, prefix="global"):
+    if not verbose or not after:
+        return
+    max_k = max(len(k) for k in after)
+    for k, v in sorted(after.items(), key=lambda i: i[0]):
+        if k[:2] == "__":
+            continue
+        if k == "config":
+            dump_dict(before[k], after[k], prefix="config")
+            continue
+        try:
+            if v != before[k]:
+                print("{}.{} {!r} (was {!r})".format(prefix, k.ljust(max_k), v, before[k]))
+                continue
+        except KeyError:
+            pass
+        print("{}.{} {!r}".format(prefix, k.ljust(max_k), v))
+
+
+def getpath(ns, keys):
+    before = copy.deepcopy(ns)
+    failed = True
+    try:
+        exec(SOURCE, ns)
+        failed = False
+    finally:
+        if failed:
+            dump_dict(before, ns)
+        else:
+            diff_dict(before, ns)
+    return {
+        k: ns['config'].get(k, ns.get(k, ...))
+        for k in keys
+    }

Lib/test/test_ntpath.py

@@ -741,8 +741,9 @@ class TestNtpath(NtpathTestCase):
             # (or any other volume root). The drive-relative
             # locations below cannot then refer to mount points
             #
-            drive, path = ntpath.splitdrive(sys.executable)
+            test_cwd = os.getenv("SystemRoot")
-            with os_helper.change_cwd(ntpath.dirname(sys.executable)):
+            drive, path = ntpath.splitdrive(test_cwd)
+            with os_helper.change_cwd(test_cwd):
                 self.assertFalse(ntpath.ismount(drive.lower()))
                 self.assertFalse(ntpath.ismount(drive.upper()))
 

Lib/test/test_site.py

@@ -11,6 +11,7 @@ from test.support import os_helper
 from test.support import socket_helper
 from test.support import captured_stderr
 from test.support.os_helper import TESTFN, EnvironmentVarGuard, change_cwd
+import ast
 import builtins
 import encodings
 import glob
@@ -300,7 +301,8 @@ class HelperFunctionsTests(unittest.TestCase):
             self.assertEqual(len(dirs), 2)
             self.assertEqual(dirs[0], 'xoxo')
             wanted = os.path.join('xoxo', 'lib', 'site-packages')
-            self.assertEqual(dirs[1], wanted)
+            self.assertEqual(os.path.normcase(dirs[1]),
+                             os.path.normcase(wanted))
 
     @unittest.skipUnless(HAS_USER_SITE, 'need user site')
     def test_no_home_directory(self):
@@ -497,13 +499,14 @@ class StartupImportTests(unittest.TestCase):
 
     def test_startup_imports(self):
         # Get sys.path in isolated mode (python3 -I)
-        popen = subprocess.Popen([sys.executable, '-I', '-c',
+        popen = subprocess.Popen([sys.executable, '-X', 'utf8', '-I',
-                                  'import sys; print(repr(sys.path))'],
+                                  '-c', 'import sys; print(repr(sys.path))'],
                                  stdout=subprocess.PIPE,
-                                 encoding='utf-8')
+                                 encoding='utf-8',
+                                 errors='surrogateescape')
         stdout = popen.communicate()[0]
         self.assertEqual(popen.returncode, 0, repr(stdout))
-        isolated_paths = eval(stdout)
+        isolated_paths = ast.literal_eval(stdout)
 
         # bpo-27807: Even with -I, the site module executes all .pth files
         # found in sys.path (see site.addpackage()). Skip the test if at least
@@ -515,14 +518,15 @@ class StartupImportTests(unittest.TestCase):
 
         # This tests checks which modules are loaded by Python when it
         # initially starts upon startup.
-        popen = subprocess.Popen([sys.executable, '-I', '-v', '-c',
+        popen = subprocess.Popen([sys.executable, '-X', 'utf8', '-I', '-v',
-                                  'import sys; print(set(sys.modules))'],
+                                  '-c', 'import sys; print(set(sys.modules))'],
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE,
-                                 encoding='utf-8')
+                                 encoding='utf-8',
+                                 errors='surrogateescape')
         stdout, stderr = popen.communicate()
         self.assertEqual(popen.returncode, 0, (stdout, stderr))
-        modules = eval(stdout)
+        modules = ast.literal_eval(stdout)
 
         self.assertIn('site', modules)
 

Lib/test/test_sysconfig.py

@@ -102,6 +102,10 @@ class TestSysConfig(unittest.TestCase):
 
     def test_get_path(self):
         config_vars = get_config_vars()
+        if os.name == 'nt':
+            # On Windows, we replace the native platlibdir name with the
+            # default so that POSIX schemes resolve correctly
+            config_vars = config_vars | {'platlibdir': 'lib'}
         for scheme in _INSTALL_SCHEMES:
             for name in _INSTALL_SCHEMES[scheme]:
                 expected = _INSTALL_SCHEMES[scheme][name].format(**config_vars)

Lib/test/test_venv.py

@@ -15,7 +15,7 @@ import subprocess
 import sys
 import tempfile
 from test.support import (captured_stdout, captured_stderr, requires_zlib,
-                          skip_if_broken_multiprocessing_synchronize)
+                          skip_if_broken_multiprocessing_synchronize, verbose)
 from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree)
 import unittest
 import venv
@@ -40,6 +40,8 @@ def check_output(cmd, encoding=None):
         encoding=encoding)
     out, err = p.communicate()
     if p.returncode:
+        if verbose and err:
+            print(err.decode('utf-8', 'backslashreplace'))
         raise subprocess.CalledProcessError(
             p.returncode, cmd, out, err)
     return out, err
@@ -194,7 +196,7 @@ class BasicTest(BaseTest):
             ('base_exec_prefix', sys.base_exec_prefix)):
             cmd[2] = 'import sys; print(sys.%s)' % prefix
             out, err = check_output(cmd)
-            self.assertEqual(out.strip(), expected.encode())
+            self.assertEqual(out.strip(), expected.encode(), prefix)
 
     if sys.platform == 'win32':
         ENV_SUBDIRS = (

Makefile.pre.in

@@ -308,7 +308,6 @@ COVERAGE_REPORT_OPTIONS=--no-branch-coverage --title "CPython lcov report"
 # Modules
 MODULE_OBJS=	\
 		Modules/config.o \
-		Modules/getpath.o \
 		Modules/main.o \
 		Modules/gcmodule.o
 
@@ -505,6 +504,7 @@ LIBRARY_OBJS_OMIT_FROZEN=	\
 LIBRARY_OBJS=	\
 		$(LIBRARY_OBJS_OMIT_FROZEN) \
 		$(DEEPFREEZE_OBJS) \
+		Modules/getpath.o \
 		Python/frozen.o
 
 ##########################################################################
@@ -1061,8 +1061,11 @@ FROZEN_FILES_OUT = \
 
 Programs/_freeze_module.o: Programs/_freeze_module.c Makefile
 
-Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN)
+Modules/getpath_noop.o: $(srcdir)/Modules/getpath_noop.c Makefile
-	$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS)
+	$(CC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Modules/getpath_noop.c
+
+Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) Modules/getpath_noop.o
+	$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) Modules/getpath_noop.o $(LIBS) $(MODLIBS) $(SYSLIBS)
 
 # BEGIN: freezing modules
 
@@ -1130,6 +1133,10 @@ Python/frozen_modules/frozen_only.h: $(FREEZE_MODULE) Tools/freeze/flag.py
 
 Tools/scripts/freeze_modules.py: $(FREEZE_MODULE)
 
+# We manually freeze getpath.py rather than through freeze_modules
+Modules/getpath.h: Programs/_freeze_module Modules/getpath.py
+	Programs/_freeze_module getpath $(srcdir)/Modules/getpath.py $(srcdir)/Modules/getpath.h
+
 .PHONY: regen-frozen
 regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES_IN)
 	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py
@@ -1170,12 +1177,13 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \
 	      -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \
 	      -o $@ $(srcdir)/Modules/getbuildinfo.c
 
-Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile
+Modules/getpath.o: $(srcdir)/Modules/getpath.c Modules/getpath.h Makefile
 	$(CC) -c $(PY_CORE_CFLAGS) -DPYTHONPATH='"$(PYTHONPATH)"' \
 		-DPREFIX='"$(prefix)"' \
 		-DEXEC_PREFIX='"$(exec_prefix)"' \
 		-DVERSION='"$(VERSION)"' \
 		-DVPATH='"$(VPATH)"' \
+		-DPLATLIBDIR='"$(PLATLIBDIR)"' \
 		-o $@ $(srcdir)/Modules/getpath.c
 
 Programs/python.o: $(srcdir)/Programs/python.c
@@ -1210,11 +1218,6 @@ Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(srcdir)/Include/pydt
 		$(MULTIARCH_CPPFLAGS) \
 		-o $@ $(srcdir)/Python/sysmodule.c
 
-Python/initconfig.o: $(srcdir)/Python/initconfig.c
-	$(CC) -c $(PY_CORE_CFLAGS) \
-		-DPLATLIBDIR='"$(PLATLIBDIR)"' \
-		-o $@ $(srcdir)/Python/initconfig.c
-
 $(IO_OBJS): $(IO_H)
 
 .PHONY: regen-pegen-metaparser

Misc/NEWS.d/next/Core and Builtins/2021-10-23-00-39-31.bpo-45582.YONPuo.rst

@@ -0,0 +1,3 @@
+Path calculation (known as ``getpath``) has been reimplemented as a frozen
+Python module. This should have no visible impact, but may affect
+calculation of all paths referenced in :mod:`sys` and :mod:`sysconfig`.

Modules/_testinternalcapi.c

@@ -14,10 +14,11 @@
 #include "Python.h"
 #include "pycore_atomic_funcs.h" // _Py_atomic_int_get()
 #include "pycore_bitutils.h"     // _Py_bswap32()
-#include "pycore_fileutils.h"    // _Py_normalize_path
+#include "pycore_fileutils.h"    // _Py_normpath
 #include "pycore_gc.h"           // PyGC_Head
 #include "pycore_hashtable.h"    // _Py_hashtable_new()
 #include "pycore_initconfig.h"   // _Py_GetConfigsAsDict()
+#include "pycore_pathconfig.h"   // _PyPathConfig_ClearGlobal()
 #include "pycore_interp.h"       // _PyInterpreterState_GetConfigCopy()
 #include "pycore_pyerrors.h"     // _Py_UTF8_Edit_Cost()
 #include "pycore_pystate.h"      // _PyThreadState_GET()
@@ -273,6 +274,14 @@ error:
 }
 
 
+static PyObject *
+test_reset_path_config(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(arg))
+{
+    _PyPathConfig_ClearGlobal();
+    Py_RETURN_NONE;
+}
+
+
 static PyObject*
 test_atomic_funcs(PyObject *self, PyObject *Py_UNUSED(args))
 {
@@ -378,15 +387,15 @@ normalize_path(PyObject *self, PyObject *filename)
         return NULL;
     }
 
-    wchar_t buf[MAXPATHLEN + 1];
+    PyObject *result = PyUnicode_FromWideChar(_Py_normpath(encoded, size), -1);
-    int res = _Py_normalize_path(encoded, buf, Py_ARRAY_LENGTH(buf));
     PyMem_Free(encoded);
-    if (res != 0) {
-        PyErr_SetString(PyExc_ValueError, "string too long");
-        return NULL;
-    }
 
-    return PyUnicode_FromWideChar(buf, -1);
+    return result;
+}
+
+static PyObject *
+get_getpath_codeobject(PyObject *self, PyObject *Py_UNUSED(args)) {
+    return _Py_Get_Getpath_CodeObject();
 }
 
 
@@ -399,9 +408,11 @@ static PyMethodDef TestMethods[] = {
     {"test_hashtable", test_hashtable, METH_NOARGS},
     {"get_config", test_get_config, METH_NOARGS},
     {"set_config", test_set_config, METH_O},
+    {"reset_path_config", test_reset_path_config, METH_NOARGS},
     {"test_atomic_funcs", test_atomic_funcs, METH_NOARGS},
     {"test_edit_cost", test_edit_cost, METH_NOARGS},
     {"normalize_path", normalize_path, METH_O, NULL},
+    {"get_getpath_codeobject", get_getpath_codeobject, METH_NOARGS, NULL},
     {NULL, NULL} /* sentinel */
 };
 

Modules/clinic/posixmodule.c.h

@@ -1317,6 +1317,38 @@ exit:
 
 #endif /* defined(MS_WINDOWS) */
 
+PyDoc_STRVAR(os__path_normpath__doc__,
+"_path_normpath($module, /, path)\n"
+"--\n"
+"\n"
+"Basic path normalization.");
+
+#define OS__PATH_NORMPATH_METHODDEF    \
+    {"_path_normpath", (PyCFunction)(void(*)(void))os__path_normpath, METH_FASTCALL|METH_KEYWORDS, os__path_normpath__doc__},
+
+static PyObject *
+os__path_normpath_impl(PyObject *module, PyObject *path);
+
+static PyObject *
+os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+    PyObject *return_value = NULL;
+    static const char * const _keywords[] = {"path", NULL};
+    static _PyArg_Parser _parser = {NULL, _keywords, "_path_normpath", 0};
+    PyObject *argsbuf[1];
+    PyObject *path;
+
+    args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    path = args[0];
+    return_value = os__path_normpath_impl(module, path);
+
+exit:
+    return return_value;
+}
+
 PyDoc_STRVAR(os_mkdir__doc__,
 "mkdir($module, /, path, mode=511, *, dir_fd=None)\n"
 "--\n"
@@ -9263,4 +9295,4 @@ exit:
 #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF
     #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF
 #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */
-/*[clinic end generated code: output=65a85d7d3f2c487e input=a9049054013a1b77]*/
+/*[clinic end generated code: output=05505f171cdcff72 input=a9049054013a1b77]*/

Modules/getpath.py

@@ -0,0 +1,727 @@
+# ******************************************************************************
+# getpath.py
+# ******************************************************************************
+
+# This script is designed to be precompiled to bytecode, frozen into the
+# main binary, and then directly evaluated. It is not an importable module,
+# and does not import any other modules (besides winreg on Windows).
+# Rather, the values listed below must be specified in the globals dict
+# used when evaluating the bytecode.
+
+# See _PyConfig_InitPathConfig in Modules/getpath.c for the execution.
+
+# ******************************************************************************
+# REQUIRED GLOBALS
+# ******************************************************************************
+
+# ** Helper functions **
+# abspath(path)     -- make relative paths absolute against CWD
+# basename(path)    -- the filename of path
+# dirname(path)     -- the directory name of path
+# hassuffix(path, suffix) -- returns True if path has suffix
+# isabs(path)       -- path is absolute or not
+# isdir(path)       -- path exists and is a directory
+# isfile(path)      -- path exists and is a file
+# isxfile(path)     -- path exists and is an executable file
+# joinpath(*paths)  -- combine the paths
+# readlines(path)   -- a list of each line of text in the UTF-8 encoded file
+# realpath(path)    -- resolves symlinks in path
+# warn(message)     -- print a warning (if enabled)
+
+# ** Values known at compile time **
+# os_name           -- [in] one of 'nt', 'posix', 'darwin'
+# PREFIX            -- [in] sysconfig.get_config_var(...)
+# EXEC_PREFIX       -- [in] sysconfig.get_config_var(...)
+# PYTHONPATH        -- [in] sysconfig.get_config_var(...)
+# VPATH             -- [in] sysconfig.get_config_var(...)
+# PLATLIBDIR        -- [in] sysconfig.get_config_var(...)
+# PYDEBUGEXT        -- [in, opt] '_d' on Windows for debug builds
+# EXE_SUFFIX        -- [in, opt] '.exe' on Windows/Cygwin/similar
+# VERSION_MAJOR     -- [in] sys.version_info.major
+# VERSION_MINOR     -- [in] sys.version_info.minor
+# PYWINVER          -- [in] the Windows platform-specific version (e.g. 3.8-32)
+
+# ** Values read from the environment **
+#   There is no need to check the use_environment flag before reading
+#   these, as the flag will be tested in this script.
+#   Also note that ENV_PYTHONPATH is read from config['pythonpath_env']
+#   to allow for embedders who choose to specify it via that struct.
+# ENV_PATH                -- [in] getenv(...)
+# ENV_PYTHONHOME          -- [in] getenv(...)
+# ENV_PYTHONEXECUTABLE    -- [in] getenv(...)
+# ENV___PYVENV_LAUNCHER__ -- [in] getenv(...)
+
+# ** Values calculated at runtime **
+# config            -- [in/out] dict of the PyConfig structure
+# real_executable   -- [in, optional] resolved path to main process
+#   On Windows and macOS, read directly from the running process
+#   Otherwise, leave None and it will be calculated from executable
+# executable_dir    -- [in, optional] real directory containing binary
+#   If None, will be calculated from real_executable or executable
+# py_setpath        -- [in] argument provided to Py_SetPath
+#   If None, 'prefix' and 'exec_prefix' may be updated in config
+# library           -- [in, optional] path of dylib/DLL/so
+#   Only used for locating ._pth files
+# winreg            -- [in, optional] the winreg module (only on Windows)
+
+# ******************************************************************************
+# HIGH-LEVEL ALGORITHM
+# ******************************************************************************
+
+# IMPORTANT: The code is the actual specification at time of writing.
+# This prose description is based on the original comment from the old
+# getpath.c to help capture the intent, but should not be considered
+# a specification.
+
+# Search in some common locations for the associated Python libraries.
+
+# Two directories must be found, the platform independent directory
+# (prefix), containing the common .py and .pyc files, and the platform
+# dependent directory (exec_prefix), containing the shared library
+# modules.  Note that prefix and exec_prefix can be the same directory,
+# but for some installations, they are different.
+
+# This script carries out separate searches for prefix and exec_prefix.
+# Each search tries a number of different locations until a ``landmark''
+# file or directory is found.  If no prefix or exec_prefix is found, a
+# warning message is issued and the preprocessor defined PREFIX and
+# EXEC_PREFIX are used (even though they will not work); python carries on
+# as best as is possible, but most imports will fail.
+
+# Before any searches are done, the location of the executable is
+# determined.  If Py_SetPath() was called, or if we are running on
+# Windows, the 'real_executable' path is used (if known).  Otherwise,
+# we use the config-specified program name or default to argv[0].
+# If this has one or more slashes in it, it is made absolute against
+# the current working directory.  If it only contains a name, it must
+# have been invoked from the shell's path, so we search $PATH for the
+# named executable and use that.  If the executable was not found on
+# $PATH (or there was no $PATH environment variable), the original
+# argv[0] string is used.
+
+# At this point, provided Py_SetPath was not used, the
+# __PYVENV_LAUNCHER__ variable may override the executable (on macOS,
+# the PYTHON_EXECUTABLE variable may also override). This allows
+# certain launchers that run Python as a subprocess to properly
+# specify the executable path. They are not intended for users.
+
+# Next, the executable location is examined to see if it is a symbolic
+# link.  If so, the link is realpath-ed and the directory of the link
+# target is used for the remaining searches.  The same steps are
+# performed for prefix and for exec_prefix, but with different landmarks.
+
+# Step 1. Are we running in a virtual environment? Unless 'home' has
+# been specified another way, check for a pyvenv.cfg and use its 'home'
+# property to override the executable dir used later for prefix searches.
+# We do not activate the venv here - that is performed later by site.py.
+
+# Step 2. Is there a ._pth file? A ._pth file lives adjacent to the
+# runtime library (if any) or the actual executable (not the symlink),
+# and contains precisely the intended contents of sys.path as relative
+# paths (to its own location). Its presence also enables isolated mode
+# and suppresses other environment variable usage. Unless already
+# specified by Py_SetHome(), the directory containing the ._pth file is
+# set as 'home'.
+
+# Step 3. Are we running python out of the build directory?  This is
+# checked by looking for the BUILDDIR_TXT file, which contains the
+# relative path to the platlib dir. The executable_dir value is
+# derived from joining the VPATH preprocessor variable to the
+# directory containing pybuilddir.txt. If it is not found, the 
+# BUILD_LANDMARK file is found, which is part of the source tree.
+# prefix is then found by searching up for a file that should only
+# exist in the source tree, and the stdlib dir is set to prefix/Lib.
+
+# Step 4. If 'home' is set, either by Py_SetHome(), ENV_PYTHONHOME,
+# a pyvenv.cfg file, ._pth file, or by detecting a build directory, it
+# is assumed to point to prefix and exec_prefix. $PYTHONHOME can be a
+# single directory, which is used for both, or the prefix and exec_prefix
+# directories separated by DELIM (colon on POSIX; semicolon on Windows).
+
+# Step 5. Try to find prefix and exec_prefix relative to executable_dir,
+# backtracking up the path until it is exhausted.  This is the most common
+# step to succeed.  Note that if prefix and exec_prefix are different,
+# exec_prefix is more likely to be found; however if exec_prefix is a
+# subdirectory of prefix, both will be found.
+
+# Step 6. Search the directories pointed to by the preprocessor variables
+# PREFIX and EXEC_PREFIX.  These are supplied by the Makefile but can be
+# passed in as options to the configure script.
+
+# That's it!
+
+# Well, almost.  Once we have determined prefix and exec_prefix, the
+# preprocessor variable PYTHONPATH is used to construct a path.  Each
+# relative path on PYTHONPATH is prefixed with prefix.  Then the directory
+# containing the shared library modules is appended.  The environment
+# variable $PYTHONPATH is inserted in front of it all. On POSIX, if we are
+# in a build directory, both prefix and exec_prefix are reset to the
+# corresponding preprocessor variables (so sys.prefix will reflect the
+# installation location, even though sys.path points into the build
+# directory).  This seems to make more sense given that currently the only
+# known use of sys.prefix and sys.exec_prefix is for the ILU installation
+# process to find the installed Python tree.
+
+# An embedding application can use Py_SetPath() to override all of
+# these automatic path computations.
+
+
+# ******************************************************************************
+# PLATFORM CONSTANTS
+# ******************************************************************************
+
+platlibdir = config.get('platlibdir') or PLATLIBDIR
+
+if os_name == 'posix' or os_name == 'darwin':
+    BUILDDIR_TXT = 'pybuilddir.txt'
+    BUILD_LANDMARK = 'Modules/Setup.local'
+    DEFAULT_PROGRAM_NAME = f'python{VERSION_MAJOR}'
+    STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}'
+    STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}/os.py', f'{STDLIB_SUBDIR}/os.pyc']
+    PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}/lib-dynload'
+    BUILDSTDLIB_LANDMARKS = ['Lib/os.py']
+    VENV_LANDMARK = 'pyvenv.cfg'
+    ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}.zip'
+    DELIM = ':'
+    SEP = '/'
+
+elif os_name == 'nt':
+    BUILDDIR_TXT = 'pybuilddir.txt'
+    BUILD_LANDMARK = r'..\..\Modules\Setup.local'
+    DEFAULT_PROGRAM_NAME = f'python'
+    STDLIB_SUBDIR = 'Lib'
+    STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}\\os.py', f'{STDLIB_SUBDIR}\\os.pyc']
+    PLATSTDLIB_LANDMARK = f'{platlibdir}'
+    BUILDSTDLIB_LANDMARKS = ['Lib\\os.py']
+    VENV_LANDMARK = 'pyvenv.cfg'
+    ZIP_LANDMARK = f'python{VERSION_MAJOR}{VERSION_MINOR}{PYDEBUGEXT or ""}.zip'
+    WINREG_KEY = f'SOFTWARE\\Python\\PythonCore\\{PYWINVER}\\PythonPath'
+    DELIM = ';'
+    SEP = '\\'
+
+
+# ******************************************************************************
+# HELPER FUNCTIONS (note that we prefer C functions for performance)
+# ******************************************************************************
+
+def search_up(prefix, *landmarks, test=isfile):
+    while prefix:
+        if any(test(joinpath(prefix, f)) for f in landmarks):
+            return prefix
+        prefix = dirname(prefix)
+
+
+# ******************************************************************************
+# READ VARIABLES FROM config
+# ******************************************************************************
+
+program_name = config.get('program_name')
+home = config.get('home')
+executable = config.get('executable')
+base_executable = config.get('base_executable')
+prefix = config.get('prefix')
+exec_prefix = config.get('exec_prefix')
+base_prefix = config.get('base_prefix')
+base_exec_prefix = config.get('base_exec_prefix')
+ENV_PYTHONPATH = config['pythonpath_env']
+use_environment = config.get('use_environment', 1)
+
+pythonpath = config.get('module_search_paths')
+
+real_executable_dir = None
+stdlib_dir = None
+platstdlib_dir = None
+
+# ******************************************************************************
+# CALCULATE program_name
+# ******************************************************************************
+
+program_name_was_set = bool(program_name)
+
+if not program_name:
+    try:
+        program_name = config.get('orig_argv', [])[0]
+    except IndexError:
+        pass
+
+if not program_name:
+    program_name = DEFAULT_PROGRAM_NAME
+
+if EXE_SUFFIX and not hassuffix(program_name, EXE_SUFFIX) and isxfile(program_name + EXE_SUFFIX):
+    program_name = program_name + EXE_SUFFIX
+
+
+# ******************************************************************************
+# CALCULATE executable
+# ******************************************************************************
+
+if py_setpath:
+    # When Py_SetPath has been called, executable defaults to
+    # the real executable path.
+    if not executable:
+        executable = real_executable
+
+if not executable and SEP in program_name:
+    # Resolve partial path program_name against current directory
+    executable = abspath(program_name)
+
+if not executable:
+    # All platforms default to real_executable if known at this
+    # stage. POSIX does not set this value.
+    executable = real_executable
+elif os_name == 'darwin':
+    # QUIRK: On macOS we may know the real executable path, but
+    # if our caller has lied to us about it (e.g. most of
+    # test_embed), we need to use their path in order to detect
+    # whether we are in a build tree. This is true even if the
+    # executable path was provided in the config.
+    real_executable = executable
+
+if not executable and program_name:
+    # Resolve names against PATH.
+    # NOTE: The use_environment value is ignored for this lookup.
+    # To properly isolate, launch Python with a full path.
+    for p in ENV_PATH.split(DELIM):
+        p = joinpath(p, program_name)
+        if isxfile(p):
+            executable = p
+            break
+
+if not executable:
+    executable = ''
+    # When we cannot calculate the executable, subsequent searches
+    # look in the current working directory. Here, we emulate that
+    # (the former getpath.c would do it apparently by accident).
+    executable_dir = abspath('.')
+    # Also need to set this fallback in case we are running from a
+    # build directory with an invalid argv0 (i.e. test_sys.test_executable)
+    real_executable_dir = executable_dir
+
+if ENV_PYTHONEXECUTABLE or ENV___PYVENV_LAUNCHER__:
+    # If set, these variables imply that we should be using them as
+    # sys.executable and when searching for venvs. However, we should
+    # use the argv0 path for prefix calculation
+    base_executable = executable
+    if not real_executable:
+        real_executable = executable
+    executable = ENV_PYTHONEXECUTABLE or ENV___PYVENV_LAUNCHER__
+    executable_dir = dirname(executable)
+
+
+# ******************************************************************************
+# CALCULATE (default) home
+# ******************************************************************************
+
+# Used later to distinguish between Py_SetPythonHome and other
+# ways that it may have been set
+home_was_set = False
+
+if home:
+    home_was_set = True
+elif use_environment and ENV_PYTHONHOME and not py_setpath:
+    home = ENV_PYTHONHOME
+
+
+# ******************************************************************************
+# READ pyvenv.cfg
+# ******************************************************************************
+
+venv_prefix = None
+
+# Calling Py_SetPythonHome(), Py_SetPath() or
+# setting $PYTHONHOME will override venv detection.
+if not home and not py_setpath:
+    try:
+        # prefix2 is just to avoid calculating dirname again later,
+        # as the path in venv_prefix is the more common case.
+        venv_prefix2 = executable_dir or dirname(executable)
+        venv_prefix = dirname(venv_prefix2)
+        try:
+            # Read pyvenv.cfg from one level above executable
+            pyvenvcfg = readlines(joinpath(venv_prefix, VENV_LANDMARK))
+        except FileNotFoundError:
+            # Try the same directory as executable
+            pyvenvcfg = readlines(joinpath(venv_prefix2, VENV_LANDMARK))
+            venv_prefix = venv_prefix2
+    except FileNotFoundError:
+        venv_prefix = None
+        pyvenvcfg = []
+
+    for line in pyvenvcfg:
+        key, had_equ, value = line.partition('=')
+        if had_equ and key.strip().lower() == 'home':
+            executable_dir = real_executable_dir = value.strip()
+            base_executable = joinpath(executable_dir, basename(executable))
+            break
+    else:
+        venv_prefix = None
+
+
+# ******************************************************************************
+# CALCULATE base_executable, real_executable AND executable_dir
+# ******************************************************************************
+
+if not base_executable:
+    base_executable = executable or real_executable or ''
+
+if not real_executable:
+    real_executable = base_executable
+
+try:
+    real_executable = realpath(real_executable)
+except OSError as ex:
+    # Only warn if the file actually exists and was unresolvable
+    # Otherwise users who specify a fake executable may get spurious warnings.
+    if isfile(real_executable):
+        warn(f'Failed to find real location of {base_executable}')
+
+if not executable_dir and os_name == 'darwin' and library:
+    # QUIRK: macOS checks adjacent to its library early
+    library_dir = dirname(library)
+    if any(isfile(joinpath(library_dir, p)) for p in STDLIB_LANDMARKS):
+        # Exceptions here should abort the whole process (to match
+        # previous behavior)
+        executable_dir = realpath(library_dir)
+        real_executable_dir = executable_dir
+
+# If we do not have the executable's directory, we can calculate it.
+# This is the directory used to find prefix/exec_prefix if necessary.
+if not executable_dir:
+    executable_dir = real_executable_dir = dirname(real_executable)
+
+# If we do not have the real executable's directory, we calculate it.
+# This is the directory used to detect build layouts.
+if not real_executable_dir:
+    real_executable_dir = dirname(real_executable)
+
+# ******************************************************************************
+# DETECT _pth FILE
+# ******************************************************************************
+
+# The contents of an optional ._pth file are used to totally override
+# sys.path calcualation. Its presence also implies isolated mode and
+# no-site (unless explicitly requested)
+pth = None
+pth_dir = None
+
+# Calling Py_SetPythonHome() or Py_SetPath() will override ._pth search,
+# but environment variables and command-line options cannot.
+if not py_setpath and not home_was_set:
+    # Check adjacent to the main DLL/dylib/so
+    if library:
+        try:
+            pth = readlines(library.rpartition('.')[0] + '._pth')
+            pth_dir = dirname(library)
+        except FileNotFoundError:
+            pass
+
+    # Check adjacent to the original executable, even if we
+    # redirected to actually launch Python. This may allow a
+    # venv to override the base_executable's ._pth file, but
+    # it cannot override the library's one.
+    if not pth_dir:
+        try:
+            pth = readlines(executable.rpartition('.')[0] + '._pth')
+            pth_dir = dirname(executable)
+        except FileNotFoundError:
+            pass
+
+    # If we found a ._pth file, disable environment and home
+    # detection now. Later, we will do the rest.
+    if pth_dir:
+        use_environment = 0
+        home = pth_dir
+        pythonpath = []
+
+
+# ******************************************************************************
+# CHECK FOR BUILD DIRECTORY
+# ******************************************************************************
+
+build_prefix = None
+
+if not home_was_set and real_executable_dir and not py_setpath:
+    # Detect a build marker and use it to infer prefix, exec_prefix,
+    # stdlib_dir and the platstdlib_dir directories.
+    try:
+        platstdlib_dir = joinpath(
+            real_executable_dir,
+            readlines(joinpath(real_executable_dir, BUILDDIR_TXT))[0],
+        )
+        build_prefix = joinpath(real_executable_dir, VPATH)
+    except FileNotFoundError:
+        if isfile(joinpath(real_executable_dir, BUILD_LANDMARK)):
+            build_prefix = joinpath(real_executable_dir, VPATH)
+            if os_name == 'nt':
+                # QUIRK: Windows builds need platstdlib_dir to be the executable
+                # dir. Normally the builddir marker handles this, but in this
+                # case we need to correct manually.
+                platstdlib_dir = real_executable_dir
+
+    if build_prefix:
+        if os_name == 'nt':
+            # QUIRK: No searching for more landmarks on Windows
+            build_stdlib_prefix = build_prefix
+        else:
+            build_stdlib_prefix = search_up(build_prefix, *BUILDSTDLIB_LANDMARKS)
+        # Always use the build prefix for stdlib
+        if build_stdlib_prefix:
+            stdlib_dir = joinpath(build_stdlib_prefix, 'Lib')
+        else:
+            stdlib_dir = joinpath(build_prefix, 'Lib')
+        # Only use the build prefix for prefix if it hasn't already been set
+        if not prefix:
+            prefix = build_stdlib_prefix
+        # Do not warn, because 'prefix' never equals 'build_prefix' on POSIX
+        #elif not venv_prefix and prefix != build_prefix:
+        #    warn('Detected development environment but prefix is already set')
+        if not exec_prefix:
+            exec_prefix = build_prefix
+        # Do not warn, because 'exec_prefix' never equals 'build_prefix' on POSIX
+        #elif not venv_prefix and exec_prefix != build_prefix:
+        #    warn('Detected development environment but exec_prefix is already set')
+        config['_is_python_build'] = 1
+
+
+# ******************************************************************************
+# CALCULATE prefix AND exec_prefix
+# ******************************************************************************
+
+if py_setpath:
+    # As documented, calling Py_SetPath will force both prefix
+    # and exec_prefix to the empty string.
+    prefix = exec_prefix = ''
+
+else:
+    # Read prefix and exec_prefix from explicitly set home
+    if home:
+        # When multiple paths are listed with ':' or ';' delimiters,
+        # split into prefix:exec_prefix
+        prefix, had_delim, exec_prefix = home.partition(DELIM)
+        if not had_delim:
+            exec_prefix = prefix
+
+
+    # First try to detect prefix by looking alongside our runtime library, if known
+    if library and not prefix:
+        library_dir = dirname(library)
+        if ZIP_LANDMARK:
+            if os_name == 'nt':
+                # QUIRK: Windows does not search up for ZIP file
+                if isfile(joinpath(library_dir, ZIP_LANDMARK)):
+                    prefix = library_dir
+            else:
+                prefix = search_up(library_dir, ZIP_LANDMARK)
+        if STDLIB_SUBDIR and STDLIB_LANDMARKS and not prefix:
+            if any(isfile(joinpath(library_dir, f)) for f in STDLIB_LANDMARKS):
+                prefix = library_dir
+                stdlib_dir = joinpath(prefix, STDLIB_SUBDIR)
+
+
+    # Detect prefix by looking for zip file
+    if ZIP_LANDMARK and executable_dir and not prefix:
+        if os_name == 'nt':
+            # QUIRK: Windows does not search up for ZIP file
+            if isfile(joinpath(executable_dir, ZIP_LANDMARK)):
+                prefix = executable_dir
+        else:
+            prefix = search_up(executable_dir, ZIP_LANDMARK)
+        if prefix:
+            stdlib_dir = joinpath(prefix, STDLIB_SUBDIR)
+            if not isdir(stdlib_dir):
+                stdlib_dir = None
+
+
+    # Detect prefix by searching from our executable location for the stdlib_dir
+    if STDLIB_SUBDIR and STDLIB_LANDMARKS and executable_dir and not prefix:
+        prefix = search_up(executable_dir, *STDLIB_LANDMARKS)
+        if prefix:
+            stdlib_dir = joinpath(prefix, STDLIB_SUBDIR)
+
+    if PREFIX and not prefix:
+        prefix = PREFIX
+        if not any(isfile(joinpath(prefix, f)) for f in STDLIB_LANDMARKS):
+            warn('Could not find platform independent libraries <prefix>')
+
+    if not prefix:
+        prefix = abspath('')
+        warn('Could not find platform independent libraries <prefix>')
+
+
+    # Detect exec_prefix by searching from executable for the platstdlib_dir
+    if PLATSTDLIB_LANDMARK and not exec_prefix:
+        if executable_dir:
+            exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir)
+        if not exec_prefix:
+            if EXEC_PREFIX:
+                exec_prefix = EXEC_PREFIX
+                if not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)):
+                    warn('Could not find platform dependent libraries <exec_prefix>')
+            else:
+                warn('Could not find platform dependent libraries <exec_prefix>')
+
+    # Fallback: assume exec_prefix == prefix
+    if not exec_prefix:
+        exec_prefix = prefix
+
+
+    if not prefix or not exec_prefix:
+        warn('Consider setting $PYTHONHOME to <prefix>[:<exec_prefix>]')
+
+
+# If we haven't set [plat]stdlib_dir already, set them now
+if not stdlib_dir:
+    if prefix:
+        stdlib_dir = joinpath(prefix, STDLIB_SUBDIR)
+    else:
+        stdlib_dir = ''
+
+if not platstdlib_dir:
+    if exec_prefix:
+        platstdlib_dir = joinpath(exec_prefix, PLATSTDLIB_LANDMARK)
+    else:
+        platstdlib_dir = ''
+
+
+# For a venv, update the main prefix/exec_prefix but leave the base ones unchanged
+# XXX: We currently do not update prefix here, but it happens in site.py
+#if venv_prefix:
+#    base_prefix = prefix
+#    base_exec_prefix = exec_prefix
+#    prefix = exec_prefix = venv_prefix
+
+
+# ******************************************************************************
+# UPDATE pythonpath (sys.path)
+# ******************************************************************************
+
+if py_setpath:
+    # If Py_SetPath was called then it overrides any existing search path
+    config['module_search_paths'] = py_setpath.split(DELIM)
+    config['module_search_paths_set'] = 1
+
+elif not pythonpath:
+    # If pythonpath was already set, we leave it alone.
+    # This won't matter in normal use, but if an embedded host is trying to
+    # recalculate paths while running then we do not want to change it.
+    pythonpath = []
+
+    # First add entries from the process environment
+    if use_environment and ENV_PYTHONPATH:
+        for p in ENV_PYTHONPATH.split(DELIM):
+            pythonpath.append(abspath(p))
+
+    # Then add the default zip file
+    if os_name == 'nt':
+        # QUIRK: Windows uses the library directory rather than the prefix
+        if library:
+            library_dir = dirname(library)
+        else:
+            library_dir = executable_dir
+        pythonpath.append(joinpath(library_dir, ZIP_LANDMARK))
+    elif build_prefix or venv_prefix:
+        # QUIRK: POSIX uses the default prefix when in the build directory
+        # or a venv
+        pythonpath.append(joinpath(PREFIX, ZIP_LANDMARK))
+    else:
+        pythonpath.append(joinpath(prefix, ZIP_LANDMARK))
+
+    if os_name == 'nt' and use_environment and winreg:
+        # QUIRK: Windows also lists paths in the registry. Paths are stored
+        # as the default value of each subkey of
+        # {HKCU,HKLM}\Software\Python\PythonCore\{winver}\PythonPath
+        # where winver is sys.winver (typically '3.x' or '3.x-32')
+        for hk in (winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE):
+            try:
+                key = winreg.OpenKeyEx(hk, WINREG_KEY)
+                try:
+                    i = 0
+                    while True:
+                        try:
+                            keyname = winreg.EnumKey(key, i)
+                            subkey = winreg.OpenKeyEx(key, keyname)
+                            if not subkey:
+                                continue
+                            try:
+                                v = winreg.QueryValue(subkey)
+                            finally:
+                                winreg.CloseKey(subkey)
+                            if isinstance(v, str):
+                                pythonpath.append(v)
+                            i += 1
+                        except OSError:
+                            break
+                finally:
+                    winreg.CloseKey(key)
+            except OSError:
+                pass
+
+    # Then add any entries compiled into the PYTHONPATH macro.
+    if PYTHONPATH:
+        for p in PYTHONPATH.split(DELIM):
+            pythonpath.append(joinpath(prefix, p))
+
+    # Then add stdlib_dir and platstdlib_dir
+    if stdlib_dir:
+        pythonpath.append(stdlib_dir)
+    if platstdlib_dir:
+        if os_name == 'nt' and venv_prefix:
+            # QUIRK: Windows appends executable_dir instead of platstdlib_dir
+            # when in a venv
+            pythonpath.append(executable_dir)
+        else:
+            pythonpath.append(platstdlib_dir)
+
+    config['module_search_paths'] = pythonpath
+    config['module_search_paths_set'] = 1
+
+
+# ******************************************************************************
+# POSIX prefix/exec_prefix QUIRKS
+# ******************************************************************************
+
+# QUIRK: Non-Windows replaces prefix/exec_prefix with defaults when running
+# in build directory. This happens after pythonpath calculation.
+if os_name != 'nt' and build_prefix:
+    prefix = config.get('prefix') or PREFIX
+    exec_prefix = config.get('exec_prefix') or EXEC_PREFIX or prefix
+
+
+# ******************************************************************************
+# SET pythonpath FROM _PTH FILE
+# ******************************************************************************
+
+if pth:
+    config['isolated'] = 1
+    config['use_environment'] = 0
+    config['site_import'] = 0
+    pythonpath = []
+    for line in pth:
+        line = line.partition('#')[0].strip()
+        if not line:
+            pass
+        elif line == 'import site':
+            config['site_import'] = 1
+        elif line.startswith('import '):
+            warn("unsupported 'import' line in ._pth file")
+        else:
+            pythonpath.append(joinpath(pth_dir, line))
+    config['module_search_paths'] = pythonpath
+    config['module_search_paths_set'] = 1
+
+# ******************************************************************************
+# UPDATE config FROM CALCULATED VALUES
+# ******************************************************************************
+
+config['program_name'] = program_name
+config['home'] = home
+config['executable'] = executable
+config['base_executable'] = base_executable
+config['prefix'] = prefix
+config['exec_prefix'] = exec_prefix
+config['base_prefix'] = base_prefix or prefix
+config['base_exec_prefix'] = base_exec_prefix or exec_prefix
+
+config['platlibdir'] = platlibdir
+config['stdlib_dir'] = stdlib_dir
+config['platstdlib_dir'] = platstdlib_dir

Modules/getpath_noop.c

@@ -0,0 +1,10 @@
+/* Implements the getpath API for compiling with no functionality */
+
+#include "Python.h"
+#include "pycore_pathconfig.h"
+
+PyStatus
+_PyConfig_InitPathConfig(PyConfig *config, int compute_path_config)
+{
+    return PyStatus_Error("path configuration is unsupported");
+}

Modules/main.c

@@ -534,11 +534,16 @@ pymain_repl(PyConfig *config, int *exitcode)
 static void
 pymain_run_python(int *exitcode)
 {
+    PyObject *main_importer_path = NULL;
     PyInterpreterState *interp = _PyInterpreterState_GET();
     /* pymain_run_stdin() modify the config */
     PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(interp);
 
-    PyObject *main_importer_path = NULL;
+    /* ensure path config is written into global variables */
+    if (_PyStatus_EXCEPTION(_PyPathConfig_UpdateGlobal(config))) {
+        goto error;
+    }
+
     if (config->run_filename != NULL) {
         /* If filename is a package (ex: directory or ZIP file) which contains
            __main__.py, main_importer_path is set to filename and will be

Modules/posixmodule.c

@@ -4444,6 +4444,33 @@ os__path_splitroot_impl(PyObject *module, path_t *path)
 #endif /* MS_WINDOWS */
 
 
+/*[clinic input]
+os._path_normpath
+
+    path: object
+
+Basic path normalization.
+[clinic start generated code]*/
+
+static PyObject *
+os__path_normpath_impl(PyObject *module, PyObject *path)
+/*[clinic end generated code: output=b94d696d828019da input=5e90c39e12549dc0]*/
+{
+    if (!PyUnicode_Check(path)) {
+        PyErr_Format(PyExc_TypeError, "expected 'str', not '%.200s'",
+            Py_TYPE(path)->tp_name);
+        return NULL;
+    }
+    Py_ssize_t len;
+    wchar_t *buffer = PyUnicode_AsWideCharString(path, &len);
+    if (!buffer) {
+        return NULL;
+    }
+    PyObject *result = PyUnicode_FromWideChar(_Py_normpath(buffer, len), -1);
+    PyMem_Free(buffer);
+    return result;
+}
+
 /*[clinic input]
 os.mkdir
 
@@ -14826,6 +14853,7 @@ static PyMethodDef posix_methods[] = {
     OS__GETFINALPATHNAME_METHODDEF
     OS__GETVOLUMEPATHNAME_METHODDEF
     OS__PATH_SPLITROOT_METHODDEF
+    OS__PATH_NORMPATH_METHODDEF
     OS_GETLOADAVG_METHODDEF
     OS_URANDOM_METHODDEF
     OS_SETRESUID_METHODDEF

PC/config_minimal.c

@@ -5,6 +5,10 @@
 
 #include "Python.h"
 
+/* Define extern variables omitted from minimal builds */
+void *PyWin_DLLhModule = NULL;
+
+
 extern PyObject* PyInit_faulthandler(void);
 extern PyObject* PyInit__tracemalloc(void);
 extern PyObject* PyInit_gc(void);

PC/pyconfig.h

@@ -66,9 +66,6 @@ WIN32 is still required for the locale module.
 
 #define MS_WIN32 /* only support win32 and greater. */
 #define MS_WINDOWS
-#ifndef PYTHONPATH
-#       define PYTHONPATH L".\\DLLs;.\\lib"
-#endif
 #define NT_THREADS
 #define WITH_THREAD
 #ifndef NETSCAPE_PI
@@ -671,6 +668,4 @@ Py_NO_ENABLE_SHARED to find out.  Also support MS_NO_COREDLL for b/w compat */
 /* Define if libssl has X509_VERIFY_PARAM_set1_host and related function */
 #define HAVE_X509_VERIFY_PARAM_SET1_HOST 1
 
-#define PLATLIBDIR "lib"
-
 #endif /* !Py_CONFIG_H */

PCbuild/_freeze_module.vcxproj

@@ -104,6 +104,7 @@
     <ClCompile Include="..\Modules\faulthandler.c" />
     <ClCompile Include="..\Modules\gcmodule.c" />
     <ClCompile Include="..\Modules\getbuildinfo.c" />
+    <ClCompile Include="..\Modules\getpath_noop.c" />
     <ClCompile Include="..\Modules\posixmodule.c" />
     <ClCompile Include="..\Modules\signalmodule.c" />
     <ClCompile Include="..\Modules\_tracemalloc.c" />
@@ -168,7 +169,6 @@
     <ClCompile Include="..\Parser\string_parser.c" />
     <ClCompile Include="..\Parser\token.c" />
     <ClCompile Include="..\Parser\tokenizer.c" />
-    <ClCompile Include="..\PC\getpathp.c" />
     <ClCompile Include="..\PC\invalid_parameter_handler.c" />
     <ClCompile Include="..\PC\msvcrtmodule.c" />
     <ClCompile Include="..\PC\winreg.c" />
@@ -374,9 +374,29 @@
     </None>
     <!-- END frozen modules -->
   </ItemGroup>
+  <ItemGroup>
+    <!-- We manually freeze getpath.py rather than through freeze_modules -->
+    <GetPath Include="..\Modules\getpath.py">
+      <ModName>getpath</ModName>
+      <IntFile>$(IntDir)getpath.g.h</IntFile>
+      <OutFile>$(PySourcePath)Modules\getpath.h</OutFile>
+    </GetPath>
+  </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
+  <Target Name="_RebuildGetPath" AfterTargets="_RebuildFrozen" Condition="$(Configuration) != 'PGUpdate'">
+    <Exec Command='"$(TargetPath)" "%(GetPath.ModName)" "%(GetPath.FullPath)" "%(GetPath.IntFile)"' />
+
+    <Copy SourceFiles="%(GetPath.IntFile)"
+          DestinationFiles="%(GetPath.OutFile)"
+          Condition="!Exists(%(GetPath.OutFile)) or (Exists(%(GetPath.IntFile)) and '$([System.IO.File]::ReadAllText(%(GetPath.OutFile)).Replace(`&#x0D;&#x0A;`, `&#x0A;`))' != '$([System.IO.File]::ReadAllText(%(GetPath.IntFile)).Replace(`&#x0D;&#x0A;`, `&#x0A;`))')">
+      <Output TaskParameter="CopiedFiles" ItemName="_UpdatedGetPath" />
+    </Copy>
+
+    <Message Text="Updated files: @(_UpdatedGetPath->'%(Filename)%(Extension)',', ')"
+             Condition="'@(_UpdatedGetPath)' != ''" Importance="high" />
+  </Target>
   <Target Name="_RebuildFrozen" AfterTargets="AfterBuild" Condition="$(Configuration) != 'PGUpdate'">
     <Exec Command='"$(TargetPath)" "%(None.ModName)" "%(None.FullPath)" "%(None.DeepIntFile)"' />
 

PCbuild/_freeze_module.vcxproj.filters

@@ -13,6 +13,395 @@
     <ClCompile Include="..\Programs\_freeze_module.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\Modules\_io\_iomodule.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\_tracemalloc.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\_warnings.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\abstract.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\accu.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\asdl.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\ast.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\ast_opt.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\ast_unparse.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\atexitmodule.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\bltinmodule.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\boolobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\bootstrap_hash.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\_io\bufferedio.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\bytearrayobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\bytes_methods.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\_io\bytesio.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\bytesobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\call.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\capsule.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\cellobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\ceval.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\classobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\codecs.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\codeobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\compile.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\complexobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\PC\config_minimal.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\context.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\descrobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\dictobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\dtoa.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\dynamic_annotations.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\dynload_win.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\enumobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\errors.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\exceptions.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\faulthandler.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\_io\fileio.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\fileobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\fileutils.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\floatobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\formatter_unicode.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\frame.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\frameobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\funcobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\future.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\gcmodule.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\genericaliasobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\genobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\getargs.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\getbuildinfo.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\getcompiler.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\getcopyright.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\getopt.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\getpath_noop.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\getplatform.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\getversion.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\hamt.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\hashtable.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\import.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\importdl.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\initconfig.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\interpreteridobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\PC\invalid_parameter_handler.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\_io\iobase.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\iterobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\listobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\longobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\marshal.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\memoryobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\methodobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\modsupport.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\moduleobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\PC\msvcrtmodule.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Parser\myreadline.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\mysnprintf.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\mystrtoul.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\namespaceobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\object.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\obmalloc.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\odictobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Parser\parser.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pathconfig.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Parser\peg_api.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Parser\pegen.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\picklebufobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\posixmodule.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\preconfig.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pyarena.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pyctype.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pyfpe.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pyhash.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pylifecycle.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pymath.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pystate.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pystrcmp.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pystrhex.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pystrtod.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\Python-ast.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pythonrun.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\Python-tokenize.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pytime.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\rangeobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\setobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\signalmodule.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\sliceobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\specialize.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Parser\string_parser.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\_io\stringio.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\structmember.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\structseq.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\suggestions.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\symtable.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\sysmodule.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\_io\textio.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\thread.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Parser\token.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Parser\tokenizer.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\traceback.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\tupleobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\typeobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\unicodectype.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\unicodeobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\unionobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\weakrefobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\_io\winconsoleio.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\PC\winreg.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="..\Modules\getpath.py">
+      <Filter>Python Files</Filter>
+    </None>
   </ItemGroup>
   <ItemGroup>
     <!-- BEGIN frozen modules -->

PCbuild/python.vcxproj

@@ -147,4 +147,11 @@ $(_PGOPath)
     </PropertyGroup>
     <WriteLinesToFile File="$(PySourcePath)python.bat" Lines="$(_Content)" Overwrite="true" Condition="'$(_Content)' != '$(_ExistingContent)'" />
   </Target>
+  <Target Name="GeneratePyBuildDirTxt" BeforeTargets="AfterBuild">
+    <PropertyGroup>
+      <_Content>$(OutDir)</_Content>
+      <_ExistingContent Condition="Exists('$(OutDir)pybuilddir.txt')">$([System.IO.File]::ReadAllText('$(OutDir)pybuilddir.txt'))</_ExistingContent>
+    </PropertyGroup>
+    <WriteLinesToFile File="$(OutDir)pybuilddir.txt" Lines="$(_Content)" Overwrite="true" Condition="'$(_Content)' != '$(_ExistingContent)'" />
+  </Target>
 </Project>

PCbuild/pythoncore.vcxproj

@@ -109,6 +109,19 @@
       <AdditionalDependencies>version.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\Modules\getpath.c">
+      <PreprocessorDefinitions>
+        PREFIX=NULL;
+        EXEC_PREFIX=NULL;
+        VERSION=NULL;
+        VPATH="..\\..";
+        PYDEBUGEXT="$(PyDebugExt)";
+        PLATLIBDIR="DLLs";
+        %(PreprocessorDefinitions)
+      </PreprocessorDefinitions>
+    </ClCompile>
+  </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\Include\Python.h" />
     <ClInclude Include="..\Include\abstract.h" />
@@ -349,6 +362,7 @@
     <ClCompile Include="..\Modules\errnomodule.c" />
     <ClCompile Include="..\Modules\faulthandler.c" />
     <ClCompile Include="..\Modules\gcmodule.c" />
+    <ClCompile Include="..\Modules\getbuildinfo.c" />
     <ClCompile Include="..\Modules\itertoolsmodule.c" />
     <ClCompile Include="..\Modules\main.c" />
     <ClCompile Include="..\Modules\mathmodule.c" />
@@ -442,7 +456,6 @@
     <ClCompile Include="..\PC\invalid_parameter_handler.c" />
     <ClCompile Include="..\PC\winreg.c" />
     <ClCompile Include="..\PC\config.c" />
-    <ClCompile Include="..\PC\getpathp.c" />
     <ClCompile Include="..\PC\msvcrtmodule.c" />
     <ClCompile Include="..\Python\pyhash.c" />
     <ClCompile Include="..\Python\_warnings.c" />
@@ -570,7 +583,7 @@
     </PropertyGroup>
     <Message Text="Building $(GitTag):$(GitVersion) $(GitBranch)" Importance="high" />
     <ItemGroup>
-      <ClCompile Include="..\Modules\getbuildinfo.c">
+      <ClCompile Condition="$(Filename) == 'getbuildinfo'">
         <PreprocessorDefinitions>GITVERSION="$(GitVersion)";GITTAG="$(GitTag)";GITBRANCH="$(GitBranch)";%(PreprocessorDefinitions)</PreprocessorDefinitions>
       </ClCompile>
     </ItemGroup>
@@ -590,4 +603,8 @@
   <Target Name="_CleanVCRuntime" AfterTargets="Clean">
     <Delete Files="@(VCRuntimeDLL->'$(OutDir)%(Filename)%(Extension)')" />
   </Target>
+
+  <Target Name="_DeletePyBuildDirTxt" BeforeTargets="PrepareForBuild">
+    <Delete Files="$(OutDir)pybuilddir.txt" />
+  </Target>
 </Project>

PCbuild/pythoncore.vcxproj.filters

@@ -648,6 +648,9 @@
     <ClInclude Include="$(zlibDir)\zutil.h">
       <Filter>Modules\zlib</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\internal\pycore_structseq.h">
+      <Filter>Include\internal</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\_abc.c">
@@ -971,9 +974,6 @@
     <ClCompile Include="..\PC\dl_nt.c">
       <Filter>PC</Filter>
     </ClCompile>
-    <ClCompile Include="..\PC\getpathp.c">
-      <Filter>PC</Filter>
-    </ClCompile>
     <ClCompile Include="..\PC\msvcrtmodule.c">
       <Filter>PC</Filter>
     </ClCompile>
@@ -1229,6 +1229,21 @@
     <ClCompile Include="..\Objects\unionobject.c">
       <Filter>Objects</Filter>
     </ClCompile>
+    <ClCompile Include="..\Python\frame.c">
+      <Filter>Python</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\getpath.c">
+      <Filter>Modules</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\suggestions.c">
+      <Filter>Python</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\Python-tokenize.c">
+      <Filter>Python</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\getbuildinfo.c">
+      <Filter>Modules</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="..\PC\python_nt.rc">

Python/dynload_win.c

@@ -2,6 +2,9 @@
 /* Support for dynamic loading of extension modules */
 
 #include "Python.h"
+#include "pycore_fileutils.h"     // _Py_add_relfile()
+#include "pycore_pathconfig.h"    // _PyPathConfig_ComputeSysPath0()
+#include "pycore_pystate.h"       // _PyInterpreterState_GET()
 
 #ifdef HAVE_DIRECT_H
 #include <direct.h>
@@ -160,6 +163,60 @@ static char *GetPythonImport (HINSTANCE hModule)
     return NULL;
 }
 
+/* Load python3.dll before loading any extension module that might refer
+   to it. That way, we can be sure that always the python3.dll corresponding
+   to this python DLL is loaded, not a python3.dll that might be on the path
+   by chance.
+   Return whether the DLL was found.
+*/
+extern HMODULE PyWin_DLLhModule;
+static int
+_Py_CheckPython3(void)
+{
+    static int python3_checked = 0;
+    static HANDLE hPython3;
+    #define MAXPATHLEN 512
+    wchar_t py3path[MAXPATHLEN+1];
+    if (python3_checked) {
+        return hPython3 != NULL;
+    }
+    python3_checked = 1;
+
+    /* If there is a python3.dll next to the python3y.dll,
+       use that DLL */
+    if (PyWin_DLLhModule && GetModuleFileNameW(PyWin_DLLhModule, py3path, MAXPATHLEN)) {
+        wchar_t *p = wcsrchr(py3path, L'\\');
+        if (p) {
+            wcscpy(p + 1, PY3_DLLNAME);
+            hPython3 = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
+            if (hPython3 != NULL) {
+                return 1;
+            }
+        }
+    }
+
+    /* If we can locate python3.dll in our application dir,
+       use that DLL */
+    hPython3 = LoadLibraryExW(PY3_DLLNAME, NULL, LOAD_LIBRARY_SEARCH_APPLICATION_DIR);
+    if (hPython3 != NULL) {
+        return 1;
+    }
+
+    /* For back-compat, also search {sys.prefix}\DLLs, though
+       that has not been a normal install layout for a while */
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(interp);
+    assert(config->prefix);
+    if (config->prefix) {
+        wcscpy_s(py3path, MAXPATHLEN, config->prefix);
+        if (py3path[0] && _Py_add_relfile(py3path, L"DLLs\\" PY3_DLLNAME, MAXPATHLEN) >= 0) {
+            hPython3 = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
+        }
+    }
+    return hPython3 != NULL;
+    #undef MAXPATHLEN
+}
+
 dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix,
                                               const char *shortname,
                                               PyObject *pathname, FILE *fp)

Python/fileutils.c

@@ -2000,13 +2000,28 @@ _Py_wrealpath(const wchar_t *path,
 #endif
 
 
-#ifndef MS_WINDOWS
 int
 _Py_isabs(const wchar_t *path)
 {
+#ifdef MS_WINDOWS
+    const wchar_t *tail;
+    HRESULT hr = PathCchSkipRoot(path, &tail);
+    if (FAILED(hr) || path == tail) {
+        return 0;
+    }
+    if (tail == &path[1] && (path[0] == SEP || path[0] == ALTSEP)) {
+        // Exclude paths with leading SEP
+        return 0;
+    }
+    if (tail == &path[2] && path[1] == L':') {
+        // Exclude drive-relative paths (e.g. C:filename.ext)
+        return 0;
+    }
+    return 1;
+#else
     return (path[0] == SEP);
-}
 #endif
+}
 
 
 /* Get an absolute path.
@@ -2017,6 +2032,22 @@ _Py_isabs(const wchar_t *path)
 int
 _Py_abspath(const wchar_t *path, wchar_t **abspath_p)
 {
+    if (path[0] == '\0' || !wcscmp(path, L".")) {
+        wchar_t cwd[MAXPATHLEN + 1];
+        cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0;
+        if (!_Py_wgetcwd(cwd, Py_ARRAY_LENGTH(cwd) - 1)) {
+            /* unable to get the current directory */
+            return -1;
+        }
+        *abspath_p = _PyMem_RawWcsdup(cwd);
+        return 0;
+    }
+
+    if (_Py_isabs(path)) {
+        *abspath_p = _PyMem_RawWcsdup(path);
+        return 0;
+    }
+
 #ifdef MS_WINDOWS
     wchar_t woutbuf[MAX_PATH], *woutbufp = woutbuf;
     DWORD result;
@@ -2028,7 +2059,7 @@ _Py_abspath(const wchar_t *path, wchar_t **abspath_p)
         return -1;
     }
 
-    if (result > Py_ARRAY_LENGTH(woutbuf)) {
+    if (result >= Py_ARRAY_LENGTH(woutbuf)) {
         if ((size_t)result <= (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) {
             woutbufp = PyMem_RawMalloc((size_t)result * sizeof(wchar_t));
         }
@@ -2055,11 +2086,6 @@ _Py_abspath(const wchar_t *path, wchar_t **abspath_p)
     *abspath_p = _PyMem_RawWcsdup(woutbufp);
     return 0;
 #else
-    if (_Py_isabs(path)) {
-        *abspath_p = _PyMem_RawWcsdup(path);
-        return 0;
-    }
-
     wchar_t cwd[MAXPATHLEN + 1];
     cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0;
     if (!_Py_wgetcwd(cwd, Py_ARRAY_LENGTH(cwd) - 1)) {
@@ -2102,7 +2128,8 @@ join_relfile(wchar_t *buffer, size_t bufsize,
              const wchar_t *dirname, const wchar_t *relfile)
 {
 #ifdef MS_WINDOWS
-    if (FAILED(PathCchCombineEx(buffer, bufsize, dirname, relfile, 0))) {
+    if (FAILED(PathCchCombineEx(buffer, bufsize, dirname, relfile, 
+        PATHCCH_ALLOW_LONG_PATHS | PATHCCH_FORCE_ENABLE_LONG_NAME_PROCESS))) {
         return -1;
     }
 #else
@@ -2180,99 +2207,125 @@ _Py_find_basename(const wchar_t *filename)
     return 0;
 }
 
-
+/* In-place path normalisation. Returns the start of the normalized
-/* Remove navigation elements such as "." and "..".
+   path, which will be within the original buffer. Guaranteed to not
-
+   make the path longer, and will not fail. 'size' is the length of
-   This is mostly a C implementation of posixpath.normpath().
+   the path, if known. If -1, the first null character will be assumed
-   Return 0 on success.  Return -1 if "orig" is too big for the buffer. */
+   to be the end of the path. */
-int
+wchar_t *
-_Py_normalize_path(const wchar_t *path, wchar_t *buf, const size_t buf_len)
+_Py_normpath(wchar_t *path, Py_ssize_t size)
 {
-    assert(path && *path != L'\0');
+    if (!path[0] || size == 0) {
-    assert(*path == SEP);  // an absolute path
+        return path;
-    if (wcslen(path) + 1 >= buf_len) {
-        return -1;
     }
+    wchar_t lastC = L'\0';
+    wchar_t *p1 = path;
+    wchar_t *pEnd = size >= 0 ? &path[size] : NULL;
+    wchar_t *p2 = path;
+    wchar_t *minP2 = path;
 
-    int dots = -1;
+#define IS_END(x) (pEnd ? (x) == pEnd : !*(x))
-    int check_leading = 1;
+#ifdef ALTSEP
-    const wchar_t *buf_start = buf;
+#define IS_SEP(x) (*(x) == SEP || *(x) == ALTSEP)
-    wchar_t *buf_next = buf;
+#else
-    // The resulting filename will never be longer than path.
+#define IS_SEP(x) (*(x) == SEP)
-    for (const wchar_t *remainder = path; *remainder != L'\0'; remainder++) {
+#endif
-        wchar_t c = *remainder;
+#define SEP_OR_END(x) (IS_SEP(x) || IS_END(x))
-        buf_next[0] = c;
+
-        buf_next++;
+    // Skip leading '.\'
-        if (c == SEP) {
+    if (p1[0] == L'.' && IS_SEP(&p1[1])) {
-            assert(dots <= 2);
+        path = &path[2];
-            if (dots == 2) {
+        while (IS_SEP(path) && !IS_END(path)) {
-                // Turn "/x/y/../z" into "/x/z".
+            path++;
-                buf_next -= 4;  // "/../"
-                assert(*buf_next == SEP);
-                // We cap it off at the root, so "/../spam" becomes "/spam".
-                if (buf_next == buf_start) {
-                    buf_next++;
-                }
-                else {
-                    // Move to the previous SEP in the buffer.
-                    while (*(buf_next - 1) != SEP) {
-                        assert(buf_next != buf_start);
-                        buf_next--;
-                    }
-                }
-            }
-            else if (dots == 1) {
-                // Turn "/./" into "/".
-                buf_next -= 2;  // "./"
-                assert(*(buf_next - 1) == SEP);
-            }
-            else if (dots == 0) {
-                // Turn "//" into "/".
-                buf_next--;
-                assert(*(buf_next - 1) == SEP);
-                if (check_leading) {
-                    if (buf_next - 1 == buf && *(remainder + 1) != SEP) {
-                        // Leave a leading "//" alone, unless "///...".
-                        buf_next++;
-                        buf_start++;
-                    }
-                    check_leading = 0;
-                }
-            }
-            dots = 0;
         }
-        else {
+        p1 = p2 = minP2 = path;
-            check_leading = 0;
+        lastC = SEP;
-            if (dots >= 0) {
+    }
-                if (c == L'.' && dots < 2) {
+#ifdef MS_WINDOWS
-                    dots++;
+    // Skip past drive segment and update minP2
-                }
+    else if (p1[0] && p1[1] == L':') {
-                else {
+        *p2++ = *p1++;
-                    dots = -1;
+        *p2++ = *p1++;
-                }
+        minP2 = p2;
+        lastC = L':';
+    }
+    // Skip past all \\-prefixed paths, including \\?\, \\.\,
+    // and network paths, including the first segment.
+    else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1])) {
+        int sepCount = 2;
+        *p2++ = SEP;
+        *p2++ = SEP;
+        p1 += 2;
+        for (; !IS_END(p1) && sepCount; ++p1) {
+            if (IS_SEP(p1)) {
+                --sepCount;
+                *p2++ = lastC = SEP;
+            } else {
+                *p2++ = lastC = *p1;
             }
         }
+        minP2 = p2;
+    }
+#else
+    // Skip past two leading SEPs
+    else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1]) && !IS_SEP(&p1[2])) {
+        *p2++ = *p1++;
+        *p2++ = *p1++;
+        minP2 = p2;
+        lastC = SEP;
+    }
+#endif /* MS_WINDOWS */
+
+    /* if pEnd is specified, check that. Else, check for null terminator */
+    for (; !IS_END(p1); ++p1) {
+        wchar_t c = *p1;
+#ifdef ALTSEP
+        if (c == ALTSEP) {
+            c = SEP;
+        }
+#endif
+        if (lastC == SEP) {
+            if (c == L'.') {
+                int sep_at_1 = SEP_OR_END(&p1[1]);
+                int sep_at_2 = !sep_at_1 && SEP_OR_END(&p1[2]);
+                if (sep_at_2 && p1[1] == L'.') {
+                    wchar_t *p3 = p2;
+                    while (p3 != minP2 && *--p3 == SEP) { }
+                    while (p3 != minP2 && *(p3 - 1) != SEP) { --p3; }
+                    if (p3[0] == L'.' && p3[1] == L'.' && IS_SEP(&p3[2])) {
+                        // Previous segment is also ../, so append instead
+                        *p2++ = L'.';
+                        *p2++ = L'.';
+                        lastC = L'.';
+                    } else if (p3[0] == SEP) {
+                        // Absolute path, so absorb segment
+                        p2 = p3 + 1;
+                    } else {
+                        p2 = p3;
+                    }
+                    p1 += 1;
+                } else if (sep_at_1) {
+                } else {
+                    *p2++ = lastC = c;
+                }
+            } else if (c == SEP) {
+            } else {
+                *p2++ = lastC = c;
+            }
+        } else {
+            *p2++ = lastC = c;
+        } 
     }
-    if (dots >= 0) {
+    *p2 = L'\0';
-        // Strip any trailing dots and trailing slash.
+    if (p2 != minP2) {
-        buf_next -= dots + 1;  // "/" or "/." or "/.."
+        while (--p2 != minP2 && *p2 == SEP) {
-        assert(*buf_next == SEP);
+            *p2 = L'\0';
-        if (buf_next == buf_start) {
-            // Leave the leading slash for root.
-            buf_next++;
-        }
-        else {
-            if (dots == 2) {
-                // Move to the previous SEP in the buffer.
-                do {
-                    assert(buf_next != buf_start);
-                    buf_next--;
-                } while (*(buf_next) != SEP);
-            }
         }
     }
-    *buf_next = L'\0';
+#undef SEP_OR_END
-    return 0;
+#undef IS_SEP
+#undef IS_END
+    return path;
 }
 
 

Python/initconfig.c

@@ -22,11 +22,6 @@
 #  endif
 #endif
 
-#ifndef PLATLIBDIR
-#  error "PLATLIBDIR macro must be defined"
-#endif
-
-
 /* --- Command line options --------------------------------------- */
 
 /* Short usage message (with %s for argv0) */
@@ -632,7 +627,6 @@ config_check_consistency(const PyConfig *config)
     assert(config->parse_argv >= 0);
     assert(config->configure_c_stdio >= 0);
     assert(config->buffered_stdio >= 0);
-    assert(config->program_name != NULL);
     assert(_PyWideStringList_CheckConsistency(&config->orig_argv));
     assert(_PyWideStringList_CheckConsistency(&config->argv));
     /* sys.argv must be non-empty: empty argv is replaced with [''] */
@@ -641,7 +635,6 @@ config_check_consistency(const PyConfig *config)
     assert(_PyWideStringList_CheckConsistency(&config->warnoptions));
     assert(_PyWideStringList_CheckConsistency(&config->module_search_paths));
     assert(config->module_search_paths_set >= 0);
-    assert(config->platlibdir != NULL);
     assert(config->filesystem_encoding != NULL);
     assert(config->filesystem_errors != NULL);
     assert(config->stdio_encoding != NULL);
@@ -740,6 +733,7 @@ _PyConfig_InitCompatConfig(PyConfig *config)
     config->legacy_windows_stdio = -1;
 #endif
     config->use_frozen_modules = -1;
+    config->_is_python_build = 0;
     config->code_debug_ranges = 1;
 }
 
@@ -962,6 +956,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
     COPY_ATTR(_isolated_interpreter);
     COPY_ATTR(use_frozen_modules);
     COPY_WSTRLIST(orig_argv);
+    COPY_ATTR(_is_python_build);
 
 #undef COPY_ATTR
 #undef COPY_WSTR_ATTR
@@ -1066,6 +1061,7 @@ _PyConfig_AsDict(const PyConfig *config)
     SET_ITEM_INT(_isolated_interpreter);
     SET_ITEM_WSTRLIST(orig_argv);
     SET_ITEM_INT(use_frozen_modules);
+    SET_ITEM_INT(_is_python_build);
 
     return dict;
 
@@ -1350,6 +1346,7 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict)
     GET_UINT(_init_main);
     GET_UINT(_isolated_interpreter);
     GET_UINT(use_frozen_modules);
+    GET_UINT(_is_python_build);
 
 #undef CHECK_VALUE
 #undef GET_UINT
@@ -1489,117 +1486,6 @@ config_set_global_vars(const PyConfig *config)
 }
 
 
-/* Get the program name: use PYTHONEXECUTABLE and __PYVENV_LAUNCHER__
-   environment variables on macOS if available. */
-static PyStatus
-config_init_program_name(PyConfig *config)
-{
-    PyStatus status;
-
-    /* If Py_SetProgramName() was called, use its value */
-    const wchar_t *program_name = _Py_path_config.program_name;
-    if (program_name != NULL) {
-        config->program_name = _PyMem_RawWcsdup(program_name);
-        if (config->program_name == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
-        return _PyStatus_OK();
-    }
-
-#ifdef __APPLE__
-    /* On MacOS X, when the Python interpreter is embedded in an
-       application bundle, it gets executed by a bootstrapping script
-       that does os.execve() with an argv[0] that's different from the
-       actual Python executable. This is needed to keep the Finder happy,
-       or rather, to work around Apple's overly strict requirements of
-       the process name. However, we still need a usable sys.executable,
-       so the actual executable path is passed in an environment variable.
-       See Lib/plat-mac/bundlebuilder.py for details about the bootstrap
-       script. */
-    const char *p = config_get_env(config, "PYTHONEXECUTABLE");
-    if (p != NULL) {
-        status = CONFIG_SET_BYTES_STR(config, &config->program_name, p,
-                                      "PYTHONEXECUTABLE environment variable");
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-        return _PyStatus_OK();
-    }
-#ifdef WITH_NEXT_FRAMEWORK
-    else {
-        const char* pyvenv_launcher = getenv("__PYVENV_LAUNCHER__");
-        if (pyvenv_launcher && *pyvenv_launcher) {
-            /* Used by Mac/Tools/pythonw.c to forward
-             * the argv0 of the stub executable
-             */
-            status = CONFIG_SET_BYTES_STR(config,
-                                          &config->program_name,
-                                          pyvenv_launcher,
-                                          "__PYVENV_LAUNCHER__ environment variable");
-            if (_PyStatus_EXCEPTION(status)) {
-                return status;
-            }
-
-            /*
-             * This environment variable is used to communicate between
-             * the stub launcher and the real interpreter and isn't needed
-             * beyond this point.
-             *
-             * Clean up to avoid problems when launching other programs
-             * later on.
-             */
-            (void)unsetenv("__PYVENV_LAUNCHER__");
-
-            return _PyStatus_OK();
-        }
-    }
-#endif   /* WITH_NEXT_FRAMEWORK */
-#endif   /* __APPLE__ */
-
-    /* Use argv[0] if available and non-empty */
-    const PyWideStringList *argv = &config->argv;
-    if (argv->length >= 1 && argv->items[0][0] != L'\0') {
-        config->program_name = _PyMem_RawWcsdup(argv->items[0]);
-        if (config->program_name == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
-        return _PyStatus_OK();
-    }
-
-    /* Last fall back: hardcoded name */
-#ifdef MS_WINDOWS
-    const wchar_t *default_program_name = L"python";
-#else
-    const wchar_t *default_program_name = L"python3";
-#endif
-    status = PyConfig_SetString(config, &config->program_name,
-                                default_program_name);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-    return _PyStatus_OK();
-}
-
-static PyStatus
-config_init_executable(PyConfig *config)
-{
-    assert(config->executable == NULL);
-
-    /* If Py_SetProgramFullPath() was called, use its value */
-    const wchar_t *program_full_path = _Py_path_config.program_full_path;
-    if (program_full_path != NULL) {
-        PyStatus status = PyConfig_SetString(config,
-                                             &config->executable,
-                                             program_full_path);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-        return _PyStatus_OK();
-    }
-    return _PyStatus_OK();
-}
-
-
 static const wchar_t*
 config_get_xoption(const PyConfig *config, wchar_t *name)
 {
@@ -1618,25 +1504,6 @@ config_get_xoption_value(const PyConfig *config, wchar_t *name)
 }
 
 
-static PyStatus
-config_init_home(PyConfig *config)
-{
-    assert(config->home == NULL);
-
-    /* If Py_SetPythonHome() was called, use its value */
-    wchar_t *home = _Py_path_config.home;
-    if (home) {
-        PyStatus status = PyConfig_SetString(config, &config->home, home);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-        return _PyStatus_OK();
-    }
-
-    return CONFIG_GET_ENV_DUP(config, &config->home,
-                              L"PYTHONHOME", "PYTHONHOME");
-}
-
 static PyStatus
 config_init_hash_seed(PyConfig *config)
 {
@@ -2092,44 +1959,6 @@ config_init_fs_encoding(PyConfig *config, const PyPreConfig *preconfig)
 }
 
 
-/* Determine if the current build is a "development" build (e.g. running
-   out of the source tree) or not.
-
-   A return value of -1 indicates that we do not know.
-  */
-static int
-is_dev_env(PyConfig *config)
-{
-    // This should only ever get called early in runtime initialization,
-    // before the global path config is written.  Otherwise we would
-    // use Py_GetProgramFullPath() and _Py_GetStdlibDir().
-    assert(config != NULL);
-
-    const wchar_t *executable = config->executable;
-    const wchar_t *stdlib = config->stdlib_dir;
-    if (executable == NULL || *executable == L'\0' ||
-            stdlib == NULL || *stdlib == L'\0') {
-        // _PyPathConfig_Calculate() hasn't run yet.
-        return -1;
-    }
-    size_t len = _Py_find_basename(executable);
-    if (wcscmp(executable + len, L"python") != 0 &&
-            wcscmp(executable + len, L"python.exe") != 0) {
-        return 0;
-    }
-    /* If dirname() is the same for both then it is a dev build. */
-    if (len != _Py_find_basename(stdlib)) {
-        return 0;
-    }
-    // We do not bother normalizing the two filenames first since
-    // for config_init_import() is does the right thing as-is.
-    if (wcsncmp(stdlib, executable, len) != 0) {
-        return 0;
-    }
-    return 1;
-}
-
-
 static PyStatus
 config_init_import(PyConfig *config, int compute_path_config)
 {
@@ -2144,10 +1973,7 @@ config_init_import(PyConfig *config, int compute_path_config)
     if (config->use_frozen_modules < 0) {
         const wchar_t *value = config_get_xoption_value(config, L"frozen_modules");
         if (value == NULL) {
-            int isdev = is_dev_env(config);
+            config->use_frozen_modules = !config->_is_python_build;
-            if (isdev >= 0) {
-                config->use_frozen_modules = !isdev;
-            }
         }
         else if (wcscmp(value, L"on") == 0) {
             config->use_frozen_modules = 1;
@@ -2246,28 +2072,6 @@ config_read(PyConfig *config, int compute_path_config)
         return status;
     }
 
-    if (config->home == NULL) {
-        status = config_init_home(config);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-    }
-
-    if (config->executable == NULL) {
-        status = config_init_executable(config);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-    }
-
-    if(config->platlibdir == NULL) {
-        status = CONFIG_SET_BYTES_STR(config, &config->platlibdir, PLATLIBDIR,
-                                      "PLATLIBDIR macro");
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-    }
-
     if (config->_install_importlib) {
         status = config_init_import(config, compute_path_config);
         if (_PyStatus_EXCEPTION(status)) {
@@ -2890,13 +2694,6 @@ config_read_cmdline(PyConfig *config)
         config->parse_argv = 1;
     }
 
-    if (config->program_name == NULL) {
-        status = config_init_program_name(config);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-    }
-
     if (config->parse_argv == 1) {
         Py_ssize_t opt_index;
         status = config_parse_cmdline(config, &cmdline_warnoptions, &opt_index);
@@ -3076,7 +2873,7 @@ done:
 PyStatus
 PyConfig_Read(PyConfig *config)
 {
-    return _PyConfig_Read(config, 1);
+    return _PyConfig_Read(config, 0);
 }
 
 
@@ -3124,16 +2921,6 @@ _Py_GetConfigsAsDict(void)
     }
     Py_CLEAR(dict);
 
-    /* path config */
-    dict = _PyPathConfig_AsDict();
-    if (dict == NULL) {
-        goto error;
-    }
-    if (PyDict_SetItemString(result, "path_config", dict) < 0) {
-        goto error;
-    }
-    Py_CLEAR(dict);
-
     return result;
 
 error:
@@ -3199,6 +2986,7 @@ _Py_DumpPathConfig(PyThreadState *tstate)
     PySys_WriteStderr("  environment = %i\n", config->use_environment);
     PySys_WriteStderr("  user site = %i\n", config->user_site_directory);
     PySys_WriteStderr("  import site = %i\n", config->site_import);
+    PySys_WriteStderr("  is in build tree = %i\n", config->_is_python_build);
     DUMP_CONFIG("stdlib dir", stdlib_dir);
 #undef DUMP_CONFIG
 

Python/pathconfig.c

@@ -1,6 +1,7 @@
 /* Path configuration like module_search_path (sys.path) */
 
 #include "Python.h"
+#include "marshal.h"              // PyMarshal_ReadObjectFromString
 #include "osdefs.h"               // DELIM
 #include "pycore_initconfig.h"
 #include "pycore_fileutils.h"
@@ -9,6 +10,8 @@
 #include <wchar.h>
 #ifdef MS_WINDOWS
 #  include <windows.h>            // GetFullPathNameW(), MAX_PATH
+#  include <pathcch.h>
+#  include <shlwapi.h>
 #endif
 
 #ifdef __cplusplus
@@ -16,86 +19,36 @@ extern "C" {
 #endif
 
 
+/* External interface */
+
+/* Stored values set by C API functions */
+typedef struct _PyPathConfig {
+    /* Full path to the Python program */
+    wchar_t *program_full_path;
+    wchar_t *prefix;
+    wchar_t *exec_prefix;
+    wchar_t *stdlib_dir;
+    /* Set by Py_SetPath */
+    wchar_t *module_search_path;
+    /* Set by _PyPathConfig_UpdateGlobal */
+    wchar_t *calculated_module_search_path;
+    /* Python program name */
+    wchar_t *program_name;
+    /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */
+    wchar_t *home;
+} _PyPathConfig;
+
+#  define _PyPathConfig_INIT \
+      {.module_search_path = NULL}
+
+
 _PyPathConfig _Py_path_config = _PyPathConfig_INIT;
 
 
-static int
+const wchar_t *
-copy_wstr(wchar_t **dst, const wchar_t *src)
+_PyPathConfig_GetGlobalModuleSearchPath(void)
 {
-    assert(*dst == NULL);
+    return _Py_path_config.module_search_path;
-    if (src != NULL) {
-        *dst = _PyMem_RawWcsdup(src);
-        if (*dst == NULL) {
-            return -1;
-        }
-    }
-    else {
-        *dst = NULL;
-    }
-    return 0;
-}
-
-
-static void
-pathconfig_clear(_PyPathConfig *config)
-{
-    /* _PyMem_SetDefaultAllocator() is needed to get a known memory allocator,
-       since Py_SetPath(), Py_SetPythonHome() and Py_SetProgramName() can be
-       called before Py_Initialize() which can changes the memory allocator. */
-    PyMemAllocatorEx old_alloc;
-    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
-
-#define CLEAR(ATTR) \
-    do { \
-        PyMem_RawFree(ATTR); \
-        ATTR = NULL; \
-    } while (0)
-
-    CLEAR(config->program_full_path);
-    CLEAR(config->prefix);
-    CLEAR(config->exec_prefix);
-    CLEAR(config->stdlib_dir);
-    CLEAR(config->module_search_path);
-    CLEAR(config->program_name);
-    CLEAR(config->home);
-#ifdef MS_WINDOWS
-    CLEAR(config->base_executable);
-#endif
-
-#undef CLEAR
-
-    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
-}
-
-
-static PyStatus
-pathconfig_copy(_PyPathConfig *config, const _PyPathConfig *config2)
-{
-    pathconfig_clear(config);
-
-#define COPY_ATTR(ATTR) \
-    do { \
-        if (copy_wstr(&config->ATTR, config2->ATTR) < 0) { \
-            return _PyStatus_NO_MEMORY(); \
-        } \
-    } while (0)
-
-    COPY_ATTR(program_full_path);
-    COPY_ATTR(prefix);
-    COPY_ATTR(exec_prefix);
-    COPY_ATTR(module_search_path);
-    COPY_ATTR(stdlib_dir);
-    COPY_ATTR(program_name);
-    COPY_ATTR(home);
-#ifdef MS_WINDOWS
-    config->isolated = config2->isolated;
-    config->site_import = config2->site_import;
-    COPY_ATTR(base_executable);
-#endif
-
-#undef COPY_ATTR
-
-    return _PyStatus_OK();
 }
 
 
@@ -105,374 +58,132 @@ _PyPathConfig_ClearGlobal(void)
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    pathconfig_clear(&_Py_path_config);
+#define CLEAR(ATTR) \
+    do { \
+        PyMem_RawFree(_Py_path_config.ATTR); \
+        _Py_path_config.ATTR = NULL; \
+    } while (0)
+
+    CLEAR(program_full_path);
+    CLEAR(prefix);
+    CLEAR(exec_prefix);
+    CLEAR(stdlib_dir);
+    CLEAR(module_search_path);
+    CLEAR(calculated_module_search_path);
+    CLEAR(program_name);
+    CLEAR(home);
+
+#undef CLEAR
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 }
 
-
+PyStatus
-static wchar_t*
+_PyPathConfig_ReadGlobal(PyConfig *config)
-_PyWideStringList_Join(const PyWideStringList *list, wchar_t sep)
 {
-    size_t len = 1;   /* NUL terminator */
+    PyStatus status = _PyStatus_OK();
-    for (Py_ssize_t i=0; i < list->length; i++) {
-        if (i != 0) {
-            len++;
-        }
-        len += wcslen(list->items[i]);
-    }
 
-    wchar_t *text = PyMem_RawMalloc(len * sizeof(wchar_t));
+#define COPY(ATTR) \
-    if (text == NULL) {
+    do { \
-        return NULL;
+        if (_Py_path_config.ATTR && !config->ATTR) { \
-    }
+            status = PyConfig_SetString(config, &config->ATTR, _Py_path_config.ATTR); \
-    wchar_t *str = text;
+            if (_PyStatus_EXCEPTION(status)) goto done; \
-    for (Py_ssize_t i=0; i < list->length; i++) {
+        } \
-        wchar_t *path = list->items[i];
+    } while (0)
-        if (i != 0) {
-            *str++ = sep;
-        }
-        len = wcslen(path);
-        memcpy(str, path, len * sizeof(wchar_t));
-        str += len;
-    }
-    *str = L'\0';
 
-    return text;
+#define COPY2(ATTR, SRCATTR) \
+    do { \
+        if (_Py_path_config.SRCATTR && !config->ATTR) { \
+            status = PyConfig_SetString(config, &config->ATTR, _Py_path_config.SRCATTR); \
+            if (_PyStatus_EXCEPTION(status)) goto done; \
+        } \
+    } while (0)
+
+    COPY(prefix);
+    COPY(exec_prefix);
+    COPY(stdlib_dir);
+    COPY(program_name);
+    COPY(home);
+    COPY2(executable, program_full_path);
+    // module_search_path must be initialised - not read
+#undef COPY
+#undef COPY2
+
+done:
+    return status;
 }
 
-
+PyStatus
-static PyStatus
+_PyPathConfig_UpdateGlobal(const PyConfig *config)
-pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config)
 {
-    PyStatus status;
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    if (config->module_search_paths_set) {
+#define COPY(ATTR) \
-        PyMem_RawFree(pathconfig->module_search_path);
+    do { \
-        pathconfig->module_search_path = _PyWideStringList_Join(&config->module_search_paths, DELIM);
+        if (config->ATTR) { \
-        if (pathconfig->module_search_path == NULL) {
+            PyMem_RawFree(_Py_path_config.ATTR); \
-            goto no_memory;
+            _Py_path_config.ATTR = _PyMem_RawWcsdup(config->ATTR); \
-        }
+            if (!_Py_path_config.ATTR) goto error; \
-    }
+        } \
+    } while (0)
 
-#define COPY_CONFIG(PATH_ATTR, CONFIG_ATTR) \
+#define COPY2(ATTR, SRCATTR) \
-        if (config->CONFIG_ATTR) { \
+    do { \
-            PyMem_RawFree(pathconfig->PATH_ATTR); \
+        if (config->SRCATTR) { \
-            pathconfig->PATH_ATTR = NULL; \
+            PyMem_RawFree(_Py_path_config.ATTR); \
-            if (copy_wstr(&pathconfig->PATH_ATTR, config->CONFIG_ATTR) < 0) { \
+            _Py_path_config.ATTR = _PyMem_RawWcsdup(config->SRCATTR); \
-                goto no_memory; \
+            if (!_Py_path_config.ATTR) goto error; \
-            } \
+        } \
+    } while (0)
+
+    COPY(prefix);
+    COPY(exec_prefix);
+    COPY(stdlib_dir);
+    COPY(program_name);
+    COPY(home);
+    COPY2(program_full_path, executable);
+#undef COPY
+#undef COPY2
+
+    PyMem_RawFree(_Py_path_config.module_search_path);
+    _Py_path_config.module_search_path = NULL;
+    PyMem_RawFree(_Py_path_config.calculated_module_search_path);
+    _Py_path_config.calculated_module_search_path = NULL;
+
+    do {
+        size_t cch = 1;
+        for (Py_ssize_t i = 0; i < config->module_search_paths.length; ++i) {
+            cch += 1 + wcslen(config->module_search_paths.items[i]);
         }
 
-    COPY_CONFIG(program_full_path, executable);
+        wchar_t *path = (wchar_t*)PyMem_RawMalloc(sizeof(wchar_t) * cch);
-    COPY_CONFIG(prefix, prefix);
+        if (!path) {
-    COPY_CONFIG(exec_prefix, exec_prefix);
+            goto error;
-    COPY_CONFIG(stdlib_dir, stdlib_dir);
+        }
-    COPY_CONFIG(program_name, program_name);
+        wchar_t *p = path;
-    COPY_CONFIG(home, home);
+        for (Py_ssize_t i = 0; i < config->module_search_paths.length; ++i) {
-#ifdef MS_WINDOWS
+            wcscpy(p, config->module_search_paths.items[i]);
-    COPY_CONFIG(base_executable, base_executable);
+            p = wcschr(p, L'\0');
-#endif
+            *p++ = DELIM;
+            *p = L'\0';
+        }
 
-#undef COPY_CONFIG
+        do {
+            *p = L'\0';
+        } while (p != path && *--p == DELIM);
+        _Py_path_config.calculated_module_search_path = path;
+    } while (0);
 
-    status = _PyStatus_OK();
-    goto done;
-
-no_memory:
-    status = _PyStatus_NO_MEMORY();
-
-done:
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
-    return status;
-}
-
-PyObject *
-_PyPathConfig_AsDict(void)
-{
-    PyObject *dict = PyDict_New();
-    if (dict == NULL) {
-        return NULL;
-    }
-
-#define SET_ITEM(KEY, EXPR) \
-        do { \
-            PyObject *obj = (EXPR); \
-            if (obj == NULL) { \
-                goto fail; \
-            } \
-            int res = PyDict_SetItemString(dict, KEY, obj); \
-            Py_DECREF(obj); \
-            if (res < 0) { \
-                goto fail; \
-            } \
-        } while (0)
-#define SET_ITEM_STR(KEY) \
-        SET_ITEM(#KEY, \
-            (_Py_path_config.KEY \
-             ? PyUnicode_FromWideChar(_Py_path_config.KEY, -1) \
-             : (Py_INCREF(Py_None), Py_None)))
-#define SET_ITEM_INT(KEY) \
-        SET_ITEM(#KEY, PyLong_FromLong(_Py_path_config.KEY))
-
-    SET_ITEM_STR(program_full_path);
-    SET_ITEM_STR(prefix);
-    SET_ITEM_STR(exec_prefix);
-    SET_ITEM_STR(module_search_path);
-    SET_ITEM_STR(stdlib_dir);
-    SET_ITEM_STR(program_name);
-    SET_ITEM_STR(home);
-#ifdef MS_WINDOWS
-    SET_ITEM_INT(isolated);
-    SET_ITEM_INT(site_import);
-    SET_ITEM_STR(base_executable);
-
-    {
-        wchar_t py3path[MAX_PATH];
-        HMODULE hPython3 = GetModuleHandleW(PY3_DLLNAME);
-        PyObject *obj;
-        if (hPython3
-            && GetModuleFileNameW(hPython3, py3path, Py_ARRAY_LENGTH(py3path)))
-        {
-            obj = PyUnicode_FromWideChar(py3path, -1);
-            if (obj == NULL) {
-                goto fail;
-            }
-        }
-        else {
-            obj = Py_None;
-            Py_INCREF(obj);
-        }
-        if (PyDict_SetItemString(dict, "python3_dll", obj) < 0) {
-            Py_DECREF(obj);
-            goto fail;
-        }
-        Py_DECREF(obj);
-    }
-#endif
-
-#undef SET_ITEM
-#undef SET_ITEM_STR
-#undef SET_ITEM_INT
-
-    return dict;
-
-fail:
-    Py_DECREF(dict);
-    return NULL;
-}
-
-
-PyStatus
-_PyConfig_WritePathConfig(const PyConfig *config)
-{
-    return pathconfig_set_from_config(&_Py_path_config, config);
-}
-
-
-static PyStatus
-config_init_module_search_paths(PyConfig *config, _PyPathConfig *pathconfig)
-{
-    assert(!config->module_search_paths_set);
-
-    _PyWideStringList_Clear(&config->module_search_paths);
-
-    const wchar_t *sys_path = pathconfig->module_search_path;
-    const wchar_t delim = DELIM;
-    while (1) {
-        const wchar_t *p = wcschr(sys_path, delim);
-        if (p == NULL) {
-            p = sys_path + wcslen(sys_path); /* End of string */
-        }
-
-        size_t path_len = (p - sys_path);
-        wchar_t *path = PyMem_RawMalloc((path_len + 1) * sizeof(wchar_t));
-        if (path == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
-        memcpy(path, sys_path, path_len * sizeof(wchar_t));
-        path[path_len] = L'\0';
-
-        PyStatus status = PyWideStringList_Append(&config->module_search_paths, path);
-        PyMem_RawFree(path);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-
-        if (*p == '\0') {
-            break;
-        }
-        sys_path = p + 1;
-    }
-    config->module_search_paths_set = 1;
     return _PyStatus_OK();
-}
 
-
+error:
-/* Calculate the path configuration:
-
-   - exec_prefix
-   - module_search_path
-   - stdlib_dir
-   - prefix
-   - program_full_path
-
-   On Windows, more fields are calculated:
-
-   - base_executable
-   - isolated
-   - site_import
-
-   On other platforms, isolated and site_import are left unchanged, and
-   _PyConfig_InitPathConfig() copies executable to base_executable (if it's not
-   set).
-
-   Priority, highest to lowest:
-
-   - PyConfig
-   - _Py_path_config: set by Py_SetPath(), Py_SetPythonHome()
-     and Py_SetProgramName()
-   - _PyPathConfig_Calculate()
-*/
-static PyStatus
-pathconfig_init(_PyPathConfig *pathconfig, const PyConfig *config,
-                int compute_path_config)
-{
-    PyStatus status;
-
-    PyMemAllocatorEx old_alloc;
-    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
-
-    status = pathconfig_copy(pathconfig, &_Py_path_config);
-    if (_PyStatus_EXCEPTION(status)) {
-        goto done;
-    }
-
-    status = pathconfig_set_from_config(pathconfig, config);
-    if (_PyStatus_EXCEPTION(status)) {
-        goto done;
-    }
-
-    if (compute_path_config) {
-        status = _PyPathConfig_Calculate(pathconfig, config);
-    }
-
-done:
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
-    return status;
+    return _PyStatus_NO_MEMORY();
 }
 
 
-static PyStatus
-config_init_pathconfig(PyConfig *config, int compute_path_config)
-{
-    _PyPathConfig pathconfig = _PyPathConfig_INIT;
-    PyStatus status;
-
-    status = pathconfig_init(&pathconfig, config, compute_path_config);
-    if (_PyStatus_EXCEPTION(status)) {
-        goto done;
-    }
-
-    if (!config->module_search_paths_set
-        && pathconfig.module_search_path != NULL)
-    {
-        status = config_init_module_search_paths(config, &pathconfig);
-        if (_PyStatus_EXCEPTION(status)) {
-            goto done;
-        }
-    }
-
-#define COPY_ATTR(PATH_ATTR, CONFIG_ATTR) \
-        if (config->CONFIG_ATTR == NULL && pathconfig.PATH_ATTR != NULL) { \
-            if (copy_wstr(&config->CONFIG_ATTR, pathconfig.PATH_ATTR) < 0) { \
-                goto no_memory; \
-            } \
-        }
-
-#ifdef MS_WINDOWS
-    if (config->executable != NULL && config->base_executable == NULL) {
-        /* If executable is set explicitly in the configuration,
-           ignore calculated base_executable: _PyConfig_InitPathConfig()
-           will copy executable to base_executable */
-    }
-    else {
-        COPY_ATTR(base_executable, base_executable);
-    }
-#endif
-
-    COPY_ATTR(program_full_path, executable);
-    COPY_ATTR(prefix, prefix);
-    COPY_ATTR(exec_prefix, exec_prefix);
-    COPY_ATTR(stdlib_dir, stdlib_dir);
-
-#undef COPY_ATTR
-
-#ifdef MS_WINDOWS
-    /* If a ._pth file is found: isolated and site_import are overridden */
-    if (pathconfig.isolated != -1) {
-        config->isolated = pathconfig.isolated;
-    }
-    if (pathconfig.site_import != -1) {
-        config->site_import = pathconfig.site_import;
-    }
-#endif
-
-    status = _PyStatus_OK();
-    goto done;
-
-no_memory:
-    status = _PyStatus_NO_MEMORY();
-
-done:
-    pathconfig_clear(&pathconfig);
-    return status;
-}
-
-
-PyStatus
-_PyConfig_InitPathConfig(PyConfig *config, int compute_path_config)
-{
-    /* Do we need to calculate the path? */
-    if (!config->module_search_paths_set
-        || config->executable == NULL
-        || config->prefix == NULL
-        || config->exec_prefix == NULL)
-    {
-        PyStatus status = config_init_pathconfig(config, compute_path_config);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-    }
-
-    if (config->base_prefix == NULL && config->prefix != NULL) {
-        if (copy_wstr(&config->base_prefix, config->prefix) < 0) {
-            return _PyStatus_NO_MEMORY();
-        }
-    }
-
-    if (config->base_exec_prefix == NULL && config->exec_prefix != NULL) {
-        if (copy_wstr(&config->base_exec_prefix,
-                      config->exec_prefix) < 0) {
-            return _PyStatus_NO_MEMORY();
-        }
-    }
-
-    if (config->base_executable == NULL && config->executable != NULL) {
-        if (copy_wstr(&config->base_executable,
-                      config->executable) < 0) {
-            return _PyStatus_NO_MEMORY();
-        }
-    }
-
-    return _PyStatus_OK();
-}
-
-
-/* External interface */
-
 static void _Py_NO_RETURN
 path_out_of_memory(const char *func)
 {
@@ -483,7 +194,7 @@ void
 Py_SetPath(const wchar_t *path)
 {
     if (path == NULL) {
-        pathconfig_clear(&_Py_path_config);
+        _PyPathConfig_ClearGlobal();
         return;
     }
 
@@ -494,6 +205,7 @@ Py_SetPath(const wchar_t *path)
     PyMem_RawFree(_Py_path_config.exec_prefix);
     PyMem_RawFree(_Py_path_config.stdlib_dir);
     PyMem_RawFree(_Py_path_config.module_search_path);
+    PyMem_RawFree(_Py_path_config.calculated_module_search_path);
 
     _Py_path_config.prefix = _PyMem_RawWcsdup(L"");
     _Py_path_config.exec_prefix = _PyMem_RawWcsdup(L"");
@@ -505,6 +217,7 @@ Py_SetPath(const wchar_t *path)
         _Py_path_config.stdlib_dir = _PyMem_RawWcsdup(L"");
     }
     _Py_path_config.module_search_path = _PyMem_RawWcsdup(path);
+    _Py_path_config.calculated_module_search_path = NULL;
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
@@ -521,19 +234,19 @@ Py_SetPath(const wchar_t *path)
 void
 Py_SetPythonHome(const wchar_t *home)
 {
-    if (home == NULL) {
+    int has_value = home && home[0];
-        return;
-    }
 
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
     PyMem_RawFree(_Py_path_config.home);
-    _Py_path_config.home = _PyMem_RawWcsdup(home);
+    if (has_value) {
+        _Py_path_config.home = _PyMem_RawWcsdup(home);
+    }
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    if (_Py_path_config.home == NULL) {
+    if (has_value && _Py_path_config.home == NULL) {
         path_out_of_memory(__func__);
     }
 }
@@ -542,19 +255,19 @@ Py_SetPythonHome(const wchar_t *home)
 void
 Py_SetProgramName(const wchar_t *program_name)
 {
-    if (program_name == NULL || program_name[0] == L'\0') {
+    int has_value = program_name && program_name[0];
-        return;
-    }
 
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
     PyMem_RawFree(_Py_path_config.program_name);
-    _Py_path_config.program_name = _PyMem_RawWcsdup(program_name);
+    if (has_value) {
+        _Py_path_config.program_name = _PyMem_RawWcsdup(program_name);
+    }
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    if (_Py_path_config.program_name == NULL) {
+    if (has_value && _Py_path_config.program_name == NULL) {
         path_out_of_memory(__func__);
     }
 }
@@ -562,19 +275,19 @@ Py_SetProgramName(const wchar_t *program_name)
 void
 _Py_SetProgramFullPath(const wchar_t *program_full_path)
 {
-    if (program_full_path == NULL || program_full_path[0] == L'\0') {
+    int has_value = program_full_path && program_full_path[0];
-        return;
-    }
 
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
     PyMem_RawFree(_Py_path_config.program_full_path);
-    _Py_path_config.program_full_path = _PyMem_RawWcsdup(program_full_path);
+    if (has_value) {
+        _Py_path_config.program_full_path = _PyMem_RawWcsdup(program_full_path);
+    }
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    if (_Py_path_config.program_full_path == NULL) {
+    if (has_value && _Py_path_config.program_full_path == NULL) {
         path_out_of_memory(__func__);
     }
 }
@@ -583,7 +296,12 @@ _Py_SetProgramFullPath(const wchar_t *program_full_path)
 wchar_t *
 Py_GetPath(void)
 {
-    return _Py_path_config.module_search_path;
+    /* If the user has provided a path, return that */
+    if (_Py_path_config.module_search_path) {
+        return _Py_path_config.module_search_path;
+    }
+    /* If we have already done calculations, return the calculated path */
+    return _Py_path_config.calculated_module_search_path;
 }
 
 
@@ -632,6 +350,8 @@ Py_GetProgramName(void)
     return _Py_path_config.program_name;
 }
 
+
+
 /* Compute module search path from argv[0] or the current working
    directory ("-m module" case) which will be prepended to sys.argv:
    sys.path[0].
@@ -772,73 +492,6 @@ _PyPathConfig_ComputeSysPath0(const PyWideStringList *argv, PyObject **path0_p)
 }
 
 
-#ifdef MS_WINDOWS
-#define WCSTOK wcstok_s
-#else
-#define WCSTOK wcstok
-#endif
-
-/* Search for a prefix value in an environment file (pyvenv.cfg).
-
-   - If found, copy it into *value_p: string which must be freed by
-     PyMem_RawFree().
-   - If not found, *value_p is set to NULL.
-*/
-PyStatus
-_Py_FindEnvConfigValue(FILE *env_file, const wchar_t *key,
-                       wchar_t **value_p)
-{
-    *value_p = NULL;
-
-    char buffer[MAXPATHLEN * 2 + 1];  /* allow extra for key, '=', etc. */
-    buffer[Py_ARRAY_LENGTH(buffer)-1] = '\0';
-
-    while (!feof(env_file)) {
-        char * p = fgets(buffer, Py_ARRAY_LENGTH(buffer) - 1, env_file);
-
-        if (p == NULL) {
-            break;
-        }
-
-        size_t n = strlen(p);
-        if (p[n - 1] != '\n') {
-            /* line has overflowed - bail */
-            break;
-        }
-        if (p[0] == '#') {
-            /* Comment - skip */
-            continue;
-        }
-
-        wchar_t *tmpbuffer = _Py_DecodeUTF8_surrogateescape(buffer, n, NULL);
-        if (tmpbuffer) {
-            wchar_t * state;
-            wchar_t * tok = WCSTOK(tmpbuffer, L" \t\r\n", &state);
-            if ((tok != NULL) && !wcscmp(tok, key)) {
-                tok = WCSTOK(NULL, L" \t", &state);
-                if ((tok != NULL) && !wcscmp(tok, L"=")) {
-                    tok = WCSTOK(NULL, L"\r\n", &state);
-                    if (tok != NULL) {
-                        *value_p = _PyMem_RawWcsdup(tok);
-                        PyMem_RawFree(tmpbuffer);
-
-                        if (*value_p == NULL) {
-                            return _PyStatus_NO_MEMORY();
-                        }
-
-                        /* found */
-                        return _PyStatus_OK();
-                    }
-                }
-            }
-            PyMem_RawFree(tmpbuffer);
-        }
-    }
-
-    /* not found */
-    return _PyStatus_OK();
-}
-
 #ifdef __cplusplus
 }
 #endif

Python/pylifecycle.c

@@ -459,7 +459,7 @@ interpreter_update_config(PyThreadState *tstate, int only_update_path_config)
     }
 
     if (_Py_IsMainInterpreter(tstate->interp)) {
-        PyStatus status = _PyConfig_WritePathConfig(config);
+        PyStatus status = _PyPathConfig_UpdateGlobal(config);
         if (_PyStatus_EXCEPTION(status)) {
             _PyErr_SetFromPyStatus(status);
             return -1;
@@ -488,7 +488,7 @@ _PyInterpreterState_SetConfig(const PyConfig *src_config)
         goto done;
     }
 
-    status = PyConfig_Read(&config);
+    status = _PyConfig_Read(&config, 1);
     if (_PyStatus_EXCEPTION(status)) {
         _PyErr_SetFromPyStatus(status);
         goto done;
@@ -549,7 +549,7 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime,
     config = _PyInterpreterState_GetConfig(interp);
 
     if (config->_install_importlib) {
-        status = _PyConfig_WritePathConfig(config);
+        status = _PyPathConfig_UpdateGlobal(config);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }