diff --git a/.gitignore b/.gitignore index 0363244bdaf..7e80446d4d4 100644 --- a/.gitignore +++ b/.gitignore @@ -74,6 +74,7 @@ Mac/pythonw Misc/python.pc Misc/python-embed.pc Misc/python-config.sh +Modules/getpath.h Modules/Setup.config Modules/Setup.local Modules/Setup.stdlib diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 3642a8bf7fb..7c8fb7b1dee 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -479,6 +479,9 @@ PyConfig Fields which are already initialized are left unchanged. + Fields for :ref:`path configuration ` are no longer + calculated or modified when calling this function, as of Python 3.11. + The :c:func:`PyConfig_Read` function only parses :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments are parsed. Since Python arguments are @@ -493,6 +496,12 @@ PyConfig parsed, and arguments are only parsed if :c:member:`PyConfig.parse_argv` equals ``1``. + .. versionchanged:: 3.11 + :c:func:`PyConfig_Read` no longer calculates all paths, and so fields + listed under :ref:`Python Path Configuration ` may + no longer be updated until :c:func:`Py_InitializeFromConfig` is + called. + .. c:function:: void PyConfig_Clear(PyConfig *config) Release configuration memory. @@ -848,12 +857,19 @@ PyConfig Default: value of the ``PLATLIBDIR`` macro which is set by the :option:`configure --with-platlibdir option <--with-platlibdir>` - (default: ``"lib"``). + (default: ``"lib"``, or ``"DLLs"`` on Windows). Part of the :ref:`Python Path Configuration ` input. .. versionadded:: 3.9 + .. versionchanged:: 3.11 + This macro is now used on Windows to locate the standard + library extension modules, typically under ``DLLs``. However, + for compatibility, note that this value is ignored for any + non-standard layouts, including in-tree builds and virtual + environments. + .. c:member:: wchar_t* pythonpath_env Module search paths (:data:`sys.path`) as a string separated by ``DELIM`` @@ -870,9 +886,9 @@ PyConfig Module search paths: :data:`sys.path`. - If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, the - function calculating the :ref:`Python Path Configuration ` - overrides the :c:member:`~PyConfig.module_search_paths` and sets + If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, + :c:func:`Py_InitializeFromConfig` will replace + :c:member:`~PyConfig.module_search_paths` and sets :c:member:`~PyConfig.module_search_paths_set` to ``1``. Default: empty list (``module_search_paths``) and ``0`` @@ -944,16 +960,16 @@ PyConfig .. c:member:: int pathconfig_warnings - On Unix, if non-zero, calculating the :ref:`Python Path Configuration - ` can log warnings into ``stderr``. If equals to 0, - suppress these warnings. - - It has no effect on Windows. + If non-zero, calculation of path configuration is allowed to log + warnings into ``stderr``. If equals to 0, suppress these warnings. Default: ``1`` in Python mode, ``0`` in isolated mode. Part of the :ref:`Python Path Configuration ` input. + .. versionchanged:: 3.11 + Now also applies on Windows. + .. c:member:: wchar_t* prefix The site-specific directory prefix where the platform independent Python @@ -1305,10 +1321,9 @@ variables, command line arguments (:c:member:`PyConfig.argv` is not parsed) and user site directory. The C standard streams (ex: ``stdout``) and the LC_CTYPE locale are left unchanged. Signal handlers are not installed. -Configuration files are still used with this configuration. Set the -:ref:`Python Path Configuration ` ("output fields") to ignore these -configuration files and avoid the function computing the default path -configuration. +Configuration files are still used with this configuration to determine +paths that are unspecified. Ensure :c:member:`PyConfig.home` is specified +to avoid computing the default path configuration. .. _init-python-config: diff --git a/Include/cpython/fileutils.h b/Include/cpython/fileutils.h index ccf37e9468d..7ab2290184a 100644 --- a/Include/cpython/fileutils.h +++ b/Include/cpython/fileutils.h @@ -135,12 +135,6 @@ PyAPI_FUNC(wchar_t*) _Py_wrealpath( size_t resolved_path_len); #endif -#ifndef MS_WINDOWS -PyAPI_FUNC(int) _Py_isabs(const wchar_t *path); -#endif - -PyAPI_FUNC(int) _Py_abspath(const wchar_t *path, wchar_t **abspath_p); - PyAPI_FUNC(wchar_t*) _Py_wgetcwd( wchar_t *buf, /* Number of characters of 'buf' buffer diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 2be4c25ec0b..2ba1224d9b0 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -213,6 +213,9 @@ typedef struct PyConfig { // If non-zero, disallow threads, subprocesses, and fork. // Default: 0. int _isolated_interpreter; + + // If non-zero, we believe we're running from a source tree. + int _is_python_build; } PyConfig; PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config); diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index d1caf9c2372..38df73b745b 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -74,14 +74,15 @@ extern int _Py_EncodeNonUnicodeWchar_InPlace( Py_ssize_t size); #endif +extern int _Py_isabs(const wchar_t *path); +extern int _Py_abspath(const wchar_t *path, wchar_t **abspath_p); extern wchar_t * _Py_join_relfile(const wchar_t *dirname, const wchar_t *relfile); extern int _Py_add_relfile(wchar_t *dirname, const wchar_t *relfile, size_t bufsize); extern size_t _Py_find_basename(const wchar_t *filename); -PyAPI_FUNC(int) _Py_normalize_path(const wchar_t *path, - wchar_t *buf, const size_t buf_len); +PyAPI_FUNC(wchar_t *) _Py_normpath(wchar_t *path, Py_ssize_t size); // Macros to protect CRT calls against instant termination when passed an diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 9014fcd41d8..be545f495dc 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -166,6 +166,10 @@ extern PyStatus _PyConfig_SetPyArgv( PyAPI_FUNC(PyObject*) _PyConfig_AsDict(const PyConfig *config); PyAPI_FUNC(int) _PyConfig_FromDict(PyConfig *config, PyObject *dict); +extern void _Py_DumpPathConfig(PyThreadState *tstate); + +PyAPI_FUNC(PyObject*) _Py_Get_Getpath_CodeObject(); + /* --- Function used for testing ---------------------------------- */ diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index a258aab2397..b8deaa0c3eb 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -8,65 +8,15 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -typedef struct _PyPathConfig { - /* Full path to the Python program */ - wchar_t *program_full_path; - wchar_t *prefix; - wchar_t *exec_prefix; - wchar_t *stdlib_dir; - /* Set by Py_SetPath(), or computed by _PyConfig_InitPathConfig() */ - wchar_t *module_search_path; - /* Python program name */ - wchar_t *program_name; - /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */ - wchar_t *home; -#ifdef MS_WINDOWS - /* isolated and site_import are used to set Py_IsolatedFlag and - Py_NoSiteFlag flags on Windows in read_pth_file(). These fields - are ignored when their value are equal to -1 (unset). */ - int isolated; - int site_import; - /* Set when a venv is detected */ - wchar_t *base_executable; -#endif -} _PyPathConfig; +PyAPI_FUNC(void) _PyPathConfig_ClearGlobal(void); +extern PyStatus _PyPathConfig_ReadGlobal(PyConfig *config); +extern PyStatus _PyPathConfig_UpdateGlobal(const PyConfig *config); +extern const wchar_t * _PyPathConfig_GetGlobalModuleSearchPath(void); -#ifdef MS_WINDOWS -# define _PyPathConfig_INIT \ - {.module_search_path = NULL, \ - .isolated = -1, \ - .site_import = -1} -#else -# define _PyPathConfig_INIT \ - {.module_search_path = NULL} -#endif -/* Note: _PyPathConfig_INIT sets other fields to 0/NULL */ - -PyAPI_DATA(_PyPathConfig) _Py_path_config; -#ifdef MS_WINDOWS -PyAPI_DATA(wchar_t*) _Py_dll_path; -#endif - -extern void _PyPathConfig_ClearGlobal(void); - -extern PyStatus _PyPathConfig_Calculate( - _PyPathConfig *pathconfig, - const PyConfig *config); extern int _PyPathConfig_ComputeSysPath0( const PyWideStringList *argv, PyObject **path0); -extern PyStatus _Py_FindEnvConfigValue( - FILE *env_file, - const wchar_t *key, - wchar_t **value_p); -#ifdef MS_WINDOWS -extern wchar_t* _Py_GetDLLPath(void); -#endif - -extern PyStatus _PyConfig_WritePathConfig(const PyConfig *config); -extern void _Py_DumpPathConfig(PyThreadState *tstate); -extern PyObject* _PyPathConfig_AsDict(void); #ifdef __cplusplus } diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 527c7ae1938..58483a0c0a9 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -70,7 +70,7 @@ def isabs(s): if s.replace('/', '\\').startswith('\\\\?\\'): return True s = splitdrive(s)[1] - return len(s) > 0 and s[0] in _get_bothseps(s) + return len(s) > 0 and s[0] and s[0] in _get_bothseps(s) # Join two (or more) paths. @@ -268,11 +268,13 @@ def ismount(path): root, rest = splitdrive(path) if root and root[0] in seps: return (not rest) or (rest in seps) - if rest in seps: + if rest and rest in seps: return True if _getvolumepathname: - return path.rstrip(seps) == _getvolumepathname(path).rstrip(seps) + x = path.rstrip(seps) + y =_getvolumepathname(path).rstrip(seps) + return x.casefold() == y.casefold() else: return False @@ -459,56 +461,68 @@ def expandvars(path): # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B. # Previously, this function also truncated pathnames to 8+3 format, # but as this module is called "ntpath", that's obviously wrong! +try: + from nt import _path_normpath -def normpath(path): - """Normalize path, eliminating double slashes, etc.""" - path = os.fspath(path) - if isinstance(path, bytes): - sep = b'\\' - altsep = b'/' - curdir = b'.' - pardir = b'..' - special_prefixes = (b'\\\\.\\', b'\\\\?\\') - else: - sep = '\\' - altsep = '/' - curdir = '.' - pardir = '..' - special_prefixes = ('\\\\.\\', '\\\\?\\') - if path.startswith(special_prefixes): - # in the case of paths with these prefixes: - # \\.\ -> device names - # \\?\ -> literal paths - # do not do any normalization, but return the path - # unchanged apart from the call to os.fspath() - return path - path = path.replace(altsep, sep) - prefix, path = splitdrive(path) +except ImportError: + def normpath(path): + """Normalize path, eliminating double slashes, etc.""" + path = os.fspath(path) + if isinstance(path, bytes): + sep = b'\\' + altsep = b'/' + curdir = b'.' + pardir = b'..' + special_prefixes = (b'\\\\.\\', b'\\\\?\\') + else: + sep = '\\' + altsep = '/' + curdir = '.' + pardir = '..' + special_prefixes = ('\\\\.\\', '\\\\?\\') + if path.startswith(special_prefixes): + # in the case of paths with these prefixes: + # \\.\ -> device names + # \\?\ -> literal paths + # do not do any normalization, but return the path + # unchanged apart from the call to os.fspath() + return path + path = path.replace(altsep, sep) + prefix, path = splitdrive(path) - # collapse initial backslashes - if path.startswith(sep): - prefix += sep - path = path.lstrip(sep) + # collapse initial backslashes + if path.startswith(sep): + prefix += sep + path = path.lstrip(sep) - comps = path.split(sep) - i = 0 - while i < len(comps): - if not comps[i] or comps[i] == curdir: - del comps[i] - elif comps[i] == pardir: - if i > 0 and comps[i-1] != pardir: - del comps[i-1:i+1] - i -= 1 - elif i == 0 and prefix.endswith(sep): + comps = path.split(sep) + i = 0 + while i < len(comps): + if not comps[i] or comps[i] == curdir: del comps[i] + elif comps[i] == pardir: + if i > 0 and comps[i-1] != pardir: + del comps[i-1:i+1] + i -= 1 + elif i == 0 and prefix.endswith(sep): + del comps[i] + else: + i += 1 else: i += 1 - else: - i += 1 - # If the path is now empty, substitute '.' - if not prefix and not comps: - comps.append(curdir) - return prefix + sep.join(comps) + # If the path is now empty, substitute '.' + if not prefix and not comps: + comps.append(curdir) + return prefix + sep.join(comps) + +else: + def normpath(path): + """Normalize path, eliminating double slashes, etc.""" + path = os.fspath(path) + if isinstance(path, bytes): + return os.fsencode(_path_normpath(os.fsdecode(path))) or b"." + return _path_normpath(path) or "." + def _abspath_fallback(path): """Return the absolute version of a path as a fallback function in case diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 195374613a7..a46c667db56 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -334,43 +334,55 @@ def expandvars(path): # It should be understood that this may change the meaning of the path # if it contains symbolic links! -def normpath(path): - """Normalize path, eliminating double slashes, etc.""" - path = os.fspath(path) - if isinstance(path, bytes): - sep = b'/' - empty = b'' - dot = b'.' - dotdot = b'..' - else: - sep = '/' - empty = '' - dot = '.' - dotdot = '..' - if path == empty: - return dot - initial_slashes = path.startswith(sep) - # POSIX allows one or two initial slashes, but treats three or more - # as single slash. - # (see http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13) - if (initial_slashes and - path.startswith(sep*2) and not path.startswith(sep*3)): - initial_slashes = 2 - comps = path.split(sep) - new_comps = [] - for comp in comps: - if comp in (empty, dot): - continue - if (comp != dotdot or (not initial_slashes and not new_comps) or - (new_comps and new_comps[-1] == dotdot)): - new_comps.append(comp) - elif new_comps: - new_comps.pop() - comps = new_comps - path = sep.join(comps) - if initial_slashes: - path = sep*initial_slashes + path - return path or dot +try: + from posix import _path_normpath + +except ImportError: + def normpath(path): + """Normalize path, eliminating double slashes, etc.""" + path = os.fspath(path) + if isinstance(path, bytes): + sep = b'/' + empty = b'' + dot = b'.' + dotdot = b'..' + else: + sep = '/' + empty = '' + dot = '.' + dotdot = '..' + if path == empty: + return dot + initial_slashes = path.startswith(sep) + # POSIX allows one or two initial slashes, but treats three or more + # as single slash. + # (see http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13) + if (initial_slashes and + path.startswith(sep*2) and not path.startswith(sep*3)): + initial_slashes = 2 + comps = path.split(sep) + new_comps = [] + for comp in comps: + if comp in (empty, dot): + continue + if (comp != dotdot or (not initial_slashes and not new_comps) or + (new_comps and new_comps[-1] == dotdot)): + new_comps.append(comp) + elif new_comps: + new_comps.pop() + comps = new_comps + path = sep.join(comps) + if initial_slashes: + path = sep*initial_slashes + path + return path or dot + +else: + def normpath(path): + """Normalize path, eliminating double slashes, etc.""" + path = os.fspath(path) + if isinstance(path, bytes): + return os.fsencode(_path_normpath(os.fsdecode(path))) or b"." + return _path_normpath(path) or "." def abspath(path): diff --git a/Lib/site.py b/Lib/site.py index e129f3b4851..b11cd48e69e 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -361,11 +361,11 @@ def getsitepackages(prefixes=None): continue seen.add(prefix) - libdirs = [sys.platlibdir] - if sys.platlibdir != "lib": - libdirs.append("lib") - if os.sep == '/': + libdirs = [sys.platlibdir] + if sys.platlibdir != "lib": + libdirs.append("lib") + for libdir in libdirs: path = os.path.join(prefix, libdir, "python%d.%d" % sys.version_info[:2], @@ -373,10 +373,7 @@ def getsitepackages(prefixes=None): sitepackages.append(path) else: sitepackages.append(prefix) - - for libdir in libdirs: - path = os.path.join(prefix, libdir, "site-packages") - sitepackages.append(path) + sitepackages.append(os.path.join(prefix, "Lib", "site-packages")) return sitepackages def addsitepackages(known_paths, prefixes=None): diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index daf9f000060..da918b7ba19 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -216,6 +216,11 @@ def _expand_vars(scheme, vars): if vars is None: vars = {} _extend_dict(vars, get_config_vars()) + if os.name == 'nt': + # On Windows we want to substitute 'lib' for schemes rather + # than the native value (without modifying vars, in case it + # was passed in) + vars = vars | {'platlibdir': 'lib'} for key, value in _INSTALL_SCHEMES[scheme].items(): if os.name in ('posix', 'nt'): diff --git a/Lib/test/_test_embed_set_config.py b/Lib/test/_test_embed_set_config.py index d05907ecfe7..e7b7106e178 100644 --- a/Lib/test/_test_embed_set_config.py +++ b/Lib/test/_test_embed_set_config.py @@ -20,6 +20,7 @@ class SetConfigTests(unittest.TestCase): self.sys_copy = dict(sys.__dict__) def tearDown(self): + _testinternalcapi.reset_path_config() _testinternalcapi.set_config(self.old_config) sys.__dict__.clear() sys.__dict__.update(self.sys_copy) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 4781c7f2a9d..3620a761960 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -13,6 +13,7 @@ import re import shutil import subprocess import sys +import sysconfig import tempfile import textwrap @@ -445,6 +446,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): '_init_main': 1, '_isolated_interpreter': 0, 'use_frozen_modules': 1, + '_is_python_build': IGNORE_CONFIG, } if MS_WINDOWS: CONFIG_COMPAT.update({ @@ -508,32 +510,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): ('Py_LegacyWindowsStdioFlag', 'legacy_windows_stdio'), )) - # path config - if MS_WINDOWS: - PATH_CONFIG = { - 'isolated': -1, - 'site_import': -1, - 'python3_dll': GET_DEFAULT_CONFIG, - } - else: - PATH_CONFIG = {} - # other keys are copied by COPY_PATH_CONFIG - - COPY_PATH_CONFIG = [ - # Copy core config to global config for expected values - 'prefix', - 'exec_prefix', - 'program_name', - 'home', - 'stdlib_dir', - # program_full_path and module_search_path are copied indirectly from - # the core configuration in check_path_config(). - ] - if MS_WINDOWS: - COPY_PATH_CONFIG.extend(( - 'base_executable', - )) - EXPECTED_CONFIG = None @classmethod @@ -599,8 +575,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): return configs def get_expected_config(self, expected_preconfig, expected, - expected_pathconfig, env, api, - modify_path_cb=None): + env, api, modify_path_cb=None): configs = self._get_expected_config() pre_config = configs['pre_config'] @@ -608,11 +583,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): if value is self.GET_DEFAULT_CONFIG: expected_preconfig[key] = pre_config[key] - path_config = configs['path_config'] - for key, value in expected_pathconfig.items(): - if value is self.GET_DEFAULT_CONFIG: - expected_pathconfig[key] = path_config[key] - if not expected_preconfig['configure_locale'] or api == API_COMPAT: # there is no easy way to get the locale encoding before # setlocale(LC_CTYPE, "") is called: don't test encodings @@ -705,18 +675,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.assertEqual(configs['global_config'], expected) - def check_path_config(self, configs, expected): - config = configs['config'] - - for key in self.COPY_PATH_CONFIG: - expected[key] = config[key] - expected['module_search_path'] = os.path.pathsep.join(config['module_search_paths']) - expected['program_full_path'] = config['executable'] - - self.assertEqual(configs['path_config'], expected) - def check_all_configs(self, testname, expected_config=None, - expected_preconfig=None, expected_pathconfig=None, + expected_preconfig=None, modify_path_cb=None, stderr=None, *, api, preconfig_api=None, env=None, ignore_stderr=False, cwd=None): @@ -740,10 +700,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): if expected_config is None: expected_config = {} - if expected_pathconfig is None: - expected_pathconfig = {} - expected_pathconfig = dict(self.PATH_CONFIG, **expected_pathconfig) - if api == API_PYTHON: default_config = self.CONFIG_PYTHON elif api == API_ISOLATED: @@ -754,7 +710,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.get_expected_config(expected_preconfig, expected_config, - expected_pathconfig, env, api, modify_path_cb) @@ -772,7 +727,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.check_pre_config(configs, expected_preconfig) self.check_config(configs, expected_config) self.check_global_config(configs) - self.check_path_config(configs, expected_pathconfig) return configs def test_init_default_config(self): @@ -1039,10 +993,13 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.check_all_configs("test_init_dont_configure_locale", {}, preconfig, api=API_PYTHON) + @unittest.skip('as of 3.11 this test no longer works because ' + 'path calculations do not occur on read') def test_init_read_set(self): config = { 'program_name': './init_read_set', 'executable': 'my_executable', + 'base_executable': 'my_executable', } def modify_path(path): path.insert(1, "test_path_insert1") @@ -1156,7 +1113,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': '', - 'use_frozen_modules': -1, } self.default_program_name(config) env = {'TESTPATH': os.path.pathsep.join(paths)} @@ -1180,7 +1136,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': '', - 'use_frozen_modules': -1, + 'use_frozen_modules': 1, # overridden by PyConfig 'program_name': 'conf_program_name', 'base_executable': 'conf_executable', @@ -1210,13 +1166,16 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): ] @contextlib.contextmanager - def tmpdir_with_python(self): + def tmpdir_with_python(self, subdir=None): # Temporary directory with a copy of the Python program with tempfile.TemporaryDirectory() as tmpdir: # bpo-38234: On macOS and FreeBSD, the temporary directory # can be symbolic link. For example, /tmp can be a symbolic link # to /var/tmp. Call realpath() to resolve all symbolic links. tmpdir = os.path.realpath(tmpdir) + if subdir: + tmpdir = os.path.normpath(os.path.join(tmpdir, subdir)) + os.makedirs(tmpdir) if MS_WINDOWS: # Copy pythonXY.dll (or pythonXY_d.dll) @@ -1260,11 +1219,14 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): prefix = exec_prefix = home if MS_WINDOWS: - stdlib = os.path.join(home, sys.platlibdir) + stdlib = os.path.join(home, "Lib") + # Because we are specifying 'home', module search paths + # are fairly static + expected_paths = [paths[0], stdlib, os.path.join(home, 'DLLs')] else: version = f'{sys.version_info.major}.{sys.version_info.minor}' stdlib = os.path.join(home, sys.platlibdir, f'python{version}') - expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) + expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) config = { 'home': home, @@ -1291,7 +1253,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): env = {'PYTHONPATH': paths_str} return env - @unittest.skipIf(MS_WINDOWS, 'Windows does not use pybuilddir.txt') + @unittest.skipIf(MS_WINDOWS, 'See test_init_pybuilddir_win32') def test_init_pybuilddir(self): # Test path configuration with pybuilddir.txt configuration file @@ -1300,6 +1262,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): # directory (tmpdir) subdir = 'libdir' libdir = os.path.join(tmpdir, subdir) + # The stdlib dir is dirname(executable) + VPATH + 'Lib' + stdlibdir = os.path.join(tmpdir, 'Lib') os.mkdir(libdir) filename = os.path.join(tmpdir, 'pybuilddir.txt') @@ -1307,21 +1271,58 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): fp.write(subdir) module_search_paths = self.module_search_paths() + module_search_paths[-2] = stdlibdir module_search_paths[-1] = libdir executable = self.test_exe config = { + 'base_exec_prefix': sysconfig.get_config_var("exec_prefix"), + 'base_prefix': sysconfig.get_config_var("prefix"), 'base_executable': executable, 'executable': executable, 'module_search_paths': module_search_paths, - 'stdlib_dir': STDLIB_INSTALL, - 'use_frozen_modules': 1 if STDLIB_INSTALL else -1, + 'stdlib_dir': stdlibdir, } env = self.copy_paths_by_env(config) self.check_all_configs("test_init_compat_config", config, api=API_COMPAT, env=env, ignore_stderr=True, cwd=tmpdir) + @unittest.skipUnless(MS_WINDOWS, 'See test_init_pybuilddir') + def test_init_pybuilddir_win32(self): + # Test path configuration with pybuilddir.txt configuration file + + with self.tmpdir_with_python(r'PCbuild\arch') as tmpdir: + # The prefix is dirname(executable) + VPATH + prefix = os.path.normpath(os.path.join(tmpdir, r'..\..')) + # The stdlib dir is dirname(executable) + VPATH + 'Lib' + stdlibdir = os.path.normpath(os.path.join(tmpdir, r'..\..\Lib')) + + filename = os.path.join(tmpdir, 'pybuilddir.txt') + with open(filename, "w", encoding="utf8") as fp: + fp.write(tmpdir) + + module_search_paths = self.module_search_paths() + module_search_paths[-3] = os.path.join(tmpdir, os.path.basename(module_search_paths[-3])) + module_search_paths[-2] = stdlibdir + module_search_paths[-1] = tmpdir + + executable = self.test_exe + config = { + 'base_exec_prefix': prefix, + 'base_prefix': prefix, + 'base_executable': executable, + 'executable': executable, + 'prefix': prefix, + 'exec_prefix': prefix, + 'module_search_paths': module_search_paths, + 'stdlib_dir': stdlibdir, + } + env = self.copy_paths_by_env(config) + self.check_all_configs("test_init_compat_config", config, + api=API_COMPAT, env=env, + ignore_stderr=False, cwd=tmpdir) + def test_init_pyvenv_cfg(self): # Test path configuration with pyvenv.cfg configuration file @@ -1336,12 +1337,12 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'lib-dynload') os.makedirs(lib_dynload) else: - lib_dynload = os.path.join(pyvenv_home, 'lib') - os.makedirs(lib_dynload) - # getpathp.c uses Lib\os.py as the LANDMARK + lib_folder = os.path.join(pyvenv_home, 'Lib') + os.makedirs(lib_folder) + # getpath.py uses Lib\os.py as the LANDMARK shutil.copyfile( os.path.join(support.STDLIB_DIR, 'os.py'), - os.path.join(lib_dynload, 'os.py'), + os.path.join(lib_folder, 'os.py'), ) filename = os.path.join(tmpdir, 'pyvenv.cfg') @@ -1355,43 +1356,38 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): else: for index, path in enumerate(paths): if index == 0: + # Because we copy the DLLs into tmpdir as well, the zip file + # entry in sys.path will be there. For a regular venv, it will + # usually be in the home directory. paths[index] = os.path.join(tmpdir, os.path.basename(path)) else: paths[index] = os.path.join(pyvenv_home, os.path.basename(path)) paths[-1] = pyvenv_home executable = self.test_exe + base_executable = os.path.join(pyvenv_home, os.path.basename(executable)) exec_prefix = pyvenv_home config = { + 'base_prefix': sysconfig.get_config_var("prefix"), 'base_exec_prefix': exec_prefix, 'exec_prefix': exec_prefix, - 'base_executable': executable, + 'base_executable': base_executable, 'executable': executable, 'module_search_paths': paths, } - path_config = {} if MS_WINDOWS: config['base_prefix'] = pyvenv_home config['prefix'] = pyvenv_home - config['stdlib_dir'] = os.path.join(pyvenv_home, 'lib') + config['stdlib_dir'] = os.path.join(pyvenv_home, 'Lib') config['use_frozen_modules'] = 1 - - ver = sys.version_info - dll = f'python{ver.major}' - if debug_build(executable): - dll += '_d' - dll += '.DLL' - dll = os.path.join(os.path.dirname(executable), dll) - path_config['python3_dll'] = dll else: - # The current getpath.c doesn't determine the stdlib dir - # in this case. - config['stdlib_dir'] = STDLIB_INSTALL - config['use_frozen_modules'] = 1 if STDLIB_INSTALL else -1 + # cannot reliably assume stdlib_dir here because it + # depends too much on our build. But it ought to be found + config['stdlib_dir'] = self.IGNORE_CONFIG + config['use_frozen_modules'] = 1 env = self.copy_paths_by_env(config) self.check_all_configs("test_init_compat_config", config, - expected_pathconfig=path_config, api=API_COMPAT, env=env, ignore_stderr=True, cwd=tmpdir) @@ -1502,10 +1498,14 @@ class SetConfigTests(unittest.TestCase): def test_set_config(self): # bpo-42260: Test _PyInterpreterState_SetConfig() import_helper.import_module('_testcapi') - cmd = [sys.executable, '-I', '-m', 'test._test_embed_set_config'] + cmd = [sys.executable, '-X', 'utf8', '-I', '-m', 'test._test_embed_set_config'] proc = subprocess.run(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + encoding='utf-8', errors='backslashreplace') + if proc.returncode and support.verbose: + print(proc.stdout) + print(proc.stderr) self.assertEqual(proc.returncode, 0, (proc.returncode, proc.stdout, proc.stderr)) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py new file mode 100644 index 00000000000..c18689c0590 --- /dev/null +++ b/Lib/test/test_getpath.py @@ -0,0 +1,879 @@ +import copy +import ntpath +import pathlib +import posixpath +import sys +import unittest + +from test.support import verbose + +try: + # If we are in a source tree, use the original source file for tests + SOURCE = (pathlib.Path(__file__).absolute().parent.parent.parent / "Modules/getpath.py").read_bytes() +except FileNotFoundError: + # Try from _testcapimodule instead + from _testinternalcapi import get_getpath_codeobject + SOURCE = get_getpath_codeobject() + + +class MockGetPathTests(unittest.TestCase): + def __init__(self, *a, **kw): + super().__init__(*a, **kw) + self.maxDiff = None + + def test_normal_win32(self): + "Test a 'standard' install layout on Windows." + ns = MockNTNamespace( + argv0=r"C:\Python\python.exe", + real_executable=r"C:\Python\python.exe", + ) + ns.add_known_xfile(r"C:\Python\python.exe") + ns.add_known_file(r"C:\Python\Lib\os.py") + ns.add_known_dir(r"C:\Python\DLLs") + expected = dict( + executable=r"C:\Python\python.exe", + base_executable=r"C:\Python\python.exe", + prefix=r"C:\Python", + exec_prefix=r"C:\Python", + module_search_paths_set=1, + module_search_paths=[ + r"C:\Python\python98.zip", + r"C:\Python\Lib", + r"C:\Python\DLLs", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_buildtree_win32(self): + "Test an in-build-tree layout on Windows." + ns = MockNTNamespace( + argv0=r"C:\CPython\PCbuild\amd64\python.exe", + real_executable=r"C:\CPython\PCbuild\amd64\python.exe", + ) + ns.add_known_xfile(r"C:\CPython\PCbuild\amd64\python.exe") + ns.add_known_file(r"C:\CPython\Lib\os.py") + ns.add_known_file(r"C:\CPython\PCbuild\amd64\pybuilddir.txt", [""]) + expected = dict( + executable=r"C:\CPython\PCbuild\amd64\python.exe", + base_executable=r"C:\CPython\PCbuild\amd64\python.exe", + prefix=r"C:\CPython", + exec_prefix=r"C:\CPython", + build_prefix=r"C:\CPython", + _is_python_build=1, + module_search_paths_set=1, + module_search_paths=[ + r"C:\CPython\PCbuild\amd64\python98.zip", + r"C:\CPython\Lib", + r"C:\CPython\PCbuild\amd64", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_venv_win32(self): + """Test a venv layout on Windows. + + This layout is discovered by the presence of %__PYVENV_LAUNCHER__%, + specifying the original launcher executable. site.py is responsible + for updating prefix and exec_prefix. + """ + ns = MockNTNamespace( + argv0=r"C:\Python\python.exe", + ENV___PYVENV_LAUNCHER__=r"C:\venv\Scripts\python.exe", + real_executable=r"C:\Python\python.exe", + ) + ns.add_known_xfile(r"C:\Python\python.exe") + ns.add_known_xfile(r"C:\venv\Scripts\python.exe") + ns.add_known_file(r"C:\Python\Lib\os.py") + ns.add_known_dir(r"C:\Python\DLLs") + ns.add_known_file(r"C:\venv\pyvenv.cfg", [ + r"home = C:\Python" + ]) + expected = dict( + executable=r"C:\venv\Scripts\python.exe", + prefix=r"C:\Python", + exec_prefix=r"C:\Python", + base_executable=r"C:\Python\python.exe", + base_prefix=r"C:\Python", + base_exec_prefix=r"C:\Python", + module_search_paths_set=1, + module_search_paths=[ + r"C:\Python\python98.zip", + r"C:\Python\Lib", + r"C:\Python", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_registry_win32(self): + """Test registry lookup on Windows. + + On Windows there are registry entries that are intended for other + applications to register search paths. + """ + hkey = rf"HKLM\Software\Python\PythonCore\9.8-XY\PythonPath" + winreg = MockWinreg({ + hkey: None, + f"{hkey}\\Path1": "path1-dir", + f"{hkey}\\Path1\\Subdir": "not-subdirs", + }) + ns = MockNTNamespace( + argv0=r"C:\Python\python.exe", + real_executable=r"C:\Python\python.exe", + winreg=winreg, + ) + ns.add_known_xfile(r"C:\Python\python.exe") + ns.add_known_file(r"C:\Python\Lib\os.py") + ns.add_known_dir(r"C:\Python\DLLs") + expected = dict( + module_search_paths_set=1, + module_search_paths=[ + r"C:\Python\python98.zip", + "path1-dir", + # should not contain not-subdirs + r"C:\Python\Lib", + r"C:\Python\DLLs", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + ns["config"]["use_environment"] = 0 + ns["config"]["module_search_paths_set"] = 0 + ns["config"]["module_search_paths"] = None + expected = dict( + module_search_paths_set=1, + module_search_paths=[ + r"C:\Python\python98.zip", + r"C:\Python\Lib", + r"C:\Python\DLLs", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_symlink_normal_win32(self): + "Test a 'standard' install layout via symlink on Windows." + ns = MockNTNamespace( + argv0=r"C:\LinkedFrom\python.exe", + real_executable=r"C:\Python\python.exe", + ) + ns.add_known_xfile(r"C:\LinkedFrom\python.exe") + ns.add_known_xfile(r"C:\Python\python.exe") + ns.add_known_link(r"C:\LinkedFrom\python.exe", r"C:\Python\python.exe") + ns.add_known_file(r"C:\Python\Lib\os.py") + ns.add_known_dir(r"C:\Python\DLLs") + expected = dict( + executable=r"C:\LinkedFrom\python.exe", + base_executable=r"C:\LinkedFrom\python.exe", + prefix=r"C:\Python", + exec_prefix=r"C:\Python", + module_search_paths_set=1, + module_search_paths=[ + r"C:\Python\python98.zip", + r"C:\Python\Lib", + r"C:\Python\DLLs", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_symlink_buildtree_win32(self): + "Test an in-build-tree layout via symlink on Windows." + ns = MockNTNamespace( + argv0=r"C:\LinkedFrom\python.exe", + real_executable=r"C:\CPython\PCbuild\amd64\python.exe", + ) + ns.add_known_xfile(r"C:\LinkedFrom\python.exe") + ns.add_known_xfile(r"C:\CPython\PCbuild\amd64\python.exe") + ns.add_known_link(r"C:\LinkedFrom\python.exe", r"C:\CPython\PCbuild\amd64\python.exe") + ns.add_known_file(r"C:\CPython\Lib\os.py") + ns.add_known_file(r"C:\CPython\PCbuild\amd64\pybuilddir.txt", [""]) + expected = dict( + executable=r"C:\LinkedFrom\python.exe", + base_executable=r"C:\LinkedFrom\python.exe", + prefix=r"C:\CPython", + exec_prefix=r"C:\CPython", + build_prefix=r"C:\CPython", + _is_python_build=1, + module_search_paths_set=1, + module_search_paths=[ + r"C:\CPython\PCbuild\amd64\python98.zip", + r"C:\CPython\Lib", + r"C:\CPython\PCbuild\amd64", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_normal_posix(self): + "Test a 'standard' install layout on *nix" + ns = MockPosixNamespace( + PREFIX="/usr", + argv0="python", + ENV_PATH="/usr/bin", + ) + ns.add_known_xfile("/usr/bin/python") + ns.add_known_file("/usr/lib/python9.8/os.py") + ns.add_known_dir("/usr/lib/python9.8/lib-dynload") + expected = dict( + executable="/usr/bin/python", + base_executable="/usr/bin/python", + prefix="/usr", + exec_prefix="/usr", + module_search_paths_set=1, + module_search_paths=[ + "/usr/lib/python98.zip", + "/usr/lib/python9.8", + "/usr/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_buildpath_posix(self): + """Test an in-build-tree layout on POSIX. + + This layout is discovered from the presence of pybuilddir.txt, which + contains the relative path from the executable's directory to the + platstdlib path. + """ + ns = MockPosixNamespace( + argv0=r"/home/cpython/python", + PREFIX="/usr/local", + ) + ns.add_known_xfile("/home/cpython/python") + ns.add_known_xfile("/usr/local/bin/python") + ns.add_known_file("/home/cpython/pybuilddir.txt", ["build/lib.linux-x86_64-9.8"]) + ns.add_known_file("/home/cpython/Lib/os.py") + ns.add_known_dir("/home/cpython/lib-dynload") + expected = dict( + executable="/home/cpython/python", + prefix="/usr/local", + exec_prefix="/usr/local", + base_executable="/home/cpython/python", + build_prefix="/home/cpython", + _is_python_build=1, + module_search_paths_set=1, + module_search_paths=[ + "/usr/local/lib/python98.zip", + "/home/cpython/Lib", + "/home/cpython/build/lib.linux-x86_64-9.8", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_venv_posix(self): + "Test a venv layout on *nix." + ns = MockPosixNamespace( + argv0="python", + PREFIX="/usr", + ENV_PATH="/venv/bin:/usr/bin", + ) + ns.add_known_xfile("/usr/bin/python") + ns.add_known_xfile("/venv/bin/python") + ns.add_known_file("/usr/lib/python9.8/os.py") + ns.add_known_dir("/usr/lib/python9.8/lib-dynload") + ns.add_known_file("/venv/pyvenv.cfg", [ + r"home = /usr/bin" + ]) + expected = dict( + executable="/venv/bin/python", + prefix="/usr", + exec_prefix="/usr", + base_executable="/usr/bin/python", + base_prefix="/usr", + base_exec_prefix="/usr", + module_search_paths_set=1, + module_search_paths=[ + "/usr/lib/python98.zip", + "/usr/lib/python9.8", + "/usr/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_symlink_normal_posix(self): + "Test a 'standard' install layout via symlink on *nix" + ns = MockPosixNamespace( + PREFIX="/usr", + argv0="/linkfrom/python", + ) + ns.add_known_xfile("/linkfrom/python") + ns.add_known_xfile("/usr/bin/python") + ns.add_known_link("/linkfrom/python", "/usr/bin/python") + ns.add_known_file("/usr/lib/python9.8/os.py") + ns.add_known_dir("/usr/lib/python9.8/lib-dynload") + expected = dict( + executable="/linkfrom/python", + base_executable="/linkfrom/python", + prefix="/usr", + exec_prefix="/usr", + module_search_paths_set=1, + module_search_paths=[ + "/usr/lib/python98.zip", + "/usr/lib/python9.8", + "/usr/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_symlink_buildpath_posix(self): + """Test an in-build-tree layout on POSIX. + + This layout is discovered from the presence of pybuilddir.txt, which + contains the relative path from the executable's directory to the + platstdlib path. + """ + ns = MockPosixNamespace( + argv0=r"/linkfrom/python", + PREFIX="/usr/local", + ) + ns.add_known_xfile("/linkfrom/python") + ns.add_known_xfile("/home/cpython/python") + ns.add_known_link("/linkfrom/python", "/home/cpython/python") + ns.add_known_xfile("/usr/local/bin/python") + ns.add_known_file("/home/cpython/pybuilddir.txt", ["build/lib.linux-x86_64-9.8"]) + ns.add_known_file("/home/cpython/Lib/os.py") + ns.add_known_dir("/home/cpython/lib-dynload") + expected = dict( + executable="/linkfrom/python", + prefix="/usr/local", + exec_prefix="/usr/local", + base_executable="/linkfrom/python", + build_prefix="/home/cpython", + _is_python_build=1, + module_search_paths_set=1, + module_search_paths=[ + "/usr/local/lib/python98.zip", + "/home/cpython/Lib", + "/home/cpython/build/lib.linux-x86_64-9.8", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_custom_platlibdir_posix(self): + "Test an install with custom platlibdir on *nix" + ns = MockPosixNamespace( + PREFIX="/usr", + argv0="/linkfrom/python", + PLATLIBDIR="lib64", + ) + ns.add_known_xfile("/usr/bin/python") + ns.add_known_file("/usr/lib64/python9.8/os.py") + ns.add_known_dir("/usr/lib64/python9.8/lib-dynload") + expected = dict( + executable="/linkfrom/python", + base_executable="/linkfrom/python", + prefix="/usr", + exec_prefix="/usr", + module_search_paths_set=1, + module_search_paths=[ + "/usr/lib64/python98.zip", + "/usr/lib64/python9.8", + "/usr/lib64/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_venv_macos(self): + """Test a venv layout on macOS. + + This layout is discovered when 'executable' and 'real_executable' match, + but $__PYVENV_LAUNCHER__ has been set to the original process. + """ + ns = MockPosixNamespace( + os_name="darwin", + argv0="/usr/bin/python", + PREFIX="/usr", + ENV___PYVENV_LAUNCHER__="/framework/Python9.8/python", + real_executable="/usr/bin/python", + ) + ns.add_known_xfile("/usr/bin/python") + ns.add_known_xfile("/framework/Python9.8/python") + ns.add_known_file("/usr/lib/python9.8/os.py") + ns.add_known_dir("/usr/lib/python9.8/lib-dynload") + ns.add_known_file("/framework/Python9.8/pyvenv.cfg", [ + "home = /usr/bin" + ]) + expected = dict( + executable="/framework/Python9.8/python", + prefix="/usr", + exec_prefix="/usr", + base_executable="/usr/bin/python", + base_prefix="/usr", + base_exec_prefix="/usr", + module_search_paths_set=1, + module_search_paths=[ + "/usr/lib/python98.zip", + "/usr/lib/python9.8", + "/usr/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_symlink_normal_macos(self): + "Test a 'standard' install layout via symlink on macOS" + ns = MockPosixNamespace( + os_name="darwin", + PREFIX="/usr", + argv0="python", + ENV_PATH="/linkfrom:/usr/bin", + # real_executable on macOS matches the invocation path + real_executable="/linkfrom/python", + ) + ns.add_known_xfile("/linkfrom/python") + ns.add_known_xfile("/usr/bin/python") + ns.add_known_link("/linkfrom/python", "/usr/bin/python") + ns.add_known_file("/usr/lib/python9.8/os.py") + ns.add_known_dir("/usr/lib/python9.8/lib-dynload") + expected = dict( + executable="/linkfrom/python", + base_executable="/linkfrom/python", + prefix="/usr", + exec_prefix="/usr", + module_search_paths_set=1, + module_search_paths=[ + "/usr/lib/python98.zip", + "/usr/lib/python9.8", + "/usr/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_symlink_buildpath_macos(self): + """Test an in-build-tree layout via symlink on macOS. + + This layout is discovered from the presence of pybuilddir.txt, which + contains the relative path from the executable's directory to the + platstdlib path. + """ + ns = MockPosixNamespace( + os_name="darwin", + argv0=r"python", + ENV_PATH="/linkfrom:/usr/bin", + PREFIX="/usr/local", + # real_executable on macOS matches the invocation path + real_executable="/linkfrom/python", + ) + ns.add_known_xfile("/linkfrom/python") + ns.add_known_xfile("/home/cpython/python") + ns.add_known_link("/linkfrom/python", "/home/cpython/python") + ns.add_known_xfile("/usr/local/bin/python") + ns.add_known_file("/home/cpython/pybuilddir.txt", ["build/lib.macos-9.8"]) + ns.add_known_file("/home/cpython/Lib/os.py") + ns.add_known_dir("/home/cpython/lib-dynload") + expected = dict( + executable="/linkfrom/python", + prefix="/usr/local", + exec_prefix="/usr/local", + base_executable="/linkfrom/python", + build_prefix="/home/cpython", + _is_python_build=1, + module_search_paths_set=1, + module_search_paths=[ + "/usr/local/lib/python98.zip", + "/home/cpython/Lib", + "/home/cpython/build/lib.macos-9.8", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + +# ****************************************************************************** + +DEFAULT_NAMESPACE = dict( + PREFIX="", + EXEC_PREFIX="", + PYTHONPATH="", + VPATH="", + PLATLIBDIR="", + PYDEBUGEXT="", + VERSION_MAJOR=9, # fixed version number for ease + VERSION_MINOR=8, # of testing + PYWINVER=None, + EXE_SUFFIX=None, + + ENV_PATH="", + ENV_PYTHONHOME="", + ENV_PYTHONEXECUTABLE="", + ENV___PYVENV_LAUNCHER__="", + argv0="", + py_setpath="", + real_executable="", + executable_dir="", + library="", + winreg=None, + build_prefix=None, + venv_prefix=None, +) + +DEFAULT_CONFIG = dict( + home=None, + platlibdir=None, + pythonpath=None, + program_name=None, + prefix=None, + exec_prefix=None, + base_prefix=None, + base_exec_prefix=None, + executable=None, + base_executable="", + stdlib_dir=None, + platstdlib_dir=None, + module_search_paths=None, + module_search_paths_set=0, + pythonpath_env=None, + argv=None, + orig_argv=None, + + isolated=0, + use_environment=1, + use_site=1, +) + +class MockNTNamespace(dict): + def __init__(self, *a, argv0=None, config=None, **kw): + self.update(DEFAULT_NAMESPACE) + self["config"] = DEFAULT_CONFIG.copy() + self["os_name"] = "nt" + self["PLATLIBDIR"] = "DLLs" + self["PYWINVER"] = "9.8-XY" + self["VPATH"] = r"..\.." + super().__init__(*a, **kw) + if argv0: + self["config"]["orig_argv"] = [argv0] + if config: + self["config"].update(config) + self._files = {} + self._links = {} + self._dirs = set() + self._warnings = [] + + def add_known_file(self, path, lines=None): + self._files[path.casefold()] = list(lines or ()) + self.add_known_dir(path.rpartition("\\")[0]) + + def add_known_xfile(self, path): + self.add_known_file(path) + + def add_known_link(self, path, target): + self._links[path.casefold()] = target + + def add_known_dir(self, path): + p = path.rstrip("\\").casefold() + while p: + self._dirs.add(p) + p = p.rpartition("\\")[0] + + def __missing__(self, key): + try: + return getattr(self, key) + except AttributeError: + raise KeyError(key) from None + + def abspath(self, path): + if self.isabs(path): + return path + return self.joinpath("C:\\Absolute", path) + + def basename(self, path): + return path.rpartition("\\")[2] + + def dirname(self, path): + name = path.rstrip("\\").rpartition("\\")[0] + if name[1:] == ":": + return name + "\\" + return name + + def hassuffix(self, path, suffix): + return path.casefold().endswith(suffix.casefold()) + + def isabs(self, path): + return path[1:3] == ":\\" + + def isdir(self, path): + if verbose: + print("Check if", path, "is a dir") + return path.casefold() in self._dirs + + def isfile(self, path): + if verbose: + print("Check if", path, "is a file") + return path.casefold() in self._files + + def ismodule(self, path): + if verbose: + print("Check if", path, "is a module") + path = path.casefold() + return path in self._files and path.rpartition(".")[2] == "py".casefold() + + def isxfile(self, path): + if verbose: + print("Check if", path, "is a executable") + path = path.casefold() + return path in self._files and path.rpartition(".")[2] == "exe".casefold() + + def joinpath(self, *path): + return ntpath.normpath(ntpath.join(*path)) + + def readlines(self, path): + try: + return self._files[path.casefold()] + except KeyError: + raise FileNotFoundError(path) from None + + def realpath(self, path, _trail=None): + if verbose: + print("Read link from", path) + try: + link = self._links[path.casefold()] + except KeyError: + return path + if _trail is None: + _trail = set() + elif link.casefold() in _trail: + raise OSError("circular link") + _trail.add(link.casefold()) + return self.realpath(link, _trail) + + def warn(self, message): + self._warnings.append(message) + if verbose: + print(message) + + +class MockWinreg: + HKEY_LOCAL_MACHINE = "HKLM" + HKEY_CURRENT_USER = "HKCU" + + def __init__(self, keys): + self.keys = {k.casefold(): v for k, v in keys.items()} + self.open = {} + + def __repr__(self): + return "" + + def __eq__(self, other): + return isinstance(other, type(self)) + + def open_keys(self): + return list(self.open) + + def OpenKeyEx(self, hkey, subkey): + if verbose: + print(f"OpenKeyEx({hkey}, {subkey})") + key = f"{hkey}\\{subkey}".casefold() + if key in self.keys: + self.open[key] = self.open.get(key, 0) + 1 + return key + raise FileNotFoundError() + + def CloseKey(self, hkey): + if verbose: + print(f"CloseKey({hkey})") + hkey = hkey.casefold() + if hkey not in self.open: + raise RuntimeError("key is not open") + self.open[hkey] -= 1 + if not self.open[hkey]: + del self.open[hkey] + + def EnumKey(self, hkey, i): + if verbose: + print(f"EnumKey({hkey}, {i})") + hkey = hkey.casefold() + if hkey not in self.open: + raise RuntimeError("key is not open") + prefix = f'{hkey}\\' + subkeys = [k[len(prefix):] for k in sorted(self.keys) if k.startswith(prefix)] + subkeys[:] = [k for k in subkeys if '\\' not in k] + for j, n in enumerate(subkeys): + if j == i: + return n.removeprefix(prefix) + raise OSError("end of enumeration") + + def QueryValue(self, hkey): + if verbose: + print(f"QueryValue({hkey})") + hkey = hkey.casefold() + if hkey not in self.open: + raise RuntimeError("key is not open") + try: + return self.keys[hkey] + except KeyError: + raise OSError() + + +class MockPosixNamespace(dict): + def __init__(self, *a, argv0=None, config=None, **kw): + self.update(DEFAULT_NAMESPACE) + self["config"] = DEFAULT_CONFIG.copy() + self["os_name"] = "posix" + self["PLATLIBDIR"] = "lib" + super().__init__(*a, **kw) + if argv0: + self["config"]["orig_argv"] = [argv0] + if config: + self["config"].update(config) + self._files = {} + self._xfiles = set() + self._links = {} + self._dirs = set() + self._warnings = [] + + def add_known_file(self, path, lines=None): + self._files[path] = list(lines or ()) + self.add_known_dir(path.rpartition("/")[0]) + + def add_known_xfile(self, path): + self.add_known_file(path) + self._xfiles.add(path) + + def add_known_link(self, path, target): + self._links[path] = target + + def add_known_dir(self, path): + p = path.rstrip("/") + while p: + self._dirs.add(p) + p = p.rpartition("/")[0] + + def __missing__(self, key): + try: + return getattr(self, key) + except AttributeError: + raise KeyError(key) from None + + def abspath(self, path): + if self.isabs(path): + return path + return self.joinpath("/Absolute", path) + + def basename(self, path): + return path.rpartition("/")[2] + + def dirname(self, path): + return path.rstrip("/").rpartition("/")[0] + + def hassuffix(self, path, suffix): + return path.endswith(suffix) + + def isabs(self, path): + return path[0:1] == "/" + + def isdir(self, path): + if verbose: + print("Check if", path, "is a dir") + return path in self._dirs + + def isfile(self, path): + if verbose: + print("Check if", path, "is a file") + return path in self._files + + def ismodule(self, path): + if verbose: + print("Check if", path, "is a module") + return path in self._files and path.rpartition(".")[2] == "py" + + def isxfile(self, path): + if verbose: + print("Check if", path, "is an xfile") + return path in self._xfiles + + def joinpath(self, *path): + return posixpath.normpath(posixpath.join(*path)) + + def readlines(self, path): + try: + return self._files[path] + except KeyError: + raise FileNotFoundError(path) from None + + def realpath(self, path, _trail=None): + if verbose: + print("Read link from", path) + try: + link = self._links[path] + except KeyError: + return path + if _trail is None: + _trail = set() + elif link in _trail: + raise OSError("circular link") + _trail.add(link) + return self.realpath(link, _trail) + + def warn(self, message): + self._warnings.append(message) + if verbose: + print(message) + + +def diff_dict(before, after, prefix="global"): + diff = [] + for k in sorted(before): + if k[:2] == "__": + continue + if k == "config": + diff_dict(before[k], after[k], prefix="config") + continue + if k in after and after[k] != before[k]: + diff.append((k, before[k], after[k])) + if not diff: + return + max_k = max(len(k) for k, _, _ in diff) + indent = " " * (len(prefix) + 1 + max_k) + if verbose: + for k, b, a in diff: + if b: + print("{}.{} -{!r}\n{} +{!r}".format(prefix, k.ljust(max_k), b, indent, a)) + else: + print("{}.{} +{!r}".format(prefix, k.ljust(max_k), a)) + + +def dump_dict(before, after, prefix="global"): + if not verbose or not after: + return + max_k = max(len(k) for k in after) + for k, v in sorted(after.items(), key=lambda i: i[0]): + if k[:2] == "__": + continue + if k == "config": + dump_dict(before[k], after[k], prefix="config") + continue + try: + if v != before[k]: + print("{}.{} {!r} (was {!r})".format(prefix, k.ljust(max_k), v, before[k])) + continue + except KeyError: + pass + print("{}.{} {!r}".format(prefix, k.ljust(max_k), v)) + + +def getpath(ns, keys): + before = copy.deepcopy(ns) + failed = True + try: + exec(SOURCE, ns) + failed = False + finally: + if failed: + dump_dict(before, ns) + else: + diff_dict(before, ns) + return { + k: ns['config'].get(k, ns.get(k, ...)) + for k in keys + } diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 661c59d6171..a8d87e53db9 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -741,8 +741,9 @@ class TestNtpath(NtpathTestCase): # (or any other volume root). The drive-relative # locations below cannot then refer to mount points # - drive, path = ntpath.splitdrive(sys.executable) - with os_helper.change_cwd(ntpath.dirname(sys.executable)): + test_cwd = os.getenv("SystemRoot") + drive, path = ntpath.splitdrive(test_cwd) + with os_helper.change_cwd(test_cwd): self.assertFalse(ntpath.ismount(drive.lower())) self.assertFalse(ntpath.ismount(drive.upper())) diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 5f06a0d4b03..76d35daed0b 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -11,6 +11,7 @@ from test.support import os_helper from test.support import socket_helper from test.support import captured_stderr from test.support.os_helper import TESTFN, EnvironmentVarGuard, change_cwd +import ast import builtins import encodings import glob @@ -300,7 +301,8 @@ class HelperFunctionsTests(unittest.TestCase): self.assertEqual(len(dirs), 2) self.assertEqual(dirs[0], 'xoxo') wanted = os.path.join('xoxo', 'lib', 'site-packages') - self.assertEqual(dirs[1], wanted) + self.assertEqual(os.path.normcase(dirs[1]), + os.path.normcase(wanted)) @unittest.skipUnless(HAS_USER_SITE, 'need user site') def test_no_home_directory(self): @@ -497,13 +499,14 @@ class StartupImportTests(unittest.TestCase): def test_startup_imports(self): # Get sys.path in isolated mode (python3 -I) - popen = subprocess.Popen([sys.executable, '-I', '-c', - 'import sys; print(repr(sys.path))'], + popen = subprocess.Popen([sys.executable, '-X', 'utf8', '-I', + '-c', 'import sys; print(repr(sys.path))'], stdout=subprocess.PIPE, - encoding='utf-8') + encoding='utf-8', + errors='surrogateescape') stdout = popen.communicate()[0] self.assertEqual(popen.returncode, 0, repr(stdout)) - isolated_paths = eval(stdout) + isolated_paths = ast.literal_eval(stdout) # bpo-27807: Even with -I, the site module executes all .pth files # found in sys.path (see site.addpackage()). Skip the test if at least @@ -515,14 +518,15 @@ class StartupImportTests(unittest.TestCase): # This tests checks which modules are loaded by Python when it # initially starts upon startup. - popen = subprocess.Popen([sys.executable, '-I', '-v', '-c', - 'import sys; print(set(sys.modules))'], + popen = subprocess.Popen([sys.executable, '-X', 'utf8', '-I', '-v', + '-c', 'import sys; print(set(sys.modules))'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding='utf-8') + encoding='utf-8', + errors='surrogateescape') stdout, stderr = popen.communicate() self.assertEqual(popen.returncode, 0, (stdout, stderr)) - modules = eval(stdout) + modules = ast.literal_eval(stdout) self.assertIn('site', modules) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 9408657c918..506266d0818 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -102,6 +102,10 @@ class TestSysConfig(unittest.TestCase): def test_get_path(self): config_vars = get_config_vars() + if os.name == 'nt': + # On Windows, we replace the native platlibdir name with the + # default so that POSIX schemes resolve correctly + config_vars = config_vars | {'platlibdir': 'lib'} for scheme in _INSTALL_SCHEMES: for name in _INSTALL_SCHEMES[scheme]: expected = _INSTALL_SCHEMES[scheme][name].format(**config_vars) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 94d626598ba..de714de2a20 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -15,7 +15,7 @@ import subprocess import sys import tempfile from test.support import (captured_stdout, captured_stderr, requires_zlib, - skip_if_broken_multiprocessing_synchronize) + skip_if_broken_multiprocessing_synchronize, verbose) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) import unittest import venv @@ -40,6 +40,8 @@ def check_output(cmd, encoding=None): encoding=encoding) out, err = p.communicate() if p.returncode: + if verbose and err: + print(err.decode('utf-8', 'backslashreplace')) raise subprocess.CalledProcessError( p.returncode, cmd, out, err) return out, err @@ -194,7 +196,7 @@ class BasicTest(BaseTest): ('base_exec_prefix', sys.base_exec_prefix)): cmd[2] = 'import sys; print(sys.%s)' % prefix out, err = check_output(cmd) - self.assertEqual(out.strip(), expected.encode()) + self.assertEqual(out.strip(), expected.encode(), prefix) if sys.platform == 'win32': ENV_SUBDIRS = ( diff --git a/Makefile.pre.in b/Makefile.pre.in index d5be6edf784..3dc131b6c96 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -308,7 +308,6 @@ COVERAGE_REPORT_OPTIONS=--no-branch-coverage --title "CPython lcov report" # Modules MODULE_OBJS= \ Modules/config.o \ - Modules/getpath.o \ Modules/main.o \ Modules/gcmodule.o @@ -505,6 +504,7 @@ LIBRARY_OBJS_OMIT_FROZEN= \ LIBRARY_OBJS= \ $(LIBRARY_OBJS_OMIT_FROZEN) \ $(DEEPFREEZE_OBJS) \ + Modules/getpath.o \ Python/frozen.o ########################################################################## @@ -1061,8 +1061,11 @@ FROZEN_FILES_OUT = \ Programs/_freeze_module.o: Programs/_freeze_module.c Makefile -Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) - $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) +Modules/getpath_noop.o: $(srcdir)/Modules/getpath_noop.c Makefile + $(CC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Modules/getpath_noop.c + +Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) Modules/getpath_noop.o + $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) Modules/getpath_noop.o $(LIBS) $(MODLIBS) $(SYSLIBS) # BEGIN: freezing modules @@ -1130,6 +1133,10 @@ Python/frozen_modules/frozen_only.h: $(FREEZE_MODULE) Tools/freeze/flag.py Tools/scripts/freeze_modules.py: $(FREEZE_MODULE) +# We manually freeze getpath.py rather than through freeze_modules +Modules/getpath.h: Programs/_freeze_module Modules/getpath.py + Programs/_freeze_module getpath $(srcdir)/Modules/getpath.py $(srcdir)/Modules/getpath.h + .PHONY: regen-frozen regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES_IN) $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py @@ -1170,12 +1177,13 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \ -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \ -o $@ $(srcdir)/Modules/getbuildinfo.c -Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile +Modules/getpath.o: $(srcdir)/Modules/getpath.c Modules/getpath.h Makefile $(CC) -c $(PY_CORE_CFLAGS) -DPYTHONPATH='"$(PYTHONPATH)"' \ -DPREFIX='"$(prefix)"' \ -DEXEC_PREFIX='"$(exec_prefix)"' \ -DVERSION='"$(VERSION)"' \ -DVPATH='"$(VPATH)"' \ + -DPLATLIBDIR='"$(PLATLIBDIR)"' \ -o $@ $(srcdir)/Modules/getpath.c Programs/python.o: $(srcdir)/Programs/python.c @@ -1210,11 +1218,6 @@ Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(srcdir)/Include/pydt $(MULTIARCH_CPPFLAGS) \ -o $@ $(srcdir)/Python/sysmodule.c -Python/initconfig.o: $(srcdir)/Python/initconfig.c - $(CC) -c $(PY_CORE_CFLAGS) \ - -DPLATLIBDIR='"$(PLATLIBDIR)"' \ - -o $@ $(srcdir)/Python/initconfig.c - $(IO_OBJS): $(IO_H) .PHONY: regen-pegen-metaparser diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-23-00-39-31.bpo-45582.YONPuo.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-23-00-39-31.bpo-45582.YONPuo.rst new file mode 100644 index 00000000000..45fa75e870d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-23-00-39-31.bpo-45582.YONPuo.rst @@ -0,0 +1,3 @@ +Path calculation (known as ``getpath``) has been reimplemented as a frozen +Python module. This should have no visible impact, but may affect +calculation of all paths referenced in :mod:`sys` and :mod:`sysconfig`. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index c230ba28d61..19babb06f56 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -14,10 +14,11 @@ #include "Python.h" #include "pycore_atomic_funcs.h" // _Py_atomic_int_get() #include "pycore_bitutils.h" // _Py_bswap32() -#include "pycore_fileutils.h" // _Py_normalize_path +#include "pycore_fileutils.h" // _Py_normpath #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() +#include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -273,6 +274,14 @@ error: } +static PyObject * +test_reset_path_config(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(arg)) +{ + _PyPathConfig_ClearGlobal(); + Py_RETURN_NONE; +} + + static PyObject* test_atomic_funcs(PyObject *self, PyObject *Py_UNUSED(args)) { @@ -378,15 +387,15 @@ normalize_path(PyObject *self, PyObject *filename) return NULL; } - wchar_t buf[MAXPATHLEN + 1]; - int res = _Py_normalize_path(encoded, buf, Py_ARRAY_LENGTH(buf)); + PyObject *result = PyUnicode_FromWideChar(_Py_normpath(encoded, size), -1); PyMem_Free(encoded); - if (res != 0) { - PyErr_SetString(PyExc_ValueError, "string too long"); - return NULL; - } - return PyUnicode_FromWideChar(buf, -1); + return result; +} + +static PyObject * +get_getpath_codeobject(PyObject *self, PyObject *Py_UNUSED(args)) { + return _Py_Get_Getpath_CodeObject(); } @@ -399,9 +408,11 @@ static PyMethodDef TestMethods[] = { {"test_hashtable", test_hashtable, METH_NOARGS}, {"get_config", test_get_config, METH_NOARGS}, {"set_config", test_set_config, METH_O}, + {"reset_path_config", test_reset_path_config, METH_NOARGS}, {"test_atomic_funcs", test_atomic_funcs, METH_NOARGS}, {"test_edit_cost", test_edit_cost, METH_NOARGS}, {"normalize_path", normalize_path, METH_O, NULL}, + {"get_getpath_codeobject", get_getpath_codeobject, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 7921c222b90..86da08711fd 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -1317,6 +1317,38 @@ exit: #endif /* defined(MS_WINDOWS) */ +PyDoc_STRVAR(os__path_normpath__doc__, +"_path_normpath($module, /, path)\n" +"--\n" +"\n" +"Basic path normalization."); + +#define OS__PATH_NORMPATH_METHODDEF \ + {"_path_normpath", (PyCFunction)(void(*)(void))os__path_normpath, METH_FASTCALL|METH_KEYWORDS, os__path_normpath__doc__}, + +static PyObject * +os__path_normpath_impl(PyObject *module, PyObject *path); + +static PyObject * +os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "_path_normpath", 0}; + PyObject *argsbuf[1]; + PyObject *path; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + path = args[0]; + return_value = os__path_normpath_impl(module, path); + +exit: + return return_value; +} + PyDoc_STRVAR(os_mkdir__doc__, "mkdir($module, /, path, mode=511, *, dir_fd=None)\n" "--\n" @@ -9263,4 +9295,4 @@ exit: #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=65a85d7d3f2c487e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=05505f171cdcff72 input=a9049054013a1b77]*/ diff --git a/Modules/getpath.c b/Modules/getpath.c index 4dbd502ddcf..32d5db9d2c4 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -1,1617 +1,935 @@ /* Return the initial module search path. */ #include "Python.h" -#include "pycore_fileutils.h" -#include "pycore_initconfig.h" -#include "pycore_pathconfig.h" +#include "marshal.h" // PyMarshal_ReadObjectFromString #include "osdefs.h" // DELIM +#include "pycore_initconfig.h" +#include "pycore_fileutils.h" +#include "pycore_pathconfig.h" +#include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() +#include -#include // getenv() -#include -#include +#ifdef MS_WINDOWS +# include // GetFullPathNameW(), MAX_PATH +# include +#endif #ifdef __APPLE__ # include #endif -/* Search in some common locations for the associated Python libraries. - * - * Two directories must be found, the platform independent directory - * (prefix), containing the common .py and .pyc files, and the platform - * dependent directory (exec_prefix), containing the shared library - * modules. Note that prefix and exec_prefix can be the same directory, - * but for some installations, they are different. - * - * Py_GetPath() carries out separate searches for prefix and exec_prefix. - * Each search tries a number of different locations until a ``landmark'' - * file or directory is found. If no prefix or exec_prefix is found, a - * warning message is issued and the preprocessor defined PREFIX and - * EXEC_PREFIX are used (even though they will not work); python carries on - * as best as is possible, but most imports will fail. - * - * Before any searches are done, the location of the executable is - * determined. If argv[0] has one or more slashes in it, it is used - * unchanged. Otherwise, it must have been invoked from the shell's path, - * so we search $PATH for the named executable and use that. If the - * executable was not found on $PATH (or there was no $PATH environment - * variable), the original argv[0] string is used. - * - * Next, the executable location is examined to see if it is a symbolic - * link. If so, the link is chased (correctly interpreting a relative - * pathname if one is found) and the directory of the link target is used. - * - * Finally, argv0_path is set to the directory containing the executable - * (i.e. the last component is stripped). - * - * With argv0_path in hand, we perform a number of steps. The same steps - * are performed for prefix and for exec_prefix, but with a different - * landmark. - * - * Step 1. Are we running python out of the build directory? This is - * checked by looking for a different kind of landmark relative to - * argv0_path. For prefix, the landmark's path is derived from the VPATH - * preprocessor variable (taking into account that its value is almost, but - * not quite, what we need). For exec_prefix, the landmark is - * pybuilddir.txt. If the landmark is found, we're done. - * - * For the remaining steps, the prefix landmark will always be - * lib/python$VERSION/os.py and the exec_prefix will always be - * lib/python$VERSION/lib-dynload, where $VERSION is Python's version - * number as supplied by the Makefile. Note that this means that no more - * build directory checking is performed; if the first step did not find - * the landmarks, the assumption is that python is running from an - * installed setup. - * - * Step 2. See if the $PYTHONHOME environment variable points to the - * installed location of the Python libraries. If $PYTHONHOME is set, then - * it points to prefix and exec_prefix. $PYTHONHOME can be a single - * directory, which is used for both, or the prefix and exec_prefix - * directories separated by a colon. - * - * Step 3. Try to find prefix and exec_prefix relative to argv0_path, - * backtracking up the path until it is exhausted. This is the most common - * step to succeed. Note that if prefix and exec_prefix are different, - * exec_prefix is more likely to be found; however if exec_prefix is a - * subdirectory of prefix, both will be found. - * - * Step 4. Search the directories pointed to by the preprocessor variables - * PREFIX and EXEC_PREFIX. These are supplied by the Makefile but can be - * passed in as options to the configure script. - * - * That's it! - * - * Well, almost. Once we have determined prefix and exec_prefix, the - * preprocessor variable PYTHONPATH is used to construct a path. Each - * relative path on PYTHONPATH is prefixed with prefix. Then the directory - * containing the shared library modules is appended. The environment - * variable $PYTHONPATH is inserted in front of it all. Finally, the - * prefix and exec_prefix globals are tweaked so they reflect the values - * expected by other code, by stripping the "lib/python$VERSION/..." stuff - * off. If either points to the build directory, the globals are reset to - * the corresponding preprocessor variables (so sys.prefix will reflect the - * installation location, even though sys.path points into the build - * directory). This seems to make more sense given that currently the only - * known use of sys.prefix and sys.exec_prefix is for the ILU installation - * process to find the installed Python tree. - * - * An embedding application can use Py_SetPath() to override all of - * these automatic path computations. - * - * NOTE: Windows MSVC builds use PC/getpathp.c instead! - */ - -#ifdef __cplusplus -extern "C" { -#endif - +/* Reference the precompiled getpath.py */ +#include "getpath.h" #if (!defined(PREFIX) || !defined(EXEC_PREFIX) \ - || !defined(VERSION) || !defined(VPATH)) -#error "PREFIX, EXEC_PREFIX, VERSION and VPATH macros must be defined" + || !defined(VERSION) || !defined(VPATH) \ + || !defined(PLATLIBDIR)) +#error "PREFIX, EXEC_PREFIX, VERSION, VPATH and PLATLIBDIR macros must be defined" #endif -#ifndef LANDMARK -#define LANDMARK L"os.py" +#if !defined(PYTHONPATH) +#define PYTHONPATH NULL #endif -#define BUILD_LANDMARK L"Modules/Setup.local" +#if !defined(PYDEBUGEXT) +#define PYDEBUGEXT NULL +#endif -#define PATHLEN_ERR() _PyStatus_ERR("path configuration: path too long") +#if !defined(PYWINVER) +#ifdef MS_DLL_ID +#define PYWINVER MS_DLL_ID +#else +#define PYWINVER NULL +#endif +#endif -typedef struct { - wchar_t *path_env; /* PATH environment variable */ - - wchar_t *pythonpath_macro; /* PYTHONPATH macro */ - wchar_t *prefix_macro; /* PREFIX macro */ - wchar_t *exec_prefix_macro; /* EXEC_PREFIX macro */ - wchar_t *vpath_macro; /* VPATH macro */ - - wchar_t *lib_python; /* / "pythonX.Y" */ - - int prefix_found; /* found platform independent libraries? */ - int exec_prefix_found; /* found the platform dependent libraries? */ - - int warnings; - const wchar_t *pythonpath_env; - const wchar_t *platlibdir; - - wchar_t *argv0_path; - wchar_t *zip_path; - wchar_t *prefix; - wchar_t *exec_prefix; -} PyCalculatePath; - -static const wchar_t delimiter[2] = {DELIM, '\0'}; -static const wchar_t separator[2] = {SEP, '\0'}; +#if !defined(EXE_SUFFIX) +#if defined(MS_WINDOWS) || defined(__CYGWIN__) || defined(__MINGW32__) +#define EXE_SUFFIX L".exe" +#else +#define EXE_SUFFIX NULL +#endif +#endif -static void -reduce(wchar_t *dir) +/* HELPER FUNCTIONS for getpath.py */ + +static PyObject * +getpath_abspath(PyObject *Py_UNUSED(self), PyObject *args) { - size_t i = wcslen(dir); - while (i > 0 && dir[i] != SEP) { - --i; + PyObject *r = NULL; + PyObject *pathobj; + const wchar_t *path; + if (!PyArg_ParseTuple(args, "U", &pathobj)) { + return NULL; } - dir[i] = '\0'; -} - - -/* Is file, not directory */ -static int -isfile(const wchar_t *filename) -{ - struct stat buf; - if (_Py_wstat(filename, &buf) != 0) { - return 0; - } - if (!S_ISREG(buf.st_mode)) { - return 0; - } - return 1; -} - - -/* Is executable file */ -static int -isxfile(const wchar_t *filename) -{ - struct stat buf; - if (_Py_wstat(filename, &buf) != 0) { - return 0; - } - if (!S_ISREG(buf.st_mode)) { - return 0; - } - if ((buf.st_mode & 0111) == 0) { - return 0; - } - return 1; -} - - -/* Is directory */ -static int -isdir(const wchar_t *filename) -{ - struct stat buf; - if (_Py_wstat(filename, &buf) != 0) { - return 0; - } - if (!S_ISDIR(buf.st_mode)) { - return 0; - } - return 1; -} - - -/* Add a path component, by appending stuff to buffer. - buflen: 'buffer' length in characters including trailing NUL. - - If path2 is empty: - - - if path doesn't end with SEP and is not empty, add SEP to path - - otherwise, do nothing. */ -static PyStatus -joinpath(wchar_t *path, const wchar_t *path2, size_t path_len) -{ - if (_Py_isabs(path2)) { - if (wcslen(path2) >= path_len) { - return PATHLEN_ERR(); + Py_ssize_t len; + path = PyUnicode_AsWideCharString(pathobj, &len); + if (path) { + wchar_t *abs; + if (_Py_abspath(path, &abs) == 0 && abs) { + r = PyUnicode_FromWideChar(_Py_normpath(abs, -1), -1); + PyMem_RawFree((void *)abs); + } else { + PyErr_SetString(PyExc_OSError, "failed to make path absolute"); } - wcscpy(path, path2); + PyMem_Free((void *)path); } - else { - if (_Py_add_relfile(path, path2, path_len) < 0) { - return PATHLEN_ERR(); - } - return _PyStatus_OK(); - } - return _PyStatus_OK(); + return r; } -static wchar_t* -substring(const wchar_t *str, size_t len) +static PyObject * +getpath_basename(PyObject *Py_UNUSED(self), PyObject *args) { - wchar_t *substr = PyMem_RawMalloc((len + 1) * sizeof(wchar_t)); - if (substr == NULL) { + const char *path; + if (!PyArg_ParseTuple(args, "s", &path)) { + return NULL; + } + const char *name = strrchr(path, SEP); + return PyUnicode_FromString(name ? name + 1 : path); +} + + +static PyObject * +getpath_dirname(PyObject *Py_UNUSED(self), PyObject *args) +{ + const char *path; + if (!PyArg_ParseTuple(args, "s", &path)) { + return NULL; + } + const char *name = strrchr(path, SEP); + if (!name) { + return PyUnicode_FromStringAndSize(NULL, 0); + } + return PyUnicode_FromStringAndSize(path, (name - path)); +} + + +static PyObject * +getpath_isabs(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *r = NULL; + PyObject *pathobj; + const wchar_t *path; + if (!PyArg_ParseTuple(args, "U", &pathobj)) { + return NULL; + } + path = PyUnicode_AsWideCharString(pathobj, NULL); + if (path) { + r = _Py_isabs(path) ? Py_True : Py_False; + PyMem_Free((void *)path); + } + Py_XINCREF(r); + return r; +} + + +static PyObject * +getpath_hassuffix(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *r = NULL; + PyObject *pathobj; + PyObject *suffixobj; + const wchar_t *path; + const wchar_t *suffix; + if (!PyArg_ParseTuple(args, "UU", &pathobj, &suffixobj)) { + return NULL; + } + Py_ssize_t len, suffixLen; + path = PyUnicode_AsWideCharString(pathobj, &len); + if (path) { + suffix = PyUnicode_AsWideCharString(suffixobj, &suffixLen); + if (suffix) { + if (suffixLen < len || +#ifdef MS_WINDOWS + wcsicmp(&path[len - suffixLen], suffix) != 0 +#else + wcscmp(&path[len - suffixLen], suffix) != 0 +#endif + ) { + r = Py_False; + } else { + r = Py_True; + } + Py_INCREF(r); + PyMem_Free((void *)suffix); + } + PyMem_Free((void *)path); + } + return r; +} + + +static PyObject * +getpath_isdir(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *r = NULL; + PyObject *pathobj; + const wchar_t *path; + if (!PyArg_ParseTuple(args, "U", &pathobj)) { + return NULL; + } + path = PyUnicode_AsWideCharString(pathobj, NULL); + if (path) { +#ifdef MS_WINDOWS + r = (GetFileAttributesW(path) & FILE_ATTRIBUTE_DIRECTORY) ? Py_True : Py_False; +#else + struct stat st; + r = (_Py_wstat(path, &st) == 0) && S_ISDIR(st.st_mode) ? Py_True : Py_False; +#endif + PyMem_Free((void *)path); + } + Py_XINCREF(r); + return r; +} + + +static PyObject * +getpath_isfile(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *r = NULL; + PyObject *pathobj; + const wchar_t *path; + if (!PyArg_ParseTuple(args, "U", &pathobj)) { + return NULL; + } + path = PyUnicode_AsWideCharString(pathobj, NULL); + if (path) { +#ifdef MS_WINDOWS + r = !(GetFileAttributesW(path) & FILE_ATTRIBUTE_DIRECTORY) ? Py_True : Py_False; +#else + struct stat st; + r = (_Py_wstat(path, &st) == 0) && S_ISREG(st.st_mode) ? Py_True : Py_False; +#endif + PyMem_Free((void *)path); + } + Py_XINCREF(r); + return r; +} + + +static PyObject * +getpath_isxfile(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *r = NULL; + PyObject *pathobj; + const wchar_t *path; + Py_ssize_t cchPath; + if (!PyArg_ParseTuple(args, "U", &pathobj)) { + return NULL; + } + path = PyUnicode_AsWideCharString(pathobj, &cchPath); + if (path) { +#ifdef MS_WINDOWS + const wchar_t *ext; + r = (GetFileAttributesW(path) & FILE_ATTRIBUTE_DIRECTORY) && + SUCCEEDED(PathCchFindExtension(path, cchPath, &ext)) && + (CompareStringOrdinal(ext, -1, L".exe", -1, 1 /* ignore case */) == CSTR_EQUAL) + ? Py_True : Py_False; +#else + struct stat st; + r = (_Py_wstat(path, &st) == 0) && + S_ISREG(st.st_mode) && + (st.st_mode & 0111) + ? Py_True : Py_False; +#endif + PyMem_Free((void *)path); + } + Py_XINCREF(r); + return r; +} + + +static PyObject * +getpath_joinpath(PyObject *Py_UNUSED(self), PyObject *args) +{ + if (!PyTuple_Check(args)) { + PyErr_SetString(PyExc_TypeError, "requires tuple of arguments"); + return NULL; + } + Py_ssize_t n = PyTuple_GET_SIZE(args); + if (n == 0) { + return PyUnicode_FromString(NULL); + } + /* Convert all parts to wchar and accumulate max final length */ + wchar_t **parts = (wchar_t **)PyMem_Malloc(n * sizeof(wchar_t *)); + memset(parts, 0, n * sizeof(wchar_t *)); + Py_ssize_t cchFinal = 0; + Py_ssize_t first = 0; + + for (Py_ssize_t i = 0; i < n; ++i) { + PyObject *s = PyTuple_GET_ITEM(args, i); + Py_ssize_t cch; + if (s == Py_None) { + cch = 0; + } else if (PyUnicode_Check(s)) { + parts[i] = PyUnicode_AsWideCharString(s, &cch); + if (!parts[i]) { + cchFinal = -1; + break; + } + if (_Py_isabs(parts[i])) { + first = i; + } + } else { + PyErr_SetString(PyExc_TypeError, "all arguments to joinpath() must be str or None"); + cchFinal = -1; + break; + } + cchFinal += cch + 1; + } + + wchar_t *final = cchFinal > 0 ? (wchar_t *)PyMem_Malloc(cchFinal * sizeof(wchar_t)) : NULL; + if (!final) { + for (Py_ssize_t i = 0; i < n; ++i) { + PyMem_Free(parts[i]); + } + PyMem_Free(parts); + if (cchFinal) { + PyErr_NoMemory(); + return NULL; + } + return PyUnicode_FromStringAndSize(NULL, 0); + } + + final[0] = '\0'; + /* Now join all the paths. The final result should be shorter than the buffer */ + for (Py_ssize_t i = 0; i < n; ++i) { + if (!parts[i]) { + continue; + } + if (i >= first && final) { + if (!final[0]) { + /* final is definitely long enough to fit any individual part */ + wcscpy(final, parts[i]); + } else if (_Py_add_relfile(final, parts[i], cchFinal) < 0) { + /* if we fail, keep iterating to free memory, but stop adding parts */ + PyMem_Free(final); + final = NULL; + } + } + PyMem_Free(parts[i]); + } + PyMem_Free(parts); + if (!final) { + PyErr_SetString(PyExc_SystemError, "failed to join paths"); + return NULL; + } + PyObject *r = PyUnicode_FromWideChar(_Py_normpath(final, -1), -1); + PyMem_Free(final); + return r; +} + + +static PyObject * +getpath_readlines(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *r = NULL; + PyObject *pathobj; + const wchar_t *path; + if (!PyArg_ParseTuple(args, "U", &pathobj)) { + return NULL; + } + path = PyUnicode_AsWideCharString(pathobj, NULL); + if (!path) { + return NULL; + } + FILE *fp = _Py_wfopen(path, L"rb"); + PyMem_Free((void *)path); + if (!fp) { + PyErr_SetFromErrno(PyExc_OSError); return NULL; } - if (len) { - memcpy(substr, str, len * sizeof(wchar_t)); + r = PyList_New(0); + if (!r) { + fclose(fp); + return NULL; } - substr[len] = L'\0'; - return substr; -} - - -static wchar_t* -joinpath2(const wchar_t *path, const wchar_t *path2) -{ - if (_Py_isabs(path2)) { - return _PyMem_RawWcsdup(path2); - } - return _Py_join_relfile(path, path2); -} - - -static inline int -safe_wcscpy(wchar_t *dst, const wchar_t *src, size_t n) -{ - size_t srclen = wcslen(src); - if (n <= srclen) { - dst[0] = L'\0'; - return -1; - } - memcpy(dst, src, (srclen + 1) * sizeof(wchar_t)); - return 0; -} - - -/* copy_absolute requires that path be allocated at least - 'abs_path_len' characters (including trailing NUL). */ -static PyStatus -copy_absolute(wchar_t *abs_path, const wchar_t *path, size_t abs_path_len) -{ - if (_Py_isabs(path)) { - if (safe_wcscpy(abs_path, path, abs_path_len) < 0) { - return PATHLEN_ERR(); - } - } - else { - if (!_Py_wgetcwd(abs_path, abs_path_len)) { - /* unable to get the current directory */ - if (safe_wcscpy(abs_path, path, abs_path_len) < 0) { - return PATHLEN_ERR(); - } - return _PyStatus_OK(); - } - if (path[0] == '.' && path[1] == SEP) { - path += 2; - } - PyStatus status = joinpath(abs_path, path, abs_path_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - return _PyStatus_OK(); -} - - -/* path_len: path length in characters including trailing NUL */ -static PyStatus -absolutize(wchar_t **path_p) -{ - assert(!_Py_isabs(*path_p)); - - wchar_t abs_path[MAXPATHLEN+1]; - wchar_t *path = *path_p; - - PyStatus status = copy_absolute(abs_path, path, Py_ARRAY_LENGTH(abs_path)); - if (_PyStatus_EXCEPTION(status)) { - return status; + const size_t MAX_FILE = 32 * 1024; + char *buffer = (char *)PyMem_Malloc(MAX_FILE); + if (!buffer) { + Py_DECREF(r); + fclose(fp); + return NULL; } - PyMem_RawFree(*path_p); - *path_p = _PyMem_RawWcsdup(abs_path); - if (*path_p == NULL) { - return _PyStatus_NO_MEMORY(); - } - return _PyStatus_OK(); -} - - -/* Is module -- check for .pyc too */ -static PyStatus -ismodule(const wchar_t *path, int *result) -{ - wchar_t *filename = joinpath2(path, LANDMARK); - if (filename == NULL) { - return _PyStatus_NO_MEMORY(); - } - - if (isfile(filename)) { - PyMem_RawFree(filename); - *result = 1; - return _PyStatus_OK(); - } - - /* Check for the compiled version of prefix. */ - size_t len = wcslen(filename); - wchar_t *pyc = PyMem_RawMalloc((len + 2) * sizeof(wchar_t)); - if (pyc == NULL) { - PyMem_RawFree(filename); - return _PyStatus_NO_MEMORY(); - } - - memcpy(pyc, filename, len * sizeof(wchar_t)); - pyc[len] = L'c'; - pyc[len + 1] = L'\0'; - *result = isfile(pyc); - - PyMem_RawFree(filename); - PyMem_RawFree(pyc); - - return _PyStatus_OK(); -} - - -#if defined(__CYGWIN__) || defined(__MINGW32__) -#ifndef EXE_SUFFIX -#define EXE_SUFFIX L".exe" -#endif - -/* pathlen: 'path' length in characters including trailing NUL */ -static PyStatus -add_exe_suffix(wchar_t **progpath_p) -{ - wchar_t *progpath = *progpath_p; - - /* Check for already have an executable suffix */ - size_t n = wcslen(progpath); - size_t s = wcslen(EXE_SUFFIX); - if (wcsncasecmp(EXE_SUFFIX, progpath + n - s, s) == 0) { - return _PyStatus_OK(); - } - - wchar_t *progpath2 = PyMem_RawMalloc((n + s + 1) * sizeof(wchar_t)); - if (progpath2 == NULL) { - return _PyStatus_NO_MEMORY(); - } - - memcpy(progpath2, progpath, n * sizeof(wchar_t)); - memcpy(progpath2 + n, EXE_SUFFIX, s * sizeof(wchar_t)); - progpath2[n+s] = L'\0'; - - if (isxfile(progpath2)) { - PyMem_RawFree(*progpath_p); - *progpath_p = progpath2; - } - else { - PyMem_RawFree(progpath2); - } - return _PyStatus_OK(); -} -#endif - - -/* search_for_prefix requires that argv0_path be no more than MAXPATHLEN - bytes long. -*/ -static PyStatus -search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig, - wchar_t *prefix, size_t prefix_len, int *found) -{ - PyStatus status; - - /* If PYTHONHOME is set, we believe it unconditionally */ - if (pathconfig->home) { - /* Path: / */ - if (safe_wcscpy(prefix, pathconfig->home, prefix_len) < 0) { - return PATHLEN_ERR(); - } - wchar_t *delim = wcschr(prefix, DELIM); - if (delim) { - *delim = L'\0'; - } - status = joinpath(prefix, calculate->lib_python, prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - *found = 1; - return _PyStatus_OK(); - } - - /* Check to see if argv0_path is in the build directory - - Path: / */ - wchar_t *path = joinpath2(calculate->argv0_path, BUILD_LANDMARK); - if (path == NULL) { - return _PyStatus_NO_MEMORY(); - } - - int is_build_dir = isfile(path); - PyMem_RawFree(path); - - if (is_build_dir) { - /* argv0_path is the build directory (BUILD_LANDMARK exists), - now also check LANDMARK using ismodule(). */ - - /* Path: / / Lib */ - /* or if VPATH is empty: / Lib */ - if (safe_wcscpy(prefix, calculate->argv0_path, prefix_len) < 0) { - return PATHLEN_ERR(); - } - - status = joinpath(prefix, calculate->vpath_macro, prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - status = joinpath(prefix, L"Lib", prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - int module; - status = ismodule(prefix, &module); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - if (module) { - /* BUILD_LANDMARK and LANDMARK found */ - *found = -1; - return _PyStatus_OK(); - } - } - - /* Search from argv0_path, until root is found */ - status = copy_absolute(prefix, calculate->argv0_path, prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - do { - /* Path: / / LANDMARK */ - size_t n = wcslen(prefix); - status = joinpath(prefix, calculate->lib_python, prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - int module; - status = ismodule(prefix, &module); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - if (module) { - *found = 1; - return _PyStatus_OK(); - } - prefix[n] = L'\0'; - reduce(prefix); - } while (prefix[0]); - - /* Look at configure's PREFIX. - Path: / / LANDMARK */ - if (safe_wcscpy(prefix, calculate->prefix_macro, prefix_len) < 0) { - return PATHLEN_ERR(); - } - status = joinpath(prefix, calculate->lib_python, prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - int module; - status = ismodule(prefix, &module); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - if (module) { - *found = 1; - return _PyStatus_OK(); - } - - /* Fail */ - *found = 0; - return _PyStatus_OK(); -} - - -static PyStatus -calculate_set_stdlib_dir(PyCalculatePath *calculate, _PyPathConfig *pathconfig) -{ - // Note that, unlike calculate_set_prefix(), here we allow a negative - // prefix_found. That means the source tree Lib dir gets used. - if (!calculate->prefix_found) { - return _PyStatus_OK(); - } - PyStatus status; - wchar_t *prefix = calculate->prefix; - if (!_Py_isabs(prefix)) { - prefix = _PyMem_RawWcsdup(prefix); - if (prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - status = absolutize(&prefix); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - wchar_t buf[MAXPATHLEN + 1]; - int res = _Py_normalize_path(prefix, buf, Py_ARRAY_LENGTH(buf)); - if (prefix != calculate->prefix) { - PyMem_RawFree(prefix); - } - if (res < 0) { - return PATHLEN_ERR(); - } - pathconfig->stdlib_dir = _PyMem_RawWcsdup(buf); - if (pathconfig->stdlib_dir == NULL) { - return _PyStatus_NO_MEMORY(); - } - return _PyStatus_OK(); -} - - -static PyStatus -calculate_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig) -{ - wchar_t prefix[MAXPATHLEN+1]; - memset(prefix, 0, sizeof(prefix)); - size_t prefix_len = Py_ARRAY_LENGTH(prefix); - - PyStatus status; - status = search_for_prefix(calculate, pathconfig, - prefix, prefix_len, - &calculate->prefix_found); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - if (!calculate->prefix_found) { - if (calculate->warnings) { - fprintf(stderr, - "Could not find platform independent libraries \n"); - } - - calculate->prefix = joinpath2(calculate->prefix_macro, - calculate->lib_python); - } - else { - calculate->prefix = _PyMem_RawWcsdup(prefix); - } - - if (calculate->prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - return _PyStatus_OK(); -} - - -static PyStatus -calculate_set_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig) -{ - /* Reduce prefix and exec_prefix to their essence, - * e.g. /usr/local/lib/python1.5 is reduced to /usr/local. - * If we're loading relative to the build directory, - * return the compiled-in defaults instead. - */ - if (calculate->prefix_found > 0) { - wchar_t *prefix = _PyMem_RawWcsdup(calculate->prefix); - if (prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - - reduce(prefix); - reduce(prefix); - if (prefix[0]) { - pathconfig->prefix = prefix; - } - else { - PyMem_RawFree(prefix); - - /* The prefix is the root directory, but reduce() chopped - off the "/". */ - pathconfig->prefix = _PyMem_RawWcsdup(separator); - if (pathconfig->prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - } - } - else { - pathconfig->prefix = _PyMem_RawWcsdup(calculate->prefix_macro); - if (pathconfig->prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - } - return _PyStatus_OK(); -} - - -static PyStatus -calculate_pybuilddir(const wchar_t *argv0_path, - wchar_t *exec_prefix, size_t exec_prefix_len, - int *found) -{ - PyStatus status; - - /* Check to see if argv[0] is in the build directory. "pybuilddir.txt" - is written by setup.py and contains the relative path to the location - of shared library modules. - - Filename: / "pybuilddir.txt" */ - wchar_t *filename = joinpath2(argv0_path, L"pybuilddir.txt"); - if (filename == NULL) { - return _PyStatus_NO_MEMORY(); - } - - FILE *fp = _Py_wfopen(filename, L"rb"); - PyMem_RawFree(filename); - if (fp == NULL) { - errno = 0; - return _PyStatus_OK(); - } - - char buf[MAXPATHLEN + 1]; - size_t n = fread(buf, 1, Py_ARRAY_LENGTH(buf) - 1, fp); - buf[n] = '\0'; + size_t cb = fread(buffer, 1, MAX_FILE, fp); fclose(fp); - - size_t dec_len; - wchar_t *pybuilddir = _Py_DecodeUTF8_surrogateescape(buf, n, &dec_len); - if (!pybuilddir) { - return DECODE_LOCALE_ERR("pybuilddir.txt", dec_len); + if (!cb) { + return r; } - - /* Path: / */ - if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) { - PyMem_RawFree(pybuilddir); - return PATHLEN_ERR(); - } - status = joinpath(exec_prefix, pybuilddir, exec_prefix_len); - PyMem_RawFree(pybuilddir); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - *found = -1; - return _PyStatus_OK(); -} - - -/* search_for_exec_prefix requires that argv0_path be no more than - MAXPATHLEN bytes long. -*/ -static PyStatus -search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig, - wchar_t *exec_prefix, size_t exec_prefix_len, - int *found) -{ - PyStatus status; - - /* If PYTHONHOME is set, we believe it unconditionally */ - if (pathconfig->home) { - /* Path: / / "lib-dynload" */ - wchar_t *delim = wcschr(pathconfig->home, DELIM); - if (delim) { - if (safe_wcscpy(exec_prefix, delim+1, exec_prefix_len) < 0) { - return PATHLEN_ERR(); - } - } - else { - if (safe_wcscpy(exec_prefix, pathconfig->home, exec_prefix_len) < 0) { - return PATHLEN_ERR(); - } - } - status = joinpath(exec_prefix, calculate->lib_python, exec_prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - status = joinpath(exec_prefix, L"lib-dynload", exec_prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - *found = 1; - return _PyStatus_OK(); - } - - /* Check for pybuilddir.txt */ - assert(*found == 0); - status = calculate_pybuilddir(calculate->argv0_path, - exec_prefix, exec_prefix_len, found); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - if (*found) { - return _PyStatus_OK(); - } - - /* Search from argv0_path, until root is found */ - status = copy_absolute(exec_prefix, calculate->argv0_path, exec_prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - do { - /* Path: / / "lib-dynload" */ - size_t n = wcslen(exec_prefix); - status = joinpath(exec_prefix, calculate->lib_python, exec_prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - status = joinpath(exec_prefix, L"lib-dynload", exec_prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - if (isdir(exec_prefix)) { - *found = 1; - return _PyStatus_OK(); - } - exec_prefix[n] = L'\0'; - reduce(exec_prefix); - } while (exec_prefix[0]); - - /* Look at configure's EXEC_PREFIX. - - Path: / / "lib-dynload" */ - if (safe_wcscpy(exec_prefix, calculate->exec_prefix_macro, exec_prefix_len) < 0) { - return PATHLEN_ERR(); - } - status = joinpath(exec_prefix, calculate->lib_python, exec_prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - status = joinpath(exec_prefix, L"lib-dynload", exec_prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - if (isdir(exec_prefix)) { - *found = 1; - return _PyStatus_OK(); - } - - /* Fail */ - *found = 0; - return _PyStatus_OK(); -} - - -static PyStatus -calculate_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig) -{ - PyStatus status; - wchar_t exec_prefix[MAXPATHLEN+1]; - memset(exec_prefix, 0, sizeof(exec_prefix)); - size_t exec_prefix_len = Py_ARRAY_LENGTH(exec_prefix); - - status = search_for_exec_prefix(calculate, pathconfig, - exec_prefix, exec_prefix_len, - &calculate->exec_prefix_found); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - if (!calculate->exec_prefix_found) { - if (calculate->warnings) { - fprintf(stderr, - "Could not find platform dependent libraries \n"); - } - - /* / "lib-dynload" */ - wchar_t *lib_dynload = joinpath2(calculate->platlibdir, - L"lib-dynload"); - if (lib_dynload == NULL) { - return _PyStatus_NO_MEMORY(); - } - - calculate->exec_prefix = joinpath2(calculate->exec_prefix_macro, - lib_dynload); - PyMem_RawFree(lib_dynload); - - if (calculate->exec_prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - } - else { - /* If we found EXEC_PREFIX do *not* reduce it! (Yet.) */ - calculate->exec_prefix = _PyMem_RawWcsdup(exec_prefix); - if (calculate->exec_prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - } - return _PyStatus_OK(); -} - - -static PyStatus -calculate_set_exec_prefix(PyCalculatePath *calculate, - _PyPathConfig *pathconfig) -{ - if (calculate->exec_prefix_found > 0) { - wchar_t *exec_prefix = _PyMem_RawWcsdup(calculate->exec_prefix); - if (exec_prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - - reduce(exec_prefix); - reduce(exec_prefix); - reduce(exec_prefix); - - if (exec_prefix[0]) { - pathconfig->exec_prefix = exec_prefix; - } - else { - /* empty string: use SEP instead */ - PyMem_RawFree(exec_prefix); - - /* The exec_prefix is the root directory, but reduce() chopped - off the "/". */ - pathconfig->exec_prefix = _PyMem_RawWcsdup(separator); - if (pathconfig->exec_prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - } - } - else { - pathconfig->exec_prefix = _PyMem_RawWcsdup(calculate->exec_prefix_macro); - if (pathconfig->exec_prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - } - return _PyStatus_OK(); -} - - -/* Similar to shutil.which(). - If found, write the path into *abs_path_p. */ -static PyStatus -calculate_which(const wchar_t *path_env, wchar_t *program_name, - wchar_t **abs_path_p) -{ - while (1) { - wchar_t *delim = wcschr(path_env, DELIM); - wchar_t *abs_path; - - if (delim) { - wchar_t *path = substring(path_env, delim - path_env); - if (path == NULL) { - return _PyStatus_NO_MEMORY(); - } - abs_path = joinpath2(path, program_name); - PyMem_RawFree(path); - } - else { - abs_path = joinpath2(path_env, program_name); - } - - if (abs_path == NULL) { - return _PyStatus_NO_MEMORY(); - } - - if (isxfile(abs_path)) { - *abs_path_p = abs_path; - return _PyStatus_OK(); - } - PyMem_RawFree(abs_path); - - if (!delim) { - break; - } - path_env = delim + 1; - } - - /* not found */ - return _PyStatus_OK(); -} - - -#ifdef __APPLE__ -static PyStatus -calculate_program_macos(wchar_t **abs_path_p) -{ - char execpath[MAXPATHLEN + 1]; - uint32_t nsexeclength = Py_ARRAY_LENGTH(execpath) - 1; - - /* On Mac OS X, if a script uses an interpreter of the form - "#!/opt/python2.3/bin/python", the kernel only passes "python" - as argv[0], which falls through to the $PATH search below. - If /opt/python2.3/bin isn't in your path, or is near the end, - this algorithm may incorrectly find /usr/bin/python. To work - around this, we can use _NSGetExecutablePath to get a better - hint of what the intended interpreter was, although this - will fail if a relative path was used. but in that case, - absolutize() should help us out below - */ - if (_NSGetExecutablePath(execpath, &nsexeclength) != 0 - || (wchar_t)execpath[0] != SEP) - { - /* _NSGetExecutablePath() failed or the path is relative */ - return _PyStatus_OK(); + if (cb >= MAX_FILE) { + Py_DECREF(r); + PyErr_SetString(PyExc_MemoryError, + "cannot read file larger than 32KB during initialization"); + return NULL; } + buffer[cb] = '\0'; size_t len; - *abs_path_p = Py_DecodeLocale(execpath, &len); - if (*abs_path_p == NULL) { - return DECODE_LOCALE_ERR("executable path", len); + wchar_t *wbuffer = _Py_DecodeUTF8_surrogateescape(buffer, cb, &len); + PyMem_Free((void *)buffer); + if (!wbuffer) { + Py_DECREF(r); + PyErr_NoMemory(); + return NULL; } - return _PyStatus_OK(); -} -#endif /* __APPLE__ */ - -static PyStatus -calculate_program_impl(PyCalculatePath *calculate, _PyPathConfig *pathconfig) -{ - assert(pathconfig->program_full_path == NULL); - - PyStatus status; - - /* If there is no slash in the argv0 path, then we have to - * assume python is on the user's $PATH, since there's no - * other way to find a directory to start the search from. If - * $PATH isn't exported, you lose. - */ - if (wcschr(pathconfig->program_name, SEP)) { - pathconfig->program_full_path = _PyMem_RawWcsdup(pathconfig->program_name); - if (pathconfig->program_full_path == NULL) { - return _PyStatus_NO_MEMORY(); + wchar_t *p1 = wbuffer; + wchar_t *p2 = p1; + while ((p2 = wcschr(p1, L'\n')) != NULL) { + size_t cb = p2 - p1; + while (cb && (p1[cb] == L'\n' || p1[cb] == L'\r')) { + --cb; } - return _PyStatus_OK(); - } - -#ifdef __APPLE__ - wchar_t *abs_path = NULL; - status = calculate_program_macos(&abs_path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - if (abs_path) { - pathconfig->program_full_path = abs_path; - return _PyStatus_OK(); - } -#endif /* __APPLE__ */ - - if (calculate->path_env) { - wchar_t *abs_path = NULL; - status = calculate_which(calculate->path_env, pathconfig->program_name, - &abs_path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - if (abs_path) { - pathconfig->program_full_path = abs_path; - return _PyStatus_OK(); - } - } - - /* In the last resort, use an empty string */ - pathconfig->program_full_path = _PyMem_RawWcsdup(L""); - if (pathconfig->program_full_path == NULL) { - return _PyStatus_NO_MEMORY(); - } - return _PyStatus_OK(); -} - - -/* Calculate pathconfig->program_full_path */ -static PyStatus -calculate_program(PyCalculatePath *calculate, _PyPathConfig *pathconfig) -{ - PyStatus status; - - status = calculate_program_impl(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - if (pathconfig->program_full_path[0] != '\0') { - /* program_full_path is not empty */ - - /* Make sure that program_full_path is an absolute path */ - if (!_Py_isabs(pathconfig->program_full_path)) { - status = absolutize(&pathconfig->program_full_path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - -#if defined(__CYGWIN__) || defined(__MINGW32__) - /* For these platforms it is necessary to ensure that the .exe suffix - * is appended to the filename, otherwise there is potential for - * sys.executable to return the name of a directory under the same - * path (bpo-28441). - */ - status = add_exe_suffix(&pathconfig->program_full_path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } -#endif - } - return _PyStatus_OK(); -} - - -#if HAVE_READLINK -static PyStatus -resolve_symlinks(wchar_t **path_p) -{ - wchar_t new_path[MAXPATHLEN + 1]; - const size_t new_path_len = Py_ARRAY_LENGTH(new_path); - unsigned int nlink = 0; - - while (1) { - int linklen = _Py_wreadlink(*path_p, new_path, new_path_len); - if (linklen == -1) { - /* not a symbolic link: we are done */ + PyObject *u = PyUnicode_FromWideChar(p1, cb + 1); + if (!u || PyList_Append(r, u) < 0) { + Py_XDECREF(u); + Py_CLEAR(r); break; } - - if (_Py_isabs(new_path)) { - PyMem_RawFree(*path_p); - *path_p = _PyMem_RawWcsdup(new_path); - if (*path_p == NULL) { - return _PyStatus_NO_MEMORY(); - } + Py_DECREF(u); + p1 = p2 + 1; + } + if (r && p1 && *p1) { + PyObject *u = PyUnicode_FromWideChar(p1, -1); + if (!u || PyList_Append(r, u) < 0) { + Py_CLEAR(r); } - else { - /* new_path is relative to path */ - reduce(*path_p); + Py_XDECREF(u); + } + PyMem_RawFree(wbuffer); + return r; +} - wchar_t *abs_path = joinpath2(*path_p, new_path); - if (abs_path == NULL) { - return _PyStatus_NO_MEMORY(); - } - PyMem_RawFree(*path_p); - *path_p = abs_path; +static PyObject * +getpath_realpath(PyObject *Py_UNUSED(self) , PyObject *args) +{ + PyObject *pathobj; + if (!PyArg_ParseTuple(args, "U", &pathobj)) { + return NULL; + } +#if defined(HAVE_READLINK) + /* This readlink calculation only resolves a symlinked file, and + does not resolve any path segments. This is consistent with + prior releases, however, the realpath implementation below is + potentially correct in more cases. */ + PyObject *r = NULL; + int nlink = 0; + wchar_t *path = PyUnicode_AsWideCharString(pathobj, NULL); + if (!path) { + goto done; + } + wchar_t *path2 = _PyMem_RawWcsdup(path); + PyMem_Free((void *)path); + path = path2; + while (path) { + wchar_t resolved[MAXPATHLEN + 1]; + int linklen = _Py_wreadlink(path, resolved, Py_ARRAY_LENGTH(resolved)); + if (linklen == -1) { + r = PyUnicode_FromWideChar(path, -1); + break; + } + if (_Py_isabs(resolved)) { + PyMem_RawFree((void *)path); + path = _PyMem_RawWcsdup(resolved); + } else { + wchar_t *s = wcsrchr(path, SEP); + if (s) { + *s = L'\0'; + } + path2 = _Py_normpath(_Py_join_relfile(path, resolved), -1); + PyMem_RawFree((void *)path); + path = path2; } - nlink++; /* 40 is the Linux kernel 4.2 limit */ if (nlink >= 40) { - return _PyStatus_ERR("maximum number of symbolic links reached"); - } - } - return _PyStatus_OK(); -} -#endif /* HAVE_READLINK */ - - -#ifdef WITH_NEXT_FRAMEWORK -static PyStatus -calculate_argv0_path_framework(PyCalculatePath *calculate, _PyPathConfig *pathconfig) -{ - NSModule pythonModule; - - /* On Mac OS X we have a special case if we're running from a framework. - This is because the python home should be set relative to the library, - which is in the framework, not relative to the executable, which may - be outside of the framework. Except when we're in the build - directory... */ - pythonModule = NSModuleForSymbol(NSLookupAndBindSymbol("_Py_Initialize")); - - /* Use dylib functions to find out where the framework was loaded from */ - const char* modPath = NSLibraryNameForModule(pythonModule); - if (modPath == NULL) { - return _PyStatus_OK(); - } - - /* We're in a framework. - See if we might be in the build directory. The framework in the - build directory is incomplete, it only has the .dylib and a few - needed symlinks, it doesn't have the Lib directories and such. - If we're running with the framework from the build directory we must - be running the interpreter in the build directory, so we use the - build-directory-specific logic to find Lib and such. */ - size_t len; - wchar_t* wbuf = Py_DecodeLocale(modPath, &len); - if (wbuf == NULL) { - return DECODE_LOCALE_ERR("framework location", len); - } - - /* Path: reduce(modPath) / lib_python / LANDMARK */ - PyStatus status; - - wchar_t *parent = _PyMem_RawWcsdup(wbuf); - if (parent == NULL) { - status = _PyStatus_NO_MEMORY(); - goto done; - } - - reduce(parent); - wchar_t *lib_python = joinpath2(parent, calculate->lib_python); - PyMem_RawFree(parent); - - if (lib_python == NULL) { - status = _PyStatus_NO_MEMORY(); - goto done; - } - - int module; - status = ismodule(lib_python, &module); - PyMem_RawFree(lib_python); - - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - if (!module) { - /* We are in the build directory so use the name of the - executable - we know that the absolute path is passed */ - PyMem_RawFree(calculate->argv0_path); - calculate->argv0_path = _PyMem_RawWcsdup(pathconfig->program_full_path); - if (calculate->argv0_path == NULL) { - status = _PyStatus_NO_MEMORY(); - goto done; - } - - status = _PyStatus_OK(); - goto done; - } - - /* Use the location of the library as argv0_path */ - PyMem_RawFree(calculate->argv0_path); - calculate->argv0_path = wbuf; - return _PyStatus_OK(); - -done: - PyMem_RawFree(wbuf); - return status; -} -#endif - - -static PyStatus -calculate_argv0_path(PyCalculatePath *calculate, - _PyPathConfig *pathconfig) -{ - PyStatus status; - - calculate->argv0_path = _PyMem_RawWcsdup(pathconfig->program_full_path); - if (calculate->argv0_path == NULL) { - return _PyStatus_NO_MEMORY(); - } - -#ifdef WITH_NEXT_FRAMEWORK - status = calculate_argv0_path_framework(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } -#endif - - status = resolve_symlinks(&calculate->argv0_path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - reduce(calculate->argv0_path); - - return _PyStatus_OK(); -} - - -static PyStatus -calculate_open_pyenv(PyCalculatePath *calculate, FILE **env_file_p) -{ - *env_file_p = NULL; - - const wchar_t *env_cfg = L"pyvenv.cfg"; - - /* Filename: / "pyvenv.cfg" */ - wchar_t *filename = joinpath2(calculate->argv0_path, env_cfg); - if (filename == NULL) { - return _PyStatus_NO_MEMORY(); - } - - *env_file_p = _Py_wfopen(filename, L"r"); - PyMem_RawFree(filename); - - if (*env_file_p != NULL) { - return _PyStatus_OK(); - - } - - /* fopen() failed: reset errno */ - errno = 0; - - /* Path: / "pyvenv.cfg" */ - wchar_t *parent = _PyMem_RawWcsdup(calculate->argv0_path); - if (parent == NULL) { - return _PyStatus_NO_MEMORY(); - } - reduce(parent); - - filename = joinpath2(parent, env_cfg); - PyMem_RawFree(parent); - if (filename == NULL) { - return _PyStatus_NO_MEMORY(); - } - - *env_file_p = _Py_wfopen(filename, L"r"); - PyMem_RawFree(filename); - - if (*env_file_p == NULL) { - /* fopen() failed: reset errno */ - errno = 0; - } - return _PyStatus_OK(); -} - - -/* Search for an "pyvenv.cfg" environment configuration file, first in the - executable's directory and then in the parent directory. - If found, open it for use when searching for prefixes. - - Write the 'home' variable of pyvenv.cfg into calculate->argv0_path. */ -static PyStatus -calculate_read_pyenv(PyCalculatePath *calculate) -{ - PyStatus status; - FILE *env_file = NULL; - - status = calculate_open_pyenv(calculate, &env_file); - if (_PyStatus_EXCEPTION(status)) { - assert(env_file == NULL); - return status; - } - if (env_file == NULL) { - /* pyvenv.cfg not found */ - return _PyStatus_OK(); - } - - /* Look for a 'home' variable and set argv0_path to it, if found */ - wchar_t *home = NULL; - status = _Py_FindEnvConfigValue(env_file, L"home", &home); - if (_PyStatus_EXCEPTION(status)) { - fclose(env_file); - return status; - } - - if (home) { - PyMem_RawFree(calculate->argv0_path); - calculate->argv0_path = home; - } - fclose(env_file); - return _PyStatus_OK(); -} - - -static PyStatus -calculate_zip_path(PyCalculatePath *calculate) -{ - PyStatus res; - - /* Path: / "pythonXY.zip" */ - wchar_t *path = joinpath2(calculate->platlibdir, - L"python" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) - L".zip"); - if (path == NULL) { - return _PyStatus_NO_MEMORY(); - } - - if (calculate->prefix_found > 0) { - /* Use the reduced prefix returned by Py_GetPrefix() - - Path: / / "pythonXY.zip" */ - wchar_t *parent = _PyMem_RawWcsdup(calculate->prefix); - if (parent == NULL) { - res = _PyStatus_NO_MEMORY(); - goto done; - } - reduce(parent); - reduce(parent); - calculate->zip_path = joinpath2(parent, path); - PyMem_RawFree(parent); - } - else { - calculate->zip_path = joinpath2(calculate->prefix_macro, path); - } - - if (calculate->zip_path == NULL) { - res = _PyStatus_NO_MEMORY(); - goto done; - } - - res = _PyStatus_OK(); - -done: - PyMem_RawFree(path); - return res; -} - - -static PyStatus -calculate_module_search_path(PyCalculatePath *calculate, - _PyPathConfig *pathconfig) -{ - /* Calculate size of return buffer */ - size_t bufsz = 0; - if (calculate->pythonpath_env != NULL) { - bufsz += wcslen(calculate->pythonpath_env) + 1; - } - - wchar_t *defpath = calculate->pythonpath_macro; - size_t prefixsz = wcslen(calculate->prefix) + 1; - while (1) { - wchar_t *delim = wcschr(defpath, DELIM); - - if (!_Py_isabs(defpath)) { - /* Paths are relative to prefix */ - bufsz += prefixsz; - } - - if (delim) { - bufsz += delim - defpath + 1; - } - else { - bufsz += wcslen(defpath) + 1; + PyErr_SetString(PyExc_OSError, "maximum number of symbolic links reached"); break; } - defpath = delim + 1; } - - bufsz += wcslen(calculate->zip_path) + 1; - bufsz += wcslen(calculate->exec_prefix) + 1; - - /* Allocate the buffer */ - wchar_t *buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); - if (buf == NULL) { - return _PyStatus_NO_MEMORY(); + if (!path) { + PyErr_NoMemory(); } - buf[0] = '\0'; +done: + PyMem_RawFree((void *)path); + return r; - /* Run-time value of $PYTHONPATH goes first */ - if (calculate->pythonpath_env) { - wcscpy(buf, calculate->pythonpath_env); - wcscat(buf, delimiter); +#elif defined(HAVE_REALPATH) + PyObject *r = NULL; + struct stat st; + const char *narrow = NULL; + wchar_t *path = PyUnicode_AsWideCharString(pathobj, NULL); + if (!path) { + goto done; } + narrow = Py_EncodeLocale(path, NULL); + if (!narrow) { + PyErr_NoMemory(); + goto done; + } + if (lstat(narrow, &st)) { + PyErr_SetFromErrno(PyExc_OSError); + goto done; + } + if (!S_ISLNK(st.st_mode)) { + Py_INCREF(pathobj); + r = pathobj; + goto done; + } + wchar_t resolved[MAXPATHLEN+1]; + if (_Py_wrealpath(path, resolved, MAXPATHLEN) == NULL) { + PyErr_SetFromErrno(PyExc_OSError); + } else { + r = PyUnicode_FromWideChar(resolved, -1); + } +done: + PyMem_Free((void *)path); + PyMem_Free((void *)narrow); + return r; +#endif - /* Next is the default zip path */ - wcscat(buf, calculate->zip_path); - wcscat(buf, delimiter); + Py_INCREF(pathobj); + return pathobj; +} - /* Next goes merge of compile-time $PYTHONPATH with - * dynamically located prefix. - */ - defpath = calculate->pythonpath_macro; - while (1) { - wchar_t *delim = wcschr(defpath, DELIM); - if (!_Py_isabs(defpath)) { - wcscat(buf, calculate->prefix); - if (prefixsz >= 2 && calculate->prefix[prefixsz - 2] != SEP && - defpath[0] != (delim ? DELIM : L'\0')) - { - /* not empty */ - wcscat(buf, separator); +static PyMethodDef getpath_methods[] = { + {"abspath", getpath_abspath, METH_VARARGS, NULL}, + {"basename", getpath_basename, METH_VARARGS, NULL}, + {"dirname", getpath_dirname, METH_VARARGS, NULL}, + {"hassuffix", getpath_hassuffix, METH_VARARGS, NULL}, + {"isabs", getpath_isabs, METH_VARARGS, NULL}, + {"isdir", getpath_isdir, METH_VARARGS, NULL}, + {"isfile", getpath_isfile, METH_VARARGS, NULL}, + {"isxfile", getpath_isxfile, METH_VARARGS, NULL}, + {"joinpath", getpath_joinpath, METH_VARARGS, NULL}, + {"readlines", getpath_readlines, METH_VARARGS, NULL}, + {"realpath", getpath_realpath, METH_VARARGS, NULL}, + {NULL, NULL, 0, NULL} +}; + + +/* Two implementations of warn() to use depending on whether warnings + are enabled or not. */ + +static PyObject * +getpath_warn(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *msgobj; + if (!PyArg_ParseTuple(args, "U", &msgobj)) { + return NULL; + } + fprintf(stderr, "%s\n", PyUnicode_AsUTF8(msgobj)); + Py_RETURN_NONE; +} + + +static PyObject * +getpath_nowarn(PyObject *Py_UNUSED(self), PyObject *args) +{ + Py_RETURN_NONE; +} + + +static PyMethodDef getpath_warn_method = {"warn", getpath_warn, METH_VARARGS, NULL}; +static PyMethodDef getpath_nowarn_method = {"warn", getpath_nowarn, METH_VARARGS, NULL}; + +/* Add the helper functions to the dict */ +static int +funcs_to_dict(PyObject *dict, int warnings) +{ + for (PyMethodDef *m = getpath_methods; m->ml_name; ++m) { + PyObject *f = PyCFunction_NewEx(m, NULL, NULL); + if (!f) { + return 0; + } + if (PyDict_SetItemString(dict, m->ml_name, f) < 0) { + Py_DECREF(f); + return 0; + } + Py_DECREF(f); + } + PyMethodDef *m2 = warnings ? &getpath_warn_method : &getpath_nowarn_method; + PyObject *f = PyCFunction_NewEx(m2, NULL, NULL); + if (!f) { + return 0; + } + if (PyDict_SetItemString(dict, m2->ml_name, f) < 0) { + Py_DECREF(f); + return 0; + } + Py_DECREF(f); + return 1; +} + + +/* Add a wide-character string constant to the dict */ +static int +wchar_to_dict(PyObject *dict, const char *key, const wchar_t *s) +{ + PyObject *u; + int r; + if (s && s[0]) { + u = PyUnicode_FromWideChar(s, -1); + if (!u) { + return 0; + } + } else { + u = Py_None; + Py_INCREF(u); + } + r = PyDict_SetItemString(dict, key, u) == 0; + Py_DECREF(u); + return r; +} + + +/* Add a narrow string constant to the dict, using default locale decoding */ +static int +decode_to_dict(PyObject *dict, const char *key, const char *s) +{ + PyObject *u = NULL; + int r; + if (s && s[0]) { + size_t len; + const wchar_t *w = Py_DecodeLocale(s, &len); + if (w) { + u = PyUnicode_FromWideChar(w, len); + PyMem_RawFree((void *)w); + } + if (!u) { + return 0; + } + } else { + u = Py_None; + Py_INCREF(u); + } + r = PyDict_SetItemString(dict, key, u) == 0; + Py_DECREF(u); + return r; +} + +/* Add an environment variable to the dict, optionally clearing it afterwards */ +static int +env_to_dict(PyObject *dict, const char *key, int and_clear) +{ + PyObject *u = NULL; + int r = 0; + assert(strncmp(key, "ENV_", 4) == 0); + assert(strlen(key) < 64); +#ifdef MS_WINDOWS + wchar_t wkey[64]; + // Quick convert to wchar_t, since we know key is ASCII + wchar_t *wp = wkey; + for (const char *p = &key[4]; *p; ++p) { + assert(*p < 128); + *wp++ = *p; + } + *wp = L'\0'; + const wchar_t *v = _wgetenv(wkey); + if (v) { + u = PyUnicode_FromWideChar(v, -1); + if (!u) { + PyErr_Clear(); + } + } +#else + const char *v = getenv(&key[4]); + if (v) { + size_t len; + const wchar_t *w = Py_DecodeLocale(v, &len); + if (w) { + u = PyUnicode_FromWideChar(w, len); + if (!u) { + PyErr_Clear(); + } + PyMem_RawFree((void *)w); + } + } +#endif + if (u) { + r = PyDict_SetItemString(dict, key, u) == 0; + Py_DECREF(u); + } else { + r = PyDict_SetItemString(dict, key, Py_None) == 0; + } + if (r && and_clear) { +#ifdef MS_WINDOWS + _wputenv_s(wkey, L""); +#else + unsetenv(&key[4]); +#endif + } + return r; +} + + +/* Add an integer constant to the dict */ +static int +int_to_dict(PyObject *dict, const char *key, int v) +{ + PyObject *o; + int r; + o = PyLong_FromLong(v); + if (!o) { + return 0; + } + r = PyDict_SetItemString(dict, key, o) == 0; + Py_DECREF(o); + return r; +} + + +#ifdef MS_WINDOWS +static int +winmodule_to_dict(PyObject *dict, const char *key, HMODULE mod) +{ + wchar_t *buffer = NULL; + for (DWORD cch = 256; buffer == NULL && cch < (1024 * 1024); cch *= 2) { + buffer = (wchar_t*)PyMem_RawMalloc(cch * sizeof(wchar_t)); + if (buffer) { + if (GetModuleFileNameW(mod, buffer, cch) == cch) { + PyMem_RawFree(buffer); + buffer = NULL; } } - - if (delim) { - size_t len = delim - defpath + 1; - size_t end = wcslen(buf) + len; - wcsncat(buf, defpath, len); - buf[end] = '\0'; - } - else { - wcscat(buf, defpath); - break; - } - defpath = delim + 1; } - wcscat(buf, delimiter); - - /* Finally, on goes the directory for dynamic-load modules */ - wcscat(buf, calculate->exec_prefix); - - pathconfig->module_search_path = buf; - return _PyStatus_OK(); -} - - -static PyStatus -calculate_init(PyCalculatePath *calculate, const PyConfig *config) -{ - size_t len; - - calculate->warnings = config->pathconfig_warnings; - calculate->pythonpath_env = config->pythonpath_env; - calculate->platlibdir = config->platlibdir; - - const char *path = getenv("PATH"); - if (path) { - calculate->path_env = Py_DecodeLocale(path, &len); - if (!calculate->path_env) { - return DECODE_LOCALE_ERR("PATH environment variable", len); - } - } - - /* Decode macros */ - calculate->pythonpath_macro = Py_DecodeLocale(PYTHONPATH, &len); - if (!calculate->pythonpath_macro) { - return DECODE_LOCALE_ERR("PYTHONPATH macro", len); - } - calculate->prefix_macro = Py_DecodeLocale(PREFIX, &len); - if (!calculate->prefix_macro) { - return DECODE_LOCALE_ERR("PREFIX macro", len); - } - calculate->exec_prefix_macro = Py_DecodeLocale(EXEC_PREFIX, &len); - if (!calculate->exec_prefix_macro) { - return DECODE_LOCALE_ERR("EXEC_PREFIX macro", len); - } - calculate->vpath_macro = Py_DecodeLocale(VPATH, &len); - if (!calculate->vpath_macro) { - return DECODE_LOCALE_ERR("VPATH macro", len); - } - - // / "pythonX.Y" - wchar_t *pyversion = Py_DecodeLocale("python" VERSION, &len); - if (!pyversion) { - return DECODE_LOCALE_ERR("VERSION macro", len); - } - calculate->lib_python = joinpath2(config->platlibdir, pyversion); - PyMem_RawFree(pyversion); - if (calculate->lib_python == NULL) { - return _PyStatus_NO_MEMORY(); - } - - return _PyStatus_OK(); -} - - -static void -calculate_free(PyCalculatePath *calculate) -{ - PyMem_RawFree(calculate->pythonpath_macro); - PyMem_RawFree(calculate->prefix_macro); - PyMem_RawFree(calculate->exec_prefix_macro); - PyMem_RawFree(calculate->vpath_macro); - PyMem_RawFree(calculate->lib_python); - PyMem_RawFree(calculate->path_env); - PyMem_RawFree(calculate->zip_path); - PyMem_RawFree(calculate->argv0_path); - PyMem_RawFree(calculate->prefix); - PyMem_RawFree(calculate->exec_prefix); -} - - -static PyStatus -calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig) -{ - PyStatus status; - - if (pathconfig->program_full_path == NULL) { - status = calculate_program(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - - status = calculate_argv0_path(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - /* If a pyvenv.cfg configure file is found, - argv0_path is overridden with its 'home' variable. */ - status = calculate_read_pyenv(calculate); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - status = calculate_prefix(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - status = calculate_zip_path(calculate); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - status = calculate_exec_prefix(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - if ((!calculate->prefix_found || !calculate->exec_prefix_found) - && calculate->warnings) - { - fprintf(stderr, - "Consider setting $PYTHONHOME to [:]\n"); - } - - if (pathconfig->module_search_path == NULL) { - status = calculate_module_search_path(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - - if (pathconfig->stdlib_dir == NULL) { - /* This must be done *before* calculate_set_prefix() is called. */ - status = calculate_set_stdlib_dir(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - - if (pathconfig->prefix == NULL) { - status = calculate_set_prefix(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - - if (pathconfig->exec_prefix == NULL) { - status = calculate_set_exec_prefix(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - return _PyStatus_OK(); -} - - -/* Calculate the Python path configuration. - - Inputs: - - - PATH environment variable - - Macros: PYTHONPATH, PREFIX, EXEC_PREFIX, VERSION (ex: "3.9"). - PREFIX and EXEC_PREFIX are generated by the configure script. - PYTHONPATH macro is the default search path. - - pybuilddir.txt file - - pyvenv.cfg configuration file - - PyConfig fields ('config' function argument): - - - pathconfig_warnings - - pythonpath_env (PYTHONPATH environment variable) - - - _PyPathConfig fields ('pathconfig' function argument): - - - program_name: see config_init_program_name() - - home: Py_SetPythonHome() or PYTHONHOME environment variable - - - current working directory: see copy_absolute() - - Outputs, 'pathconfig' fields: - - - program_full_path - - module_search_path - - prefix - - exec_prefix - - If a field is already set (non NULL), it is left unchanged. */ -PyStatus -_PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config) -{ - PyStatus status; - PyCalculatePath calculate; - memset(&calculate, 0, sizeof(calculate)); - - status = calculate_init(&calculate, config); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - - status = calculate_path(&calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - - /* program_full_path must an either an empty string or an absolute path */ - assert(wcslen(pathconfig->program_full_path) == 0 - || _Py_isabs(pathconfig->program_full_path)); - - status = _PyStatus_OK(); - -done: - calculate_free(&calculate); - return status; -} - -#ifdef __cplusplus + int r = wchar_to_dict(dict, key, buffer); + PyMem_RawFree(buffer); + return r; } #endif + + +/* Add the current executable's path to the dict */ +static int +progname_to_dict(PyObject *dict, const char *key) +{ +#ifdef MS_WINDOWS + return winmodule_to_dict(dict, key, NULL); +#elif defined(__APPLE__) + char *path; + uint32_t pathLen = 256; + while (pathLen) { + path = PyMem_RawMalloc((pathLen + 1) * sizeof(char)); + if (!path) { + return 0; + } + if (_NSGetExecutablePath(path, &pathLen) != 0) { + PyMem_RawFree(path); + continue; + } + // Only keep if the path is absolute + if (path[0] == SEP) { + int r = decode_to_dict(dict, key, path); + PyMem_RawFree(path); + return r; + } + // Fall back and store None + PyMem_RawFree(path); + break; + } +#endif + return PyDict_SetItemString(dict, key, Py_None) == 0; +} + + +/* Add the runtime library's path to the dict */ +static int +library_to_dict(PyObject *dict, const char *key) +{ +#ifdef MS_WINDOWS + extern HMODULE PyWin_DLLhModule; + if (PyWin_DLLhModule) { + return winmodule_to_dict(dict, key, PyWin_DLLhModule); + } +#elif defined(WITH_NEXT_FRAMEWORK) + static const char modPath[MAXPATHLEN + 1]; + static int modPathInitialized = -1; + if (modPathInitialized < 0) { + NSModule pythonModule; + modPathInitialized = 0; + + /* On Mac OS X we have a special case if we're running from a framework. + This is because the python home should be set relative to the library, + which is in the framework, not relative to the executable, which may + be outside of the framework. Except when we're in the build + directory... */ + pythonModule = NSModuleForSymbol(NSLookupAndBindSymbol("_Py_Initialize")); + + /* Use dylib functions to find out where the framework was loaded from */ + const char *path = NSLibraryNameForModule(pythonModule); + if (path) { + strncpy(modPath, path, MAXPATHLEN); + } + } + if (modPathInitialized > 0) { + return decode_to_dict(dict, key, modPath); + } +#endif + return PyDict_SetItemString(dict, key, Py_None) == 0; +} + + +PyObject * +_Py_Get_Getpath_CodeObject() +{ + return PyMarshal_ReadObjectFromString( + (const char*)_Py_M__getpath, sizeof(_Py_M__getpath)); +} + + +/* Perform the actual path calculation. + + When compute_path_config is 0, this only reads any initialised path + config values into the PyConfig struct. For example, Py_SetHome() or + Py_SetPath(). The only error should be due to failed memory allocation. + + When compute_path_config is 1, full path calculation is performed. + The GIL must be held, and there may be filesystem access, side + effects, and potential unraisable errors that are reported directly + to stderr. + + Calling this function multiple times on the same PyConfig is only + safe because already-configured values are not recalculated. To + actually recalculate paths, you need a clean PyConfig. +*/ +PyStatus +_PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) +{ + PyStatus status = _PyPathConfig_ReadGlobal(config); + + if (_PyStatus_EXCEPTION(status) || !compute_path_config) { + return status; + } + + if (!_PyThreadState_UncheckedGet()) { + return PyStatus_Error("cannot calculate path configuration without GIL"); + } + + PyObject *configDict = _PyConfig_AsDict(config); + if (!configDict) { + PyErr_Clear(); + return PyStatus_NoMemory(); + } + + PyObject *dict = PyDict_New(); + if (!dict) { + PyErr_Clear(); + Py_DECREF(configDict); + return PyStatus_NoMemory(); + } + + if (PyDict_SetItemString(dict, "config", configDict) < 0) { + PyErr_Clear(); + Py_DECREF(configDict); + Py_DECREF(dict); + return PyStatus_NoMemory(); + } + /* reference now held by dict */ + Py_DECREF(configDict); + + PyObject *co = _Py_Get_Getpath_CodeObject(); + if (!co || !PyCode_Check(co)) { + PyErr_Clear(); + Py_XDECREF(co); + Py_DECREF(dict); + return PyStatus_Error("error reading frozen getpath.py"); + } + +#ifdef MS_WINDOWS + PyObject *winreg = PyImport_ImportModule("winreg"); + if (!winreg || PyDict_SetItemString(dict, "winreg", winreg) < 0) { + PyErr_Clear(); + Py_XDECREF(winreg); + if (PyDict_SetItemString(dict, "winreg", Py_None) < 0) { + PyErr_Clear(); + Py_DECREF(co); + Py_DECREF(dict); + return PyStatus_Error("error importing winreg module"); + } + } else { + Py_DECREF(winreg); + } +#endif + + if ( +#ifdef MS_WINDOWS + !decode_to_dict(dict, "os_name", "nt") || +#elif defined(__APPLE__) + !decode_to_dict(dict, "os_name", "darwin") || +#else + !decode_to_dict(dict, "os_name", "posix") || +#endif + !decode_to_dict(dict, "PREFIX", PREFIX) || + !decode_to_dict(dict, "EXEC_PREFIX", EXEC_PREFIX) || + !decode_to_dict(dict, "PYTHONPATH", PYTHONPATH) || + !decode_to_dict(dict, "VPATH", VPATH) || + !decode_to_dict(dict, "PLATLIBDIR", PLATLIBDIR) || + !decode_to_dict(dict, "PYDEBUGEXT", PYDEBUGEXT) || + !int_to_dict(dict, "VERSION_MAJOR", PY_MAJOR_VERSION) || + !int_to_dict(dict, "VERSION_MINOR", PY_MINOR_VERSION) || + !decode_to_dict(dict, "PYWINVER", PYWINVER) || + !wchar_to_dict(dict, "EXE_SUFFIX", EXE_SUFFIX) || + !env_to_dict(dict, "ENV_PATH", 0) || + !env_to_dict(dict, "ENV_PYTHONHOME", 0) || + !env_to_dict(dict, "ENV_PYTHONEXECUTABLE", 0) || + !env_to_dict(dict, "ENV___PYVENV_LAUNCHER__", 1) || + !progname_to_dict(dict, "real_executable") || + !library_to_dict(dict, "library") || + !wchar_to_dict(dict, "executable_dir", NULL) || + !wchar_to_dict(dict, "py_setpath", _PyPathConfig_GetGlobalModuleSearchPath()) || + !funcs_to_dict(dict, config->pathconfig_warnings) || +#ifndef MS_WINDOWS + PyDict_SetItemString(dict, "winreg", Py_None) < 0 || +#endif + PyDict_SetItemString(dict, "__builtins__", PyEval_GetBuiltins()) < 0 + ) { + Py_DECREF(co); + Py_DECREF(dict); + _PyErr_WriteUnraisableMsg("error evaluating initial values", NULL); + return PyStatus_Error("error evaluating initial values"); + } + + PyObject *r = PyEval_EvalCode(co, dict, dict); + Py_DECREF(co); + + if (!r) { + Py_DECREF(dict); + _PyErr_WriteUnraisableMsg("error evaluating path", NULL); + return PyStatus_Error("error evaluating path"); + } + Py_DECREF(r); + +#if 0 + PyObject *it = PyObject_GetIter(configDict); + for (PyObject *k = PyIter_Next(it); k; k = PyIter_Next(it)) { + if (!strcmp("__builtins__", PyUnicode_AsUTF8(k))) { + Py_DECREF(k); + continue; + } + fprintf(stderr, "%s = ", PyUnicode_AsUTF8(k)); + PyObject *o = PyDict_GetItem(configDict, k); + o = PyObject_Repr(o); + fprintf(stderr, "%s\n", PyUnicode_AsUTF8(o)); + Py_DECREF(o); + Py_DECREF(k); + } + Py_DECREF(it); +#endif + + if (_PyConfig_FromDict(config, configDict) < 0) { + _PyErr_WriteUnraisableMsg("reading getpath results", NULL); + Py_DECREF(dict); + return PyStatus_Error("error getting getpath results"); + } + + Py_DECREF(dict); + + return _PyStatus_OK(); +} diff --git a/Modules/getpath.py b/Modules/getpath.py new file mode 100644 index 00000000000..2eadfba1dfd --- /dev/null +++ b/Modules/getpath.py @@ -0,0 +1,727 @@ +# ****************************************************************************** +# getpath.py +# ****************************************************************************** + +# This script is designed to be precompiled to bytecode, frozen into the +# main binary, and then directly evaluated. It is not an importable module, +# and does not import any other modules (besides winreg on Windows). +# Rather, the values listed below must be specified in the globals dict +# used when evaluating the bytecode. + +# See _PyConfig_InitPathConfig in Modules/getpath.c for the execution. + +# ****************************************************************************** +# REQUIRED GLOBALS +# ****************************************************************************** + +# ** Helper functions ** +# abspath(path) -- make relative paths absolute against CWD +# basename(path) -- the filename of path +# dirname(path) -- the directory name of path +# hassuffix(path, suffix) -- returns True if path has suffix +# isabs(path) -- path is absolute or not +# isdir(path) -- path exists and is a directory +# isfile(path) -- path exists and is a file +# isxfile(path) -- path exists and is an executable file +# joinpath(*paths) -- combine the paths +# readlines(path) -- a list of each line of text in the UTF-8 encoded file +# realpath(path) -- resolves symlinks in path +# warn(message) -- print a warning (if enabled) + +# ** Values known at compile time ** +# os_name -- [in] one of 'nt', 'posix', 'darwin' +# PREFIX -- [in] sysconfig.get_config_var(...) +# EXEC_PREFIX -- [in] sysconfig.get_config_var(...) +# PYTHONPATH -- [in] sysconfig.get_config_var(...) +# VPATH -- [in] sysconfig.get_config_var(...) +# PLATLIBDIR -- [in] sysconfig.get_config_var(...) +# PYDEBUGEXT -- [in, opt] '_d' on Windows for debug builds +# EXE_SUFFIX -- [in, opt] '.exe' on Windows/Cygwin/similar +# VERSION_MAJOR -- [in] sys.version_info.major +# VERSION_MINOR -- [in] sys.version_info.minor +# PYWINVER -- [in] the Windows platform-specific version (e.g. 3.8-32) + +# ** Values read from the environment ** +# There is no need to check the use_environment flag before reading +# these, as the flag will be tested in this script. +# Also note that ENV_PYTHONPATH is read from config['pythonpath_env'] +# to allow for embedders who choose to specify it via that struct. +# ENV_PATH -- [in] getenv(...) +# ENV_PYTHONHOME -- [in] getenv(...) +# ENV_PYTHONEXECUTABLE -- [in] getenv(...) +# ENV___PYVENV_LAUNCHER__ -- [in] getenv(...) + +# ** Values calculated at runtime ** +# config -- [in/out] dict of the PyConfig structure +# real_executable -- [in, optional] resolved path to main process +# On Windows and macOS, read directly from the running process +# Otherwise, leave None and it will be calculated from executable +# executable_dir -- [in, optional] real directory containing binary +# If None, will be calculated from real_executable or executable +# py_setpath -- [in] argument provided to Py_SetPath +# If None, 'prefix' and 'exec_prefix' may be updated in config +# library -- [in, optional] path of dylib/DLL/so +# Only used for locating ._pth files +# winreg -- [in, optional] the winreg module (only on Windows) + +# ****************************************************************************** +# HIGH-LEVEL ALGORITHM +# ****************************************************************************** + +# IMPORTANT: The code is the actual specification at time of writing. +# This prose description is based on the original comment from the old +# getpath.c to help capture the intent, but should not be considered +# a specification. + +# Search in some common locations for the associated Python libraries. + +# Two directories must be found, the platform independent directory +# (prefix), containing the common .py and .pyc files, and the platform +# dependent directory (exec_prefix), containing the shared library +# modules. Note that prefix and exec_prefix can be the same directory, +# but for some installations, they are different. + +# This script carries out separate searches for prefix and exec_prefix. +# Each search tries a number of different locations until a ``landmark'' +# file or directory is found. If no prefix or exec_prefix is found, a +# warning message is issued and the preprocessor defined PREFIX and +# EXEC_PREFIX are used (even though they will not work); python carries on +# as best as is possible, but most imports will fail. + +# Before any searches are done, the location of the executable is +# determined. If Py_SetPath() was called, or if we are running on +# Windows, the 'real_executable' path is used (if known). Otherwise, +# we use the config-specified program name or default to argv[0]. +# If this has one or more slashes in it, it is made absolute against +# the current working directory. If it only contains a name, it must +# have been invoked from the shell's path, so we search $PATH for the +# named executable and use that. If the executable was not found on +# $PATH (or there was no $PATH environment variable), the original +# argv[0] string is used. + +# At this point, provided Py_SetPath was not used, the +# __PYVENV_LAUNCHER__ variable may override the executable (on macOS, +# the PYTHON_EXECUTABLE variable may also override). This allows +# certain launchers that run Python as a subprocess to properly +# specify the executable path. They are not intended for users. + +# Next, the executable location is examined to see if it is a symbolic +# link. If so, the link is realpath-ed and the directory of the link +# target is used for the remaining searches. The same steps are +# performed for prefix and for exec_prefix, but with different landmarks. + +# Step 1. Are we running in a virtual environment? Unless 'home' has +# been specified another way, check for a pyvenv.cfg and use its 'home' +# property to override the executable dir used later for prefix searches. +# We do not activate the venv here - that is performed later by site.py. + +# Step 2. Is there a ._pth file? A ._pth file lives adjacent to the +# runtime library (if any) or the actual executable (not the symlink), +# and contains precisely the intended contents of sys.path as relative +# paths (to its own location). Its presence also enables isolated mode +# and suppresses other environment variable usage. Unless already +# specified by Py_SetHome(), the directory containing the ._pth file is +# set as 'home'. + +# Step 3. Are we running python out of the build directory? This is +# checked by looking for the BUILDDIR_TXT file, which contains the +# relative path to the platlib dir. The executable_dir value is +# derived from joining the VPATH preprocessor variable to the +# directory containing pybuilddir.txt. If it is not found, the +# BUILD_LANDMARK file is found, which is part of the source tree. +# prefix is then found by searching up for a file that should only +# exist in the source tree, and the stdlib dir is set to prefix/Lib. + +# Step 4. If 'home' is set, either by Py_SetHome(), ENV_PYTHONHOME, +# a pyvenv.cfg file, ._pth file, or by detecting a build directory, it +# is assumed to point to prefix and exec_prefix. $PYTHONHOME can be a +# single directory, which is used for both, or the prefix and exec_prefix +# directories separated by DELIM (colon on POSIX; semicolon on Windows). + +# Step 5. Try to find prefix and exec_prefix relative to executable_dir, +# backtracking up the path until it is exhausted. This is the most common +# step to succeed. Note that if prefix and exec_prefix are different, +# exec_prefix is more likely to be found; however if exec_prefix is a +# subdirectory of prefix, both will be found. + +# Step 6. Search the directories pointed to by the preprocessor variables +# PREFIX and EXEC_PREFIX. These are supplied by the Makefile but can be +# passed in as options to the configure script. + +# That's it! + +# Well, almost. Once we have determined prefix and exec_prefix, the +# preprocessor variable PYTHONPATH is used to construct a path. Each +# relative path on PYTHONPATH is prefixed with prefix. Then the directory +# containing the shared library modules is appended. The environment +# variable $PYTHONPATH is inserted in front of it all. On POSIX, if we are +# in a build directory, both prefix and exec_prefix are reset to the +# corresponding preprocessor variables (so sys.prefix will reflect the +# installation location, even though sys.path points into the build +# directory). This seems to make more sense given that currently the only +# known use of sys.prefix and sys.exec_prefix is for the ILU installation +# process to find the installed Python tree. + +# An embedding application can use Py_SetPath() to override all of +# these automatic path computations. + + +# ****************************************************************************** +# PLATFORM CONSTANTS +# ****************************************************************************** + +platlibdir = config.get('platlibdir') or PLATLIBDIR + +if os_name == 'posix' or os_name == 'darwin': + BUILDDIR_TXT = 'pybuilddir.txt' + BUILD_LANDMARK = 'Modules/Setup.local' + DEFAULT_PROGRAM_NAME = f'python{VERSION_MAJOR}' + STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}' + STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}/os.py', f'{STDLIB_SUBDIR}/os.pyc'] + PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}/lib-dynload' + BUILDSTDLIB_LANDMARKS = ['Lib/os.py'] + VENV_LANDMARK = 'pyvenv.cfg' + ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}.zip' + DELIM = ':' + SEP = '/' + +elif os_name == 'nt': + BUILDDIR_TXT = 'pybuilddir.txt' + BUILD_LANDMARK = r'..\..\Modules\Setup.local' + DEFAULT_PROGRAM_NAME = f'python' + STDLIB_SUBDIR = 'Lib' + STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}\\os.py', f'{STDLIB_SUBDIR}\\os.pyc'] + PLATSTDLIB_LANDMARK = f'{platlibdir}' + BUILDSTDLIB_LANDMARKS = ['Lib\\os.py'] + VENV_LANDMARK = 'pyvenv.cfg' + ZIP_LANDMARK = f'python{VERSION_MAJOR}{VERSION_MINOR}{PYDEBUGEXT or ""}.zip' + WINREG_KEY = f'SOFTWARE\\Python\\PythonCore\\{PYWINVER}\\PythonPath' + DELIM = ';' + SEP = '\\' + + +# ****************************************************************************** +# HELPER FUNCTIONS (note that we prefer C functions for performance) +# ****************************************************************************** + +def search_up(prefix, *landmarks, test=isfile): + while prefix: + if any(test(joinpath(prefix, f)) for f in landmarks): + return prefix + prefix = dirname(prefix) + + +# ****************************************************************************** +# READ VARIABLES FROM config +# ****************************************************************************** + +program_name = config.get('program_name') +home = config.get('home') +executable = config.get('executable') +base_executable = config.get('base_executable') +prefix = config.get('prefix') +exec_prefix = config.get('exec_prefix') +base_prefix = config.get('base_prefix') +base_exec_prefix = config.get('base_exec_prefix') +ENV_PYTHONPATH = config['pythonpath_env'] +use_environment = config.get('use_environment', 1) + +pythonpath = config.get('module_search_paths') + +real_executable_dir = None +stdlib_dir = None +platstdlib_dir = None + +# ****************************************************************************** +# CALCULATE program_name +# ****************************************************************************** + +program_name_was_set = bool(program_name) + +if not program_name: + try: + program_name = config.get('orig_argv', [])[0] + except IndexError: + pass + +if not program_name: + program_name = DEFAULT_PROGRAM_NAME + +if EXE_SUFFIX and not hassuffix(program_name, EXE_SUFFIX) and isxfile(program_name + EXE_SUFFIX): + program_name = program_name + EXE_SUFFIX + + +# ****************************************************************************** +# CALCULATE executable +# ****************************************************************************** + +if py_setpath: + # When Py_SetPath has been called, executable defaults to + # the real executable path. + if not executable: + executable = real_executable + +if not executable and SEP in program_name: + # Resolve partial path program_name against current directory + executable = abspath(program_name) + +if not executable: + # All platforms default to real_executable if known at this + # stage. POSIX does not set this value. + executable = real_executable +elif os_name == 'darwin': + # QUIRK: On macOS we may know the real executable path, but + # if our caller has lied to us about it (e.g. most of + # test_embed), we need to use their path in order to detect + # whether we are in a build tree. This is true even if the + # executable path was provided in the config. + real_executable = executable + +if not executable and program_name: + # Resolve names against PATH. + # NOTE: The use_environment value is ignored for this lookup. + # To properly isolate, launch Python with a full path. + for p in ENV_PATH.split(DELIM): + p = joinpath(p, program_name) + if isxfile(p): + executable = p + break + +if not executable: + executable = '' + # When we cannot calculate the executable, subsequent searches + # look in the current working directory. Here, we emulate that + # (the former getpath.c would do it apparently by accident). + executable_dir = abspath('.') + # Also need to set this fallback in case we are running from a + # build directory with an invalid argv0 (i.e. test_sys.test_executable) + real_executable_dir = executable_dir + +if ENV_PYTHONEXECUTABLE or ENV___PYVENV_LAUNCHER__: + # If set, these variables imply that we should be using them as + # sys.executable and when searching for venvs. However, we should + # use the argv0 path for prefix calculation + base_executable = executable + if not real_executable: + real_executable = executable + executable = ENV_PYTHONEXECUTABLE or ENV___PYVENV_LAUNCHER__ + executable_dir = dirname(executable) + + +# ****************************************************************************** +# CALCULATE (default) home +# ****************************************************************************** + +# Used later to distinguish between Py_SetPythonHome and other +# ways that it may have been set +home_was_set = False + +if home: + home_was_set = True +elif use_environment and ENV_PYTHONHOME and not py_setpath: + home = ENV_PYTHONHOME + + +# ****************************************************************************** +# READ pyvenv.cfg +# ****************************************************************************** + +venv_prefix = None + +# Calling Py_SetPythonHome(), Py_SetPath() or +# setting $PYTHONHOME will override venv detection. +if not home and not py_setpath: + try: + # prefix2 is just to avoid calculating dirname again later, + # as the path in venv_prefix is the more common case. + venv_prefix2 = executable_dir or dirname(executable) + venv_prefix = dirname(venv_prefix2) + try: + # Read pyvenv.cfg from one level above executable + pyvenvcfg = readlines(joinpath(venv_prefix, VENV_LANDMARK)) + except FileNotFoundError: + # Try the same directory as executable + pyvenvcfg = readlines(joinpath(venv_prefix2, VENV_LANDMARK)) + venv_prefix = venv_prefix2 + except FileNotFoundError: + venv_prefix = None + pyvenvcfg = [] + + for line in pyvenvcfg: + key, had_equ, value = line.partition('=') + if had_equ and key.strip().lower() == 'home': + executable_dir = real_executable_dir = value.strip() + base_executable = joinpath(executable_dir, basename(executable)) + break + else: + venv_prefix = None + + +# ****************************************************************************** +# CALCULATE base_executable, real_executable AND executable_dir +# ****************************************************************************** + +if not base_executable: + base_executable = executable or real_executable or '' + +if not real_executable: + real_executable = base_executable + +try: + real_executable = realpath(real_executable) +except OSError as ex: + # Only warn if the file actually exists and was unresolvable + # Otherwise users who specify a fake executable may get spurious warnings. + if isfile(real_executable): + warn(f'Failed to find real location of {base_executable}') + +if not executable_dir and os_name == 'darwin' and library: + # QUIRK: macOS checks adjacent to its library early + library_dir = dirname(library) + if any(isfile(joinpath(library_dir, p)) for p in STDLIB_LANDMARKS): + # Exceptions here should abort the whole process (to match + # previous behavior) + executable_dir = realpath(library_dir) + real_executable_dir = executable_dir + +# If we do not have the executable's directory, we can calculate it. +# This is the directory used to find prefix/exec_prefix if necessary. +if not executable_dir: + executable_dir = real_executable_dir = dirname(real_executable) + +# If we do not have the real executable's directory, we calculate it. +# This is the directory used to detect build layouts. +if not real_executable_dir: + real_executable_dir = dirname(real_executable) + +# ****************************************************************************** +# DETECT _pth FILE +# ****************************************************************************** + +# The contents of an optional ._pth file are used to totally override +# sys.path calcualation. Its presence also implies isolated mode and +# no-site (unless explicitly requested) +pth = None +pth_dir = None + +# Calling Py_SetPythonHome() or Py_SetPath() will override ._pth search, +# but environment variables and command-line options cannot. +if not py_setpath and not home_was_set: + # Check adjacent to the main DLL/dylib/so + if library: + try: + pth = readlines(library.rpartition('.')[0] + '._pth') + pth_dir = dirname(library) + except FileNotFoundError: + pass + + # Check adjacent to the original executable, even if we + # redirected to actually launch Python. This may allow a + # venv to override the base_executable's ._pth file, but + # it cannot override the library's one. + if not pth_dir: + try: + pth = readlines(executable.rpartition('.')[0] + '._pth') + pth_dir = dirname(executable) + except FileNotFoundError: + pass + + # If we found a ._pth file, disable environment and home + # detection now. Later, we will do the rest. + if pth_dir: + use_environment = 0 + home = pth_dir + pythonpath = [] + + +# ****************************************************************************** +# CHECK FOR BUILD DIRECTORY +# ****************************************************************************** + +build_prefix = None + +if not home_was_set and real_executable_dir and not py_setpath: + # Detect a build marker and use it to infer prefix, exec_prefix, + # stdlib_dir and the platstdlib_dir directories. + try: + platstdlib_dir = joinpath( + real_executable_dir, + readlines(joinpath(real_executable_dir, BUILDDIR_TXT))[0], + ) + build_prefix = joinpath(real_executable_dir, VPATH) + except FileNotFoundError: + if isfile(joinpath(real_executable_dir, BUILD_LANDMARK)): + build_prefix = joinpath(real_executable_dir, VPATH) + if os_name == 'nt': + # QUIRK: Windows builds need platstdlib_dir to be the executable + # dir. Normally the builddir marker handles this, but in this + # case we need to correct manually. + platstdlib_dir = real_executable_dir + + if build_prefix: + if os_name == 'nt': + # QUIRK: No searching for more landmarks on Windows + build_stdlib_prefix = build_prefix + else: + build_stdlib_prefix = search_up(build_prefix, *BUILDSTDLIB_LANDMARKS) + # Always use the build prefix for stdlib + if build_stdlib_prefix: + stdlib_dir = joinpath(build_stdlib_prefix, 'Lib') + else: + stdlib_dir = joinpath(build_prefix, 'Lib') + # Only use the build prefix for prefix if it hasn't already been set + if not prefix: + prefix = build_stdlib_prefix + # Do not warn, because 'prefix' never equals 'build_prefix' on POSIX + #elif not venv_prefix and prefix != build_prefix: + # warn('Detected development environment but prefix is already set') + if not exec_prefix: + exec_prefix = build_prefix + # Do not warn, because 'exec_prefix' never equals 'build_prefix' on POSIX + #elif not venv_prefix and exec_prefix != build_prefix: + # warn('Detected development environment but exec_prefix is already set') + config['_is_python_build'] = 1 + + +# ****************************************************************************** +# CALCULATE prefix AND exec_prefix +# ****************************************************************************** + +if py_setpath: + # As documented, calling Py_SetPath will force both prefix + # and exec_prefix to the empty string. + prefix = exec_prefix = '' + +else: + # Read prefix and exec_prefix from explicitly set home + if home: + # When multiple paths are listed with ':' or ';' delimiters, + # split into prefix:exec_prefix + prefix, had_delim, exec_prefix = home.partition(DELIM) + if not had_delim: + exec_prefix = prefix + + + # First try to detect prefix by looking alongside our runtime library, if known + if library and not prefix: + library_dir = dirname(library) + if ZIP_LANDMARK: + if os_name == 'nt': + # QUIRK: Windows does not search up for ZIP file + if isfile(joinpath(library_dir, ZIP_LANDMARK)): + prefix = library_dir + else: + prefix = search_up(library_dir, ZIP_LANDMARK) + if STDLIB_SUBDIR and STDLIB_LANDMARKS and not prefix: + if any(isfile(joinpath(library_dir, f)) for f in STDLIB_LANDMARKS): + prefix = library_dir + stdlib_dir = joinpath(prefix, STDLIB_SUBDIR) + + + # Detect prefix by looking for zip file + if ZIP_LANDMARK and executable_dir and not prefix: + if os_name == 'nt': + # QUIRK: Windows does not search up for ZIP file + if isfile(joinpath(executable_dir, ZIP_LANDMARK)): + prefix = executable_dir + else: + prefix = search_up(executable_dir, ZIP_LANDMARK) + if prefix: + stdlib_dir = joinpath(prefix, STDLIB_SUBDIR) + if not isdir(stdlib_dir): + stdlib_dir = None + + + # Detect prefix by searching from our executable location for the stdlib_dir + if STDLIB_SUBDIR and STDLIB_LANDMARKS and executable_dir and not prefix: + prefix = search_up(executable_dir, *STDLIB_LANDMARKS) + if prefix: + stdlib_dir = joinpath(prefix, STDLIB_SUBDIR) + + if PREFIX and not prefix: + prefix = PREFIX + if not any(isfile(joinpath(prefix, f)) for f in STDLIB_LANDMARKS): + warn('Could not find platform independent libraries ') + + if not prefix: + prefix = abspath('') + warn('Could not find platform independent libraries ') + + + # Detect exec_prefix by searching from executable for the platstdlib_dir + if PLATSTDLIB_LANDMARK and not exec_prefix: + if executable_dir: + exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir) + if not exec_prefix: + if EXEC_PREFIX: + exec_prefix = EXEC_PREFIX + if not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)): + warn('Could not find platform dependent libraries ') + else: + warn('Could not find platform dependent libraries ') + + # Fallback: assume exec_prefix == prefix + if not exec_prefix: + exec_prefix = prefix + + + if not prefix or not exec_prefix: + warn('Consider setting $PYTHONHOME to [:]') + + +# If we haven't set [plat]stdlib_dir already, set them now +if not stdlib_dir: + if prefix: + stdlib_dir = joinpath(prefix, STDLIB_SUBDIR) + else: + stdlib_dir = '' + +if not platstdlib_dir: + if exec_prefix: + platstdlib_dir = joinpath(exec_prefix, PLATSTDLIB_LANDMARK) + else: + platstdlib_dir = '' + + +# For a venv, update the main prefix/exec_prefix but leave the base ones unchanged +# XXX: We currently do not update prefix here, but it happens in site.py +#if venv_prefix: +# base_prefix = prefix +# base_exec_prefix = exec_prefix +# prefix = exec_prefix = venv_prefix + + +# ****************************************************************************** +# UPDATE pythonpath (sys.path) +# ****************************************************************************** + +if py_setpath: + # If Py_SetPath was called then it overrides any existing search path + config['module_search_paths'] = py_setpath.split(DELIM) + config['module_search_paths_set'] = 1 + +elif not pythonpath: + # If pythonpath was already set, we leave it alone. + # This won't matter in normal use, but if an embedded host is trying to + # recalculate paths while running then we do not want to change it. + pythonpath = [] + + # First add entries from the process environment + if use_environment and ENV_PYTHONPATH: + for p in ENV_PYTHONPATH.split(DELIM): + pythonpath.append(abspath(p)) + + # Then add the default zip file + if os_name == 'nt': + # QUIRK: Windows uses the library directory rather than the prefix + if library: + library_dir = dirname(library) + else: + library_dir = executable_dir + pythonpath.append(joinpath(library_dir, ZIP_LANDMARK)) + elif build_prefix or venv_prefix: + # QUIRK: POSIX uses the default prefix when in the build directory + # or a venv + pythonpath.append(joinpath(PREFIX, ZIP_LANDMARK)) + else: + pythonpath.append(joinpath(prefix, ZIP_LANDMARK)) + + if os_name == 'nt' and use_environment and winreg: + # QUIRK: Windows also lists paths in the registry. Paths are stored + # as the default value of each subkey of + # {HKCU,HKLM}\Software\Python\PythonCore\{winver}\PythonPath + # where winver is sys.winver (typically '3.x' or '3.x-32') + for hk in (winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE): + try: + key = winreg.OpenKeyEx(hk, WINREG_KEY) + try: + i = 0 + while True: + try: + keyname = winreg.EnumKey(key, i) + subkey = winreg.OpenKeyEx(key, keyname) + if not subkey: + continue + try: + v = winreg.QueryValue(subkey) + finally: + winreg.CloseKey(subkey) + if isinstance(v, str): + pythonpath.append(v) + i += 1 + except OSError: + break + finally: + winreg.CloseKey(key) + except OSError: + pass + + # Then add any entries compiled into the PYTHONPATH macro. + if PYTHONPATH: + for p in PYTHONPATH.split(DELIM): + pythonpath.append(joinpath(prefix, p)) + + # Then add stdlib_dir and platstdlib_dir + if stdlib_dir: + pythonpath.append(stdlib_dir) + if platstdlib_dir: + if os_name == 'nt' and venv_prefix: + # QUIRK: Windows appends executable_dir instead of platstdlib_dir + # when in a venv + pythonpath.append(executable_dir) + else: + pythonpath.append(platstdlib_dir) + + config['module_search_paths'] = pythonpath + config['module_search_paths_set'] = 1 + + +# ****************************************************************************** +# POSIX prefix/exec_prefix QUIRKS +# ****************************************************************************** + +# QUIRK: Non-Windows replaces prefix/exec_prefix with defaults when running +# in build directory. This happens after pythonpath calculation. +if os_name != 'nt' and build_prefix: + prefix = config.get('prefix') or PREFIX + exec_prefix = config.get('exec_prefix') or EXEC_PREFIX or prefix + + +# ****************************************************************************** +# SET pythonpath FROM _PTH FILE +# ****************************************************************************** + +if pth: + config['isolated'] = 1 + config['use_environment'] = 0 + config['site_import'] = 0 + pythonpath = [] + for line in pth: + line = line.partition('#')[0].strip() + if not line: + pass + elif line == 'import site': + config['site_import'] = 1 + elif line.startswith('import '): + warn("unsupported 'import' line in ._pth file") + else: + pythonpath.append(joinpath(pth_dir, line)) + config['module_search_paths'] = pythonpath + config['module_search_paths_set'] = 1 + +# ****************************************************************************** +# UPDATE config FROM CALCULATED VALUES +# ****************************************************************************** + +config['program_name'] = program_name +config['home'] = home +config['executable'] = executable +config['base_executable'] = base_executable +config['prefix'] = prefix +config['exec_prefix'] = exec_prefix +config['base_prefix'] = base_prefix or prefix +config['base_exec_prefix'] = base_exec_prefix or exec_prefix + +config['platlibdir'] = platlibdir +config['stdlib_dir'] = stdlib_dir +config['platstdlib_dir'] = platstdlib_dir diff --git a/Modules/getpath_noop.c b/Modules/getpath_noop.c new file mode 100644 index 00000000000..c10e41d07f2 --- /dev/null +++ b/Modules/getpath_noop.c @@ -0,0 +1,10 @@ +/* Implements the getpath API for compiling with no functionality */ + +#include "Python.h" +#include "pycore_pathconfig.h" + +PyStatus +_PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) +{ + return PyStatus_Error("path configuration is unsupported"); +} diff --git a/Modules/main.c b/Modules/main.c index c537e6b6785..b9bcea393ab 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -534,11 +534,16 @@ pymain_repl(PyConfig *config, int *exitcode) static void pymain_run_python(int *exitcode) { + PyObject *main_importer_path = NULL; PyInterpreterState *interp = _PyInterpreterState_GET(); /* pymain_run_stdin() modify the config */ PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(interp); - PyObject *main_importer_path = NULL; + /* ensure path config is written into global variables */ + if (_PyStatus_EXCEPTION(_PyPathConfig_UpdateGlobal(config))) { + goto error; + } + if (config->run_filename != NULL) { /* If filename is a package (ex: directory or ZIP file) which contains __main__.py, main_importer_path is set to filename and will be diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index a89cff72951..b1c2914fb0f 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4444,6 +4444,33 @@ os__path_splitroot_impl(PyObject *module, path_t *path) #endif /* MS_WINDOWS */ +/*[clinic input] +os._path_normpath + + path: object + +Basic path normalization. +[clinic start generated code]*/ + +static PyObject * +os__path_normpath_impl(PyObject *module, PyObject *path) +/*[clinic end generated code: output=b94d696d828019da input=5e90c39e12549dc0]*/ +{ + if (!PyUnicode_Check(path)) { + PyErr_Format(PyExc_TypeError, "expected 'str', not '%.200s'", + Py_TYPE(path)->tp_name); + return NULL; + } + Py_ssize_t len; + wchar_t *buffer = PyUnicode_AsWideCharString(path, &len); + if (!buffer) { + return NULL; + } + PyObject *result = PyUnicode_FromWideChar(_Py_normpath(buffer, len), -1); + PyMem_Free(buffer); + return result; +} + /*[clinic input] os.mkdir @@ -14826,6 +14853,7 @@ static PyMethodDef posix_methods[] = { OS__GETFINALPATHNAME_METHODDEF OS__GETVOLUMEPATHNAME_METHODDEF OS__PATH_SPLITROOT_METHODDEF + OS__PATH_NORMPATH_METHODDEF OS_GETLOADAVG_METHODDEF OS_URANDOM_METHODDEF OS_SETRESUID_METHODDEF diff --git a/PC/config_minimal.c b/PC/config_minimal.c index adb1c44a724..928a4efd32e 100644 --- a/PC/config_minimal.c +++ b/PC/config_minimal.c @@ -5,6 +5,10 @@ #include "Python.h" +/* Define extern variables omitted from minimal builds */ +void *PyWin_DLLhModule = NULL; + + extern PyObject* PyInit_faulthandler(void); extern PyObject* PyInit__tracemalloc(void); extern PyObject* PyInit_gc(void); diff --git a/PC/getpathp.c b/PC/getpathp.c deleted file mode 100644 index a27348024d2..00000000000 --- a/PC/getpathp.c +++ /dev/null @@ -1,1174 +0,0 @@ - -/* Return the initial module search path. */ -/* Used by DOS, Windows 3.1, Windows 95/98, Windows NT. */ - -/* ---------------------------------------------------------------- - PATH RULES FOR WINDOWS: - This describes how sys.path is formed on Windows. It describes the - functionality, not the implementation (ie, the order in which these - are actually fetched is different). The presence of a python._pth or - pythonXY._pth file alongside the program overrides these rules - see - below. - - * Python always adds an empty entry at the start, which corresponds - to the current directory. - - * If the PYTHONPATH env. var. exists, its entries are added next. - - * We look in the registry for "application paths" - that is, sub-keys - under the main PythonPath registry key. These are added next (the - order of sub-key processing is undefined). - HKEY_CURRENT_USER is searched and added first. - HKEY_LOCAL_MACHINE is searched and added next. - (Note that all known installers only use HKLM, so HKCU is typically - empty) - - * We attempt to locate the "Python Home" - if the PYTHONHOME env var - is set, we believe it. Otherwise, we use the path of our host .EXE's - to try and locate one of our "landmarks" and deduce our home. - - If we DO have a Python Home: The relevant sub-directories (Lib, - DLLs, etc) are based on the Python Home - - If we DO NOT have a Python Home, the core Python Path is - loaded from the registry. (This is the main PythonPath key, - and both HKLM and HKCU are combined to form the path) - - * Iff - we can not locate the Python Home, have not had a PYTHONPATH - specified, and can't locate any Registry entries (ie, we have _nothing_ - we can assume is a good path), a default path with relative entries is - used (eg. .\Lib;.\DLLs, etc) - - - If a '._pth' file exists adjacent to the executable with the same base name - (e.g. python._pth adjacent to python.exe) or adjacent to the shared library - (e.g. python36._pth adjacent to python36.dll), it is used in preference to - the above process. The shared library file takes precedence over the - executable. The path file must contain a list of paths to add to sys.path, - one per line. Each path is relative to the directory containing the file. - Blank lines and comments beginning with '#' are permitted. - - In the presence of this ._pth file, no other paths are added to the search - path, the registry finder is not enabled, site.py is not imported and - isolated mode is enabled. The site package can be enabled by including a - line reading "import site"; no other imports are recognized. Any invalid - entry (other than directories that do not exist) will result in immediate - termination of the program. - - - The end result of all this is: - * When running python.exe, or any other .exe in the main Python directory - (either an installed version, or directly from the PCbuild directory), - the core path is deduced, and the core paths in the registry are - ignored. Other "application paths" in the registry are always read. - - * When Python is hosted in another exe (different directory, embedded via - COM, etc), the Python Home will not be deduced, so the core path from - the registry is used. Other "application paths" in the registry are - always read. - - * If Python can't find its home and there is no registry (eg, frozen - exe, some very strange installation setup) you get a path with - some default, but relative, paths. - - * An embedding application can use Py_SetPath() to override all of - these automatic path computations. - - * An install of Python can fully specify the contents of sys.path using - either a 'EXENAME._pth' or 'DLLNAME._pth' file, optionally including - "import site" to enable the site module. - - ---------------------------------------------------------------- */ - - -#include "Python.h" -#include "pycore_fileutils.h" // _Py_add_relfile() -#include "pycore_initconfig.h" // PyStatus -#include "pycore_pathconfig.h" // _PyPathConfig -#include "osdefs.h" // SEP, ALTSEP -#include - -#ifndef MS_WINDOWS -#error getpathp.c should only be built on Windows -#endif - -#include -#include - -#ifdef HAVE_SYS_TYPES_H -#include -#endif /* HAVE_SYS_TYPES_H */ - -#ifdef HAVE_SYS_STAT_H -#include -#endif /* HAVE_SYS_STAT_H */ - -#include - -/* Search in some common locations for the associated Python libraries. - * - * Py_GetPath() tries to return a sensible Python module search path. - * - * The approach is an adaptation for Windows of the strategy used in - * ../Modules/getpath.c; it uses the Windows Registry as one of its - * information sources. - * - * Py_SetPath() can be used to override this mechanism. Call Py_SetPath - * with a semicolon separated path prior to calling Py_Initialize. - */ - -#define STDLIB_SUBDIR L"lib" - -#define INIT_ERR_BUFFER_OVERFLOW() _PyStatus_ERR("buffer overflow") - - -typedef struct { - const wchar_t *path_env; /* PATH environment variable */ - const wchar_t *home; /* PYTHONHOME environment variable */ - - /* Registry key "Software\Python\PythonCore\X.Y\PythonPath" - where X.Y is the Python version (major.minor) */ - wchar_t *machine_path; /* from HKEY_LOCAL_MACHINE */ - wchar_t *user_path; /* from HKEY_CURRENT_USER */ - - const wchar_t *pythonpath_env; -} PyCalculatePath; - - -/* determine if "ch" is a separator character */ -static int -is_sep(wchar_t ch) -{ -#ifdef ALTSEP - return ch == SEP || ch == ALTSEP; -#else - return ch == SEP; -#endif -} - - -/* assumes 'dir' null terminated in bounds. Never writes - beyond existing terminator. */ -static void -reduce(wchar_t *dir) -{ - size_t i = wcsnlen_s(dir, MAXPATHLEN+1); - if (i >= MAXPATHLEN+1) { - Py_FatalError("buffer overflow in getpathp.c's reduce()"); - } - - while (i > 0 && !is_sep(dir[i])) - --i; - dir[i] = '\0'; -} - - -static int -change_ext(wchar_t *dest, const wchar_t *src, const wchar_t *ext) -{ - if (src && src != dest) { - size_t src_len = wcsnlen_s(src, MAXPATHLEN+1); - size_t i = src_len; - if (i >= MAXPATHLEN+1) { - Py_FatalError("buffer overflow in getpathp.c's reduce()"); - } - - while (i > 0 && src[i] != '.' && !is_sep(src[i])) - --i; - - if (i == 0) { - dest[0] = '\0'; - return -1; - } - - if (is_sep(src[i])) { - i = src_len; - } - - if (wcsncpy_s(dest, MAXPATHLEN+1, src, i)) { - dest[0] = '\0'; - return -1; - } - } else { - wchar_t *s = wcsrchr(dest, L'.'); - if (s) { - s[0] = '\0'; - } - } - - if (wcscat_s(dest, MAXPATHLEN+1, ext)) { - dest[0] = '\0'; - return -1; - } - - return 0; -} - - -static int -exists(const wchar_t *filename) -{ - return GetFileAttributesW(filename) != 0xFFFFFFFF; -} - - -/* Is module -- check for .pyc too. - Assumes 'filename' MAXPATHLEN+1 bytes long - - may extend 'filename' by one character. */ -static int -ismodule(wchar_t *filename) -{ - size_t n; - - if (exists(filename)) { - return 1; - } - - /* Check for the compiled version of prefix. */ - n = wcsnlen_s(filename, MAXPATHLEN+1); - if (n < MAXPATHLEN) { - int exist = 0; - filename[n] = L'c'; - filename[n + 1] = L'\0'; - exist = exists(filename); - // Drop the 'c' we just added. - filename[n] = L'\0'; - return exist; - } - return 0; -} - - -/* Add a path component, by appending stuff to buffer. - buffer must have at least MAXPATHLEN + 1 bytes allocated, and contain a - NUL-terminated string with no more than MAXPATHLEN characters (not counting - the trailing NUL). It's a fatal error if it contains a string longer than - that (callers must be careful!). If these requirements are met, it's - guaranteed that buffer will still be a NUL-terminated string with no more - than MAXPATHLEN characters at exit. If stuff is too long, only as much of - stuff as fits will be appended. -*/ - -static void -join(wchar_t *buffer, const wchar_t *stuff) -{ - if (_Py_add_relfile(buffer, stuff, MAXPATHLEN+1) < 0) { - Py_FatalError("buffer overflow in getpathp.c's join()"); - } -} - -/* Call PathCchCanonicalizeEx(path): remove navigation elements such as "." - and ".." to produce a direct, well-formed path. */ -static PyStatus -canonicalize(wchar_t *buffer, const wchar_t *path) -{ - if (buffer == NULL) { - return _PyStatus_NO_MEMORY(); - } - - const wchar_t *pathTail; - if (FAILED(PathCchSkipRoot(path, &pathTail)) || path == pathTail) { - wchar_t buff[MAXPATHLEN + 1]; - if (!GetCurrentDirectoryW(MAXPATHLEN, buff)) { - return _PyStatus_ERR("unable to find current working directory"); - } - if (FAILED(PathCchCombineEx(buff, MAXPATHLEN + 1, buff, path, PATHCCH_ALLOW_LONG_PATHS))) { - return INIT_ERR_BUFFER_OVERFLOW(); - } - if (FAILED(PathCchCanonicalizeEx(buffer, MAXPATHLEN + 1, buff, PATHCCH_ALLOW_LONG_PATHS))) { - return INIT_ERR_BUFFER_OVERFLOW(); - } - return _PyStatus_OK(); - } - - if (FAILED(PathCchCanonicalizeEx(buffer, MAXPATHLEN + 1, path, PATHCCH_ALLOW_LONG_PATHS))) { - return INIT_ERR_BUFFER_OVERFLOW(); - } - return _PyStatus_OK(); -} - -static int -is_stdlibdir(wchar_t *stdlibdir) -{ - wchar_t *filename = stdlibdir; -#ifndef LANDMARK -# define LANDMARK L"os.py" -#endif - /* join() ensures 'landmark' can not overflow prefix if too long. */ - join(filename, LANDMARK); - return ismodule(filename); -} - -/* assumes argv0_path is MAXPATHLEN+1 bytes long, already \0 term'd. - assumption provided by only caller, calculate_path() */ -static int -search_for_prefix(wchar_t *prefix, const wchar_t *argv0_path) -{ - /* Search from argv0_path, until LANDMARK is found. - We guarantee 'prefix' is null terminated in bounds. */ - wcscpy_s(prefix, MAXPATHLEN+1, argv0_path); - if (!prefix[0]) { - return 0; - } - wchar_t stdlibdir[MAXPATHLEN+1]; - wcscpy_s(stdlibdir, Py_ARRAY_LENGTH(stdlibdir), prefix); - /* We initialize with the longest possible path, in case it doesn't fit. - This also gives us an initial SEP at stdlibdir[wcslen(prefix)]. */ - join(stdlibdir, STDLIB_SUBDIR); - do { - assert(stdlibdir[wcslen(prefix)] == SEP); - /* Due to reduce() and our initial value, this result - is guaranteed to fit. */ - wcscpy(&stdlibdir[wcslen(prefix) + 1], STDLIB_SUBDIR); - if (is_stdlibdir(stdlibdir)) { - return 1; - } - reduce(prefix); - } while (prefix[0]); - return 0; -} - - -static int -get_dllpath(wchar_t *dllpath) -{ -#ifdef Py_ENABLE_SHARED - extern HANDLE PyWin_DLLhModule; - if (PyWin_DLLhModule && GetModuleFileNameW(PyWin_DLLhModule, dllpath, MAXPATHLEN)) { - return 0; - } -#endif - return -1; -} - - -#ifdef Py_ENABLE_SHARED - -/* a string loaded from the DLL at startup.*/ -extern const char *PyWin_DLLVersionString; - -/* Load a PYTHONPATH value from the registry. - Load from either HKEY_LOCAL_MACHINE or HKEY_CURRENT_USER. - - Works in both Unicode and 8bit environments. Only uses the - Ex family of functions so it also works with Windows CE. - - Returns NULL, or a pointer that should be freed. - - XXX - this code is pretty strange, as it used to also - work on Win16, where the buffer sizes were not available - in advance. It could be simplied now Win16/Win32s is dead! -*/ -static wchar_t * -getpythonregpath(HKEY keyBase, int skipcore) -{ - HKEY newKey = 0; - DWORD dataSize = 0; - DWORD numKeys = 0; - LONG rc; - wchar_t *retval = NULL; - WCHAR *dataBuf = NULL; - static const WCHAR keyPrefix[] = L"Software\\Python\\PythonCore\\"; - static const WCHAR keySuffix[] = L"\\PythonPath"; - size_t versionLen, keyBufLen; - DWORD index; - WCHAR *keyBuf = NULL; - WCHAR *keyBufPtr; - WCHAR **ppPaths = NULL; - - /* Tried to use sysget("winver") but here is too early :-( */ - versionLen = strlen(PyWin_DLLVersionString); - /* Space for all the chars, plus one \0 */ - keyBufLen = sizeof(keyPrefix) + - sizeof(WCHAR)*(versionLen-1) + - sizeof(keySuffix); - keyBuf = keyBufPtr = PyMem_RawMalloc(keyBufLen); - if (keyBuf==NULL) { - goto done; - } - - memcpy_s(keyBufPtr, keyBufLen, keyPrefix, sizeof(keyPrefix)-sizeof(WCHAR)); - keyBufPtr += Py_ARRAY_LENGTH(keyPrefix) - 1; - mbstowcs(keyBufPtr, PyWin_DLLVersionString, versionLen); - keyBufPtr += versionLen; - /* NULL comes with this one! */ - memcpy(keyBufPtr, keySuffix, sizeof(keySuffix)); - /* Open the root Python key */ - rc=RegOpenKeyExW(keyBase, - keyBuf, /* subkey */ - 0, /* reserved */ - KEY_READ, - &newKey); - if (rc!=ERROR_SUCCESS) { - goto done; - } - /* Find out how big our core buffer is, and how many subkeys we have */ - rc = RegQueryInfoKeyW(newKey, NULL, NULL, NULL, &numKeys, NULL, NULL, - NULL, NULL, &dataSize, NULL, NULL); - if (rc!=ERROR_SUCCESS) { - goto done; - } - if (skipcore) { - dataSize = 0; /* Only count core ones if we want them! */ - } - /* Allocate a temp array of char buffers, so we only need to loop - reading the registry once - */ - ppPaths = PyMem_RawCalloc(numKeys, sizeof(WCHAR *)); - if (ppPaths==NULL) { - goto done; - } - /* Loop over all subkeys, allocating a temp sub-buffer. */ - for(index=0;index 0) { - *(szCur++) = L';'; - dataSize--; - } - if (ppPaths[index]) { - Py_ssize_t len = wcslen(ppPaths[index]); - wcsncpy(szCur, ppPaths[index], len); - szCur += len; - assert(dataSize > (DWORD)len); - dataSize -= (DWORD)len; - } - } - if (skipcore) { - *szCur = '\0'; - } - else { - /* If we have no values, we don't need a ';' */ - if (numKeys) { - *(szCur++) = L';'; - dataSize--; - } - /* Now append the core path entries - - this will include the NULL - */ - rc = RegQueryValueExW(newKey, NULL, 0, NULL, - (LPBYTE)szCur, &dataSize); - if (rc != ERROR_SUCCESS) { - PyMem_RawFree(dataBuf); - goto done; - } - } - /* And set the result - caller must free */ - retval = dataBuf; - } -done: - /* Loop freeing my temp buffers */ - if (ppPaths) { - for(index=0; indexbase_executable == NULL) { - pathconfig->base_executable = PyMem_RawMalloc( - sizeof(wchar_t) * (MAXPATHLEN + 1)); - if (pathconfig->base_executable == NULL) { - return _PyStatus_NO_MEMORY(); - } - - status = canonicalize(pathconfig->base_executable, - program_full_path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - - wcscpy_s(program_full_path, MAXPATHLEN+1, pyvenv_launcher); - /* bpo-35873: Clear the environment variable to avoid it being - * inherited by child processes. */ - _wputenv_s(L"__PYVENV_LAUNCHER__", L""); - } - - if (pathconfig->program_full_path == NULL) { - pathconfig->program_full_path = PyMem_RawMalloc( - sizeof(wchar_t) * (MAXPATHLEN + 1)); - if (pathconfig->program_full_path == NULL) { - return _PyStatus_NO_MEMORY(); - } - - status = canonicalize(pathconfig->program_full_path, - program_full_path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - return _PyStatus_OK(); -} - - -static PyStatus -read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path, - int *found) -{ - PyStatus status; - wchar_t *buf = NULL; - wchar_t *wline = NULL; - FILE *sp_file; - - sp_file = _Py_wfopen(path, L"r"); - if (sp_file == NULL) { - return _PyStatus_OK(); - } - - wcscpy_s(prefix, MAXPATHLEN+1, path); - reduce(prefix); - pathconfig->isolated = 1; - pathconfig->site_import = 0; - - size_t bufsiz = MAXPATHLEN; - size_t prefixlen = wcslen(prefix); - - buf = (wchar_t*)PyMem_RawMalloc(bufsiz * sizeof(wchar_t)); - if (buf == NULL) { - status = _PyStatus_NO_MEMORY(); - goto done; - } - buf[0] = '\0'; - - while (!feof(sp_file)) { - char line[MAXPATHLEN + 1]; - char *p = fgets(line, Py_ARRAY_LENGTH(line), sp_file); - if (!p) { - break; - } - if (*p == '\0' || *p == '\r' || *p == '\n' || *p == '#') { - continue; - } - while (*++p) { - if (*p == '\r' || *p == '\n') { - *p = '\0'; - break; - } - } - - if (strcmp(line, "import site") == 0) { - pathconfig->site_import = 1; - continue; - } - else if (strncmp(line, "import ", 7) == 0) { - status = _PyStatus_ERR("only 'import site' is supported " - "in ._pth file"); - goto done; - } - - DWORD wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, NULL, 0); - wchar_t *wline = (wchar_t*)PyMem_RawMalloc((wn + 1) * sizeof(wchar_t)); - if (wline == NULL) { - status = _PyStatus_NO_MEMORY(); - goto done; - } - wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, wline, wn + 1); - wline[wn] = '\0'; - - size_t usedsiz = wcslen(buf); - while (usedsiz + wn + prefixlen + 4 > bufsiz) { - bufsiz += MAXPATHLEN; - wchar_t *tmp = (wchar_t*)PyMem_RawRealloc(buf, (bufsiz + 1) * - sizeof(wchar_t)); - if (tmp == NULL) { - status = _PyStatus_NO_MEMORY(); - goto done; - } - buf = tmp; - } - - if (usedsiz) { - wcscat_s(buf, bufsiz, L";"); - usedsiz += 1; - } - - errno_t result; - _Py_BEGIN_SUPPRESS_IPH - result = wcscat_s(buf, bufsiz, prefix); - _Py_END_SUPPRESS_IPH - - if (result == EINVAL) { - status = _PyStatus_ERR("invalid argument during ._pth processing"); - goto done; - } else if (result == ERANGE) { - status = _PyStatus_ERR("buffer overflow during ._pth processing"); - goto done; - } - - wchar_t *b = &buf[usedsiz]; - join(b, wline); - - PyMem_RawFree(wline); - wline = NULL; - } - - if (pathconfig->module_search_path == NULL) { - pathconfig->module_search_path = _PyMem_RawWcsdup(buf); - if (pathconfig->module_search_path == NULL) { - status = _PyStatus_NO_MEMORY(); - goto done; - } - } - - *found = 1; - status = _PyStatus_OK(); - goto done; - -done: - PyMem_RawFree(buf); - PyMem_RawFree(wline); - fclose(sp_file); - return status; -} - - -static int -get_pth_filename(PyCalculatePath *calculate, wchar_t *filename, - const _PyPathConfig *pathconfig) -{ - if (!get_dllpath(filename) && - !change_ext(filename, filename, L"._pth") && - exists(filename)) - { - return 1; - } - if (pathconfig->program_full_path[0] && - !change_ext(filename, pathconfig->program_full_path, L"._pth") && - exists(filename)) - { - return 1; - } - return 0; -} - - -static PyStatus -calculate_pth_file(PyCalculatePath *calculate, _PyPathConfig *pathconfig, - wchar_t *prefix, int *found) -{ - wchar_t filename[MAXPATHLEN+1]; - - if (!get_pth_filename(calculate, filename, pathconfig)) { - return _PyStatus_OK(); - } - - return read_pth_file(pathconfig, prefix, filename, found); -} - - -/* Search for an environment configuration file, first in the - executable's directory and then in the parent directory. - If found, open it for use when searching for prefixes. -*/ -static PyStatus -calculate_pyvenv_file(PyCalculatePath *calculate, - wchar_t *argv0_path, size_t argv0_path_len) -{ - wchar_t filename[MAXPATHLEN+1]; - const wchar_t *env_cfg = L"pyvenv.cfg"; - - /* Filename: / "pyvenv.cfg" */ - wcscpy_s(filename, MAXPATHLEN+1, argv0_path); - join(filename, env_cfg); - - FILE *env_file = _Py_wfopen(filename, L"r"); - if (env_file == NULL) { - errno = 0; - - /* Filename: / "pyvenv.cfg" */ - reduce(filename); - reduce(filename); - join(filename, env_cfg); - - env_file = _Py_wfopen(filename, L"r"); - if (env_file == NULL) { - errno = 0; - return _PyStatus_OK(); - } - } - - /* Look for a 'home' variable and set argv0_path to it, if found */ - wchar_t *home = NULL; - PyStatus status = _Py_FindEnvConfigValue(env_file, L"home", &home); - if (_PyStatus_EXCEPTION(status)) { - fclose(env_file); - return status; - } - if (home) { - wcscpy_s(argv0_path, argv0_path_len, home); - PyMem_RawFree(home); - } - fclose(env_file); - return _PyStatus_OK(); -} - - -static void -calculate_home_prefix(PyCalculatePath *calculate, - const wchar_t *argv0_path, - const wchar_t *zip_path, - wchar_t *prefix) -{ - if (calculate->home == NULL || *calculate->home == '\0') { - if (zip_path[0] && exists(zip_path)) { - wcscpy_s(prefix, MAXPATHLEN+1, zip_path); - reduce(prefix); - calculate->home = prefix; - } - else if (search_for_prefix(prefix, argv0_path)) { - calculate->home = prefix; - } - else { - calculate->home = NULL; - } - } - else { - wcscpy_s(prefix, MAXPATHLEN+1, calculate->home); - } -} - - -static PyStatus -calculate_module_search_path(PyCalculatePath *calculate, - _PyPathConfig *pathconfig, - const wchar_t *argv0_path, - wchar_t *prefix, - const wchar_t *zip_path) -{ - int skiphome = calculate->home==NULL ? 0 : 1; -#ifdef Py_ENABLE_SHARED - if (!Py_IgnoreEnvironmentFlag) { - calculate->machine_path = getpythonregpath(HKEY_LOCAL_MACHINE, - skiphome); - calculate->user_path = getpythonregpath(HKEY_CURRENT_USER, skiphome); - } -#endif - /* We only use the default relative PYTHONPATH if we haven't - anything better to use! */ - int skipdefault = (calculate->pythonpath_env != NULL || - calculate->home != NULL || - calculate->machine_path != NULL || - calculate->user_path != NULL); - - /* We need to construct a path from the following parts. - (1) the PYTHONPATH environment variable, if set; - (2) for Win32, the zip archive file path; - (3) for Win32, the machine_path and user_path, if set; - (4) the PYTHONPATH config macro, with the leading "." - of each component replaced with home, if set; - (5) the directory containing the executable (argv0_path). - The length calculation calculates #4 first. - Extra rules: - - If PYTHONHOME is set (in any way) item (3) is ignored. - - If registry values are used, (4) and (5) are ignored. - */ - - /* Calculate size of return buffer */ - size_t bufsz = 0; - if (calculate->home != NULL) { - const wchar_t *p; - bufsz = 1; - for (p = PYTHONPATH; *p; p++) { - if (*p == DELIM) { - bufsz++; /* number of DELIM plus one */ - } - } - bufsz *= wcslen(calculate->home); - } - bufsz += wcslen(PYTHONPATH) + 1; - bufsz += wcslen(argv0_path) + 1; - if (calculate->user_path) { - bufsz += wcslen(calculate->user_path) + 1; - } - if (calculate->machine_path) { - bufsz += wcslen(calculate->machine_path) + 1; - } - bufsz += wcslen(zip_path) + 1; - if (calculate->pythonpath_env != NULL) { - bufsz += wcslen(calculate->pythonpath_env) + 1; - } - - wchar_t *buf, *start_buf; - buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); - if (buf == NULL) { - return _PyStatus_NO_MEMORY(); - } - start_buf = buf; - - if (calculate->pythonpath_env) { - if (wcscpy_s(buf, bufsz - (buf - start_buf), - calculate->pythonpath_env)) { - return INIT_ERR_BUFFER_OVERFLOW(); - } - buf = wcschr(buf, L'\0'); - *buf++ = DELIM; - } - if (zip_path[0]) { - if (wcscpy_s(buf, bufsz - (buf - start_buf), zip_path)) { - return INIT_ERR_BUFFER_OVERFLOW(); - } - buf = wcschr(buf, L'\0'); - *buf++ = DELIM; - } - if (calculate->user_path) { - if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->user_path)) { - return INIT_ERR_BUFFER_OVERFLOW(); - } - buf = wcschr(buf, L'\0'); - *buf++ = DELIM; - } - if (calculate->machine_path) { - if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->machine_path)) { - return INIT_ERR_BUFFER_OVERFLOW(); - } - buf = wcschr(buf, L'\0'); - *buf++ = DELIM; - } - if (calculate->home == NULL) { - if (!skipdefault) { - if (wcscpy_s(buf, bufsz - (buf - start_buf), PYTHONPATH)) { - return INIT_ERR_BUFFER_OVERFLOW(); - } - buf = wcschr(buf, L'\0'); - *buf++ = DELIM; - } - } else { - const wchar_t *p = PYTHONPATH; - const wchar_t *q; - size_t n; - for (;;) { - q = wcschr(p, DELIM); - if (q == NULL) { - n = wcslen(p); - } - else { - n = q-p; - } - if (p[0] == '.' && is_sep(p[1])) { - if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->home)) { - return INIT_ERR_BUFFER_OVERFLOW(); - } - buf = wcschr(buf, L'\0'); - p++; - n--; - } - wcsncpy(buf, p, n); - buf += n; - *buf++ = DELIM; - if (q == NULL) { - break; - } - p = q+1; - } - } - if (argv0_path) { - wcscpy(buf, argv0_path); - buf = wcschr(buf, L'\0'); - *buf++ = DELIM; - } - *(buf - 1) = L'\0'; - - /* Now to pull one last hack/trick. If sys.prefix is - empty, then try and find it somewhere on the paths - we calculated. We scan backwards, as our general policy - is that Python core directories are at the *end* of - sys.path. We assume that our "lib" directory is - on the path, and that our 'prefix' directory is - the parent of that. - */ - if (prefix[0] == L'\0') { - PyStatus status; - wchar_t lookBuf[MAXPATHLEN+1]; - const wchar_t *look = buf - 1; /* 'buf' is at the end of the buffer */ - while (1) { - Py_ssize_t nchars; - const wchar_t *lookEnd = look; - /* 'look' will end up one character before the - start of the path in question - even if this - is one character before the start of the buffer - */ - while (look >= start_buf && *look != DELIM) - look--; - nchars = lookEnd-look; - wcsncpy(lookBuf, look+1, nchars); - lookBuf[nchars] = L'\0'; - status = canonicalize(lookBuf, lookBuf); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - /* Up one level to the parent */ - reduce(lookBuf); - if (search_for_prefix(prefix, lookBuf)) { - break; - } - /* If we are out of paths to search - give up */ - if (look < start_buf) { - break; - } - look--; - } - } - - pathconfig->module_search_path = start_buf; - return _PyStatus_OK(); -} - - -static PyStatus -calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig) -{ - PyStatus status; - - status = get_program_full_path(pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - /* program_full_path guaranteed \0 terminated in MAXPATH+1 bytes. */ - wchar_t argv0_path[MAXPATHLEN+1]; - memset(argv0_path, 0, sizeof(argv0_path)); - - wcscpy_s(argv0_path, MAXPATHLEN+1, pathconfig->program_full_path); - reduce(argv0_path); - - wchar_t prefix[MAXPATHLEN+1]; - memset(prefix, 0, sizeof(prefix)); - - /* Search for a sys.path file */ - int pth_found = 0; - status = calculate_pth_file(calculate, pathconfig, prefix, &pth_found); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - if (pth_found) { - goto done; - } - - status = calculate_pyvenv_file(calculate, - argv0_path, Py_ARRAY_LENGTH(argv0_path)); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - /* Calculate zip archive path from DLL or exe path */ - wchar_t zip_path[MAXPATHLEN+1]; - memset(zip_path, 0, sizeof(zip_path)); - - if (get_dllpath(zip_path) || change_ext(zip_path, zip_path, L".zip")) - { - if (change_ext(zip_path, pathconfig->program_full_path, L".zip")) { - zip_path[0] = L'\0'; - } - } - - calculate_home_prefix(calculate, argv0_path, zip_path, prefix); - - if (pathconfig->module_search_path == NULL) { - status = calculate_module_search_path(calculate, pathconfig, - argv0_path, prefix, zip_path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - -done: - if (pathconfig->stdlib_dir == NULL) { - pathconfig->stdlib_dir = _Py_join_relfile(prefix, STDLIB_SUBDIR); - if (pathconfig->stdlib_dir == NULL) { - return _PyStatus_NO_MEMORY(); - } - } - if (pathconfig->prefix == NULL) { - pathconfig->prefix = _PyMem_RawWcsdup(prefix); - if (pathconfig->prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - } - if (pathconfig->exec_prefix == NULL) { - pathconfig->exec_prefix = _PyMem_RawWcsdup(prefix); - if (pathconfig->exec_prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - } - - return _PyStatus_OK(); -} - - -static PyStatus -calculate_init(PyCalculatePath *calculate, _PyPathConfig *pathconfig, - const PyConfig *config) -{ - calculate->home = pathconfig->home; - calculate->path_env = _wgetenv(L"PATH"); - - calculate->pythonpath_env = config->pythonpath_env; - - return _PyStatus_OK(); -} - - -static void -calculate_free(PyCalculatePath *calculate) -{ - PyMem_RawFree(calculate->machine_path); - PyMem_RawFree(calculate->user_path); -} - - -/* Calculate the Python path configuration. - - Inputs: - - - PyConfig.pythonpath_env: PYTHONPATH environment variable - - _PyPathConfig.home: Py_SetPythonHome() or PYTHONHOME environment variable - - PATH environment variable - - __PYVENV_LAUNCHER__ environment variable - - GetModuleFileNameW(NULL): fully qualified path of the executable file of - the current process - - ._pth configuration file - - pyvenv.cfg configuration file - - Registry key "Software\Python\PythonCore\X.Y\PythonPath" - of HKEY_CURRENT_USER and HKEY_LOCAL_MACHINE where X.Y is the Python - version. - - Outputs, 'pathconfig' fields: - - - base_executable - - program_full_path - - module_search_path - - prefix - - exec_prefix - - isolated - - site_import - - If a field is already set (non NULL), it is left unchanged. */ -PyStatus -_PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config) -{ - PyStatus status; - PyCalculatePath calculate; - memset(&calculate, 0, sizeof(calculate)); - - status = calculate_init(&calculate, pathconfig, config); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - - status = calculate_path(&calculate, pathconfig); - -done: - calculate_free(&calculate); - return status; -} - - -/* Load python3.dll before loading any extension module that might refer - to it. That way, we can be sure that always the python3.dll corresponding - to this python DLL is loaded, not a python3.dll that might be on the path - by chance. - Return whether the DLL was found. -*/ -static int python3_checked = 0; -static HANDLE hPython3; -int -_Py_CheckPython3(void) -{ - wchar_t py3path[MAXPATHLEN+1]; - if (python3_checked) { - return hPython3 != NULL; - } - python3_checked = 1; - - /* If there is a python3.dll next to the python3y.dll, - use that DLL */ - if (!get_dllpath(py3path)) { - reduce(py3path); - join(py3path, PY3_DLLNAME); - hPython3 = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); - if (hPython3 != NULL) { - return 1; - } - } - - /* If we can locate python3.dll in our application dir, - use that DLL */ - hPython3 = LoadLibraryExW(PY3_DLLNAME, NULL, LOAD_LIBRARY_SEARCH_APPLICATION_DIR); - if (hPython3 != NULL) { - return 1; - } - - /* For back-compat, also search {sys.prefix}\DLLs, though - that has not been a normal install layout for a while */ - wcscpy(py3path, Py_GetPrefix()); - if (py3path[0]) { - join(py3path, L"DLLs\\" PY3_DLLNAME); - hPython3 = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); - } - return hPython3 != NULL; -} diff --git a/PC/pyconfig.h b/PC/pyconfig.h index 5d8d9f36184..e0d875adf2e 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -66,9 +66,6 @@ WIN32 is still required for the locale module. #define MS_WIN32 /* only support win32 and greater. */ #define MS_WINDOWS -#ifndef PYTHONPATH -# define PYTHONPATH L".\\DLLs;.\\lib" -#endif #define NT_THREADS #define WITH_THREAD #ifndef NETSCAPE_PI @@ -671,6 +668,4 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ /* Define if libssl has X509_VERIFY_PARAM_set1_host and related function */ #define HAVE_X509_VERIFY_PARAM_SET1_HOST 1 -#define PLATLIBDIR "lib" - #endif /* !Py_CONFIG_H */ diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 54fef9ca629..7b2df4b8afc 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -104,6 +104,7 @@ + @@ -168,7 +169,6 @@ - @@ -374,9 +374,29 @@ + + + + getpath + $(IntDir)getpath.g.h + $(PySourcePath)Modules\getpath.h + + + + + + + + + + + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 5894909e0fb..1c8f1b0dcf4 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -13,6 +13,395 @@ Source Files + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + + + Python Files + @@ -78,4 +467,4 @@ - + \ No newline at end of file diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index b58945a4d19..0b4329dd076 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -147,4 +147,11 @@ $(_PGOPath) + + + <_Content>$(OutDir) + <_ExistingContent Condition="Exists('$(OutDir)pybuilddir.txt')">$([System.IO.File]::ReadAllText('$(OutDir)pybuilddir.txt')) + + + diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 9d96f4bd536..b446e099556 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -109,6 +109,19 @@ version.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies) + + + + PREFIX=NULL; + EXEC_PREFIX=NULL; + VERSION=NULL; + VPATH="..\\.."; + PYDEBUGEXT="$(PyDebugExt)"; + PLATLIBDIR="DLLs"; + %(PreprocessorDefinitions) + + + @@ -349,6 +362,7 @@ + @@ -442,7 +456,6 @@ - @@ -570,7 +583,7 @@ - + GITVERSION="$(GitVersion)";GITTAG="$(GitTag)";GITBRANCH="$(GitBranch)";%(PreprocessorDefinitions) @@ -590,4 +603,8 @@ + + + + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index b19f0279ec3..c1667e3fb74 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -648,6 +648,9 @@ Modules\zlib + + Include\internal + @@ -971,9 +974,6 @@ PC - - PC - PC @@ -1229,10 +1229,25 @@ Objects + + Python + + + Modules + + + Python + + + Python + + + Modules + Resource Files - + \ No newline at end of file diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 5702ab2cd71..854b1e64c15 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -2,6 +2,9 @@ /* Support for dynamic loading of extension modules */ #include "Python.h" +#include "pycore_fileutils.h" // _Py_add_relfile() +#include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() +#include "pycore_pystate.h" // _PyInterpreterState_GET() #ifdef HAVE_DIRECT_H #include @@ -160,6 +163,60 @@ static char *GetPythonImport (HINSTANCE hModule) return NULL; } +/* Load python3.dll before loading any extension module that might refer + to it. That way, we can be sure that always the python3.dll corresponding + to this python DLL is loaded, not a python3.dll that might be on the path + by chance. + Return whether the DLL was found. +*/ +extern HMODULE PyWin_DLLhModule; +static int +_Py_CheckPython3(void) +{ + static int python3_checked = 0; + static HANDLE hPython3; + #define MAXPATHLEN 512 + wchar_t py3path[MAXPATHLEN+1]; + if (python3_checked) { + return hPython3 != NULL; + } + python3_checked = 1; + + /* If there is a python3.dll next to the python3y.dll, + use that DLL */ + if (PyWin_DLLhModule && GetModuleFileNameW(PyWin_DLLhModule, py3path, MAXPATHLEN)) { + wchar_t *p = wcsrchr(py3path, L'\\'); + if (p) { + wcscpy(p + 1, PY3_DLLNAME); + hPython3 = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + if (hPython3 != NULL) { + return 1; + } + } + } + + /* If we can locate python3.dll in our application dir, + use that DLL */ + hPython3 = LoadLibraryExW(PY3_DLLNAME, NULL, LOAD_LIBRARY_SEARCH_APPLICATION_DIR); + if (hPython3 != NULL) { + return 1; + } + + /* For back-compat, also search {sys.prefix}\DLLs, though + that has not been a normal install layout for a while */ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(interp); + assert(config->prefix); + if (config->prefix) { + wcscpy_s(py3path, MAXPATHLEN, config->prefix); + if (py3path[0] && _Py_add_relfile(py3path, L"DLLs\\" PY3_DLLNAME, MAXPATHLEN) >= 0) { + hPython3 = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + } + } + return hPython3 != NULL; + #undef MAXPATHLEN +} + dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, const char *shortname, PyObject *pathname, FILE *fp) diff --git a/Python/fileutils.c b/Python/fileutils.c index ac0046cdac3..cae6b75b6ae 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2000,13 +2000,28 @@ _Py_wrealpath(const wchar_t *path, #endif -#ifndef MS_WINDOWS int _Py_isabs(const wchar_t *path) { +#ifdef MS_WINDOWS + const wchar_t *tail; + HRESULT hr = PathCchSkipRoot(path, &tail); + if (FAILED(hr) || path == tail) { + return 0; + } + if (tail == &path[1] && (path[0] == SEP || path[0] == ALTSEP)) { + // Exclude paths with leading SEP + return 0; + } + if (tail == &path[2] && path[1] == L':') { + // Exclude drive-relative paths (e.g. C:filename.ext) + return 0; + } + return 1; +#else return (path[0] == SEP); -} #endif +} /* Get an absolute path. @@ -2017,6 +2032,22 @@ _Py_isabs(const wchar_t *path) int _Py_abspath(const wchar_t *path, wchar_t **abspath_p) { + if (path[0] == '\0' || !wcscmp(path, L".")) { + wchar_t cwd[MAXPATHLEN + 1]; + cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0; + if (!_Py_wgetcwd(cwd, Py_ARRAY_LENGTH(cwd) - 1)) { + /* unable to get the current directory */ + return -1; + } + *abspath_p = _PyMem_RawWcsdup(cwd); + return 0; + } + + if (_Py_isabs(path)) { + *abspath_p = _PyMem_RawWcsdup(path); + return 0; + } + #ifdef MS_WINDOWS wchar_t woutbuf[MAX_PATH], *woutbufp = woutbuf; DWORD result; @@ -2028,7 +2059,7 @@ _Py_abspath(const wchar_t *path, wchar_t **abspath_p) return -1; } - if (result > Py_ARRAY_LENGTH(woutbuf)) { + if (result >= Py_ARRAY_LENGTH(woutbuf)) { if ((size_t)result <= (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { woutbufp = PyMem_RawMalloc((size_t)result * sizeof(wchar_t)); } @@ -2055,11 +2086,6 @@ _Py_abspath(const wchar_t *path, wchar_t **abspath_p) *abspath_p = _PyMem_RawWcsdup(woutbufp); return 0; #else - if (_Py_isabs(path)) { - *abspath_p = _PyMem_RawWcsdup(path); - return 0; - } - wchar_t cwd[MAXPATHLEN + 1]; cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0; if (!_Py_wgetcwd(cwd, Py_ARRAY_LENGTH(cwd) - 1)) { @@ -2102,7 +2128,8 @@ join_relfile(wchar_t *buffer, size_t bufsize, const wchar_t *dirname, const wchar_t *relfile) { #ifdef MS_WINDOWS - if (FAILED(PathCchCombineEx(buffer, bufsize, dirname, relfile, 0))) { + if (FAILED(PathCchCombineEx(buffer, bufsize, dirname, relfile, + PATHCCH_ALLOW_LONG_PATHS | PATHCCH_FORCE_ENABLE_LONG_NAME_PROCESS))) { return -1; } #else @@ -2180,99 +2207,125 @@ _Py_find_basename(const wchar_t *filename) return 0; } - -/* Remove navigation elements such as "." and "..". - - This is mostly a C implementation of posixpath.normpath(). - Return 0 on success. Return -1 if "orig" is too big for the buffer. */ -int -_Py_normalize_path(const wchar_t *path, wchar_t *buf, const size_t buf_len) +/* In-place path normalisation. Returns the start of the normalized + path, which will be within the original buffer. Guaranteed to not + make the path longer, and will not fail. 'size' is the length of + the path, if known. If -1, the first null character will be assumed + to be the end of the path. */ +wchar_t * +_Py_normpath(wchar_t *path, Py_ssize_t size) { - assert(path && *path != L'\0'); - assert(*path == SEP); // an absolute path - if (wcslen(path) + 1 >= buf_len) { - return -1; + if (!path[0] || size == 0) { + return path; } + wchar_t lastC = L'\0'; + wchar_t *p1 = path; + wchar_t *pEnd = size >= 0 ? &path[size] : NULL; + wchar_t *p2 = path; + wchar_t *minP2 = path; - int dots = -1; - int check_leading = 1; - const wchar_t *buf_start = buf; - wchar_t *buf_next = buf; - // The resulting filename will never be longer than path. - for (const wchar_t *remainder = path; *remainder != L'\0'; remainder++) { - wchar_t c = *remainder; - buf_next[0] = c; - buf_next++; - if (c == SEP) { - assert(dots <= 2); - if (dots == 2) { - // Turn "/x/y/../z" into "/x/z". - buf_next -= 4; // "/../" - assert(*buf_next == SEP); - // We cap it off at the root, so "/../spam" becomes "/spam". - if (buf_next == buf_start) { - buf_next++; - } - else { - // Move to the previous SEP in the buffer. - while (*(buf_next - 1) != SEP) { - assert(buf_next != buf_start); - buf_next--; - } - } - } - else if (dots == 1) { - // Turn "/./" into "/". - buf_next -= 2; // "./" - assert(*(buf_next - 1) == SEP); - } - else if (dots == 0) { - // Turn "//" into "/". - buf_next--; - assert(*(buf_next - 1) == SEP); - if (check_leading) { - if (buf_next - 1 == buf && *(remainder + 1) != SEP) { - // Leave a leading "//" alone, unless "///...". - buf_next++; - buf_start++; - } - check_leading = 0; - } - } - dots = 0; +#define IS_END(x) (pEnd ? (x) == pEnd : !*(x)) +#ifdef ALTSEP +#define IS_SEP(x) (*(x) == SEP || *(x) == ALTSEP) +#else +#define IS_SEP(x) (*(x) == SEP) +#endif +#define SEP_OR_END(x) (IS_SEP(x) || IS_END(x)) + + // Skip leading '.\' + if (p1[0] == L'.' && IS_SEP(&p1[1])) { + path = &path[2]; + while (IS_SEP(path) && !IS_END(path)) { + path++; } - else { - check_leading = 0; - if (dots >= 0) { - if (c == L'.' && dots < 2) { - dots++; - } - else { - dots = -1; - } + p1 = p2 = minP2 = path; + lastC = SEP; + } +#ifdef MS_WINDOWS + // Skip past drive segment and update minP2 + else if (p1[0] && p1[1] == L':') { + *p2++ = *p1++; + *p2++ = *p1++; + minP2 = p2; + lastC = L':'; + } + // Skip past all \\-prefixed paths, including \\?\, \\.\, + // and network paths, including the first segment. + else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1])) { + int sepCount = 2; + *p2++ = SEP; + *p2++ = SEP; + p1 += 2; + for (; !IS_END(p1) && sepCount; ++p1) { + if (IS_SEP(p1)) { + --sepCount; + *p2++ = lastC = SEP; + } else { + *p2++ = lastC = *p1; } } + minP2 = p2; + } +#else + // Skip past two leading SEPs + else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1]) && !IS_SEP(&p1[2])) { + *p2++ = *p1++; + *p2++ = *p1++; + minP2 = p2; + lastC = SEP; + } +#endif /* MS_WINDOWS */ + + /* if pEnd is specified, check that. Else, check for null terminator */ + for (; !IS_END(p1); ++p1) { + wchar_t c = *p1; +#ifdef ALTSEP + if (c == ALTSEP) { + c = SEP; + } +#endif + if (lastC == SEP) { + if (c == L'.') { + int sep_at_1 = SEP_OR_END(&p1[1]); + int sep_at_2 = !sep_at_1 && SEP_OR_END(&p1[2]); + if (sep_at_2 && p1[1] == L'.') { + wchar_t *p3 = p2; + while (p3 != minP2 && *--p3 == SEP) { } + while (p3 != minP2 && *(p3 - 1) != SEP) { --p3; } + if (p3[0] == L'.' && p3[1] == L'.' && IS_SEP(&p3[2])) { + // Previous segment is also ../, so append instead + *p2++ = L'.'; + *p2++ = L'.'; + lastC = L'.'; + } else if (p3[0] == SEP) { + // Absolute path, so absorb segment + p2 = p3 + 1; + } else { + p2 = p3; + } + p1 += 1; + } else if (sep_at_1) { + } else { + *p2++ = lastC = c; + } + } else if (c == SEP) { + } else { + *p2++ = lastC = c; + } + } else { + *p2++ = lastC = c; + } + } + *p2 = L'\0'; + if (p2 != minP2) { + while (--p2 != minP2 && *p2 == SEP) { + *p2 = L'\0'; + } } - if (dots >= 0) { - // Strip any trailing dots and trailing slash. - buf_next -= dots + 1; // "/" or "/." or "/.." - assert(*buf_next == SEP); - if (buf_next == buf_start) { - // Leave the leading slash for root. - buf_next++; - } - else { - if (dots == 2) { - // Move to the previous SEP in the buffer. - do { - assert(buf_next != buf_start); - buf_next--; - } while (*(buf_next) != SEP); - } - } - } - *buf_next = L'\0'; - return 0; +#undef SEP_OR_END +#undef IS_SEP +#undef IS_END + return path; } diff --git a/Python/initconfig.c b/Python/initconfig.c index 53b624fdd72..47ebc64c847 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -22,11 +22,6 @@ # endif #endif -#ifndef PLATLIBDIR -# error "PLATLIBDIR macro must be defined" -#endif - - /* --- Command line options --------------------------------------- */ /* Short usage message (with %s for argv0) */ @@ -632,7 +627,6 @@ config_check_consistency(const PyConfig *config) assert(config->parse_argv >= 0); assert(config->configure_c_stdio >= 0); assert(config->buffered_stdio >= 0); - assert(config->program_name != NULL); assert(_PyWideStringList_CheckConsistency(&config->orig_argv)); assert(_PyWideStringList_CheckConsistency(&config->argv)); /* sys.argv must be non-empty: empty argv is replaced with [''] */ @@ -641,7 +635,6 @@ config_check_consistency(const PyConfig *config) assert(_PyWideStringList_CheckConsistency(&config->warnoptions)); assert(_PyWideStringList_CheckConsistency(&config->module_search_paths)); assert(config->module_search_paths_set >= 0); - assert(config->platlibdir != NULL); assert(config->filesystem_encoding != NULL); assert(config->filesystem_errors != NULL); assert(config->stdio_encoding != NULL); @@ -740,6 +733,7 @@ _PyConfig_InitCompatConfig(PyConfig *config) config->legacy_windows_stdio = -1; #endif config->use_frozen_modules = -1; + config->_is_python_build = 0; config->code_debug_ranges = 1; } @@ -962,6 +956,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) COPY_ATTR(_isolated_interpreter); COPY_ATTR(use_frozen_modules); COPY_WSTRLIST(orig_argv); + COPY_ATTR(_is_python_build); #undef COPY_ATTR #undef COPY_WSTR_ATTR @@ -1066,6 +1061,7 @@ _PyConfig_AsDict(const PyConfig *config) SET_ITEM_INT(_isolated_interpreter); SET_ITEM_WSTRLIST(orig_argv); SET_ITEM_INT(use_frozen_modules); + SET_ITEM_INT(_is_python_build); return dict; @@ -1350,6 +1346,7 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) GET_UINT(_init_main); GET_UINT(_isolated_interpreter); GET_UINT(use_frozen_modules); + GET_UINT(_is_python_build); #undef CHECK_VALUE #undef GET_UINT @@ -1489,117 +1486,6 @@ config_set_global_vars(const PyConfig *config) } -/* Get the program name: use PYTHONEXECUTABLE and __PYVENV_LAUNCHER__ - environment variables on macOS if available. */ -static PyStatus -config_init_program_name(PyConfig *config) -{ - PyStatus status; - - /* If Py_SetProgramName() was called, use its value */ - const wchar_t *program_name = _Py_path_config.program_name; - if (program_name != NULL) { - config->program_name = _PyMem_RawWcsdup(program_name); - if (config->program_name == NULL) { - return _PyStatus_NO_MEMORY(); - } - return _PyStatus_OK(); - } - -#ifdef __APPLE__ - /* On MacOS X, when the Python interpreter is embedded in an - application bundle, it gets executed by a bootstrapping script - that does os.execve() with an argv[0] that's different from the - actual Python executable. This is needed to keep the Finder happy, - or rather, to work around Apple's overly strict requirements of - the process name. However, we still need a usable sys.executable, - so the actual executable path is passed in an environment variable. - See Lib/plat-mac/bundlebuilder.py for details about the bootstrap - script. */ - const char *p = config_get_env(config, "PYTHONEXECUTABLE"); - if (p != NULL) { - status = CONFIG_SET_BYTES_STR(config, &config->program_name, p, - "PYTHONEXECUTABLE environment variable"); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - return _PyStatus_OK(); - } -#ifdef WITH_NEXT_FRAMEWORK - else { - const char* pyvenv_launcher = getenv("__PYVENV_LAUNCHER__"); - if (pyvenv_launcher && *pyvenv_launcher) { - /* Used by Mac/Tools/pythonw.c to forward - * the argv0 of the stub executable - */ - status = CONFIG_SET_BYTES_STR(config, - &config->program_name, - pyvenv_launcher, - "__PYVENV_LAUNCHER__ environment variable"); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - /* - * This environment variable is used to communicate between - * the stub launcher and the real interpreter and isn't needed - * beyond this point. - * - * Clean up to avoid problems when launching other programs - * later on. - */ - (void)unsetenv("__PYVENV_LAUNCHER__"); - - return _PyStatus_OK(); - } - } -#endif /* WITH_NEXT_FRAMEWORK */ -#endif /* __APPLE__ */ - - /* Use argv[0] if available and non-empty */ - const PyWideStringList *argv = &config->argv; - if (argv->length >= 1 && argv->items[0][0] != L'\0') { - config->program_name = _PyMem_RawWcsdup(argv->items[0]); - if (config->program_name == NULL) { - return _PyStatus_NO_MEMORY(); - } - return _PyStatus_OK(); - } - - /* Last fall back: hardcoded name */ -#ifdef MS_WINDOWS - const wchar_t *default_program_name = L"python"; -#else - const wchar_t *default_program_name = L"python3"; -#endif - status = PyConfig_SetString(config, &config->program_name, - default_program_name); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - return _PyStatus_OK(); -} - -static PyStatus -config_init_executable(PyConfig *config) -{ - assert(config->executable == NULL); - - /* If Py_SetProgramFullPath() was called, use its value */ - const wchar_t *program_full_path = _Py_path_config.program_full_path; - if (program_full_path != NULL) { - PyStatus status = PyConfig_SetString(config, - &config->executable, - program_full_path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - return _PyStatus_OK(); - } - return _PyStatus_OK(); -} - - static const wchar_t* config_get_xoption(const PyConfig *config, wchar_t *name) { @@ -1618,25 +1504,6 @@ config_get_xoption_value(const PyConfig *config, wchar_t *name) } -static PyStatus -config_init_home(PyConfig *config) -{ - assert(config->home == NULL); - - /* If Py_SetPythonHome() was called, use its value */ - wchar_t *home = _Py_path_config.home; - if (home) { - PyStatus status = PyConfig_SetString(config, &config->home, home); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - return _PyStatus_OK(); - } - - return CONFIG_GET_ENV_DUP(config, &config->home, - L"PYTHONHOME", "PYTHONHOME"); -} - static PyStatus config_init_hash_seed(PyConfig *config) { @@ -2092,44 +1959,6 @@ config_init_fs_encoding(PyConfig *config, const PyPreConfig *preconfig) } -/* Determine if the current build is a "development" build (e.g. running - out of the source tree) or not. - - A return value of -1 indicates that we do not know. - */ -static int -is_dev_env(PyConfig *config) -{ - // This should only ever get called early in runtime initialization, - // before the global path config is written. Otherwise we would - // use Py_GetProgramFullPath() and _Py_GetStdlibDir(). - assert(config != NULL); - - const wchar_t *executable = config->executable; - const wchar_t *stdlib = config->stdlib_dir; - if (executable == NULL || *executable == L'\0' || - stdlib == NULL || *stdlib == L'\0') { - // _PyPathConfig_Calculate() hasn't run yet. - return -1; - } - size_t len = _Py_find_basename(executable); - if (wcscmp(executable + len, L"python") != 0 && - wcscmp(executable + len, L"python.exe") != 0) { - return 0; - } - /* If dirname() is the same for both then it is a dev build. */ - if (len != _Py_find_basename(stdlib)) { - return 0; - } - // We do not bother normalizing the two filenames first since - // for config_init_import() is does the right thing as-is. - if (wcsncmp(stdlib, executable, len) != 0) { - return 0; - } - return 1; -} - - static PyStatus config_init_import(PyConfig *config, int compute_path_config) { @@ -2144,10 +1973,7 @@ config_init_import(PyConfig *config, int compute_path_config) if (config->use_frozen_modules < 0) { const wchar_t *value = config_get_xoption_value(config, L"frozen_modules"); if (value == NULL) { - int isdev = is_dev_env(config); - if (isdev >= 0) { - config->use_frozen_modules = !isdev; - } + config->use_frozen_modules = !config->_is_python_build; } else if (wcscmp(value, L"on") == 0) { config->use_frozen_modules = 1; @@ -2246,28 +2072,6 @@ config_read(PyConfig *config, int compute_path_config) return status; } - if (config->home == NULL) { - status = config_init_home(config); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - - if (config->executable == NULL) { - status = config_init_executable(config); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - - if(config->platlibdir == NULL) { - status = CONFIG_SET_BYTES_STR(config, &config->platlibdir, PLATLIBDIR, - "PLATLIBDIR macro"); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - if (config->_install_importlib) { status = config_init_import(config, compute_path_config); if (_PyStatus_EXCEPTION(status)) { @@ -2890,13 +2694,6 @@ config_read_cmdline(PyConfig *config) config->parse_argv = 1; } - if (config->program_name == NULL) { - status = config_init_program_name(config); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - if (config->parse_argv == 1) { Py_ssize_t opt_index; status = config_parse_cmdline(config, &cmdline_warnoptions, &opt_index); @@ -3076,7 +2873,7 @@ done: PyStatus PyConfig_Read(PyConfig *config) { - return _PyConfig_Read(config, 1); + return _PyConfig_Read(config, 0); } @@ -3124,16 +2921,6 @@ _Py_GetConfigsAsDict(void) } Py_CLEAR(dict); - /* path config */ - dict = _PyPathConfig_AsDict(); - if (dict == NULL) { - goto error; - } - if (PyDict_SetItemString(result, "path_config", dict) < 0) { - goto error; - } - Py_CLEAR(dict); - return result; error: @@ -3199,6 +2986,7 @@ _Py_DumpPathConfig(PyThreadState *tstate) PySys_WriteStderr(" environment = %i\n", config->use_environment); PySys_WriteStderr(" user site = %i\n", config->user_site_directory); PySys_WriteStderr(" import site = %i\n", config->site_import); + PySys_WriteStderr(" is in build tree = %i\n", config->_is_python_build); DUMP_CONFIG("stdlib dir", stdlib_dir); #undef DUMP_CONFIG diff --git a/Python/pathconfig.c b/Python/pathconfig.c index ad22222e000..4271928571f 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -1,6 +1,7 @@ /* Path configuration like module_search_path (sys.path) */ #include "Python.h" +#include "marshal.h" // PyMarshal_ReadObjectFromString #include "osdefs.h" // DELIM #include "pycore_initconfig.h" #include "pycore_fileutils.h" @@ -9,6 +10,8 @@ #include #ifdef MS_WINDOWS # include // GetFullPathNameW(), MAX_PATH +# include +# include #endif #ifdef __cplusplus @@ -16,86 +19,36 @@ extern "C" { #endif +/* External interface */ + +/* Stored values set by C API functions */ +typedef struct _PyPathConfig { + /* Full path to the Python program */ + wchar_t *program_full_path; + wchar_t *prefix; + wchar_t *exec_prefix; + wchar_t *stdlib_dir; + /* Set by Py_SetPath */ + wchar_t *module_search_path; + /* Set by _PyPathConfig_UpdateGlobal */ + wchar_t *calculated_module_search_path; + /* Python program name */ + wchar_t *program_name; + /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */ + wchar_t *home; +} _PyPathConfig; + +# define _PyPathConfig_INIT \ + {.module_search_path = NULL} + + _PyPathConfig _Py_path_config = _PyPathConfig_INIT; -static int -copy_wstr(wchar_t **dst, const wchar_t *src) +const wchar_t * +_PyPathConfig_GetGlobalModuleSearchPath(void) { - assert(*dst == NULL); - if (src != NULL) { - *dst = _PyMem_RawWcsdup(src); - if (*dst == NULL) { - return -1; - } - } - else { - *dst = NULL; - } - return 0; -} - - -static void -pathconfig_clear(_PyPathConfig *config) -{ - /* _PyMem_SetDefaultAllocator() is needed to get a known memory allocator, - since Py_SetPath(), Py_SetPythonHome() and Py_SetProgramName() can be - called before Py_Initialize() which can changes the memory allocator. */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - -#define CLEAR(ATTR) \ - do { \ - PyMem_RawFree(ATTR); \ - ATTR = NULL; \ - } while (0) - - CLEAR(config->program_full_path); - CLEAR(config->prefix); - CLEAR(config->exec_prefix); - CLEAR(config->stdlib_dir); - CLEAR(config->module_search_path); - CLEAR(config->program_name); - CLEAR(config->home); -#ifdef MS_WINDOWS - CLEAR(config->base_executable); -#endif - -#undef CLEAR - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); -} - - -static PyStatus -pathconfig_copy(_PyPathConfig *config, const _PyPathConfig *config2) -{ - pathconfig_clear(config); - -#define COPY_ATTR(ATTR) \ - do { \ - if (copy_wstr(&config->ATTR, config2->ATTR) < 0) { \ - return _PyStatus_NO_MEMORY(); \ - } \ - } while (0) - - COPY_ATTR(program_full_path); - COPY_ATTR(prefix); - COPY_ATTR(exec_prefix); - COPY_ATTR(module_search_path); - COPY_ATTR(stdlib_dir); - COPY_ATTR(program_name); - COPY_ATTR(home); -#ifdef MS_WINDOWS - config->isolated = config2->isolated; - config->site_import = config2->site_import; - COPY_ATTR(base_executable); -#endif - -#undef COPY_ATTR - - return _PyStatus_OK(); + return _Py_path_config.module_search_path; } @@ -105,374 +58,132 @@ _PyPathConfig_ClearGlobal(void) PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - pathconfig_clear(&_Py_path_config); +#define CLEAR(ATTR) \ + do { \ + PyMem_RawFree(_Py_path_config.ATTR); \ + _Py_path_config.ATTR = NULL; \ + } while (0) + + CLEAR(program_full_path); + CLEAR(prefix); + CLEAR(exec_prefix); + CLEAR(stdlib_dir); + CLEAR(module_search_path); + CLEAR(calculated_module_search_path); + CLEAR(program_name); + CLEAR(home); + +#undef CLEAR PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } - -static wchar_t* -_PyWideStringList_Join(const PyWideStringList *list, wchar_t sep) +PyStatus +_PyPathConfig_ReadGlobal(PyConfig *config) { - size_t len = 1; /* NUL terminator */ - for (Py_ssize_t i=0; i < list->length; i++) { - if (i != 0) { - len++; - } - len += wcslen(list->items[i]); - } + PyStatus status = _PyStatus_OK(); - wchar_t *text = PyMem_RawMalloc(len * sizeof(wchar_t)); - if (text == NULL) { - return NULL; - } - wchar_t *str = text; - for (Py_ssize_t i=0; i < list->length; i++) { - wchar_t *path = list->items[i]; - if (i != 0) { - *str++ = sep; - } - len = wcslen(path); - memcpy(str, path, len * sizeof(wchar_t)); - str += len; - } - *str = L'\0'; +#define COPY(ATTR) \ + do { \ + if (_Py_path_config.ATTR && !config->ATTR) { \ + status = PyConfig_SetString(config, &config->ATTR, _Py_path_config.ATTR); \ + if (_PyStatus_EXCEPTION(status)) goto done; \ + } \ + } while (0) - return text; +#define COPY2(ATTR, SRCATTR) \ + do { \ + if (_Py_path_config.SRCATTR && !config->ATTR) { \ + status = PyConfig_SetString(config, &config->ATTR, _Py_path_config.SRCATTR); \ + if (_PyStatus_EXCEPTION(status)) goto done; \ + } \ + } while (0) + + COPY(prefix); + COPY(exec_prefix); + COPY(stdlib_dir); + COPY(program_name); + COPY(home); + COPY2(executable, program_full_path); + // module_search_path must be initialised - not read +#undef COPY +#undef COPY2 + +done: + return status; } - -static PyStatus -pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config) +PyStatus +_PyPathConfig_UpdateGlobal(const PyConfig *config) { - PyStatus status; PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - if (config->module_search_paths_set) { - PyMem_RawFree(pathconfig->module_search_path); - pathconfig->module_search_path = _PyWideStringList_Join(&config->module_search_paths, DELIM); - if (pathconfig->module_search_path == NULL) { - goto no_memory; - } - } +#define COPY(ATTR) \ + do { \ + if (config->ATTR) { \ + PyMem_RawFree(_Py_path_config.ATTR); \ + _Py_path_config.ATTR = _PyMem_RawWcsdup(config->ATTR); \ + if (!_Py_path_config.ATTR) goto error; \ + } \ + } while (0) -#define COPY_CONFIG(PATH_ATTR, CONFIG_ATTR) \ - if (config->CONFIG_ATTR) { \ - PyMem_RawFree(pathconfig->PATH_ATTR); \ - pathconfig->PATH_ATTR = NULL; \ - if (copy_wstr(&pathconfig->PATH_ATTR, config->CONFIG_ATTR) < 0) { \ - goto no_memory; \ - } \ +#define COPY2(ATTR, SRCATTR) \ + do { \ + if (config->SRCATTR) { \ + PyMem_RawFree(_Py_path_config.ATTR); \ + _Py_path_config.ATTR = _PyMem_RawWcsdup(config->SRCATTR); \ + if (!_Py_path_config.ATTR) goto error; \ + } \ + } while (0) + + COPY(prefix); + COPY(exec_prefix); + COPY(stdlib_dir); + COPY(program_name); + COPY(home); + COPY2(program_full_path, executable); +#undef COPY +#undef COPY2 + + PyMem_RawFree(_Py_path_config.module_search_path); + _Py_path_config.module_search_path = NULL; + PyMem_RawFree(_Py_path_config.calculated_module_search_path); + _Py_path_config.calculated_module_search_path = NULL; + + do { + size_t cch = 1; + for (Py_ssize_t i = 0; i < config->module_search_paths.length; ++i) { + cch += 1 + wcslen(config->module_search_paths.items[i]); } - COPY_CONFIG(program_full_path, executable); - COPY_CONFIG(prefix, prefix); - COPY_CONFIG(exec_prefix, exec_prefix); - COPY_CONFIG(stdlib_dir, stdlib_dir); - COPY_CONFIG(program_name, program_name); - COPY_CONFIG(home, home); -#ifdef MS_WINDOWS - COPY_CONFIG(base_executable, base_executable); -#endif + wchar_t *path = (wchar_t*)PyMem_RawMalloc(sizeof(wchar_t) * cch); + if (!path) { + goto error; + } + wchar_t *p = path; + for (Py_ssize_t i = 0; i < config->module_search_paths.length; ++i) { + wcscpy(p, config->module_search_paths.items[i]); + p = wcschr(p, L'\0'); + *p++ = DELIM; + *p = L'\0'; + } -#undef COPY_CONFIG + do { + *p = L'\0'; + } while (p != path && *--p == DELIM); + _Py_path_config.calculated_module_search_path = path; + } while (0); - status = _PyStatus_OK(); - goto done; - -no_memory: - status = _PyStatus_NO_MEMORY(); - -done: PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - return status; -} - -PyObject * -_PyPathConfig_AsDict(void) -{ - PyObject *dict = PyDict_New(); - if (dict == NULL) { - return NULL; - } - -#define SET_ITEM(KEY, EXPR) \ - do { \ - PyObject *obj = (EXPR); \ - if (obj == NULL) { \ - goto fail; \ - } \ - int res = PyDict_SetItemString(dict, KEY, obj); \ - Py_DECREF(obj); \ - if (res < 0) { \ - goto fail; \ - } \ - } while (0) -#define SET_ITEM_STR(KEY) \ - SET_ITEM(#KEY, \ - (_Py_path_config.KEY \ - ? PyUnicode_FromWideChar(_Py_path_config.KEY, -1) \ - : (Py_INCREF(Py_None), Py_None))) -#define SET_ITEM_INT(KEY) \ - SET_ITEM(#KEY, PyLong_FromLong(_Py_path_config.KEY)) - - SET_ITEM_STR(program_full_path); - SET_ITEM_STR(prefix); - SET_ITEM_STR(exec_prefix); - SET_ITEM_STR(module_search_path); - SET_ITEM_STR(stdlib_dir); - SET_ITEM_STR(program_name); - SET_ITEM_STR(home); -#ifdef MS_WINDOWS - SET_ITEM_INT(isolated); - SET_ITEM_INT(site_import); - SET_ITEM_STR(base_executable); - - { - wchar_t py3path[MAX_PATH]; - HMODULE hPython3 = GetModuleHandleW(PY3_DLLNAME); - PyObject *obj; - if (hPython3 - && GetModuleFileNameW(hPython3, py3path, Py_ARRAY_LENGTH(py3path))) - { - obj = PyUnicode_FromWideChar(py3path, -1); - if (obj == NULL) { - goto fail; - } - } - else { - obj = Py_None; - Py_INCREF(obj); - } - if (PyDict_SetItemString(dict, "python3_dll", obj) < 0) { - Py_DECREF(obj); - goto fail; - } - Py_DECREF(obj); - } -#endif - -#undef SET_ITEM -#undef SET_ITEM_STR -#undef SET_ITEM_INT - - return dict; - -fail: - Py_DECREF(dict); - return NULL; -} - - -PyStatus -_PyConfig_WritePathConfig(const PyConfig *config) -{ - return pathconfig_set_from_config(&_Py_path_config, config); -} - - -static PyStatus -config_init_module_search_paths(PyConfig *config, _PyPathConfig *pathconfig) -{ - assert(!config->module_search_paths_set); - - _PyWideStringList_Clear(&config->module_search_paths); - - const wchar_t *sys_path = pathconfig->module_search_path; - const wchar_t delim = DELIM; - while (1) { - const wchar_t *p = wcschr(sys_path, delim); - if (p == NULL) { - p = sys_path + wcslen(sys_path); /* End of string */ - } - - size_t path_len = (p - sys_path); - wchar_t *path = PyMem_RawMalloc((path_len + 1) * sizeof(wchar_t)); - if (path == NULL) { - return _PyStatus_NO_MEMORY(); - } - memcpy(path, sys_path, path_len * sizeof(wchar_t)); - path[path_len] = L'\0'; - - PyStatus status = PyWideStringList_Append(&config->module_search_paths, path); - PyMem_RawFree(path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - if (*p == '\0') { - break; - } - sys_path = p + 1; - } - config->module_search_paths_set = 1; return _PyStatus_OK(); -} - -/* Calculate the path configuration: - - - exec_prefix - - module_search_path - - stdlib_dir - - prefix - - program_full_path - - On Windows, more fields are calculated: - - - base_executable - - isolated - - site_import - - On other platforms, isolated and site_import are left unchanged, and - _PyConfig_InitPathConfig() copies executable to base_executable (if it's not - set). - - Priority, highest to lowest: - - - PyConfig - - _Py_path_config: set by Py_SetPath(), Py_SetPythonHome() - and Py_SetProgramName() - - _PyPathConfig_Calculate() -*/ -static PyStatus -pathconfig_init(_PyPathConfig *pathconfig, const PyConfig *config, - int compute_path_config) -{ - PyStatus status; - - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - status = pathconfig_copy(pathconfig, &_Py_path_config); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - - status = pathconfig_set_from_config(pathconfig, config); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - - if (compute_path_config) { - status = _PyPathConfig_Calculate(pathconfig, config); - } - -done: +error: PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - return status; + return _PyStatus_NO_MEMORY(); } -static PyStatus -config_init_pathconfig(PyConfig *config, int compute_path_config) -{ - _PyPathConfig pathconfig = _PyPathConfig_INIT; - PyStatus status; - - status = pathconfig_init(&pathconfig, config, compute_path_config); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - - if (!config->module_search_paths_set - && pathconfig.module_search_path != NULL) - { - status = config_init_module_search_paths(config, &pathconfig); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - } - -#define COPY_ATTR(PATH_ATTR, CONFIG_ATTR) \ - if (config->CONFIG_ATTR == NULL && pathconfig.PATH_ATTR != NULL) { \ - if (copy_wstr(&config->CONFIG_ATTR, pathconfig.PATH_ATTR) < 0) { \ - goto no_memory; \ - } \ - } - -#ifdef MS_WINDOWS - if (config->executable != NULL && config->base_executable == NULL) { - /* If executable is set explicitly in the configuration, - ignore calculated base_executable: _PyConfig_InitPathConfig() - will copy executable to base_executable */ - } - else { - COPY_ATTR(base_executable, base_executable); - } -#endif - - COPY_ATTR(program_full_path, executable); - COPY_ATTR(prefix, prefix); - COPY_ATTR(exec_prefix, exec_prefix); - COPY_ATTR(stdlib_dir, stdlib_dir); - -#undef COPY_ATTR - -#ifdef MS_WINDOWS - /* If a ._pth file is found: isolated and site_import are overridden */ - if (pathconfig.isolated != -1) { - config->isolated = pathconfig.isolated; - } - if (pathconfig.site_import != -1) { - config->site_import = pathconfig.site_import; - } -#endif - - status = _PyStatus_OK(); - goto done; - -no_memory: - status = _PyStatus_NO_MEMORY(); - -done: - pathconfig_clear(&pathconfig); - return status; -} - - -PyStatus -_PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) -{ - /* Do we need to calculate the path? */ - if (!config->module_search_paths_set - || config->executable == NULL - || config->prefix == NULL - || config->exec_prefix == NULL) - { - PyStatus status = config_init_pathconfig(config, compute_path_config); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - - if (config->base_prefix == NULL && config->prefix != NULL) { - if (copy_wstr(&config->base_prefix, config->prefix) < 0) { - return _PyStatus_NO_MEMORY(); - } - } - - if (config->base_exec_prefix == NULL && config->exec_prefix != NULL) { - if (copy_wstr(&config->base_exec_prefix, - config->exec_prefix) < 0) { - return _PyStatus_NO_MEMORY(); - } - } - - if (config->base_executable == NULL && config->executable != NULL) { - if (copy_wstr(&config->base_executable, - config->executable) < 0) { - return _PyStatus_NO_MEMORY(); - } - } - - return _PyStatus_OK(); -} - - -/* External interface */ - static void _Py_NO_RETURN path_out_of_memory(const char *func) { @@ -483,7 +194,7 @@ void Py_SetPath(const wchar_t *path) { if (path == NULL) { - pathconfig_clear(&_Py_path_config); + _PyPathConfig_ClearGlobal(); return; } @@ -494,6 +205,7 @@ Py_SetPath(const wchar_t *path) PyMem_RawFree(_Py_path_config.exec_prefix); PyMem_RawFree(_Py_path_config.stdlib_dir); PyMem_RawFree(_Py_path_config.module_search_path); + PyMem_RawFree(_Py_path_config.calculated_module_search_path); _Py_path_config.prefix = _PyMem_RawWcsdup(L""); _Py_path_config.exec_prefix = _PyMem_RawWcsdup(L""); @@ -505,6 +217,7 @@ Py_SetPath(const wchar_t *path) _Py_path_config.stdlib_dir = _PyMem_RawWcsdup(L""); } _Py_path_config.module_search_path = _PyMem_RawWcsdup(path); + _Py_path_config.calculated_module_search_path = NULL; PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); @@ -521,19 +234,19 @@ Py_SetPath(const wchar_t *path) void Py_SetPythonHome(const wchar_t *home) { - if (home == NULL) { - return; - } + int has_value = home && home[0]; PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); PyMem_RawFree(_Py_path_config.home); - _Py_path_config.home = _PyMem_RawWcsdup(home); + if (has_value) { + _Py_path_config.home = _PyMem_RawWcsdup(home); + } PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - if (_Py_path_config.home == NULL) { + if (has_value && _Py_path_config.home == NULL) { path_out_of_memory(__func__); } } @@ -542,19 +255,19 @@ Py_SetPythonHome(const wchar_t *home) void Py_SetProgramName(const wchar_t *program_name) { - if (program_name == NULL || program_name[0] == L'\0') { - return; - } + int has_value = program_name && program_name[0]; PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); PyMem_RawFree(_Py_path_config.program_name); - _Py_path_config.program_name = _PyMem_RawWcsdup(program_name); + if (has_value) { + _Py_path_config.program_name = _PyMem_RawWcsdup(program_name); + } PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - if (_Py_path_config.program_name == NULL) { + if (has_value && _Py_path_config.program_name == NULL) { path_out_of_memory(__func__); } } @@ -562,19 +275,19 @@ Py_SetProgramName(const wchar_t *program_name) void _Py_SetProgramFullPath(const wchar_t *program_full_path) { - if (program_full_path == NULL || program_full_path[0] == L'\0') { - return; - } + int has_value = program_full_path && program_full_path[0]; PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); PyMem_RawFree(_Py_path_config.program_full_path); - _Py_path_config.program_full_path = _PyMem_RawWcsdup(program_full_path); + if (has_value) { + _Py_path_config.program_full_path = _PyMem_RawWcsdup(program_full_path); + } PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - if (_Py_path_config.program_full_path == NULL) { + if (has_value && _Py_path_config.program_full_path == NULL) { path_out_of_memory(__func__); } } @@ -583,7 +296,12 @@ _Py_SetProgramFullPath(const wchar_t *program_full_path) wchar_t * Py_GetPath(void) { - return _Py_path_config.module_search_path; + /* If the user has provided a path, return that */ + if (_Py_path_config.module_search_path) { + return _Py_path_config.module_search_path; + } + /* If we have already done calculations, return the calculated path */ + return _Py_path_config.calculated_module_search_path; } @@ -632,6 +350,8 @@ Py_GetProgramName(void) return _Py_path_config.program_name; } + + /* Compute module search path from argv[0] or the current working directory ("-m module" case) which will be prepended to sys.argv: sys.path[0]. @@ -772,73 +492,6 @@ _PyPathConfig_ComputeSysPath0(const PyWideStringList *argv, PyObject **path0_p) } -#ifdef MS_WINDOWS -#define WCSTOK wcstok_s -#else -#define WCSTOK wcstok -#endif - -/* Search for a prefix value in an environment file (pyvenv.cfg). - - - If found, copy it into *value_p: string which must be freed by - PyMem_RawFree(). - - If not found, *value_p is set to NULL. -*/ -PyStatus -_Py_FindEnvConfigValue(FILE *env_file, const wchar_t *key, - wchar_t **value_p) -{ - *value_p = NULL; - - char buffer[MAXPATHLEN * 2 + 1]; /* allow extra for key, '=', etc. */ - buffer[Py_ARRAY_LENGTH(buffer)-1] = '\0'; - - while (!feof(env_file)) { - char * p = fgets(buffer, Py_ARRAY_LENGTH(buffer) - 1, env_file); - - if (p == NULL) { - break; - } - - size_t n = strlen(p); - if (p[n - 1] != '\n') { - /* line has overflowed - bail */ - break; - } - if (p[0] == '#') { - /* Comment - skip */ - continue; - } - - wchar_t *tmpbuffer = _Py_DecodeUTF8_surrogateescape(buffer, n, NULL); - if (tmpbuffer) { - wchar_t * state; - wchar_t * tok = WCSTOK(tmpbuffer, L" \t\r\n", &state); - if ((tok != NULL) && !wcscmp(tok, key)) { - tok = WCSTOK(NULL, L" \t", &state); - if ((tok != NULL) && !wcscmp(tok, L"=")) { - tok = WCSTOK(NULL, L"\r\n", &state); - if (tok != NULL) { - *value_p = _PyMem_RawWcsdup(tok); - PyMem_RawFree(tmpbuffer); - - if (*value_p == NULL) { - return _PyStatus_NO_MEMORY(); - } - - /* found */ - return _PyStatus_OK(); - } - } - } - PyMem_RawFree(tmpbuffer); - } - } - - /* not found */ - return _PyStatus_OK(); -} - #ifdef __cplusplus } #endif diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index c6b2a1eb961..84b76ea3275 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -459,7 +459,7 @@ interpreter_update_config(PyThreadState *tstate, int only_update_path_config) } if (_Py_IsMainInterpreter(tstate->interp)) { - PyStatus status = _PyConfig_WritePathConfig(config); + PyStatus status = _PyPathConfig_UpdateGlobal(config); if (_PyStatus_EXCEPTION(status)) { _PyErr_SetFromPyStatus(status); return -1; @@ -488,7 +488,7 @@ _PyInterpreterState_SetConfig(const PyConfig *src_config) goto done; } - status = PyConfig_Read(&config); + status = _PyConfig_Read(&config, 1); if (_PyStatus_EXCEPTION(status)) { _PyErr_SetFromPyStatus(status); goto done; @@ -549,7 +549,7 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime, config = _PyInterpreterState_GetConfig(interp); if (config->_install_importlib) { - status = _PyConfig_WritePathConfig(config); + status = _PyPathConfig_UpdateGlobal(config); if (_PyStatus_EXCEPTION(status)) { return status; }