From 96c8475362acb41decd1d7db9243f328973e5de7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 Sep 2019 16:17:34 +0200 Subject: [PATCH] [3.8] bpo-38234: Backport init path config changes from master (GH-16423) * bpo-38234: Py_SetPath() uses the program full path (GH-16357) Py_SetPath() now sets sys.executable to the program full path (Py_GetProgramFullPath()), rather than to the program name (Py_GetProgramName()). Fix also memory leaks in pathconfig_set_from_config(). (cherry picked from commit 1ce152a42eaa917d7763bce93f1e1ca72530d7ca) * bpo-38234: Add tests for Python init path config (GH-16358) (cherry picked from commit bb6bf7d342b4503a6227fd209fac934905b6a1aa) * bpo-38234: test_embed: test pyvenv.cfg and pybuilddir.txt (GH-16366) Add test_init_pybuilddir() and test_init_pyvenv_cfg() to test_embed to test pyvenv.cfg and pybuilddir.txt configuration files. Fix sysconfig._generate_posix_vars(): pybuilddir.txt uses UTF-8 encoding, not ASCII. (cherry picked from commit 52ad33abbfb6637d74932617c7013bae0ccf6e32) * bpo-38234: Cleanup getpath.c (GH-16367) * search_for_prefix() directly calls reduce() if found is greater than 0. * Add calculate_pybuilddir() subfunction. * search_for_prefix(): add path string buffer for readability. * Fix some error handling code paths: release resources on error. * calculate_read_pyenv(): rename tmpbuffer to filename. * test.pythoninfo now also logs windows.dll_path (cherry picked from commit 221fd84703c545408bbb4a6e0b58459651331f5c) * bpo-38234: Fix test_embed pathconfig tests (GH-16390) bpo-38234: On macOS and FreeBSD, the temporary directory can be symbolic link. For example, /tmp can be a symbolic link to /var/tmp. Call realpath() to resolve all symbolic links. (cherry picked from commit 00508a7407d7d300b487532e2271534b20e378a7) * bpo-38234: Add test_init_setpath_config() to test_embed (GH-16402) * Add test_embed.test_init_setpath_config(): test Py_SetPath() with PyConfig. * test_init_setpath() and test_init_setpythonhome() no longer call Py_SetProgramName(), but use the default program name. * _PyPathConfig: isolated, site_import and base_executable fields are now only available on Windows. * If executable is set explicitly in the configuration, ignore calculated base_executable: _PyConfig_InitPathConfig() copies executable to base_executable. * Complete path config documentation. (cherry picked from commit 8bf39b606ef7b02c0279a80789f3c4824b0da5e9) * bpo-38234: Complete init config documentation (GH-16404) (cherry picked from commit 88feaecd46a8f427e30ef7ad8cfcddfe392a2402) * bpo-38234: Fix test_embed.test_init_setpath_config() on FreeBSD (GH-16406) Explicitly preinitializes with a Python preconfiguration to avoid Py_SetPath() implicit preinitialization with a compat preconfiguration. Fix also test_init_setpath() and test_init_setpythonhome() on macOS: use self.test_exe as the executable (and base_executable), rather than shutil.which('python3'). (cherry picked from commit 49d99f01e6e51acec5ca57a02e857f0796bc418b) * bpo-38234: Py_Initialize() sets global path configuration (GH-16421) * Py_InitializeFromConfig() now writes PyConfig path configuration to the global path configuration (_Py_path_config). * Add test_embed.test_get_pathconfig(). * Fix typo in _PyWideStringList_Join(). (cherry picked from commit 12f2f177fc483723406d7917194e7f655a20631b) --- Doc/c-api/init.rst | 8 +- Doc/c-api/init_config.rst | 79 ++-- Doc/whatsnew/3.8.rst | 5 + Include/internal/pycore_pathconfig.h | 17 +- Lib/sysconfig.py | 2 +- Lib/test/pythoninfo.py | 130 +++++-- Lib/test/test_embed.py | 340 +++++++++++++++++- .../2019-09-24-17-09-48.bpo-38234.d0bhEA.rst | 3 + Modules/getpath.c | 195 ++++++---- PC/getpathp.c | 36 +- Programs/_testembed.c | 113 +++++- Python/pathconfig.c | 75 +++- Python/pylifecycle.c | 4 +- 13 files changed, 797 insertions(+), 210 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2019-09-24-17-09-48.bpo-38234.d0bhEA.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 0f8ff3b0dde..0b7a84d0315 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -472,8 +472,8 @@ Process-wide parameters dependent delimiter character, which is ``':'`` on Unix and Mac OS X, ``';'`` on Windows. - This also causes :data:`sys.executable` to be set only to the raw program - name (see :c:func:`Py_SetProgramName`) and for :data:`sys.prefix` and + This also causes :data:`sys.executable` to be set to the program + full path (see :c:func:`Py_GetProgramFullPath`) and for :data:`sys.prefix` and :data:`sys.exec_prefix` to be empty. It is up to the caller to modify these if required after calling :c:func:`Py_Initialize`. @@ -483,6 +483,10 @@ Process-wide parameters The path argument is copied internally, so the caller may free it after the call completes. + .. versionchanged:: 3.8 + The program full path is now used for :data:`sys.executable`, instead + of the program name. + .. c:function:: const char* Py_GetVersion() diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index bc24fa08131..0c3c725c841 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -241,6 +241,7 @@ PyPreConfig locale to decide if it should be coerced. .. c:member:: int coerce_c_locale_warn + If non-zero, emit a warning if the C locale is coerced. .. c:member:: int dev_mode @@ -300,7 +301,7 @@ For :ref:`Python Configuration ` (:c:func:`PyPreConfig_InitPythonConfig`), if Python is initialized with command line arguments, the command line arguments must also be passed to preinitialize Python, since they have an effect on the pre-configuration -like encodings. For example, the :option:`-X` ``utf8`` command line option +like encodings. For example, the :option:`-X utf8 <-X>` command line option enables the UTF-8 Mode. ``PyMem_SetAllocator()`` can be called after :c:func:`Py_PreInitialize` and @@ -464,7 +465,7 @@ PyConfig .. c:member:: int dev_mode - Development mode: see :option:`-X` ``dev``. + Development mode: see :option:`-X dev <-X>`. .. c:member:: int dump_refs @@ -482,7 +483,7 @@ PyConfig .. c:member:: int faulthandler - If non-zero, call :func:`faulthandler.enable`. + If non-zero, call :func:`faulthandler.enable` at startup. .. c:member:: wchar_t* filesystem_encoding @@ -504,6 +505,9 @@ PyConfig Python home directory. + Initialized from :envvar:`PYTHONHOME` environment variable value by + default. + .. c:member:: int import_time If non-zero, profile import time. @@ -561,7 +565,7 @@ PyConfig :data:`sys.path`. If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, the :c:member:`~PyConfig.module_search_paths` is overridden - by the function computing the :ref:`Path Configuration + by the function calculating the :ref:`Path Configuration `. .. c:member:: int optimization_level @@ -586,9 +590,9 @@ PyConfig .. c:member:: int pathconfig_warnings - If equal to 0, suppress warnings when computing the path configuration - (Unix only, Windows does not log any warning). Otherwise, warnings are - written into ``stderr``. + If equal to 0, suppress warnings when calculating the :ref:`Path + Configuration ` (Unix only, Windows does not log any + warning). Otherwise, warnings are written into ``stderr``. .. c:member:: wchar_t* prefix @@ -596,39 +600,46 @@ PyConfig .. c:member:: wchar_t* program_name - Program name. + Program name. Used to initialize :c:member:`~PyConfig.executable`, and in + early error messages. .. c:member:: wchar_t* pycache_prefix - ``.pyc`` cache prefix. + :data:`sys.pycache_prefix`: ``.pyc`` cache prefix. + + If NULL, :data:`sys.pycache_prefix` is set to ``None``. .. c:member:: int quiet Quiet mode. For example, don't display the copyright and version messages - even in interactive mode. + in interactive mode. .. c:member:: wchar_t* run_command - ``python3 -c COMMAND`` argument. + ``python3 -c COMMAND`` argument. Used by :c:func:`Py_RunMain`. .. c:member:: wchar_t* run_filename - ``python3 FILENAME`` argument. + ``python3 FILENAME`` argument. Used by :c:func:`Py_RunMain`. .. c:member:: wchar_t* run_module - ``python3 -m MODULE`` argument. + ``python3 -m MODULE`` argument. Used by :c:func:`Py_RunMain`. .. c:member:: int show_alloc_count Show allocation counts at exit? + Set to 1 by :option:`-X showalloccount <-X>` command line option. + Need a special Python build with ``COUNT_ALLOCS`` macro defined. .. c:member:: int show_ref_count Show total reference count at exit? + Set to 1 by :option:`-X showrefcount <-X>` command line option. + Need a debug build of Python (``Py_REF_DEBUG`` macro must be defined). .. c:member:: int site_import @@ -647,7 +658,7 @@ PyConfig .. c:member:: int tracemalloc - If non-zero, call :func:`tracemalloc.start`. + If non-zero, call :func:`tracemalloc.start` at startup. .. c:member:: int use_environment @@ -669,6 +680,9 @@ PyConfig If non-zero, write ``.pyc`` files. + :data:`sys.dont_write_bytecode` is initialized to the inverted value of + :c:member:`~PyConfig.write_bytecode`. + .. c:member:: PyWideStringList xoptions :data:`sys._xoptions`. @@ -694,8 +708,8 @@ Function to initialize Python: The caller is responsible to handle exceptions (error or exit) using :c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. -``PyImport_FrozenModules``, ``PyImport_AppendInittab()`` or -``PyImport_ExtendInittab()`` is used: they must be set or called after Python +If ``PyImport_FrozenModules``, ``PyImport_AppendInittab()`` or +``PyImport_ExtendInittab()`` are used, they must be set or called after Python preinitialization and before the Python initialization. Example setting the program name:: @@ -760,7 +774,7 @@ configuration, and then override some parameters:: /* Append our custom search path to sys.path */ status = PyWideStringList_Append(&config.module_search_paths, - L"/path/to/more/modules"); + L"/path/to/more/modules"); if (PyStatus_Exception(status)) { goto done; } @@ -791,9 +805,9 @@ isolate Python from the system. For example, to embed Python into an application. This configuration ignores global configuration variables, environments -variables and command line arguments (:c:member:`PyConfig.argv` is not parsed). -The C standard streams (ex: ``stdout``) and the LC_CTYPE locale are left -unchanged by default. +variables, command line arguments (:c:member:`PyConfig.argv` is not parsed) +and user site directory. The C standard streams (ex: ``stdout``) and the +LC_CTYPE locale are left unchanged. Signal handlers are not installed. Configuration files are still used with this configuration. Set the :ref:`Path Configuration ` ("output fields") to ignore these @@ -864,29 +878,38 @@ Path Configuration :c:type:`PyConfig` contains multiple fields for the path configuration: -* Path configuration input fields: +* Path configuration inputs: * :c:member:`PyConfig.home` * :c:member:`PyConfig.pathconfig_warnings` * :c:member:`PyConfig.program_name` * :c:member:`PyConfig.pythonpath_env` + * current working directory: to get absolute paths + * ``PATH`` environment variable to get the program full path + (from :c:member:`PyConfig.program_name`) + * ``__PYVENV_LAUNCHER__`` environment variable + * (Windows only) Application paths in the registry under + "Software\Python\PythonCore\X.Y\PythonPath" of HKEY_CURRENT_USER and + HKEY_LOCAL_MACHINE (where X.Y is the Python version). * Path configuration output fields: + * :c:member:`PyConfig.base_exec_prefix` * :c:member:`PyConfig.base_executable` + * :c:member:`PyConfig.base_prefix` * :c:member:`PyConfig.exec_prefix` * :c:member:`PyConfig.executable` - * :c:member:`PyConfig.prefix` * :c:member:`PyConfig.module_search_paths_set`, :c:member:`PyConfig.module_search_paths` + * :c:member:`PyConfig.prefix` -If at least one "output field" is not set, Python computes the path +If at least one "output field" is not set, Python calculates the path configuration to fill unset fields. If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, :c:member:`~PyConfig.module_search_paths` is overridden and :c:member:`~PyConfig.module_search_paths_set` is set to 1. -It is possible to completely ignore the function computing the default +It is possible to completely ignore the function calculating the default path configuration by setting explicitly all path configuration output fields listed above. A string is considered as set even if it is non-empty. ``module_search_paths`` is considered as set if @@ -894,7 +917,7 @@ fields listed above. A string is considered as set even if it is non-empty. configuration input fields are ignored as well. Set :c:member:`~PyConfig.pathconfig_warnings` to 0 to suppress warnings when -computing the path configuration (Unix only, Windows does not log any warning). +calculating the path configuration (Unix only, Windows does not log any warning). If :c:member:`~PyConfig.base_prefix` or :c:member:`~PyConfig.base_exec_prefix` fields are not set, they inherit their value from :c:member:`~PyConfig.prefix` @@ -961,7 +984,7 @@ initialization, the core feature of the :pep:`432`: * Builtin exceptions; * Builtin and frozen modules; * The :mod:`sys` module is only partially initialized - (ex: :data:`sys.path` doesn't exist yet); + (ex: :data:`sys.path` doesn't exist yet). * "Main" initialization phase, Python is fully initialized: @@ -987,9 +1010,9 @@ No module is imported during the "Core" phase and the ``importlib`` module is not configured: the :ref:`Path Configuration ` is only applied during the "Main" phase. It may allow to customize Python in Python to override or tune the :ref:`Path Configuration `, maybe -install a custom sys.meta_path importer or an import hook, etc. +install a custom :data:`sys.meta_path` importer or an import hook, etc. -It may become possible to compute the :ref:`Path Configuration +It may become possible to calculatin the :ref:`Path Configuration ` in Python, after the Core phase and before the Main phase, which is one of the :pep:`432` motivation. diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index c2455f487b1..0995cb3b911 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -1347,6 +1347,11 @@ Build and C API Changes parameter for indicating the number of positional-only arguments. (Contributed by Pablo Galindo in :issue:`37221`.) +* :c:func:`Py_SetPath` now sets :data:`sys.executable` to the program full + path (:c:func:`Py_GetProgramFullPath`) rather than to the program name + (:c:func:`Py_GetProgramName`). + (Contributed by Victor Stinner in :issue:`38234`.) + Deprecated ========== diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index 61b3790fe1f..ce75ccee835 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -19,6 +19,7 @@ typedef struct _PyPathConfig { wchar_t *program_name; /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */ wchar_t *home; +#ifdef MS_WINDOWS /* isolated and site_import are used to set Py_IsolatedFlag and Py_NoSiteFlag flags on Windows in read_pth_file(). These fields are ignored when their value are equal to -1 (unset). */ @@ -26,12 +27,18 @@ typedef struct _PyPathConfig { int site_import; /* Set when a venv is detected */ wchar_t *base_executable; +#endif } _PyPathConfig; -#define _PyPathConfig_INIT \ - {.module_search_path = NULL, \ - .isolated = -1, \ - .site_import = -1} +#ifdef MS_WINDOWS +# define _PyPathConfig_INIT \ + {.module_search_path = NULL, \ + .isolated = -1, \ + .site_import = -1} +#else +# define _PyPathConfig_INIT \ + {.module_search_path = NULL} +#endif /* Note: _PyPathConfig_INIT sets other fields to 0/NULL */ PyAPI_DATA(_PyPathConfig) _Py_path_config; @@ -59,7 +66,7 @@ extern int _Py_FindEnvConfigValue( extern wchar_t* _Py_GetDLLPath(void); #endif -extern PyStatus _PyPathConfig_Init(void); +extern PyStatus _PyConfig_WritePathConfig(const PyConfig *config); extern void _Py_DumpPathConfig(PyThreadState *tstate); #ifdef __cplusplus diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index e76e6927cb1..b9e2fafbc08 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -412,7 +412,7 @@ def _generate_posix_vars(): pprint.pprint(vars, stream=f) # Create file used for sys.path fixup -- see Modules/getpath.c - with open('pybuilddir.txt', 'w', encoding='ascii') as f: + with open('pybuilddir.txt', 'w', encoding='utf8') as f: f.write(pybuilddir) def _init_posix(vars): diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index e9edf675b91..d2fa6c59365 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -161,6 +161,25 @@ def collect_builtins(info_add): info_add('builtins.float.double_format', float.__getformat__("double")) +def collect_urandom(info_add): + import os + + if hasattr(os, 'getrandom'): + # PEP 524: Check if system urandom is initialized + try: + try: + os.getrandom(1, os.GRND_NONBLOCK) + state = 'ready (initialized)' + except BlockingIOError as exc: + state = 'not seeded yet (%s)' % exc + info_add('os.getrandom', state) + except OSError as exc: + # Python was compiled on a more recent Linux version + # than the current Linux kernel: ignore OSError(ENOSYS) + if exc.errno != errno.ENOSYS: + raise + + def collect_os(info_add): import os @@ -180,16 +199,16 @@ def collect_os(info_add): ) copy_attributes(info_add, os, 'os.%s', attributes, formatter=format_attr) - call_func(info_add, 'os.cwd', os, 'getcwd') + call_func(info_add, 'os.getcwd', os, 'getcwd') - call_func(info_add, 'os.uid', os, 'getuid') - call_func(info_add, 'os.gid', os, 'getgid') + call_func(info_add, 'os.getuid', os, 'getuid') + call_func(info_add, 'os.getgid', os, 'getgid') call_func(info_add, 'os.uname', os, 'uname') def format_groups(groups): return ', '.join(map(str, groups)) - call_func(info_add, 'os.groups', os, 'getgroups', formatter=format_groups) + call_func(info_add, 'os.getgroups', os, 'getgroups', formatter=format_groups) if hasattr(os, 'getlogin'): try: @@ -202,7 +221,7 @@ def collect_os(info_add): info_add("os.login", login) call_func(info_add, 'os.cpu_count', os, 'cpu_count') - call_func(info_add, 'os.loadavg', os, 'getloadavg') + call_func(info_add, 'os.getloadavg', os, 'getloadavg') # Environment variables used by the stdlib and tests. Don't log the full # environment: filter to list to not leak sensitive information. @@ -286,20 +305,32 @@ def collect_os(info_add): os.umask(mask) info_add("os.umask", '%03o' % mask) - if hasattr(os, 'getrandom'): - # PEP 524: Check if system urandom is initialized - try: - try: - os.getrandom(1, os.GRND_NONBLOCK) - state = 'ready (initialized)' - except BlockingIOError as exc: - state = 'not seeded yet (%s)' % exc - info_add('os.getrandom', state) - except OSError as exc: - # Python was compiled on a more recent Linux version - # than the current Linux kernel: ignore OSError(ENOSYS) - if exc.errno != errno.ENOSYS: - raise + +def collect_pwd(info_add): + try: + import pwd + except ImportError: + return + import os + + uid = os.getuid() + try: + entry = pwd.getpwuid(uid) + except KeyError: + entry = None + + info_add('pwd.getpwuid(%s)'% uid, + entry if entry is not None else '') + + if entry is None: + # there is nothing interesting to read if the current user identifier + # is not the password database + return + + if hasattr(os, 'getgrouplist'): + groups = os.getgrouplist(entry.pw_name, entry.pw_gid) + groups = ', '.join(map(str, groups)) + info_add('os.getgrouplist', groups) def collect_readline(info_add): @@ -620,36 +651,71 @@ def collect_subprocess(info_add): copy_attributes(info_add, subprocess, 'subprocess.%s', ('_USE_POSIX_SPAWN',)) +def collect_windows(info_add): + try: + import ctypes + except ImportError: + return + + if not hasattr(ctypes, 'WinDLL'): + return + + ntdll = ctypes.WinDLL('ntdll') + BOOLEAN = ctypes.c_ubyte + + try: + RtlAreLongPathsEnabled = ntdll.RtlAreLongPathsEnabled + except AttributeError: + res = '' + else: + RtlAreLongPathsEnabled.restype = BOOLEAN + RtlAreLongPathsEnabled.argtypes = () + res = bool(RtlAreLongPathsEnabled()) + info_add('windows.RtlAreLongPathsEnabled', res) + + try: + import _winapi + dll_path = _winapi.GetModuleFileName(sys.dllhandle) + info_add('windows.dll_path', dll_path) + except (ImportError, AttributeError): + pass + + def collect_info(info): error = False info_add = info.add for collect_func in ( - # collect_os() should be the first, to check the getrandom() status - collect_os, + # collect_urandom() must be the first, to check the getrandom() status. + # Other functions may block on os.urandom() indirectly and so change + # its state. + collect_urandom, collect_builtins, + collect_cc, + collect_datetime, + collect_decimal, + collect_expat, collect_gdb, + collect_gdbm, + collect_get_config, collect_locale, + collect_os, collect_platform, + collect_pwd, collect_readline, + collect_resource, collect_socket, collect_sqlite, collect_ssl, + collect_subprocess, collect_sys, collect_sysconfig, - collect_time, - collect_datetime, - collect_tkinter, - collect_zlib, - collect_expat, - collect_decimal, collect_testcapi, - collect_resource, - collect_cc, - collect_gdbm, - collect_get_config, - collect_subprocess, + collect_time, + collect_tkinter, + collect_windows, + collect_zlib, # Collecting from tests should be last as they have side effects. collect_test_socket, diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index e02acbc6bef..ed2b96fbfc1 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -3,15 +3,19 @@ from test import support import unittest from collections import namedtuple +import contextlib import json import os import re +import shutil import subprocess import sys +import tempfile import textwrap MS_WINDOWS = (os.name == 'nt') +MACOS = (sys.platform == 'darwin') PYMEM_ALLOCATOR_NOT_SET = 0 PYMEM_ALLOCATOR_DEBUG = 2 @@ -25,6 +29,12 @@ API_PYTHON = 2 API_ISOLATED = 3 +def debug_build(program): + program = os.path.basename(program) + name = os.path.splitext(program)[0] + return name.endswith("_d") + + def remove_python_envvars(): env = dict(os.environ) # Remove PYTHON* environment variables to get deterministic environment @@ -40,7 +50,7 @@ class EmbeddingTestsMixin: basepath = os.path.dirname(os.path.dirname(os.path.dirname(here))) exename = "_testembed" if MS_WINDOWS: - ext = ("_d" if "_d" in sys.executable else "") + ".exe" + ext = ("_d" if debug_build(sys.executable) else "") + ".exe" exename += ext exepath = os.path.dirname(sys.executable) else: @@ -58,7 +68,8 @@ class EmbeddingTestsMixin: os.chdir(self.oldcwd) def run_embedded_interpreter(self, *args, env=None, - timeout=None, returncode=0, input=None): + timeout=None, returncode=0, input=None, + cwd=None): """Runs a test in the embedded interpreter""" cmd = [self.test_exe] cmd.extend(args) @@ -72,7 +83,8 @@ class EmbeddingTestsMixin: stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, - env=env) + env=env, + cwd=cwd) try: (out, err) = p.communicate(input=input, timeout=timeout) except: @@ -460,6 +472,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): EXPECTED_CONFIG = None + @classmethod + def tearDownClass(cls): + # clear cache + cls.EXPECTED_CONFIG = None + def main_xoptions(self, xoptions_list): xoptions = {} for opt in xoptions_list: @@ -470,7 +487,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): xoptions[opt] = True return xoptions - def _get_expected_config(self, env): + def _get_expected_config_impl(self): + env = remove_python_envvars() code = textwrap.dedent(''' import json import sys @@ -489,23 +507,37 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): args = [sys.executable, '-S', '-c', code] proc = subprocess.run(args, env=env, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + stderr=subprocess.PIPE) if proc.returncode: raise Exception(f"failed to get the default config: " f"stdout={proc.stdout!r} stderr={proc.stderr!r}") stdout = proc.stdout.decode('utf-8') + # ignore stderr try: return json.loads(stdout) except json.JSONDecodeError: self.fail(f"fail to decode stdout: {stdout!r}") + def _get_expected_config(self): + cls = InitConfigTests + if cls.EXPECTED_CONFIG is None: + cls.EXPECTED_CONFIG = self._get_expected_config_impl() + + # get a copy + configs = {} + for config_key, config_value in cls.EXPECTED_CONFIG.items(): + config = {} + for key, value in config_value.items(): + if isinstance(value, list): + value = value.copy() + config[key] = value + configs[config_key] = config + return configs + def get_expected_config(self, expected_preconfig, expected, env, api, modify_path_cb=None): cls = self.__class__ - if cls.EXPECTED_CONFIG is None: - cls.EXPECTED_CONFIG = self._get_expected_config(env) - configs = {key: dict(value) - for key, value in self.EXPECTED_CONFIG.items()} + configs = self._get_expected_config() pre_config = configs['pre_config'] for key, value in expected_preconfig.items(): @@ -553,9 +585,10 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): if value is self.GET_DEFAULT_CONFIG: expected[key] = config[key] - prepend_path = expected['pythonpath_env'] - if prepend_path is not None: - expected['module_search_paths'] = [prepend_path, *expected['module_search_paths']] + pythonpath_env = expected['pythonpath_env'] + if pythonpath_env is not None: + paths = pythonpath_env.split(os.path.pathsep) + expected['module_search_paths'] = [*paths, *expected['module_search_paths']] if modify_path_cb is not None: expected['module_search_paths'] = expected['module_search_paths'].copy() modify_path_cb(expected['module_search_paths']) @@ -603,13 +636,19 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.assertEqual(configs['global_config'], expected) def check_all_configs(self, testname, expected_config=None, - expected_preconfig=None, modify_path_cb=None, stderr=None, - *, api): - env = remove_python_envvars() + expected_preconfig=None, modify_path_cb=None, + stderr=None, *, api, preconfig_api=None, + env=None, ignore_stderr=False, cwd=None): + new_env = remove_python_envvars() + if env is not None: + new_env.update(env) + env = new_env - if api == API_ISOLATED: + if preconfig_api is None: + preconfig_api = api + if preconfig_api == API_ISOLATED: default_preconfig = self.PRE_CONFIG_ISOLATED - elif api == API_PYTHON: + elif preconfig_api == API_PYTHON: default_preconfig = self.PRE_CONFIG_PYTHON else: default_preconfig = self.PRE_CONFIG_COMPAT @@ -631,10 +670,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): expected_config, env, api, modify_path_cb) - out, err = self.run_embedded_interpreter(testname, env=env) + out, err = self.run_embedded_interpreter(testname, + env=env, cwd=cwd) if stderr is None and not expected_config['verbose']: stderr = "" - if stderr is not None: + if stderr is not None and not ignore_stderr: self.assertEqual(err.rstrip(), stderr) try: configs = json.loads(out) @@ -965,6 +1005,268 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.check_all_configs("test_init_dont_parse_argv", config, pre_config, api=API_PYTHON) + def default_program_name(self, config): + if MS_WINDOWS: + program_name = 'python' + executable = self.test_exe + else: + program_name = 'python3' + if MACOS: + executable = self.test_exe + else: + executable = shutil.which(program_name) or '' + config.update({ + 'program_name': program_name, + 'base_executable': executable, + 'executable': executable, + }) + + def test_init_setpath(self): + # Test Py_SetPath() + config = self._get_expected_config() + paths = config['config']['module_search_paths'] + + config = { + 'module_search_paths': paths, + 'prefix': '', + 'base_prefix': '', + 'exec_prefix': '', + 'base_exec_prefix': '', + } + self.default_program_name(config) + env = {'TESTPATH': os.path.pathsep.join(paths)} + self.check_all_configs("test_init_setpath", config, + api=API_COMPAT, env=env, + ignore_stderr=True) + + def test_init_setpath_config(self): + # Test Py_SetPath() with PyConfig + config = self._get_expected_config() + paths = config['config']['module_search_paths'] + + config = { + # set by Py_SetPath() + 'module_search_paths': paths, + 'prefix': '', + 'base_prefix': '', + 'exec_prefix': '', + 'base_exec_prefix': '', + # overriden by PyConfig + 'program_name': 'conf_program_name', + 'base_executable': 'conf_executable', + 'executable': 'conf_executable', + } + env = {'TESTPATH': os.path.pathsep.join(paths)} + self.check_all_configs("test_init_setpath_config", config, + api=API_PYTHON, env=env, ignore_stderr=True) + + def module_search_paths(self, prefix=None, exec_prefix=None): + config = self._get_expected_config() + if prefix is None: + prefix = config['config']['prefix'] + if exec_prefix is None: + exec_prefix = config['config']['prefix'] + if MS_WINDOWS: + return config['config']['module_search_paths'] + else: + ver = sys.version_info + return [ + os.path.join(prefix, 'lib', + f'python{ver.major}{ver.minor}.zip'), + os.path.join(prefix, 'lib', + f'python{ver.major}.{ver.minor}'), + os.path.join(exec_prefix, 'lib', + f'python{ver.major}.{ver.minor}', 'lib-dynload'), + ] + + @contextlib.contextmanager + def tmpdir_with_python(self): + # Temporary directory with a copy of the Python program + with tempfile.TemporaryDirectory() as tmpdir: + # bpo-38234: On macOS and FreeBSD, the temporary directory + # can be symbolic link. For example, /tmp can be a symbolic link + # to /var/tmp. Call realpath() to resolve all symbolic links. + tmpdir = os.path.realpath(tmpdir) + + if MS_WINDOWS: + # Copy pythonXY.dll (or pythonXY_d.dll) + ver = sys.version_info + dll = f'python{ver.major}{ver.minor}' + if debug_build(sys.executable): + dll += '_d' + dll += '.dll' + dll = os.path.join(os.path.dirname(self.test_exe), dll) + dll_copy = os.path.join(tmpdir, os.path.basename(dll)) + shutil.copyfile(dll, dll_copy) + + # Copy Python program + exec_copy = os.path.join(tmpdir, os.path.basename(self.test_exe)) + shutil.copyfile(self.test_exe, exec_copy) + shutil.copystat(self.test_exe, exec_copy) + self.test_exe = exec_copy + + yield tmpdir + + def test_init_setpythonhome(self): + # Test Py_SetPythonHome(home) with PYTHONPATH env var + config = self._get_expected_config() + paths = config['config']['module_search_paths'] + paths_str = os.path.pathsep.join(paths) + + for path in paths: + if not os.path.isdir(path): + continue + if os.path.exists(os.path.join(path, 'os.py')): + home = os.path.dirname(path) + break + else: + self.fail(f"Unable to find home in {paths!r}") + + prefix = exec_prefix = home + ver = sys.version_info + expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) + + config = { + 'home': home, + 'module_search_paths': expected_paths, + 'prefix': prefix, + 'base_prefix': prefix, + 'exec_prefix': exec_prefix, + 'base_exec_prefix': exec_prefix, + 'pythonpath_env': paths_str, + } + self.default_program_name(config) + env = {'TESTHOME': home, 'PYTHONPATH': paths_str} + self.check_all_configs("test_init_setpythonhome", config, + api=API_COMPAT, env=env) + + def copy_paths_by_env(self, config): + all_configs = self._get_expected_config() + paths = all_configs['config']['module_search_paths'] + paths_str = os.path.pathsep.join(paths) + config['pythonpath_env'] = paths_str + env = {'PYTHONPATH': paths_str} + return env + + @unittest.skipIf(MS_WINDOWS, 'Windows does not use pybuilddir.txt') + def test_init_pybuilddir(self): + # Test path configuration with pybuilddir.txt configuration file + + with self.tmpdir_with_python() as tmpdir: + # pybuilddir.txt is a sub-directory relative to the current + # directory (tmpdir) + subdir = 'libdir' + libdir = os.path.join(tmpdir, subdir) + os.mkdir(libdir) + + filename = os.path.join(tmpdir, 'pybuilddir.txt') + with open(filename, "w", encoding="utf8") as fp: + fp.write(subdir) + + module_search_paths = self.module_search_paths() + module_search_paths[-1] = libdir + + executable = self.test_exe + config = { + 'base_executable': executable, + 'executable': executable, + 'module_search_paths': module_search_paths, + } + env = self.copy_paths_by_env(config) + self.check_all_configs("test_init_compat_config", config, + api=API_COMPAT, env=env, + ignore_stderr=True, cwd=tmpdir) + + def test_init_pyvenv_cfg(self): + # Test path configuration with pyvenv.cfg configuration file + + with self.tmpdir_with_python() as tmpdir, \ + tempfile.TemporaryDirectory() as pyvenv_home: + ver = sys.version_info + + if not MS_WINDOWS: + lib_dynload = os.path.join(pyvenv_home, + 'lib', + f'python{ver.major}.{ver.minor}', + 'lib-dynload') + os.makedirs(lib_dynload) + else: + lib_dynload = os.path.join(pyvenv_home, 'lib') + os.makedirs(lib_dynload) + # getpathp.c uses Lib\os.py as the LANDMARK + shutil.copyfile(os.__file__, os.path.join(lib_dynload, 'os.py')) + + filename = os.path.join(tmpdir, 'pyvenv.cfg') + with open(filename, "w", encoding="utf8") as fp: + print("home = %s" % pyvenv_home, file=fp) + print("include-system-site-packages = false", file=fp) + + paths = self.module_search_paths() + if not MS_WINDOWS: + paths[-1] = lib_dynload + else: + for index, path in enumerate(paths): + if index == 0: + paths[index] = os.path.join(tmpdir, os.path.basename(path)) + else: + paths[index] = os.path.join(pyvenv_home, os.path.basename(path)) + paths[-1] = pyvenv_home + + executable = self.test_exe + exec_prefix = pyvenv_home + config = { + 'base_exec_prefix': exec_prefix, + 'exec_prefix': exec_prefix, + 'base_executable': executable, + 'executable': executable, + 'module_search_paths': paths, + } + if MS_WINDOWS: + config['base_prefix'] = pyvenv_home + config['prefix'] = pyvenv_home + env = self.copy_paths_by_env(config) + self.check_all_configs("test_init_compat_config", config, + api=API_COMPAT, env=env, + ignore_stderr=True, cwd=tmpdir) + + def test_global_pathconfig(self): + # Test C API functions getting the path configuration: + # + # - Py_GetExecPrefix() + # - Py_GetPath() + # - Py_GetPrefix() + # - Py_GetProgramFullPath() + # - Py_GetProgramName() + # - Py_GetPythonHome() + # + # The global path configuration (_Py_path_config) must be a copy + # of the path configuration of PyInterpreter.config (PyConfig). + ctypes = support.import_module('ctypes') + _testinternalcapi = support.import_module('_testinternalcapi') + + def get_func(name): + func = getattr(ctypes.pythonapi, name) + func.argtypes = () + func.restype = ctypes.c_wchar_p + return func + + Py_GetPath = get_func('Py_GetPath') + Py_GetPrefix = get_func('Py_GetPrefix') + Py_GetExecPrefix = get_func('Py_GetExecPrefix') + Py_GetProgramName = get_func('Py_GetProgramName') + Py_GetProgramFullPath = get_func('Py_GetProgramFullPath') + Py_GetPythonHome = get_func('Py_GetPythonHome') + + config = _testinternalcapi.get_configs()['config'] + + self.assertEqual(Py_GetPath().split(os.path.pathsep), + config['module_search_paths']) + self.assertEqual(Py_GetPrefix(), config['prefix']) + self.assertEqual(Py_GetExecPrefix(), config['exec_prefix']) + self.assertEqual(Py_GetProgramName(), config['program_name']) + self.assertEqual(Py_GetProgramFullPath(), config['executable']) + self.assertEqual(Py_GetPythonHome(), config['home']) + class AuditingTests(EmbeddingTestsMixin, unittest.TestCase): def test_open_code_hook(self): diff --git a/Misc/NEWS.d/next/C API/2019-09-24-17-09-48.bpo-38234.d0bhEA.rst b/Misc/NEWS.d/next/C API/2019-09-24-17-09-48.bpo-38234.d0bhEA.rst new file mode 100644 index 00000000000..ba4cc312e69 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2019-09-24-17-09-48.bpo-38234.d0bhEA.rst @@ -0,0 +1,3 @@ +:c:func:`Py_SetPath` now sets :data:`sys.executable` to the program full +path (:c:func:`Py_GetProgramFullPath`) rather than to the program name +(:c:func:`Py_GetProgramName`). diff --git a/Modules/getpath.c b/Modules/getpath.c index de32c3d5170..b727f66953b 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -370,12 +370,15 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig, const wchar_t *argv0_path, wchar_t *prefix, size_t prefix_len, int *found) { + wchar_t path[MAXPATHLEN+1]; + memset(path, 0, sizeof(path)); + size_t path_len = Py_ARRAY_LENGTH(path); + PyStatus status; - size_t n; - wchar_t *vpath; /* If PYTHONHOME is set, we believe it unconditionally */ if (pathconfig->home) { + /* Path: / */ if (safe_wcscpy(prefix, pathconfig->home, prefix_len) < 0) { return PATHLEN_ERR(); } @@ -387,27 +390,25 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig, if (_PyStatus_EXCEPTION(status)) { return status; } - status = joinpath(prefix, LANDMARK, prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } *found = 1; return _PyStatus_OK(); } /* Check to see if argv[0] is in the build directory */ - if (safe_wcscpy(prefix, argv0_path, prefix_len) < 0) { + if (safe_wcscpy(path, argv0_path, path_len) < 0) { return PATHLEN_ERR(); } - status = joinpath(prefix, L"Modules/Setup.local", prefix_len); + status = joinpath(path, L"Modules/Setup.local", path_len); if (_PyStatus_EXCEPTION(status)) { return status; } - if (isfile(prefix)) { - /* Check VPATH to see if argv0_path is in the build directory. */ - vpath = Py_DecodeLocale(VPATH, NULL); + if (isfile(path)) { + /* Check VPATH to see if argv0_path is in the build directory. + VPATH can be empty. */ + wchar_t *vpath = Py_DecodeLocale(VPATH, NULL); if (vpath != NULL) { + /* Path: / / Lib / LANDMARK */ if (safe_wcscpy(prefix, argv0_path, prefix_len) < 0) { return PATHLEN_ERR(); } @@ -428,6 +429,7 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig, if (ismodule(prefix, prefix_len)) { *found = -1; + reduce(prefix); return _PyStatus_OK(); } } @@ -440,7 +442,8 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig, } do { - n = wcslen(prefix); + /* Path: / / LANDMARK */ + size_t n = wcslen(prefix); status = joinpath(prefix, calculate->lib_python, prefix_len); if (_PyStatus_EXCEPTION(status)) { return status; @@ -452,13 +455,15 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig, if (ismodule(prefix, prefix_len)) { *found = 1; + reduce(prefix); return _PyStatus_OK(); } prefix[n] = L'\0'; reduce(prefix); } while (prefix[0]); - /* Look at configure's PREFIX */ + /* Look at configure's PREFIX. + Path: / / LANDMARK */ if (safe_wcscpy(prefix, calculate->prefix, prefix_len) < 0) { return PATHLEN_ERR(); } @@ -473,6 +478,7 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig, if (ismodule(prefix, prefix_len)) { *found = 1; + reduce(prefix); return _PyStatus_OK(); } @@ -509,9 +515,6 @@ calculate_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig, return status; } } - else { - reduce(prefix); - } return _PyStatus_OK(); } @@ -546,6 +549,67 @@ calculate_set_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig, } +static PyStatus +calculate_pybuilddir(const wchar_t *argv0_path, + wchar_t *exec_prefix, size_t exec_prefix_len, + int *found) +{ + PyStatus status; + + wchar_t filename[MAXPATHLEN+1]; + memset(filename, 0, sizeof(filename)); + size_t filename_len = Py_ARRAY_LENGTH(filename); + + /* Check to see if argv[0] is in the build directory. "pybuilddir.txt" + is written by setup.py and contains the relative path to the location + of shared library modules. + + Filename: / "pybuilddir.txt" */ + if (safe_wcscpy(filename, argv0_path, filename_len) < 0) { + return PATHLEN_ERR(); + } + status = joinpath(filename, L"pybuilddir.txt", filename_len); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + + if (!isfile(filename)) { + return _PyStatus_OK(); + } + + FILE *fp = _Py_wfopen(filename, L"rb"); + if (fp == NULL) { + errno = 0; + return _PyStatus_OK(); + } + + char buf[MAXPATHLEN + 1]; + size_t n = fread(buf, 1, Py_ARRAY_LENGTH(buf) - 1, fp); + buf[n] = '\0'; + fclose(fp); + + size_t dec_len; + wchar_t *pybuilddir = _Py_DecodeUTF8_surrogateescape(buf, n, &dec_len); + if (!pybuilddir) { + return DECODE_LOCALE_ERR("pybuilddir.txt", dec_len); + } + + /* Path: / */ + if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) { + PyMem_RawFree(pybuilddir); + return PATHLEN_ERR(); + } + status = joinpath(exec_prefix, pybuilddir, exec_prefix_len); + PyMem_RawFree(pybuilddir); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + + *found = -1; + return _PyStatus_OK(); +} + + /* search_for_exec_prefix requires that argv0_path be no more than MAXPATHLEN bytes long. */ @@ -556,10 +620,10 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig, int *found) { PyStatus status; - size_t n; /* If PYTHONHOME is set, we believe it unconditionally */ if (pathconfig->home) { + /* Path: / / "lib-dynload" */ wchar_t *delim = wcschr(pathconfig->home, DELIM); if (delim) { if (safe_wcscpy(exec_prefix, delim+1, exec_prefix_len) < 0) { @@ -583,47 +647,15 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig, return _PyStatus_OK(); } - /* Check to see if argv[0] is in the build directory. "pybuilddir.txt" - is written by setup.py and contains the relative path to the location - of shared library modules. */ - if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) { - return PATHLEN_ERR(); - } - status = joinpath(exec_prefix, L"pybuilddir.txt", exec_prefix_len); + /* Check for pybuilddir.txt */ + assert(*found == 0); + status = calculate_pybuilddir(argv0_path, exec_prefix, exec_prefix_len, + found); if (_PyStatus_EXCEPTION(status)) { return status; } - - if (isfile(exec_prefix)) { - FILE *f = _Py_wfopen(exec_prefix, L"rb"); - if (f == NULL) { - errno = 0; - } - else { - char buf[MAXPATHLEN + 1]; - n = fread(buf, 1, Py_ARRAY_LENGTH(buf) - 1, f); - buf[n] = '\0'; - fclose(f); - - wchar_t *pybuilddir; - size_t dec_len; - pybuilddir = _Py_DecodeUTF8_surrogateescape(buf, n, &dec_len); - if (!pybuilddir) { - return DECODE_LOCALE_ERR("pybuilddir.txt", dec_len); - } - - if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) { - return PATHLEN_ERR(); - } - status = joinpath(exec_prefix, pybuilddir, exec_prefix_len); - PyMem_RawFree(pybuilddir ); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - *found = -1; - return _PyStatus_OK(); - } + if (*found) { + return _PyStatus_OK(); } /* Search from argv0_path, until root is found */ @@ -633,7 +665,8 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig, } do { - n = wcslen(exec_prefix); + /* Path: / / "lib-dynload" */ + size_t n = wcslen(exec_prefix); status = joinpath(exec_prefix, calculate->lib_python, exec_prefix_len); if (_PyStatus_EXCEPTION(status)) { return status; @@ -650,7 +683,9 @@ search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig, reduce(exec_prefix); } while (exec_prefix[0]); - /* Look at configure's EXEC_PREFIX */ + /* Look at configure's EXEC_PREFIX. + + Path: / / "lib-dynload" */ if (safe_wcscpy(exec_prefix, calculate->exec_prefix, exec_prefix_len) < 0) { return PATHLEN_ERR(); } @@ -962,43 +997,49 @@ calculate_read_pyenv(PyCalculatePath *calculate, wchar_t *argv0_path, size_t argv0_path_len) { PyStatus status; - wchar_t tmpbuffer[MAXPATHLEN+1]; - const size_t buflen = Py_ARRAY_LENGTH(tmpbuffer); - wchar_t *env_cfg = L"pyvenv.cfg"; + const wchar_t *env_cfg = L"pyvenv.cfg"; FILE *env_file; - if (safe_wcscpy(tmpbuffer, argv0_path, buflen) < 0) { + wchar_t filename[MAXPATHLEN+1]; + const size_t filename_len = Py_ARRAY_LENGTH(filename); + memset(filename, 0, sizeof(filename)); + + /* Filename: / "pyvenv.cfg" */ + if (safe_wcscpy(filename, argv0_path, filename_len) < 0) { return PATHLEN_ERR(); } - status = joinpath(tmpbuffer, env_cfg, buflen); + status = joinpath(filename, env_cfg, filename_len); if (_PyStatus_EXCEPTION(status)) { return status; } - env_file = _Py_wfopen(tmpbuffer, L"r"); + env_file = _Py_wfopen(filename, L"r"); if (env_file == NULL) { errno = 0; - reduce(tmpbuffer); - reduce(tmpbuffer); - status = joinpath(tmpbuffer, env_cfg, buflen); + /* Filename: / "pyvenv.cfg" */ + reduce(filename); + reduce(filename); + status = joinpath(filename, env_cfg, filename_len); if (_PyStatus_EXCEPTION(status)) { return status; } - env_file = _Py_wfopen(tmpbuffer, L"r"); + env_file = _Py_wfopen(filename, L"r"); if (env_file == NULL) { errno = 0; + return _PyStatus_OK(); } } - if (env_file == NULL) { - return _PyStatus_OK(); - } - /* Look for a 'home' variable and set argv0_path to it, if found */ - if (_Py_FindEnvConfigValue(env_file, L"home", tmpbuffer, buflen)) { - if (safe_wcscpy(argv0_path, tmpbuffer, argv0_path_len) < 0) { + wchar_t home[MAXPATHLEN+1]; + memset(home, 0, sizeof(home)); + + if (_Py_FindEnvConfigValue(env_file, L"home", + home, Py_ARRAY_LENGTH(home))) { + if (safe_wcscpy(argv0_path, home, argv0_path_len) < 0) { + fclose(env_file); return PATHLEN_ERR(); } } @@ -1012,12 +1053,12 @@ calculate_zip_path(PyCalculatePath *calculate, const wchar_t *prefix, wchar_t *zip_path, size_t zip_path_len) { PyStatus status; - if (safe_wcscpy(zip_path, prefix, zip_path_len) < 0) { - return PATHLEN_ERR(); - } if (calculate->prefix_found > 0) { /* Use the reduced prefix returned by Py_GetPrefix() */ + if (safe_wcscpy(zip_path, prefix, zip_path_len) < 0) { + return PATHLEN_ERR(); + } reduce(zip_path); reduce(zip_path); } @@ -1200,6 +1241,8 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig) return status; } + /* If a pyvenv.cfg configure file is found, + argv0_path is overriden with its 'home' variable. */ status = calculate_read_pyenv(calculate, argv0_path, Py_ARRAY_LENGTH(argv0_path)); if (_PyStatus_EXCEPTION(status)) { diff --git a/PC/getpathp.c b/PC/getpathp.c index c4c0636ddde..8bac592aefd 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -757,34 +757,34 @@ static void calculate_pyvenv_file(PyCalculatePath *calculate, wchar_t *argv0_path, size_t argv0_path_len) { - wchar_t envbuffer[MAXPATHLEN+1]; + wchar_t filename[MAXPATHLEN+1]; const wchar_t *env_cfg = L"pyvenv.cfg"; - wcscpy_s(envbuffer, MAXPATHLEN+1, argv0_path); - join(envbuffer, env_cfg); + /* Filename: / "pyvenv.cfg" */ + wcscpy_s(filename, MAXPATHLEN+1, argv0_path); + join(filename, env_cfg); - FILE *env_file = _Py_wfopen(envbuffer, L"r"); + FILE *env_file = _Py_wfopen(filename, L"r"); if (env_file == NULL) { errno = 0; - reduce(envbuffer); - reduce(envbuffer); - join(envbuffer, env_cfg); + /* Filename: / "pyvenv.cfg" */ + reduce(filename); + reduce(filename); + join(filename, env_cfg); - env_file = _Py_wfopen(envbuffer, L"r"); + env_file = _Py_wfopen(filename, L"r"); if (env_file == NULL) { errno = 0; + return; } } - if (env_file == NULL) { - return; - } - /* Look for a 'home' variable and set argv0_path to it, if found */ - wchar_t tmpbuffer[MAXPATHLEN+1]; - if (_Py_FindEnvConfigValue(env_file, L"home", tmpbuffer, MAXPATHLEN)) { - wcscpy_s(argv0_path, argv0_path_len, tmpbuffer); + wchar_t home[MAXPATHLEN+1]; + if (_Py_FindEnvConfigValue(env_file, L"home", + home, Py_ARRAY_LENGTH(home))) { + wcscpy_s(argv0_path, argv0_path_len, home); } fclose(env_file); } @@ -1099,11 +1099,11 @@ calculate_free(PyCalculatePath *calculate) - __PYVENV_LAUNCHER__ environment variable - GetModuleFileNameW(NULL): fully qualified path of the executable file of the current process - - .pth configuration file + - ._pth configuration file - pyvenv.cfg configuration file - Registry key "Software\Python\PythonCore\X.Y\PythonPath" - of HKEY_LOCAL_MACHINE and HKEY_CURRENT_USER where X.Y is the Python - version (major.minor). + of HKEY_CURRENT_USER and HKEY_LOCAL_MACHINE where X.Y is the Python + version. Outputs, 'pathconfig' fields: diff --git a/Programs/_testembed.c b/Programs/_testembed.c index c3ccc0ec325..83c266b885a 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -20,11 +20,12 @@ * Executed via 'EmbeddingTests' in Lib/test/test_capi.py *********************************************************/ +/* Use path starting with "./" avoids a search along the PATH */ +#define PROGRAM_NAME L"./_testembed" + static void _testembed_Py_Initialize(void) { - /* HACK: the "./" at front avoids a search along the PATH in - Modules/getpath.c */ - Py_SetProgramName(L"./_testembed"); + Py_SetProgramName(PROGRAM_NAME); Py_Initialize(); } @@ -363,8 +364,7 @@ config_set_wide_string_list(PyConfig *config, PyWideStringList *list, static void config_set_program_name(PyConfig *config) { - /* Use path starting with "./" avoids a search along the PATH */ - const wchar_t *program_name = L"./_testembed"; + const wchar_t *program_name = PROGRAM_NAME; config_set_string(config, &config->program_name, program_name); } @@ -1263,7 +1263,7 @@ static int _audit_hook_run(const char *eventName, PyObject *args, void *userData static int test_audit_run_command(void) { AuditRunCommandTest test = {"cpython.run_command"}; - wchar_t *argv[] = {L"./_testembed", L"-c", L"pass"}; + wchar_t *argv[] = {PROGRAM_NAME, L"-c", L"pass"}; Py_IgnoreEnvironmentFlag = 0; PySys_AddAuditHook(_audit_hook_run, (void*)&test); @@ -1274,7 +1274,7 @@ static int test_audit_run_command(void) static int test_audit_run_file(void) { AuditRunCommandTest test = {"cpython.run_file"}; - wchar_t *argv[] = {L"./_testembed", L"filename.py"}; + wchar_t *argv[] = {PROGRAM_NAME, L"filename.py"}; Py_IgnoreEnvironmentFlag = 0; PySys_AddAuditHook(_audit_hook_run, (void*)&test); @@ -1312,21 +1312,21 @@ static int run_audit_run_test(int argc, wchar_t **argv, void *test) static int test_audit_run_interactivehook(void) { AuditRunCommandTest test = {"cpython.run_interactivehook", 10}; - wchar_t *argv[] = {L"./_testembed"}; + wchar_t *argv[] = {PROGRAM_NAME}; return run_audit_run_test(Py_ARRAY_LENGTH(argv), argv, &test); } static int test_audit_run_startup(void) { AuditRunCommandTest test = {"cpython.run_startup", 10}; - wchar_t *argv[] = {L"./_testembed"}; + wchar_t *argv[] = {PROGRAM_NAME}; return run_audit_run_test(Py_ARRAY_LENGTH(argv), argv, &test); } static int test_audit_run_stdin(void) { AuditRunCommandTest test = {"cpython.run_stdin"}; - wchar_t *argv[] = {L"./_testembed"}; + wchar_t *argv[] = {PROGRAM_NAME}; return run_audit_run_test(Py_ARRAY_LENGTH(argv), argv, &test); } @@ -1423,6 +1423,95 @@ fail: } +static int test_init_setpath(void) +{ + char *env = getenv("TESTPATH"); + if (!env) { + fprintf(stderr, "missing TESTPATH env var\n"); + return 1; + } + wchar_t *path = Py_DecodeLocale(env, NULL); + if (path == NULL) { + fprintf(stderr, "failed to decode TESTPATH\n"); + return 1; + } + Py_SetPath(path); + PyMem_RawFree(path); + putenv("TESTPATH="); + + Py_Initialize(); + dump_config(); + Py_Finalize(); + return 0; +} + + +static int test_init_setpath_config(void) +{ + PyStatus status; + PyPreConfig preconfig; + PyPreConfig_InitPythonConfig(&preconfig); + + /* Explicitly preinitializes with Python preconfiguration to avoid + Py_SetPath() implicit preinitialization with compat preconfiguration. */ + status = Py_PreInitialize(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + + char *env = getenv("TESTPATH"); + if (!env) { + fprintf(stderr, "missing TESTPATH env var\n"); + return 1; + } + wchar_t *path = Py_DecodeLocale(env, NULL); + if (path == NULL) { + fprintf(stderr, "failed to decode TESTPATH\n"); + return 1; + } + Py_SetPath(path); + PyMem_RawFree(path); + putenv("TESTPATH="); + + PyConfig config; + + status = PyConfig_InitPythonConfig(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + config_set_string(&config, &config.program_name, L"conf_program_name"); + config_set_string(&config, &config.executable, L"conf_executable"); + init_from_config_clear(&config); + + dump_config(); + Py_Finalize(); + return 0; +} + + +static int test_init_setpythonhome(void) +{ + char *env = getenv("TESTHOME"); + if (!env) { + fprintf(stderr, "missing TESTHOME env var\n"); + return 1; + } + wchar_t *home = Py_DecodeLocale(env, NULL); + if (home == NULL) { + fprintf(stderr, "failed to decode TESTHOME\n"); + return 1; + } + Py_SetPythonHome(home); + PyMem_RawFree(home); + putenv("TESTHOME="); + + Py_Initialize(); + dump_config(); + Py_Finalize(); + return 0; +} + + static void configure_init_main(PyConfig *config) { wchar_t* argv[] = { @@ -1559,7 +1648,11 @@ static struct TestCase TestCases[] = { {"test_init_run_main", test_init_run_main}, {"test_init_main", test_init_main}, {"test_init_sys_add", test_init_sys_add}, + {"test_init_setpath", test_init_setpath}, + {"test_init_setpath_config", test_init_setpath_config}, + {"test_init_setpythonhome", test_init_setpythonhome}, {"test_run_main", test_run_main}, + {"test_open_code_hook", test_open_code_hook}, {"test_audit", test_audit}, {"test_audit_subinterpreter", test_audit_subinterpreter}, diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 09533100b46..d5b8b1acff8 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -23,6 +23,7 @@ wchar_t *_Py_dll_path = NULL; static int copy_wstr(wchar_t **dst, const wchar_t *src) { + assert(*dst == NULL); if (src != NULL) { *dst = _PyMem_RawWcsdup(src); if (*dst == NULL) { @@ -57,7 +58,10 @@ pathconfig_clear(_PyPathConfig *config) CLEAR(config->module_search_path); CLEAR(config->program_name); CLEAR(config->home); +#ifdef MS_WINDOWS CLEAR(config->base_executable); +#endif + #undef CLEAR PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); @@ -82,9 +86,11 @@ pathconfig_copy(_PyPathConfig *config, const _PyPathConfig *config2) COPY_ATTR(module_search_path); COPY_ATTR(program_name); COPY_ATTR(home); +#ifdef MS_WINDOWS config->isolated = config2->isolated; config->site_import = config2->site_import; COPY_ATTR(base_executable); +#endif #undef COPY_ATTR @@ -127,7 +133,7 @@ _PyWideStringList_Join(const PyWideStringList *list, wchar_t sep) for (Py_ssize_t i=0; i < list->length; i++) { wchar_t *path = list->items[i]; if (i != 0) { - *str++ = SEP; + *str++ = sep; } len = wcslen(path); memcpy(str, path, len * sizeof(wchar_t)); @@ -139,11 +145,11 @@ _PyWideStringList_Join(const PyWideStringList *list, wchar_t sep) } -/* Initialize _Py_dll_path on Windows. Do nothing on other platforms. */ -PyStatus -_PyPathConfig_Init(void) -{ #ifdef MS_WINDOWS +/* Initialize _Py_dll_path on Windows. Do nothing on other platforms. */ +static PyStatus +_PyPathConfig_InitDLLPath(void) +{ if (_Py_dll_path == NULL) { /* Already set: nothing to do */ return _PyStatus_OK(); @@ -159,9 +165,9 @@ _PyPathConfig_Init(void) if (_Py_dll_path == NULL) { return _PyStatus_NO_MEMORY(); } -#endif return _PyStatus_OK(); } +#endif static PyStatus @@ -172,6 +178,7 @@ pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config) _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); if (config->module_search_paths_set) { + PyMem_RawFree(pathconfig->module_search_path); pathconfig->module_search_path = _PyWideStringList_Join(&config->module_search_paths, DELIM); if (pathconfig->module_search_path == NULL) { goto no_memory; @@ -180,17 +187,21 @@ pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config) #define COPY_CONFIG(PATH_ATTR, CONFIG_ATTR) \ if (config->CONFIG_ATTR) { \ + PyMem_RawFree(pathconfig->PATH_ATTR); \ + pathconfig->PATH_ATTR = NULL; \ if (copy_wstr(&pathconfig->PATH_ATTR, config->CONFIG_ATTR) < 0) { \ goto no_memory; \ } \ } - COPY_CONFIG(base_executable, base_executable); COPY_CONFIG(program_full_path, executable); COPY_CONFIG(prefix, prefix); COPY_CONFIG(exec_prefix, exec_prefix); COPY_CONFIG(program_name, program_name); COPY_CONFIG(home, home); +#ifdef MS_WINDOWS + COPY_CONFIG(base_executable, base_executable); +#endif #undef COPY_CONFIG @@ -206,6 +217,20 @@ done: } +PyStatus +_PyConfig_WritePathConfig(const PyConfig *config) +{ +#ifdef MS_WINDOWS + PyStatus status = _PyPathConfig_InitDLLPath(); + if (_PyStatus_EXCEPTION(status)) { + return status; + } +#endif + + return pathconfig_set_from_config(&_Py_path_config, config); +} + + static PyStatus config_init_module_search_paths(PyConfig *config, _PyPathConfig *pathconfig) { @@ -326,18 +351,32 @@ config_calculate_pathconfig(PyConfig *config) } \ } +#ifdef MS_WINDOWS + if (config->executable != NULL && config->base_executable == NULL) { + /* If executable is set explicitly in the configuration, + ignore calculated base_executable: _PyConfig_InitPathConfig() + will copy executable to base_executable */ + } + else { + COPY_ATTR(base_executable, base_executable); + } +#endif + COPY_ATTR(program_full_path, executable); COPY_ATTR(prefix, prefix); COPY_ATTR(exec_prefix, exec_prefix); - COPY_ATTR(base_executable, base_executable); + #undef COPY_ATTR +#ifdef MS_WINDOWS + /* If a ._pth file is found: isolated and site_import are overriden */ if (pathconfig.isolated != -1) { config->isolated = pathconfig.isolated; } if (pathconfig.site_import != -1) { config->site_import = pathconfig.site_import; } +#endif status = _PyStatus_OK(); goto done; @@ -356,9 +395,9 @@ _PyConfig_InitPathConfig(PyConfig *config) { /* Do we need to calculate the path? */ if (!config->module_search_paths_set - || (config->executable == NULL) - || (config->prefix == NULL) - || (config->exec_prefix == NULL)) + || config->executable == NULL + || config->prefix == NULL + || config->exec_prefix == NULL) { PyStatus status = config_calculate_pathconfig(config); if (_PyStatus_EXCEPTION(status)) { @@ -416,11 +455,12 @@ pathconfig_global_init(void) { PyStatus status; - /* Initialize _Py_dll_path if needed */ - status = _PyPathConfig_Init(); +#ifdef MS_WINDOWS + status = _PyPathConfig_InitDLLPath(); if (_PyStatus_EXCEPTION(status)) { Py_ExitStatusException(status); } +#endif if (_Py_path_config.module_search_path == NULL) { status = pathconfig_global_read(&_Py_path_config); @@ -438,7 +478,9 @@ pathconfig_global_init(void) assert(_Py_path_config.module_search_path != NULL); assert(_Py_path_config.program_name != NULL); /* home can be NULL */ +#ifdef MS_WINDOWS assert(_Py_path_config.base_executable != NULL); +#endif } @@ -455,16 +497,15 @@ Py_SetPath(const wchar_t *path) PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - /* Getting the program name calls pathconfig_global_init() */ - wchar_t *program_name = _PyMem_RawWcsdup(Py_GetProgramName()); + /* Getting the program full path calls pathconfig_global_init() */ + wchar_t *program_full_path = _PyMem_RawWcsdup(Py_GetProgramFullPath()); PyMem_RawFree(_Py_path_config.program_full_path); PyMem_RawFree(_Py_path_config.prefix); PyMem_RawFree(_Py_path_config.exec_prefix); PyMem_RawFree(_Py_path_config.module_search_path); - /* Copy program_name to program_full_path */ - _Py_path_config.program_full_path = program_name; + _Py_path_config.program_full_path = program_full_path; _Py_path_config.prefix = _PyMem_RawWcsdup(L""); _Py_path_config.exec_prefix = _PyMem_RawWcsdup(L""); _Py_path_config.module_search_path = _PyMem_RawWcsdup(path); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 4d7873fd522..42a062e1086 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -472,7 +472,7 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime, config = &interp->config; if (config->_install_importlib) { - status = _PyPathConfig_Init(); + status = _PyConfig_WritePathConfig(config); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -641,7 +641,7 @@ pycore_init_import_warnings(PyInterpreterState *interp, PyObject *sysmod) } if (config->_install_importlib) { - status = _PyPathConfig_Init(); + status = _PyConfig_WritePathConfig(config); if (_PyStatus_EXCEPTION(status)) { return status; }