From 8f023a2f664f902a3d0b7a6f64d63afc0d1c15ae Mon Sep 17 00:00:00 2001 From: Sandro Mani Date: Mon, 8 Jun 2020 17:28:11 +0200 Subject: [PATCH] bpo-40854: Allow overriding sys.platlibdir via PYTHONPLATLIBDIR env-var (GH-20605) --- Doc/c-api/init_config.rst | 9 +++++++ Doc/using/cmdline.rst | 8 ++++++ Include/cpython/initconfig.h | 1 + Lib/test/test_embed.py | 22 +++++++++++----- Makefile.pre.in | 5 ++++ .../2020-06-03-13-53-24.bpo-40854.O6vfQU.rst | 1 + Misc/python.man | 2 ++ Modules/getpath.c | 16 +++++------- Programs/_testembed.c | 8 ++++++ Python/initconfig.c | 26 +++++++++++++++++++ Python/sysmodule.c | 8 +----- 11 files changed, 82 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-06-03-13-53-24.bpo-40854.O6vfQU.rst diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index fc82c3eb590..7b8e894fe22 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -436,6 +436,14 @@ PyConfig :data:`sys.base_prefix`. + .. c:member:: wchar_t* platlibdir + + :data:`sys.platlibdir`: platform library directory name, set at configure time + by ``--with-platlibdir``, overrideable by the ``PYTHONPLATLIBDIR`` + environment variable. + + .. versionadded:: 3.10 + .. c:member:: int buffered_stdio If equals to 0, enable unbuffered mode, making the stdout and stderr @@ -884,6 +892,7 @@ Path Configuration * Path configuration inputs: * :c:member:`PyConfig.home` + * :c:member:`PyConfig.platlibdir` * :c:member:`PyConfig.pathconfig_warnings` * :c:member:`PyConfig.program_name` * :c:member:`PyConfig.pythonpath_env` diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index b0911956a9e..3e0797279d6 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -538,6 +538,14 @@ conflict. within a Python program as the variable :data:`sys.path`. +.. envvar:: PYTHONPLATLIBDIR + + If this is set to a non-empty string, it overrides the :data:`sys.platlibdir` + value. + + .. versionadded:: 3.10 + + .. envvar:: PYTHONSTARTUP If this is the name of a readable file, the Python commands in that file are diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index e9c2e6bec38..563c2bacfa4 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -385,6 +385,7 @@ typedef struct { wchar_t *base_prefix; /* sys.base_prefix */ wchar_t *exec_prefix; /* sys.exec_prefix */ wchar_t *base_exec_prefix; /* sys.base_exec_prefix */ + wchar_t *platlibdir; /* sys.platlibdir */ /* --- Parameter only used by Py_Main() ---------- */ diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 3d60b2f330c..f1371db8669 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -380,6 +380,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'exec_prefix': GET_DEFAULT_CONFIG, 'base_exec_prefix': GET_DEFAULT_CONFIG, 'module_search_paths': GET_DEFAULT_CONFIG, + 'platlibdir': sys.platlibdir, 'site_import': 1, 'bytes_warning': 0, @@ -585,13 +586,14 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): if value is self.GET_DEFAULT_CONFIG: expected[key] = config[key] - pythonpath_env = expected['pythonpath_env'] - if pythonpath_env is not None: - paths = pythonpath_env.split(os.path.pathsep) - expected['module_search_paths'] = [*paths, *expected['module_search_paths']] - if modify_path_cb is not None: - expected['module_search_paths'] = expected['module_search_paths'].copy() - modify_path_cb(expected['module_search_paths']) + if expected['module_search_paths'] is not self.IGNORE_CONFIG: + pythonpath_env = expected['pythonpath_env'] + if pythonpath_env is not None: + paths = pythonpath_env.split(os.path.pathsep) + expected['module_search_paths'] = [*paths, *expected['module_search_paths']] + if modify_path_cb is not None: + expected['module_search_paths'] = expected['module_search_paths'].copy() + modify_path_cb(expected['module_search_paths']) for key in self.COPY_PRE_CONFIG: if key not in expected_preconfig: @@ -764,6 +766,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'buffered_stdio': 0, 'user_site_directory': 0, 'faulthandler': 1, + 'platlibdir': 'my_platlibdir', + 'module_search_paths': self.IGNORE_CONFIG, 'check_hash_pycs_mode': 'always', 'pathconfig_warnings': 0, @@ -795,6 +799,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'user_site_directory': 0, 'faulthandler': 1, 'warnoptions': ['EnvVar'], + 'platlibdir': 'env_platlibdir', + 'module_search_paths': self.IGNORE_CONFIG, '_use_peg_parser': 0, } self.check_all_configs("test_init_compat_env", config, preconfig, @@ -823,6 +829,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'user_site_directory': 0, 'faulthandler': 1, 'warnoptions': ['EnvVar'], + 'platlibdir': 'env_platlibdir', + 'module_search_paths': self.IGNORE_CONFIG, '_use_peg_parser': 0, } self.check_all_configs("test_init_python_env", config, preconfig, diff --git a/Makefile.pre.in b/Makefile.pre.in index b115e7fc01f..9cb7a23eea5 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -811,6 +811,11 @@ Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(srcdir)/Include/pydt $(MULTIARCH_CPPFLAGS) \ -o $@ $(srcdir)/Python/sysmodule.c +Python/initconfig.o: $(srcdir)/Python/initconfig.c + $(CC) -c $(PY_CORE_CFLAGS) \ + -DPLATLIBDIR='"$(PLATLIBDIR)"' \ + -o $@ $(srcdir)/Python/initconfig.c + $(IO_OBJS): $(IO_H) .PHONY: regen-grammar diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-06-03-13-53-24.bpo-40854.O6vfQU.rst b/Misc/NEWS.d/next/Core and Builtins/2020-06-03-13-53-24.bpo-40854.O6vfQU.rst new file mode 100644 index 00000000000..6ef4ed5af73 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-06-03-13-53-24.bpo-40854.O6vfQU.rst @@ -0,0 +1 @@ +Allow overriding :data:`sys.platlibdir` via a new :envvar:`PYTHONPLATLIBDIR` environment variable. diff --git a/Misc/python.man b/Misc/python.man index 89a15a5e7b2..74b2d72939e 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -413,6 +413,8 @@ inserted in the path in front of $PYTHONPATH. The search path can be manipulated from within a Python program as the variable .IR sys.path . +.IP PYTHONPLATLIBDIR +Override sys.platlibdir. .IP PYTHONSTARTUP If this is the name of a readable file, the Python commands in that file are executed before the first prompt is displayed in interactive diff --git a/Modules/getpath.c b/Modules/getpath.c index d9829f8ad3d..469c9ca0106 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -105,8 +105,8 @@ extern "C" { #if (!defined(PREFIX) || !defined(EXEC_PREFIX) \ - || !defined(VERSION) || !defined(VPATH) || !defined(PLATLIBDIR)) -#error "PREFIX, EXEC_PREFIX, VERSION, VPATH and PLATLIBDIR macros must be defined" + || !defined(VERSION) || !defined(VPATH)) +#error "PREFIX, EXEC_PREFIX, VERSION and VPATH macros must be defined" #endif #ifndef LANDMARK @@ -128,7 +128,6 @@ typedef struct { wchar_t *pythonpath_macro; /* PYTHONPATH macro */ wchar_t *prefix_macro; /* PREFIX macro */ wchar_t *exec_prefix_macro; /* EXEC_PREFIX macro */ - wchar_t *platlibdir_macro; /* PLATLIBDIR macro */ wchar_t *vpath_macro; /* VPATH macro */ wchar_t *lib_python; /* "lib/pythonX.Y" */ @@ -138,6 +137,7 @@ typedef struct { int warnings; const wchar_t *pythonpath_env; + const wchar_t *platlibdir; wchar_t *argv0_path; wchar_t *zip_path; @@ -811,7 +811,7 @@ calculate_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig) } /* / "lib-dynload" */ - wchar_t *lib_dynload = joinpath2(calculate->platlibdir_macro, + wchar_t *lib_dynload = joinpath2(calculate->platlibdir, L"lib-dynload"); if (lib_dynload == NULL) { return _PyStatus_NO_MEMORY(); @@ -1297,7 +1297,7 @@ calculate_zip_path(PyCalculatePath *calculate) PyStatus res; /* Path: / "pythonXY.zip" */ - wchar_t *path = joinpath2(calculate->platlibdir_macro, L"python" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) L".zip"); + wchar_t *path = joinpath2(calculate->platlibdir, L"python" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) L".zip"); if (path == NULL) { return _PyStatus_NO_MEMORY(); } @@ -1451,10 +1451,6 @@ calculate_init(PyCalculatePath *calculate, const PyConfig *config) if (!calculate->vpath_macro) { return DECODE_LOCALE_ERR("VPATH macro", len); } - calculate->platlibdir_macro = Py_DecodeLocale(PLATLIBDIR, &len); - if (!calculate->platlibdir_macro) { - return DECODE_LOCALE_ERR("PLATLIBDIR macro", len); - } calculate->lib_python = Py_DecodeLocale(PLATLIBDIR "/python" VERSION, &len); if (!calculate->lib_python) { @@ -1463,6 +1459,7 @@ calculate_init(PyCalculatePath *calculate, const PyConfig *config) calculate->warnings = config->pathconfig_warnings; calculate->pythonpath_env = config->pythonpath_env; + calculate->platlibdir = config->platlibdir; return _PyStatus_OK(); } @@ -1475,7 +1472,6 @@ calculate_free(PyCalculatePath *calculate) PyMem_RawFree(calculate->prefix_macro); PyMem_RawFree(calculate->exec_prefix_macro); PyMem_RawFree(calculate->vpath_macro); - PyMem_RawFree(calculate->platlibdir_macro); PyMem_RawFree(calculate->lib_python); PyMem_RawFree(calculate->path_env); PyMem_RawFree(calculate->zip_path); diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 5c83678f650..11524dfbc0d 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -548,6 +548,13 @@ static int test_init_from_config(void) /* FIXME: test home */ /* FIXME: test path config: module_search_path .. dll_path */ + putenv("PYTHONPLATLIBDIR=env_platlibdir"); + status = PyConfig_SetBytesString(&config, &config.platlibdir, "my_platlibdir"); + if (PyStatus_Exception(status)) { + PyConfig_Clear(&config); + Py_ExitStatusException(status); + } + putenv("PYTHONVERBOSE=0"); Py_VerboseFlag = 0; config.verbose = 1; @@ -668,6 +675,7 @@ static void set_most_env_vars(void) putenv("PYTHONFAULTHANDLER=1"); putenv("PYTHONIOENCODING=iso8859-1:replace"); putenv("PYTHONOLDPARSER=1"); + putenv("PYTHONPLATLIBDIR=env_platlibdir"); } diff --git a/Python/initconfig.c b/Python/initconfig.c index 185935c05fb..834b8ed9430 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -24,6 +24,10 @@ # endif #endif +#ifndef PLATLIBDIR +# error "PLATLIBDIR macro must be defined" +#endif + /* --- Command line options --------------------------------------- */ @@ -110,6 +114,7 @@ PYTHONPATH : '%lc'-separated list of directories prefixed to the\n\ static const char usage_5[] = "PYTHONHOME : alternate directory (or %lc).\n" " The default module search path uses %s.\n" +"PYTHONPLATLIBDIR : override sys.platlibdir.\n" "PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" "PYTHONUTF8: if set to 1, enable the UTF-8 mode.\n" "PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n" @@ -588,6 +593,7 @@ PyConfig_Clear(PyConfig *config) CLEAR(config->base_prefix); CLEAR(config->exec_prefix); CLEAR(config->base_exec_prefix); + CLEAR(config->platlibdir); CLEAR(config->filesystem_encoding); CLEAR(config->filesystem_errors); @@ -824,6 +830,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) COPY_WSTR_ATTR(base_prefix); COPY_WSTR_ATTR(exec_prefix); COPY_WSTR_ATTR(base_exec_prefix); + COPY_WSTR_ATTR(platlibdir); COPY_ATTR(site_import); COPY_ATTR(bytes_warning); @@ -926,6 +933,7 @@ config_as_dict(const PyConfig *config) SET_ITEM_WSTR(base_prefix); SET_ITEM_WSTR(exec_prefix); SET_ITEM_WSTR(base_exec_prefix); + SET_ITEM_WSTR(platlibdir); SET_ITEM_INT(site_import); SET_ITEM_INT(bytes_warning); SET_ITEM_INT(inspect); @@ -1336,6 +1344,14 @@ config_read_env_vars(PyConfig *config) } } + if(config->platlibdir == NULL) { + status = CONFIG_GET_ENV_DUP(config, &config->platlibdir, + L"PYTHONPLATLIBDIR", "PYTHONPLATLIBDIR"); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } + if (config->use_hash_seed < 0) { status = config_init_hash_seed(config); if (_PyStatus_EXCEPTION(status)) { @@ -1731,6 +1747,14 @@ config_read(PyConfig *config) } } + if(config->platlibdir == NULL) { + status = CONFIG_SET_BYTES_STR(config, &config->platlibdir, PLATLIBDIR, + "PLATLIBDIR macro"); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } + if (config->_install_importlib) { status = _PyConfig_InitPathConfig(config); if (_PyStatus_EXCEPTION(status)) { @@ -2554,6 +2578,7 @@ PyConfig_Read(PyConfig *config) assert(config->exec_prefix != NULL); assert(config->base_exec_prefix != NULL); } + assert(config->platlibdir != NULL); assert(config->filesystem_encoding != NULL); assert(config->filesystem_errors != NULL); assert(config->stdio_encoding != NULL); @@ -2704,6 +2729,7 @@ _Py_DumpPathConfig(PyThreadState *tstate) DUMP_SYS(_base_executable); DUMP_SYS(base_prefix); DUMP_SYS(base_exec_prefix); + DUMP_SYS(platlibdir); DUMP_SYS(executable); DUMP_SYS(prefix); DUMP_SYS(exec_prefix); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index e3fe1436145..3e4115fe8e1 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2922,13 +2922,7 @@ _PySys_InitMain(PyThreadState *tstate) SET_SYS_FROM_WSTR("base_prefix", config->base_prefix); SET_SYS_FROM_WSTR("exec_prefix", config->exec_prefix); SET_SYS_FROM_WSTR("base_exec_prefix", config->base_exec_prefix); - { - PyObject *str = PyUnicode_FromString(PLATLIBDIR); - if (str == NULL) { - return -1; - } - SET_SYS_FROM_STRING("platlibdir", str); - } + SET_SYS_FROM_WSTR("platlibdir", config->platlibdir); if (config->pycache_prefix != NULL) { SET_SYS_FROM_WSTR("pycache_prefix", config->pycache_prefix);