Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

bpo-45020: Default to using frozen modules unless running from source tree. (gh-28940) 

The default was "off".  Switching it to "on" means users get the benefit of frozen stdlib modules without having to do anything.  There's a special-case for running-in-source-tree, so contributors don't get surprised when their stdlib changes don't get used.

https://bugs.python.org/issue45020
This commit is contained in:
Eric Snow 2021-10-16 13:16:08 -06:00 committed by GitHub
parent fe0d9e22a5
commit b9cdd0fb9c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 91 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Doc/using/cmdline.rst
View File

@ -483,7 +483,8 @@ Miscellaneous options
* ``-X frozen_modules`` determines whether or not frozen modules are
ignored by the import machinery. A value of "on" means they get
imported and "off" means they are ignored. The default is "on"
for non-debug builds (the normal case) and "off" for debug builds.
if this is an installed Python (the normal case). If it's under
development (running from the source tree) then the default is "off".
Note that the "importlib_bootstrap" and "importlib_bootstrap_external"
frozen modules are always used, even if this flag is set to "off".

1
Include/internal/pycore_fileutils.h
View File

@ -79,6 +79,7 @@ extern wchar_t * _Py_join_relfile(const wchar_t *dirname,
extern int _Py_add_relfile(wchar_t *dirname,
const wchar_t *relfile,
size_t bufsize);
extern size_t _Py_find_basename(const wchar_t *filename);
// Macros to protect CRT calls against instant termination when passed an
// invalid parameter (bpo-23524). IPH stands for Invalid Parameter Handler.

4
Lib/site.py
View File

@ -418,8 +418,10 @@ def setcopyright():
files, dirs = [], []
# Not all modules are required to have a __file__ attribute. See
# PEP 420 for more details.
if hasattr(os, '__file__'):
here = getattr(sys, '_stdlib_dir', None)
if not here and hasattr(os, '__file__'):
here = os.path.dirname(os.__file__)
if here:
files.extend(["LICENSE.txt", "LICENSE"])
dirs.extend([os.path.join(here, os.pardir), here, os.curdir])
builtins.license = _sitebuiltins._Printer(

14
Lib/test/test_embed.py
View File

@ -53,12 +53,13 @@ def remove_python_envvars():
class EmbeddingTestsMixin:
def setUp(self):
exename = "_testembed"
builddir = os.path.dirname(sys.executable)
if MS_WINDOWS:
ext = ("_d" if debug_build(sys.executable) else "") + ".exe"
exename += ext
exepath = os.path.dirname(sys.executable)
exepath = builddir
else:
exepath = os.path.join(support.REPO_ROOT, "Programs")
exepath = os.path.join(builddir, 'Programs')
self.test_exe = exe = os.path.join(exepath, exename)
if not os.path.exists(exe):
self.skipTest("%r doesn't exist" % exe)
@ -434,7 +435,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'pathconfig_warnings': 1,
'_init_main': 1,
'_isolated_interpreter': 0,
'use_frozen_modules': 0,
'use_frozen_modules': 1,
}
if MS_WINDOWS:
CONFIG_COMPAT.update({
@ -1146,6 +1147,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
# The current getpath.c doesn't determine the stdlib dir
# in this case.
'stdlib_dir': '',
'use_frozen_modules': -1,
}
self.default_program_name(config)
env = {'TESTPATH': os.path.pathsep.join(paths)}
@ -1169,6 +1171,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
# The current getpath.c doesn't determine the stdlib dir
# in this case.
'stdlib_dir': '',
'use_frozen_modules': -1,
# overridden by PyConfig
'program_name': 'conf_program_name',
'base_executable': 'conf_executable',
@ -1265,6 +1268,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'stdlib_dir': stdlib,
}
self.default_program_name(config)
if not config['executable']:
config['use_frozen_modules'] = -1
env = {'TESTHOME': home, 'PYTHONPATH': paths_str}
self.check_all_configs("test_init_setpythonhome", config,
api=API_COMPAT, env=env)
@ -1303,6 +1308,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
# The current getpath.c doesn't determine the stdlib dir
# in this case.
'stdlib_dir': None,
'use_frozen_modules': -1,
}
env = self.copy_paths_by_env(config)
self.check_all_configs("test_init_compat_config", config,
@ -1361,6 +1367,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
config['base_prefix'] = pyvenv_home
config['prefix'] = pyvenv_home
config['stdlib_dir'] = os.path.join(pyvenv_home, 'lib')
config['use_frozen_modules'] = 1
ver = sys.version_info
dll = f'python{ver.major}'
@ -1373,6 +1380,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
# The current getpath.c doesn't determine the stdlib dir
# in this case.
config['stdlib_dir'] = None
config['use_frozen_modules'] = -1
env = self.copy_paths_by_env(config)
self.check_all_configs("test_init_compat_config", config,

2
Misc/NEWS.d/3.11.0a1.rst
View File

@ -262,7 +262,7 @@ Compiler now removes trailing unused constants from co_consts.
Add a new command line option, "-X frozen_modules=[on|off]" to opt out of
(or into) using optional frozen modules. This defaults to "on" (or "off" if
it's a debug build).
it's running out of the source tree).
..

12
Python/fileutils.c
View File

@ -2169,6 +2169,18 @@ _Py_add_relfile(wchar_t *dirname, const wchar_t *relfile, size_t bufsize)
}
size_t
_Py_find_basename(const wchar_t *filename)
{
for (size_t i = wcslen(filename); i > 0; --i) {
if (filename[i] == SEP) {
return i + 1;
}
}
return 0;
}
/* Get the current directory. buflen is the buffer size in wide characters
including the null character. Decode the path from the locale encoding.

80
Python/initconfig.c
View File

@ -739,6 +739,7 @@ _PyConfig_InitCompatConfig(PyConfig *config)
#ifdef MS_WINDOWS
config->legacy_windows_stdio = -1;
#endif
config->use_frozen_modules = -1;
}
@ -2090,6 +2091,44 @@ config_init_fs_encoding(PyConfig *config, const PyPreConfig *preconfig)
}
/* Determine if the current build is a "development" build (e.g. running
out of the source tree) or not.
A return value of -1 indicates that we do not know.
*/
static int
is_dev_env(PyConfig *config)
{
// This should only ever get called early in runtime initialization,
// before the global path config is written. Otherwise we would
// use Py_GetProgramFullPath() and _Py_GetStdlibDir().
assert(config != NULL);
const wchar_t *executable = config->executable;
const wchar_t *stdlib = config->stdlib_dir;
if (executable == NULL || *executable == L'\0' ||
stdlib == NULL || *stdlib == L'\0') {
// _PyPathConfig_Calculate() hasn't run yet.
return -1;
}
size_t len = _Py_find_basename(executable);
if (wcscmp(executable + len, L"python") != 0 &&
wcscmp(executable + len, L"python.exe") != 0) {
return 0;
}
/* If dirname() is the same for both then it is a dev build. */
if (len != _Py_find_basename(stdlib)) {
return 0;
}
// We do not bother normalizing the two filenames first since
// for config_init_import() is does the right thing as-is.
if (wcsncmp(stdlib, executable, len) != 0) {
return 0;
}
return 1;
}
static PyStatus
config_init_import(PyConfig *config, int compute_path_config)
{
@ -2101,25 +2140,28 @@ config_init_import(PyConfig *config, int compute_path_config)
}
/* -X frozen_modules=[on|off] */
const wchar_t *value = config_get_xoption_value(config, L"frozen_modules");
if (value == NULL) {
// For now we always default to "off".
// In the near future we will be factoring in PGO and in-development.
config->use_frozen_modules = 0;
}
else if (wcscmp(value, L"on") == 0) {
config->use_frozen_modules = 1;
}
else if (wcscmp(value, L"off") == 0) {
config->use_frozen_modules = 0;
}
else if (wcslen(value) == 0) {
// "-X frozen_modules" and "-X frozen_modules=" both imply "on".
config->use_frozen_modules = 1;
}
else {
return PyStatus_Error("bad value for option -X frozen_modules "
"(expected \"on\" or \"off\")");
if (config->use_frozen_modules < 0) {
const wchar_t *value = config_get_xoption_value(config, L"frozen_modules");
if (value == NULL) {
int isdev = is_dev_env(config);
if (isdev >= 0) {
config->use_frozen_modules = !isdev;
}
}
else if (wcscmp(value, L"on") == 0) {
config->use_frozen_modules = 1;
}
else if (wcscmp(value, L"off") == 0) {
config->use_frozen_modules = 0;
}
else if (wcslen(value) == 0) {
// "-X frozen_modules" and "-X frozen_modules=" both imply "on".
config->use_frozen_modules = 1;
}
else {
return PyStatus_Error("bad value for option -X frozen_modules "
"(expected \"on\" or \"off\")");
}
}
return _PyStatus_OK();