traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[3.7] bpo-34247: Fix Python 3.7 initialization (#8659) 

* -X dev: it is now possible to override the memory allocator using
  PYTHONMALLOC even if the developer mode is enabled.
* Add _Py_InitializeFromConfig()
* Add _Py_Initialize_ReadEnvVars() to set global configuration
  variables from environment variables
* Fix the code to initialize Python: Py_Initialize() now also reads
  environment variables
* _Py_InitializeCore() can now be called twice: the second call
  only replaces the configuration.
* Write unit tests on Py_Initialize() and the different ways to
  configure Python
* The isolated mode now always sets Py_IgnoreEnvironmentFlag and
  Py_NoUserSiteDirectory to 1.
* pymain_read_conf() now saves/restores the configuration
  if the encoding changed
This commit is contained in:
Victor Stinner 2018-08-05 12:31:59 +02:00 committed by GitHub
parent e65ec491fb
commit 0c90d6f759
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 781 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Include/pylifecycle.h
View File

@ -51,14 +51,24 @@ PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding,
const char *errors);
/* PEP 432 Multi-phase initialization API (Private while provisional!) */
PyAPI_FUNC(_PyInitError) _Py_InitializeCore(const _PyCoreConfig *);
PyAPI_FUNC(_PyInitError) _Py_InitializeCore(
PyInterpreterState **interp_p,
const _PyCoreConfig *config);
PyAPI_FUNC(int) _Py_IsCoreInitialized(void);
PyAPI_FUNC(_PyInitError) _Py_InitializeFromConfig(
const _PyCoreConfig *config);
#ifdef Py_BUILD_CORE
PyAPI_FUNC(void) _Py_Initialize_ReadEnvVarsNoAlloc(void);
#endif
PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *);
PyAPI_FUNC(void) _PyCoreConfig_Clear(_PyCoreConfig *);
PyAPI_FUNC(int) _PyCoreConfig_Copy(
_PyCoreConfig *config,
const _PyCoreConfig *config2);
PyAPI_FUNC(void) _PyCoreConfig_SetGlobalConfig(
const _PyCoreConfig *config);
PyAPI_FUNC(_PyInitError) _PyMainInterpreterConfig_Read(
_PyMainInterpreterConfig *config,
@ -68,14 +78,16 @@ PyAPI_FUNC(int) _PyMainInterpreterConfig_Copy(
_PyMainInterpreterConfig *config,
const _PyMainInterpreterConfig *config2);
PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *);
#endif
PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter(
PyInterpreterState *interp,
const _PyMainInterpreterConfig *config);
#endif /* !defined(Py_LIMITED_API) */
/* Initialization and finalization */
PyAPI_FUNC(void) Py_Initialize(void);
PyAPI_FUNC(void) Py_InitializeEx(int);
#ifndef Py_LIMITED_API
PyAPI_FUNC(_PyInitError) _Py_InitializeEx_Private(int, int);
PyAPI_FUNC(void) _Py_FatalInitError(_PyInitError err) _Py_NO_RETURN;
#endif
PyAPI_FUNC(void) Py_Finalize(void);

3
Include/pystate.h
View File

@ -79,8 +79,11 @@ typedef struct {
#define _PyCoreConfig_INIT \
(_PyCoreConfig){ \
.install_signal_handlers = -1, \
.ignore_environment = -1, \
.use_hash_seed = -1, \
.coerce_c_locale = -1, \
.faulthandler = -1, \
.tracemalloc = -1, \
.utf8_mode = -1, \
.argc = -1, \
.nmodule_search_path = -1}

4
Lib/test/test_cmd_line.py
View File

@ -523,9 +523,7 @@ class CmdLineTest(unittest.TestCase):
env = dict(os.environ)
env.pop('PYTHONWARNINGS', None)
env.pop('PYTHONDEVMODE', None)
# Force malloc() to disable the debug hooks which are enabled
# by default for Python compiled in debug mode
env['PYTHONMALLOC'] = 'malloc'
env.pop('PYTHONMALLOC', None)
if xdev:
args = (sys.executable, '-X', 'dev', *args)

148
Lib/test/test_embed.py
View File

@ -9,7 +9,7 @@ import subprocess
import sys
class EmbeddingTests(unittest.TestCase):
class EmbeddingTestsMixin:
def setUp(self):
here = os.path.abspath(__file__)
basepath = os.path.dirname(os.path.dirname(os.path.dirname(here)))
@ -110,6 +110,8 @@ class EmbeddingTests(unittest.TestCase):
yield current_run
current_run = []
class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase):
def test_subinterps_main(self):
for run in self.run_repeated_init_and_subinterpreters():
main = run[0]
@ -247,5 +249,149 @@ class EmbeddingTests(unittest.TestCase):
self.assertEqual(err, '')
class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
maxDiff = 4096
DEFAULT_CONFIG = {
'install_signal_handlers': 1,
'Py_IgnoreEnvironmentFlag': 0,
'use_hash_seed': 0,
'hash_seed': 0,
'allocator': '(null)',
'dev_mode': 0,
'faulthandler': 0,
'tracemalloc': 0,
'import_time': 0,
'show_ref_count': 0,
'show_alloc_count': 0,
'dump_refs': 0,
'malloc_stats': 0,
'utf8_mode': 0,
'coerce_c_locale': 0,
'coerce_c_locale_warn': 0,
'program_name': './_testembed',
'argc': 0,
'argv': '[]',
'program': '(null)',
'Py_IsolatedFlag': 0,
'Py_NoSiteFlag': 0,
'Py_BytesWarningFlag': 0,
'Py_InspectFlag': 0,
'Py_InteractiveFlag': 0,
'Py_OptimizeFlag': 0,
'Py_DebugFlag': 0,
'Py_DontWriteBytecodeFlag': 0,
'Py_VerboseFlag': 0,
'Py_QuietFlag': 0,
'Py_NoUserSiteDirectory': 0,
'Py_UnbufferedStdioFlag': 0,
'_disable_importlib': 0,
'Py_FrozenFlag': 0,
}
def check_config(self, testname, expected):
env = dict(os.environ)
for key in list(env):
if key.startswith('PYTHON'):
del env[key]
# Disable C locale coercion and UTF-8 mode to not depend
# on the current locale
env['PYTHONCOERCECLOCALE'] = '0'
env['PYTHONUTF8'] = '0'
out, err = self.run_embedded_interpreter(testname, env=env)
# Ignore err
expected = dict(self.DEFAULT_CONFIG, **expected)
for key, value in expected.items():
expected[key] = str(value)
config = {}
for line in out.splitlines():
key, value = line.split(' = ', 1)
config[key] = value
self.assertEqual(config, expected)
def test_init_default_config(self):
self.check_config("init_default_config", {})
def test_init_global_config(self):
config = {
'program_name': './globalvar',
'Py_NoSiteFlag': 1,
'Py_BytesWarningFlag': 1,
'Py_InspectFlag': 1,
'Py_InteractiveFlag': 1,
'Py_OptimizeFlag': 2,
'Py_DontWriteBytecodeFlag': 1,
'Py_VerboseFlag': 1,
'Py_QuietFlag': 1,
'Py_UnbufferedStdioFlag': 1,
'utf8_mode': 1,
'Py_NoUserSiteDirectory': 1,
'Py_FrozenFlag': 1,
}
self.check_config("init_global_config", config)
def test_init_from_config(self):
config = {
'install_signal_handlers': 0,
'use_hash_seed': 1,
'hash_seed': 123,
'allocator': 'malloc_debug',
'tracemalloc': 2,
'import_time': 1,
'show_ref_count': 1,
'show_alloc_count': 1,
'malloc_stats': 1,
'utf8_mode': 1,
'program_name': './conf_program_name',
'program': 'conf_program',
'faulthandler': 1,
}
self.check_config("init_from_config", config)
def test_init_env(self):
config = {
'use_hash_seed': 1,
'hash_seed': 42,
'allocator': 'malloc_debug',
'tracemalloc': 2,
'import_time': 1,
'malloc_stats': 1,
'utf8_mode': 1,
'Py_InspectFlag': 1,
'Py_OptimizeFlag': 2,
'Py_DontWriteBytecodeFlag': 1,
'Py_VerboseFlag': 1,
'Py_UnbufferedStdioFlag': 1,
'Py_NoUserSiteDirectory': 1,
'faulthandler': 1,
'dev_mode': 1,
}
self.check_config("init_env", config)
def test_init_dev_mode(self):
config = {
'dev_mode': 1,
'faulthandler': 1,
'allocator': 'debug',
}
self.check_config("init_dev_mode", config)
def test_init_isolated(self):
config = {
'Py_IsolatedFlag': 1,
'Py_IgnoreEnvironmentFlag': 1,
'Py_NoUserSiteDirectory': 1,
}
self.check_config("init_isolated", config)
if __name__ == "__main__":
unittest.main()

2
Misc/NEWS.d/next/C API/2018-08-05-00-21-38.bpo-34247._Sn92u.rst Normal file
View File

@ -0,0 +1,2 @@
Fix Py_Initialize() regression introduced in 3.7.0: read environment
variables like PYTHONOPTIMIZE.

2
Misc/NEWS.d/next/Core and Builtins/2018-08-03-21-59-06.bpo-34170.v1h_H2.rst Normal file
View File

@ -0,0 +1,2 @@
-X dev: it is now possible to override the memory allocator using
PYTHONMALLOC even if the developer mode is enabled.

301
Modules/main.c
View File

@ -1364,9 +1364,24 @@ pymain_update_sys_path(_PyMain *pymain, PyObject *path0)
}
void
_PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config)
{
#define COPY_FLAG(ATTR, VALUE) \
if (config->ATTR == -1) { \
config->ATTR = VALUE; \
}
COPY_FLAG(ignore_environment, Py_IgnoreEnvironmentFlag);
COPY_FLAG(utf8_mode, Py_UTF8Mode);
#undef COPY_FLAG
}
/* Get Py_xxx global configuration variables */
static void
pymain_get_global_config(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
cmdline_get_global_config(_Py_CommandLineDetails *cmdline)
{
cmdline->bytes_warning = Py_BytesWarningFlag;
cmdline->debug = Py_DebugFlag;
@ -1385,15 +1400,24 @@ pymain_get_global_config(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
cmdline->legacy_windows_stdio = Py_LegacyWindowsStdioFlag;
#endif
cmdline->check_hash_pycs_mode = _Py_CheckHashBasedPycsMode ;
}
pymain->config.ignore_environment = Py_IgnoreEnvironmentFlag;
pymain->config.utf8_mode = Py_UTF8Mode;
void
_PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config)
{
Py_IgnoreEnvironmentFlag = config->ignore_environment;
Py_UTF8Mode = config->utf8_mode;
/* Random or non-zero hash seed */
Py_HashRandomizationFlag = (config->use_hash_seed == 0 ||
config->hash_seed != 0);
}
/* Set Py_xxx global configuration variables */
static void
pymain_set_global_config(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
cmdline_set_global_config(_Py_CommandLineDetails *cmdline)
{
Py_BytesWarningFlag = cmdline->bytes_warning;
Py_DebugFlag = cmdline->debug;
@ -1412,13 +1436,6 @@ pymain_set_global_config(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
Py_LegacyWindowsFSEncodingFlag = cmdline->legacy_windows_fs_encoding;
Py_LegacyWindowsStdioFlag = cmdline->legacy_windows_stdio;
#endif
Py_IgnoreEnvironmentFlag = pymain->config.ignore_environment;
Py_UTF8Mode = pymain->config.utf8_mode;
/* Random or non-zero hash seed */
Py_HashRandomizationFlag = (pymain->config.use_hash_seed == 0 ||
pymain->config.hash_seed != 0);
}
@ -1632,7 +1649,7 @@ pymain_wstr_to_int(const wchar_t *wstr, int *result)
static _PyInitError
pymain_init_tracemalloc(_PyCoreConfig *config)
config_init_tracemalloc(_PyCoreConfig *config)
{
int nframe;
int valid;
@ -1714,6 +1731,27 @@ cmdline_get_env_flags(_Py_CommandLineDetails *cmdline)
}
/* Set global variable variables from environment variables */
void
_Py_Initialize_ReadEnvVarsNoAlloc(void)
{
_Py_CommandLineDetails cmdline;
memset(&cmdline, 0, sizeof(cmdline));
cmdline_get_global_config(&cmdline);
if (cmdline.isolated) {
Py_IgnoreEnvironmentFlag = 1;
cmdline.no_user_site_directory = 1;
}
if (!Py_IgnoreEnvironmentFlag) {
cmdline_get_env_flags(&cmdline);
}
cmdline_set_global_config(&cmdline);
/* no need to call pymain_clear_cmdline(), no memory has been allocated */
}
static _PyInitError
config_init_home(_PyCoreConfig *config)
{
@ -1741,17 +1779,15 @@ config_init_home(_PyCoreConfig *config)
static _PyInitError
config_init_hash_seed(_PyCoreConfig *config)
{
if (config->use_hash_seed < 0) {
const char *seed_text = config_get_env_var("PYTHONHASHSEED");
int use_hash_seed;
unsigned long hash_seed;
if (_Py_ReadHashSeed(seed_text, &use_hash_seed, &hash_seed) < 0) {
return _Py_INIT_USER_ERR("PYTHONHASHSEED must be \"random\" "
"or an integer in range [0; 4294967295]");
}
config->use_hash_seed = use_hash_seed;
config->hash_seed = hash_seed;
const char *seed_text = config_get_env_var("PYTHONHASHSEED");
int use_hash_seed;
unsigned long hash_seed;
if (_Py_ReadHashSeed(seed_text, &use_hash_seed, &hash_seed) < 0) {
return _Py_INIT_USER_ERR("PYTHONHASHSEED must be \"random\" "
"or an integer in range [0; 4294967295]");
}
config->use_hash_seed = use_hash_seed;
config->hash_seed = hash_seed;
return _Py_INIT_OK();
}
@ -1759,12 +1795,6 @@ config_init_hash_seed(_PyCoreConfig *config)
static _PyInitError
config_init_utf8_mode(_PyCoreConfig *config)
{
/* The option was already set by Py_UTF8Mode,
Py_LegacyWindowsFSEncodingFlag or PYTHONLEGACYWINDOWSFSENCODING. */
if (config->utf8_mode >= 0) {
return _Py_INIT_OK();
}
const wchar_t *xopt = config_get_xoption(config, L"utf8");
if (xopt) {
wchar_t *sep = wcschr(xopt, L'=');
@ -1808,7 +1838,11 @@ config_init_utf8_mode(_PyCoreConfig *config)
static _PyInitError
config_read_env_vars(_PyCoreConfig *config)
{
config->allocator = config_get_env_var("PYTHONMALLOC");
assert(!config->ignore_environment);
if (config->allocator == NULL) {
config->allocator = config_get_env_var("PYTHONMALLOC");
}
if (config_get_env_var("PYTHONDUMPREFS")) {
config->dump_refs = 1;
@ -1817,16 +1851,18 @@ config_read_env_vars(_PyCoreConfig *config)
config->malloc_stats = 1;
}
const char *env = config_get_env_var("PYTHONCOERCECLOCALE");
if (env) {
if (strcmp(env, "0") == 0) {
config->coerce_c_locale = 0;
}
else if (strcmp(env, "warn") == 0) {
config->coerce_c_locale_warn = 1;
}
else {
config->coerce_c_locale = 1;
if (config->coerce_c_locale < 0) {
const char *env = config_get_env_var("PYTHONCOERCECLOCALE");
if (env) {
if (strcmp(env, "0") == 0) {
config->coerce_c_locale = 0;
}
else if (strcmp(env, "warn") == 0) {
config->coerce_c_locale_warn = 1;
}
else {
config->coerce_c_locale = 1;
}
}
}
@ -1837,9 +1873,11 @@ config_read_env_vars(_PyCoreConfig *config)
}
config->module_search_path_env = path;
_PyInitError err = config_init_hash_seed(config);
if (_Py_INIT_FAILED(err)) {
return err;
if (config->use_hash_seed < 0) {
_PyInitError err = config_init_hash_seed(config);
if (_Py_INIT_FAILED(err)) {
return err;
}
}
return _Py_INIT_OK();
@ -1850,9 +1888,11 @@ static _PyInitError
config_read_complex_options(_PyCoreConfig *config)
{
/* More complex options configured by env var and -X option */
if (config_get_env_var("PYTHONFAULTHANDLER")
|| config_get_xoption(config, L"faulthandler")) {
config->faulthandler = 1;
if (config->faulthandler < 0) {
if (config_get_env_var("PYTHONFAULTHANDLER")
|| config_get_xoption(config, L"faulthandler")) {
config->faulthandler = 1;
}
}
if (config_get_env_var("PYTHONPROFILEIMPORTTIME")
|| config_get_xoption(config, L"importtime")) {
@ -1862,13 +1902,13 @@ config_read_complex_options(_PyCoreConfig *config)
config_get_env_var("PYTHONDEVMODE"))
{
config->dev_mode = 1;
config->faulthandler = 1;
config->allocator = "debug";
}
_PyInitError err = pymain_init_tracemalloc(config);
if (_Py_INIT_FAILED(err)) {
return err;
if (config->tracemalloc < 0) {
_PyInitError err = config_init_tracemalloc(config);
if (_Py_INIT_FAILED(err)) {
return err;
}
}
return _Py_INIT_OK();
}
@ -1892,10 +1932,12 @@ pymain_read_conf_impl(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
/* Set Py_IgnoreEnvironmentFlag for Py_GETENV() */
_PyCoreConfig *config = &pymain->config;
Py_IgnoreEnvironmentFlag = config->ignore_environment;
Py_IgnoreEnvironmentFlag = config->ignore_environment || cmdline->isolated;
/* Get environment variables */
cmdline_get_env_flags(cmdline);
if (!Py_IgnoreEnvironmentFlag) {
cmdline_get_env_flags(cmdline);
}
err = cmdline_init_env_warnoptions(cmdline);
if (_Py_INIT_FAILED(err)) {
@ -1940,6 +1982,8 @@ pymain_read_conf_impl(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
static int
pymain_read_conf(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
{
_PyCoreConfig *config = &pymain->config;
_PyCoreConfig save_config = _PyCoreConfig_INIT;
int res = -1;
char *oldloc = _PyMem_RawStrdup(setlocale(LC_ALL, NULL));
@ -1953,10 +1997,15 @@ pymain_read_conf(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
int locale_coerced = 0;
int loops = 0;
int init_ignore_env = pymain->config.ignore_environment;
int init_ignore_env = config->ignore_environment;
if (_PyCoreConfig_Copy(&save_config, config) < 0) {
pymain->err = _Py_INIT_NO_MEMORY();
goto done;
}
while (1) {
int utf8_mode = pymain->config.utf8_mode;
int init_utf8_mode = config->utf8_mode;
int encoding_changed = 0;
/* Watchdog to prevent an infinite loop */
@ -1987,20 +2036,20 @@ pymain_read_conf(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
* See the documentation of the PYTHONCOERCECLOCALE setting for more
* details.
*/
if (pymain->config.coerce_c_locale == 1 && !locale_coerced) {
if (config->coerce_c_locale == 1 && !locale_coerced) {
locale_coerced = 1;
_Py_CoerceLegacyLocale(&pymain->config);
_Py_CoerceLegacyLocale(config);
encoding_changed = 1;
}
if (utf8_mode == -1) {
if (pymain->config.utf8_mode == 1) {
if (init_utf8_mode == -1) {
if (config->utf8_mode == 1) {
/* UTF-8 Mode enabled */
encoding_changed = 1;
}
}
else {
if (pymain->config.utf8_mode != utf8_mode) {
if (config->utf8_mode != init_utf8_mode) {
encoding_changed = 1;
}
}
@ -2013,12 +2062,18 @@ pymain_read_conf(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
Py_DecodeLocale(). Reset Py_IgnoreEnvironmentFlag, modified by
pymain_read_conf_impl(). Reset Py_IsolatedFlag and Py_NoSiteFlag
modified by _PyCoreConfig_Read(). */
Py_UTF8Mode = pymain->config.utf8_mode;
int new_utf8_mode = config->utf8_mode;
Py_IgnoreEnvironmentFlag = init_ignore_env;
_PyCoreConfig_Clear(&pymain->config);
if (_PyCoreConfig_Copy(config, &save_config) < 0) {
pymain->err = _Py_INIT_NO_MEMORY();
goto done;
}
pymain_clear_cmdline(pymain, cmdline);
memset(cmdline, 0, sizeof(*cmdline));
pymain_get_global_config(pymain, cmdline);
cmdline_get_global_config(cmdline);
_PyCoreConfig_GetGlobalConfig(config);
config->utf8_mode = new_utf8_mode;
/* The encoding changed: read again the configuration
with the new encoding */
@ -2026,6 +2081,7 @@ pymain_read_conf(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
res = 0;
done:
_PyCoreConfig_Clear(&save_config);
if (oldloc != NULL) {
setlocale(LC_ALL, oldloc);
PyMem_RawFree(oldloc);
@ -2038,10 +2094,6 @@ done:
static void
config_init_locale(_PyCoreConfig *config)
{
if (config->utf8_mode >= 0 && config->coerce_c_locale >= 0) {
return;
}
if (_Py_LegacyLocaleDetected()) {
/* POSIX locale: enable C locale coercion and UTF-8 Mode */
if (config->utf8_mode < 0) {
@ -2050,15 +2102,6 @@ config_init_locale(_PyCoreConfig *config)
if (config->coerce_c_locale < 0) {
config->coerce_c_locale = 1;
}
return;
}
/* By default, C locale coercion and UTF-8 Mode are disabled */
if (config->coerce_c_locale < 0) {
config->coerce_c_locale = 0;
}
if (config->utf8_mode < 0) {
config->utf8_mode = 0;
}
}
@ -2175,9 +2218,14 @@ _PyCoreConfig_Read(_PyCoreConfig *config)
{
_PyInitError err;
err = config_read_env_vars(config);
if (_Py_INIT_FAILED(err)) {
return err;
_PyCoreConfig_GetGlobalConfig(config);
assert(config->ignore_environment >= 0);
if (!config->ignore_environment) {
err = config_read_env_vars(config);
if (_Py_INIT_FAILED(err)) {
return err;
}
}
/* -X options */
@ -2193,26 +2241,29 @@ _PyCoreConfig_Read(_PyCoreConfig *config)
return err;
}
err = config_init_utf8_mode(config);
if (_Py_INIT_FAILED(err)) {
return err;
if (config->utf8_mode < 0) {
err = config_init_utf8_mode(config);
if (_Py_INIT_FAILED(err)) {
return err;
}
}
err = config_init_home(config);
if (_Py_INIT_FAILED(err)) {
return err;
if (config->home == NULL) {
err = config_init_home(config);
if (_Py_INIT_FAILED(err)) {
return err;
}
}
err = config_init_program_name(config);
if (_Py_INIT_FAILED(err)) {
return err;
if (config->program_name == NULL) {
err = config_init_program_name(config);
if (_Py_INIT_FAILED(err)) {
return err;
}
}
config_init_locale(config);
/* Signal handlers are installed by default */
if (config->install_signal_handlers < 0) {
config->install_signal_handlers = 1;
if (config->utf8_mode < 0 || config->coerce_c_locale < 0) {
config_init_locale(config);
}
if (!config->_disable_importlib) {
@ -2221,6 +2272,39 @@ _PyCoreConfig_Read(_PyCoreConfig *config)
return err;
}
}
/* default values */
if (config->dev_mode) {
if (config->faulthandler < 0) {
config->faulthandler = 1;
}
if (config->allocator == NULL) {
config->allocator = "debug";
}
}
if (config->install_signal_handlers < 0) {
config->install_signal_handlers = 1;
}
if (config->use_hash_seed < 0) {
config->use_hash_seed = 0;
config->hash_seed = 0;
}
if (config->faulthandler < 0) {
config->faulthandler = 0;
}
if (config->tracemalloc < 0) {
config->tracemalloc = 0;
}
if (config->coerce_c_locale < 0) {
config->coerce_c_locale = 0;
}
if (config->utf8_mode < 0) {
config->utf8_mode = 0;
}
if (config->argc < 0) {
config->argc = 0;
}
return _Py_INIT_OK();
}
@ -2289,6 +2373,7 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2)
config->LEN = config2->LEN; \
} while (0)
COPY_ATTR(install_signal_handlers);
COPY_ATTR(ignore_environment);
COPY_ATTR(use_hash_seed);
COPY_ATTR(hash_seed);
@ -2302,6 +2387,9 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2)
COPY_ATTR(show_alloc_count);
COPY_ATTR(dump_refs);
COPY_ATTR(malloc_stats);
COPY_ATTR(coerce_c_locale);
COPY_ATTR(coerce_c_locale_warn);
COPY_ATTR(utf8_mode);
COPY_STR_ATTR(module_search_path_env);
@ -2457,14 +2545,14 @@ _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *main_config,
static int
pymain_init_python_main(_PyMain *pymain)
pymain_init_python_main(_PyMain *pymain, PyInterpreterState *interp)
{
_PyInitError err;
_PyMainInterpreterConfig main_config = _PyMainInterpreterConfig_INIT;
err = _PyMainInterpreterConfig_Read(&main_config, &pymain->config);
err = _PyMainInterpreterConfig_Read(&main_config, &interp->core_config);
if (!_Py_INIT_FAILED(err)) {
err = _Py_InitializeMainInterpreter(&main_config);
err = _Py_InitializeMainInterpreter(interp, &main_config);
}
_PyMainInterpreterConfig_Clear(&main_config);
@ -2615,11 +2703,17 @@ pymain_cmdline(_PyMain *pymain)
_Py_CommandLineDetails cmdline;
memset(&cmdline, 0, sizeof(cmdline));
pymain_get_global_config(pymain, &cmdline);
cmdline_get_global_config(&cmdline);
_PyCoreConfig_GetGlobalConfig(&pymain->config);
int res = pymain_cmdline_impl(pymain, &cmdline);
pymain_set_global_config(pymain, &cmdline);
cmdline_set_global_config(&cmdline);
_PyCoreConfig_SetGlobalConfig(&pymain->config);
if (Py_IsolatedFlag) {
Py_IgnoreEnvironmentFlag = 1;
Py_NoUserSiteDirectory = 1;
}
pymain_clear_cmdline(pymain, &cmdline);
@ -2649,16 +2743,13 @@ pymain_main(_PyMain *pymain)
pymain_init_stdio(pymain);
/* bpo-34008: For backward compatibility reasons, calling Py_Main() after
Py_Initialize() ignores the new configuration. */
if (!_PyRuntime.initialized) {
pymain->err = _Py_InitializeCore(&pymain->config);
if (_Py_INIT_FAILED(pymain->err)) {
_Py_FatalInitError(pymain->err);
}
PyInterpreterState *interp;
pymain->err = _Py_InitializeCore(&interp, &pymain->config);
if (_Py_INIT_FAILED(pymain->err)) {
_Py_FatalInitError(pymain->err);
}
if (pymain_init_python_main(pymain) < 0) {
if (pymain_init_python_main(pymain, interp) < 0) {
_Py_FatalInitError(pymain->err);
}

13
Programs/_freeze_importlib.c
View File

@ -74,18 +74,25 @@ main(int argc, char *argv[])
}
text[text_size] = '\0';
_PyCoreConfig config = _PyCoreConfig_INIT;
config.program_name = L"./_freeze_importlib";
/* Don't install importlib, since it could execute outdated bytecode. */
config._disable_importlib = 1;
Py_NoUserSiteDirectory++;
Py_NoSiteFlag++;
Py_IgnoreEnvironmentFlag++;
Py_FrozenFlag++;
Py_SetProgramName(L"./_freeze_importlib");
/* Don't install importlib, since it could execute outdated bytecode. */
_PyInitError err = _Py_InitializeEx_Private(1, 0);
_PyInitError err = _Py_InitializeFromConfig(&config);
/* No need to call _PyCoreConfig_Clear() since we didn't allocate any
memory: program_name is a constant string. */
if (_Py_INIT_FAILED(err)) {
_Py_FatalInitError(err);
}
if (strstr(inpath, "_external") != NULL) {
is_bootstrap = 0;
}

294
Programs/_testembed.c
View File

@ -1,4 +1,5 @@
#include <Python.h>
#include "internal/import.h"
#include "pythread.h"
#include <inttypes.h>
#include <stdio.h>
@ -292,6 +293,293 @@ static int test_initialize_pymain(void)
}
static void
dump_config(void)
{
#define ASSERT_EQUAL(a, b) \
if ((a) != (b)) { \
printf("ERROR: %s != %s (%i != %i)\n", #a, #b, (a), (b)); \
exit(1); \
}
#define ASSERT_STR_EQUAL(a, b) \
if ((a) == NULL || (b == NULL) || wcscmp((a), (b)) != 0) { \
printf("ERROR: %s != %s ('%ls' != '%ls')\n", #a, #b, (a), (b)); \
exit(1); \
}
PyInterpreterState *interp = PyThreadState_Get()->interp;
_PyCoreConfig *config = &interp->core_config;
printf("install_signal_handlers = %i\n", config->install_signal_handlers);
printf("Py_IgnoreEnvironmentFlag = %i\n", Py_IgnoreEnvironmentFlag);
printf("use_hash_seed = %i\n", config->use_hash_seed);
printf("hash_seed = %lu\n", config->hash_seed);
printf("allocator = %s\n", config->allocator);
printf("dev_mode = %i\n", config->dev_mode);
printf("faulthandler = %i\n", config->faulthandler);
printf("tracemalloc = %i\n", config->tracemalloc);
printf("import_time = %i\n", config->import_time);
printf("show_ref_count = %i\n", config->show_ref_count);
printf("show_alloc_count = %i\n", config->show_alloc_count);
printf("dump_refs = %i\n", config->dump_refs);
printf("malloc_stats = %i\n", config->malloc_stats);
printf("coerce_c_locale = %i\n", config->coerce_c_locale);
printf("coerce_c_locale_warn = %i\n", config->coerce_c_locale_warn);
printf("utf8_mode = %i\n", config->utf8_mode);
printf("program_name = %ls\n", config->program_name);
ASSERT_STR_EQUAL(config->program_name, Py_GetProgramName());
printf("argc = %i\n", config->argc);
printf("argv = [");
for (int i=0; i < config->argc; i++) {
if (i) {
printf(", ");
}
printf("\"%ls\"", config->argv[i]);
}
printf("]\n");
printf("program = %ls\n", config->program);
/* FIXME: test xoptions */
/* FIXME: test warnoptions */
/* FIXME: test module_search_path_env */
/* FIXME: test home */
/* FIXME: test module_search_paths */
/* FIXME: test executable */
/* FIXME: test prefix */
/* FIXME: test base_prefix */
/* FIXME: test exec_prefix */
/* FIXME: test base_exec_prefix */
/* FIXME: test dll_path */
printf("Py_IsolatedFlag = %i\n", Py_IsolatedFlag);
printf("Py_NoSiteFlag = %i\n", Py_NoSiteFlag);
printf("Py_BytesWarningFlag = %i\n", Py_BytesWarningFlag);
printf("Py_InspectFlag = %i\n", Py_InspectFlag);
printf("Py_InteractiveFlag = %i\n", Py_InteractiveFlag);
printf("Py_OptimizeFlag = %i\n", Py_OptimizeFlag);
printf("Py_DebugFlag = %i\n", Py_DebugFlag);
printf("Py_DontWriteBytecodeFlag = %i\n", Py_DontWriteBytecodeFlag);
printf("Py_VerboseFlag = %i\n", Py_VerboseFlag);
printf("Py_QuietFlag = %i\n", Py_QuietFlag);
printf("Py_NoUserSiteDirectory = %i\n", Py_NoUserSiteDirectory);
printf("Py_UnbufferedStdioFlag = %i\n", Py_UnbufferedStdioFlag);
/* FIXME: test legacy_windows_fs_encoding */
/* FIXME: test legacy_windows_stdio */
printf("_disable_importlib = %i\n", config->_disable_importlib);
/* cannot test _Py_CheckHashBasedPycsMode: the symbol is not exported */
printf("Py_FrozenFlag = %i\n", Py_FrozenFlag);
#undef ASSERT_EQUAL
#undef ASSERT_STR_EQUAL
}
static int test_init_default_config(void)
{
_testembed_Py_Initialize();
dump_config();
Py_Finalize();
return 0;
}
static int test_init_global_config(void)
{
/* FIXME: test Py_IgnoreEnvironmentFlag */
putenv("PYTHONUTF8=0");
Py_UTF8Mode = 1;
/* Test initialization from global configuration variables (Py_xxx) */
Py_SetProgramName(L"./globalvar");
/* Py_IsolatedFlag is not tested */
Py_NoSiteFlag = 1;
Py_BytesWarningFlag = 1;
putenv("PYTHONINSPECT=");
Py_InspectFlag = 1;
putenv("PYTHONOPTIMIZE=0");
Py_InteractiveFlag = 1;
putenv("PYTHONDEBUG=0");
Py_OptimizeFlag = 2;
/* Py_DebugFlag is not tested */
putenv("PYTHONDONTWRITEBYTECODE=");
Py_DontWriteBytecodeFlag = 1;
putenv("PYTHONVERBOSE=0");
Py_VerboseFlag = 1;
Py_QuietFlag = 1;
Py_NoUserSiteDirectory = 1;
putenv("PYTHONUNBUFFERED=");
Py_UnbufferedStdioFlag = 1;
Py_FrozenFlag = 1;
/* FIXME: test Py_LegacyWindowsFSEncodingFlag */
/* FIXME: test Py_LegacyWindowsStdioFlag */
Py_Initialize();
dump_config();
Py_Finalize();
return 0;
}
static int test_init_from_config(void)
{
/* Test _Py_InitializeFromConfig() */
_PyCoreConfig config = _PyCoreConfig_INIT;
config.install_signal_handlers = 0;
/* FIXME: test ignore_environment */
putenv("PYTHONHASHSEED=42");
config.use_hash_seed = 1;
config.hash_seed = 123;
putenv("PYTHONMALLOC=malloc");
config.allocator = "malloc_debug";
/* dev_mode=1 is tested in test_init_dev_mode() */
putenv("PYTHONFAULTHANDLER=");
config.faulthandler = 1;
putenv("PYTHONTRACEMALLOC=0");
config.tracemalloc = 2;
putenv("PYTHONPROFILEIMPORTTIME=0");
config.import_time = 1;
config.show_ref_count = 1;
config.show_alloc_count = 1;
/* FIXME: test dump_refs: bpo-34223 */
putenv("PYTHONMALLOCSTATS=0");
config.malloc_stats = 1;
/* FIXME: test coerce_c_locale and coerce_c_locale_warn */
putenv("PYTHONUTF8=0");
Py_UTF8Mode = 0;
config.utf8_mode = 1;
Py_SetProgramName(L"./globalvar");
config.program_name = L"./conf_program_name";
/* FIXME: test argc/argv */
config.program = L"conf_program";
/* FIXME: test xoptions */
/* FIXME: test warnoptions */
/* FIXME: test module_search_path_env */
/* FIXME: test home */
/* FIXME: test path config: module_search_path .. dll_path */
_PyInitError err = _Py_InitializeFromConfig(&config);
/* Don't call _PyCoreConfig_Clear() since all strings are static */
if (_Py_INIT_FAILED(err)) {
_Py_FatalInitError(err);
}
dump_config();
Py_Finalize();
return 0;
}
static void test_init_env_putenvs(void)
{
putenv("PYTHONHASHSEED=42");
putenv("PYTHONMALLOC=malloc_debug");
putenv("PYTHONTRACEMALLOC=2");
putenv("PYTHONPROFILEIMPORTTIME=1");
putenv("PYTHONMALLOCSTATS=1");
putenv("PYTHONUTF8=1");
putenv("PYTHONVERBOSE=1");
putenv("PYTHONINSPECT=1");
putenv("PYTHONOPTIMIZE=2");
putenv("PYTHONDONTWRITEBYTECODE=1");
putenv("PYTHONUNBUFFERED=1");
putenv("PYTHONNOUSERSITE=1");
putenv("PYTHONFAULTHANDLER=1");
putenv("PYTHONDEVMODE=1");
/* FIXME: test PYTHONWARNINGS */
/* FIXME: test PYTHONEXECUTABLE */
/* FIXME: test PYTHONHOME */
/* FIXME: test PYTHONDEBUG */
/* FIXME: test PYTHONDUMPREFS */
/* FIXME: test PYTHONCOERCECLOCALE */
/* FIXME: test PYTHONPATH */
}
static int test_init_env(void)
{
/* Test initialization from environment variables */
Py_IgnoreEnvironmentFlag = 0;
test_init_env_putenvs();
_testembed_Py_Initialize();
dump_config();
Py_Finalize();
return 0;
}
static int test_init_isolated(void)
{
/* Test _PyCoreConfig.isolated=1 */
_PyCoreConfig config = _PyCoreConfig_INIT;
/* Set coerce_c_locale and utf8_mode to not depend on the locale */
config.coerce_c_locale = 0;
config.utf8_mode = 0;
/* Use path starting with "./" avoids a search along the PATH */
config.program_name = L"./_testembed";
Py_IsolatedFlag = 1;
test_init_env_putenvs();
_PyInitError err = _Py_InitializeFromConfig(&config);
if (_Py_INIT_FAILED(err)) {
_Py_FatalInitError(err);
}
dump_config();
Py_Finalize();
return 0;
}
static int test_init_dev_mode(void)
{
_PyCoreConfig config = _PyCoreConfig_INIT;
putenv("PYTHONFAULTHANDLER=");
putenv("PYTHONMALLOC=");
config.dev_mode = 1;
config.program_name = L"./_testembed";
_PyInitError err = _Py_InitializeFromConfig(&config);
if (_Py_INIT_FAILED(err)) {
_Py_FatalInitError(err);
}
dump_config();
Py_Finalize();
return 0;
}
/* *********************************************************
* List of test cases and the function that implements it.
*
@ -318,6 +606,12 @@ static struct TestCase TestCases[] = {
{ "bpo20891", test_bpo20891 },
{ "initialize_twice", test_initialize_twice },
{ "initialize_pymain", test_initialize_pymain },
{ "init_default_config", test_init_default_config },
{ "init_global_config", test_init_global_config },
{ "init_from_config", test_init_from_config },
{ "init_env", test_init_env },
{ "init_dev_mode", test_init_dev_mode },
{ "init_isolated", test_init_isolated },
{ NULL, NULL }
};

156
Python/pylifecycle.c
View File

@ -576,6 +576,27 @@ _Py_SetLocaleFromEnv(int category)
*/
static _PyInitError
_Py_Initialize_ReconfigureCore(PyInterpreterState *interp,
const _PyCoreConfig *core_config)
{
if (core_config->allocator != NULL) {
const char *allocator = _PyMem_GetAllocatorsName();
if (allocator == NULL || strcmp(core_config->allocator, allocator) != 0) {
return _Py_INIT_USER_ERR("cannot modify memory allocator "
"after first Py_Initialize()");
}
}
_PyCoreConfig_SetGlobalConfig(core_config);
if (_PyCoreConfig_Copy(&interp->core_config, core_config) < 0) {
return _Py_INIT_ERR("failed to copy core config");
}
return _Py_INIT_OK();
}
/* Begin interpreter initialization
*
* On return, the first thread and interpreter state have been created,
@ -595,15 +616,32 @@ _Py_SetLocaleFromEnv(int category)
*/
_PyInitError
_Py_InitializeCore(const _PyCoreConfig *core_config)
_Py_InitializeCore_impl(PyInterpreterState **interp_p,
const _PyCoreConfig *core_config)
{
assert(core_config != NULL);
PyInterpreterState *interp;
PyThreadState *tstate;
PyObject *bimod, *sysmod, *pstderr;
_PyInitError err;
/* bpo-34008: For backward compatibility reasons, calling Py_Main() after
Py_Initialize() ignores the new configuration. */
if (_PyRuntime.core_initialized) {
PyThreadState *tstate = PyThreadState_GET();
if (!tstate) {
return _Py_INIT_ERR("no thread state found");
}
interp = tstate->interp;
if (interp == NULL) {
return _Py_INIT_ERR("no main interpreter found");
}
*interp_p = interp;
return _Py_Initialize_ReconfigureCore(interp, core_config);
}
_PyCoreConfig_SetGlobalConfig(core_config);
err = _PyRuntime_Initialize();
if (_Py_INIT_FAILED(err)) {
return err;
@ -660,12 +698,13 @@ _Py_InitializeCore(const _PyCoreConfig *core_config)
if (interp == NULL) {
return _Py_INIT_ERR("can't make main interpreter");
}
*interp_p = interp;
if (_PyCoreConfig_Copy(&interp->core_config, core_config) < 0) {
return _Py_INIT_ERR("failed to copy core config");
}
tstate = PyThreadState_New(interp);
PyThreadState *tstate = PyThreadState_New(interp);
if (tstate == NULL)
return _Py_INIT_ERR("can't make first thread");
(void) PyThreadState_Swap(tstate);
@ -702,6 +741,7 @@ _Py_InitializeCore(const _PyCoreConfig *core_config)
return _Py_INIT_ERR("can't make modules dictionary");
interp->modules = modules;
PyObject *sysmod;
err = _PySys_BeginInit(&sysmod);
if (_Py_INIT_FAILED(err)) {
return err;
@ -723,7 +763,7 @@ _Py_InitializeCore(const _PyCoreConfig *core_config)
if (_PyStructSequence_Init() < 0)
return _Py_INIT_ERR("can't initialize structseq");
bimod = _PyBuiltin_Init();
PyObject *bimod = _PyBuiltin_Init();
if (bimod == NULL)
return _Py_INIT_ERR("can't initialize builtins modules");
_PyImport_FixupBuiltin(bimod, "builtins", modules);
@ -737,7 +777,7 @@ _Py_InitializeCore(const _PyCoreConfig *core_config)
/* Set up a preliminary stderr printer until we have enough
infrastructure for the io module in place. */
pstderr = PyFile_NewStdPrinter(fileno(stderr));
PyObject *pstderr = PyFile_NewStdPrinter(fileno(stderr));
if (pstderr == NULL)
return _Py_INIT_ERR("can't set preliminary stderr");
_PySys_SetObjectId(&PyId_stderr, pstderr);
@ -775,6 +815,43 @@ _Py_InitializeCore(const _PyCoreConfig *core_config)
return _Py_INIT_OK();
}
_PyInitError
_Py_InitializeCore(PyInterpreterState **interp_p,
const _PyCoreConfig *src_config)
{
assert(src_config != NULL);
PyMemAllocatorEx old_alloc;
_PyInitError err;
/* Copy the configuration, since _PyCoreConfig_Read() modifies it
(and the input configuration is read only). */
_PyCoreConfig config = _PyCoreConfig_INIT;
_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
if (_PyCoreConfig_Copy(&config, src_config) >= 0) {
err = _PyCoreConfig_Read(&config);
}
else {
err = _Py_INIT_ERR("failed to copy core config");
}
PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
if (_Py_INIT_FAILED(err)) {
goto done;
}
err = _Py_InitializeCore_impl(interp_p, &config);
done:
_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
_PyCoreConfig_Clear(&config);
PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
return err;
}
/* Py_Initialize() has already been called: update the main interpreter
configuration. Example of bpo-34008: Py_Main() called after
Py_Initialize(). */
@ -803,24 +880,15 @@ _Py_ReconfigureMainInterpreter(PyInterpreterState *interp,
* non-zero return code.
*/
_PyInitError
_Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config)
_Py_InitializeMainInterpreter(PyInterpreterState *interp,
const _PyMainInterpreterConfig *config)
{
PyInterpreterState *interp;
PyThreadState *tstate;
_PyInitError err;
if (!_PyRuntime.core_initialized) {
return _Py_INIT_ERR("runtime core not initialized");
}
/* Get current thread state and interpreter pointer */
tstate = PyThreadState_GET();
if (!tstate)
return _Py_INIT_ERR("failed to read thread state");
interp = tstate->interp;
if (!interp)
return _Py_INIT_ERR("failed to get interpreter");
/* Now finish configuring the main interpreter */
if (_PyMainInterpreterConfig_Copy(&interp->config, config) < 0) {
return _Py_INIT_ERR("failed to copy main interpreter config");
@ -909,53 +977,49 @@ _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config)
#undef _INIT_DEBUG_PRINT
_PyInitError
_Py_InitializeEx_Private(int install_sigs, int install_importlib)
_Py_InitializeFromConfig(const _PyCoreConfig *config)
{
if (_PyRuntime.initialized) {
/* bpo-33932: Calling Py_Initialize() twice does nothing. */
return _Py_INIT_OK();
}
_Py_Initialize_ReadEnvVarsNoAlloc();
_PyCoreConfig config = _PyCoreConfig_INIT;
PyInterpreterState *interp;
_PyInitError err;
config.ignore_environment = Py_IgnoreEnvironmentFlag;
config._disable_importlib = !install_importlib;
config.install_signal_handlers = install_sigs;
err = _PyCoreConfig_Read(&config);
err = _Py_InitializeCore(&interp, config);
if (_Py_INIT_FAILED(err)) {
goto done;
}
err = _Py_InitializeCore(&config);
if (_Py_INIT_FAILED(err)) {
goto done;
return err;
}
config = &interp->core_config;
_PyMainInterpreterConfig main_config = _PyMainInterpreterConfig_INIT;
err = _PyMainInterpreterConfig_Read(&main_config, &config);
err = _PyMainInterpreterConfig_Read(&main_config, config);
if (!_Py_INIT_FAILED(err)) {
err = _Py_InitializeMainInterpreter(&main_config);
err = _Py_InitializeMainInterpreter(interp, &main_config);
}
_PyMainInterpreterConfig_Clear(&main_config);
if (_Py_INIT_FAILED(err)) {
goto done;
return err;
}
err = _Py_INIT_OK();
done:
_PyCoreConfig_Clear(&config);
return err;
return _Py_INIT_OK();
}
void
Py_InitializeEx(int install_sigs)
{
_PyInitError err = _Py_InitializeEx_Private(install_sigs, 1);
if (_PyRuntime.initialized) {
/* bpo-33932: Calling Py_Initialize() twice does nothing. */
return;
}
_PyInitError err;
_PyCoreConfig config = _PyCoreConfig_INIT;
config.install_signal_handlers = install_sigs;
err = _Py_InitializeFromConfig(&config);
_PyCoreConfig_Clear(&config);
if (_Py_INIT_FAILED(err)) {
_Py_FatalInitError(err);
}