bpo-22257: Private C-API for core runtime initialization (PEP 432). (#1772)

(patch by Nick Coghlan)
This commit is contained in:
Eric Snow 2017-05-23 21:46:51 -07:00 committed by GitHub
parent c842efc6ae
commit 1abcf6700b
5 changed files with 224 additions and 62 deletions

View File

@ -19,8 +19,14 @@ PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void);
*/
PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding,
const char *errors);
/* PEP 432 Multi-phase initialization API (Private while provisional!) */
PyAPI_FUNC(void) _Py_InitializeCore(const _PyCoreConfig *);
PyAPI_FUNC(int) _Py_IsCoreInitialized(void);
PyAPI_FUNC(int) _Py_InitializeMainInterpreter(int install_sigs);
#endif
/* Initialization and finalization */
PyAPI_FUNC(void) Py_Initialize(void);
PyAPI_FUNC(void) Py_InitializeEx(int);
#ifndef Py_LIMITED_API
@ -29,6 +35,8 @@ PyAPI_FUNC(void) _Py_InitializeEx_Private(int, int);
PyAPI_FUNC(void) Py_Finalize(void);
PyAPI_FUNC(int) Py_FinalizeEx(void);
PyAPI_FUNC(int) Py_IsInitialized(void);
/* Subinterpreter support */
PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void);
PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *);
@ -85,7 +93,7 @@ PyAPI_FUNC(void) _PyImportHooks_Init(void);
PyAPI_FUNC(int) _PyFrame_Init(void);
PyAPI_FUNC(int) _PyFloat_Init(void);
PyAPI_FUNC(int) PyByteArray_Init(void);
PyAPI_FUNC(void) _Py_HashRandomization_Init(void);
PyAPI_FUNC(void) _Py_HashRandomization_Init(_PyCoreConfig *core_config);
#endif
/* Various internal finalizers */

View File

@ -23,6 +23,16 @@ typedef struct _is PyInterpreterState;
#else
typedef PyObject* (*_PyFrameEvalFunction)(struct _frame *, int);
typedef struct {
int ignore_environment;
int use_hash_seed;
unsigned long hash_seed;
int _disable_importlib; /* Needed by freeze_importlib */
} _PyCoreConfig;
#define _PyCoreConfig_INIT {0, -1, 0, 0}
typedef struct _is {
struct _is *next;
@ -42,6 +52,7 @@ typedef struct _is {
int codecs_initialized;
int fscodec_initialized;
_PyCoreConfig core_config;
#ifdef HAVE_DLOPEN
int dlopenflags;
#endif

View File

@ -380,19 +380,6 @@ read_command_line(int argc, wchar_t **argv, _Py_CommandLineDetails *cmdline)
wchar_t *command = NULL;
wchar_t *module = NULL;
int c;
char *opt;
opt = Py_GETENV("PYTHONMALLOC");
if (_PyMem_SetupAllocators(opt) < 0) {
fprintf(stderr,
"Error in PYTHONMALLOC: unknown allocator \"%s\"!\n", opt);
exit(1);
}
// TODO: Move these to core runtime init.
Py_HashRandomizationFlag = 1;
_Py_HashRandomization_Init();
PySys_ResetWarnOptions();
_PyOS_ResetGetOpt();
@ -584,6 +571,7 @@ Py_Main(int argc, wchar_t **argv)
#endif
int stdin_is_interactive = 0;
_Py_CommandLineDetails cmdline = _Py_CommandLineDetails_INIT;
_PyCoreConfig core_config = _PyCoreConfig_INIT;
PyCompilerFlags cf;
PyObject *main_importer_path = NULL;
@ -602,11 +590,23 @@ Py_Main(int argc, wchar_t **argv)
break;
}
if (c == 'E' || c == 'I') {
Py_IgnoreEnvironmentFlag++;
core_config.ignore_environment++;
break;
}
}
char *pymalloc = Py_GETENV("PYTHONMALLOC");
if (_PyMem_SetupAllocators(pymalloc) < 0) {
fprintf(stderr,
"Error in PYTHONMALLOC: unknown allocator \"%s\"!\n", pymalloc);
exit(1);
}
/* Initialize the core language runtime */
Py_IgnoreEnvironmentFlag = core_config.ignore_environment;
core_config._disable_importlib = 0;
_Py_InitializeCore(&core_config);
/* Reprocess the command line with the language runtime available */
if (read_command_line(argc, argv, &cmdline)) {
return usage(2, argv[0]);
@ -680,6 +680,7 @@ Py_Main(int argc, wchar_t **argv)
for (i = 0; i < PyList_GET_SIZE(cmdline.warning_options); i++) {
PySys_AddWarnOptionUnicode(PyList_GET_ITEM(cmdline.warning_options, i));
}
Py_DECREF(cmdline.warning_options);
}
stdin_is_interactive = Py_FdIsInteractive(stdin, (char *)0);
@ -767,9 +768,10 @@ Py_Main(int argc, wchar_t **argv)
#else
Py_SetProgramName(argv[0]);
#endif
Py_Initialize();
Py_XDECREF(cmdline.warning_options);
if (_Py_InitializeMainInterpreter(1))
Py_FatalError("Py_Main: Py_InitializeMainInterpreter failed");
/* TODO: Move this to _PyRun_PrepareMain */
if (!Py_QuietFlag && (Py_VerboseFlag ||
(cmdline.command == NULL && cmdline.filename == NULL &&
cmdline.module == NULL && stdin_is_interactive))) {
@ -779,6 +781,7 @@ Py_Main(int argc, wchar_t **argv)
fprintf(stderr, "%s\n", COPYRIGHT);
}
/* TODO: Move this to _Py_InitializeMainInterpreter */
if (cmdline.command != NULL) {
/* Backup _PyOS_optind and force sys.argv[0] = '-c' */
_PyOS_optind--;

View File

@ -599,11 +599,11 @@ init_hash_secret(int use_hash_seed,
}
void
_Py_HashRandomization_Init(void)
_Py_HashRandomization_Init(_PyCoreConfig *core_config)
{
char *seed_text;
int use_hash_seed = -1;
unsigned long hash_seed;
int use_hash_seed = core_config->use_hash_seed;
unsigned long hash_seed = core_config->hash_seed;
if (use_hash_seed < 0) {
seed_text = Py_GETENV("PYTHONHASHSEED");
@ -611,6 +611,8 @@ _Py_HashRandomization_Init(void)
Py_FatalError("PYTHONHASHSEED must be \"random\" or an integer "
"in range [0; 4294967295]");
}
core_config->use_hash_seed = use_hash_seed;
core_config->hash_seed = hash_seed;
}
init_hash_secret(use_hash_seed, hash_seed);
}

View File

@ -112,14 +112,23 @@ PyModule_GetWarningsModule(void)
return PyImport_ImportModule("warnings");
}
static int initialized = 0;
/* APIs to access the initialization flags
*
* Can be called prior to Py_Initialize.
*/
int _Py_CoreInitialized = 0;
int _Py_Initialized = 0;
/* API to access the initialized flag -- useful for esoteric use */
int
_Py_IsCoreInitialized(void)
{
return _Py_CoreInitialized;
}
int
Py_IsInitialized(void)
{
return initialized;
return _Py_Initialized;
}
/* Helper to allow an embedding application to override the normal
@ -246,7 +255,7 @@ get_locale_encoding(void)
}
static void
import_init(PyInterpreterState *interp, PyObject *sysmod)
initimport(PyInterpreterState *interp, PyObject *sysmod)
{
PyObject *importlib;
PyObject *impmod;
@ -304,19 +313,82 @@ import_init(PyInterpreterState *interp, PyObject *sysmod)
_PyImportZip_Init();
}
static void
initexternalimport(PyInterpreterState *interp)
{
PyObject *value;
value = PyObject_CallMethod(interp->importlib,
"_install_external_importers", "");
if (value == NULL) {
PyErr_Print();
Py_FatalError("Py_EndInitialization: external importer setup failed");
}
}
void
_Py_InitializeEx_Private(int install_sigs, int install_importlib)
/* Global initializations. Can be undone by Py_Finalize(). Don't
call this twice without an intervening Py_Finalize() call.
Every call to Py_InitializeCore, Py_Initialize or Py_InitializeEx
must have a corresponding call to Py_Finalize.
Locking: you must hold the interpreter lock while calling these APIs.
(If the lock has not yet been initialized, that's equivalent to
having the lock, but you cannot use multiple threads.)
*/
/* Begin interpreter initialization
*
* On return, the first thread and interpreter state have been created,
* but the compiler, signal handling, multithreading and
* multiple interpreter support, and codec infrastructure are not yet
* available.
*
* The import system will support builtin and frozen modules only.
* The only supported io is writing to sys.stderr
*
* If any operation invoked by this function fails, a fatal error is
* issued and the function does not return.
*
* Any code invoked from this function should *not* assume it has access
* to the Python C API (unless the API is explicitly listed as being
* safe to call without calling Py_Initialize first)
*/
/* TODO: Progresively move functionality from Py_BeginInitialization to
* Py_ReadConfig and Py_EndInitialization
*/
void _Py_InitializeCore(const _PyCoreConfig *config)
{
PyInterpreterState *interp;
PyThreadState *tstate;
PyObject *bimod, *sysmod, *pstderr;
char *p;
extern void _Py_ReadyTypes(void);
_PyCoreConfig core_config = _PyCoreConfig_INIT;
if (initialized)
return;
initialized = 1;
if (config != NULL) {
core_config = *config;
}
if (_Py_Initialized) {
Py_FatalError("Py_InitializeCore: main interpreter already initialized");
}
if (_Py_CoreInitialized) {
Py_FatalError("Py_InitializeCore: runtime core already initialized");
}
/* Py_Finalize leaves _Py_Finalizing set in order to help daemon
* threads behave a little more gracefully at interpreter shutdown.
* We clobber it here so the new interpreter can start with a clean
* slate.
*
* However, this may still lead to misbehaviour if there are daemon
* threads still hanging around from a previous Py_Initialize/Finalize
* pair :(
*/
_Py_Finalizing = NULL;
#ifdef HAVE_SETLOCALE
@ -345,16 +417,21 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
Py_LegacyWindowsStdioFlag = add_flag(Py_LegacyWindowsStdioFlag, p);
#endif
_Py_HashRandomization_Init();
_Py_HashRandomization_Init(&core_config);
if (!core_config.use_hash_seed || core_config.hash_seed) {
/* Random or non-zero hash seed */
Py_HashRandomizationFlag = 1;
}
_PyInterpreterState_Init();
interp = PyInterpreterState_New();
if (interp == NULL)
Py_FatalError("Py_Initialize: can't make first interpreter");
Py_FatalError("Py_InitializeCore: can't make main interpreter");
interp->core_config = core_config;
tstate = PyThreadState_New(interp);
if (tstate == NULL)
Py_FatalError("Py_Initialize: can't make first thread");
Py_FatalError("Py_InitializeCore: can't make first thread");
(void) PyThreadState_Swap(tstate);
#ifdef WITH_THREAD
@ -364,7 +441,6 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
Instead we destroy the previously created GIL here, which ensures
that we can call Py_Initialize / Py_FinalizeEx multiple times. */
_PyEval_FiniThreads();
/* Auto-thread-state API */
_PyGILState_Init(interp, tstate);
#endif /* WITH_THREAD */
@ -372,34 +448,35 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
_Py_ReadyTypes();
if (!_PyFrame_Init())
Py_FatalError("Py_Initialize: can't init frames");
Py_FatalError("Py_InitializeCore: can't init frames");
if (!_PyLong_Init())
Py_FatalError("Py_Initialize: can't init longs");
Py_FatalError("Py_InitializeCore: can't init longs");
if (!PyByteArray_Init())
Py_FatalError("Py_Initialize: can't init bytearray");
Py_FatalError("Py_InitializeCore: can't init bytearray");
if (!_PyFloat_Init())
Py_FatalError("Py_Initialize: can't init float");
Py_FatalError("Py_InitializeCore: can't init float");
interp->modules = PyDict_New();
if (interp->modules == NULL)
Py_FatalError("Py_Initialize: can't make modules dictionary");
Py_FatalError("Py_InitializeCore: can't make modules dictionary");
/* Init Unicode implementation; relies on the codec registry */
if (_PyUnicode_Init() < 0)
Py_FatalError("Py_Initialize: can't initialize unicode");
Py_FatalError("Py_InitializeCore: can't initialize unicode");
if (_PyStructSequence_Init() < 0)
Py_FatalError("Py_Initialize: can't initialize structseq");
Py_FatalError("Py_InitializeCore: can't initialize structseq");
bimod = _PyBuiltin_Init();
if (bimod == NULL)
Py_FatalError("Py_Initialize: can't initialize builtins modules");
Py_FatalError("Py_InitializeCore: can't initialize builtins modules");
_PyImport_FixupBuiltin(bimod, "builtins");
interp->builtins = PyModule_GetDict(bimod);
if (interp->builtins == NULL)
Py_FatalError("Py_Initialize: can't initialize builtins dict");
Py_FatalError("Py_InitializeCore: can't initialize builtins dict");
Py_INCREF(interp->builtins);
/* initialize builtin exceptions */
@ -407,19 +484,11 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
sysmod = _PySys_BeginInit();
if (sysmod == NULL)
Py_FatalError("Py_Initialize: can't initialize sys");
Py_FatalError("Py_InitializeCore: can't initialize sys");
interp->sysdict = PyModule_GetDict(sysmod);
if (interp->sysdict == NULL)
Py_FatalError("Py_Initialize: can't initialize sys dict");
Py_FatalError("Py_InitializeCore: can't initialize sys dict");
Py_INCREF(interp->sysdict);
/* GetPath may initialize state that _PySys_EndInit locks
in, and so has to be called first.
Hopefully one day Eric Snow will fix this. */
PySys_SetPath(Py_GetPath());
if (_PySys_EndInit(interp->sysdict) < 0)
Py_FatalError("Py_Initialize: can't initialize sys");
_PyImport_FixupBuiltin(sysmod, "sys");
PyDict_SetItemString(interp->sysdict, "modules",
interp->modules);
@ -428,7 +497,7 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
infrastructure for the io module in place. */
pstderr = PyFile_NewStdPrinter(fileno(stderr));
if (pstderr == NULL)
Py_FatalError("Py_Initialize: can't set preliminary stderr");
Py_FatalError("Py_InitializeCore: can't set preliminary stderr");
_PySys_SetObjectId(&PyId_stderr, pstderr);
PySys_SetObject("__stderr__", pstderr);
Py_DECREF(pstderr);
@ -440,13 +509,53 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
/* Initialize _warnings. */
_PyWarnings_Init();
if (!install_importlib)
return;
/* This call sets up builtin and frozen import support */
if (!interp->core_config._disable_importlib) {
initimport(interp, sysmod);
}
/* Only when we get here is the runtime core fully initialized */
_Py_CoreInitialized = 1;
}
int
_Py_InitializeMainInterpreter(int install_sigs)
{
PyInterpreterState *interp;
PyThreadState *tstate;
/* Get current thread state and interpreter pointer */
tstate = PyThreadState_GET();
if (!tstate)
Py_FatalError("Py_Initialize: failed to read thread state");
interp = tstate->interp;
if (!interp)
Py_FatalError("Py_Initialize: failed to get interpreter");
/* Now finish configuring the main interpreter */
if (interp->core_config._disable_importlib) {
/* Special mode for freeze_importlib: run with no import system
*
* This means anything which needs support from extension modules
* or pure Python code in the standard library won't work.
*/
_Py_Initialized = 1;
return 0;
}
/* TODO: Report exceptions rather than fatal errors below here */
if (_PyTime_Init() < 0)
Py_FatalError("Py_Initialize: can't initialize time");
import_init(interp, sysmod);
/* Finish setting up the sys module and import system */
/* GetPath may initialize state that _PySys_EndInit locks
in, and so has to be called first. */
PySys_SetPath(Py_GetPath());
if (_PySys_EndInit(interp->sysdict) < 0)
Py_FatalError("Py_InitializeMainInterpreter: can't finish initializing sys");
/* TODO: Call Py_GetPath() in Py_ReadConfig, rather than here */
PySys_SetPath(Py_GetPath());
initexternalimport(interp);
/* initialize the faulthandler module */
if (_PyFaulthandler_Init())
@ -476,10 +585,27 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
Py_XDECREF(warnings_module);
}
_Py_Initialized = 1;
if (!Py_NoSiteFlag)
initsite(); /* Module site */
return 0;
}
void
_Py_InitializeEx_Private(int install_sigs, int install_importlib)
{
_PyCoreConfig core_config = _PyCoreConfig_INIT;
/* TODO: Moar config options! */
core_config.ignore_environment = Py_IgnoreEnvironmentFlag;
core_config._disable_importlib = !install_importlib;
_Py_InitializeCore(&core_config);
_Py_InitializeMainInterpreter(install_sigs);
}
void
Py_InitializeEx(int install_sigs)
{
@ -567,7 +693,7 @@ Py_FinalizeEx(void)
PyThreadState *tstate;
int status = 0;
if (!initialized)
if (!_Py_Initialized)
return status;
wait_for_thread_shutdown();
@ -590,7 +716,8 @@ Py_FinalizeEx(void)
/* Remaining threads (e.g. daemon threads) will automatically exit
after taking the GIL (in PyEval_RestoreThread()). */
_Py_Finalizing = tstate;
initialized = 0;
_Py_Initialized = 0;
_Py_CoreInitialized = 0;
/* Flush sys.stdout and sys.stderr */
if (flush_std_files() < 0) {
@ -781,7 +908,7 @@ Py_NewInterpreter(void)
PyThreadState *tstate, *save_tstate;
PyObject *bimod, *sysmod;
if (!initialized)
if (!_Py_Initialized)
Py_FatalError("Py_NewInterpreter: call Py_Initialize first");
#ifdef WITH_THREAD
@ -802,6 +929,15 @@ Py_NewInterpreter(void)
save_tstate = PyThreadState_Swap(tstate);
/* Copy the current interpreter config into the new interpreter */
if (save_tstate != NULL) {
interp->core_config = save_tstate->interp->core_config;
} else {
/* No current thread state, copy from the main interpreter */
PyInterpreterState *main_interp = PyInterpreterState_Main();
interp->core_config = main_interp->core_config;
}
/* XXX The following is lax in error checking */
interp->modules = PyDict_New();
@ -825,6 +961,7 @@ Py_NewInterpreter(void)
if (interp->sysdict == NULL)
goto handle_error;
Py_INCREF(interp->sysdict);
_PySys_EndInit(interp->sysdict);
PySys_SetPath(Py_GetPath());
PyDict_SetItemString(interp->sysdict, "modules",
interp->modules);
@ -832,21 +969,22 @@ Py_NewInterpreter(void)
infrastructure for the io module in place. */
pstderr = PyFile_NewStdPrinter(fileno(stderr));
if (pstderr == NULL)
Py_FatalError("Py_Initialize: can't set preliminary stderr");
Py_FatalError("Py_NewInterpreter: can't set preliminary stderr");
_PySys_SetObjectId(&PyId_stderr, pstderr);
PySys_SetObject("__stderr__", pstderr);
Py_DECREF(pstderr);
_PyImportHooks_Init();
import_init(interp, sysmod);
initimport(interp, sysmod);
initexternalimport(interp);
if (initfsencoding(interp) < 0)
goto handle_error;
if (initstdio() < 0)
Py_FatalError(
"Py_Initialize: can't initialize sys standard streams");
"Py_NewInterpreter: can't initialize sys standard streams");
initmain(interp);
if (!Py_NoSiteFlag)
initsite();