From b2fc5492789623d656953d458f3eeaac03c1ef56 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 15 Feb 2023 15:32:31 -0700 Subject: [PATCH] gh-101758: Clean Up Uses of Import State (gh-101919) This change is almost entirely moving code around and hiding import state behind internal API. We introduce no changes to behavior, nor to non-internal API. (Since there was already going to be a lot of churn, I took this as an opportunity to re-organize import.c into topically-grouped sections of code.) The motivation is to simplify a number of upcoming changes. Specific changes: * move existing import-related code to import.c, wherever possible * add internal API for interacting with import state (both global and per-interpreter) * use only API outside of import.c (to limit churn there when changing the location, etc.) * consolidate the import-related state of PyInterpreterState into a single struct field (this changes layout slightly) * add macros for import state in import.c (to simplify changing the location) * group code in import.c into sections *remove _PyState_AddModule() https://github.com/python/cpython/issues/101758 --- Include/internal/pycore_import.h | 103 +- Include/internal/pycore_interp.h | 33 +- Include/internal/pycore_pylifecycle.h | 4 - Include/internal/pycore_pystate.h | 6 - Include/internal/pycore_runtime_init.h | 15 +- Include/internal/pycore_sysmodule.h | 3 + Lib/test/test_imp.py | 2 +- Lib/test/test_stable_abi_ctypes.py | 1 - Misc/stable_abi.toml | 3 - Objects/moduleobject.c | 25 +- PC/python3dll.c | 1 - Python/_warnings.c | 11 +- Python/ceval.c | 2 +- Python/dynload_shlib.c | 2 +- Python/import.c | 2032 +++++++++++++++--------- Python/importdl.c | 17 +- Python/pylifecycle.c | 127 +- Python/pystate.c | 151 +- Python/pythonrun.c | 10 +- Python/sysmodule.c | 40 +- 20 files changed, 1496 insertions(+), 1092 deletions(-) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 9036dff6725..da766253ef6 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -36,11 +36,112 @@ struct _import_runtime_state { const char * pkgcontext; }; +struct _import_state { + /* cached sys.modules dictionary */ + PyObject *modules; + /* This is the list of module objects for all legacy (single-phase init) + extension modules ever loaded in this process (i.e. imported + in this interpreter or in any other). Py_None stands in for + modules that haven't actually been imported in this interpreter. + + A module's index (PyModuleDef.m_base.m_index) is used to look up + the corresponding module object for this interpreter, if any. + (See PyState_FindModule().) When any extension module + is initialized during import, its moduledef gets initialized by + PyModuleDef_Init(), and the first time that happens for each + PyModuleDef, its index gets set to the current value of + a global counter (see _PyRuntimeState.imports.last_module_index). + The entry for that index in this interpreter remains unset until + the module is actually imported here. (Py_None is used as + a placeholder.) Note that multi-phase init modules always get + an index for which there will never be a module set. + + This is initialized lazily in PyState_AddModule(), which is also + where modules get added. */ + PyObject *modules_by_index; + /* importlib module._bootstrap */ + PyObject *importlib; + /* override for config->use_frozen_modules (for tests) + (-1: "off", 1: "on", 0: no override) */ + int override_frozen_modules; +#ifdef HAVE_DLOPEN + int dlopenflags; +#endif + PyObject *import_func; +}; + +#ifdef HAVE_DLOPEN +# include +# if HAVE_DECL_RTLD_NOW +# define _Py_DLOPEN_FLAGS RTLD_NOW +# else +# define _Py_DLOPEN_FLAGS RTLD_LAZY +# endif +# define DLOPENFLAGS_INIT .dlopenflags = _Py_DLOPEN_FLAGS, +#else +# define _Py_DLOPEN_FLAGS 0 +# define DLOPENFLAGS_INIT +#endif + +#define IMPORTS_INIT \ + { \ + .override_frozen_modules = 0, \ + DLOPENFLAGS_INIT \ + } + +extern void _PyImport_ClearCore(PyInterpreterState *interp); + +extern Py_ssize_t _PyImport_GetNextModuleIndex(void); +extern const char * _PyImport_ResolveNameWithPackageContext(const char *name); +extern const char * _PyImport_SwapPackageContext(const char *newcontext); + +extern int _PyImport_GetDLOpenFlags(PyInterpreterState *interp); +extern void _PyImport_SetDLOpenFlags(PyInterpreterState *interp, int new_val); + +extern PyObject * _PyImport_InitModules(PyInterpreterState *interp); +extern PyObject * _PyImport_GetModules(PyInterpreterState *interp); +extern void _PyImport_ClearModules(PyInterpreterState *interp); + +extern void _PyImport_ClearModulesByIndex(PyInterpreterState *interp); + +extern int _PyImport_InitDefaultImportFunc(PyInterpreterState *interp); +extern int _PyImport_IsDefaultImportFunc( + PyInterpreterState *interp, + PyObject *func); + +extern PyObject * _PyImport_GetImportlibLoader( + PyInterpreterState *interp, + const char *loader_name); +extern PyObject * _PyImport_GetImportlibExternalLoader( + PyInterpreterState *interp, + const char *loader_name); +extern PyObject * _PyImport_BlessMyLoader( + PyInterpreterState *interp, + PyObject *module_globals); +extern PyObject * _PyImport_ImportlibModuleRepr( + PyInterpreterState *interp, + PyObject *module); + + +extern PyStatus _PyImport_Init(void); +extern void _PyImport_Fini(void); +extern void _PyImport_Fini2(void); + +extern PyStatus _PyImport_InitCore( + PyThreadState *tstate, + PyObject *sysmod, + int importlib); +extern PyStatus _PyImport_InitExternal(PyThreadState *tstate); +extern void _PyImport_FiniCore(PyInterpreterState *interp); +extern void _PyImport_FiniExternal(PyInterpreterState *interp); + #ifdef HAVE_FORK extern PyStatus _PyImport_ReInitLock(void); #endif -extern PyObject* _PyImport_BootstrapImp(PyThreadState *tstate); + + +extern PyObject* _PyImport_GetBuiltinModuleNames(void); struct _module_alias { const char *name; /* ASCII encoded string */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 0e3d46852f2..60de31b336f 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -21,6 +21,7 @@ extern "C" { #include "pycore_function.h" // FUNC_MAX_WATCHERS #include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_gc.h" // struct _gc_runtime_state +#include "pycore_import.h" // struct _import_state #include "pycore_list.h" // struct _Py_list_state #include "pycore_global_objects.h" // struct _Py_interp_static_objects #include "pycore_tuple.h" // struct _Py_tuple_state @@ -92,37 +93,12 @@ struct _is { struct _ceval_state ceval; struct _gc_runtime_state gc; - // sys.modules dictionary - PyObject *modules; - /* This is the list of module objects for all legacy (single-phase init) - extension modules ever loaded in this process (i.e. imported - in this interpreter or in any other). Py_None stands in for - modules that haven't actually been imported in this interpreter. + struct _import_state imports; - A module's index (PyModuleDef.m_base.m_index) is used to look up - the corresponding module object for this interpreter, if any. - (See PyState_FindModule().) When any extension module - is initialized during import, its moduledef gets initialized by - PyModuleDef_Init(), and the first time that happens for each - PyModuleDef, its index gets set to the current value of - a global counter (see _PyRuntimeState.imports.last_module_index). - The entry for that index in this interpreter remains unset until - the module is actually imported here. (Py_None is used as - a placeholder.) Note that multi-phase init modules always get - an index for which there will never be a module set. - - This is initialized lazily in _PyState_AddModule(), which is also - where modules get added. */ - PyObject *modules_by_index; // Dictionary of the sys module PyObject *sysdict; // Dictionary of the builtins module PyObject *builtins; - // importlib module - PyObject *importlib; - // override for config->use_frozen_modules (for tests) - // (-1: "off", 1: "on", 0: no override) - int override_frozen_modules; PyObject *codec_search_path; PyObject *codec_search_cache; @@ -130,15 +106,11 @@ struct _is { int codecs_initialized; PyConfig config; -#ifdef HAVE_DLOPEN - int dlopenflags; -#endif unsigned long feature_flags; PyObject *dict; /* Stores per-interpreter state */ PyObject *builtins_copy; - PyObject *import_func; // Initialized to _PyEval_EvalFrameDefault(). _PyFrameEvalFunction eval_frame; @@ -205,7 +177,6 @@ struct _is { /* other API */ -extern void _PyInterpreterState_ClearModules(PyInterpreterState *interp); extern void _PyInterpreterState_Clear(PyThreadState *tstate); diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 2d431befd74..e7a31807205 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -30,7 +30,6 @@ PyAPI_FUNC(int) _Py_IsLocaleCoercionTarget(const char *ctype_loc); /* Various one-time initializers */ extern void _Py_InitVersion(void); -extern PyStatus _PyImport_Init(void); extern PyStatus _PyFaulthandler_Init(int enable); extern int _PyTraceMalloc_Init(int enable); extern PyObject * _PyBuiltin_Init(PyInterpreterState *interp); @@ -45,7 +44,6 @@ extern int _PyBuiltins_AddExceptions(PyObject * bltinmod); extern PyStatus _Py_HashRandomization_Init(const PyConfig *); extern PyStatus _PyTime_Init(void); -extern PyStatus _PyImportZip_Init(PyThreadState *tstate); extern PyStatus _PyGC_Init(PyInterpreterState *interp); extern PyStatus _PyAtExit_Init(PyInterpreterState *interp); extern int _Py_Deepfreeze_Init(void); @@ -55,8 +53,6 @@ extern int _Py_Deepfreeze_Init(void); extern int _PySignal_Init(int install_signal_handlers); extern void _PySignal_Fini(void); -extern void _PyImport_Fini(void); -extern void _PyImport_Fini2(void); extern void _PyGC_Fini(PyInterpreterState *interp); extern void _Py_HashRandomization_Fini(void); extern void _PyFaulthandler_Fini(void); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 7046ec8d9ad..638b8625387 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -152,12 +152,6 @@ extern void _PySignal_AfterFork(void); #endif -PyAPI_FUNC(int) _PyState_AddModule( - PyThreadState *tstate, - PyObject* module, - PyModuleDef* def); - - PyAPI_FUNC(int) _PyOS_InterruptOccurred(PyThreadState *tstate); #define HEAD_LOCK(runtime) \ diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index c6a27d076ea..a8d5953ff98 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -97,23 +97,10 @@ extern "C" { ._main_interpreter = _PyInterpreterState_INIT, \ } -#ifdef HAVE_DLOPEN -# include -# if HAVE_DECL_RTLD_NOW -# define _Py_DLOPEN_FLAGS RTLD_NOW -# else -# define _Py_DLOPEN_FLAGS RTLD_LAZY -# endif -# define DLOPENFLAGS_INIT .dlopenflags = _Py_DLOPEN_FLAGS, -#else -# define _Py_DLOPEN_FLAGS 0 -# define DLOPENFLAGS_INIT -#endif - #define _PyInterpreterState_INIT \ { \ .id_refcount = -1, \ - DLOPENFLAGS_INIT \ + .imports = IMPORTS_INIT, \ .ceval = { \ .recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \ }, \ diff --git a/Include/internal/pycore_sysmodule.h b/Include/internal/pycore_sysmodule.h index 10d092cdc30..b4b1febafa4 100644 --- a/Include/internal/pycore_sysmodule.h +++ b/Include/internal/pycore_sysmodule.h @@ -20,6 +20,9 @@ extern void _PySys_ClearAuditHooks(PyThreadState *tstate); PyAPI_FUNC(int) _PySys_SetAttr(PyObject *, PyObject *); +extern int _PySys_ClearAttrString(PyInterpreterState *interp, + const char *name, int verbose); + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 31dce21587e..c85ab92307d 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -387,7 +387,7 @@ class ImportTests(unittest.TestCase): check_basic_reloaded(mod, lookedup, initialized, init_count, before, reloaded) - # Currently _PyState_AddModule() always replaces the cached module. + # Currently PyState_AddModule() always replaces the cached module. self.assertIs(basic.look_up_self(), mod) self.assertEqual(basic.initialized_count(), expected_init_count) diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index e77c1c84098..7e50fbda2c0 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -864,7 +864,6 @@ SYMBOL_NAMES = ( "_PyObject_GC_Resize", "_PyObject_New", "_PyObject_NewVar", - "_PyState_AddModule", "_PyThreadState_Init", "_PyThreadState_Prealloc", "_PyWeakref_CallableProxyType", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 21ff9616133..c04a3a228ca 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -1684,9 +1684,6 @@ [function._PyObject_NewVar] added = '3.2' abi_only = true -[function._PyState_AddModule] - added = '3.2' - abi_only = true [function._PyThreadState_Init] added = '3.2' abi_only = true diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 24190e320ee..a0be19a3ca8 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -42,10 +42,9 @@ PyModuleDef_Init(PyModuleDef* def) { assert(PyModuleDef_Type.tp_flags & Py_TPFLAGS_READY); if (def->m_base.m_index == 0) { - _PyRuntime.imports.last_module_index++; Py_SET_REFCNT(def, 1); Py_SET_TYPE(def, &PyModuleDef_Type); - def->m_base.m_index = _PyRuntime.imports.last_module_index; + def->m_base.m_index = _PyImport_GetNextModuleIndex(); } return (PyObject*)def; } @@ -209,24 +208,7 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) "module %s: PyModule_Create is incompatible with m_slots", name); return NULL; } - /* Make sure name is fully qualified. - - This is a bit of a hack: when the shared library is loaded, - the module name is "package.module", but the module calls - PyModule_Create*() with just "module" for the name. The shared - library loader squirrels away the true name of the module in - _Py_PackageContext, and PyModule_Create*() will substitute this - (if the name actually matches). - */ -#define _Py_PackageContext (_PyRuntime.imports.pkgcontext) - if (_Py_PackageContext != NULL) { - const char *p = strrchr(_Py_PackageContext, '.'); - if (p != NULL && strcmp(module->m_name, p+1) == 0) { - name = _Py_PackageContext; - _Py_PackageContext = NULL; - } - } -#undef _Py_PackageContext + name = _PyImport_ResolveNameWithPackageContext(name); if ((m = (PyModuleObject*)PyModule_New(name)) == NULL) return NULL; @@ -710,8 +692,7 @@ static PyObject * module_repr(PyModuleObject *m) { PyInterpreterState *interp = _PyInterpreterState_GET(); - - return PyObject_CallMethod(interp->importlib, "_module_repr", "O", m); + return _PyImport_ImportlibModuleRepr(interp, (PyObject *)m); } /* Check if the "_initializing" attribute of the module spec is set to true. diff --git a/PC/python3dll.c b/PC/python3dll.c index e3008193657..79f09037282 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -34,7 +34,6 @@ EXPORT_FUNC(_PyObject_GC_NewVar) EXPORT_FUNC(_PyObject_GC_Resize) EXPORT_FUNC(_PyObject_New) EXPORT_FUNC(_PyObject_NewVar) -EXPORT_FUNC(_PyState_AddModule) EXPORT_FUNC(_PyThreadState_Init) EXPORT_FUNC(_PyThreadState_Prealloc) EXPORT_FUNC(Py_AddPendingCall) diff --git a/Python/_warnings.c b/Python/_warnings.c index e78f21644f3..d510381c365 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -214,7 +214,7 @@ get_warnings_attr(PyInterpreterState *interp, PyObject *attr, int try_import) gone, then we can't even use PyImport_GetModule without triggering an interpreter abort. */ - if (!interp->modules) { + if (!_PyImport_GetModules(interp)) { return NULL; } warnings_module = PyImport_GetModule(&_Py_ID(warnings)); @@ -1050,7 +1050,6 @@ warnings_warn_impl(PyObject *module, PyObject *message, PyObject *category, static PyObject * get_source_line(PyInterpreterState *interp, PyObject *module_globals, int lineno) { - PyObject *external; PyObject *loader; PyObject *module_name; PyObject *get_source; @@ -1059,13 +1058,7 @@ get_source_line(PyInterpreterState *interp, PyObject *module_globals, int lineno PyObject *source_line; /* stolen from import.c */ - external = PyObject_GetAttrString(interp->importlib, "_bootstrap_external"); - if (external == NULL) { - return NULL; - } - - loader = PyObject_CallMethod(external, "_bless_my_loader", "O", module_globals, NULL); - Py_DECREF(external); + loader = _PyImport_BlessMyLoader(interp, module_globals); if (loader == NULL) { return NULL; } diff --git a/Python/ceval.c b/Python/ceval.c index 611d62b0eba..09fd2f29266 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2688,7 +2688,7 @@ import_name(PyThreadState *tstate, _PyInterpreterFrame *frame, } PyObject *locals = frame->f_locals; /* Fast path for not overloaded __import__. */ - if (import_func == tstate->interp->import_func) { + if (_PyImport_IsDefaultImportFunc(tstate->interp, import_func)) { int ilevel = _PyLong_AsInt(level); if (ilevel == -1 && _PyErr_Occurred(tstate)) { return NULL; diff --git a/Python/dynload_shlib.c b/Python/dynload_shlib.c index 3c5fd83df58..6761bba4579 100644 --- a/Python/dynload_shlib.c +++ b/Python/dynload_shlib.c @@ -75,7 +75,7 @@ _PyImport_FindSharedFuncptr(const char *prefix, return NULL; } - dlopenflags = _PyInterpreterState_GET()->dlopenflags; + dlopenflags = _PyImport_GetDLOpenFlags(_PyInterpreterState_GET()); handle = dlopen(pathname, dlopenflags); diff --git a/Python/import.c b/Python/import.c index 63ed2443657..ae27aaf5684 100644 --- a/Python/import.c +++ b/Python/import.c @@ -4,7 +4,7 @@ #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() -#include "pycore_interp.h" // _PyInterpreterState_ClearModules() +#include "pycore_interp.h" // struct _import_runtime_state #include "pycore_namespace.h" // _PyNamespace_Type #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_pyhash.h" // _Py_KeyedHash() @@ -24,8 +24,18 @@ extern "C" { #endif -/* Forward references */ -static PyObject *import_add_module(PyThreadState *tstate, PyObject *name); + +/*[clinic input] +module _imp +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=9c332475d8686284]*/ + +#include "clinic/import.c.h" + + +/*******************************/ +/* process-global import state */ +/*******************************/ /* This table is defined in config.c: */ extern struct _inittab _PyImport_Inittab[]; @@ -37,67 +47,51 @@ struct _inittab *PyImport_Inittab = _PyImport_Inittab; // we track the pointer here so we can deallocate it during finalization. static struct _inittab *inittab_copy = NULL; -/*[clinic input] -module _imp -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=9c332475d8686284]*/ -#include "clinic/import.c.h" +/*******************************/ +/* runtime-global import state */ +/*******************************/ -/* Initialize things */ - -PyStatus -_PyImportZip_Init(PyThreadState *tstate) -{ - PyObject *path_hooks; - int err = 0; - - path_hooks = PySys_GetObject("path_hooks"); - if (path_hooks == NULL) { - _PyErr_SetString(tstate, PyExc_RuntimeError, - "unable to get sys.path_hooks"); - goto error; - } - - int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose; - if (verbose) { - PySys_WriteStderr("# installing zipimport hook\n"); - } - - PyObject *zipimporter = _PyImport_GetModuleAttrString("zipimport", "zipimporter"); - if (zipimporter == NULL) { - _PyErr_Clear(tstate); /* No zipimporter object -- okay */ - if (verbose) { - PySys_WriteStderr("# can't import zipimport.zipimporter\n"); - } - } - else { - /* sys.path_hooks.insert(0, zipimporter) */ - err = PyList_Insert(path_hooks, 0, zipimporter); - Py_DECREF(zipimporter); - if (err < 0) { - goto error; - } - if (verbose) { - PySys_WriteStderr("# installed zipimport hook\n"); - } - } - - return _PyStatus_OK(); - - error: - PyErr_Print(); - return _PyStatus_ERR("initializing zipimport failed"); -} - -/* Locking primitives to prevent parallel imports of the same module - in different threads to return with a partially loaded module. - These calls are serialized by the global interpreter lock. */ +#define INITTAB _PyRuntime.imports.inittab +#define LAST_MODULE_INDEX _PyRuntime.imports.last_module_index +#define EXTENSIONS _PyRuntime.imports.extensions #define import_lock _PyRuntime.imports.lock.mutex #define import_lock_thread _PyRuntime.imports.lock.thread #define import_lock_level _PyRuntime.imports.lock.level +#define FIND_AND_LOAD _PyRuntime.imports.find_and_load +#define PKGCONTEXT (_PyRuntime.imports.pkgcontext) + + +/*******************************/ +/* interpreter import state */ +/*******************************/ + +#define MODULES(interp) \ + (interp)->imports.modules +#define MODULES_BY_INDEX(interp) \ + (interp)->imports.modules_by_index +#define IMPORTLIB(interp) \ + (interp)->imports.importlib +#define OVERRIDE_FROZEN_MODULES(interp) \ + (interp)->imports.override_frozen_modules +#ifdef HAVE_DLOPEN +# define DLOPENFLAGS(interp) \ + (interp)->imports.dlopenflags +#endif +#define IMPORT_FUNC(interp) \ + (interp)->imports.import_func + + +/*******************/ +/* the import lock */ +/*******************/ + +/* Locking primitives to prevent parallel imports of the same module + in different threads to return with a partially loaded module. + These calls are serialized by the global interpreter lock. */ + void _PyImport_AcquireLock(void) { @@ -170,154 +164,45 @@ _PyImport_ReInitLock(void) } #endif -/*[clinic input] -_imp.lock_held -Return True if the import lock is currently held, else False. +/***************/ +/* sys.modules */ +/***************/ -On platforms without threads, return False. -[clinic start generated code]*/ - -static PyObject * -_imp_lock_held_impl(PyObject *module) -/*[clinic end generated code: output=8b89384b5e1963fc input=9b088f9b217d9bdf]*/ +PyObject * +_PyImport_InitModules(PyInterpreterState *interp) { - return PyBool_FromLong(import_lock_thread != PYTHREAD_INVALID_THREAD_ID); -} - -/*[clinic input] -_imp.acquire_lock - -Acquires the interpreter's import lock for the current thread. - -This lock should be used by import hooks to ensure thread-safety when importing -modules. On platforms without threads, this function does nothing. -[clinic start generated code]*/ - -static PyObject * -_imp_acquire_lock_impl(PyObject *module) -/*[clinic end generated code: output=1aff58cb0ee1b026 input=4a2d4381866d5fdc]*/ -{ - _PyImport_AcquireLock(); - Py_RETURN_NONE; -} - -/*[clinic input] -_imp.release_lock - -Release the interpreter's import lock. - -On platforms without threads, this function does nothing. -[clinic start generated code]*/ - -static PyObject * -_imp_release_lock_impl(PyObject *module) -/*[clinic end generated code: output=7faab6d0be178b0a input=934fb11516dd778b]*/ -{ - if (_PyImport_ReleaseLock() < 0) { - PyErr_SetString(PyExc_RuntimeError, - "not holding the import lock"); + assert(MODULES(interp) == NULL); + MODULES(interp) = PyDict_New(); + if (MODULES(interp) == NULL) { return NULL; } - Py_RETURN_NONE; + return MODULES(interp); } -PyStatus -_PyImport_Init(void) +PyObject * +_PyImport_GetModules(PyInterpreterState *interp) { - if (_PyRuntime.imports.inittab != NULL) { - return _PyStatus_ERR("global import state already initialized"); - } - PyStatus status = _PyStatus_OK(); - - size_t size; - for (size = 0; PyImport_Inittab[size].name != NULL; size++) - ; - size++; - - /* Force default raw memory allocator to get a known allocator to be able - to release the memory in _PyImport_Fini() */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - /* Make the copy. */ - struct _inittab *copied = PyMem_RawMalloc(size * sizeof(struct _inittab)); - if (copied == NULL) { - status = PyStatus_NoMemory(); - goto done; - } - memcpy(copied, PyImport_Inittab, size * sizeof(struct _inittab)); - _PyRuntime.imports.inittab = copied; - -done: - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - return status; -} - -static inline void _extensions_cache_clear(void); - -void -_PyImport_Fini(void) -{ - _extensions_cache_clear(); - if (import_lock != NULL) { - PyThread_free_lock(import_lock); - import_lock = NULL; - } - - /* Use the same memory allocator as _PyImport_Init(). */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - /* Free memory allocated by _PyImport_Init() */ - struct _inittab *inittab = _PyRuntime.imports.inittab; - _PyRuntime.imports.inittab = NULL; - PyMem_RawFree(inittab); - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + return MODULES(interp); } void -_PyImport_Fini2(void) +_PyImport_ClearModules(PyInterpreterState *interp) { - /* Use the same memory allocator than PyImport_ExtendInittab(). */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - // Reset PyImport_Inittab - PyImport_Inittab = _PyImport_Inittab; - - /* Free memory allocated by PyImport_ExtendInittab() */ - PyMem_RawFree(inittab_copy); - inittab_copy = NULL; - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + Py_SETREF(MODULES(interp), NULL); } -/* Helper for sys */ - PyObject * PyImport_GetModuleDict(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->modules == NULL) { + if (MODULES(interp) == NULL) { Py_FatalError("interpreter has no modules dictionary"); } - return interp->modules; -} - -/* In some corner cases it is important to be sure that the import - machinery has been initialized (or not cleaned up yet). For - example, see issue #4236 and PyModule_Create2(). */ - -int -_PyImport_IsInitialized(PyInterpreterState *interp) -{ - if (interp->modules == NULL) - return 0; - return 1; + return MODULES(interp); } +// This is only kept around for extensions that use _Py_IDENTIFIER. PyObject * _PyImport_GetModuleId(_Py_Identifier *nameid) { @@ -332,7 +217,7 @@ int _PyImport_SetModule(PyObject *name, PyObject *m) { PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *modules = interp->modules; + PyObject *modules = MODULES(interp); return PyObject_SetItem(modules, name, m); } @@ -340,14 +225,14 @@ int _PyImport_SetModuleString(const char *name, PyObject *m) { PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *modules = interp->modules; + PyObject *modules = MODULES(interp); return PyMapping_SetItemString(modules, name, m); } static PyObject * import_get_module(PyThreadState *tstate, PyObject *name) { - PyObject *modules = tstate->interp->modules; + PyObject *modules = MODULES(tstate->interp); if (modules == NULL) { _PyErr_SetString(tstate, PyExc_RuntimeError, "unable to get sys.modules"); @@ -370,7 +255,6 @@ import_get_module(PyThreadState *tstate, PyObject *name) return m; } - static int import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *name) { @@ -387,7 +271,7 @@ import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *n if (busy) { /* Wait until module is done importing. */ PyObject *value = _PyObject_CallMethodOneArg( - interp->importlib, &_Py_ID(_lock_unlock_module), name); + IMPORTLIB(interp), &_Py_ID(_lock_unlock_module), name); if (value == NULL) { return -1; } @@ -396,37 +280,371 @@ import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *n return 0; } +static void remove_importlib_frames(PyThreadState *tstate); -/* Helper for pythonrun.c -- return magic number and tag. */ - -long -PyImport_GetMagicNumber(void) +PyObject * +PyImport_GetModule(PyObject *name) { - long res; - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *external, *pyc_magic; + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *mod; - external = PyObject_GetAttrString(interp->importlib, "_bootstrap_external"); - if (external == NULL) - return -1; - pyc_magic = PyObject_GetAttrString(external, "_RAW_MAGIC_NUMBER"); - Py_DECREF(external); - if (pyc_magic == NULL) - return -1; - res = PyLong_AsLong(pyc_magic); - Py_DECREF(pyc_magic); - return res; + mod = import_get_module(tstate, name); + if (mod != NULL && mod != Py_None) { + if (import_ensure_initialized(tstate->interp, mod, name) < 0) { + Py_DECREF(mod); + remove_importlib_frames(tstate); + return NULL; + } + } + return mod; +} + +/* Get the module object corresponding to a module name. + First check the modules dictionary if there's one there, + if not, create a new one and insert it in the modules dictionary. */ + +static PyObject * +import_add_module(PyThreadState *tstate, PyObject *name) +{ + PyObject *modules = MODULES(tstate->interp); + if (modules == NULL) { + _PyErr_SetString(tstate, PyExc_RuntimeError, + "no import module dictionary"); + return NULL; + } + + PyObject *m; + if (PyDict_CheckExact(modules)) { + m = Py_XNewRef(PyDict_GetItemWithError(modules, name)); + } + else { + m = PyObject_GetItem(modules, name); + // For backward-compatibility we copy the behavior + // of PyDict_GetItemWithError(). + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + _PyErr_Clear(tstate); + } + } + if (_PyErr_Occurred(tstate)) { + return NULL; + } + if (m != NULL && PyModule_Check(m)) { + return m; + } + Py_XDECREF(m); + m = PyModule_NewObject(name); + if (m == NULL) + return NULL; + if (PyObject_SetItem(modules, name, m) != 0) { + Py_DECREF(m); + return NULL; + } + + return m; +} + +PyObject * +PyImport_AddModuleObject(PyObject *name) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *mod = import_add_module(tstate, name); + if (mod) { + PyObject *ref = PyWeakref_NewRef(mod, NULL); + Py_DECREF(mod); + if (ref == NULL) { + return NULL; + } + mod = PyWeakref_GetObject(ref); + Py_DECREF(ref); + } + return mod; /* borrowed reference */ } -extern const char * _PySys_ImplCacheTag; +PyObject * +PyImport_AddModule(const char *name) +{ + PyObject *nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) { + return NULL; + } + PyObject *module = PyImport_AddModuleObject(nameobj); + Py_DECREF(nameobj); + return module; +} + + +/* Remove name from sys.modules, if it's there. + * Can be called with an exception raised. + * If fail to remove name a new exception will be chained with the old + * exception, otherwise the old exception is preserved. + */ +static void +remove_module(PyThreadState *tstate, PyObject *name) +{ + PyObject *type, *value, *traceback; + _PyErr_Fetch(tstate, &type, &value, &traceback); + + PyObject *modules = MODULES(tstate->interp); + if (PyDict_CheckExact(modules)) { + PyObject *mod = _PyDict_Pop(modules, name, Py_None); + Py_XDECREF(mod); + } + else if (PyMapping_DelItem(modules, name) < 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + _PyErr_Clear(tstate); + } + } + + _PyErr_ChainExceptions(type, value, traceback); +} + + +/************************************/ +/* per-interpreter modules-by-index */ +/************************************/ + +Py_ssize_t +_PyImport_GetNextModuleIndex(void) +{ + LAST_MODULE_INDEX++; + return LAST_MODULE_INDEX; +} + +static const char * +_modules_by_index_check(PyInterpreterState *interp, Py_ssize_t index) +{ + if (index == 0) { + return "invalid module index"; + } + if (MODULES_BY_INDEX(interp) == NULL) { + return "Interpreters module-list not accessible."; + } + if (index > PyList_GET_SIZE(MODULES_BY_INDEX(interp))) { + return "Module index out of bounds."; + } + return NULL; +} + +static PyObject * +_modules_by_index_get(PyInterpreterState *interp, PyModuleDef *def) +{ + Py_ssize_t index = def->m_base.m_index; + if (_modules_by_index_check(interp, index) != NULL) { + return NULL; + } + PyObject *res = PyList_GET_ITEM(MODULES_BY_INDEX(interp), index); + return res==Py_None ? NULL : res; +} + +static int +_modules_by_index_set(PyInterpreterState *interp, + PyModuleDef *def, PyObject *module) +{ + assert(def != NULL); + assert(def->m_slots == NULL); + assert(def->m_base.m_index > 0); + + if (MODULES_BY_INDEX(interp) == NULL) { + MODULES_BY_INDEX(interp) = PyList_New(0); + if (MODULES_BY_INDEX(interp) == NULL) { + return -1; + } + } + + Py_ssize_t index = def->m_base.m_index; + while (PyList_GET_SIZE(MODULES_BY_INDEX(interp)) <= index) { + if (PyList_Append(MODULES_BY_INDEX(interp), Py_None) < 0) { + return -1; + } + } + + return PyList_SetItem(MODULES_BY_INDEX(interp), index, Py_NewRef(module)); +} + +static int +_modules_by_index_clear(PyInterpreterState *interp, PyModuleDef *def) +{ + Py_ssize_t index = def->m_base.m_index; + const char *err = _modules_by_index_check(interp, index); + if (err != NULL) { + Py_FatalError(err); + return -1; + } + return PyList_SetItem(MODULES_BY_INDEX(interp), index, Py_NewRef(Py_None)); +} + + +PyObject* +PyState_FindModule(PyModuleDef* module) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (module->m_slots) { + return NULL; + } + return _modules_by_index_get(interp, module); +} + +int +PyState_AddModule(PyObject* module, PyModuleDef* def) +{ + if (!def) { + Py_FatalError("module definition is NULL"); + return -1; + } + + PyThreadState *tstate = _PyThreadState_GET(); + if (def->m_slots) { + _PyErr_SetString(tstate, + PyExc_SystemError, + "PyState_AddModule called on module with slots"); + return -1; + } + + PyInterpreterState *interp = tstate->interp; + Py_ssize_t index = def->m_base.m_index; + if (MODULES_BY_INDEX(interp) && + index < PyList_GET_SIZE(MODULES_BY_INDEX(interp)) && + module == PyList_GET_ITEM(MODULES_BY_INDEX(interp), index)) + { + _Py_FatalErrorFormat(__func__, "module %p already added", module); + return -1; + } + + return _modules_by_index_set(interp, def, module); +} + +int +PyState_RemoveModule(PyModuleDef* def) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (def->m_slots) { + _PyErr_SetString(tstate, + PyExc_SystemError, + "PyState_RemoveModule called on module with slots"); + return -1; + } + return _modules_by_index_clear(tstate->interp, def); +} + + +// Used by finalize_modules() +void +_PyImport_ClearModulesByIndex(PyInterpreterState *interp) +{ + if (!MODULES_BY_INDEX(interp)) { + return; + } + + Py_ssize_t i; + for (i = 0; i < PyList_GET_SIZE(MODULES_BY_INDEX(interp)); i++) { + PyObject *m = PyList_GET_ITEM(MODULES_BY_INDEX(interp), i); + if (PyModule_Check(m)) { + /* cleanup the saved copy of module dicts */ + PyModuleDef *md = PyModule_GetDef(m); + if (md) { + Py_CLEAR(md->m_base.m_copy); + } + } + } + + /* Setting modules_by_index to NULL could be dangerous, so we + clear the list instead. */ + if (PyList_SetSlice(MODULES_BY_INDEX(interp), + 0, PyList_GET_SIZE(MODULES_BY_INDEX(interp)), + NULL)) { + PyErr_WriteUnraisable(MODULES_BY_INDEX(interp)); + } +} + + +/*********************/ +/* extension modules */ +/*********************/ + +/* Make sure name is fully qualified. + + This is a bit of a hack: when the shared library is loaded, + the module name is "package.module", but the module calls + PyModule_Create*() with just "module" for the name. The shared + library loader squirrels away the true name of the module in + _PyRuntime.imports.pkgcontext, and PyModule_Create*() will + substitute this (if the name actually matches). +*/ +const char * +_PyImport_ResolveNameWithPackageContext(const char *name) +{ + if (PKGCONTEXT != NULL) { + const char *p = strrchr(PKGCONTEXT, '.'); + if (p != NULL && strcmp(name, p+1) == 0) { + name = PKGCONTEXT; + PKGCONTEXT = NULL; + } + } + return name; +} const char * -PyImport_GetMagicTag(void) +_PyImport_SwapPackageContext(const char *newcontext) { - return _PySys_ImplCacheTag; + const char *oldcontext = PKGCONTEXT; + PKGCONTEXT = newcontext; + return oldcontext; } +#ifdef HAVE_DLOPEN +int +_PyImport_GetDLOpenFlags(PyInterpreterState *interp) +{ + return DLOPENFLAGS(interp); +} + +void +_PyImport_SetDLOpenFlags(PyInterpreterState *interp, int new_val) +{ + DLOPENFLAGS(interp) = new_val; +} +#endif // HAVE_DLOPEN + + +/* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */ +static int +exec_builtin_or_dynamic(PyObject *mod) { + PyModuleDef *def; + void *state; + + if (!PyModule_Check(mod)) { + return 0; + } + + def = PyModule_GetDef(mod); + if (def == NULL) { + return 0; + } + + state = PyModule_GetState(mod); + if (state) { + /* Already initialized; skip reload */ + return 0; + } + + return PyModule_ExecDef(mod, def); +} + + +/*******************/ + +#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) +#include +EM_JS(PyObject*, _PyImport_InitFunc_TrampolineCall, (PyModInitFunction func), { + return wasmTable.get(func)(); +}); +#endif // __EMSCRIPTEN__ && PY_CALL_TRAMPOLINE + + +/*****************************/ +/* single-phase init modules */ +/*****************************/ /* We support a number of kinds of single-phase init builtin/extension modules: @@ -512,7 +730,7 @@ gets even messier. static PyModuleDef * _extensions_cache_get(PyObject *filename, PyObject *name) { - PyObject *extensions = _PyRuntime.imports.extensions; + PyObject *extensions = EXTENSIONS; if (extensions == NULL) { return NULL; } @@ -528,13 +746,13 @@ _extensions_cache_get(PyObject *filename, PyObject *name) static int _extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def) { - PyObject *extensions = _PyRuntime.imports.extensions; + PyObject *extensions = EXTENSIONS; if (extensions == NULL) { extensions = PyDict_New(); if (extensions == NULL) { return -1; } - _PyRuntime.imports.extensions = extensions; + EXTENSIONS = extensions; } PyObject *key = PyTuple_Pack(2, filename, name); if (key == NULL) { @@ -551,7 +769,7 @@ _extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def) static void _extensions_cache_clear(void) { - Py_CLEAR(_PyRuntime.imports.extensions); + Py_CLEAR(EXTENSIONS); } static int @@ -569,7 +787,7 @@ fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) } PyThreadState *tstate = _PyThreadState_GET(); - if (_PyState_AddModule(tstate, mod, def) < 0) { + if (_modules_by_index_set(tstate->interp, def, mod) < 0) { return -1; } @@ -616,40 +834,19 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, return 0; } -int -_PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules) -{ - int res = -1; - PyObject *nameobj; - nameobj = PyUnicode_InternFromString(name); - if (nameobj == NULL) { - return -1; - } - if (PyObject_SetItem(modules, nameobj, mod) < 0) { - goto finally; - } - if (fix_up_extension(mod, nameobj, nameobj) < 0) { - PyMapping_DelItem(modules, nameobj); - goto finally; - } - res = 0; - -finally: - Py_DECREF(nameobj); - return res; -} static PyObject * import_find_extension(PyThreadState *tstate, PyObject *name, PyObject *filename) { + /* Only single-phase init modules will be in the cache. */ PyModuleDef *def = _extensions_cache_get(filename, name); if (def == NULL) { return NULL; } PyObject *mod, *mdict; - PyObject *modules = tstate->interp->modules; + PyObject *modules = MODULES(tstate->interp); if (def->m_size == -1) { /* Module does not support repeated initialization */ @@ -679,7 +876,7 @@ import_find_extension(PyThreadState *tstate, PyObject *name, return NULL; } } - if (_PyState_AddModule(tstate, mod, def) < 0) { + if (_modules_by_index_set(tstate->interp, def, mod) < 0) { PyMapping_DelItem(modules, name); Py_DECREF(mod); return NULL; @@ -694,107 +891,268 @@ import_find_extension(PyThreadState *tstate, PyObject *name, } -/* Get the module object corresponding to a module name. - First check the modules dictionary if there's one there, - if not, create a new one and insert it in the modules dictionary. */ +/*******************/ +/* builtin modules */ +/*******************/ -static PyObject * -import_add_module(PyThreadState *tstate, PyObject *name) +int +_PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules) { - PyObject *modules = tstate->interp->modules; - if (modules == NULL) { - _PyErr_SetString(tstate, PyExc_RuntimeError, - "no import module dictionary"); - return NULL; + int res = -1; + PyObject *nameobj; + nameobj = PyUnicode_InternFromString(name); + if (nameobj == NULL) { + return -1; } + if (PyObject_SetItem(modules, nameobj, mod) < 0) { + goto finally; + } + if (fix_up_extension(mod, nameobj, nameobj) < 0) { + PyMapping_DelItem(modules, nameobj); + goto finally; + } + res = 0; - PyObject *m; - if (PyDict_CheckExact(modules)) { - m = Py_XNewRef(PyDict_GetItemWithError(modules, name)); - } - else { - m = PyObject_GetItem(modules, name); - // For backward-compatibility we copy the behavior - // of PyDict_GetItemWithError(). - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - _PyErr_Clear(tstate); +finally: + Py_DECREF(nameobj); + return res; +} + +/* Helper to test for built-in module */ + +static int +is_builtin(PyObject *name) +{ + int i; + struct _inittab *inittab = INITTAB; + for (i = 0; inittab[i].name != NULL; i++) { + if (_PyUnicode_EqualToASCIIString(name, inittab[i].name)) { + if (inittab[i].initfunc == NULL) + return -1; + else + return 1; } } - if (_PyErr_Occurred(tstate)) { - return NULL; - } - if (m != NULL && PyModule_Check(m)) { - return m; - } - Py_XDECREF(m); - m = PyModule_NewObject(name); - if (m == NULL) - return NULL; - if (PyObject_SetItem(modules, name, m) != 0) { - Py_DECREF(m); - return NULL; + return 0; +} + +static PyObject* +create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) +{ + PyObject *mod = import_find_extension(tstate, name, name); + if (mod || _PyErr_Occurred(tstate)) { + return mod; } - return m; + PyObject *modules = MODULES(tstate->interp); + for (struct _inittab *p = INITTAB; p->name != NULL; p++) { + if (_PyUnicode_EqualToASCIIString(name, p->name)) { + if (p->initfunc == NULL) { + /* Cannot re-init internal module ("sys" or "builtins") */ + mod = PyImport_AddModuleObject(name); + return Py_XNewRef(mod); + } + mod = _PyImport_InitFunc_TrampolineCall(*p->initfunc); + if (mod == NULL) { + return NULL; + } + + if (PyObject_TypeCheck(mod, &PyModuleDef_Type)) { + return PyModule_FromDefAndSpec((PyModuleDef*)mod, spec); + } + else { + /* Remember pointer to module init function. */ + PyModuleDef *def = PyModule_GetDef(mod); + if (def == NULL) { + return NULL; + } + + def->m_base.m_init = p->initfunc; + if (_PyImport_FixupExtensionObject(mod, name, name, + modules) < 0) { + return NULL; + } + return mod; + } + } + } + + // not found + Py_RETURN_NONE; +} + + +/*****************************/ +/* the builtin modules table */ +/*****************************/ + +/* API for embedding applications that want to add their own entries + to the table of built-in modules. This should normally be called + *before* Py_Initialize(). When the table resize fails, -1 is + returned and the existing table is unchanged. + + After a similar function by Just van Rossum. */ + +int +PyImport_ExtendInittab(struct _inittab *newtab) +{ + struct _inittab *p; + size_t i, n; + int res = 0; + + if (INITTAB != NULL) { + Py_FatalError("PyImport_ExtendInittab() may not be called after Py_Initialize()"); + } + + /* Count the number of entries in both tables */ + for (n = 0; newtab[n].name != NULL; n++) + ; + if (n == 0) + return 0; /* Nothing to do */ + for (i = 0; PyImport_Inittab[i].name != NULL; i++) + ; + + /* Force default raw memory allocator to get a known allocator to be able + to release the memory in _PyImport_Fini2() */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + /* Allocate new memory for the combined table */ + p = NULL; + if (i + n <= SIZE_MAX / sizeof(struct _inittab) - 1) { + size_t size = sizeof(struct _inittab) * (i + n + 1); + p = PyMem_RawRealloc(inittab_copy, size); + } + if (p == NULL) { + res = -1; + goto done; + } + + /* Copy the tables into the new memory at the first call + to PyImport_ExtendInittab(). */ + if (inittab_copy != PyImport_Inittab) { + memcpy(p, PyImport_Inittab, (i+1) * sizeof(struct _inittab)); + } + memcpy(p + i, newtab, (n + 1) * sizeof(struct _inittab)); + PyImport_Inittab = inittab_copy = p; + +done: + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + return res; +} + +/* Shorthand to add a single entry given a name and a function */ + +int +PyImport_AppendInittab(const char *name, PyObject* (*initfunc)(void)) +{ + struct _inittab newtab[2]; + + if (INITTAB != NULL) { + Py_FatalError("PyImport_AppendInittab() may not be called after Py_Initialize()"); + } + + memset(newtab, '\0', sizeof newtab); + + newtab[0].name = name; + newtab[0].initfunc = initfunc; + + return PyImport_ExtendInittab(newtab); +} + + +/* the internal table */ + +static int +init_builtin_modules_table(void) +{ + size_t size; + for (size = 0; PyImport_Inittab[size].name != NULL; size++) + ; + size++; + + /* Make the copy. */ + struct _inittab *copied = PyMem_RawMalloc(size * sizeof(struct _inittab)); + if (copied == NULL) { + return -1; + } + memcpy(copied, PyImport_Inittab, size * sizeof(struct _inittab)); + INITTAB = copied; + return 0; +} + +static void +fini_builtin_modules_table(void) +{ + struct _inittab *inittab = INITTAB; + INITTAB = NULL; + PyMem_RawFree(inittab); } PyObject * -PyImport_AddModuleObject(PyObject *name) +_PyImport_GetBuiltinModuleNames(void) { - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *mod = import_add_module(tstate, name); - if (mod) { - PyObject *ref = PyWeakref_NewRef(mod, NULL); - Py_DECREF(mod); - if (ref == NULL) { + PyObject *list = PyList_New(0); + if (list == NULL) { + return NULL; + } + struct _inittab *inittab = INITTAB; + for (Py_ssize_t i = 0; inittab[i].name != NULL; i++) { + PyObject *name = PyUnicode_FromString(inittab[i].name); + if (name == NULL) { + Py_DECREF(list); return NULL; } - mod = PyWeakref_GetObject(ref); - Py_DECREF(ref); - } - return mod; /* borrowed reference */ -} - - -PyObject * -PyImport_AddModule(const char *name) -{ - PyObject *nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) { - return NULL; - } - PyObject *module = PyImport_AddModuleObject(nameobj); - Py_DECREF(nameobj); - return module; -} - - -/* Remove name from sys.modules, if it's there. - * Can be called with an exception raised. - * If fail to remove name a new exception will be chained with the old - * exception, otherwise the old exception is preserved. - */ -static void -remove_module(PyThreadState *tstate, PyObject *name) -{ - PyObject *type, *value, *traceback; - _PyErr_Fetch(tstate, &type, &value, &traceback); - - PyObject *modules = tstate->interp->modules; - if (PyDict_CheckExact(modules)) { - PyObject *mod = _PyDict_Pop(modules, name, Py_None); - Py_XDECREF(mod); - } - else if (PyMapping_DelItem(modules, name) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - _PyErr_Clear(tstate); + if (PyList_Append(list, name) < 0) { + Py_DECREF(name); + Py_DECREF(list); + return NULL; } + Py_DECREF(name); } - - _PyErr_ChainExceptions(type, value, traceback); + return list; } +/********************/ +/* the magic number */ +/********************/ + +/* Helper for pythonrun.c -- return magic number and tag. */ + +long +PyImport_GetMagicNumber(void) +{ + long res; + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *external, *pyc_magic; + + external = PyObject_GetAttrString(IMPORTLIB(interp), "_bootstrap_external"); + if (external == NULL) + return -1; + pyc_magic = PyObject_GetAttrString(external, "_RAW_MAGIC_NUMBER"); + Py_DECREF(external); + if (pyc_magic == NULL) + return -1; + res = PyLong_AsLong(pyc_magic); + Py_DECREF(pyc_magic); + return res; +} + + +extern const char * _PySys_ImplCacheTag; + +const char * +PyImport_GetMagicTag(void) +{ + return _PySys_ImplCacheTag; +} + + +/*********************************/ +/* a Python module's code object */ +/*********************************/ + /* Execute a code object in a module and return the module object * WITH INCREMENTED REFERENCE COUNT. If an error occurs, name is * removed from sys.modules, to avoid leaving damaged module objects @@ -851,7 +1209,7 @@ PyImport_ExecCodeModuleWithPathnames(const char *name, PyObject *co, Py_FatalError("no current interpreter"); } - external= PyObject_GetAttrString(interp->importlib, + external= PyObject_GetAttrString(IMPORTLIB(interp), "_bootstrap_external"); if (external != NULL) { pathobj = _PyObject_CallMethodOneArg( @@ -936,7 +1294,7 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, if (pathname == NULL) { pathname = ((PyCodeObject *)co)->co_filename; } - external = PyObject_GetAttrString(tstate->interp->importlib, + external = PyObject_GetAttrString(IMPORTLIB(tstate->interp), "_bootstrap_external"); if (external == NULL) { Py_DECREF(d); @@ -989,204 +1347,10 @@ update_compiled_module(PyCodeObject *co, PyObject *newname) Py_DECREF(oldname); } -/*[clinic input] -_imp._fix_co_filename - - code: object(type="PyCodeObject *", subclass_of="&PyCode_Type") - Code object to change. - - path: unicode - File path to use. - / - -Changes code.co_filename to specify the passed-in file path. -[clinic start generated code]*/ - -static PyObject * -_imp__fix_co_filename_impl(PyObject *module, PyCodeObject *code, - PyObject *path) -/*[clinic end generated code: output=1d002f100235587d input=895ba50e78b82f05]*/ - -{ - update_compiled_module(code, path); - - Py_RETURN_NONE; -} - - -/* Helper to test for built-in module */ - -static int -is_builtin(PyObject *name) -{ - int i; - struct _inittab *inittab = _PyRuntime.imports.inittab; - for (i = 0; inittab[i].name != NULL; i++) { - if (_PyUnicode_EqualToASCIIString(name, inittab[i].name)) { - if (inittab[i].initfunc == NULL) - return -1; - else - return 1; - } - } - return 0; -} - - -/* Return a finder object for a sys.path/pkg.__path__ item 'p', - possibly by fetching it from the path_importer_cache dict. If it - wasn't yet cached, traverse path_hooks until a hook is found - that can handle the path item. Return None if no hook could; - this tells our caller that the path based finder could not find - a finder for this path item. Cache the result in - path_importer_cache. */ - -static PyObject * -get_path_importer(PyThreadState *tstate, PyObject *path_importer_cache, - PyObject *path_hooks, PyObject *p) -{ - PyObject *importer; - Py_ssize_t j, nhooks; - - /* These conditions are the caller's responsibility: */ - assert(PyList_Check(path_hooks)); - assert(PyDict_Check(path_importer_cache)); - - nhooks = PyList_Size(path_hooks); - if (nhooks < 0) - return NULL; /* Shouldn't happen */ - - importer = PyDict_GetItemWithError(path_importer_cache, p); - if (importer != NULL || _PyErr_Occurred(tstate)) { - return Py_XNewRef(importer); - } - - /* set path_importer_cache[p] to None to avoid recursion */ - if (PyDict_SetItem(path_importer_cache, p, Py_None) != 0) - return NULL; - - for (j = 0; j < nhooks; j++) { - PyObject *hook = PyList_GetItem(path_hooks, j); - if (hook == NULL) - return NULL; - importer = PyObject_CallOneArg(hook, p); - if (importer != NULL) - break; - - if (!_PyErr_ExceptionMatches(tstate, PyExc_ImportError)) { - return NULL; - } - _PyErr_Clear(tstate); - } - if (importer == NULL) { - Py_RETURN_NONE; - } - if (PyDict_SetItem(path_importer_cache, p, importer) < 0) { - Py_DECREF(importer); - return NULL; - } - return importer; -} - -PyObject * -PyImport_GetImporter(PyObject *path) -{ - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *path_importer_cache = PySys_GetObject("path_importer_cache"); - PyObject *path_hooks = PySys_GetObject("path_hooks"); - if (path_importer_cache == NULL || path_hooks == NULL) { - return NULL; - } - return get_path_importer(tstate, path_importer_cache, path_hooks, path); -} - -#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) -#include -EM_JS(PyObject*, _PyImport_InitFunc_TrampolineCall, (PyModInitFunction func), { - return wasmTable.get(func)(); -}); -#endif // __EMSCRIPTEN__ && PY_CALL_TRAMPOLINE - -static PyObject* -create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) -{ - PyObject *mod = import_find_extension(tstate, name, name); - if (mod || _PyErr_Occurred(tstate)) { - return mod; - } - - PyObject *modules = tstate->interp->modules; - for (struct _inittab *p = _PyRuntime.imports.inittab; p->name != NULL; p++) { - if (_PyUnicode_EqualToASCIIString(name, p->name)) { - if (p->initfunc == NULL) { - /* Cannot re-init internal module ("sys" or "builtins") */ - mod = PyImport_AddModuleObject(name); - return Py_XNewRef(mod); - } - mod = _PyImport_InitFunc_TrampolineCall(*p->initfunc); - if (mod == NULL) { - return NULL; - } - - if (PyObject_TypeCheck(mod, &PyModuleDef_Type)) { - return PyModule_FromDefAndSpec((PyModuleDef*)mod, spec); - } - else { - /* Remember pointer to module init function. */ - PyModuleDef *def = PyModule_GetDef(mod); - if (def == NULL) { - return NULL; - } - - def->m_base.m_init = p->initfunc; - if (_PyImport_FixupExtensionObject(mod, name, name, - modules) < 0) { - return NULL; - } - return mod; - } - } - } - - // not found - Py_RETURN_NONE; -} - - - -/*[clinic input] -_imp.create_builtin - - spec: object - / - -Create an extension module. -[clinic start generated code]*/ - -static PyObject * -_imp_create_builtin(PyObject *module, PyObject *spec) -/*[clinic end generated code: output=ace7ff22271e6f39 input=37f966f890384e47]*/ -{ - PyThreadState *tstate = _PyThreadState_GET(); - - PyObject *name = PyObject_GetAttrString(spec, "name"); - if (name == NULL) { - return NULL; - } - - if (!PyUnicode_Check(name)) { - PyErr_Format(PyExc_TypeError, - "name must be string, not %.200s", - Py_TYPE(name)->tp_name); - Py_DECREF(name); - return NULL; - } - - PyObject *mod = create_builtin(tstate, name, spec); - Py_DECREF(name); - return mod; -} +/******************/ +/* frozen modules */ +/******************/ /* Return true if the name is an alias. In that case, "alias" is set to the original module name. If it is an alias but the original @@ -1210,14 +1374,11 @@ resolve_module_alias(const char *name, const struct _module_alias *aliases, } } - -/* Frozen modules */ - static bool use_frozen(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); - int override = interp->override_frozen_modules; + int override = OVERRIDE_FROZEN_MODULES(interp); if (override > 0) { return true; } @@ -1587,6 +1748,267 @@ PyImport_ImportFrozenModule(const char *name) } +/*************/ +/* importlib */ +/*************/ + +/* Import the _imp extension by calling manually _imp.create_builtin() and + _imp.exec_builtin() since importlib is not initialized yet. Initializing + importlib requires the _imp module: this function fix the bootstrap issue. + */ +static PyObject* +bootstrap_imp(PyThreadState *tstate) +{ + PyObject *name = PyUnicode_FromString("_imp"); + if (name == NULL) { + return NULL; + } + + // Mock a ModuleSpec object just good enough for PyModule_FromDefAndSpec(): + // an object with just a name attribute. + // + // _imp.__spec__ is overridden by importlib._bootstrap._instal() anyway. + PyObject *attrs = Py_BuildValue("{sO}", "name", name); + if (attrs == NULL) { + goto error; + } + PyObject *spec = _PyNamespace_New(attrs); + Py_DECREF(attrs); + if (spec == NULL) { + goto error; + } + + // Create the _imp module from its definition. + PyObject *mod = create_builtin(tstate, name, spec); + Py_CLEAR(name); + Py_DECREF(spec); + if (mod == NULL) { + goto error; + } + assert(mod != Py_None); // not found + + // Execute the _imp module: call imp_module_exec(). + if (exec_builtin_or_dynamic(mod) < 0) { + Py_DECREF(mod); + goto error; + } + return mod; + +error: + Py_XDECREF(name); + return NULL; +} + +/* Global initializations. Can be undone by Py_FinalizeEx(). Don't + call this twice without an intervening Py_FinalizeEx() call. When + initializations fail, a fatal error is issued and the function does + not return. On return, the first thread and interpreter state have + been created. + + Locking: you must hold the interpreter lock while calling this. + (If the lock has not yet been initialized, that's equivalent to + having the lock, but you cannot use multiple threads.) + +*/ +static int +init_importlib(PyThreadState *tstate, PyObject *sysmod) +{ + assert(!_PyErr_Occurred(tstate)); + + PyInterpreterState *interp = tstate->interp; + int verbose = _PyInterpreterState_GetConfig(interp)->verbose; + + // Import _importlib through its frozen version, _frozen_importlib. + if (verbose) { + PySys_FormatStderr("import _frozen_importlib # frozen\n"); + } + if (PyImport_ImportFrozenModule("_frozen_importlib") <= 0) { + return -1; + } + PyObject *importlib = PyImport_AddModule("_frozen_importlib"); // borrowed + if (importlib == NULL) { + return -1; + } + IMPORTLIB(interp) = Py_NewRef(importlib); + + // Import the _imp module + if (verbose) { + PySys_FormatStderr("import _imp # builtin\n"); + } + PyObject *imp_mod = bootstrap_imp(tstate); + if (imp_mod == NULL) { + return -1; + } + if (_PyImport_SetModuleString("_imp", imp_mod) < 0) { + Py_DECREF(imp_mod); + return -1; + } + + // Install importlib as the implementation of import + PyObject *value = PyObject_CallMethod(importlib, "_install", + "OO", sysmod, imp_mod); + Py_DECREF(imp_mod); + if (value == NULL) { + return -1; + } + Py_DECREF(value); + + assert(!_PyErr_Occurred(tstate)); + return 0; +} + + +static int +init_importlib_external(PyInterpreterState *interp) +{ + PyObject *value; + value = PyObject_CallMethod(IMPORTLIB(interp), + "_install_external_importers", ""); + if (value == NULL) { + return -1; + } + Py_DECREF(value); + return 0; +} + +PyObject * +_PyImport_GetImportlibLoader(PyInterpreterState *interp, + const char *loader_name) +{ + return PyObject_GetAttrString(IMPORTLIB(interp), loader_name); +} + +PyObject * +_PyImport_GetImportlibExternalLoader(PyInterpreterState *interp, + const char *loader_name) +{ + PyObject *bootstrap = PyObject_GetAttrString(IMPORTLIB(interp), + "_bootstrap_external"); + if (bootstrap == NULL) { + return NULL; + } + + PyObject *loader_type = PyObject_GetAttrString(bootstrap, loader_name); + Py_DECREF(bootstrap); + return loader_type; +} + +PyObject * +_PyImport_BlessMyLoader(PyInterpreterState *interp, PyObject *module_globals) +{ + PyObject *external = PyObject_GetAttrString(IMPORTLIB(interp), + "_bootstrap_external"); + if (external == NULL) { + return NULL; + } + + PyObject *loader = PyObject_CallMethod(external, "_bless_my_loader", + "O", module_globals, NULL); + Py_DECREF(external); + return loader; +} + +PyObject * +_PyImport_ImportlibModuleRepr(PyInterpreterState *interp, PyObject *m) +{ + return PyObject_CallMethod(IMPORTLIB(interp), "_module_repr", "O", m); +} + + +/*******************/ + +/* Return a finder object for a sys.path/pkg.__path__ item 'p', + possibly by fetching it from the path_importer_cache dict. If it + wasn't yet cached, traverse path_hooks until a hook is found + that can handle the path item. Return None if no hook could; + this tells our caller that the path based finder could not find + a finder for this path item. Cache the result in + path_importer_cache. */ + +static PyObject * +get_path_importer(PyThreadState *tstate, PyObject *path_importer_cache, + PyObject *path_hooks, PyObject *p) +{ + PyObject *importer; + Py_ssize_t j, nhooks; + + /* These conditions are the caller's responsibility: */ + assert(PyList_Check(path_hooks)); + assert(PyDict_Check(path_importer_cache)); + + nhooks = PyList_Size(path_hooks); + if (nhooks < 0) + return NULL; /* Shouldn't happen */ + + importer = PyDict_GetItemWithError(path_importer_cache, p); + if (importer != NULL || _PyErr_Occurred(tstate)) { + return Py_XNewRef(importer); + } + + /* set path_importer_cache[p] to None to avoid recursion */ + if (PyDict_SetItem(path_importer_cache, p, Py_None) != 0) + return NULL; + + for (j = 0; j < nhooks; j++) { + PyObject *hook = PyList_GetItem(path_hooks, j); + if (hook == NULL) + return NULL; + importer = PyObject_CallOneArg(hook, p); + if (importer != NULL) + break; + + if (!_PyErr_ExceptionMatches(tstate, PyExc_ImportError)) { + return NULL; + } + _PyErr_Clear(tstate); + } + if (importer == NULL) { + Py_RETURN_NONE; + } + if (PyDict_SetItem(path_importer_cache, p, importer) < 0) { + Py_DECREF(importer); + return NULL; + } + return importer; +} + +PyObject * +PyImport_GetImporter(PyObject *path) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *path_importer_cache = PySys_GetObject("path_importer_cache"); + PyObject *path_hooks = PySys_GetObject("path_hooks"); + if (path_importer_cache == NULL || path_hooks == NULL) { + return NULL; + } + return get_path_importer(tstate, path_importer_cache, path_hooks, path); +} + + +/*********************/ +/* importing modules */ +/*********************/ + +int +_PyImport_InitDefaultImportFunc(PyInterpreterState *interp) +{ + // Get the __import__ function + PyObject *import_func = _PyDict_GetItemStringWithError(interp->builtins, + "__import__"); + if (import_func == NULL) { + return -1; + } + IMPORT_FUNC(interp) = Py_NewRef(import_func); + return 0; +} + +int +_PyImport_IsDefaultImportFunc(PyInterpreterState *interp, PyObject *func) +{ + return func == IMPORT_FUNC(interp); +} + + /* Import a module, either built-in, frozen, or external, and return its module object WITH INCREMENTED REFERENCE COUNT */ @@ -1851,8 +2273,8 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) PyObject *mod = NULL; PyInterpreterState *interp = tstate->interp; int import_time = _PyInterpreterState_GetConfig(interp)->import_time; -#define import_level _PyRuntime.imports.find_and_load.import_level -#define accumulated _PyRuntime.imports.find_and_load.accumulated +#define import_level FIND_AND_LOAD.import_level +#define accumulated FIND_AND_LOAD.accumulated _PyTime_t t1 = 0, accumulated_copy = accumulated; @@ -1873,7 +2295,7 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) * _PyDict_GetItemIdWithError(). */ if (import_time) { -#define header _PyRuntime.imports.find_and_load.header +#define header FIND_AND_LOAD.header if (header) { fputs("import time: self [us] | cumulative | imported package\n", stderr); @@ -1889,8 +2311,8 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) if (PyDTrace_IMPORT_FIND_LOAD_START_ENABLED()) PyDTrace_IMPORT_FIND_LOAD_START(PyUnicode_AsUTF8(abs_name)); - mod = PyObject_CallMethodObjArgs(interp->importlib, &_Py_ID(_find_and_load), - abs_name, interp->import_func, NULL); + mod = PyObject_CallMethodObjArgs(IMPORTLIB(interp), &_Py_ID(_find_and_load), + abs_name, IMPORT_FUNC(interp), NULL); if (PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED()) PyDTrace_IMPORT_FIND_LOAD_DONE(PyUnicode_AsUTF8(abs_name), @@ -1913,23 +2335,6 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) #undef accumulated } -PyObject * -PyImport_GetModule(PyObject *name) -{ - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *mod; - - mod = import_get_module(tstate, name); - if (mod != NULL && mod != Py_None) { - if (import_ensure_initialized(tstate->interp, mod, name) < 0) { - Py_DECREF(mod); - remove_importlib_frames(tstate); - return NULL; - } - } - return mod; -} - PyObject * PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, @@ -2059,8 +2464,8 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, if (path) { Py_DECREF(path); final_mod = PyObject_CallMethodObjArgs( - interp->importlib, &_Py_ID(_handle_fromlist), - mod, fromlist, interp->import_func, NULL); + IMPORTLIB(interp), &_Py_ID(_handle_fromlist), + mod, fromlist, IMPORT_FUNC(interp), NULL); } else { final_mod = Py_NewRef(mod); @@ -2195,6 +2600,363 @@ PyImport_Import(PyObject *module_name) return r; } + +/*********************/ +/* runtime lifecycle */ +/*********************/ + +PyStatus +_PyImport_Init(void) +{ + if (INITTAB != NULL) { + return _PyStatus_ERR("global import state already initialized"); + } + + PyStatus status = _PyStatus_OK(); + + /* Force default raw memory allocator to get a known allocator to be able + to release the memory in _PyImport_Fini() */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + if (init_builtin_modules_table() != 0) { + status = PyStatus_NoMemory(); + goto done; + } + +done: + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + return status; +} + +void +_PyImport_Fini(void) +{ + /* Destroy the database used by _PyImport_{Fixup,Find}Extension */ + _extensions_cache_clear(); + if (import_lock != NULL) { + PyThread_free_lock(import_lock); + import_lock = NULL; + } + + /* Use the same memory allocator as _PyImport_Init(). */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + /* Free memory allocated by _PyImport_Init() */ + fini_builtin_modules_table(); + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); +} + +void +_PyImport_Fini2(void) +{ + /* Use the same memory allocator than PyImport_ExtendInittab(). */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + // Reset PyImport_Inittab + PyImport_Inittab = _PyImport_Inittab; + + /* Free memory allocated by PyImport_ExtendInittab() */ + PyMem_RawFree(inittab_copy); + inittab_copy = NULL; + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); +} + + +/*************************/ +/* interpreter lifecycle */ +/*************************/ + +PyStatus +_PyImport_InitCore(PyThreadState *tstate, PyObject *sysmod, int importlib) +{ + // XXX Initialize here: interp->modules and interp->import_func. + // XXX Initialize here: sys.modules and sys.meta_path. + + if (importlib) { + /* This call sets up builtin and frozen import support */ + if (init_importlib(tstate, sysmod) < 0) { + return _PyStatus_ERR("failed to initialize importlib"); + } + } + + return _PyStatus_OK(); +} + +/* In some corner cases it is important to be sure that the import + machinery has been initialized (or not cleaned up yet). For + example, see issue #4236 and PyModule_Create2(). */ + +int +_PyImport_IsInitialized(PyInterpreterState *interp) +{ + if (MODULES(interp) == NULL) + return 0; + return 1; +} + +/* Clear the direct per-interpreter import state, if not cleared already. */ +void +_PyImport_ClearCore(PyInterpreterState *interp) +{ + /* interp->modules should have been cleaned up and cleared already + by _PyImport_FiniCore(). */ + Py_CLEAR(MODULES(interp)); + Py_CLEAR(MODULES_BY_INDEX(interp)); + Py_CLEAR(IMPORTLIB(interp)); + Py_CLEAR(IMPORT_FUNC(interp)); +} + +void +_PyImport_FiniCore(PyInterpreterState *interp) +{ + int verbose = _PyInterpreterState_GetConfig(interp)->verbose; + + if (_PySys_ClearAttrString(interp, "meta_path", verbose) < 0) { + PyErr_WriteUnraisable(NULL); + } + + // XXX Pull in most of finalize_modules() in pylifecycle.c. + + if (_PySys_ClearAttrString(interp, "modules", verbose) < 0) { + PyErr_WriteUnraisable(NULL); + } + + _PyImport_ClearCore(interp); +} + +// XXX Add something like _PyImport_Disable() for use early in interp fini? + + +/* "external" imports */ + +static int +init_zipimport(PyThreadState *tstate, int verbose) +{ + PyObject *path_hooks = PySys_GetObject("path_hooks"); + if (path_hooks == NULL) { + _PyErr_SetString(tstate, PyExc_RuntimeError, + "unable to get sys.path_hooks"); + return -1; + } + + if (verbose) { + PySys_WriteStderr("# installing zipimport hook\n"); + } + + PyObject *zipimporter = _PyImport_GetModuleAttrString("zipimport", "zipimporter"); + if (zipimporter == NULL) { + _PyErr_Clear(tstate); /* No zipimporter object -- okay */ + if (verbose) { + PySys_WriteStderr("# can't import zipimport.zipimporter\n"); + } + } + else { + /* sys.path_hooks.insert(0, zipimporter) */ + int err = PyList_Insert(path_hooks, 0, zipimporter); + Py_DECREF(zipimporter); + if (err < 0) { + return -1; + } + if (verbose) { + PySys_WriteStderr("# installed zipimport hook\n"); + } + } + + return 0; +} + +PyStatus +_PyImport_InitExternal(PyThreadState *tstate) +{ + int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose; + + // XXX Initialize here: sys.path_hooks and sys.path_importer_cache. + + if (init_importlib_external(tstate->interp) != 0) { + _PyErr_Print(tstate); + return _PyStatus_ERR("external importer setup failed"); + } + + if (init_zipimport(tstate, verbose) != 0) { + PyErr_Print(); + return _PyStatus_ERR("initializing zipimport failed"); + } + + return _PyStatus_OK(); +} + +void +_PyImport_FiniExternal(PyInterpreterState *interp) +{ + int verbose = _PyInterpreterState_GetConfig(interp)->verbose; + + // XXX Uninstall importlib metapath importers here? + + if (_PySys_ClearAttrString(interp, "path_importer_cache", verbose) < 0) { + PyErr_WriteUnraisable(NULL); + } + if (_PySys_ClearAttrString(interp, "path_hooks", verbose) < 0) { + PyErr_WriteUnraisable(NULL); + } +} + + +/******************/ +/* module helpers */ +/******************/ + +PyObject * +_PyImport_GetModuleAttr(PyObject *modname, PyObject *attrname) +{ + PyObject *mod = PyImport_Import(modname); + if (mod == NULL) { + return NULL; + } + PyObject *result = PyObject_GetAttr(mod, attrname); + Py_DECREF(mod); + return result; +} + +PyObject * +_PyImport_GetModuleAttrString(const char *modname, const char *attrname) +{ + PyObject *pmodname = PyUnicode_FromString(modname); + if (pmodname == NULL) { + return NULL; + } + PyObject *pattrname = PyUnicode_FromString(attrname); + if (pattrname == NULL) { + Py_DECREF(pmodname); + return NULL; + } + PyObject *result = _PyImport_GetModuleAttr(pmodname, pattrname); + Py_DECREF(pattrname); + Py_DECREF(pmodname); + return result; +} + + +/**************/ +/* the module */ +/**************/ + +/*[clinic input] +_imp.lock_held + +Return True if the import lock is currently held, else False. + +On platforms without threads, return False. +[clinic start generated code]*/ + +static PyObject * +_imp_lock_held_impl(PyObject *module) +/*[clinic end generated code: output=8b89384b5e1963fc input=9b088f9b217d9bdf]*/ +{ + return PyBool_FromLong(import_lock_thread != PYTHREAD_INVALID_THREAD_ID); +} + +/*[clinic input] +_imp.acquire_lock + +Acquires the interpreter's import lock for the current thread. + +This lock should be used by import hooks to ensure thread-safety when importing +modules. On platforms without threads, this function does nothing. +[clinic start generated code]*/ + +static PyObject * +_imp_acquire_lock_impl(PyObject *module) +/*[clinic end generated code: output=1aff58cb0ee1b026 input=4a2d4381866d5fdc]*/ +{ + _PyImport_AcquireLock(); + Py_RETURN_NONE; +} + +/*[clinic input] +_imp.release_lock + +Release the interpreter's import lock. + +On platforms without threads, this function does nothing. +[clinic start generated code]*/ + +static PyObject * +_imp_release_lock_impl(PyObject *module) +/*[clinic end generated code: output=7faab6d0be178b0a input=934fb11516dd778b]*/ +{ + if (_PyImport_ReleaseLock() < 0) { + PyErr_SetString(PyExc_RuntimeError, + "not holding the import lock"); + return NULL; + } + Py_RETURN_NONE; +} + + +/*[clinic input] +_imp._fix_co_filename + + code: object(type="PyCodeObject *", subclass_of="&PyCode_Type") + Code object to change. + + path: unicode + File path to use. + / + +Changes code.co_filename to specify the passed-in file path. +[clinic start generated code]*/ + +static PyObject * +_imp__fix_co_filename_impl(PyObject *module, PyCodeObject *code, + PyObject *path) +/*[clinic end generated code: output=1d002f100235587d input=895ba50e78b82f05]*/ + +{ + update_compiled_module(code, path); + + Py_RETURN_NONE; +} + + +/*[clinic input] +_imp.create_builtin + + spec: object + / + +Create an extension module. +[clinic start generated code]*/ + +static PyObject * +_imp_create_builtin(PyObject *module, PyObject *spec) +/*[clinic end generated code: output=ace7ff22271e6f39 input=37f966f890384e47]*/ +{ + PyThreadState *tstate = _PyThreadState_GET(); + + PyObject *name = PyObject_GetAttrString(spec, "name"); + if (name == NULL) { + return NULL; + } + + if (!PyUnicode_Check(name)) { + PyErr_Format(PyExc_TypeError, + "name must be string, not %.200s", + Py_TYPE(name)->tp_name); + Py_DECREF(name); + return NULL; + } + + PyObject *mod = create_builtin(tstate, name, spec); + Py_DECREF(name); + return mod; +} + + /*[clinic input] _imp.extension_suffixes @@ -2459,34 +3221,10 @@ _imp__override_frozen_modules_for_tests_impl(PyObject *module, int override) /*[clinic end generated code: output=36d5cb1594160811 input=8f1f95a3ef21aec3]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - interp->override_frozen_modules = override; + OVERRIDE_FROZEN_MODULES(interp) = override; Py_RETURN_NONE; } -/* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */ -static int -exec_builtin_or_dynamic(PyObject *mod) { - PyModuleDef *def; - void *state; - - if (!PyModule_Check(mod)) { - return 0; - } - - def = PyModule_GetDef(mod); - if (def == NULL) { - return 0; - } - - state = PyModule_GetState(mod); - if (state) { - /* Already initialized; skip reload */ - return 0; - } - - return PyModule_ExecDef(mod, def); -} - #ifdef HAVE_DYNAMIC_LOADING /*[clinic input] @@ -2674,158 +3412,6 @@ PyInit__imp(void) } -// Import the _imp extension by calling manually _imp.create_builtin() and -// _imp.exec_builtin() since importlib is not initialized yet. Initializing -// importlib requires the _imp module: this function fix the bootstrap issue. -PyObject* -_PyImport_BootstrapImp(PyThreadState *tstate) -{ - PyObject *name = PyUnicode_FromString("_imp"); - if (name == NULL) { - return NULL; - } - - // Mock a ModuleSpec object just good enough for PyModule_FromDefAndSpec(): - // an object with just a name attribute. - // - // _imp.__spec__ is overridden by importlib._bootstrap._instal() anyway. - PyObject *attrs = Py_BuildValue("{sO}", "name", name); - if (attrs == NULL) { - goto error; - } - PyObject *spec = _PyNamespace_New(attrs); - Py_DECREF(attrs); - if (spec == NULL) { - goto error; - } - - // Create the _imp module from its definition. - PyObject *mod = create_builtin(tstate, name, spec); - Py_CLEAR(name); - Py_DECREF(spec); - if (mod == NULL) { - goto error; - } - assert(mod != Py_None); // not found - - // Execute the _imp module: call imp_module_exec(). - if (exec_builtin_or_dynamic(mod) < 0) { - Py_DECREF(mod); - goto error; - } - return mod; - -error: - Py_XDECREF(name); - return NULL; -} - - -/* API for embedding applications that want to add their own entries - to the table of built-in modules. This should normally be called - *before* Py_Initialize(). When the table resize fails, -1 is - returned and the existing table is unchanged. - - After a similar function by Just van Rossum. */ - -int -PyImport_ExtendInittab(struct _inittab *newtab) -{ - struct _inittab *p; - size_t i, n; - int res = 0; - - if (_PyRuntime.imports.inittab != NULL) { - Py_FatalError("PyImport_ExtendInittab() may not be called after Py_Initialize()"); - } - - /* Count the number of entries in both tables */ - for (n = 0; newtab[n].name != NULL; n++) - ; - if (n == 0) - return 0; /* Nothing to do */ - for (i = 0; PyImport_Inittab[i].name != NULL; i++) - ; - - /* Force default raw memory allocator to get a known allocator to be able - to release the memory in _PyImport_Fini2() */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - /* Allocate new memory for the combined table */ - p = NULL; - if (i + n <= SIZE_MAX / sizeof(struct _inittab) - 1) { - size_t size = sizeof(struct _inittab) * (i + n + 1); - p = PyMem_RawRealloc(inittab_copy, size); - } - if (p == NULL) { - res = -1; - goto done; - } - - /* Copy the tables into the new memory at the first call - to PyImport_ExtendInittab(). */ - if (inittab_copy != PyImport_Inittab) { - memcpy(p, PyImport_Inittab, (i+1) * sizeof(struct _inittab)); - } - memcpy(p + i, newtab, (n + 1) * sizeof(struct _inittab)); - PyImport_Inittab = inittab_copy = p; - -done: - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - return res; -} - -/* Shorthand to add a single entry given a name and a function */ - -int -PyImport_AppendInittab(const char *name, PyObject* (*initfunc)(void)) -{ - struct _inittab newtab[2]; - - if (_PyRuntime.imports.inittab != NULL) { - Py_FatalError("PyImport_AppendInittab() may not be called after Py_Initialize()"); - } - - memset(newtab, '\0', sizeof newtab); - - newtab[0].name = name; - newtab[0].initfunc = initfunc; - - return PyImport_ExtendInittab(newtab); -} - - -PyObject * -_PyImport_GetModuleAttr(PyObject *modname, PyObject *attrname) -{ - PyObject *mod = PyImport_Import(modname); - if (mod == NULL) { - return NULL; - } - PyObject *result = PyObject_GetAttr(mod, attrname); - Py_DECREF(mod); - return result; -} - -PyObject * -_PyImport_GetModuleAttrString(const char *modname, const char *attrname) -{ - PyObject *pmodname = PyUnicode_FromString(modname); - if (pmodname == NULL) { - return NULL; - } - PyObject *pattrname = PyUnicode_FromString(attrname); - if (pattrname == NULL) { - Py_DECREF(pmodname); - return NULL; - } - PyObject *result = _PyImport_GetModuleAttr(pmodname, pattrname); - Py_DECREF(pattrname); - Py_DECREF(pmodname); - return result; -} - #ifdef __cplusplus } #endif diff --git a/Python/importdl.c b/Python/importdl.c index 91fa06f49c2..6dafb454148 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -99,7 +99,7 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) #endif PyObject *name_unicode = NULL, *name = NULL, *path = NULL, *m = NULL; const char *name_buf, *hook_prefix; - const char *oldcontext; + const char *oldcontext, *newcontext; dl_funcptr exportfunc; PyModuleDef *def; PyModInitFunction p0; @@ -113,6 +113,10 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) "spec.name must be a string"); goto error; } + newcontext = PyUnicode_AsUTF8(name_unicode); + if (newcontext == NULL) { + goto error; + } name = get_encoded_name(name_unicode, &hook_prefix); if (name == NULL) { @@ -160,16 +164,9 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) p0 = (PyModInitFunction)exportfunc; /* Package context is needed for single-phase init */ -#define _Py_PackageContext (_PyRuntime.imports.pkgcontext) - oldcontext = _Py_PackageContext; - _Py_PackageContext = PyUnicode_AsUTF8(name_unicode); - if (_Py_PackageContext == NULL) { - _Py_PackageContext = oldcontext; - goto error; - } + oldcontext = _PyImport_SwapPackageContext(newcontext); m = _PyImport_InitFunc_TrampolineCall(p0); - _Py_PackageContext = oldcontext; -#undef _Py_PackageContext + _PyImport_SwapPackageContext(oldcontext); if (m == NULL) { if (!PyErr_Occurred()) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 045a2996e89..281035dafa9 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -156,79 +156,6 @@ Py_IsInitialized(void) } -/* Global initializations. Can be undone by Py_FinalizeEx(). Don't - call this twice without an intervening Py_FinalizeEx() call. When - initializations fail, a fatal error is issued and the function does - not return. On return, the first thread and interpreter state have - been created. - - Locking: you must hold the interpreter lock while calling this. - (If the lock has not yet been initialized, that's equivalent to - having the lock, but you cannot use multiple threads.) - -*/ -static int -init_importlib(PyThreadState *tstate, PyObject *sysmod) -{ - assert(!_PyErr_Occurred(tstate)); - - PyInterpreterState *interp = tstate->interp; - int verbose = _PyInterpreterState_GetConfig(interp)->verbose; - - // Import _importlib through its frozen version, _frozen_importlib. - if (verbose) { - PySys_FormatStderr("import _frozen_importlib # frozen\n"); - } - if (PyImport_ImportFrozenModule("_frozen_importlib") <= 0) { - return -1; - } - PyObject *importlib = PyImport_AddModule("_frozen_importlib"); // borrowed - if (importlib == NULL) { - return -1; - } - interp->importlib = Py_NewRef(importlib); - - // Import the _imp module - if (verbose) { - PySys_FormatStderr("import _imp # builtin\n"); - } - PyObject *imp_mod = _PyImport_BootstrapImp(tstate); - if (imp_mod == NULL) { - return -1; - } - if (_PyImport_SetModuleString("_imp", imp_mod) < 0) { - Py_DECREF(imp_mod); - return -1; - } - - // Install importlib as the implementation of import - PyObject *value = PyObject_CallMethod(importlib, "_install", - "OO", sysmod, imp_mod); - Py_DECREF(imp_mod); - if (value == NULL) { - return -1; - } - Py_DECREF(value); - - assert(!_PyErr_Occurred(tstate)); - return 0; -} - - -static PyStatus -init_importlib_external(PyThreadState *tstate) -{ - PyObject *value; - value = PyObject_CallMethod(tstate->interp->importlib, - "_install_external_importers", ""); - if (value == NULL) { - _PyErr_Print(tstate); - return _PyStatus_ERR("external importer setup failed"); - } - Py_DECREF(value); - return _PyImportZip_Init(tstate); -} - /* Helper functions to better handle the legacy C locale * * The legacy C locale assumes ASCII as the default text encoding, which @@ -814,7 +741,8 @@ pycore_init_builtins(PyThreadState *tstate) goto error; } - if (_PyImport_FixupBuiltin(bimod, "builtins", interp->modules) < 0) { + PyObject *modules = _PyImport_GetModules(interp); + if (_PyImport_FixupBuiltin(bimod, "builtins", modules) < 0) { goto error; } @@ -850,13 +778,9 @@ pycore_init_builtins(PyThreadState *tstate) } Py_DECREF(bimod); - // Get the __import__ function - PyObject *import_func = _PyDict_GetItemStringWithError(interp->builtins, - "__import__"); - if (import_func == NULL) { + if (_PyImport_InitDefaultImportFunc(interp) < 0) { goto error; } - interp->import_func = Py_NewRef(import_func); assert(!_PyErr_Occurred(tstate)); return _PyStatus_OK(); @@ -918,11 +842,10 @@ pycore_interp_init(PyThreadState *tstate) } const PyConfig *config = _PyInterpreterState_GetConfig(interp); - if (config->_install_importlib) { - /* This call sets up builtin and frozen import support */ - if (init_importlib(tstate, sysmod) < 0) { - return _PyStatus_ERR("failed to initialize importlib"); - } + + status = _PyImport_InitCore(tstate, sysmod, config->_install_importlib); + if (_PyStatus_EXCEPTION(status)) { + goto done; } done: @@ -1172,7 +1095,7 @@ init_interp_main(PyThreadState *tstate) return _PyStatus_ERR("failed to update the Python config"); } - status = init_importlib_external(tstate); + status = _PyImport_InitExternal(tstate); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -1379,8 +1302,11 @@ finalize_modules_delete_special(PyThreadState *tstate, int verbose) static const char * const sys_deletes[] = { "path", "argv", "ps1", "ps2", "last_type", "last_value", "last_traceback", - "path_hooks", "path_importer_cache", "meta_path", "__interactivehook__", + // path_hooks and path_importer_cache are cleared + // by _PyImport_FiniExternal(). + // XXX Clear meta_path in _PyImport_FiniCore(). + "meta_path", NULL }; @@ -1401,10 +1327,7 @@ finalize_modules_delete_special(PyThreadState *tstate, int verbose) const char * const *p; for (p = sys_deletes; *p != NULL; p++) { - if (verbose) { - PySys_WriteStderr("# clear sys.%s\n", *p); - } - if (PyDict_SetItemString(interp->sysdict, *p, Py_None) < 0) { + if (_PySys_ClearAttrString(interp, *p, verbose) < 0) { PyErr_WriteUnraisable(NULL); } } @@ -1576,11 +1499,12 @@ finalize_clear_sys_builtins_dict(PyInterpreterState *interp, int verbose) /* Clear modules, as good as we can */ +// XXX Move most of this to import.c. static void finalize_modules(PyThreadState *tstate) { PyInterpreterState *interp = tstate->interp; - PyObject *modules = interp->modules; + PyObject *modules = _PyImport_GetModules(interp); if (modules == NULL) { // Already done return; @@ -1645,12 +1569,12 @@ finalize_modules(PyThreadState *tstate) // clear PyInterpreterState.modules_by_index and // clear PyModuleDef.m_base.m_copy (of extensions not using the multi-phase // initialization API) - _PyInterpreterState_ClearModules(interp); + _PyImport_ClearModulesByIndex(interp); // Clear and delete the modules directory. Actual modules will // still be there only if imported during the execution of some // destructor. - Py_SETREF(interp->modules, NULL); + _PyImport_ClearModules(interp); // Collect garbage once more _PyGC_CollectNoFail(tstate); @@ -1861,6 +1785,8 @@ Py_FinalizeEx(void) runtime->initialized = 0; runtime->core_initialized = 0; + // XXX Call something like _PyImport_Disable() here? + /* Destroy the state of all threads of the interpreter, except of the current thread. In practice, only daemon threads should still be alive, except if wait_for_thread_shutdown() has been cancelled by CTRL+C. @@ -1910,6 +1836,7 @@ Py_FinalizeEx(void) PyGC_Collect(); /* Destroy all modules */ + _PyImport_FiniExternal(tstate->interp); finalize_modules(tstate); /* Print debug stats if any */ @@ -1943,7 +1870,9 @@ Py_FinalizeEx(void) so it is possible to use tracemalloc in objects destructor. */ _PyTraceMalloc_Fini(); - /* Destroy the database used by _PyImport_{Fixup,Find}Extension */ + /* Finalize any remaining import state */ + // XXX Move these up to where finalize_modules() is currently. + _PyImport_FiniCore(tstate->interp); _PyImport_Fini(); /* unload faulthandler module */ @@ -2183,7 +2112,11 @@ Py_EndInterpreter(PyThreadState *tstate) Py_FatalError("not the last thread"); } + // XXX Call something like _PyImport_Disable() here? + + _PyImport_FiniExternal(tstate->interp); finalize_modules(tstate); + _PyImport_FiniCore(tstate->interp); finalize_interp_clear(tstate); finalize_interp_delete(tstate->interp); @@ -2232,8 +2165,8 @@ add_main_module(PyInterpreterState *interp) if (PyErr_Occurred()) { return _PyStatus_ERR("Failed to test __main__.__loader__"); } - PyObject *loader = PyObject_GetAttrString(interp->importlib, - "BuiltinImporter"); + PyObject *loader = _PyImport_GetImportlibLoader(interp, + "BuiltinImporter"); if (loader == NULL) { return _PyStatus_ERR("Failed to retrieve BuiltinImporter"); } @@ -2739,7 +2672,7 @@ _Py_DumpExtensionModules(int fd, PyInterpreterState *interp) if (interp == NULL) { return; } - PyObject *modules = interp->modules; + PyObject *modules = _PyImport_GetModules(interp); if (modules == NULL || !PyDict_Check(modules)) { return; } diff --git a/Python/pystate.c b/Python/pystate.c index 1261092d143..4770caaed0a 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -772,11 +772,13 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->codec_search_path); Py_CLEAR(interp->codec_search_cache); Py_CLEAR(interp->codec_error_registry); - Py_CLEAR(interp->modules); - Py_CLEAR(interp->modules_by_index); + + assert(interp->imports.modules == NULL); + assert(interp->imports.modules_by_index == NULL); + assert(interp->imports.importlib == NULL); + assert(interp->imports.import_func == NULL); + Py_CLEAR(interp->builtins_copy); - Py_CLEAR(interp->importlib); - Py_CLEAR(interp->import_func); Py_CLEAR(interp->dict); #ifdef HAVE_FORK Py_CLEAR(interp->before_forkers); @@ -836,6 +838,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp) // garbage. It can be different than the current Python thread state // of 'interp'. PyThreadState *current_tstate = current_fast_get(interp->runtime); + _PyImport_ClearCore(interp); interpreter_clear(interp, current_tstate); } @@ -843,6 +846,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp) void _PyInterpreterState_Clear(PyThreadState *tstate) { + _PyImport_ClearCore(tstate->interp); interpreter_clear(tstate->interp, tstate); } @@ -945,36 +949,6 @@ _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime) #endif -// Used by finalize_modules() -void -_PyInterpreterState_ClearModules(PyInterpreterState *interp) -{ - if (!interp->modules_by_index) { - return; - } - - Py_ssize_t i; - for (i = 0; i < PyList_GET_SIZE(interp->modules_by_index); i++) { - PyObject *m = PyList_GET_ITEM(interp->modules_by_index, i); - if (PyModule_Check(m)) { - /* cleanup the saved copy of module dicts */ - PyModuleDef *md = PyModule_GetDef(m); - if (md) { - Py_CLEAR(md->m_base.m_copy); - } - } - } - - /* Setting modules_by_index to NULL could be dangerous, so we - clear the list instead. */ - if (PyList_SetSlice(interp->modules_by_index, - 0, PyList_GET_SIZE(interp->modules_by_index), - NULL)) { - PyErr_WriteUnraisable(interp->modules_by_index); - } -} - - //---------- // accessors //---------- @@ -1058,11 +1032,12 @@ _PyInterpreterState_RequireIDRef(PyInterpreterState *interp, int required) PyObject * _PyInterpreterState_GetMainModule(PyInterpreterState *interp) { - if (interp->modules == NULL) { + PyObject *modules = _PyImport_GetModules(interp); + if (modules == NULL) { PyErr_SetString(PyExc_RuntimeError, "interpreter not initialized"); return NULL; } - return PyMapping_GetItemString(interp->modules, "__main__"); + return PyMapping_GetItemString(modules, "__main__"); } PyObject * @@ -1922,110 +1897,6 @@ done: } -/****************/ -/* module state */ -/****************/ - -PyObject* -PyState_FindModule(PyModuleDef* module) -{ - Py_ssize_t index = module->m_base.m_index; - PyInterpreterState *state = _PyInterpreterState_GET(); - PyObject *res; - if (module->m_slots) { - return NULL; - } - if (index == 0) - return NULL; - if (state->modules_by_index == NULL) - return NULL; - if (index >= PyList_GET_SIZE(state->modules_by_index)) - return NULL; - res = PyList_GET_ITEM(state->modules_by_index, index); - return res==Py_None ? NULL : res; -} - -int -_PyState_AddModule(PyThreadState *tstate, PyObject* module, PyModuleDef* def) -{ - if (!def) { - assert(_PyErr_Occurred(tstate)); - return -1; - } - if (def->m_slots) { - _PyErr_SetString(tstate, - PyExc_SystemError, - "PyState_AddModule called on module with slots"); - return -1; - } - - PyInterpreterState *interp = tstate->interp; - if (!interp->modules_by_index) { - interp->modules_by_index = PyList_New(0); - if (!interp->modules_by_index) { - return -1; - } - } - - while (PyList_GET_SIZE(interp->modules_by_index) <= def->m_base.m_index) { - if (PyList_Append(interp->modules_by_index, Py_None) < 0) { - return -1; - } - } - - return PyList_SetItem(interp->modules_by_index, - def->m_base.m_index, Py_NewRef(module)); -} - -int -PyState_AddModule(PyObject* module, PyModuleDef* def) -{ - if (!def) { - Py_FatalError("module definition is NULL"); - return -1; - } - - PyThreadState *tstate = current_fast_get(&_PyRuntime); - PyInterpreterState *interp = tstate->interp; - Py_ssize_t index = def->m_base.m_index; - if (interp->modules_by_index && - index < PyList_GET_SIZE(interp->modules_by_index) && - module == PyList_GET_ITEM(interp->modules_by_index, index)) - { - _Py_FatalErrorFormat(__func__, "module %p already added", module); - return -1; - } - return _PyState_AddModule(tstate, module, def); -} - -int -PyState_RemoveModule(PyModuleDef* def) -{ - PyThreadState *tstate = current_fast_get(&_PyRuntime); - PyInterpreterState *interp = tstate->interp; - - if (def->m_slots) { - _PyErr_SetString(tstate, - PyExc_SystemError, - "PyState_RemoveModule called on module with slots"); - return -1; - } - - Py_ssize_t index = def->m_base.m_index; - if (index == 0) { - Py_FatalError("invalid module index"); - } - if (interp->modules_by_index == NULL) { - Py_FatalError("Interpreters module-list not accessible."); - } - if (index > PyList_GET_SIZE(interp->modules_by_index)) { - Py_FatalError("Module index out of bounds."); - } - - return PyList_SetItem(interp->modules_by_index, index, Py_NewRef(Py_None)); -} - - /***********************************/ /* Python "auto thread state" API. */ /***********************************/ diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 6a4d5937686..ce993ea8796 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -350,14 +350,8 @@ static int set_main_loader(PyObject *d, PyObject *filename, const char *loader_name) { PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *bootstrap = PyObject_GetAttrString(interp->importlib, - "_bootstrap_external"); - if (bootstrap == NULL) { - return -1; - } - - PyObject *loader_type = PyObject_GetAttrString(bootstrap, loader_name); - Py_DECREF(bootstrap); + PyObject *loader_type = _PyImport_GetImportlibExternalLoader(interp, + loader_name); if (loader_type == NULL) { return -1; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 6e81ef92b67..b69b8035609 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -142,6 +142,20 @@ PySys_SetObject(const char *name, PyObject *v) return sys_set_object_str(interp, name, v); } +int +_PySys_ClearAttrString(PyInterpreterState *interp, + const char *name, int verbose) +{ + if (verbose) { + PySys_WriteStderr("# clear sys.%s\n", name); + } + /* To play it safe, we set the attr to None instead of deleting it. */ + if (PyDict_SetItemString(interp->sysdict, name, Py_None) < 0) { + return -1; + } + return 0; +} + static int should_audit(PyInterpreterState *interp) @@ -1650,7 +1664,7 @@ sys_setdlopenflags_impl(PyObject *module, int new_val) /*[clinic end generated code: output=ec918b7fe0a37281 input=4c838211e857a77f]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - interp->dlopenflags = new_val; + _PyImport_SetDLOpenFlags(interp, new_val); Py_RETURN_NONE; } @@ -1668,7 +1682,8 @@ sys_getdlopenflags_impl(PyObject *module) /*[clinic end generated code: output=e92cd1bc5005da6e input=dc4ea0899c53b4b6]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - return PyLong_FromLong(interp->dlopenflags); + return PyLong_FromLong( + _PyImport_GetDLOpenFlags(interp)); } #endif /* HAVE_DLOPEN */ @@ -2279,22 +2294,10 @@ static PyMethodDef sys_methods[] = { static PyObject * list_builtin_module_names(void) { - PyObject *list = PyList_New(0); + PyObject *list = _PyImport_GetBuiltinModuleNames(); if (list == NULL) { return NULL; } - struct _inittab *inittab = _PyRuntime.imports.inittab; - for (Py_ssize_t i = 0; inittab[i].name != NULL; i++) { - PyObject *name = PyUnicode_FromString(inittab[i].name); - if (name == NULL) { - goto error; - } - if (PyList_Append(list, name) < 0) { - Py_DECREF(name); - goto error; - } - Py_DECREF(name); - } if (PyList_Sort(list) != 0) { goto error; } @@ -3411,11 +3414,10 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) PyInterpreterState *interp = tstate->interp; - PyObject *modules = PyDict_New(); + PyObject *modules = _PyImport_InitModules(interp); if (modules == NULL) { goto error; } - interp->modules = modules; PyObject *sysmod = _PyModule_CreateInitialized(&sysmodule, PYTHON_API_VERSION); if (sysmod == NULL) { @@ -3428,7 +3430,7 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) } interp->sysdict = Py_NewRef(sysdict); - if (PyDict_SetItemString(sysdict, "modules", interp->modules) < 0) { + if (PyDict_SetItemString(sysdict, "modules", modules) < 0) { goto error; } @@ -3442,7 +3444,7 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) return status; } - if (_PyImport_FixupBuiltin(sysmod, "sys", interp->modules) < 0) { + if (_PyImport_FixupBuiltin(sysmod, "sys", modules) < 0) { goto error; }