gh-101758: Clean Up Uses of Import State (gh-101919)

This change is almost entirely moving code around and hiding import state behind internal API.  We introduce no changes to behavior, nor to non-internal API.  (Since there was already going to be a lot of churn, I took this as an opportunity to re-organize import.c into topically-grouped sections of code.)  The motivation is to simplify a number of upcoming changes.

Specific changes:

* move existing import-related code to import.c, wherever possible
* add internal API for interacting with import state (both global and per-interpreter)
* use only API outside of import.c (to limit churn there when changing the location, etc.)
* consolidate the import-related state of PyInterpreterState into a single struct field (this changes layout slightly)
* add macros for import state in import.c (to simplify changing the location)
* group code in import.c into sections
*remove _PyState_AddModule()

https://github.com/python/cpython/issues/101758

Include/internal/pycore_import.h

@@ -36,11 +36,112 @@ struct _import_runtime_state {
     const char * pkgcontext;
 };
 
+struct _import_state {
+    /* cached sys.modules dictionary */
+    PyObject *modules;
+    /* This is the list of module objects for all legacy (single-phase init)
+       extension modules ever loaded in this process (i.e. imported
+       in this interpreter or in any other).  Py_None stands in for
+       modules that haven't actually been imported in this interpreter.
+
+       A module's index (PyModuleDef.m_base.m_index) is used to look up
+       the corresponding module object for this interpreter, if any.
+       (See PyState_FindModule().)  When any extension module
+       is initialized during import, its moduledef gets initialized by
+       PyModuleDef_Init(), and the first time that happens for each
+       PyModuleDef, its index gets set to the current value of
+       a global counter (see _PyRuntimeState.imports.last_module_index).
+       The entry for that index in this interpreter remains unset until
+       the module is actually imported here.  (Py_None is used as
+       a placeholder.)  Note that multi-phase init modules always get
+       an index for which there will never be a module set.
+
+       This is initialized lazily in PyState_AddModule(), which is also
+       where modules get added. */
+    PyObject *modules_by_index;
+    /* importlib module._bootstrap */
+    PyObject *importlib;
+    /* override for config->use_frozen_modules (for tests)
+       (-1: "off", 1: "on", 0: no override) */
+    int override_frozen_modules;
+#ifdef HAVE_DLOPEN
+    int dlopenflags;
+#endif
+    PyObject *import_func;
+};
+
+#ifdef HAVE_DLOPEN
+#  include <dlfcn.h>
+#  if HAVE_DECL_RTLD_NOW
+#    define _Py_DLOPEN_FLAGS RTLD_NOW
+#  else
+#    define _Py_DLOPEN_FLAGS RTLD_LAZY
+#  endif
+#  define DLOPENFLAGS_INIT .dlopenflags = _Py_DLOPEN_FLAGS,
+#else
+#  define _Py_DLOPEN_FLAGS 0
+#  define DLOPENFLAGS_INIT
+#endif
+
+#define IMPORTS_INIT \
+    { \
+        .override_frozen_modules = 0, \
+        DLOPENFLAGS_INIT \
+    }
+
+extern void _PyImport_ClearCore(PyInterpreterState *interp);
+
+extern Py_ssize_t _PyImport_GetNextModuleIndex(void);
+extern const char * _PyImport_ResolveNameWithPackageContext(const char *name);
+extern const char * _PyImport_SwapPackageContext(const char *newcontext);
+
+extern int _PyImport_GetDLOpenFlags(PyInterpreterState *interp);
+extern void _PyImport_SetDLOpenFlags(PyInterpreterState *interp, int new_val);
+
+extern PyObject * _PyImport_InitModules(PyInterpreterState *interp);
+extern PyObject * _PyImport_GetModules(PyInterpreterState *interp);
+extern void _PyImport_ClearModules(PyInterpreterState *interp);
+
+extern void _PyImport_ClearModulesByIndex(PyInterpreterState *interp);
+
+extern int _PyImport_InitDefaultImportFunc(PyInterpreterState *interp);
+extern int _PyImport_IsDefaultImportFunc(
+        PyInterpreterState *interp,
+        PyObject *func);
+
+extern PyObject * _PyImport_GetImportlibLoader(
+        PyInterpreterState *interp,
+        const char *loader_name);
+extern PyObject * _PyImport_GetImportlibExternalLoader(
+        PyInterpreterState *interp,
+        const char *loader_name);
+extern PyObject * _PyImport_BlessMyLoader(
+        PyInterpreterState *interp,
+        PyObject *module_globals);
+extern PyObject * _PyImport_ImportlibModuleRepr(
+        PyInterpreterState *interp,
+        PyObject *module);
+
+
+extern PyStatus _PyImport_Init(void);
+extern void _PyImport_Fini(void);
+extern void _PyImport_Fini2(void);
+
+extern PyStatus _PyImport_InitCore(
+        PyThreadState *tstate,
+        PyObject *sysmod,
+        int importlib);
+extern PyStatus _PyImport_InitExternal(PyThreadState *tstate);
+extern void _PyImport_FiniCore(PyInterpreterState *interp);
+extern void _PyImport_FiniExternal(PyInterpreterState *interp);
+
 
 #ifdef HAVE_FORK
 extern PyStatus _PyImport_ReInitLock(void);
 #endif
-extern PyObject* _PyImport_BootstrapImp(PyThreadState *tstate);
+
+
+extern PyObject* _PyImport_GetBuiltinModuleNames(void);
 
 struct _module_alias {
     const char *name;                 /* ASCII encoded string */

Include/internal/pycore_interp.h

@@ -21,6 +21,7 @@ extern "C" {
 #include "pycore_function.h"      // FUNC_MAX_WATCHERS
 #include "pycore_genobject.h"     // struct _Py_async_gen_state
 #include "pycore_gc.h"            // struct _gc_runtime_state
+#include "pycore_import.h"        // struct _import_state
 #include "pycore_list.h"          // struct _Py_list_state
 #include "pycore_global_objects.h"  // struct _Py_interp_static_objects
 #include "pycore_tuple.h"         // struct _Py_tuple_state
@@ -92,37 +93,12 @@ struct _is {
     struct _ceval_state ceval;
     struct _gc_runtime_state gc;
 
-    // sys.modules dictionary
+    struct _import_state imports;
-    PyObject *modules;
-    /* This is the list of module objects for all legacy (single-phase init)
-       extension modules ever loaded in this process (i.e. imported
-       in this interpreter or in any other).  Py_None stands in for
-       modules that haven't actually been imported in this interpreter.
 
-       A module's index (PyModuleDef.m_base.m_index) is used to look up
-       the corresponding module object for this interpreter, if any.
-       (See PyState_FindModule().)  When any extension module
-       is initialized during import, its moduledef gets initialized by
-       PyModuleDef_Init(), and the first time that happens for each
-       PyModuleDef, its index gets set to the current value of
-       a global counter (see _PyRuntimeState.imports.last_module_index).
-       The entry for that index in this interpreter remains unset until
-       the module is actually imported here.  (Py_None is used as
-       a placeholder.)  Note that multi-phase init modules always get
-       an index for which there will never be a module set.
-
-       This is initialized lazily in _PyState_AddModule(), which is also
-       where modules get added. */
-    PyObject *modules_by_index;
     // Dictionary of the sys module
     PyObject *sysdict;
     // Dictionary of the builtins module
     PyObject *builtins;
-    // importlib module
-    PyObject *importlib;
-    // override for config->use_frozen_modules (for tests)
-    // (-1: "off", 1: "on", 0: no override)
-    int override_frozen_modules;
 
     PyObject *codec_search_path;
     PyObject *codec_search_cache;
@@ -130,15 +106,11 @@ struct _is {
     int codecs_initialized;
 
     PyConfig config;
-#ifdef HAVE_DLOPEN
-    int dlopenflags;
-#endif
     unsigned long feature_flags;
 
     PyObject *dict;  /* Stores per-interpreter state */
 
     PyObject *builtins_copy;
-    PyObject *import_func;
     // Initialized to _PyEval_EvalFrameDefault().
     _PyFrameEvalFunction eval_frame;
 
@@ -205,7 +177,6 @@ struct _is {
 
 /* other API */
 
-extern void _PyInterpreterState_ClearModules(PyInterpreterState *interp);
 extern void _PyInterpreterState_Clear(PyThreadState *tstate);
 
 

Include/internal/pycore_pylifecycle.h

@@ -30,7 +30,6 @@ PyAPI_FUNC(int) _Py_IsLocaleCoercionTarget(const char *ctype_loc);
 /* Various one-time initializers */
 
 extern void _Py_InitVersion(void);
-extern PyStatus _PyImport_Init(void);
 extern PyStatus _PyFaulthandler_Init(int enable);
 extern int _PyTraceMalloc_Init(int enable);
 extern PyObject * _PyBuiltin_Init(PyInterpreterState *interp);
@@ -45,7 +44,6 @@ extern int _PyBuiltins_AddExceptions(PyObject * bltinmod);
 extern PyStatus _Py_HashRandomization_Init(const PyConfig *);
 
 extern PyStatus _PyTime_Init(void);
-extern PyStatus _PyImportZip_Init(PyThreadState *tstate);
 extern PyStatus _PyGC_Init(PyInterpreterState *interp);
 extern PyStatus _PyAtExit_Init(PyInterpreterState *interp);
 extern int _Py_Deepfreeze_Init(void);
@@ -55,8 +53,6 @@ extern int _Py_Deepfreeze_Init(void);
 extern int _PySignal_Init(int install_signal_handlers);
 extern void _PySignal_Fini(void);
 
-extern void _PyImport_Fini(void);
-extern void _PyImport_Fini2(void);
 extern void _PyGC_Fini(PyInterpreterState *interp);
 extern void _Py_HashRandomization_Fini(void);
 extern void _PyFaulthandler_Fini(void);

Include/internal/pycore_pystate.h

@@ -152,12 +152,6 @@ extern void _PySignal_AfterFork(void);
 #endif
 
 
-PyAPI_FUNC(int) _PyState_AddModule(
-    PyThreadState *tstate,
-    PyObject* module,
-    PyModuleDef* def);
-
-
 PyAPI_FUNC(int) _PyOS_InterruptOccurred(PyThreadState *tstate);
 
 #define HEAD_LOCK(runtime) \

Include/internal/pycore_runtime_init.h

@@ -97,23 +97,10 @@ extern "C" {
         ._main_interpreter = _PyInterpreterState_INIT, \
     }
 
-#ifdef HAVE_DLOPEN
-#  include <dlfcn.h>
-#  if HAVE_DECL_RTLD_NOW
-#    define _Py_DLOPEN_FLAGS RTLD_NOW
-#  else
-#    define _Py_DLOPEN_FLAGS RTLD_LAZY
-#  endif
-#  define DLOPENFLAGS_INIT .dlopenflags = _Py_DLOPEN_FLAGS,
-#else
-#  define _Py_DLOPEN_FLAGS 0
-#  define DLOPENFLAGS_INIT
-#endif
-
 #define _PyInterpreterState_INIT \
     { \
         .id_refcount = -1, \
-        DLOPENFLAGS_INIT \
+        .imports = IMPORTS_INIT, \
         .ceval = { \
             .recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \
         }, \

Include/internal/pycore_sysmodule.h

@@ -20,6 +20,9 @@ extern void _PySys_ClearAuditHooks(PyThreadState *tstate);
 
 PyAPI_FUNC(int) _PySys_SetAttr(PyObject *, PyObject *);
 
+extern int _PySys_ClearAttrString(PyInterpreterState *interp,
+                                  const char *name, int verbose);
+
 #ifdef __cplusplus
 }
 #endif

Lib/test/test_imp.py

@@ -387,7 +387,7 @@ class ImportTests(unittest.TestCase):
             check_basic_reloaded(mod, lookedup, initialized, init_count,
                                  before, reloaded)
 
-            # Currently _PyState_AddModule() always replaces the cached module.
+            # Currently PyState_AddModule() always replaces the cached module.
             self.assertIs(basic.look_up_self(), mod)
             self.assertEqual(basic.initialized_count(), expected_init_count)
 

Lib/test/test_stable_abi_ctypes.py

@@ -864,7 +864,6 @@ SYMBOL_NAMES = (
     "_PyObject_GC_Resize",
     "_PyObject_New",
     "_PyObject_NewVar",
-    "_PyState_AddModule",
     "_PyThreadState_Init",
     "_PyThreadState_Prealloc",
     "_PyWeakref_CallableProxyType",

Misc/stable_abi.toml

@@ -1684,9 +1684,6 @@
 [function._PyObject_NewVar]
     added = '3.2'
     abi_only = true
-[function._PyState_AddModule]
-    added = '3.2'
-    abi_only = true
 [function._PyThreadState_Init]
     added = '3.2'
     abi_only = true

Objects/moduleobject.c

@@ -42,10 +42,9 @@ PyModuleDef_Init(PyModuleDef* def)
 {
     assert(PyModuleDef_Type.tp_flags & Py_TPFLAGS_READY);
     if (def->m_base.m_index == 0) {
-        _PyRuntime.imports.last_module_index++;
         Py_SET_REFCNT(def, 1);
         Py_SET_TYPE(def, &PyModuleDef_Type);
-        def->m_base.m_index = _PyRuntime.imports.last_module_index;
+        def->m_base.m_index = _PyImport_GetNextModuleIndex();
     }
     return (PyObject*)def;
 }
@@ -209,24 +208,7 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version)
             "module %s: PyModule_Create is incompatible with m_slots", name);
         return NULL;
     }
-    /* Make sure name is fully qualified.
+    name = _PyImport_ResolveNameWithPackageContext(name);
-
-       This is a bit of a hack: when the shared library is loaded,
-       the module name is "package.module", but the module calls
-       PyModule_Create*() with just "module" for the name.  The shared
-       library loader squirrels away the true name of the module in
-       _Py_PackageContext, and PyModule_Create*() will substitute this
-       (if the name actually matches).
-    */
-#define _Py_PackageContext (_PyRuntime.imports.pkgcontext)
-    if (_Py_PackageContext != NULL) {
-        const char *p = strrchr(_Py_PackageContext, '.');
-        if (p != NULL && strcmp(module->m_name, p+1) == 0) {
-            name = _Py_PackageContext;
-            _Py_PackageContext = NULL;
-        }
-    }
-#undef _Py_PackageContext
     if ((m = (PyModuleObject*)PyModule_New(name)) == NULL)
         return NULL;
 
@@ -710,8 +692,7 @@ static PyObject *
 module_repr(PyModuleObject *m)
 {
     PyInterpreterState *interp = _PyInterpreterState_GET();
-
+    return _PyImport_ImportlibModuleRepr(interp, (PyObject *)m);
-    return PyObject_CallMethod(interp->importlib, "_module_repr", "O", m);
 }
 
 /* Check if the "_initializing" attribute of the module spec is set to true.

PC/python3dll.c

@@ -34,7 +34,6 @@ EXPORT_FUNC(_PyObject_GC_NewVar)
 EXPORT_FUNC(_PyObject_GC_Resize)
 EXPORT_FUNC(_PyObject_New)
 EXPORT_FUNC(_PyObject_NewVar)
-EXPORT_FUNC(_PyState_AddModule)
 EXPORT_FUNC(_PyThreadState_Init)
 EXPORT_FUNC(_PyThreadState_Prealloc)
 EXPORT_FUNC(Py_AddPendingCall)

Python/_warnings.c

@@ -214,7 +214,7 @@ get_warnings_attr(PyInterpreterState *interp, PyObject *attr, int try_import)
            gone, then we can't even use PyImport_GetModule without triggering
            an interpreter abort.
         */
-        if (!interp->modules) {
+        if (!_PyImport_GetModules(interp)) {
             return NULL;
         }
         warnings_module = PyImport_GetModule(&_Py_ID(warnings));
@@ -1050,7 +1050,6 @@ warnings_warn_impl(PyObject *module, PyObject *message, PyObject *category,
 static PyObject *
 get_source_line(PyInterpreterState *interp, PyObject *module_globals, int lineno)
 {
-    PyObject *external;
     PyObject *loader;
     PyObject *module_name;
     PyObject *get_source;
@@ -1059,13 +1058,7 @@ get_source_line(PyInterpreterState *interp, PyObject *module_globals, int lineno
     PyObject *source_line;
 
     /* stolen from import.c */
-    external = PyObject_GetAttrString(interp->importlib, "_bootstrap_external");
+    loader = _PyImport_BlessMyLoader(interp, module_globals);
-    if (external == NULL) {
-        return NULL;
-    }
-
-    loader = PyObject_CallMethod(external, "_bless_my_loader", "O", module_globals, NULL);
-    Py_DECREF(external);
     if (loader == NULL) {
         return NULL;
     }

Python/ceval.c

@@ -2688,7 +2688,7 @@ import_name(PyThreadState *tstate, _PyInterpreterFrame *frame,
     }
     PyObject *locals = frame->f_locals;
     /* Fast path for not overloaded __import__. */
-    if (import_func == tstate->interp->import_func) {
+    if (_PyImport_IsDefaultImportFunc(tstate->interp, import_func)) {
         int ilevel = _PyLong_AsInt(level);
         if (ilevel == -1 && _PyErr_Occurred(tstate)) {
             return NULL;

Python/dynload_shlib.c

@@ -75,7 +75,7 @@ _PyImport_FindSharedFuncptr(const char *prefix,
             return NULL;
     }
 
-    dlopenflags = _PyInterpreterState_GET()->dlopenflags;
+    dlopenflags = _PyImport_GetDLOpenFlags(_PyInterpreterState_GET());
 
     handle = dlopen(pathname, dlopenflags);
 

Python/importdl.c

@@ -99,7 +99,7 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp)
 #endif
     PyObject *name_unicode = NULL, *name = NULL, *path = NULL, *m = NULL;
     const char *name_buf, *hook_prefix;
-    const char *oldcontext;
+    const char *oldcontext, *newcontext;
     dl_funcptr exportfunc;
     PyModuleDef *def;
     PyModInitFunction p0;
@@ -113,6 +113,10 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp)
                         "spec.name must be a string");
         goto error;
     }
+    newcontext = PyUnicode_AsUTF8(name_unicode);
+    if (newcontext == NULL) {
+        goto error;
+    }
 
     name = get_encoded_name(name_unicode, &hook_prefix);
     if (name == NULL) {
@@ -160,16 +164,9 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp)
     p0 = (PyModInitFunction)exportfunc;
 
     /* Package context is needed for single-phase init */
-#define _Py_PackageContext (_PyRuntime.imports.pkgcontext)
+    oldcontext = _PyImport_SwapPackageContext(newcontext);
-    oldcontext = _Py_PackageContext;
-    _Py_PackageContext = PyUnicode_AsUTF8(name_unicode);
-    if (_Py_PackageContext == NULL) {
-        _Py_PackageContext = oldcontext;
-        goto error;
-    }
     m = _PyImport_InitFunc_TrampolineCall(p0);
-    _Py_PackageContext = oldcontext;
+    _PyImport_SwapPackageContext(oldcontext);
-#undef _Py_PackageContext
 
     if (m == NULL) {
         if (!PyErr_Occurred()) {

Python/pylifecycle.c

@@ -156,79 +156,6 @@ Py_IsInitialized(void)
 }
 
 
-/* Global initializations.  Can be undone by Py_FinalizeEx().  Don't
-   call this twice without an intervening Py_FinalizeEx() call.  When
-   initializations fail, a fatal error is issued and the function does
-   not return.  On return, the first thread and interpreter state have
-   been created.
-
-   Locking: you must hold the interpreter lock while calling this.
-   (If the lock has not yet been initialized, that's equivalent to
-   having the lock, but you cannot use multiple threads.)
-
-*/
-static int
-init_importlib(PyThreadState *tstate, PyObject *sysmod)
-{
-    assert(!_PyErr_Occurred(tstate));
-
-    PyInterpreterState *interp = tstate->interp;
-    int verbose = _PyInterpreterState_GetConfig(interp)->verbose;
-
-    // Import _importlib through its frozen version, _frozen_importlib.
-    if (verbose) {
-        PySys_FormatStderr("import _frozen_importlib # frozen\n");
-    }
-    if (PyImport_ImportFrozenModule("_frozen_importlib") <= 0) {
-        return -1;
-    }
-    PyObject *importlib = PyImport_AddModule("_frozen_importlib"); // borrowed
-    if (importlib == NULL) {
-        return -1;
-    }
-    interp->importlib = Py_NewRef(importlib);
-
-    // Import the _imp module
-    if (verbose) {
-        PySys_FormatStderr("import _imp # builtin\n");
-    }
-    PyObject *imp_mod = _PyImport_BootstrapImp(tstate);
-    if (imp_mod == NULL) {
-        return -1;
-    }
-    if (_PyImport_SetModuleString("_imp", imp_mod) < 0) {
-        Py_DECREF(imp_mod);
-        return -1;
-    }
-
-    // Install importlib as the implementation of import
-    PyObject *value = PyObject_CallMethod(importlib, "_install",
-                                          "OO", sysmod, imp_mod);
-    Py_DECREF(imp_mod);
-    if (value == NULL) {
-        return -1;
-    }
-    Py_DECREF(value);
-
-    assert(!_PyErr_Occurred(tstate));
-    return 0;
-}
-
-
-static PyStatus
-init_importlib_external(PyThreadState *tstate)
-{
-    PyObject *value;
-    value = PyObject_CallMethod(tstate->interp->importlib,
-                                "_install_external_importers", "");
-    if (value == NULL) {
-        _PyErr_Print(tstate);
-        return _PyStatus_ERR("external importer setup failed");
-    }
-    Py_DECREF(value);
-    return _PyImportZip_Init(tstate);
-}
-
 /* Helper functions to better handle the legacy C locale
  *
  * The legacy C locale assumes ASCII as the default text encoding, which
@@ -814,7 +741,8 @@ pycore_init_builtins(PyThreadState *tstate)
         goto error;
     }
 
-    if (_PyImport_FixupBuiltin(bimod, "builtins", interp->modules) < 0) {
+    PyObject *modules = _PyImport_GetModules(interp);
+    if (_PyImport_FixupBuiltin(bimod, "builtins", modules) < 0) {
         goto error;
     }
 
@@ -850,13 +778,9 @@ pycore_init_builtins(PyThreadState *tstate)
     }
     Py_DECREF(bimod);
 
-    // Get the __import__ function
+    if (_PyImport_InitDefaultImportFunc(interp) < 0) {
-    PyObject *import_func = _PyDict_GetItemStringWithError(interp->builtins,
-                                                           "__import__");
-    if (import_func == NULL) {
         goto error;
     }
-    interp->import_func = Py_NewRef(import_func);
 
     assert(!_PyErr_Occurred(tstate));
     return _PyStatus_OK();
@@ -918,11 +842,10 @@ pycore_interp_init(PyThreadState *tstate)
     }
 
     const PyConfig *config = _PyInterpreterState_GetConfig(interp);
-    if (config->_install_importlib) {
+
-        /* This call sets up builtin and frozen import support */
+    status = _PyImport_InitCore(tstate, sysmod, config->_install_importlib);
-        if (init_importlib(tstate, sysmod) < 0) {
+    if (_PyStatus_EXCEPTION(status)) {
-            return _PyStatus_ERR("failed to initialize importlib");
+        goto done;
-        }
     }
 
 done:
@@ -1172,7 +1095,7 @@ init_interp_main(PyThreadState *tstate)
         return _PyStatus_ERR("failed to update the Python config");
     }
 
-    status = init_importlib_external(tstate);
+    status = _PyImport_InitExternal(tstate);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
@@ -1379,8 +1302,11 @@ finalize_modules_delete_special(PyThreadState *tstate, int verbose)
     static const char * const sys_deletes[] = {
         "path", "argv", "ps1", "ps2",
         "last_type", "last_value", "last_traceback",
-        "path_hooks", "path_importer_cache", "meta_path",
         "__interactivehook__",
+        // path_hooks and path_importer_cache are cleared
+        // by _PyImport_FiniExternal().
+        // XXX Clear meta_path in _PyImport_FiniCore().
+        "meta_path",
         NULL
     };
 
@@ -1401,10 +1327,7 @@ finalize_modules_delete_special(PyThreadState *tstate, int verbose)
 
     const char * const *p;
     for (p = sys_deletes; *p != NULL; p++) {
-        if (verbose) {
+        if (_PySys_ClearAttrString(interp, *p, verbose) < 0) {
-            PySys_WriteStderr("# clear sys.%s\n", *p);
-        }
-        if (PyDict_SetItemString(interp->sysdict, *p, Py_None) < 0) {
             PyErr_WriteUnraisable(NULL);
         }
     }
@@ -1576,11 +1499,12 @@ finalize_clear_sys_builtins_dict(PyInterpreterState *interp, int verbose)
 
 
 /* Clear modules, as good as we can */
+// XXX Move most of this to import.c.
 static void
 finalize_modules(PyThreadState *tstate)
 {
     PyInterpreterState *interp = tstate->interp;
-    PyObject *modules = interp->modules;
+    PyObject *modules = _PyImport_GetModules(interp);
     if (modules == NULL) {
         // Already done
         return;
@@ -1645,12 +1569,12 @@ finalize_modules(PyThreadState *tstate)
     // clear PyInterpreterState.modules_by_index and
     // clear PyModuleDef.m_base.m_copy (of extensions not using the multi-phase
     // initialization API)
-    _PyInterpreterState_ClearModules(interp);
+    _PyImport_ClearModulesByIndex(interp);
 
     // Clear and delete the modules directory.  Actual modules will
     // still be there only if imported during the execution of some
     // destructor.
-    Py_SETREF(interp->modules, NULL);
+    _PyImport_ClearModules(interp);
 
     // Collect garbage once more
     _PyGC_CollectNoFail(tstate);
@@ -1861,6 +1785,8 @@ Py_FinalizeEx(void)
     runtime->initialized = 0;
     runtime->core_initialized = 0;
 
+    // XXX Call something like _PyImport_Disable() here?
+
     /* Destroy the state of all threads of the interpreter, except of the
        current thread. In practice, only daemon threads should still be alive,
        except if wait_for_thread_shutdown() has been cancelled by CTRL+C.
@@ -1910,6 +1836,7 @@ Py_FinalizeEx(void)
     PyGC_Collect();
 
     /* Destroy all modules */
+    _PyImport_FiniExternal(tstate->interp);
     finalize_modules(tstate);
 
     /* Print debug stats if any */
@@ -1943,7 +1870,9 @@ Py_FinalizeEx(void)
        so it is possible to use tracemalloc in objects destructor. */
     _PyTraceMalloc_Fini();
 
-    /* Destroy the database used by _PyImport_{Fixup,Find}Extension */
+    /* Finalize any remaining import state */
+    // XXX Move these up to where finalize_modules() is currently.
+    _PyImport_FiniCore(tstate->interp);
     _PyImport_Fini();
 
     /* unload faulthandler module */
@@ -2183,7 +2112,11 @@ Py_EndInterpreter(PyThreadState *tstate)
         Py_FatalError("not the last thread");
     }
 
+    // XXX Call something like _PyImport_Disable() here?
+
+    _PyImport_FiniExternal(tstate->interp);
     finalize_modules(tstate);
+    _PyImport_FiniCore(tstate->interp);
 
     finalize_interp_clear(tstate);
     finalize_interp_delete(tstate->interp);
@@ -2232,7 +2165,7 @@ add_main_module(PyInterpreterState *interp)
         if (PyErr_Occurred()) {
             return _PyStatus_ERR("Failed to test __main__.__loader__");
         }
-        PyObject *loader = PyObject_GetAttrString(interp->importlib,
+        PyObject *loader = _PyImport_GetImportlibLoader(interp,
                                                         "BuiltinImporter");
         if (loader == NULL) {
             return _PyStatus_ERR("Failed to retrieve BuiltinImporter");
@@ -2739,7 +2672,7 @@ _Py_DumpExtensionModules(int fd, PyInterpreterState *interp)
     if (interp == NULL) {
         return;
     }
-    PyObject *modules = interp->modules;
+    PyObject *modules = _PyImport_GetModules(interp);
     if (modules == NULL || !PyDict_Check(modules)) {
         return;
     }

Python/pystate.c

@@ -772,11 +772,13 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate)
     Py_CLEAR(interp->codec_search_path);
     Py_CLEAR(interp->codec_search_cache);
     Py_CLEAR(interp->codec_error_registry);
-    Py_CLEAR(interp->modules);
+
-    Py_CLEAR(interp->modules_by_index);
+    assert(interp->imports.modules == NULL);
+    assert(interp->imports.modules_by_index == NULL);
+    assert(interp->imports.importlib == NULL);
+    assert(interp->imports.import_func == NULL);
+
     Py_CLEAR(interp->builtins_copy);
-    Py_CLEAR(interp->importlib);
-    Py_CLEAR(interp->import_func);
     Py_CLEAR(interp->dict);
 #ifdef HAVE_FORK
     Py_CLEAR(interp->before_forkers);
@@ -836,6 +838,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp)
     // garbage. It can be different than the current Python thread state
     // of 'interp'.
     PyThreadState *current_tstate = current_fast_get(interp->runtime);
+    _PyImport_ClearCore(interp);
     interpreter_clear(interp, current_tstate);
 }
 
@@ -843,6 +846,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp)
 void
 _PyInterpreterState_Clear(PyThreadState *tstate)
 {
+    _PyImport_ClearCore(tstate->interp);
     interpreter_clear(tstate->interp, tstate);
 }
 
@@ -945,36 +949,6 @@ _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime)
 #endif
 
 
-// Used by finalize_modules()
-void
-_PyInterpreterState_ClearModules(PyInterpreterState *interp)
-{
-    if (!interp->modules_by_index) {
-        return;
-    }
-
-    Py_ssize_t i;
-    for (i = 0; i < PyList_GET_SIZE(interp->modules_by_index); i++) {
-        PyObject *m = PyList_GET_ITEM(interp->modules_by_index, i);
-        if (PyModule_Check(m)) {
-            /* cleanup the saved copy of module dicts */
-            PyModuleDef *md = PyModule_GetDef(m);
-            if (md) {
-                Py_CLEAR(md->m_base.m_copy);
-            }
-        }
-    }
-
-    /* Setting modules_by_index to NULL could be dangerous, so we
-       clear the list instead. */
-    if (PyList_SetSlice(interp->modules_by_index,
-                        0, PyList_GET_SIZE(interp->modules_by_index),
-                        NULL)) {
-        PyErr_WriteUnraisable(interp->modules_by_index);
-    }
-}
-
-
 //----------
 // accessors
 //----------
@@ -1058,11 +1032,12 @@ _PyInterpreterState_RequireIDRef(PyInterpreterState *interp, int required)
 PyObject *
 _PyInterpreterState_GetMainModule(PyInterpreterState *interp)
 {
-    if (interp->modules == NULL) {
+    PyObject *modules = _PyImport_GetModules(interp);
+    if (modules == NULL) {
         PyErr_SetString(PyExc_RuntimeError, "interpreter not initialized");
         return NULL;
     }
-    return PyMapping_GetItemString(interp->modules, "__main__");
+    return PyMapping_GetItemString(modules, "__main__");
 }
 
 PyObject *
@@ -1922,110 +1897,6 @@ done:
 }
 
 
-/****************/
-/* module state */
-/****************/
-
-PyObject*
-PyState_FindModule(PyModuleDef* module)
-{
-    Py_ssize_t index = module->m_base.m_index;
-    PyInterpreterState *state = _PyInterpreterState_GET();
-    PyObject *res;
-    if (module->m_slots) {
-        return NULL;
-    }
-    if (index == 0)
-        return NULL;
-    if (state->modules_by_index == NULL)
-        return NULL;
-    if (index >= PyList_GET_SIZE(state->modules_by_index))
-        return NULL;
-    res = PyList_GET_ITEM(state->modules_by_index, index);
-    return res==Py_None ? NULL : res;
-}
-
-int
-_PyState_AddModule(PyThreadState *tstate, PyObject* module, PyModuleDef* def)
-{
-    if (!def) {
-        assert(_PyErr_Occurred(tstate));
-        return -1;
-    }
-    if (def->m_slots) {
-        _PyErr_SetString(tstate,
-                         PyExc_SystemError,
-                         "PyState_AddModule called on module with slots");
-        return -1;
-    }
-
-    PyInterpreterState *interp = tstate->interp;
-    if (!interp->modules_by_index) {
-        interp->modules_by_index = PyList_New(0);
-        if (!interp->modules_by_index) {
-            return -1;
-        }
-    }
-
-    while (PyList_GET_SIZE(interp->modules_by_index) <= def->m_base.m_index) {
-        if (PyList_Append(interp->modules_by_index, Py_None) < 0) {
-            return -1;
-        }
-    }
-
-    return PyList_SetItem(interp->modules_by_index,
-                          def->m_base.m_index, Py_NewRef(module));
-}
-
-int
-PyState_AddModule(PyObject* module, PyModuleDef* def)
-{
-    if (!def) {
-        Py_FatalError("module definition is NULL");
-        return -1;
-    }
-
-    PyThreadState *tstate = current_fast_get(&_PyRuntime);
-    PyInterpreterState *interp = tstate->interp;
-    Py_ssize_t index = def->m_base.m_index;
-    if (interp->modules_by_index &&
-        index < PyList_GET_SIZE(interp->modules_by_index) &&
-        module == PyList_GET_ITEM(interp->modules_by_index, index))
-    {
-        _Py_FatalErrorFormat(__func__, "module %p already added", module);
-        return -1;
-    }
-    return _PyState_AddModule(tstate, module, def);
-}
-
-int
-PyState_RemoveModule(PyModuleDef* def)
-{
-    PyThreadState *tstate = current_fast_get(&_PyRuntime);
-    PyInterpreterState *interp = tstate->interp;
-
-    if (def->m_slots) {
-        _PyErr_SetString(tstate,
-                         PyExc_SystemError,
-                         "PyState_RemoveModule called on module with slots");
-        return -1;
-    }
-
-    Py_ssize_t index = def->m_base.m_index;
-    if (index == 0) {
-        Py_FatalError("invalid module index");
-    }
-    if (interp->modules_by_index == NULL) {
-        Py_FatalError("Interpreters module-list not accessible.");
-    }
-    if (index > PyList_GET_SIZE(interp->modules_by_index)) {
-        Py_FatalError("Module index out of bounds.");
-    }
-
-    return PyList_SetItem(interp->modules_by_index, index, Py_NewRef(Py_None));
-}
-
-
 /***********************************/
 /* Python "auto thread state" API. */
 /***********************************/

Python/pythonrun.c

@@ -350,14 +350,8 @@ static int
 set_main_loader(PyObject *d, PyObject *filename, const char *loader_name)
 {
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    PyObject *bootstrap = PyObject_GetAttrString(interp->importlib,
+    PyObject *loader_type = _PyImport_GetImportlibExternalLoader(interp,
-                                                 "_bootstrap_external");
+                                                                 loader_name);
-    if (bootstrap == NULL) {
-        return -1;
-    }
-
-    PyObject *loader_type = PyObject_GetAttrString(bootstrap, loader_name);
-    Py_DECREF(bootstrap);
     if (loader_type == NULL) {
         return -1;
     }

Python/sysmodule.c

@@ -142,6 +142,20 @@ PySys_SetObject(const char *name, PyObject *v)
     return sys_set_object_str(interp, name, v);
 }
 
+int
+_PySys_ClearAttrString(PyInterpreterState *interp,
+                       const char *name, int verbose)
+{
+    if (verbose) {
+        PySys_WriteStderr("# clear sys.%s\n", name);
+    }
+    /* To play it safe, we set the attr to None instead of deleting it. */
+    if (PyDict_SetItemString(interp->sysdict, name, Py_None) < 0) {
+        return -1;
+    }
+    return 0;
+}
+
 
 static int
 should_audit(PyInterpreterState *interp)
@@ -1650,7 +1664,7 @@ sys_setdlopenflags_impl(PyObject *module, int new_val)
 /*[clinic end generated code: output=ec918b7fe0a37281 input=4c838211e857a77f]*/
 {
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    interp->dlopenflags = new_val;
+    _PyImport_SetDLOpenFlags(interp, new_val);
     Py_RETURN_NONE;
 }
 
@@ -1668,7 +1682,8 @@ sys_getdlopenflags_impl(PyObject *module)
 /*[clinic end generated code: output=e92cd1bc5005da6e input=dc4ea0899c53b4b6]*/
 {
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    return PyLong_FromLong(interp->dlopenflags);
+    return PyLong_FromLong(
+            _PyImport_GetDLOpenFlags(interp));
 }
 
 #endif  /* HAVE_DLOPEN */
@@ -2279,22 +2294,10 @@ static PyMethodDef sys_methods[] = {
 static PyObject *
 list_builtin_module_names(void)
 {
-    PyObject *list = PyList_New(0);
+    PyObject *list = _PyImport_GetBuiltinModuleNames();
     if (list == NULL) {
         return NULL;
     }
-    struct _inittab *inittab = _PyRuntime.imports.inittab;
-    for (Py_ssize_t i = 0; inittab[i].name != NULL; i++) {
-        PyObject *name = PyUnicode_FromString(inittab[i].name);
-        if (name == NULL) {
-            goto error;
-        }
-        if (PyList_Append(list, name) < 0) {
-            Py_DECREF(name);
-            goto error;
-        }
-        Py_DECREF(name);
-    }
     if (PyList_Sort(list) != 0) {
         goto error;
     }
@@ -3411,11 +3414,10 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p)
 
     PyInterpreterState *interp = tstate->interp;
 
-    PyObject *modules = PyDict_New();
+    PyObject *modules = _PyImport_InitModules(interp);
     if (modules == NULL) {
         goto error;
     }
-    interp->modules = modules;
 
     PyObject *sysmod = _PyModule_CreateInitialized(&sysmodule, PYTHON_API_VERSION);
     if (sysmod == NULL) {
@@ -3428,7 +3430,7 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p)
     }
     interp->sysdict = Py_NewRef(sysdict);
 
-    if (PyDict_SetItemString(sysdict, "modules", interp->modules) < 0) {
+    if (PyDict_SetItemString(sysdict, "modules", modules) < 0) {
         goto error;
     }
 
@@ -3442,7 +3444,7 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p)
         return status;
     }
 
-    if (_PyImport_FixupBuiltin(sysmod, "sys", interp->modules) < 0) {
+    if (_PyImport_FixupBuiltin(sysmod, "sys", modules) < 0) {
         goto error;
     }