Issue #26186: Remove the restriction that built-in and extension

modules  can't be lazily loaded.

Thanks to Python 3.6 allowing for types.ModuleType to have its
__class__ mutated, the restriction can be lifted by calling
create_module() on the wrapped loader.

Doc/library/importlib.rst

@@ -379,10 +379,14 @@ ABC hierarchy::
 
        An abstract method that executes the module in its own namespace
        when a module is imported or reloaded.  The module should already
-       be initialized when exec_module() is called.
+       be initialized when ``exec_module()`` is called. When this method exists,
+       :meth:`~importlib.abc.Loader.create_module` must be defined.
 
        .. versionadded:: 3.4
 
+       .. versionchanged:: 3.6
+          :meth:`~importlib.abc.Loader.create_module` must also be defined.
+
     .. method:: load_module(fullname)
 
         A legacy method for loading a module. If the module cannot be
@@ -1200,12 +1204,13 @@ an :term:`importer`.
 
 .. function:: module_from_spec(spec)
 
-   Create a new module based on **spec** and ``spec.loader.create_module()``.
+   Create a new module based on **spec** and
+   :meth:`spec.loader.create_module <importlib.abc.Loader.create_module>`.
 
-   If ``spec.loader.create_module()`` does not return ``None``, then any
+   If :meth:`spec.loader.create_module <importlib.abc.Loader.create_module>`
-   pre-existing attributes will not be reset. Also, no :exc:`AttributeError`
+   does not return ``None``, then any pre-existing attributes will not be reset.
-   will be raised if triggered while accessing **spec** or setting an attribute
+   Also, no :exc:`AttributeError` will be raised if triggered while accessing
-   on the module.
+   **spec** or setting an attribute on the module.
 
    This function is preferred over using :class:`types.ModuleType` to create a
    new module as **spec** is used to set as many import-controlled attributes on
@@ -1267,7 +1272,8 @@ an :term:`importer`.
 
 .. decorator:: set_package
 
-   A :term:`decorator` for :meth:`importlib.abc.Loader.load_module` to set the :attr:`__package__` attribute on the returned module. If :attr:`__package__`
+   A :term:`decorator` for :meth:`importlib.abc.Loader.load_module` to set the
+   :attr:`__package__` attribute on the returned module. If :attr:`__package__`
    is set and has a value other than ``None`` it will not be changed.
 
    .. deprecated:: 3.4
@@ -1300,13 +1306,12 @@ an :term:`importer`.
    This class **only** works with loaders that define
    :meth:`~importlib.abc.Loader.exec_module` as control over what module type
    is used for the module is required. For those same reasons, the loader's
-   :meth:`~importlib.abc.Loader.create_module` method will be ignored (i.e., the
+   :meth:`~importlib.abc.Loader.create_module` method must return ``None`` or a
-   loader's method should only return ``None``; this excludes
+   type for which its ``__class__`` attribute can be mutated along with not
-   :class:`BuiltinImporter` and :class:`ExtensionFileLoader`). Finally,
+   using :term:`slots <__slots__>`. Finally, modules which substitute the object
-   modules which substitute the object placed into :attr:`sys.modules` will
+   placed into :attr:`sys.modules` will not work as there is no way to properly
-   not work as there is no way to properly replace the module references
+   replace the module references throughout the interpreter safely;
-   throughout the interpreter safely; :exc:`ValueError` is raised if such a
+   :exc:`ValueError` is raised if such a substitution is detected.
-   substitution is detected.
 
    .. note::
       For projects where startup time is critical, this class allows for
@@ -1317,6 +1322,11 @@ an :term:`importer`.
 
    .. versionadded:: 3.5
 
+   .. versionchanged:: 3.6
+      Began calling :meth:`~importlib.abc.Loader.create_module`, removing the
+      compatibility warning for :class:`importlib.machinery.BuiltinImporter` and
+      :class:`importlib.machinery.ExtensionFileLoader`.
+
    .. classmethod:: factory(loader)
 
       A static method which returns a callable that creates a lazy loader. This

Doc/whatsnew/3.6.rst

@@ -291,6 +291,16 @@ The idlelib package is being modernized and refactored to make IDLE look and wor
 In compensation, the eventual result with be that some idlelib classes will be easier to use, with better APIs and docstrings explaining them.  Additional useful information will be added to idlelib when available.
 
 
+importlib
+---------
+
+:class:`importlib.util.LazyLoader` now calls
+:meth:`~importlib.abc.Loader.create_module` on the wrapped loader, removing the
+restriction that :class:`importlib.machinery.BuiltinImporter` and
+:class:`importlib.machinery.ExtensionFileLoader` couldn't be used with
+:class:`importlib.util.LazyLoader`.
+
+
 os
 --
 
@@ -620,6 +630,9 @@ that may require changes to your code.
 Changes in the Python API
 -------------------------
 
+* When :meth:`importlib.abc.Loader.exec_module` is defined,
+  :meth:`importlib.abc.Loader.create_module` must also be defined.
+
 * The format of the ``co_lnotab`` attribute of code objects changed to support
   negative line number delta. By default, Python does not emit bytecode with
   negative line number delta. Functions using ``frame.f_lineno``,

Lib/importlib/util.py

@@ -204,11 +204,6 @@ def module_for_loader(fxn):
     return module_for_loader_wrapper
 
 
-class _Module(types.ModuleType):
-
-    """A subclass of the module type to allow __class__ manipulation."""
-
-
 class _LazyModule(types.ModuleType):
 
     """A subclass of the module type which triggers loading upon attribute access."""
@@ -218,13 +213,14 @@ class _LazyModule(types.ModuleType):
         # All module metadata must be garnered from __spec__ in order to avoid
         # using mutated values.
         # Stop triggering this method.
-        self.__class__ = _Module
+        self.__class__ = types.ModuleType
         # Get the original name to make sure no object substitution occurred
         # in sys.modules.
         original_name = self.__spec__.name
         # Figure out exactly what attributes were mutated between the creation
         # of the module and now.
-        attrs_then = self.__spec__.loader_state
+        attrs_then = self.__spec__.loader_state['__dict__']
+        original_type = self.__spec__.loader_state['__class__']
         attrs_now = self.__dict__
         attrs_updated = {}
         for key, value in attrs_now.items():
@@ -239,9 +235,9 @@ class _LazyModule(types.ModuleType):
         # object was put into sys.modules.
         if original_name in sys.modules:
             if id(self) != id(sys.modules[original_name]):
-                msg = ('module object for {!r} substituted in sys.modules '
+                raise ValueError(f"module object for {original_name!r} "
-                       'during a lazy load')
+                                  "substituted in sys.modules during a lazy "
-                raise ValueError(msg.format(original_name))
+                                  "load")
         # Update after loading since that's what would happen in an eager
         # loading situation.
         self.__dict__.update(attrs_updated)
@@ -275,8 +271,7 @@ class LazyLoader(abc.Loader):
         self.loader = loader
 
     def create_module(self, spec):
-        """Create a module which can have its __class__ manipulated."""
+        return self.loader.create_module(spec)
-        return _Module(spec.name)
 
     def exec_module(self, module):
         """Make the module load lazily."""
@@ -286,5 +281,8 @@ class LazyLoader(abc.Loader):
         # on an object would have triggered the load,
         # e.g. ``module.__spec__.loader = None`` would trigger a load from
         # trying to access module.__spec__.
-        module.__spec__.loader_state = module.__dict__.copy()
+        loader_state = {}
+        loader_state['__dict__'] = module.__dict__.copy()
+        loader_state['__class__'] = module.__class__
+        module.__spec__.loader_state = loader_state
         module.__class__ = _LazyModule

Lib/test/test_importlib/test_lazy.py

@@ -66,6 +66,8 @@ class LazyLoaderTests(unittest.TestCase):
         spec = util.spec_from_loader(TestingImporter.module_name,
                                      util.LazyLoader(loader))
         module = spec.loader.create_module(spec)
+        if module is None:
+            module = types.ModuleType(TestingImporter.module_name)
         module.__spec__ = spec
         module.__loader__ = spec.loader
         spec.loader.exec_module(module)