From 696c35e86bffea1f2bc6179a29e46416899e3de6 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 25 Jun 2016 10:58:17 -0700 Subject: [PATCH] Issue #26186: Remove the restriction that built-in and extension modules can't be lazily loaded. Thanks to Python 3.6 allowing for types.ModuleType to have its __class__ mutated, the restriction can be lifted by calling create_module() on the wrapped loader. --- Doc/library/importlib.rst | 38 ++++++++++++++++++---------- Doc/whatsnew/3.6.rst | 13 ++++++++++ Lib/importlib/util.py | 24 ++++++++---------- Lib/test/test_importlib/test_lazy.py | 2 ++ 4 files changed, 50 insertions(+), 27 deletions(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 656d51ac0e6..c1c3b124d8e 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -379,10 +379,14 @@ ABC hierarchy:: An abstract method that executes the module in its own namespace when a module is imported or reloaded. The module should already - be initialized when exec_module() is called. + be initialized when ``exec_module()`` is called. When this method exists, + :meth:`~importlib.abc.Loader.create_module` must be defined. .. versionadded:: 3.4 + .. versionchanged:: 3.6 + :meth:`~importlib.abc.Loader.create_module` must also be defined. + .. method:: load_module(fullname) A legacy method for loading a module. If the module cannot be @@ -1200,12 +1204,13 @@ an :term:`importer`. .. function:: module_from_spec(spec) - Create a new module based on **spec** and ``spec.loader.create_module()``. + Create a new module based on **spec** and + :meth:`spec.loader.create_module `. - If ``spec.loader.create_module()`` does not return ``None``, then any - pre-existing attributes will not be reset. Also, no :exc:`AttributeError` - will be raised if triggered while accessing **spec** or setting an attribute - on the module. + If :meth:`spec.loader.create_module ` + does not return ``None``, then any pre-existing attributes will not be reset. + Also, no :exc:`AttributeError` will be raised if triggered while accessing + **spec** or setting an attribute on the module. This function is preferred over using :class:`types.ModuleType` to create a new module as **spec** is used to set as many import-controlled attributes on @@ -1267,7 +1272,8 @@ an :term:`importer`. .. decorator:: set_package - A :term:`decorator` for :meth:`importlib.abc.Loader.load_module` to set the :attr:`__package__` attribute on the returned module. If :attr:`__package__` + A :term:`decorator` for :meth:`importlib.abc.Loader.load_module` to set the + :attr:`__package__` attribute on the returned module. If :attr:`__package__` is set and has a value other than ``None`` it will not be changed. .. deprecated:: 3.4 @@ -1300,13 +1306,12 @@ an :term:`importer`. This class **only** works with loaders that define :meth:`~importlib.abc.Loader.exec_module` as control over what module type is used for the module is required. For those same reasons, the loader's - :meth:`~importlib.abc.Loader.create_module` method will be ignored (i.e., the - loader's method should only return ``None``; this excludes - :class:`BuiltinImporter` and :class:`ExtensionFileLoader`). Finally, - modules which substitute the object placed into :attr:`sys.modules` will - not work as there is no way to properly replace the module references - throughout the interpreter safely; :exc:`ValueError` is raised if such a - substitution is detected. + :meth:`~importlib.abc.Loader.create_module` method must return ``None`` or a + type for which its ``__class__`` attribute can be mutated along with not + using :term:`slots <__slots__>`. Finally, modules which substitute the object + placed into :attr:`sys.modules` will not work as there is no way to properly + replace the module references throughout the interpreter safely; + :exc:`ValueError` is raised if such a substitution is detected. .. note:: For projects where startup time is critical, this class allows for @@ -1317,6 +1322,11 @@ an :term:`importer`. .. versionadded:: 3.5 + .. versionchanged:: 3.6 + Began calling :meth:`~importlib.abc.Loader.create_module`, removing the + compatibility warning for :class:`importlib.machinery.BuiltinImporter` and + :class:`importlib.machinery.ExtensionFileLoader`. + .. classmethod:: factory(loader) A static method which returns a callable that creates a lazy loader. This diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 177b17fb720..ea4dd8c5f15 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -291,6 +291,16 @@ The idlelib package is being modernized and refactored to make IDLE look and wor In compensation, the eventual result with be that some idlelib classes will be easier to use, with better APIs and docstrings explaining them. Additional useful information will be added to idlelib when available. +importlib +--------- + +:class:`importlib.util.LazyLoader` now calls +:meth:`~importlib.abc.Loader.create_module` on the wrapped loader, removing the +restriction that :class:`importlib.machinery.BuiltinImporter` and +:class:`importlib.machinery.ExtensionFileLoader` couldn't be used with +:class:`importlib.util.LazyLoader`. + + os -- @@ -620,6 +630,9 @@ that may require changes to your code. Changes in the Python API ------------------------- +* When :meth:`importlib.abc.Loader.exec_module` is defined, + :meth:`importlib.abc.Loader.create_module` must also be defined. + * The format of the ``co_lnotab`` attribute of code objects changed to support negative line number delta. By default, Python does not emit bytecode with negative line number delta. Functions using ``frame.f_lineno``, diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index a9d0f1e5147..6bdf0d445db 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -204,11 +204,6 @@ def module_for_loader(fxn): return module_for_loader_wrapper -class _Module(types.ModuleType): - - """A subclass of the module type to allow __class__ manipulation.""" - - class _LazyModule(types.ModuleType): """A subclass of the module type which triggers loading upon attribute access.""" @@ -218,13 +213,14 @@ class _LazyModule(types.ModuleType): # All module metadata must be garnered from __spec__ in order to avoid # using mutated values. # Stop triggering this method. - self.__class__ = _Module + self.__class__ = types.ModuleType # Get the original name to make sure no object substitution occurred # in sys.modules. original_name = self.__spec__.name # Figure out exactly what attributes were mutated between the creation # of the module and now. - attrs_then = self.__spec__.loader_state + attrs_then = self.__spec__.loader_state['__dict__'] + original_type = self.__spec__.loader_state['__class__'] attrs_now = self.__dict__ attrs_updated = {} for key, value in attrs_now.items(): @@ -239,9 +235,9 @@ class _LazyModule(types.ModuleType): # object was put into sys.modules. if original_name in sys.modules: if id(self) != id(sys.modules[original_name]): - msg = ('module object for {!r} substituted in sys.modules ' - 'during a lazy load') - raise ValueError(msg.format(original_name)) + raise ValueError(f"module object for {original_name!r} " + "substituted in sys.modules during a lazy " + "load") # Update after loading since that's what would happen in an eager # loading situation. self.__dict__.update(attrs_updated) @@ -275,8 +271,7 @@ class LazyLoader(abc.Loader): self.loader = loader def create_module(self, spec): - """Create a module which can have its __class__ manipulated.""" - return _Module(spec.name) + return self.loader.create_module(spec) def exec_module(self, module): """Make the module load lazily.""" @@ -286,5 +281,8 @@ class LazyLoader(abc.Loader): # on an object would have triggered the load, # e.g. ``module.__spec__.loader = None`` would trigger a load from # trying to access module.__spec__. - module.__spec__.loader_state = module.__dict__.copy() + loader_state = {} + loader_state['__dict__'] = module.__dict__.copy() + loader_state['__class__'] = module.__class__ + module.__spec__.loader_state = loader_state module.__class__ = _LazyModule diff --git a/Lib/test/test_importlib/test_lazy.py b/Lib/test/test_importlib/test_lazy.py index cc383c286de..ffd8dc6cb04 100644 --- a/Lib/test/test_importlib/test_lazy.py +++ b/Lib/test/test_importlib/test_lazy.py @@ -66,6 +66,8 @@ class LazyLoaderTests(unittest.TestCase): spec = util.spec_from_loader(TestingImporter.module_name, util.LazyLoader(loader)) module = spec.loader.create_module(spec) + if module is None: + module = types.ModuleType(TestingImporter.module_name) module.__spec__ = spec module.__loader__ = spec.loader spec.loader.exec_module(module)