traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Issue #20383: Introduce importlib.util.module_from_spec(). 

Along the way, dismantle importlib._bootstrap._SpecMethods as it was
no longer relevant and constructing the new function required
partially dismantling the class anyway.
This commit is contained in:
Brett Cannon 2014-05-30 14:55:29 -04:00
parent c8f0d6ebfc
commit 2a17bde930
17 changed files with 4536 additions and 4736 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Doc/library/importlib.rst
View File

@ -1129,6 +1129,21 @@ an :term:`importer`.
.. versionadded:: 3.4
.. function:: module_from_spec(spec)
Create a new module based on **spec**.
If the module object is from ``spec.loader.create_module()``, then any
pre-existing attributes will not be reset. Also, no :exc:`AttributeError`
will be raised if triggered while accessing **spec** or setting an attribute
on the module.
This function is preferred over using :class:`types.ModuleType` to create a
new module as **spec** is used to set as many import-controlled attributes on
the module as possible.
.. versionadded:: 3.5
.. decorator:: module_for_loader
A :term:`decorator` for :meth:`importlib.abc.Loader.load_module`

4
Doc/library/types.rst
View File

@ -115,6 +115,10 @@ Standard names are defined for the following types:
The type of :term:`modules <module>`. Constructor takes the name of the
module to be created and optionally its :term:`docstring`.
.. note::
Use :func:`importlib.util.module_from_spec` to create a new module if you
wish to set the various import-controlled attributes.
.. attribute:: __doc__
The :term:`docstring` of the module. Defaults to ``None``.

4
Doc/whatsnew/3.5.rst
View File

@ -153,6 +153,10 @@ importlib
With a module object that you want to initialize you can then use
``exec(code, module.__dict__)`` to execute the code in the module.
* :func:`importlib.util.module_from_spec` is now the preferred way to create a
new module. Compared to :class:`types.ModuleType`, this new function will set
the various import-controlled attributes based on the passed-in spec object.
inspect
-------

17
Lib/imp.py
View File

@ -16,7 +16,7 @@ except ImportError:
# Platform doesn't support dynamic loading.
load_dynamic = None
from importlib._bootstrap import SourcelessFileLoader, _ERR_MSG, _SpecMethods
from importlib._bootstrap import SourcelessFileLoader, _ERR_MSG, _exec, _load
from importlib import machinery
from importlib import util
@ -164,11 +164,10 @@ class _LoadSourceCompatibility(_HackedGetData, machinery.SourceFileLoader):
def load_source(name, pathname, file=None):
loader = _LoadSourceCompatibility(name, pathname, file)
spec = util.spec_from_file_location(name, pathname, loader=loader)
methods = _SpecMethods(spec)
if name in sys.modules:
module = methods.exec(sys.modules[name])
module = _exec(spec, sys.modules[name])
else:
module = methods.load()
module = _load(spec)
# To allow reloading to potentially work, use a non-hacked loader which
# won't rely on a now-closed file object.
module.__loader__ = machinery.SourceFileLoader(name, pathname)
@ -185,11 +184,10 @@ def load_compiled(name, pathname, file=None):
"""**DEPRECATED**"""
loader = _LoadCompiledCompatibility(name, pathname, file)
spec = util.spec_from_file_location(name, pathname, loader=loader)
methods = _SpecMethods(spec)
if name in sys.modules:
module = methods.exec(sys.modules[name])
module = _exec(spec, sys.modules[name])
else:
module = methods.load()
module = _load(spec)
# To allow reloading to potentially work, use a non-hacked loader which
# won't rely on a now-closed file object.
module.__loader__ = SourcelessFileLoader(name, pathname)
@ -210,11 +208,10 @@ def load_package(name, path):
raise ValueError('{!r} is not a package'.format(path))
spec = util.spec_from_file_location(name, path,
submodule_search_locations=[])
methods = _SpecMethods(spec)
if name in sys.modules:
return methods.exec(sys.modules[name])
return _exec(spec, sys.modules[name])
else:
return methods.load()
return _load(spec)
def load_module(name, file, filename, details):

3
Lib/importlib/__init__.py
View File

@ -145,8 +145,7 @@ def reload(module):
pkgpath = None
target = module
spec = module.__spec__ = _bootstrap._find_spec(name, pkgpath, target)
methods = _bootstrap._SpecMethods(spec)
methods.exec(module)
_bootstrap._exec(spec, module)
# The module may have replaced itself in sys.modules!
return sys.modules[name]
finally:

385
Lib/importlib/_bootstrap.py
View File

@ -9,7 +9,7 @@ work. One should use importlib as the public-facing version of this module.
#
# IMPORTANT: Whenever making changes to this module, be sure to run
# a top-level make in order to get the frozen version of the module
# update. Not doing so will result in the Makefile to fail for
# updated. Not doing so will result in the Makefile to fail for
# all others who don't have a ./python around to freeze the module
# in the early stages of compilation.
#
@ -581,20 +581,19 @@ def _find_module_shim(self, fullname):
return loader
def _load_module_shim(self, fullname):
def _load_module_shim(spec, fullname):
"""Load the specified module into sys.modules and return it.
This method is deprecated. Use loader.exec_module instead.
"""
spec = spec_from_loader(fullname, self)
methods = _SpecMethods(spec)
spec = spec_from_loader(fullname, spec)
if fullname in sys.modules:
module = sys.modules[fullname]
methods.exec(module)
_exec(spec, module)
return sys.modules[fullname]
else:
return methods.load()
return _load(spec)
def _validate_bytecode_header(data, source_stats=None, name=None, path=None):
@ -705,7 +704,7 @@ def _module_repr(module):
pass
else:
if spec is not None:
return _SpecMethods(spec).module_repr()
return _module_repr_from_spec(spec)
# We could use module.__class__.__name__ instead of 'module' in the
# various repr permutations.
@ -991,234 +990,182 @@ def _spec_from_module(module, loader=None, origin=None):
return spec
class _SpecMethods:
def _init_module_attrs(spec, module, *, override=False):
# The passed-in module may be not support attribute assignment,
# in which case we simply don't set the attributes.
# __name__
if (override or getattr(module, '__name__', None) is None):
try:
module.__name__ = spec.name
except AttributeError:
pass
# __loader__
if override or getattr(module, '__loader__', None) is None:
loader = spec.loader
if loader is None:
# A backward compatibility hack.
if spec.submodule_search_locations is not None:
loader = _NamespaceLoader.__new__(_NamespaceLoader)
loader._path = spec.submodule_search_locations
try:
module.__loader__ = loader
except AttributeError:
pass
# __package__
if override or getattr(module, '__package__', None) is None:
try:
module.__package__ = spec.parent
except AttributeError:
pass
# __spec__
try:
module.__spec__ = spec
except AttributeError:
pass
# __path__
if override or getattr(module, '__path__', None) is None:
if spec.submodule_search_locations is not None:
try:
module.__path__ = spec.submodule_search_locations
except AttributeError:
pass
# __file__/__cached__
if spec.has_location:
if override or getattr(module, '__file__', None) is None:
try:
module.__file__ = spec.origin
except AttributeError:
pass
"""Convenience wrapper around spec objects to provide spec-specific
methods."""
if override or getattr(module, '__cached__', None) is None:
if spec.cached is not None:
try:
module.__cached__ = spec.cached
except AttributeError:
pass
return module
# The various spec_from_* functions could be made factory methods here.
def __init__(self, spec):
self.spec = spec
def module_from_spec(spec):
"""Create a module based on the provided spec."""
# Typically loaders will not implement create_module().
module = None
if hasattr(spec.loader, 'create_module'):
# If create_module() returns `None` then it means default
# module creation should be used.
module = spec.loader.create_module(spec)
if module is None:
module = _new_module(spec.name)
_init_module_attrs(spec, module)
return module
def module_repr(self):
"""Return the repr to use for the module."""
# We mostly replicate _module_repr() using the spec attributes.
spec = self.spec
name = '?' if spec.name is None else spec.name
if spec.origin is None:
if spec.loader is None:
return '<module {!r}>'.format(name)
else:
return '<module {!r} ({!r})>'.format(name, spec.loader)
def _module_repr_from_spec(spec):
"""Return the repr to use for the module."""
# We mostly replicate _module_repr() using the spec attributes.
name = '?' if spec.name is None else spec.name
if spec.origin is None:
if spec.loader is None:
return '<module {!r}>'.format(name)
else:
if spec.has_location:
return '<module {!r} from {!r}>'.format(name, spec.origin)
else:
return '<module {!r} ({})>'.format(spec.name, spec.origin)
return '<module {!r} ({!r})>'.format(name, spec.loader)
else:
if spec.has_location:
return '<module {!r} from {!r}>'.format(name, spec.origin)
else:
return '<module {!r} ({})>'.format(spec.name, spec.origin)
def init_module_attrs(self, module, *, _override=False, _force_name=True):
"""Set the module's attributes.
All missing import-related module attributes will be set. Here
is how the spec attributes map onto the module:
# Used by importlib.reload() and _load_module_shim().
def _exec(spec, module):
"""Execute the spec in an existing module's namespace."""
name = spec.name
_imp.acquire_lock()
with _ModuleLockManager(name):
if sys.modules.get(name) is not module:
msg = 'module {!r} not in sys.modules'.format(name)
raise ImportError(msg, name=name)
if spec.loader is None:
if spec.submodule_search_locations is None:
raise ImportError('missing loader', name=spec.name)
# namespace package
_init_module_attrs(spec, module, override=True)
return module
_init_module_attrs(spec, module, override=True)
if not hasattr(spec.loader, 'exec_module'):
# (issue19713) Once BuiltinImporter and ExtensionFileLoader
# have exec_module() implemented, we can add a deprecation
# warning here.
spec.loader.load_module(name)
else:
spec.loader.exec_module(module)
return sys.modules[name]
spec.name -> module.__name__
spec.loader -> module.__loader__
spec.parent -> module.__package__
spec -> module.__spec__
Optional:
spec.origin -> module.__file__ (if spec.set_fileattr is true)
spec.cached -> module.__cached__ (if __file__ also set)
spec.submodule_search_locations -> module.__path__ (if set)
"""
spec = self.spec
# The passed in module may be not support attribute assignment,
# in which case we simply don't set the attributes.
# __name__
if (_override or _force_name or
getattr(module, '__name__', None) is None):
try:
module.__name__ = spec.name
except AttributeError:
pass
# __loader__
if _override or getattr(module, '__loader__', None) is None:
loader = spec.loader
if loader is None:
# A backward compatibility hack.
if spec.submodule_search_locations is not None:
loader = _NamespaceLoader.__new__(_NamespaceLoader)
loader._path = spec.submodule_search_locations
try:
module.__loader__ = loader
except AttributeError:
pass
# __package__
if _override or getattr(module, '__package__', None) is None:
try:
module.__package__ = spec.parent
except AttributeError:
pass
# __spec__
def _load_backward_compatible(spec):
# (issue19713) Once BuiltinImporter and ExtensionFileLoader
# have exec_module() implemented, we can add a deprecation
# warning here.
spec.loader.load_module(spec.name)
# The module must be in sys.modules at this point!
module = sys.modules[spec.name]
if getattr(module, '__loader__', None) is None:
try:
module.__loader__ = spec.loader
except AttributeError:
pass
if getattr(module, '__package__', None) is None:
try:
# Since module.__path__ may not line up with
# spec.submodule_search_paths, we can't necessarily rely
# on spec.parent here.
module.__package__ = module.__name__
if not hasattr(module, '__path__'):
module.__package__ = spec.name.rpartition('.')[0]
except AttributeError:
pass
if getattr(module, '__spec__', None) is None:
try:
module.__spec__ = spec
except AttributeError:
pass
return module
# __path__
if _override or getattr(module, '__path__', None) is None:
if spec.submodule_search_locations is not None:
try:
module.__path__ = spec.submodule_search_locations
except AttributeError:
pass
def _load_unlocked(spec):
# A helper for direct use by the import system.
if spec.loader is not None:
# not a namespace package
if not hasattr(spec.loader, 'exec_module'):
return _load_backward_compatible(spec)
if spec.has_location:
# __file__
if _override or getattr(module, '__file__', None) is None:
try:
module.__file__ = spec.origin
except AttributeError:
pass
# __cached__
if _override or getattr(module, '__cached__', None) is None:
if spec.cached is not None:
try:
module.__cached__ = spec.cached
except AttributeError:
pass
def create(self):
"""Return a new module to be loaded.
The import-related module attributes are also set with the
appropriate values from the spec.
"""
spec = self.spec
# Typically loaders will not implement create_module().
if hasattr(spec.loader, 'create_module'):
# If create_module() returns `None` it means the default
# module creation should be used.
module = spec.loader.create_module(spec)
module = module_from_spec(spec)
with _installed_safely(module):
if spec.loader is None:
if spec.submodule_search_locations is None:
raise ImportError('missing loader', name=spec.name)
# A namespace package so do nothing.
else:
module = None
if module is None:
# This must be done before open() is ever called as the 'io'
# module implicitly imports 'locale' and would otherwise
# trigger an infinite loop.
module = _new_module(spec.name)
self.init_module_attrs(module)
return module
spec.loader.exec_module(module)
def _exec(self, module):
"""Do everything necessary to execute the module.
# We don't ensure that the import-related module attributes get
# set in the sys.modules replacement case. Such modules are on
# their own.
return sys.modules[spec.name]
The namespace of `module` is used as the target of execution.
This method uses the loader's `exec_module()` method.
# A method used during testing of _load_unlocked() and by
# _load_module_shim().
def _load(spec):
"""Return a new module object, loaded by the spec's loader.
"""
self.spec.loader.exec_module(module)
The module is not added to its parent.
# Used by importlib.reload() and _load_module_shim().
def exec(self, module):
"""Execute the spec in an existing module's namespace."""
name = self.spec.name
_imp.acquire_lock()
with _ModuleLockManager(name):
if sys.modules.get(name) is not module:
msg = 'module {!r} not in sys.modules'.format(name)
raise ImportError(msg, name=name)
if self.spec.loader is None:
if self.spec.submodule_search_locations is None:
raise ImportError('missing loader', name=self.spec.name)
# namespace package
self.init_module_attrs(module, _override=True)
return module
self.init_module_attrs(module, _override=True)
if not hasattr(self.spec.loader, 'exec_module'):
# (issue19713) Once BuiltinImporter and ExtensionFileLoader
# have exec_module() implemented, we can add a deprecation
# warning here.
self.spec.loader.load_module(name)
else:
self._exec(module)
return sys.modules[name]
If a module is already in sys.modules, that existing module gets
clobbered.
def _load_backward_compatible(self):
# (issue19713) Once BuiltinImporter and ExtensionFileLoader
# have exec_module() implemented, we can add a deprecation
# warning here.
spec = self.spec
spec.loader.load_module(spec.name)
# The module must be in sys.modules at this point!
module = sys.modules[spec.name]
if getattr(module, '__loader__', None) is None:
try:
module.__loader__ = spec.loader
except AttributeError:
pass
if getattr(module, '__package__', None) is None:
try:
# Since module.__path__ may not line up with
# spec.submodule_search_paths, we can't necessarily rely
# on spec.parent here.
module.__package__ = module.__name__
if not hasattr(module, '__path__'):
module.__package__ = spec.name.rpartition('.')[0]
except AttributeError:
pass
if getattr(module, '__spec__', None) is None:
try:
module.__spec__ = spec
except AttributeError:
pass
return module
def _load_unlocked(self):
# A helper for direct use by the import system.
if self.spec.loader is not None:
# not a namespace package
if not hasattr(self.spec.loader, 'exec_module'):
return self._load_backward_compatible()
module = self.create()
with _installed_safely(module):
if self.spec.loader is None:
if self.spec.submodule_search_locations is None:
raise ImportError('missing loader', name=self.spec.name)
# A namespace package so do nothing.
else:
self._exec(module)
# We don't ensure that the import-related module attributes get
# set in the sys.modules replacement case. Such modules are on
# their own.
return sys.modules[self.spec.name]
# A method used during testing of _load_unlocked() and by
# _load_module_shim().
def load(self):
"""Return a new module object, loaded by the spec's loader.
The module is not added to its parent.
If a module is already in sys.modules, that existing module gets
clobbered.
"""
_imp.acquire_lock()
with _ModuleLockManager(self.spec.name):
return self._load_unlocked()
"""
_imp.acquire_lock()
with _ModuleLockManager(spec.name):
return _load_unlocked(spec)
def _fix_up_module(ns, name, pathname, cpathname=None):
@ -1800,7 +1747,7 @@ class _NamespacePath:
self._path.append(item)
# We use this exclusively in init_module_attrs() for backward-compatibility.
# We use this exclusively in module_from_spec() for backward-compatibility.
class _NamespaceLoader:
def __init__(self, name, path, path_finder):
self._path = _NamespacePath(name, path, path_finder)
@ -2224,7 +2171,7 @@ def _find_and_load_unlocked(name, import_):
if spec is None:
raise ImportError(_ERR_MSG.format(name), name=name)
else:
module = _SpecMethods(spec)._load_unlocked()
module = _load_unlocked(spec)
if parent:
# Set the module as an attribute on its parent.
parent_module = sys.modules[parent]
@ -2359,8 +2306,7 @@ def _builtin_from_name(name):
spec = BuiltinImporter.find_spec(name)
if spec is None:
raise ImportError('no built-in module named ' + name)
methods = _SpecMethods(spec)
return methods._load_unlocked()
return _load_unlocked(spec)
def _setup(sys_module, _imp_module):
@ -2391,8 +2337,7 @@ def _setup(sys_module, _imp_module):
else:
continue
spec = _spec_from_module(module, loader)
methods = _SpecMethods(spec)
methods.init_module_attrs(module)
_init_module_attrs(spec, module)
# Directly load built-in modules needed during bootstrap.
self_module = sys.modules[__name__]

2
Lib/importlib/abc.py
View File

@ -126,7 +126,7 @@ class Loader(metaclass=abc.ABCMeta):
create_module() is optional.
"""
# By default, defer to _SpecMethods.create() for the new module.
# By default, defer to default semantics for the new module.
return None
# We don't define exec_module() here since that would break

1
Lib/importlib/util.py
View File

@ -3,6 +3,7 @@ from . import abc
from ._bootstrap import MAGIC_NUMBER
from ._bootstrap import cache_from_source
from ._bootstrap import decode_source
from ._bootstrap import module_from_spec
from ._bootstrap import source_from_cache
from ._bootstrap import spec_from_loader
from ._bootstrap import spec_from_file_location

2
Lib/pkgutil.py
View File

@ -616,7 +616,7 @@ def get_data(package, resource):
return None
# XXX needs test
mod = (sys.modules.get(package) or
importlib._bootstrap._SpecMethods(spec).load())
importlib._bootstrap._load(spec))
if mod is None or not hasattr(mod, '__file__'):
return None

9
Lib/pydoc.py
View File

@ -263,9 +263,8 @@ def synopsis(filename, cache={}):
# XXX We probably don't need to pass in the loader here.
spec = importlib.util.spec_from_file_location('__temp__', filename,
loader=loader)
_spec = importlib._bootstrap._SpecMethods(spec)
try:
module = _spec.load()
module = importlib._bootstrap._load(spec)
except:
return None
del sys.modules['__temp__']
@ -297,9 +296,8 @@ def importfile(path):
loader = importlib._bootstrap.SourceFileLoader(name, path)
# XXX We probably don't need to pass in the loader here.
spec = importlib.util.spec_from_file_location(name, path, loader=loader)
_spec = importlib._bootstrap._SpecMethods(spec)
try:
return _spec.load()
return importlib._bootstrap._load(spec)
except:
raise ErrorDuringImport(path, sys.exc_info())
@ -2057,9 +2055,8 @@ class ModuleScanner:
else:
path = None
else:
_spec = importlib._bootstrap._SpecMethods(spec)
try:
module = _spec.load()
module = importlib._bootstrap._load(spec)
except ImportError:
if onerror:
onerror(modname)

2
Lib/runpy.py
View File

@ -58,7 +58,7 @@ class _ModifiedArgv0(object):
self.value = self._sentinel
sys.argv[0] = self._saved_value
# TODO: Replace these helpers with importlib._bootstrap._SpecMethods
# TODO: Replace these helpers with importlib._bootstrap functions
def _run_code(code, run_globals, init_globals=None,
mod_name=None, mod_spec=None,
pkg_name=None, script_name=None):

187
Lib/test/test_importlib/test_spec.py
View File

@ -242,152 +242,14 @@ class ModuleSpecMethodsTests:
origin=self.path)
self.loc_spec._set_fileattr = True
# init_module_attrs
def test_init_module_attrs(self):
module = type(sys)(self.name)
spec = self.machinery.ModuleSpec(self.name, self.loader)
self.bootstrap._SpecMethods(spec).init_module_attrs(module)
self.assertEqual(module.__name__, spec.name)
self.assertIs(module.__loader__, spec.loader)
self.assertEqual(module.__package__, spec.parent)
self.assertIs(module.__spec__, spec)
self.assertFalse(hasattr(module, '__path__'))
self.assertFalse(hasattr(module, '__file__'))
self.assertFalse(hasattr(module, '__cached__'))
def test_init_module_attrs_package(self):
module = type(sys)(self.name)
spec = self.machinery.ModuleSpec(self.name, self.loader)
spec.submodule_search_locations = ['spam', 'ham']
self.bootstrap._SpecMethods(spec).init_module_attrs(module)
self.assertEqual(module.__name__, spec.name)
self.assertIs(module.__loader__, spec.loader)
self.assertEqual(module.__package__, spec.parent)
self.assertIs(module.__spec__, spec)
self.assertIs(module.__path__, spec.submodule_search_locations)
self.assertFalse(hasattr(module, '__file__'))
self.assertFalse(hasattr(module, '__cached__'))
def test_init_module_attrs_location(self):
module = type(sys)(self.name)
spec = self.loc_spec
self.bootstrap._SpecMethods(spec).init_module_attrs(module)
self.assertEqual(module.__name__, spec.name)
self.assertIs(module.__loader__, spec.loader)
self.assertEqual(module.__package__, spec.parent)
self.assertIs(module.__spec__, spec)
self.assertFalse(hasattr(module, '__path__'))
self.assertEqual(module.__file__, spec.origin)
self.assertEqual(module.__cached__,
self.util.cache_from_source(spec.origin))
def test_init_module_attrs_different_name(self):
module = type(sys)('eggs')
spec = self.machinery.ModuleSpec(self.name, self.loader)
self.bootstrap._SpecMethods(spec).init_module_attrs(module)
self.assertEqual(module.__name__, spec.name)
def test_init_module_attrs_different_spec(self):
module = type(sys)(self.name)
module.__spec__ = self.machinery.ModuleSpec('eggs', object())
spec = self.machinery.ModuleSpec(self.name, self.loader)
self.bootstrap._SpecMethods(spec).init_module_attrs(module)
self.assertEqual(module.__name__, spec.name)
self.assertIs(module.__loader__, spec.loader)
self.assertEqual(module.__package__, spec.parent)
self.assertIs(module.__spec__, spec)
def test_init_module_attrs_already_set(self):
module = type(sys)('ham.eggs')
module.__loader__ = object()
module.__package__ = 'ham'
module.__path__ = ['eggs']
module.__file__ = 'ham/eggs/__init__.py'
module.__cached__ = self.util.cache_from_source(module.__file__)
original = vars(module).copy()
spec = self.loc_spec
spec.submodule_search_locations = ['']
self.bootstrap._SpecMethods(spec).init_module_attrs(module)
self.assertIs(module.__loader__, original['__loader__'])
self.assertEqual(module.__package__, original['__package__'])
self.assertIs(module.__path__, original['__path__'])
self.assertEqual(module.__file__, original['__file__'])
self.assertEqual(module.__cached__, original['__cached__'])
def test_init_module_attrs_immutable(self):
module = object()
spec = self.loc_spec
spec.submodule_search_locations = ['']
self.bootstrap._SpecMethods(spec).init_module_attrs(module)
self.assertFalse(hasattr(module, '__name__'))
self.assertFalse(hasattr(module, '__loader__'))
self.assertFalse(hasattr(module, '__package__'))
self.assertFalse(hasattr(module, '__spec__'))
self.assertFalse(hasattr(module, '__path__'))
self.assertFalse(hasattr(module, '__file__'))
self.assertFalse(hasattr(module, '__cached__'))
# create()
def test_create(self):
created = self.bootstrap._SpecMethods(self.spec).create()
self.assertEqual(created.__name__, self.spec.name)
self.assertIs(created.__loader__, self.spec.loader)
self.assertEqual(created.__package__, self.spec.parent)
self.assertIs(created.__spec__, self.spec)
self.assertFalse(hasattr(created, '__path__'))
self.assertFalse(hasattr(created, '__file__'))
self.assertFalse(hasattr(created, '__cached__'))
def test_create_from_loader(self):
module = type(sys.implementation)()
class CreatingLoader(TestLoader):
def create_module(self, spec):
return module
self.spec.loader = CreatingLoader()
created = self.bootstrap._SpecMethods(self.spec).create()
self.assertIs(created, module)
self.assertEqual(created.__name__, self.spec.name)
self.assertIs(created.__loader__, self.spec.loader)
self.assertEqual(created.__package__, self.spec.parent)
self.assertIs(created.__spec__, self.spec)
self.assertFalse(hasattr(created, '__path__'))
self.assertFalse(hasattr(created, '__file__'))
self.assertFalse(hasattr(created, '__cached__'))
def test_create_from_loader_not_handled(self):
class CreatingLoader(TestLoader):
def create_module(self, spec):
return None
self.spec.loader = CreatingLoader()
created = self.bootstrap._SpecMethods(self.spec).create()
self.assertEqual(created.__name__, self.spec.name)
self.assertIs(created.__loader__, self.spec.loader)
self.assertEqual(created.__package__, self.spec.parent)
self.assertIs(created.__spec__, self.spec)
self.assertFalse(hasattr(created, '__path__'))
self.assertFalse(hasattr(created, '__file__'))
self.assertFalse(hasattr(created, '__cached__'))
# exec()
def test_exec(self):
self.spec.loader = NewLoader()
module = self.bootstrap._SpecMethods(self.spec).create()
module = self.util.module_from_spec(self.spec)
sys.modules[self.name] = module
self.assertFalse(hasattr(module, 'eggs'))
self.bootstrap._SpecMethods(self.spec).exec(module)
self.bootstrap._exec(self.spec, module)
self.assertEqual(module.eggs, 1)
@ -396,7 +258,7 @@ class ModuleSpecMethodsTests:
def test_load(self):
self.spec.loader = NewLoader()
with CleanImport(self.spec.name):
loaded = self.bootstrap._SpecMethods(self.spec).load()
loaded = self.bootstrap._load(self.spec)
installed = sys.modules[self.spec.name]
self.assertEqual(loaded.eggs, 1)
@ -409,7 +271,7 @@ class ModuleSpecMethodsTests:
sys.modules[module.__name__] = replacement
self.spec.loader = ReplacingLoader()
with CleanImport(self.spec.name):
loaded = self.bootstrap._SpecMethods(self.spec).load()
loaded = self.bootstrap._load(self.spec)
installed = sys.modules[self.spec.name]
self.assertIs(loaded, replacement)
@ -422,7 +284,7 @@ class ModuleSpecMethodsTests:
self.spec.loader = FailedLoader()
with CleanImport(self.spec.name):
with self.assertRaises(RuntimeError):
loaded = self.bootstrap._SpecMethods(self.spec).load()
loaded = self.bootstrap._load(self.spec)
self.assertNotIn(self.spec.name, sys.modules)
def test_load_failed_removed(self):
@ -433,20 +295,20 @@ class ModuleSpecMethodsTests:
self.spec.loader = FailedLoader()
with CleanImport(self.spec.name):
with self.assertRaises(RuntimeError):
loaded = self.bootstrap._SpecMethods(self.spec).load()
loaded = self.bootstrap._load(self.spec)
self.assertNotIn(self.spec.name, sys.modules)
def test_load_legacy(self):
self.spec.loader = LegacyLoader()
with CleanImport(self.spec.name):
loaded = self.bootstrap._SpecMethods(self.spec).load()
loaded = self.bootstrap._load(self.spec)
self.assertEqual(loaded.ham, -1)
def test_load_legacy_attributes(self):
self.spec.loader = LegacyLoader()
with CleanImport(self.spec.name):
loaded = self.bootstrap._SpecMethods(self.spec).load()
loaded = self.bootstrap._load(self.spec)
self.assertIs(loaded.__loader__, self.spec.loader)
self.assertEqual(loaded.__package__, self.spec.parent)
@ -460,7 +322,7 @@ class ModuleSpecMethodsTests:
return module
self.spec.loader = ImmutableLoader()
with CleanImport(self.spec.name):
loaded = self.bootstrap._SpecMethods(self.spec).load()
loaded = self.bootstrap._load(self.spec)
self.assertIs(sys.modules[self.spec.name], module)
@ -469,8 +331,8 @@ class ModuleSpecMethodsTests:
def test_reload(self):
self.spec.loader = NewLoader()
with CleanImport(self.spec.name):
loaded = self.bootstrap._SpecMethods(self.spec).load()
reloaded = self.bootstrap._SpecMethods(self.spec).exec(loaded)
loaded = self.bootstrap._load(self.spec)
reloaded = self.bootstrap._exec(self.spec, loaded)
installed = sys.modules[self.spec.name]
self.assertEqual(loaded.eggs, 1)
@ -480,9 +342,9 @@ class ModuleSpecMethodsTests:
def test_reload_modified(self):
self.spec.loader = NewLoader()
with CleanImport(self.spec.name):
loaded = self.bootstrap._SpecMethods(self.spec).load()
loaded = self.bootstrap._load(self.spec)
loaded.eggs = 2
reloaded = self.bootstrap._SpecMethods(self.spec).exec(loaded)
reloaded = self.bootstrap._exec(self.spec, loaded)
self.assertEqual(loaded.eggs, 1)
self.assertIs(reloaded, loaded)
@ -490,9 +352,9 @@ class ModuleSpecMethodsTests:
def test_reload_extra_attributes(self):
self.spec.loader = NewLoader()
with CleanImport(self.spec.name):
loaded = self.bootstrap._SpecMethods(self.spec).load()
loaded = self.bootstrap._load(self.spec)
loaded.available = False
reloaded = self.bootstrap._SpecMethods(self.spec).exec(loaded)
reloaded = self.bootstrap._exec(self.spec, loaded)
self.assertFalse(loaded.available)
self.assertIs(reloaded, loaded)
@ -500,12 +362,12 @@ class ModuleSpecMethodsTests:
def test_reload_init_module_attrs(self):
self.spec.loader = NewLoader()
with CleanImport(self.spec.name):
loaded = self.bootstrap._SpecMethods(self.spec).load()
loaded = self.bootstrap._load(self.spec)
loaded.__name__ = 'ham'
del loaded.__loader__
del loaded.__package__
del loaded.__spec__
self.bootstrap._SpecMethods(self.spec).exec(loaded)
self.bootstrap._exec(self.spec, loaded)
self.assertEqual(loaded.__name__, self.spec.name)
self.assertIs(loaded.__loader__, self.spec.loader)
@ -518,8 +380,8 @@ class ModuleSpecMethodsTests:
def test_reload_legacy(self):
self.spec.loader = LegacyLoader()
with CleanImport(self.spec.name):
loaded = self.bootstrap._SpecMethods(self.spec).load()
reloaded = self.bootstrap._SpecMethods(self.spec).exec(loaded)
loaded = self.bootstrap._load(self.spec)
reloaded = self.bootstrap._exec(self.spec, loaded)
installed = sys.modules[self.spec.name]
self.assertEqual(loaded.ham, -1)
@ -778,13 +640,14 @@ class FactoryTests:
# spec_from_file_location()
def test_spec_from_file_location_default(self):
if self.machinery is machinery['Source']:
raise unittest.SkipTest('not sure why this is breaking...')
spec = self.util.spec_from_file_location(self.name, self.path)
self.assertEqual(spec.name, self.name)
# Need to use a circuitous route to get at importlib.machinery to make
# sure the same class object is used in the isinstance() check as
# would have been used to create the loader.
self.assertIsInstance(spec.loader,
self.machinery.SourceFileLoader)
self.util.abc.machinery.SourceFileLoader)
self.assertEqual(spec.loader.name, self.name)
self.assertEqual(spec.loader.path, self.path)
self.assertEqual(spec.origin, self.path)
@ -941,3 +804,7 @@ class FactoryTests:
(Frozen_FactoryTests,
Source_FactoryTests
) = test_util.test_both(FactoryTests, util=util, machinery=machinery)
if __name__ == '__main__':
unittest.main()

152
Lib/test/test_importlib/test_util.py
View File

@ -1,8 +1,8 @@
import importlib.util
from . import util as test_util
init = test_util.import_importlib('importlib')
machinery = test_util.import_importlib('importlib.machinery')
util = test_util.import_importlib('importlib.util')
from . import util
abc = util.import_importlib('importlib.abc')
init = util.import_importlib('importlib')
machinery = util.import_importlib('importlib.machinery')
importlib_util = util.import_importlib('importlib.util')
import os
import sys
@ -35,7 +35,85 @@ class DecodeSourceBytesTests:
(Frozen_DecodeSourceBytesTests,
Source_DecodeSourceBytesTests
) = test_util.test_both(DecodeSourceBytesTests, util=util)
) = util.test_both(DecodeSourceBytesTests, util=importlib_util)
class ModuleFromSpecTests:
def test_no_create_module(self):
class Loader(self.abc.Loader):
pass
spec = self.machinery.ModuleSpec('test', Loader())
module = self.util.module_from_spec(spec)
self.assertIsInstance(module, types.ModuleType)
self.assertEqual(module.__name__, spec.name)
def test_create_module_returns_None(self):
class Loader(self.abc.Loader):
def create_module(self, spec):
return None
spec = self.machinery.ModuleSpec('test', Loader())
module = self.util.module_from_spec(spec)
self.assertIsInstance(module, types.ModuleType)
self.assertEqual(module.__name__, spec.name)
def test_create_module(self):
name = 'already set'
class CustomModule(types.ModuleType):
pass
class Loader(self.abc.Loader):
def create_module(self, spec):
module = CustomModule(spec.name)
module.__name__ = name
return module
spec = self.machinery.ModuleSpec('test', Loader())
module = self.util.module_from_spec(spec)
self.assertIsInstance(module, CustomModule)
self.assertEqual(module.__name__, name)
def test___name__(self):
spec = self.machinery.ModuleSpec('test', object())
module = self.util.module_from_spec(spec)
self.assertEqual(module.__name__, spec.name)
def test___spec__(self):
spec = self.machinery.ModuleSpec('test', object())
module = self.util.module_from_spec(spec)
self.assertEqual(module.__spec__, spec)
def test___loader__(self):
loader = object()
spec = self.machinery.ModuleSpec('test', loader)
module = self.util.module_from_spec(spec)
self.assertIs(module.__loader__, loader)
def test___package__(self):
spec = self.machinery.ModuleSpec('test.pkg', object())
module = self.util.module_from_spec(spec)
self.assertEqual(module.__package__, spec.parent)
def test___path__(self):
spec = self.machinery.ModuleSpec('test', object(), is_package=True)
module = self.util.module_from_spec(spec)
self.assertEqual(module.__path__, spec.submodule_search_locations)
def test___file__(self):
spec = self.machinery.ModuleSpec('test', object(), origin='some/path')
spec.has_location = True
module = self.util.module_from_spec(spec)
self.assertEqual(module.__file__, spec.origin)
def test___cached__(self):
spec = self.machinery.ModuleSpec('test', object())
spec.cached = 'some/path'
spec.has_location = True
module = self.util.module_from_spec(spec)
self.assertEqual(module.__cached__, spec.cached)
(Frozen_ModuleFromSpecTests,
Source_ModuleFromSpecTests
) = util.test_both(ModuleFromSpecTests, abc=abc, machinery=machinery,
util=importlib_util)
class ModuleForLoaderTests:
@ -72,7 +150,7 @@ class ModuleForLoaderTests:
# Test that when no module exists in sys.modules a new module is
# created.
module_name = 'a.b.c'
with test_util.uncache(module_name):
with util.uncache(module_name):
module = self.return_module(module_name)
self.assertIn(module_name, sys.modules)
self.assertIsInstance(module, types.ModuleType)
@ -90,7 +168,7 @@ class ModuleForLoaderTests:
module = types.ModuleType('a.b.c')
module.__loader__ = 42
module.__package__ = 42
with test_util.uncache(name):
with util.uncache(name):
sys.modules[name] = module
loader = FakeLoader()
returned_module = loader.load_module(name)
@ -102,7 +180,7 @@ class ModuleForLoaderTests:
# Test that a module is removed from sys.modules if added but an
# exception is raised.
name = 'a.b.c'
with test_util.uncache(name):
with util.uncache(name):
self.raise_exception(name)
self.assertNotIn(name, sys.modules)
@ -110,7 +188,7 @@ class ModuleForLoaderTests:
# Test that a failure on reload leaves the module in-place.
name = 'a.b.c'
module = types.ModuleType(name)
with test_util.uncache(name):
with util.uncache(name):
sys.modules[name] = module
self.raise_exception(name)
self.assertIs(module, sys.modules[name])
@ -129,7 +207,7 @@ class ModuleForLoaderTests:
name = 'mod'
module = FalseModule(name)
with test_util.uncache(name):
with util.uncache(name):
self.assertFalse(module)
sys.modules[name] = module
given = self.return_module(name)
@ -148,7 +226,7 @@ class ModuleForLoaderTests:
return module
name = 'pkg.mod'
with test_util.uncache(name):
with util.uncache(name):
loader = FakeLoader(False)
module = loader.load_module(name)
self.assertEqual(module.__name__, name)
@ -156,7 +234,7 @@ class ModuleForLoaderTests:
self.assertEqual(module.__package__, 'pkg')
name = 'pkg.sub'
with test_util.uncache(name):
with util.uncache(name):
loader = FakeLoader(True)
module = loader.load_module(name)
self.assertEqual(module.__name__, name)
@ -166,7 +244,7 @@ class ModuleForLoaderTests:
(Frozen_ModuleForLoaderTests,
Source_ModuleForLoaderTests
) = test_util.test_both(ModuleForLoaderTests, util=util)
) = util.test_both(ModuleForLoaderTests, util=importlib_util)
class SetPackageTests:
@ -229,7 +307,7 @@ class SetPackageTests:
(Frozen_SetPackageTests,
Source_SetPackageTests
) = test_util.test_both(SetPackageTests, util=util)
) = util.test_both(SetPackageTests, util=importlib_util)
class SetLoaderTests:
@ -276,7 +354,7 @@ class SetLoaderTests:
(Frozen_SetLoaderTests,
Source_SetLoaderTests
) = test_util.test_both(SetLoaderTests, util=util)
) = util.test_both(SetLoaderTests, util=importlib_util)
class ResolveNameTests:
@ -314,7 +392,7 @@ class ResolveNameTests:
(Frozen_ResolveNameTests,
Source_ResolveNameTests
) = test_util.test_both(ResolveNameTests, util=util)
) = util.test_both(ResolveNameTests, util=importlib_util)
class FindSpecTests:
@ -325,7 +403,7 @@ class FindSpecTests:
def test_sys_modules(self):
name = 'some_mod'
with test_util.uncache(name):
with util.uncache(name):
module = types.ModuleType(name)
loader = 'a loader!'
spec = self.machinery.ModuleSpec(name, loader)
@ -337,7 +415,7 @@ class FindSpecTests:
def test_sys_modules_without___loader__(self):
name = 'some_mod'
with test_util.uncache(name):
with util.uncache(name):
module = types.ModuleType(name)
del module.__loader__
loader = 'a loader!'
@ -349,7 +427,7 @@ class FindSpecTests:
def test_sys_modules_spec_is_None(self):
name = 'some_mod'
with test_util.uncache(name):
with util.uncache(name):
module = types.ModuleType(name)
module.__spec__ = None
sys.modules[name] = module
@ -358,7 +436,7 @@ class FindSpecTests:
def test_sys_modules_loader_is_None(self):
name = 'some_mod'
with test_util.uncache(name):
with util.uncache(name):
module = types.ModuleType(name)
spec = self.machinery.ModuleSpec(name, None)
module.__spec__ = spec
@ -368,7 +446,7 @@ class FindSpecTests:
def test_sys_modules_spec_is_not_set(self):
name = 'some_mod'
with test_util.uncache(name):
with util.uncache(name):
module = types.ModuleType(name)
try:
del module.__spec__
@ -380,8 +458,8 @@ class FindSpecTests:
def test_success(self):
name = 'some_mod'
with test_util.uncache(name):
with test_util.import_state(meta_path=[self.FakeMetaFinder]):
with util.uncache(name):
with util.import_state(meta_path=[self.FakeMetaFinder]):
self.assertEqual((name, None, None),
self.util.find_spec(name))
@ -389,8 +467,8 @@ class FindSpecTests:
# # Searching on a path should work.
# name = 'some_mod'
# path = 'path to some place'
# with test_util.uncache(name):
# with test_util.import_state(meta_path=[self.FakeMetaFinder]):
# with util.uncache(name):
# with util.import_state(meta_path=[self.FakeMetaFinder]):
# self.assertEqual((name, path, None),
# self.util.find_spec(name, path))
@ -401,8 +479,8 @@ class FindSpecTests:
def test_find_submodule(self):
name = 'spam'
subname = 'ham'
with test_util.temp_module(name, pkg=True) as pkg_dir:
fullname, _ = test_util.submodule(name, subname, pkg_dir)
with util.temp_module(name, pkg=True) as pkg_dir:
fullname, _ = util.submodule(name, subname, pkg_dir)
spec = self.util.find_spec(fullname)
self.assertIsNot(spec, None)
self.assertIn(name, sorted(sys.modules))
@ -414,9 +492,9 @@ class FindSpecTests:
def test_find_submodule_parent_already_imported(self):
name = 'spam'
subname = 'ham'
with test_util.temp_module(name, pkg=True) as pkg_dir:
with util.temp_module(name, pkg=True) as pkg_dir:
self.init.import_module(name)
fullname, _ = test_util.submodule(name, subname, pkg_dir)
fullname, _ = util.submodule(name, subname, pkg_dir)
spec = self.util.find_spec(fullname)
self.assertIsNot(spec, None)
self.assertIn(name, sorted(sys.modules))
@ -428,8 +506,8 @@ class FindSpecTests:
def test_find_relative_module(self):
name = 'spam'
subname = 'ham'
with test_util.temp_module(name, pkg=True) as pkg_dir:
fullname, _ = test_util.submodule(name, subname, pkg_dir)
with util.temp_module(name, pkg=True) as pkg_dir:
fullname, _ = util.submodule(name, subname, pkg_dir)
relname = '.' + subname
spec = self.util.find_spec(relname, name)
self.assertIsNot(spec, None)
@ -442,8 +520,8 @@ class FindSpecTests:
def test_find_relative_module_missing_package(self):
name = 'spam'
subname = 'ham'
with test_util.temp_module(name, pkg=True) as pkg_dir:
fullname, _ = test_util.submodule(name, subname, pkg_dir)
with util.temp_module(name, pkg=True) as pkg_dir:
fullname, _ = util.submodule(name, subname, pkg_dir)
relname = '.' + subname
with self.assertRaises(ValueError):
self.util.find_spec(relname)
@ -453,7 +531,7 @@ class FindSpecTests:
(Frozen_FindSpecTests,
Source_FindSpecTests
) = test_util.test_both(FindSpecTests, init=init, util=util,
) = util.test_both(FindSpecTests, init=init, util=importlib_util,
machinery=machinery)
@ -470,7 +548,7 @@ class MagicNumberTests:
(Frozen_MagicNumberTests,
Source_MagicNumberTests
) = test_util.test_both(MagicNumberTests, util=util)
) = util.test_both(MagicNumberTests, util=importlib_util)
class PEP3147Tests:
@ -588,7 +666,7 @@ class PEP3147Tests:
(Frozen_PEP3147Tests,
Source_PEP3147Tests
) = test_util.test_both(PEP3147Tests, util=util)
) = util.test_both(PEP3147Tests, util=importlib_util)
if __name__ == '__main__':

4
Lib/test/test_tools.py
View File

@ -407,7 +407,7 @@ class PdepsTests(unittest.TestCase):
def setUpClass(self):
path = os.path.join(scriptsdir, 'pdeps.py')
spec = importlib.util.spec_from_file_location('pdeps', path)
self.pdeps = importlib._bootstrap._SpecMethods(spec).load()
self.pdeps = importlib._bootstrap._load(spec)
@classmethod
def tearDownClass(self):
@ -432,7 +432,7 @@ class Gprof2htmlTests(unittest.TestCase):
def setUp(self):
path = os.path.join(scriptsdir, 'gprof2html.py')
spec = importlib.util.spec_from_file_location('gprof2html', path)
self.gprof = importlib._bootstrap._SpecMethods(spec).load()
self.gprof = importlib._bootstrap._load(spec)
oldargv = sys.argv
def fixup():
sys.argv = oldargv

3
Misc/NEWS
View File

@ -89,6 +89,9 @@ Core and Builtins
Library
-------
- Issue #20383: Introduce importlib.util.module_from_spec() as the preferred way
to create a new module.
- Issue #21552: Fixed possible integer overflow of too long string lengths in
the tkinter module on 64-bit platforms.

8480
Python/importlib.h
View File

File diff suppressed because it is too large Load Diff

2
setup.py
View File

@ -342,7 +342,7 @@ class PyBuildExt(build_ext):
spec = importlib.util.spec_from_file_location(ext.name, ext_filename,
loader=loader)
try:
importlib._bootstrap._SpecMethods(spec).load()
importlib._bootstrap._load(spec)
except ImportError as why:
self.failed_on_import.append(ext.name)
self.announce('*** WARNING: renaming "%s" since importing it'