cpython/Lib/test/test_importlib/frozen/test_finder.py

184 lines
6.4 KiB
Python
Raw Normal View History

from test.test_importlib import abc, util
machinery = util.import_importlib('importlib.machinery')
import os.path
import unittest
from test.support import import_helper, REPO_ROOT, STDLIB_DIR
def resolve_stdlib_file(name, ispkg=False):
assert name
if ispkg:
return os.path.join(STDLIB_DIR, *name.split('.'), '__init__.py')
else:
return os.path.join(STDLIB_DIR, *name.split('.')) + '.py'
2013-11-22 12:05:39 -04:00
class FindSpecTests(abc.FinderTests):
"""Test finding frozen modules."""
def find(self, name, **kwargs):
2013-11-22 12:05:39 -04:00
finder = self.machinery.FrozenImporter
with import_helper.frozen_modules():
return finder.find_spec(name, **kwargs)
2013-11-22 12:05:39 -04:00
def check_basic(self, spec, name, ispkg=False):
self.assertEqual(spec.name, name)
self.assertIs(spec.loader, self.machinery.FrozenImporter)
2013-11-22 12:05:39 -04:00
self.assertEqual(spec.origin, 'frozen')
self.assertFalse(spec.has_location)
if ispkg:
self.assertIsNotNone(spec.submodule_search_locations)
else:
self.assertIsNone(spec.submodule_search_locations)
self.assertIsNotNone(spec.loader_state)
def check_loader_state(self, spec, origname=None, filename=None):
if not filename:
if not origname:
origname = spec.name
bpo-21736: Set __file__ on frozen stdlib modules. (gh-28656) Currently frozen modules do not have __file__ set. In their spec, origin is set to "frozen" and they are marked as not having a location. (Similarly, for frozen packages __path__ is set to an empty list.) However, for frozen stdlib modules we are able to extrapolate __file__ as long as we can determine the stdlib directory at runtime. (We now do so since gh-28586.) Having __file__ set is helpful for a number of reasons. Likewise, having a non-empty __path__ means we can import submodules of a frozen package from the filesystem (e.g. we could partially freeze the encodings module). This change sets __file__ (and adds to __path__) for frozen stdlib modules. It uses sys._stdlibdir (from gh-28586) and the frozen module alias information (from gh-28655). All that work is done in FrozenImporter (in Lib/importlib/_bootstrap.py). Also, if a frozen module is imported before importlib is bootstrapped (during interpreter initialization) then we fix up that module and its spec during the importlib bootstrapping step (i.e. imporlib._bootstrap._setup()) to match what gets set by FrozenImporter, including setting the file info (if the stdlib dir is known). To facilitate this, modules imported using PyImport_ImportFrozenModule() have __origname__ set using the frozen module alias info. __origname__ is popped off during importlib bootstrap. (To be clear, even with this change the new code to set __file__ during fixups in imporlib._bootstrap._setup() doesn't actually get triggered yet. This is because sys._stdlibdir hasn't been set yet in interpreter initialization at the point importlib is bootstrapped. However, we do fix up such modules at that point to otherwise match the result of importing through FrozenImporter, just not the __file__ and __path__ parts. Doing so will require changes in the order in which things happen during interpreter initialization. That can be addressed separately. Once it is, the file-related fixup code from this PR will kick in.) Here are things this change does not do: * set __file__ for non-stdlib modules (no way of knowing the parent dir) * set __file__ if the stdlib dir is not known (nor assume the expense of finding it) * relatedly, set __file__ if the stdlib is in a zip file * verify that the filename set to __file__ actually exists (too expensive) * update __path__ for frozen packages that alias a non-package (since there is no package dir) Other things this change skips, but we may do later: * set __file__ on modules imported using PyImport_ImportFrozenModule() * set co_filename when we unmarshal the frozen code object while importing the module (e.g. in FrozenImporter.exec_module()) -- this would allow tracebacks to show source lines * implement FrozenImporter.get_filename() and FrozenImporter.get_source() https://bugs.python.org/issue21736
2021-10-14 18:32:18 -03:00
filename = resolve_stdlib_file(origname)
actual = dict(vars(spec.loader_state))
# Check the rest of spec.loader_state.
expected = dict(
origname=origname,
bpo-21736: Set __file__ on frozen stdlib modules. (gh-28656) Currently frozen modules do not have __file__ set. In their spec, origin is set to "frozen" and they are marked as not having a location. (Similarly, for frozen packages __path__ is set to an empty list.) However, for frozen stdlib modules we are able to extrapolate __file__ as long as we can determine the stdlib directory at runtime. (We now do so since gh-28586.) Having __file__ set is helpful for a number of reasons. Likewise, having a non-empty __path__ means we can import submodules of a frozen package from the filesystem (e.g. we could partially freeze the encodings module). This change sets __file__ (and adds to __path__) for frozen stdlib modules. It uses sys._stdlibdir (from gh-28586) and the frozen module alias information (from gh-28655). All that work is done in FrozenImporter (in Lib/importlib/_bootstrap.py). Also, if a frozen module is imported before importlib is bootstrapped (during interpreter initialization) then we fix up that module and its spec during the importlib bootstrapping step (i.e. imporlib._bootstrap._setup()) to match what gets set by FrozenImporter, including setting the file info (if the stdlib dir is known). To facilitate this, modules imported using PyImport_ImportFrozenModule() have __origname__ set using the frozen module alias info. __origname__ is popped off during importlib bootstrap. (To be clear, even with this change the new code to set __file__ during fixups in imporlib._bootstrap._setup() doesn't actually get triggered yet. This is because sys._stdlibdir hasn't been set yet in interpreter initialization at the point importlib is bootstrapped. However, we do fix up such modules at that point to otherwise match the result of importing through FrozenImporter, just not the __file__ and __path__ parts. Doing so will require changes in the order in which things happen during interpreter initialization. That can be addressed separately. Once it is, the file-related fixup code from this PR will kick in.) Here are things this change does not do: * set __file__ for non-stdlib modules (no way of knowing the parent dir) * set __file__ if the stdlib dir is not known (nor assume the expense of finding it) * relatedly, set __file__ if the stdlib is in a zip file * verify that the filename set to __file__ actually exists (too expensive) * update __path__ for frozen packages that alias a non-package (since there is no package dir) Other things this change skips, but we may do later: * set __file__ on modules imported using PyImport_ImportFrozenModule() * set co_filename when we unmarshal the frozen code object while importing the module (e.g. in FrozenImporter.exec_module()) -- this would allow tracebacks to show source lines * implement FrozenImporter.get_filename() and FrozenImporter.get_source() https://bugs.python.org/issue21736
2021-10-14 18:32:18 -03:00
filename=filename if origname else None,
)
self.assertDictEqual(actual, expected)
def check_search_locations(self, spec):
bpo-21736: Set __file__ on frozen stdlib modules. (gh-28656) Currently frozen modules do not have __file__ set. In their spec, origin is set to "frozen" and they are marked as not having a location. (Similarly, for frozen packages __path__ is set to an empty list.) However, for frozen stdlib modules we are able to extrapolate __file__ as long as we can determine the stdlib directory at runtime. (We now do so since gh-28586.) Having __file__ set is helpful for a number of reasons. Likewise, having a non-empty __path__ means we can import submodules of a frozen package from the filesystem (e.g. we could partially freeze the encodings module). This change sets __file__ (and adds to __path__) for frozen stdlib modules. It uses sys._stdlibdir (from gh-28586) and the frozen module alias information (from gh-28655). All that work is done in FrozenImporter (in Lib/importlib/_bootstrap.py). Also, if a frozen module is imported before importlib is bootstrapped (during interpreter initialization) then we fix up that module and its spec during the importlib bootstrapping step (i.e. imporlib._bootstrap._setup()) to match what gets set by FrozenImporter, including setting the file info (if the stdlib dir is known). To facilitate this, modules imported using PyImport_ImportFrozenModule() have __origname__ set using the frozen module alias info. __origname__ is popped off during importlib bootstrap. (To be clear, even with this change the new code to set __file__ during fixups in imporlib._bootstrap._setup() doesn't actually get triggered yet. This is because sys._stdlibdir hasn't been set yet in interpreter initialization at the point importlib is bootstrapped. However, we do fix up such modules at that point to otherwise match the result of importing through FrozenImporter, just not the __file__ and __path__ parts. Doing so will require changes in the order in which things happen during interpreter initialization. That can be addressed separately. Once it is, the file-related fixup code from this PR will kick in.) Here are things this change does not do: * set __file__ for non-stdlib modules (no way of knowing the parent dir) * set __file__ if the stdlib dir is not known (nor assume the expense of finding it) * relatedly, set __file__ if the stdlib is in a zip file * verify that the filename set to __file__ actually exists (too expensive) * update __path__ for frozen packages that alias a non-package (since there is no package dir) Other things this change skips, but we may do later: * set __file__ on modules imported using PyImport_ImportFrozenModule() * set co_filename when we unmarshal the frozen code object while importing the module (e.g. in FrozenImporter.exec_module()) -- this would allow tracebacks to show source lines * implement FrozenImporter.get_filename() and FrozenImporter.get_source() https://bugs.python.org/issue21736
2021-10-14 18:32:18 -03:00
"""This is only called when testing packages."""
missing = object()
filename = getattr(spec.loader_state, 'filename', missing)
origname = getattr(spec.loader_state, 'origname', None)
if not origname or filename is missing:
# We deal with this in check_loader_state().
return
if not filename:
expected = []
elif origname != spec.name and not origname.startswith('<'):
expected = []
else:
expected = [os.path.dirname(filename)]
self.assertListEqual(spec.submodule_search_locations, expected)
def test_module(self):
modules = [
'__hello__',
'__phello__.spam',
'__phello__.ham.eggs',
]
for name in modules:
with self.subTest(f'{name} -> {name}'):
spec = self.find(name)
self.check_basic(spec, name)
self.check_loader_state(spec)
modules = {
'__hello_alias__': '__hello__',
'_frozen_importlib': 'importlib._bootstrap',
}
for name, origname in modules.items():
with self.subTest(f'{name} -> {origname}'):
spec = self.find(name)
self.check_basic(spec, name)
self.check_loader_state(spec, origname)
modules = [
'__phello__.__init__',
'__phello__.ham.__init__',
]
for name in modules:
origname = '<' + name.rpartition('.')[0]
filename = resolve_stdlib_file(name)
with self.subTest(f'{name} -> {origname}'):
spec = self.find(name)
self.check_basic(spec, name)
self.check_loader_state(spec, origname, filename)
modules = {
'__hello_only__': ('Tools', 'freeze', 'flag.py'),
}
for name, path in modules.items():
origname = None
filename = os.path.join(REPO_ROOT, *path)
with self.subTest(f'{name} -> {filename}'):
spec = self.find(name)
self.check_basic(spec, name)
self.check_loader_state(spec, origname, filename)
2013-11-22 12:05:39 -04:00
def test_package(self):
packages = [
'__phello__',
'__phello__.ham',
]
for name in packages:
filename = resolve_stdlib_file(name, ispkg=True)
with self.subTest(f'{name} -> {name}'):
spec = self.find(name)
self.check_basic(spec, name, ispkg=True)
self.check_loader_state(spec, name, filename)
self.check_search_locations(spec)
packages = {
'__phello_alias__': '__hello__',
}
for name, origname in packages.items():
filename = resolve_stdlib_file(origname, ispkg=False)
with self.subTest(f'{name} -> {origname}'):
spec = self.find(name)
self.check_basic(spec, name, ispkg=True)
self.check_loader_state(spec, origname, filename)
self.check_search_locations(spec)
# These are covered by test_module() and test_package().
test_module_in_package = None
2013-11-22 12:05:39 -04:00
test_package_in_package = None
# No easy way to test.
test_package_over_module = None
def test_path_ignored(self):
for name in ('__hello__', '__phello__', '__phello__.spam'):
actual = self.find(name)
for path in (None, object(), '', 'eggs', [], [''], ['eggs']):
with self.subTest((name, path)):
spec = self.find(name, path=path)
self.assertEqual(spec, actual)
def test_target_ignored(self):
imported = ('__hello__', '__phello__')
with import_helper.CleanImport(*imported, usefrozen=True):
import __hello__ as match
import __phello__ as nonmatch
name = '__hello__'
actual = self.find(name)
for target in (None, match, nonmatch, object(), 'not-a-module-object'):
with self.subTest(target):
spec = self.find(name, target=target)
self.assertEqual(spec, actual)
2013-11-22 12:05:39 -04:00
def test_failure(self):
spec = self.find('<not real>')
self.assertIsNone(spec)
def test_not_using_frozen(self):
finder = self.machinery.FrozenImporter
with import_helper.frozen_modules(enabled=False):
# both frozen and not frozen
spec1 = finder.find_spec('__hello__')
# only frozen
spec2 = finder.find_spec('__hello_only__')
self.assertIsNone(spec1)
self.assertIsNone(spec2)
(Frozen_FindSpecTests,
Source_FindSpecTests
) = util.test_both(FindSpecTests, machinery=machinery)
2013-11-22 12:05:39 -04:00
if __name__ == '__main__':
unittest.main()