diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 399191301a3..4c6aa59bf9f 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -14,12 +14,13 @@ This module leverages Python's import system to provide access to *resources* within *packages*. "Resources" are file-like resources associated with a module or package in -Python. The resources may be contained directly in a package or within a -subdirectory contained in that package. Resources may be text or binary. As a -result, Python module sources (.py) of a package and compilation artifacts -(pycache) are technically de-facto resources of that package. In practice, -however, resources are primarily those non-Python artifacts exposed -specifically by the package author. +Python. The resources may be contained directly in a package, within a +subdirectory contained in that package, or adjacent to modules outside a +package. Resources may be text or binary. As a result, Python module sources +(.py) of a package and compilation artifacts (pycache) are technically +de-facto resources of that package. In practice, however, resources are +primarily those non-Python artifacts exposed specifically by the package +author. Resources can be opened or read in either binary or text mode. @@ -49,27 +50,35 @@ for example, a package and its resources can be imported from a zip file using ``get_resource_reader(fullname)`` method as specified by :class:`importlib.resources.abc.ResourceReader`. -.. data:: Package +.. data:: Anchor - Whenever a function accepts a ``Package`` argument, you can pass in - either a :class:`module object ` or a module name - as a string. You can only pass module objects whose - ``__spec__.submodule_search_locations`` is not ``None``. + Represents an anchor for resources, either a :class:`module object + ` or a module name as a string. Defined as + ``Union[str, ModuleType]``. - The ``Package`` type is defined as ``Union[str, ModuleType]``. - -.. function:: files(package) +.. function:: files(anchor: Optional[Anchor] = None) Returns a :class:`~importlib.resources.abc.Traversable` object - representing the resource container for the package (think directory) - and its resources (think files). A Traversable may contain other - containers (think subdirectories). + representing the resource container (think directory) and its resources + (think files). A Traversable may contain other containers (think + subdirectories). - *package* is either a name or a module object which conforms to the - :data:`Package` requirements. + *anchor* is an optional :data:`Anchor`. If the anchor is a + package, resources are resolved from that package. If a module, + resources are resolved adjacent to that module (in the same package + or the package root). If the anchor is omitted, the caller's module + is used. .. versionadded:: 3.9 + .. versionchanged:: 3.12 + "package" parameter was renamed to "anchor". "anchor" can now + be a non-package module and if omitted will default to the caller's + module. "package" is still accepted for compatibility but will raise + a DeprecationWarning. Consider passing the anchor positionally or + using ``importlib_resources >= 5.10`` for a compatible interface + on older Pythons. + .. function:: as_file(traversable) Given a :class:`~importlib.resources.abc.Traversable` object representing @@ -86,6 +95,7 @@ for example, a package and its resources can be imported from a zip file using .. versionadded:: 3.9 + Deprecated functions -------------------- @@ -94,6 +104,18 @@ scheduled for removal in a future version of Python. The main drawback of these functions is that they do not support directories: they assume all resources are located directly within a *package*. +.. data:: Package + + Whenever a function accepts a ``Package`` argument, you can pass in + either a :class:`module object ` or a module name + as a string. You can only pass module objects whose + ``__spec__.submodule_search_locations`` is not ``None``. + + The ``Package`` type is defined as ``Union[str, ModuleType]``. + + .. deprecated:: 3.12 + + .. data:: Resource For *resource* arguments of the functions below, you can pass in @@ -102,6 +124,7 @@ directories: they assume all resources are located directly within a *package*. The ``Resource`` type is defined as ``Union[str, os.PathLike]``. + .. function:: open_binary(package, resource) Open for binary reading the *resource* within *package*. diff --git a/Lib/importlib/resources/_common.py b/Lib/importlib/resources/_common.py index 92a37e2c12b..a3902535342 100644 --- a/Lib/importlib/resources/_common.py +++ b/Lib/importlib/resources/_common.py @@ -5,25 +5,58 @@ import functools import contextlib import types import importlib +import inspect +import warnings +import itertools -from typing import Union, Optional +from typing import Union, Optional, cast from .abc import ResourceReader, Traversable from ._adapters import wrap_spec Package = Union[types.ModuleType, str] +Anchor = Package -def files(package): - # type: (Package) -> Traversable +def package_to_anchor(func): """ - Get a Traversable resource from a package + Replace 'package' parameter as 'anchor' and warn about the change. + + Other errors should fall through. + + >>> files('a', 'b') + Traceback (most recent call last): + TypeError: files() takes from 0 to 1 positional arguments but 2 were given """ - return from_package(get_package(package)) + undefined = object() + + @functools.wraps(func) + def wrapper(anchor=undefined, package=undefined): + if package is not undefined: + if anchor is not undefined: + return func(anchor, package) + warnings.warn( + "First parameter to files is renamed to 'anchor'", + DeprecationWarning, + stacklevel=2, + ) + return func(package) + elif anchor is undefined: + return func() + return func(anchor) + + return wrapper -def get_resource_reader(package): - # type: (types.ModuleType) -> Optional[ResourceReader] +@package_to_anchor +def files(anchor: Optional[Anchor] = None) -> Traversable: + """ + Get a Traversable resource for an anchor. + """ + return from_package(resolve(anchor)) + + +def get_resource_reader(package: types.ModuleType) -> Optional[ResourceReader]: """ Return the package's loader if it's a ResourceReader. """ @@ -39,24 +72,39 @@ def get_resource_reader(package): return reader(spec.name) # type: ignore -def resolve(cand): - # type: (Package) -> types.ModuleType - return cand if isinstance(cand, types.ModuleType) else importlib.import_module(cand) +@functools.singledispatch +def resolve(cand: Optional[Anchor]) -> types.ModuleType: + return cast(types.ModuleType, cand) -def get_package(package): - # type: (Package) -> types.ModuleType - """Take a package name or module object and return the module. +@resolve.register +def _(cand: str) -> types.ModuleType: + return importlib.import_module(cand) - Raise an exception if the resolved module is not a package. + +@resolve.register +def _(cand: None) -> types.ModuleType: + return resolve(_infer_caller().f_globals['__name__']) + + +def _infer_caller(): """ - resolved = resolve(package) - if wrap_spec(resolved).submodule_search_locations is None: - raise TypeError(f'{package!r} is not a package') - return resolved + Walk the stack and find the frame of the first caller not in this module. + """ + + def is_this_file(frame_info): + return frame_info.filename == __file__ + + def is_wrapper(frame_info): + return frame_info.function == 'wrapper' + + not_this_file = itertools.filterfalse(is_this_file, inspect.stack()) + # also exclude 'wrapper' due to singledispatch in the call stack + callers = itertools.filterfalse(is_wrapper, not_this_file) + return next(callers).frame -def from_package(package): +def from_package(package: types.ModuleType): """ Return a Traversable object for the given package. diff --git a/Lib/importlib/resources/_legacy.py b/Lib/importlib/resources/_legacy.py index 1d5d3f1fbb1..b1ea8105dad 100644 --- a/Lib/importlib/resources/_legacy.py +++ b/Lib/importlib/resources/_legacy.py @@ -27,8 +27,7 @@ def deprecated(func): return wrapper -def normalize_path(path): - # type: (Any) -> str +def normalize_path(path: Any) -> str: """Normalize a path by ensuring it is a string. If the resulting string contains path separators, an exception is raised. diff --git a/Lib/importlib/resources/abc.py b/Lib/importlib/resources/abc.py index 67c78c07856..6750a7aaf14 100644 --- a/Lib/importlib/resources/abc.py +++ b/Lib/importlib/resources/abc.py @@ -142,7 +142,8 @@ class Traversable(Protocol): accepted by io.TextIOWrapper. """ - @abc.abstractproperty + @property + @abc.abstractmethod def name(self) -> str: """ The base name of this object without any parent references. diff --git a/Lib/importlib/resources/simple.py b/Lib/importlib/resources/simple.py index b85e4694ace..7770c922c84 100644 --- a/Lib/importlib/resources/simple.py +++ b/Lib/importlib/resources/simple.py @@ -16,31 +16,28 @@ class SimpleReader(abc.ABC): provider. """ - @abc.abstractproperty - def package(self): - # type: () -> str + @property + @abc.abstractmethod + def package(self) -> str: """ The name of the package for which this reader loads resources. """ @abc.abstractmethod - def children(self): - # type: () -> List['SimpleReader'] + def children(self) -> List['SimpleReader']: """ Obtain an iterable of SimpleReader for available child containers (e.g. directories). """ @abc.abstractmethod - def resources(self): - # type: () -> List[str] + def resources(self) -> List[str]: """ Obtain available named resources for this virtual package. """ @abc.abstractmethod - def open_binary(self, resource): - # type: (str) -> BinaryIO + def open_binary(self, resource: str) -> BinaryIO: """ Obtain a File-like for a named resource. """ @@ -50,13 +47,35 @@ class SimpleReader(abc.ABC): return self.package.split('.')[-1] +class ResourceContainer(Traversable): + """ + Traversable container for a package's resources via its reader. + """ + + def __init__(self, reader: SimpleReader): + self.reader = reader + + def is_dir(self): + return True + + def is_file(self): + return False + + def iterdir(self): + files = (ResourceHandle(self, name) for name in self.reader.resources) + dirs = map(ResourceContainer, self.reader.children()) + return itertools.chain(files, dirs) + + def open(self, *args, **kwargs): + raise IsADirectoryError() + + class ResourceHandle(Traversable): """ Handle to a named resource in a ResourceReader. """ - def __init__(self, parent, name): - # type: (ResourceContainer, str) -> None + def __init__(self, parent: ResourceContainer, name: str): self.parent = parent self.name = name # type: ignore @@ -76,30 +95,6 @@ class ResourceHandle(Traversable): raise RuntimeError("Cannot traverse into a resource") -class ResourceContainer(Traversable): - """ - Traversable container for a package's resources via its reader. - """ - - def __init__(self, reader): - # type: (SimpleReader) -> None - self.reader = reader - - def is_dir(self): - return True - - def is_file(self): - return False - - def iterdir(self): - files = (ResourceHandle(self, name) for name in self.reader.resources) - dirs = map(ResourceContainer, self.reader.children()) - return itertools.chain(files, dirs) - - def open(self, *args, **kwargs): - raise IsADirectoryError() - - class TraversableReader(TraversableResources, SimpleReader): """ A TraversableResources based on SimpleReader. Resource providers diff --git a/Lib/test/test_importlib/resources/_path.py b/Lib/test/test_importlib/resources/_path.py new file mode 100644 index 00000000000..c630e4d3d3f --- /dev/null +++ b/Lib/test/test_importlib/resources/_path.py @@ -0,0 +1,50 @@ +import pathlib +import functools + + +#### +# from jaraco.path 3.4 + + +def build(spec, prefix=pathlib.Path()): + """ + Build a set of files/directories, as described by the spec. + + Each key represents a pathname, and the value represents + the content. Content may be a nested directory. + + >>> spec = { + ... 'README.txt': "A README file", + ... "foo": { + ... "__init__.py": "", + ... "bar": { + ... "__init__.py": "", + ... }, + ... "baz.py": "# Some code", + ... } + ... } + >>> tmpdir = getfixture('tmpdir') + >>> build(spec, tmpdir) + """ + for name, contents in spec.items(): + create(contents, pathlib.Path(prefix) / name) + + +@functools.singledispatch +def create(content, path): + path.mkdir(exist_ok=True) + build(content, prefix=path) # type: ignore + + +@create.register +def _(content: bytes, path): + path.write_bytes(content) + + +@create.register +def _(content: str, path): + path.write_text(content) + + +# end from jaraco.path +#### diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 779e5a12b5d..fe813ae7d08 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -1,10 +1,24 @@ import typing +import textwrap import unittest +import warnings +import importlib +import contextlib from importlib import resources from importlib.resources.abc import Traversable from . import data01 from . import util +from . import _path +from test.support import os_helper +from test.support import import_helper + + +@contextlib.contextmanager +def suppress_known_deprecation(): + with warnings.catch_warnings(record=True) as ctx: + warnings.simplefilter('default', category=DeprecationWarning) + yield ctx class FilesTests: @@ -25,6 +39,14 @@ class FilesTests: def test_traversable(self): assert isinstance(resources.files(self.data), Traversable) + def test_old_parameter(self): + """ + Files used to take a 'package' parameter. Make sure anyone + passing by name is still supported. + """ + with suppress_known_deprecation(): + resources.files(package=self.data) + class OpenDiskTests(FilesTests, unittest.TestCase): def setUp(self): @@ -42,5 +64,50 @@ class OpenNamespaceTests(FilesTests, unittest.TestCase): self.data = namespacedata01 +class SiteDir: + def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + self.site_dir = self.fixtures.enter_context(os_helper.temp_dir()) + self.fixtures.enter_context(import_helper.DirsOnSysPath(self.site_dir)) + self.fixtures.enter_context(import_helper.CleanImport()) + + +class ModulesFilesTests(SiteDir, unittest.TestCase): + def test_module_resources(self): + """ + A module can have resources found adjacent to the module. + """ + spec = { + 'mod.py': '', + 'res.txt': 'resources are the best', + } + _path.build(spec, self.site_dir) + import mod + + actual = resources.files(mod).joinpath('res.txt').read_text() + assert actual == spec['res.txt'] + + +class ImplicitContextFilesTests(SiteDir, unittest.TestCase): + def test_implicit_files(self): + """ + Without any parameter, files() will infer the location as the caller. + """ + spec = { + 'somepkg': { + '__init__.py': textwrap.dedent( + """ + import importlib.resources as res + val = res.files().joinpath('res.txt').read_text() + """ + ), + 'res.txt': 'resources are the best', + }, + } + _path.build(spec, self.site_dir) + assert importlib.import_module('somepkg').val == 'resources are the best' + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/resources/util.py b/Lib/test/test_importlib/resources/util.py index eb2291f15de..1e72b91ff6b 100644 --- a/Lib/test/test_importlib/resources/util.py +++ b/Lib/test/test_importlib/resources/util.py @@ -3,7 +3,7 @@ import importlib import io import sys import types -from pathlib import Path, PurePath +import pathlib from . import data01 from . import zipdata01 @@ -94,7 +94,7 @@ class CommonTests(metaclass=abc.ABCMeta): def test_pathlib_path(self): # Passing in a pathlib.PurePath object for the path should succeed. - path = PurePath('utf-8.file') + path = pathlib.PurePath('utf-8.file') self.execute(data01, path) def test_importing_module_as_side_effect(self): @@ -102,17 +102,6 @@ class CommonTests(metaclass=abc.ABCMeta): del sys.modules[data01.__name__] self.execute(data01.__name__, 'utf-8.file') - def test_non_package_by_name(self): - # The anchor package cannot be a module. - with self.assertRaises(TypeError): - self.execute(__name__, 'utf-8.file') - - def test_non_package_by_package(self): - # The anchor package cannot be a module. - with self.assertRaises(TypeError): - module = sys.modules['test.test_importlib.resources.util'] - self.execute(module, 'utf-8.file') - def test_missing_path(self): # Attempting to open or read or request the path for a # non-existent path should succeed if open_resource @@ -144,7 +133,7 @@ class ZipSetupBase: @classmethod def setUpClass(cls): - data_path = Path(cls.ZIP_MODULE.__file__) + data_path = pathlib.Path(cls.ZIP_MODULE.__file__) data_dir = data_path.parent cls._zip_path = str(data_dir / 'ziptestdata.zip') sys.path.append(cls._zip_path) diff --git a/Misc/NEWS.d/next/Library/2022-12-29-11-45-22.gh-issue-97930.hrtmJe.rst b/Misc/NEWS.d/next/Library/2022-12-29-11-45-22.gh-issue-97930.hrtmJe.rst new file mode 100644 index 00000000000..1fe4a9d6b77 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-29-11-45-22.gh-issue-97930.hrtmJe.rst @@ -0,0 +1,6 @@ +``importlib.resources.files`` now accepts a module as an anchor instead of +only accepting packages. If a module is passed, resources are resolved +adjacent to that module (in the same package or at the package root). The +parameter was renamed from ``package`` to ``anchor`` with a compatibility +shim for those passing by keyword. Additionally, the new ``anchor`` +parameter is now optional and will default to the caller's module.