diff --git a/Doc/library/imp.rst b/Doc/library/imp.rst index 1345b254d62..7270e3c6906 100644 --- a/Doc/library/imp.rst +++ b/Doc/library/imp.rst @@ -30,6 +30,9 @@ This module provides an interface to the mechanisms used to implement the :const:`PY_SOURCE`, :const:`PY_COMPILED`, or :const:`C_EXTENSION`, described below. + .. deprecated:: 3.3 + Use the constants defined on :mod:`importlib.machinery` instead. + .. function:: find_module(name[, path]) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 6855a7918aa..ddfaa211e71 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -477,6 +477,43 @@ are also provided to help in implementing the core ABCs. This module contains the various objects that help :keyword:`import` find and load modules. +.. attribute:: SOURCE_SUFFIXES + + A list of strings representing the recognized file suffixes for source + modules. + + .. versionadded:: 3.3 + +.. attribute:: DEBUG_BYTECODE_SUFFIXES + + A list of strings representing the file suffixes for non-optimized bytecode + modules. + + .. versionadded:: 3.3 + +.. attribute:: OPTIMIZED_BYTECODE_SUFFIXES + + A list of strings representing the file suffixes for optimized bytecode + modules. + + .. versionadded:: 3.3 + +.. attribute:: BYTECODE_SUFFIXES + + A list of strings representing the recognized file suffixes for bytecode + modules. Set to either :attr:`DEBUG_BYTECODE_SUFFIXES` or + :attr:`OPTIMIZED_BYTECODE_SUFFIXES` based on whether ``__debug__`` is true. + + .. versionadded:: 3.3 + +.. attribute:: EXTENSION_SUFFIXES + + A list of strings representing the the recognized file suffixes for + extension modules. + + .. versionadded:: 3.3 + + .. class:: BuiltinImporter An :term:`importer` for built-in modules. All known built-in modules are diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 90b7220a6b0..611f780254f 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -190,6 +190,10 @@ attributes: compared to the constants defined in the :mod:`imp` module; see the documentation for that module for more information on module types. + .. deprecated:: 3.3 + You may check the file path's suffix against the supported suffixes + listed in :mod:`importlib.machinery` to infer the same information. + .. function:: getmodulename(path) diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 97d0c2f3fa4..61eb09454b8 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -40,8 +40,8 @@ if os.name == "nt": clibname = 'msvcr%d' % (version * 10) # If python was built with in debug mode - import imp - if imp.get_suffixes()[0][0] == '_d.pyd': + import importlib.machinery + if '_d.pyd' in importlib.machinery.EXTENSION_SUFFIXES: clibname += 'd' return clibname+'.dll' diff --git a/Lib/idlelib/PathBrowser.py b/Lib/idlelib/PathBrowser.py index d88a48e344e..9954f070c0a 100644 --- a/Lib/idlelib/PathBrowser.py +++ b/Lib/idlelib/PathBrowser.py @@ -1,6 +1,7 @@ import os import sys import imp +import importlib.machinery from idlelib.TreeWidget import TreeItem from idlelib.ClassBrowser import ClassBrowser, ModuleBrowserTreeItem @@ -70,7 +71,9 @@ class DirBrowserTreeItem(TreeItem): def listmodules(self, allnames): modules = {} - suffixes = imp.get_suffixes() + suffixes = importlib.machinery.EXTENSION_SUFFIXES[:] + suffixes += importlib.machinery.SOURCE_SUFFIXES[:] + suffixes += importlib.machinery.BYTECODE_SUFFIXES[:] sorted = [] for suff, mode, flag in suffixes: i = -len(suff) diff --git a/Lib/imp.py b/Lib/imp.py index 717b6a43432..f60f0508bac 100644 --- a/Lib/imp.py +++ b/Lib/imp.py @@ -17,9 +17,11 @@ from importlib._bootstrap import new_module from importlib._bootstrap import cache_from_source from importlib import _bootstrap +from importlib import machinery import os import sys import tokenize +import warnings # XXX "deprecate" once find_module(), load_module(), and get_suffixes() are @@ -37,9 +39,12 @@ IMP_HOOK = 9 def get_suffixes(): + warnings.warn('imp.get_suffixes() is deprecated; use the constants ' + 'defined on importlib.machinery instead', + DeprecationWarning, 2) extensions = [(s, 'rb', C_EXTENSION) for s in extension_suffixes()] - source = [(s, 'U', PY_SOURCE) for s in _bootstrap._SOURCE_SUFFIXES] - bytecode = [(_bootstrap._BYTECODE_SUFFIX, 'rb', PY_COMPILED)] + source = [(s, 'U', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES] + bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES] return extensions + source + bytecode @@ -61,7 +66,7 @@ def source_from_cache(path): raise ValueError('expected only 2 dots in ' '{!r}'.format(pycache_filename)) base_filename = pycache_filename.partition('.')[0] - return os.path.join(head, base_filename + _bootstrap._SOURCE_SUFFIXES[0]) + return os.path.join(head, base_filename + machinery.SOURCE_SUFFIXES[0]) class NullImporter: @@ -126,7 +131,7 @@ def load_compiled(name, pathname, file=None): # XXX deprecate def load_package(name, path): if os.path.isdir(path): - extensions = _bootstrap._SOURCE_SUFFIXES + [_bootstrap._BYTECODE_SUFFIX] + extensions = machinery.SOURCE_SUFFIXES[:] + [machinery.BYTECODE_SUFFIXES] for extension in extensions: path = os.path.join(path, '__init__'+extension) if os.path.exists(path): @@ -190,19 +195,21 @@ def find_module(name, path=None): for entry in path: package_directory = os.path.join(entry, name) - for suffix in ['.py', _bootstrap._BYTECODE_SUFFIX]: + for suffix in ['.py', machinery.BYTECODE_SUFFIXES[0]]: package_file_name = '__init__' + suffix file_path = os.path.join(package_directory, package_file_name) if os.path.isfile(file_path): return None, package_directory, ('', '', PKG_DIRECTORY) - for suffix, mode, type_ in get_suffixes(): - file_name = name + suffix - file_path = os.path.join(entry, file_name) - if os.path.isfile(file_path): - break - else: - continue - break # Break out of outer loop when breaking out of inner loop. + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + for suffix, mode, type_ in get_suffixes(): + file_name = name + suffix + file_path = os.path.join(entry, file_name) + if os.path.isfile(file_path): + break + else: + continue + break # Break out of outer loop when breaking out of inner loop. else: raise ImportError('No module name {!r}'.format(name), name=name) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 9952b067f13..ce23043665f 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -163,11 +163,14 @@ def new_module(name): _PYCACHE = '__pycache__' -_SOURCE_SUFFIXES = ['.py'] # _setup() adds .pyw as needed. +SOURCE_SUFFIXES = ['.py'] # _setup() adds .pyw as needed. -_DEBUG_BYTECODE_SUFFIX = '.pyc' -_OPT_BYTECODE_SUFFIX = '.pyo' -_BYTECODE_SUFFIX = _DEBUG_BYTECODE_SUFFIX if __debug__ else _OPT_BYTECODE_SUFFIX +DEBUG_BYTECODE_SUFFIXES = ['.pyc'] +OPTIMIZED_BYTECODE_SUFFIXES = ['.pyo'] +if __debug__: + BYTECODE_SUFFIXES = DEBUG_BYTECODE_SUFFIXES +else: + BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES def cache_from_source(path, debug_override=None): """Given the path to a .py file, return the path to its .pyc/.pyo file. @@ -181,10 +184,13 @@ def cache_from_source(path, debug_override=None): """ debug = __debug__ if debug_override is None else debug_override - suffix = _DEBUG_BYTECODE_SUFFIX if debug else _OPT_BYTECODE_SUFFIX + if debug: + suffixes = DEBUG_BYTECODE_SUFFIXES + else: + suffixes = OPTIMIZED_BYTECODE_SUFFIXES head, tail = _path_split(path) base_filename, sep, _ = tail.partition('.') - filename = ''.join([base_filename, sep, _TAG, suffix]) + filename = ''.join([base_filename, sep, _TAG, suffixes[0]]) return _path_join(head, _PYCACHE, filename) @@ -1147,15 +1153,15 @@ def _setup(sys_module, _imp_module): setattr(self_module, '_MAGIC_NUMBER', _imp_module.get_magic()) setattr(self_module, '_TAG', _imp.get_tag()) if builtin_os == 'nt': - _SOURCE_SUFFIXES.append('.pyw') + SOURCE_SUFFIXES.append('.pyw') def _install(sys_module, _imp_module): """Install importlib as the implementation of import.""" _setup(sys_module, _imp_module) extensions = ExtensionFileLoader, _imp_module.extension_suffixes(), False - source = SourceFileLoader, _SOURCE_SUFFIXES, True - bytecode = SourcelessFileLoader, [_BYTECODE_SUFFIX], True + source = SourceFileLoader, SOURCE_SUFFIXES, True + bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES, True supported_loaders = [extensions, source, bytecode] sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)]) sys.meta_path.extend([BuiltinImporter, FrozenImporter, PathFinder]) diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index 07465ced68a..d5e7250479c 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -1,5 +1,9 @@ """The machinery of importlib: finders, loaders, hooks, etc.""" +import _imp + +from ._bootstrap import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES, + OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES) from ._bootstrap import BuiltinImporter from ._bootstrap import FrozenImporter from ._bootstrap import PathFinder @@ -7,3 +11,5 @@ from ._bootstrap import FileFinder from ._bootstrap import SourceFileLoader from ._bootstrap import SourcelessFileLoader from ._bootstrap import ExtensionFileLoader + +EXTENSION_SUFFIXES = _imp.extension_suffixes() diff --git a/Lib/importlib/test/benchmark.py b/Lib/importlib/test/benchmark.py index 16863146b5a..183e8180c94 100644 --- a/Lib/importlib/test/benchmark.py +++ b/Lib/importlib/test/benchmark.py @@ -9,7 +9,6 @@ from .source import util as source_util import decimal import imp import importlib -import importlib._bootstrap import importlib.machinery import json import os @@ -72,7 +71,7 @@ def source_wo_bytecode(seconds, repeat): assert not os.path.exists(imp.cache_from_source(mapping[name])) sys.meta_path.append(importlib.machinery.PathFinder) loader = (importlib.machinery.SourceFileLoader, - importlib._bootstrap._SOURCE_SUFFIXES, True) + importlib.machinery.SOURCE_SUFFIXES, True) sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader)) for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat, seconds=seconds): @@ -110,7 +109,7 @@ def source_writing_bytecode(seconds, repeat): with source_util.create_modules(name) as mapping: sys.meta_path.append(importlib.machinery.PathFinder) loader = (importlib.machinery.SourceFileLoader, - importlib._bootstrap._SOURCE_SUFFIXES, True) + importlib.machinery.SOURCE_SUFFIXES, True) sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader)) def cleanup(): sys.modules.pop(name) @@ -145,7 +144,7 @@ def source_using_bytecode(seconds, repeat): with source_util.create_modules(name) as mapping: sys.meta_path.append(importlib.machinery.PathFinder) loader = (importlib.machinery.SourceFileLoader, - importlib._bootstrap._SOURCE_SUFFIXES, True) + importlib.machinery.SOURCE_SUFFIXES, True) sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader)) py_compile.compile(mapping[name]) assert os.path.exists(imp.cache_from_source(mapping[name])) diff --git a/Lib/importlib/test/extension/util.py b/Lib/importlib/test/extension/util.py index d1491697483..a266dd98c83 100644 --- a/Lib/importlib/test/extension/util.py +++ b/Lib/importlib/test/extension/util.py @@ -1,4 +1,5 @@ import imp +from importlib import machinery import os import sys @@ -6,10 +7,9 @@ PATH = None EXT = None FILENAME = None NAME = '_testcapi' -_file_exts = [x[0] for x in imp.get_suffixes() if x[2] == imp.C_EXTENSION] try: for PATH in sys.path: - for EXT in _file_exts: + for EXT in machinery.EXTENSION_SUFFIXES: FILENAME = NAME + EXT FILEPATH = os.path.join(PATH, FILENAME) if os.path.exists(os.path.join(PATH, FILENAME)): @@ -18,4 +18,3 @@ try: PATH = EXT = FILENAME = FILEPATH = None except StopIteration: pass -del _file_exts diff --git a/Lib/importlib/test/source/test_case_sensitivity.py b/Lib/importlib/test/source/test_case_sensitivity.py index fade25fe104..21a4378ce9d 100644 --- a/Lib/importlib/test/source/test_case_sensitivity.py +++ b/Lib/importlib/test/source/test_case_sensitivity.py @@ -1,5 +1,6 @@ """Test case-sensitivity (PEP 235).""" from importlib import _bootstrap +from importlib import machinery from .. import util from . import util as source_util import imp @@ -20,12 +21,12 @@ class CaseSensitivityTest(unittest.TestCase): assert name != name.lower() def find(self, path): - finder = _bootstrap.FileFinder(path, - (_bootstrap.SourceFileLoader, - _bootstrap._SOURCE_SUFFIXES, + finder = machinery.FileFinder(path, + (machinery.SourceFileLoader, + machinery.SOURCE_SUFFIXES, True), - (_bootstrap.SourcelessFileLoader, - [_bootstrap._BYTECODE_SUFFIX], + (machinery.SourcelessFileLoader, + machinery.BYTECODE_SUFFIXES, True)) return finder.find_module(self.name) diff --git a/Lib/importlib/test/source/test_finder.py b/Lib/importlib/test/source/test_finder.py index 45b804ffdff..bbe01639fde 100644 --- a/Lib/importlib/test/source/test_finder.py +++ b/Lib/importlib/test/source/test_finder.py @@ -1,7 +1,7 @@ from .. import abc from . import util as source_util -from importlib import _bootstrap +from importlib import machinery import errno import imp import os @@ -36,11 +36,11 @@ class FinderTests(abc.FinderTests): """ def import_(self, root, module): - loader_details = [(_bootstrap.SourceFileLoader, - _bootstrap._SOURCE_SUFFIXES, True), - (_bootstrap.SourcelessFileLoader, - [_bootstrap._BYTECODE_SUFFIX], True)] - finder = _bootstrap.FileFinder(root, *loader_details) + loader_details = [(machinery.SourceFileLoader, + machinery.SOURCE_SUFFIXES, True), + (machinery.SourcelessFileLoader, + machinery.BYTECODE_SUFFIXES, True)] + finder = machinery.FileFinder(root, *loader_details) return finder.find_module(module) def run_test(self, test, create=None, *, compile_=None, unlink=None): @@ -138,8 +138,8 @@ class FinderTests(abc.FinderTests): def test_empty_string_for_dir(self): # The empty string from sys.path means to search in the cwd. - finder = _bootstrap.FileFinder('', (_bootstrap.SourceFileLoader, - _bootstrap._SOURCE_SUFFIXES, True)) + finder = machinery.FileFinder('', (machinery.SourceFileLoader, + machinery.SOURCE_SUFFIXES, True)) with open('mod.py', 'w') as file: file.write("# test file for importlib") try: @@ -150,8 +150,8 @@ class FinderTests(abc.FinderTests): def test_invalidate_caches(self): # invalidate_caches() should reset the mtime. - finder = _bootstrap.FileFinder('', (_bootstrap.SourceFileLoader, - _bootstrap._SOURCE_SUFFIXES, True)) + finder = machinery.FileFinder('', (machinery.SourceFileLoader, + machinery.SOURCE_SUFFIXES, True)) finder._path_mtime = 42 finder.invalidate_caches() self.assertEqual(finder._path_mtime, -1) diff --git a/Lib/importlib/test/source/test_path_hook.py b/Lib/importlib/test/source/test_path_hook.py index df69b4d61ed..54c06995014 100644 --- a/Lib/importlib/test/source/test_path_hook.py +++ b/Lib/importlib/test/source/test_path_hook.py @@ -1,6 +1,6 @@ from . import util as source_util -from importlib import _bootstrap +from importlib import machinery import imp import unittest @@ -10,8 +10,8 @@ class PathHookTest(unittest.TestCase): """Test the path hook for source.""" def path_hook(self): - return _bootstrap.FileFinder.path_hook((_bootstrap.SourceFileLoader, - _bootstrap._SOURCE_SUFFIXES, True)) + return machinery.FileFinder.path_hook((machinery.SourceFileLoader, + machinery.SOURCE_SUFFIXES, True)) def test_success(self): with source_util.create_modules('dummy') as mapping: diff --git a/Lib/inspect.py b/Lib/inspect.py index fc9f6120446..56d05fb9d4f 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -29,14 +29,15 @@ Here are some of the useful functions provided by this module: __author__ = 'Ka-Ping Yee ' __date__ = '1 Jan 2001' -import sys -import os -import types -import itertools -import re import imp -import tokenize +import importlib.machinery +import itertools import linecache +import os +import re +import sys +import tokenize +import types from operator import attrgetter from collections import namedtuple @@ -432,6 +433,8 @@ ModuleInfo = namedtuple('ModuleInfo', 'name suffix mode module_type') def getmoduleinfo(path): """Get the module name, suffix, mode, and module type for a given file.""" + warnings.warn('inspect.getmoduleinfo() is deprecated', DeprecationWarning, + 2) filename = os.path.basename(path) suffixes = [(-len(suffix), suffix, mode, mtype) for suffix, mode, mtype in imp.get_suffixes()] @@ -450,12 +453,14 @@ def getsourcefile(object): Return None if no way can be identified to get the source. """ filename = getfile(object) - if filename[-4:].lower() in ('.pyc', '.pyo'): - filename = filename[:-4] + '.py' - for suffix, mode, kind in imp.get_suffixes(): - if 'b' in mode and filename[-len(suffix):].lower() == suffix: - # Looks like a binary file. We want to only return a text file. - return None + all_bytecode_suffixes = importlib.machinery.DEBUG_BYTECODE_SUFFIXES[:] + all_bytecode_suffixes += importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES[:] + if any(filename.endswith(s) for s in all_bytecode_suffixes): + filename = (os.path.splitext(filename)[0] + + importlib.machinery.SOURCE_SUFFIXES[0]) + elif any(filename.endswith(s) for s in + importlib.machinery.EXTENSION_SUFFIXES): + return None if os.path.exists(filename): return filename # only return a non-existent filename if the module has a PEP 302 loader diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index c0910aa8774..683e3054964 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -222,8 +222,9 @@ class ModuleFinder: # But we must also collect Python extension modules - although # we cannot separate normal dlls from Python extensions. suffixes = [] - for triple in imp.get_suffixes(): - suffixes.append(triple[0]) + suffixes += importlib.machinery.EXTENSION_SUFFIXES[:] + suffixes += importlib.machinery.SOURCE_SUFFIXES[:] + suffixes += importlib.machinery.BYTECODE_SUFFIXES[:] for dir in m.__path__: try: names = os.listdir(dir) diff --git a/Lib/pydoc.py b/Lib/pydoc.py index b319d11b5e4..9c9658cef59 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -53,6 +53,7 @@ Richard Chamberlain, for the first implementation of textdoc. import builtins import imp +import importlib.machinery import inspect import io import os @@ -220,20 +221,34 @@ def synopsis(filename, cache={}): mtime = os.stat(filename).st_mtime lastupdate, result = cache.get(filename, (None, None)) if lastupdate is None or lastupdate < mtime: - info = inspect.getmoduleinfo(filename) try: file = tokenize.open(filename) except IOError: # module can't be opened, so skip it return None - if info and 'b' in info[2]: # binary modules have to be imported - try: module = imp.load_module('__temp__', file, filename, info[1:]) - except: return None + binary_suffixes = importlib.machinery.BYTECODE_SUFFIXES[:] + binary_suffixes += importlib.machinery.EXTENSION_SUFFIXES[:] + if any(filename.endswith(x) for x in binary_suffixes): + # binary modules have to be imported + file.close() + if any(filename.endswith(x) for x in + importlib.machinery.BYTECODE_SUFFIXES): + loader = importlib.machinery.SourcelessFileLoader('__temp__', + filename) + else: + loader = importlib.machinery.ExtensionFileLoader('__temp__', + filename) + try: + module = loader.load_module('__temp__') + except: + return None result = (module.__doc__ or '').splitlines()[0] del sys.modules['__temp__'] - else: # text modules can be directly examined + else: + # text modules can be directly examined result = source_synopsis(file) file.close() + cache[filename] = (mtime, result) return result diff --git a/Misc/NEWS b/Misc/NEWS index 2f11eefd58a..29b2d861ed4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -23,6 +23,11 @@ Core and Builtins Library ------- +- Issue #13959: imp.get_suffixes() has been deprecated in favour of the new + attributes on importlib.machinery: SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES, + OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES, and EXTENSION_SUFFIXES. This + led to an indirect deprecation of inspect.getmoduleinfo(). + - Issue #14662: Prevent shutil failures on OS X when destination does not support chflag operations. Patch by Hynek Schlawack.