From cb66eb0deca1d5cd232f97c76a215ecaab958d30 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 11 May 2012 12:58:42 -0400 Subject: [PATCH] Issue #13959: Deprecate imp.get_suffixes() for new attributes on importlib.machinery that provide the suffix details for import. The attributes were not put on imp so as to compartmentalize everything importlib needs for setting up imports in importlib.machinery. This also led to an indirect deprecation of inspect.getmoduleinfo() as it directly returned imp.get_suffix's returned tuple which no longer makes sense. --- Doc/library/imp.rst | 3 ++ Doc/library/importlib.rst | 37 +++++++++++++++++++ Doc/library/inspect.rst | 4 ++ Lib/ctypes/util.py | 4 +- Lib/idlelib/PathBrowser.py | 5 ++- Lib/imp.py | 33 ++++++++++------- Lib/importlib/_bootstrap.py | 24 +++++++----- Lib/importlib/machinery.py | 6 +++ Lib/importlib/test/benchmark.py | 7 ++-- Lib/importlib/test/extension/util.py | 5 +-- .../test/source/test_case_sensitivity.py | 11 +++--- Lib/importlib/test/source/test_finder.py | 20 +++++----- Lib/importlib/test/source/test_path_hook.py | 6 +-- Lib/inspect.py | 29 +++++++++------ Lib/modulefinder.py | 5 ++- Lib/pydoc.py | 25 ++++++++++--- Misc/NEWS | 5 +++ 17 files changed, 160 insertions(+), 69 deletions(-) diff --git a/Doc/library/imp.rst b/Doc/library/imp.rst index 1345b254d62..7270e3c6906 100644 --- a/Doc/library/imp.rst +++ b/Doc/library/imp.rst @@ -30,6 +30,9 @@ This module provides an interface to the mechanisms used to implement the :const:`PY_SOURCE`, :const:`PY_COMPILED`, or :const:`C_EXTENSION`, described below. + .. deprecated:: 3.3 + Use the constants defined on :mod:`importlib.machinery` instead. + .. function:: find_module(name[, path]) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 6855a7918aa..ddfaa211e71 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -477,6 +477,43 @@ are also provided to help in implementing the core ABCs. This module contains the various objects that help :keyword:`import` find and load modules. +.. attribute:: SOURCE_SUFFIXES + + A list of strings representing the recognized file suffixes for source + modules. + + .. versionadded:: 3.3 + +.. attribute:: DEBUG_BYTECODE_SUFFIXES + + A list of strings representing the file suffixes for non-optimized bytecode + modules. + + .. versionadded:: 3.3 + +.. attribute:: OPTIMIZED_BYTECODE_SUFFIXES + + A list of strings representing the file suffixes for optimized bytecode + modules. + + .. versionadded:: 3.3 + +.. attribute:: BYTECODE_SUFFIXES + + A list of strings representing the recognized file suffixes for bytecode + modules. Set to either :attr:`DEBUG_BYTECODE_SUFFIXES` or + :attr:`OPTIMIZED_BYTECODE_SUFFIXES` based on whether ``__debug__`` is true. + + .. versionadded:: 3.3 + +.. attribute:: EXTENSION_SUFFIXES + + A list of strings representing the the recognized file suffixes for + extension modules. + + .. versionadded:: 3.3 + + .. class:: BuiltinImporter An :term:`importer` for built-in modules. All known built-in modules are diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 90b7220a6b0..611f780254f 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -190,6 +190,10 @@ attributes: compared to the constants defined in the :mod:`imp` module; see the documentation for that module for more information on module types. + .. deprecated:: 3.3 + You may check the file path's suffix against the supported suffixes + listed in :mod:`importlib.machinery` to infer the same information. + .. function:: getmodulename(path) diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 97d0c2f3fa4..61eb09454b8 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -40,8 +40,8 @@ if os.name == "nt": clibname = 'msvcr%d' % (version * 10) # If python was built with in debug mode - import imp - if imp.get_suffixes()[0][0] == '_d.pyd': + import importlib.machinery + if '_d.pyd' in importlib.machinery.EXTENSION_SUFFIXES: clibname += 'd' return clibname+'.dll' diff --git a/Lib/idlelib/PathBrowser.py b/Lib/idlelib/PathBrowser.py index d88a48e344e..9954f070c0a 100644 --- a/Lib/idlelib/PathBrowser.py +++ b/Lib/idlelib/PathBrowser.py @@ -1,6 +1,7 @@ import os import sys import imp +import importlib.machinery from idlelib.TreeWidget import TreeItem from idlelib.ClassBrowser import ClassBrowser, ModuleBrowserTreeItem @@ -70,7 +71,9 @@ class DirBrowserTreeItem(TreeItem): def listmodules(self, allnames): modules = {} - suffixes = imp.get_suffixes() + suffixes = importlib.machinery.EXTENSION_SUFFIXES[:] + suffixes += importlib.machinery.SOURCE_SUFFIXES[:] + suffixes += importlib.machinery.BYTECODE_SUFFIXES[:] sorted = [] for suff, mode, flag in suffixes: i = -len(suff) diff --git a/Lib/imp.py b/Lib/imp.py index 717b6a43432..f60f0508bac 100644 --- a/Lib/imp.py +++ b/Lib/imp.py @@ -17,9 +17,11 @@ from importlib._bootstrap import new_module from importlib._bootstrap import cache_from_source from importlib import _bootstrap +from importlib import machinery import os import sys import tokenize +import warnings # XXX "deprecate" once find_module(), load_module(), and get_suffixes() are @@ -37,9 +39,12 @@ IMP_HOOK = 9 def get_suffixes(): + warnings.warn('imp.get_suffixes() is deprecated; use the constants ' + 'defined on importlib.machinery instead', + DeprecationWarning, 2) extensions = [(s, 'rb', C_EXTENSION) for s in extension_suffixes()] - source = [(s, 'U', PY_SOURCE) for s in _bootstrap._SOURCE_SUFFIXES] - bytecode = [(_bootstrap._BYTECODE_SUFFIX, 'rb', PY_COMPILED)] + source = [(s, 'U', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES] + bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES] return extensions + source + bytecode @@ -61,7 +66,7 @@ def source_from_cache(path): raise ValueError('expected only 2 dots in ' '{!r}'.format(pycache_filename)) base_filename = pycache_filename.partition('.')[0] - return os.path.join(head, base_filename + _bootstrap._SOURCE_SUFFIXES[0]) + return os.path.join(head, base_filename + machinery.SOURCE_SUFFIXES[0]) class NullImporter: @@ -126,7 +131,7 @@ def load_compiled(name, pathname, file=None): # XXX deprecate def load_package(name, path): if os.path.isdir(path): - extensions = _bootstrap._SOURCE_SUFFIXES + [_bootstrap._BYTECODE_SUFFIX] + extensions = machinery.SOURCE_SUFFIXES[:] + [machinery.BYTECODE_SUFFIXES] for extension in extensions: path = os.path.join(path, '__init__'+extension) if os.path.exists(path): @@ -190,19 +195,21 @@ def find_module(name, path=None): for entry in path: package_directory = os.path.join(entry, name) - for suffix in ['.py', _bootstrap._BYTECODE_SUFFIX]: + for suffix in ['.py', machinery.BYTECODE_SUFFIXES[0]]: package_file_name = '__init__' + suffix file_path = os.path.join(package_directory, package_file_name) if os.path.isfile(file_path): return None, package_directory, ('', '', PKG_DIRECTORY) - for suffix, mode, type_ in get_suffixes(): - file_name = name + suffix - file_path = os.path.join(entry, file_name) - if os.path.isfile(file_path): - break - else: - continue - break # Break out of outer loop when breaking out of inner loop. + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + for suffix, mode, type_ in get_suffixes(): + file_name = name + suffix + file_path = os.path.join(entry, file_name) + if os.path.isfile(file_path): + break + else: + continue + break # Break out of outer loop when breaking out of inner loop. else: raise ImportError('No module name {!r}'.format(name), name=name) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 9952b067f13..ce23043665f 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -163,11 +163,14 @@ def new_module(name): _PYCACHE = '__pycache__' -_SOURCE_SUFFIXES = ['.py'] # _setup() adds .pyw as needed. +SOURCE_SUFFIXES = ['.py'] # _setup() adds .pyw as needed. -_DEBUG_BYTECODE_SUFFIX = '.pyc' -_OPT_BYTECODE_SUFFIX = '.pyo' -_BYTECODE_SUFFIX = _DEBUG_BYTECODE_SUFFIX if __debug__ else _OPT_BYTECODE_SUFFIX +DEBUG_BYTECODE_SUFFIXES = ['.pyc'] +OPTIMIZED_BYTECODE_SUFFIXES = ['.pyo'] +if __debug__: + BYTECODE_SUFFIXES = DEBUG_BYTECODE_SUFFIXES +else: + BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES def cache_from_source(path, debug_override=None): """Given the path to a .py file, return the path to its .pyc/.pyo file. @@ -181,10 +184,13 @@ def cache_from_source(path, debug_override=None): """ debug = __debug__ if debug_override is None else debug_override - suffix = _DEBUG_BYTECODE_SUFFIX if debug else _OPT_BYTECODE_SUFFIX + if debug: + suffixes = DEBUG_BYTECODE_SUFFIXES + else: + suffixes = OPTIMIZED_BYTECODE_SUFFIXES head, tail = _path_split(path) base_filename, sep, _ = tail.partition('.') - filename = ''.join([base_filename, sep, _TAG, suffix]) + filename = ''.join([base_filename, sep, _TAG, suffixes[0]]) return _path_join(head, _PYCACHE, filename) @@ -1147,15 +1153,15 @@ def _setup(sys_module, _imp_module): setattr(self_module, '_MAGIC_NUMBER', _imp_module.get_magic()) setattr(self_module, '_TAG', _imp.get_tag()) if builtin_os == 'nt': - _SOURCE_SUFFIXES.append('.pyw') + SOURCE_SUFFIXES.append('.pyw') def _install(sys_module, _imp_module): """Install importlib as the implementation of import.""" _setup(sys_module, _imp_module) extensions = ExtensionFileLoader, _imp_module.extension_suffixes(), False - source = SourceFileLoader, _SOURCE_SUFFIXES, True - bytecode = SourcelessFileLoader, [_BYTECODE_SUFFIX], True + source = SourceFileLoader, SOURCE_SUFFIXES, True + bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES, True supported_loaders = [extensions, source, bytecode] sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)]) sys.meta_path.extend([BuiltinImporter, FrozenImporter, PathFinder]) diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index 07465ced68a..d5e7250479c 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -1,5 +1,9 @@ """The machinery of importlib: finders, loaders, hooks, etc.""" +import _imp + +from ._bootstrap import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES, + OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES) from ._bootstrap import BuiltinImporter from ._bootstrap import FrozenImporter from ._bootstrap import PathFinder @@ -7,3 +11,5 @@ from ._bootstrap import FileFinder from ._bootstrap import SourceFileLoader from ._bootstrap import SourcelessFileLoader from ._bootstrap import ExtensionFileLoader + +EXTENSION_SUFFIXES = _imp.extension_suffixes() diff --git a/Lib/importlib/test/benchmark.py b/Lib/importlib/test/benchmark.py index 16863146b5a..183e8180c94 100644 --- a/Lib/importlib/test/benchmark.py +++ b/Lib/importlib/test/benchmark.py @@ -9,7 +9,6 @@ from .source import util as source_util import decimal import imp import importlib -import importlib._bootstrap import importlib.machinery import json import os @@ -72,7 +71,7 @@ def source_wo_bytecode(seconds, repeat): assert not os.path.exists(imp.cache_from_source(mapping[name])) sys.meta_path.append(importlib.machinery.PathFinder) loader = (importlib.machinery.SourceFileLoader, - importlib._bootstrap._SOURCE_SUFFIXES, True) + importlib.machinery.SOURCE_SUFFIXES, True) sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader)) for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat, seconds=seconds): @@ -110,7 +109,7 @@ def source_writing_bytecode(seconds, repeat): with source_util.create_modules(name) as mapping: sys.meta_path.append(importlib.machinery.PathFinder) loader = (importlib.machinery.SourceFileLoader, - importlib._bootstrap._SOURCE_SUFFIXES, True) + importlib.machinery.SOURCE_SUFFIXES, True) sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader)) def cleanup(): sys.modules.pop(name) @@ -145,7 +144,7 @@ def source_using_bytecode(seconds, repeat): with source_util.create_modules(name) as mapping: sys.meta_path.append(importlib.machinery.PathFinder) loader = (importlib.machinery.SourceFileLoader, - importlib._bootstrap._SOURCE_SUFFIXES, True) + importlib.machinery.SOURCE_SUFFIXES, True) sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader)) py_compile.compile(mapping[name]) assert os.path.exists(imp.cache_from_source(mapping[name])) diff --git a/Lib/importlib/test/extension/util.py b/Lib/importlib/test/extension/util.py index d1491697483..a266dd98c83 100644 --- a/Lib/importlib/test/extension/util.py +++ b/Lib/importlib/test/extension/util.py @@ -1,4 +1,5 @@ import imp +from importlib import machinery import os import sys @@ -6,10 +7,9 @@ PATH = None EXT = None FILENAME = None NAME = '_testcapi' -_file_exts = [x[0] for x in imp.get_suffixes() if x[2] == imp.C_EXTENSION] try: for PATH in sys.path: - for EXT in _file_exts: + for EXT in machinery.EXTENSION_SUFFIXES: FILENAME = NAME + EXT FILEPATH = os.path.join(PATH, FILENAME) if os.path.exists(os.path.join(PATH, FILENAME)): @@ -18,4 +18,3 @@ try: PATH = EXT = FILENAME = FILEPATH = None except StopIteration: pass -del _file_exts diff --git a/Lib/importlib/test/source/test_case_sensitivity.py b/Lib/importlib/test/source/test_case_sensitivity.py index fade25fe104..21a4378ce9d 100644 --- a/Lib/importlib/test/source/test_case_sensitivity.py +++ b/Lib/importlib/test/source/test_case_sensitivity.py @@ -1,5 +1,6 @@ """Test case-sensitivity (PEP 235).""" from importlib import _bootstrap +from importlib import machinery from .. import util from . import util as source_util import imp @@ -20,12 +21,12 @@ class CaseSensitivityTest(unittest.TestCase): assert name != name.lower() def find(self, path): - finder = _bootstrap.FileFinder(path, - (_bootstrap.SourceFileLoader, - _bootstrap._SOURCE_SUFFIXES, + finder = machinery.FileFinder(path, + (machinery.SourceFileLoader, + machinery.SOURCE_SUFFIXES, True), - (_bootstrap.SourcelessFileLoader, - [_bootstrap._BYTECODE_SUFFIX], + (machinery.SourcelessFileLoader, + machinery.BYTECODE_SUFFIXES, True)) return finder.find_module(self.name) diff --git a/Lib/importlib/test/source/test_finder.py b/Lib/importlib/test/source/test_finder.py index 45b804ffdff..bbe01639fde 100644 --- a/Lib/importlib/test/source/test_finder.py +++ b/Lib/importlib/test/source/test_finder.py @@ -1,7 +1,7 @@ from .. import abc from . import util as source_util -from importlib import _bootstrap +from importlib import machinery import errno import imp import os @@ -36,11 +36,11 @@ class FinderTests(abc.FinderTests): """ def import_(self, root, module): - loader_details = [(_bootstrap.SourceFileLoader, - _bootstrap._SOURCE_SUFFIXES, True), - (_bootstrap.SourcelessFileLoader, - [_bootstrap._BYTECODE_SUFFIX], True)] - finder = _bootstrap.FileFinder(root, *loader_details) + loader_details = [(machinery.SourceFileLoader, + machinery.SOURCE_SUFFIXES, True), + (machinery.SourcelessFileLoader, + machinery.BYTECODE_SUFFIXES, True)] + finder = machinery.FileFinder(root, *loader_details) return finder.find_module(module) def run_test(self, test, create=None, *, compile_=None, unlink=None): @@ -138,8 +138,8 @@ class FinderTests(abc.FinderTests): def test_empty_string_for_dir(self): # The empty string from sys.path means to search in the cwd. - finder = _bootstrap.FileFinder('', (_bootstrap.SourceFileLoader, - _bootstrap._SOURCE_SUFFIXES, True)) + finder = machinery.FileFinder('', (machinery.SourceFileLoader, + machinery.SOURCE_SUFFIXES, True)) with open('mod.py', 'w') as file: file.write("# test file for importlib") try: @@ -150,8 +150,8 @@ class FinderTests(abc.FinderTests): def test_invalidate_caches(self): # invalidate_caches() should reset the mtime. - finder = _bootstrap.FileFinder('', (_bootstrap.SourceFileLoader, - _bootstrap._SOURCE_SUFFIXES, True)) + finder = machinery.FileFinder('', (machinery.SourceFileLoader, + machinery.SOURCE_SUFFIXES, True)) finder._path_mtime = 42 finder.invalidate_caches() self.assertEqual(finder._path_mtime, -1) diff --git a/Lib/importlib/test/source/test_path_hook.py b/Lib/importlib/test/source/test_path_hook.py index df69b4d61ed..54c06995014 100644 --- a/Lib/importlib/test/source/test_path_hook.py +++ b/Lib/importlib/test/source/test_path_hook.py @@ -1,6 +1,6 @@ from . import util as source_util -from importlib import _bootstrap +from importlib import machinery import imp import unittest @@ -10,8 +10,8 @@ class PathHookTest(unittest.TestCase): """Test the path hook for source.""" def path_hook(self): - return _bootstrap.FileFinder.path_hook((_bootstrap.SourceFileLoader, - _bootstrap._SOURCE_SUFFIXES, True)) + return machinery.FileFinder.path_hook((machinery.SourceFileLoader, + machinery.SOURCE_SUFFIXES, True)) def test_success(self): with source_util.create_modules('dummy') as mapping: diff --git a/Lib/inspect.py b/Lib/inspect.py index fc9f6120446..56d05fb9d4f 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -29,14 +29,15 @@ Here are some of the useful functions provided by this module: __author__ = 'Ka-Ping Yee ' __date__ = '1 Jan 2001' -import sys -import os -import types -import itertools -import re import imp -import tokenize +import importlib.machinery +import itertools import linecache +import os +import re +import sys +import tokenize +import types from operator import attrgetter from collections import namedtuple @@ -432,6 +433,8 @@ ModuleInfo = namedtuple('ModuleInfo', 'name suffix mode module_type') def getmoduleinfo(path): """Get the module name, suffix, mode, and module type for a given file.""" + warnings.warn('inspect.getmoduleinfo() is deprecated', DeprecationWarning, + 2) filename = os.path.basename(path) suffixes = [(-len(suffix), suffix, mode, mtype) for suffix, mode, mtype in imp.get_suffixes()] @@ -450,12 +453,14 @@ def getsourcefile(object): Return None if no way can be identified to get the source. """ filename = getfile(object) - if filename[-4:].lower() in ('.pyc', '.pyo'): - filename = filename[:-4] + '.py' - for suffix, mode, kind in imp.get_suffixes(): - if 'b' in mode and filename[-len(suffix):].lower() == suffix: - # Looks like a binary file. We want to only return a text file. - return None + all_bytecode_suffixes = importlib.machinery.DEBUG_BYTECODE_SUFFIXES[:] + all_bytecode_suffixes += importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES[:] + if any(filename.endswith(s) for s in all_bytecode_suffixes): + filename = (os.path.splitext(filename)[0] + + importlib.machinery.SOURCE_SUFFIXES[0]) + elif any(filename.endswith(s) for s in + importlib.machinery.EXTENSION_SUFFIXES): + return None if os.path.exists(filename): return filename # only return a non-existent filename if the module has a PEP 302 loader diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index c0910aa8774..683e3054964 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -222,8 +222,9 @@ class ModuleFinder: # But we must also collect Python extension modules - although # we cannot separate normal dlls from Python extensions. suffixes = [] - for triple in imp.get_suffixes(): - suffixes.append(triple[0]) + suffixes += importlib.machinery.EXTENSION_SUFFIXES[:] + suffixes += importlib.machinery.SOURCE_SUFFIXES[:] + suffixes += importlib.machinery.BYTECODE_SUFFIXES[:] for dir in m.__path__: try: names = os.listdir(dir) diff --git a/Lib/pydoc.py b/Lib/pydoc.py index b319d11b5e4..9c9658cef59 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -53,6 +53,7 @@ Richard Chamberlain, for the first implementation of textdoc. import builtins import imp +import importlib.machinery import inspect import io import os @@ -220,20 +221,34 @@ def synopsis(filename, cache={}): mtime = os.stat(filename).st_mtime lastupdate, result = cache.get(filename, (None, None)) if lastupdate is None or lastupdate < mtime: - info = inspect.getmoduleinfo(filename) try: file = tokenize.open(filename) except IOError: # module can't be opened, so skip it return None - if info and 'b' in info[2]: # binary modules have to be imported - try: module = imp.load_module('__temp__', file, filename, info[1:]) - except: return None + binary_suffixes = importlib.machinery.BYTECODE_SUFFIXES[:] + binary_suffixes += importlib.machinery.EXTENSION_SUFFIXES[:] + if any(filename.endswith(x) for x in binary_suffixes): + # binary modules have to be imported + file.close() + if any(filename.endswith(x) for x in + importlib.machinery.BYTECODE_SUFFIXES): + loader = importlib.machinery.SourcelessFileLoader('__temp__', + filename) + else: + loader = importlib.machinery.ExtensionFileLoader('__temp__', + filename) + try: + module = loader.load_module('__temp__') + except: + return None result = (module.__doc__ or '').splitlines()[0] del sys.modules['__temp__'] - else: # text modules can be directly examined + else: + # text modules can be directly examined result = source_synopsis(file) file.close() + cache[filename] = (mtime, result) return result diff --git a/Misc/NEWS b/Misc/NEWS index 2f11eefd58a..29b2d861ed4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -23,6 +23,11 @@ Core and Builtins Library ------- +- Issue #13959: imp.get_suffixes() has been deprecated in favour of the new + attributes on importlib.machinery: SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES, + OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES, and EXTENSION_SUFFIXES. This + led to an indirect deprecation of inspect.getmoduleinfo(). + - Issue #14662: Prevent shutil failures on OS X when destination does not support chflag operations. Patch by Hynek Schlawack.