"""Check extension modules The script checks shared and built-in extension modules. It verifies that the modules have been built and that they can be imported successfully. Missing modules and failed imports are reported to the user. Shared extension files are renamed on failed import. Module information is parsed from several sources: - core modules hard-coded in Modules/config.c.in - Windows-specific modules that are hard-coded in PC/config.c - MODULE_{name}_STATE entries in Makefile (provided through sysconfig) - Various makesetup files: - $(srcdir)/Modules/Setup - Modules/Setup.[local|bootstrap|stdlib] files, which are generated from $(srcdir)/Modules/Setup.*.in files See --help for more information """ import argparse import collections import enum import logging import os import pathlib import re import sys import sysconfig import warnings import _imp from importlib._bootstrap import _load as bootstrap_load from importlib.machinery import BuiltinImporter, ExtensionFileLoader, ModuleSpec from importlib.util import spec_from_file_location, spec_from_loader from typing import Iterable SRC_DIR = pathlib.Path(__file__).parent.parent.parent # core modules, hard-coded in Modules/config.h.in CORE_MODULES = { "_ast", "_imp", "_string", "_tokenize", "_warnings", "builtins", "gc", "marshal", "sys", } # Windows-only modules WINDOWS_MODULES = { "_overlapped", "_testconsole", "_winapi", "_wmi", "msvcrt", "nt", "winreg", "winsound", } logger = logging.getLogger(__name__) parser = argparse.ArgumentParser( prog="check_extension_modules", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--verbose", action="store_true", help="Verbose, report builtin, shared, and unavailable modules", ) parser.add_argument( "--debug", action="store_true", help="Enable debug logging", ) parser.add_argument( "--strict", action=argparse.BooleanOptionalAction, help=( "Strict check, fail when a module is missing or fails to import" "(default: no, unless env var PYTHONSTRICTEXTENSIONBUILD is set)" ), default=bool(os.environ.get("PYTHONSTRICTEXTENSIONBUILD")), ) parser.add_argument( "--cross-compiling", action=argparse.BooleanOptionalAction, help=( "Use cross-compiling checks " "(default: no, unless env var _PYTHON_HOST_PLATFORM is set)." ), default="_PYTHON_HOST_PLATFORM" in os.environ, ) parser.add_argument( "--list-module-names", action="store_true", help="Print a list of module names to stdout and exit", ) class ModuleState(enum.Enum): # Makefile state "yes" BUILTIN = "builtin" SHARED = "shared" DISABLED = "disabled" MISSING = "missing" NA = "n/a" # disabled by Setup / makesetup rule DISABLED_SETUP = "disabled_setup" def __bool__(self): return self.value in {"builtin", "shared"} ModuleInfo = collections.namedtuple("ModuleInfo", "name state") class ModuleChecker: pybuilddir_txt = "pybuilddir.txt" setup_files = ( # see end of configure.ac "Modules/Setup.local", "Modules/Setup.stdlib", "Modules/Setup.bootstrap", SRC_DIR / "Modules/Setup", ) def __init__(self, cross_compiling: bool = False, strict: bool = False): self.cross_compiling = cross_compiling self.strict_extensions_build = strict self.ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") self.platform = sysconfig.get_platform() self.builddir = self.get_builddir() self.modules = self.get_modules() self.builtin_ok = [] self.shared_ok = [] self.failed_on_import = [] self.missing = [] self.disabled_configure = [] self.disabled_setup = [] self.notavailable = [] def check(self): if not hasattr(_imp, 'create_dynamic'): logger.warning( ('Dynamic extensions not supported ' '(HAVE_DYNAMIC_LOADING not defined)'), ) for modinfo in self.modules: logger.debug("Checking '%s' (%s)", modinfo.name, self.get_location(modinfo)) if modinfo.state == ModuleState.DISABLED: self.disabled_configure.append(modinfo) elif modinfo.state == ModuleState.DISABLED_SETUP: self.disabled_setup.append(modinfo) elif modinfo.state == ModuleState.MISSING: self.missing.append(modinfo) elif modinfo.state == ModuleState.NA: self.notavailable.append(modinfo) else: try: if self.cross_compiling: self.check_module_cross(modinfo) else: self.check_module_import(modinfo) except (ImportError, FileNotFoundError): self.rename_module(modinfo) self.failed_on_import.append(modinfo) else: if modinfo.state == ModuleState.BUILTIN: self.builtin_ok.append(modinfo) else: assert modinfo.state == ModuleState.SHARED self.shared_ok.append(modinfo) def summary(self, *, verbose: bool = False): longest = max([len(e.name) for e in self.modules], default=0) def print_three_column(modinfos: list[ModuleInfo]): names = [modinfo.name for modinfo in modinfos] names.sort(key=str.lower) # guarantee zip() doesn't drop anything while len(names) % 3: names.append("") for l, m, r in zip(names[::3], names[1::3], names[2::3]): print("%-*s %-*s %-*s" % (longest, l, longest, m, longest, r)) if verbose and self.builtin_ok: print("The following *built-in* modules have been successfully built:") print_three_column(self.builtin_ok) print() if verbose and self.shared_ok: print("The following *shared* modules have been successfully built:") print_three_column(self.shared_ok) print() if self.disabled_configure: print("The following modules are *disabled* in configure script:") print_three_column(self.disabled_configure) print() if self.disabled_setup: print("The following modules are *disabled* in Modules/Setup files:") print_three_column(self.disabled_setup) print() if verbose and self.notavailable: print( f"The following modules are not available on platform '{self.platform}':" ) print_three_column(self.notavailable) print() if self.missing: print("The necessary bits to build these optional modules were not found:") print_three_column(self.missing) print("To find the necessary bits, look in configure.ac and config.log.") print() if self.failed_on_import: print( "Following modules built successfully " "but were removed because they could not be imported:" ) print_three_column(self.failed_on_import) print() if any( modinfo.name == "_ssl" for modinfo in self.missing + self.failed_on_import ): print("Could not build the ssl module!") print("Python requires a OpenSSL 1.1.1 or newer") if sysconfig.get_config_var("OPENSSL_LDFLAGS"): print("Custom linker flags may require --with-openssl-rpath=auto") print() disabled = len(self.disabled_configure) + len(self.disabled_setup) print( f"Checked {len(self.modules)} modules (" f"{len(self.builtin_ok)} built-in, " f"{len(self.shared_ok)} shared, " f"{len(self.notavailable)} n/a on {self.platform}, " f"{disabled} disabled, " f"{len(self.missing)} missing, " f"{len(self.failed_on_import)} failed on import)" ) def check_strict_build(self): """Fail if modules are missing and it's a strict build""" if self.strict_extensions_build and (self.failed_on_import or self.missing): raise RuntimeError("Failed to build some stdlib modules") def list_module_names(self, *, all: bool = False) -> set: names = {modinfo.name for modinfo in self.modules} if all: names.update(WINDOWS_MODULES) return names def get_builddir(self) -> pathlib.Path: try: with open(self.pybuilddir_txt, encoding="utf-8") as f: builddir = f.read() except FileNotFoundError: logger.error("%s must be run from the top build directory", __file__) raise builddir = pathlib.Path(builddir) logger.debug("%s: %s", self.pybuilddir_txt, builddir) return builddir def get_modules(self) -> list[ModuleInfo]: """Get module info from sysconfig and Modules/Setup* files""" seen = set() modules = [] # parsing order is important, first entry wins for modinfo in self.get_core_modules(): modules.append(modinfo) seen.add(modinfo.name) for setup_file in self.setup_files: for modinfo in self.parse_setup_file(setup_file): if modinfo.name not in seen: modules.append(modinfo) seen.add(modinfo.name) for modinfo in self.get_sysconfig_modules(): if modinfo.name not in seen: modules.append(modinfo) seen.add(modinfo.name) logger.debug("Found %i modules in total", len(modules)) modules.sort() return modules def get_core_modules(self) -> Iterable[ModuleInfo]: """Get hard-coded core modules""" for name in CORE_MODULES: modinfo = ModuleInfo(name, ModuleState.BUILTIN) logger.debug("Found core module %s", modinfo) yield modinfo def get_sysconfig_modules(self) -> Iterable[ModuleInfo]: """Get modules defined in Makefile through sysconfig MODBUILT_NAMES: modules in *static* block MODSHARED_NAMES: modules in *shared* block MODDISABLED_NAMES: modules in *disabled* block """ moddisabled = set(sysconfig.get_config_var("MODDISABLED_NAMES").split()) if self.cross_compiling: modbuiltin = set(sysconfig.get_config_var("MODBUILT_NAMES").split()) else: modbuiltin = set(sys.builtin_module_names) for key, value in sysconfig.get_config_vars().items(): if not key.startswith("MODULE_") or not key.endswith("_STATE"): continue if value not in {"yes", "disabled", "missing", "n/a"}: raise ValueError(f"Unsupported value '{value}' for {key}") modname = key[7:-6].lower() if modname in moddisabled: # Setup "*disabled*" rule state = ModuleState.DISABLED_SETUP elif value in {"disabled", "missing", "n/a"}: state = ModuleState(value) elif modname in modbuiltin: assert value == "yes" state = ModuleState.BUILTIN else: assert value == "yes" state = ModuleState.SHARED modinfo = ModuleInfo(modname, state) logger.debug("Found %s in Makefile", modinfo) yield modinfo def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]: """Parse a Modules/Setup file""" assign_var = re.compile(r"^\w+=") # EGG_SPAM=foo # default to static module state = ModuleState.BUILTIN logger.debug("Parsing Setup file %s", setup_file) with open(setup_file, encoding="utf-8") as f: for line in f: line = line.strip() if not line or line.startswith("#") or assign_var.match(line): continue match line.split(): case ["*shared*"]: state = ModuleState.SHARED case ["*static*"]: state = ModuleState.BUILTIN case ["*disabled*"]: state = ModuleState.DISABLED case ["*noconfig*"]: state = None case [*items]: if state == ModuleState.DISABLED: # *disabled* can disable multiple modules per line for item in items: modinfo = ModuleInfo(item, state) logger.debug("Found %s in %s", modinfo, setup_file) yield modinfo elif state in {ModuleState.SHARED, ModuleState.BUILTIN}: # *shared* and *static*, first item is the name of the module. modinfo = ModuleInfo(items[0], state) logger.debug("Found %s in %s", modinfo, setup_file) yield modinfo def get_spec(self, modinfo: ModuleInfo) -> ModuleSpec: """Get ModuleSpec for builtin or extension module""" if modinfo.state == ModuleState.SHARED: location = os.fspath(self.get_location(modinfo)) loader = ExtensionFileLoader(modinfo.name, location) return spec_from_file_location(modinfo.name, location, loader=loader) elif modinfo.state == ModuleState.BUILTIN: return spec_from_loader(modinfo.name, loader=BuiltinImporter) else: raise ValueError(modinfo) def get_location(self, modinfo: ModuleInfo) -> pathlib.Path: """Get shared library location in build directory""" if modinfo.state == ModuleState.SHARED: return self.builddir / f"{modinfo.name}{self.ext_suffix}" else: return None def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec): """Check that the module file is present and not empty""" if spec.loader is BuiltinImporter: return try: st = os.stat(spec.origin) except FileNotFoundError: logger.error("%s (%s) is missing", modinfo.name, spec.origin) raise if not st.st_size: raise ImportError(f"{spec.origin} is an empty file") def check_module_import(self, modinfo: ModuleInfo): """Attempt to import module and report errors""" spec = self.get_spec(modinfo) self._check_file(modinfo, spec) try: with warnings.catch_warnings(): # ignore deprecation warning from deprecated modules warnings.simplefilter("ignore", DeprecationWarning) bootstrap_load(spec) except ImportError as e: logger.error("%s failed to import: %s", modinfo.name, e) raise except Exception as e: if not hasattr(_imp, 'create_dynamic'): logger.warning("Dynamic extension '%s' ignored", modinfo.name) return logger.exception("Importing extension '%s' failed!", modinfo.name) raise def check_module_cross(self, modinfo: ModuleInfo): """Sanity check for cross compiling""" spec = self.get_spec(modinfo) self._check_file(modinfo, spec) def rename_module(self, modinfo: ModuleInfo) -> None: """Rename module file""" if modinfo.state == ModuleState.BUILTIN: logger.error("Cannot mark builtin module '%s' as failed!", modinfo.name) return failed_name = f"{modinfo.name}_failed{self.ext_suffix}" builddir_path = self.get_location(modinfo) if builddir_path.is_symlink(): symlink = builddir_path module_path = builddir_path.resolve().relative_to(os.getcwd()) failed_path = module_path.parent / failed_name else: symlink = None module_path = builddir_path failed_path = self.builddir / failed_name # remove old failed file failed_path.unlink(missing_ok=True) # remove symlink if symlink is not None: symlink.unlink(missing_ok=True) # rename shared extension file try: module_path.rename(failed_path) except FileNotFoundError: logger.debug("Shared extension file '%s' does not exist.", module_path) else: logger.debug("Rename '%s' -> '%s'", module_path, failed_path) def main(): args = parser.parse_args() if args.debug: args.verbose = True logging.basicConfig( level=logging.DEBUG if args.debug else logging.INFO, format="[%(levelname)s] %(message)s", ) checker = ModuleChecker( cross_compiling=args.cross_compiling, strict=args.strict, ) if args.list_module_names: names = checker.list_module_names(all=True) for name in sorted(names): print(name) else: checker.check() checker.summary(verbose=args.verbose) try: checker.check_strict_build() except RuntimeError as e: parser.exit(1, f"\nError: {e}\n") if __name__ == "__main__": main()