bpo-45019: Do some cleanup related to frozen modules. (gh-28319)

There are a few things I missed in gh-27980. This is a follow-up that will make subsequent PRs cleaner. It includes fixes to tests and tools that reference the frozen modules.

https://bugs.python.org/issue45019

.gitattributes

@@ -47,6 +47,7 @@ Objects/clinic/*.h          linguist-generated=true
 PC/clinic/*.h               linguist-generated=true
 Python/clinic/*.h           linguist-generated=true
 Python/frozen_modules/*.h   linguist-generated=true
+Python/frozen_modules/MANIFEST  linguist-generated=true
 Include/internal/pycore_ast.h   linguist-generated=true
 Python/Python-ast.c         linguist-generated=true
 Include/opcode.h            linguist-generated=true

.gitignore

@@ -121,6 +121,13 @@ Tools/msi/obj
 Tools/ssl/amd64
 Tools/ssl/win32
 
+# TODO: Once we auto-regen frozem modules for Windows builds
+# we can drop the .h files from the repo and ignore them here.
+# At that point we will rely the frozen manifest file to identify
+# changed generated files.  We'll drop the entry for it then.
+# See: Tools/scripts/freeze_modules.py.
+#Python/frozen_modules/*.h
+
 # Two-trick pony for OSX and other case insensitive file systems:
 # Ignore ./python binary on Unix but still look into ./Python/ directory.
 /python

Lib/ctypes/test/test_values.py

@@ -2,9 +2,12 @@
 A testcase which accesses *values* in a dll.
 """
 
+import imp
+import importlib.util
 import unittest
 import sys
 from ctypes import *
+from test.support import import_helper, captured_stdout
 
 import _ctypes_test
 
@@ -55,41 +58,32 @@ class PythonValuesTestCase(unittest.TestCase):
 
         ft = FrozenTable.in_dll(pythonapi, "PyImport_FrozenModules")
         # ft is a pointer to the struct_frozen entries:
-        items = []
+        modules = []
-        # _frozen_importlib changes size whenever importlib._bootstrap
-        # changes, so it gets a special case.  We should make sure it's
-        # found, but don't worry about its size too much.  The same
-        # applies to _frozen_importlib_external.
-        bootstrap_seen = []
-        bootstrap_expected = [
-                b'_frozen_importlib',
-                b'_frozen_importlib_external',
-                b'zipimport',
-                ]
         for entry in ft:
             # This is dangerous. We *can* iterate over a pointer, but
             # the loop will not terminate (maybe with an access
             # violation;-) because the pointer instance has no size.
             if entry.name is None:
                 break
+            modname = entry.name.decode("ascii")
+            modules.append(modname)
+            with self.subTest(modname):
+                # Do a sanity check on entry.size and entry.code.
+                self.assertGreater(abs(entry.size), 10)
+                self.assertTrue([entry.code[i] for i in range(abs(entry.size))])
+                # Check the module's package-ness.
+                spec = importlib.util.find_spec(modname)
+                if entry.size < 0:
+                    # It's a package.
+                    self.assertIsNotNone(spec.submodule_search_locations)
+                else:
+                    self.assertIsNone(spec.submodule_search_locations)
 
-            if entry.name in bootstrap_expected:
+        expected = imp._frozen_module_names()
-                bootstrap_seen.append(entry.name)
+        self.maxDiff = None
-                self.assertTrue(entry.size,
+        self.assertEqual(modules, expected, "PyImport_FrozenModules example "
-                    "{!r} was reported as having no size".format(entry.name))
-                continue
-            items.append((entry.name.decode("ascii"), entry.size))
-
-        expected = [("__hello__", 164),
-                    ("__phello__", -164),
-                    ("__phello__.spam", 164),
-                    ]
-        self.assertEqual(items, expected, "PyImport_FrozenModules example "
             "in Doc/library/ctypes.rst may be out of date")
 
-        self.assertEqual(sorted(bootstrap_seen), bootstrap_expected,
-            "frozen bootstrap modules did not match PyImport_FrozenModules")
-
         from ctypes import _pointer_type_cache
         del _pointer_type_cache[struct_frozen]
 

Lib/imp.py

@@ -9,7 +9,7 @@ functionality over this module.
 from _imp import (lock_held, acquire_lock, release_lock,
                   get_frozen_object, is_frozen_package,
                   init_frozen, is_builtin, is_frozen,
-                  _fix_co_filename)
+                  _fix_co_filename, _frozen_module_names)
 try:
     from _imp import create_dynamic
 except ImportError:

Makefile.pre.in

@@ -750,22 +750,22 @@ regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES)
 
 # BEGIN: freezing modules
 
-Python/frozen_modules/importlib__bootstrap.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap.py
+Python/frozen_modules/importlib__bootstrap.h: Programs/_freeze_module Lib/importlib/_bootstrap.py
 	$(srcdir)/Programs/_freeze_module importlib._bootstrap \
 		$(srcdir)/Lib/importlib/_bootstrap.py \
 		$(srcdir)/Python/frozen_modules/importlib__bootstrap.h
 
-Python/frozen_modules/importlib__bootstrap_external.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap_external.py
+Python/frozen_modules/importlib__bootstrap_external.h: Programs/_freeze_module Lib/importlib/_bootstrap_external.py
 	$(srcdir)/Programs/_freeze_module importlib._bootstrap_external \
 		$(srcdir)/Lib/importlib/_bootstrap_external.py \
 		$(srcdir)/Python/frozen_modules/importlib__bootstrap_external.h
 
-Python/frozen_modules/zipimport.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/zipimport.py
+Python/frozen_modules/zipimport.h: Programs/_freeze_module Lib/zipimport.py
 	$(srcdir)/Programs/_freeze_module zipimport \
 		$(srcdir)/Lib/zipimport.py \
 		$(srcdir)/Python/frozen_modules/zipimport.h
 
-Python/frozen_modules/hello.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Tools/freeze/flag.py
+Python/frozen_modules/hello.h: Programs/_freeze_module Tools/freeze/flag.py
 	$(srcdir)/Programs/_freeze_module hello \
 		$(srcdir)/Tools/freeze/flag.py \
 		$(srcdir)/Python/frozen_modules/hello.h

Python/clinic/import.c.h

@@ -297,6 +297,24 @@ exit:
     return return_value;
 }
 
+PyDoc_STRVAR(_imp__frozen_module_names__doc__,
+"_frozen_module_names($module, /)\n"
+"--\n"
+"\n"
+"Returns the list of available frozen modules.");
+
+#define _IMP__FROZEN_MODULE_NAMES_METHODDEF    \
+    {"_frozen_module_names", (PyCFunction)_imp__frozen_module_names, METH_NOARGS, _imp__frozen_module_names__doc__},
+
+static PyObject *
+_imp__frozen_module_names_impl(PyObject *module);
+
+static PyObject *
+_imp__frozen_module_names(PyObject *module, PyObject *Py_UNUSED(ignored))
+{
+    return _imp__frozen_module_names_impl(module);
+}
+
 #if defined(HAVE_DYNAMIC_LOADING)
 
 PyDoc_STRVAR(_imp_create_dynamic__doc__,
@@ -449,4 +467,4 @@ exit:
 #ifndef _IMP_EXEC_DYNAMIC_METHODDEF
     #define _IMP_EXEC_DYNAMIC_METHODDEF
 #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */
-/*[clinic end generated code: output=7c31c433af88af6b input=a9049054013a1b77]*/
+/*[clinic end generated code: output=0ab3fa7c5808bba4 input=a9049054013a1b77]*/

Python/frozen.c

@@ -47,7 +47,7 @@
 /* Note that a negative size indicates a package. */
 
 static const struct _frozen _PyImport_FrozenModules[] = {
-    /* importlib */
+    /* import system */
     {"_frozen_importlib", _Py_M__importlib__bootstrap,
         (int)sizeof(_Py_M__importlib__bootstrap)},
     {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external,

Python/frozen_modules/MANIFEST

@@ -0,0 +1,12 @@
+# The list of frozen modules with key information.
+# Note that the "check_generated_files" CI job will identify
+# when source files were changed but regen-frozen wasn't run.
+# This file is auto-generated by Tools/scripts/freeze_modules.py.
+          module           ispkg              source                          frozen               checksum
+-------------------------- ----- ------------------------------- ------------------------------- ------------
+_frozen_importlib            no  <importlib._bootstrap>          importlib__bootstrap.h          749d553f858d
+_frozen_importlib_external   no  <importlib._bootstrap_external> importlib__bootstrap_external.h e4539e6347d7
+zipimport                    no  <zipimport>                     zipimport.h                     374879e5d43d
+__hello__                    no  Tools/freeze/flag.py            hello.h                         af6fb665713f
+__phello__                  YES  Tools/freeze/flag.py            hello.h                         af6fb665713f
+__phello__.spam              no  Tools/freeze/flag.py            hello.h                         af6fb665713f

Python/frozen_modules/README.txt

@@ -0,0 +1,7 @@
+This directory contains the generated .h files for all the frozen
+modules.  Python/frozen.c depends on these files.
+
+Note that, other than the required frozen modules, none of these files
+are committed into the repo.
+
+See Tools/scripts/freeze_modules.py for more info.

Python/import.c

@@ -16,6 +16,7 @@
 #include "code.h"
 #include "importdl.h"
 #include "pydtrace.h"
+#include <stdbool.h>
 
 #ifdef HAVE_FCNTL_H
 #include <fcntl.h>
@@ -1049,6 +1050,32 @@ _imp_create_builtin(PyObject *module, PyObject *spec)
 
 /* Frozen modules */
 
+static PyObject *
+list_frozen_module_names(bool force)
+{
+    PyObject *names = PyList_New(0);
+    if (names == NULL) {
+        return NULL;
+    }
+    for (const struct _frozen *p = PyImport_FrozenModules; ; p++) {
+        if (p->name == NULL) {
+            break;
+        }
+        PyObject *name = PyUnicode_FromString(p->name);
+        if (name == NULL) {
+            Py_DECREF(names);
+            return NULL;
+        }
+        int res = PyList_Append(names, name);
+        Py_DECREF(name);
+        if (res != 0) {
+            Py_DECREF(names);
+            return NULL;
+        }
+    }
+    return names;
+}
+
 static const struct _frozen *
 find_frozen(PyObject *name)
 {
@@ -1954,6 +1981,19 @@ _imp_is_frozen_impl(PyObject *module, PyObject *name)
     return PyBool_FromLong((long) (p == NULL ? 0 : p->size));
 }
 
+/*[clinic input]
+_imp._frozen_module_names
+
+Returns the list of available frozen modules.
+[clinic start generated code]*/
+
+static PyObject *
+_imp__frozen_module_names_impl(PyObject *module)
+/*[clinic end generated code: output=80609ef6256310a8 input=76237fbfa94460d2]*/
+{
+    return list_frozen_module_names(true);
+}
+
 /* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */
 static int
 exec_builtin_or_dynamic(PyObject *mod) {
@@ -2114,6 +2154,7 @@ static PyMethodDef imp_methods[] = {
     _IMP_INIT_FROZEN_METHODDEF
     _IMP_IS_BUILTIN_METHODDEF
     _IMP_IS_FROZEN_METHODDEF
+    _IMP__FROZEN_MODULE_NAMES_METHODDEF
     _IMP_CREATE_DYNAMIC_METHODDEF
     _IMP_EXEC_DYNAMIC_METHODDEF
     _IMP_EXEC_BUILTIN_METHODDEF

Tools/scripts/freeze_modules.py

@@ -3,6 +3,8 @@
 See the notes at the top of Python/frozen.c for more info.
 """
 
+from collections import namedtuple
+import hashlib
 import os
 import os.path
 import subprocess
@@ -21,18 +23,24 @@ STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib')
 MODULES_DIR = os.path.join(ROOT_DIR, 'Python/frozen_modules')
 TOOL = os.path.join(ROOT_DIR, 'Programs', '_freeze_module')
 
+MANIFEST = os.path.join(MODULES_DIR, 'MANIFEST')
 FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c')
 MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in')
 PCBUILD_PROJECT = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj')
 PCBUILD_FILTERS = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj.filters')
+TEST_CTYPES = os.path.join(STDLIB_DIR, 'ctypes', 'test', 'test_values.py')
 
 # These are modules that get frozen.
 FROZEN = [
     # See parse_frozen_spec() for the format.
     # In cases where the frozenid is duplicated, the first one is re-used.
-    ('importlib', [
+    ('import system', [
+        # These frozen modules are necessary for bootstrapping
+        # the import system.
         'importlib._bootstrap : _frozen_importlib',
         'importlib._bootstrap_external : _frozen_importlib_external',
+        # This module is important because some Python builds rely
+        # on a builtin zip file instead of a filesystem.
         'zipimport',
         ]),
     ('Test module', [
@@ -41,13 +49,43 @@ FROZEN = [
         'hello : __phello__.spam',
         ]),
 ]
+ESSENTIAL = {
+    'importlib._bootstrap',
+    'importlib._bootstrap_external',
+    'zipimport',
+}
 
 
 #######################################
 # specs
 
-def parse_frozen_spec(rawspec, knownids=None, section=None):
+def parse_frozen_specs(sectionalspecs=FROZEN, destdir=None):
-    """Yield (frozenid, pyfile, modname, ispkg) for the corresponding modules.
+    seen = {}
+    for section, specs in sectionalspecs:
+        parsed = _parse_specs(specs, section, seen)
+        for frozenid, pyfile, modname, ispkg, section in parsed:
+            try:
+                source = seen[frozenid]
+            except KeyError:
+                source = FrozenSource.from_id(frozenid, pyfile, destdir)
+                seen[frozenid] = source
+            else:
+                assert not pyfile
+            yield FrozenModule(modname, ispkg, section, source)
+
+
+def _parse_specs(specs, section, seen):
+    for spec in specs:
+        info, subs = _parse_spec(spec, seen, section)
+        yield info
+        for info in subs or ():
+            yield info
+
+
+def _parse_spec(spec, knownids=None, section=None):
+    """Yield an info tuple for each module corresponding to the given spec.
+
+    The info consists of: (frozenid, pyfile, modname, ispkg, section).
 
     Supported formats:
 
@@ -74,7 +112,7 @@ def parse_frozen_spec(rawspec, knownids=None, section=None):
     Also, if "modname" has brackets then "frozenid" should not,
     and "pyfile" should have been provided..
     """
-    frozenid, _, remainder = rawspec.partition(':')
+    frozenid, _, remainder = spec.partition(':')
     modname, _, pyfile = remainder.partition('=')
     frozenid = frozenid.strip()
     modname = modname.strip()
@@ -82,28 +120,28 @@ def parse_frozen_spec(rawspec, knownids=None, section=None):
 
     submodules = None
     if modname.startswith('<') and modname.endswith('>'):
-        assert check_modname(frozenid), rawspec
+        assert check_modname(frozenid), spec
         modname = modname[1:-1]
-        assert check_modname(modname), rawspec
+        assert check_modname(modname), spec
         if frozenid in knownids:
             pass
         elif pyfile:
-            assert not os.path.isdir(pyfile), rawspec
+            assert not os.path.isdir(pyfile), spec
         else:
             pyfile = _resolve_module(frozenid, ispkg=False)
         ispkg = True
     elif pyfile:
-        assert check_modname(frozenid), rawspec
+        assert check_modname(frozenid), spec
-        assert not knownids or frozenid not in knownids, rawspec
+        assert not knownids or frozenid not in knownids, spec
-        assert check_modname(modname), rawspec
+        assert check_modname(modname), spec
-        assert not os.path.isdir(pyfile), rawspec
+        assert not os.path.isdir(pyfile), spec
         ispkg = False
     elif knownids and frozenid in knownids:
-        assert check_modname(frozenid), rawspec
+        assert check_modname(frozenid), spec
-        assert check_modname(modname), rawspec
+        assert check_modname(modname), spec
         ispkg = False
     else:
-        assert not modname or check_modname(modname), rawspec
+        assert not modname or check_modname(modname), spec
         resolved = iter(resolve_modules(frozenid))
         frozenid, pyfile, ispkg = next(resolved)
         if not modname:
@@ -113,7 +151,7 @@ def parse_frozen_spec(rawspec, knownids=None, section=None):
             pkgname = modname
             def iter_subs():
                 for frozenid, pyfile, ispkg in resolved:
-                    assert not knownids or frozenid not in knownids, (frozenid, rawspec)
+                    assert not knownids or frozenid not in knownids, (frozenid, spec)
                     if pkgname:
                         modname = frozenid.replace(pkgid, pkgname, 1)
                     else:
@@ -121,59 +159,104 @@ def parse_frozen_spec(rawspec, knownids=None, section=None):
                     yield frozenid, pyfile, modname, ispkg, section
             submodules = iter_subs()
 
-    spec = (frozenid, pyfile or None, modname, ispkg, section)
+    info = (frozenid, pyfile or None, modname, ispkg, section)
-    return spec, submodules
+    return info, submodules
 
 
-def parse_frozen_specs(rawspecs=FROZEN):
+#######################################
-    seen = set()
+# frozen source files
-    for section, _specs in rawspecs:
-        for spec in _parse_frozen_specs(_specs, section, seen):
-            frozenid = spec[0]
-            yield spec
-            seen.add(frozenid)
 
+class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile')):
 
-def _parse_frozen_specs(rawspecs, section, seen):
+    @classmethod
-    for rawspec in rawspecs:
+    def from_id(cls, frozenid, pyfile=None, destdir=MODULES_DIR):
-        spec, subs = parse_frozen_spec(rawspec, seen, section)
+        if not pyfile:
-        yield spec
+            pyfile = os.path.join(STDLIB_DIR, *frozenid.split('.')) + '.py'
-        for spec in subs or ():
+            #assert os.path.exists(pyfile), (frozenid, pyfile)
-            yield spec
-
-
-def resolve_frozen_file(spec, destdir=MODULES_DIR):
-    if isinstance(spec, str):
-        modname = spec
-    else:
-        _, frozenid, _, _, _= spec
-        modname = frozenid
-    # We use a consistent naming convention for all frozen modules.
-    return os.path.join(destdir, modname.replace('.', '_')) + '.h'
-
-
-def resolve_frozen_files(specs, destdir=MODULES_DIR):
-    frozen = {}
-    frozenids = []
-    lastsection = None
-    for spec in specs:
-        frozenid, pyfile, *_, section = spec
-        if frozenid in frozen:
-            if section is None:
-                lastsection = None
-            else:
-                assert section == lastsection
-            continue
-        lastsection = section
         frozenfile = resolve_frozen_file(frozenid, destdir)
-        frozen[frozenid] = (pyfile, frozenfile)
+        return cls(frozenid, pyfile, frozenfile)
-        frozenids.append(frozenid)
+
-    return frozen, frozenids
+    @property
+    def frozenid(self):
+        return self.id
+
+    @property
+    def modname(self):
+        if self.pyfile.startswith(STDLIB_DIR):
+            return self.id
+        return None
+
+    @property
+    def symbol(self):
+        # This matches what we do in Programs/_freeze_module.c:
+        name = self.frozenid.replace('.', '_')
+        return '_Py_M__' + name
+
+
+def resolve_frozen_file(frozenid, destdir=MODULES_DIR):
+    """Return the filename corresponding to the given frozen ID.
+
+    For stdlib modules the ID will always be the full name
+    of the source module.
+    """
+    if not isinstance(frozenid, str):
+        try:
+            frozenid = frozenid.frozenid
+        except AttributeError:
+            raise ValueError(f'unsupported frozenid {frozenid!r}')
+    # We use a consistent naming convention for all frozen modules.
+    frozenfile = frozenid.replace('.', '_') + '.h'
+    if not destdir:
+        return frozenfile
+    return os.path.join(destdir, frozenfile)
+
+
+#######################################
+# frozen modules
+
+class FrozenModule(namedtuple('FrozenModule', 'name ispkg section source')):
+
+    def __getattr__(self, name):
+        return getattr(self.source, name)
+
+    @property
+    def modname(self):
+        return self.name
+
+    def summarize(self):
+        source = self.source.modname
+        if source:
+            source = f'<{source}>'
+        else:
+            source = os.path.relpath(self.pyfile, ROOT_DIR)
+        return {
+            'module': self.name,
+            'ispkg': self.ispkg,
+            'source': source,
+            'frozen': os.path.basename(self.frozenfile),
+            'checksum': _get_checksum(self.frozenfile),
+        }
+
+
+def _iter_sources(modules):
+    seen = set()
+    for mod in modules:
+        if mod.source not in seen:
+            yield mod.source
+            seen.add(mod.source)
 
 
 #######################################
 # generic helpers
 
+def _get_checksum(filename):
+    with open(filename) as infile:
+        text = infile.read()
+    m = hashlib.sha256()
+    m.update(text.encode('utf8'))
+    return m.hexdigest()
+
+
 def resolve_modules(modname, pyfile=None):
     if modname.startswith('<') and modname.endswith('>'):
         if pyfile:
@@ -293,38 +376,68 @@ def replace_block(lines, start_marker, end_marker, replacements, file):
     return lines[:start_pos + 1] + replacements + lines[end_pos:]
 
 
-def regen_frozen(specs, dest=MODULES_DIR):
+def regen_manifest(modules):
-    if isinstance(dest, str):
+    header = 'module ispkg source frozen checksum'.split()
-        frozen, frozenids = resolve_frozen_files(specs, destdir)
+    widths = [5] * len(header)
-    else:
+    rows = []
-        frozenids, frozen = dest
+    for mod in modules:
+        info = mod.summarize()
+        row = []
+        for i, col in enumerate(header):
+            value = info[col]
+            if col == 'checksum':
+                value = value[:12]
+            elif col == 'ispkg':
+                value = 'YES' if value else 'no'
+            widths[i] = max(widths[i], len(value))
+            row.append(value or '-')
+        rows.append(row)
 
+    modlines = [
+        '# The list of frozen modules with key information.',
+        '# Note that the "check_generated_files" CI job will identify',
+        '# when source files were changed but regen-frozen wasn\'t run.',
+        '# This file is auto-generated by Tools/scripts/freeze_modules.py.',
+        ' '.join(c.center(w) for c, w in zip(header, widths)).rstrip(),
+        ' '.join('-' * w for w in widths),
+    ]
+    for row in rows:
+        for i, w in enumerate(widths):
+            if header[i] == 'ispkg':
+                row[i] = row[i].center(w)
+            else:
+                row[i] = row[i].ljust(w)
+        modlines.append(' '.join(row).rstrip())
+
+    print(f'# Updating {os.path.relpath(MANIFEST)}')
+    with open(MANIFEST, 'w') as outfile:
+        lines = (l + '\n' for l in modlines)
+        outfile.writelines(lines)
+
+
+def regen_frozen(modules):
     headerlines = []
     parentdir = os.path.dirname(FROZEN_FILE)
-    for frozenid in frozenids:
+    for src in _iter_sources(modules):
         # Adding a comment to separate sections here doesn't add much,
         # so we don't.
-        _, frozenfile = frozen[frozenid]
+        header = os.path.relpath(src.frozenfile, parentdir)
-        header = os.path.relpath(frozenfile, parentdir)
         headerlines.append(f'#include "{header}"')
 
     deflines = []
     indent = '    '
     lastsection = None
-    for spec in specs:
+    for mod in modules:
-        frozenid, _, modname, ispkg, section = spec
+        if mod.section != lastsection:
-        if section != lastsection:
             if lastsection is not None:
                 deflines.append('')
-            deflines.append(f'/* {section} */')
+            deflines.append(f'/* {mod.section} */')
-        lastsection = section
+        lastsection = mod.section
 
-        # This matches what we do in Programs/_freeze_module.c:
+        symbol = mod.symbol
-        name = frozenid.replace('.', '_')
+        pkg = '-' if mod.ispkg else ''
-        symbol = '_Py_M__' + name
-        pkg = '-' if ispkg else ''
         line = ('{"%s", %s, %s(int)sizeof(%s)},'
-                % (modname, symbol, pkg, symbol))
+                ) % (mod.name, symbol, pkg, symbol)
         # TODO: Consider not folding lines
         if len(line) < 80:
             deflines.append(line)
@@ -361,22 +474,20 @@ def regen_frozen(specs, dest=MODULES_DIR):
         outfile.writelines(lines)
 
 
-def regen_makefile(frozenids, frozen):
+def regen_makefile(modules):
     frozenfiles = []
     rules = ['']
-    for frozenid in frozenids:
+    for src in _iter_sources(modules):
-        pyfile, frozenfile = frozen[frozenid]
+        header = os.path.relpath(src.frozenfile, ROOT_DIR)
-        header = os.path.relpath(frozenfile, ROOT_DIR)
         relfile = header.replace('\\', '/')
         frozenfiles.append(f'\t\t$(srcdir)/{relfile} \\')
 
-        _pyfile = os.path.relpath(pyfile, ROOT_DIR)
+        pyfile = os.path.relpath(src.pyfile, ROOT_DIR)
-        tmpfile = f'{header}.new'
         # Note that we freeze the module to the target .h file
         # instead of going through an intermediate file like we used to.
-        rules.append(f'{header}: $(srcdir)/Programs/_freeze_module $(srcdir)/{_pyfile}')
+        rules.append(f'{header}: Programs/_freeze_module {pyfile}')
-        rules.append(f'\t$(srcdir)/Programs/_freeze_module {frozenid} \\')
+        rules.append(f'\t$(srcdir)/Programs/_freeze_module {src.frozenid} \\')
-        rules.append(f'\t\t$(srcdir)/{_pyfile} \\')
+        rules.append(f'\t\t$(srcdir)/{pyfile} \\')
         rules.append(f'\t\t$(srcdir)/{header}')
         rules.append('')
 
@@ -402,22 +513,24 @@ def regen_makefile(frozenids, frozen):
         outfile.writelines(lines)
 
 
-def regen_pcbuild(frozenids, frozen):
+def regen_pcbuild(modules):
     projlines = []
     filterlines = []
-    for frozenid in frozenids:
+    for src in _iter_sources(modules):
-        pyfile, frozenfile = frozen[frozenid]
+        # For now we only require the essential frozen modules on Windows.
-
+        # See bpo-45186 and bpo-45188.
-        _pyfile = os.path.relpath(pyfile, ROOT_DIR).replace('/', '\\')
+        if src.id not in ESSENTIAL and src.id != 'hello':
-        header = os.path.relpath(frozenfile, ROOT_DIR).replace('/', '\\')
+            continue
+        pyfile = os.path.relpath(src.pyfile, ROOT_DIR).replace('/', '\\')
+        header = os.path.relpath(src.frozenfile, ROOT_DIR).replace('/', '\\')
         intfile = header.split('\\')[-1].strip('.h') + '.g.h'
-        projlines.append(f'    <None Include="..\\{_pyfile}">')
+        projlines.append(f'    <None Include="..\\{pyfile}">')
-        projlines.append(f'      <ModName>{frozenid}</ModName>')
+        projlines.append(f'      <ModName>{src.frozenid}</ModName>')
         projlines.append(f'      <IntFile>$(IntDir){intfile}</IntFile>')
         projlines.append(f'      <OutFile>$(PySourcePath){header}</OutFile>')
         projlines.append(f'    </None>')
 
-        filterlines.append(f'    <None Include="..\\{_pyfile}">')
+        filterlines.append(f'    <None Include="..\\{pyfile}">')
         filterlines.append('      <Filter>Python Files</Filter>')
         filterlines.append('    </None>')
 
@@ -451,7 +564,7 @@ def regen_pcbuild(frozenids, frozen):
 def freeze_module(modname, pyfile=None, destdir=MODULES_DIR):
     """Generate the frozen module .h file for the given module."""
     for modname, pyfile, ispkg in resolve_modules(modname, pyfile):
-        frozenfile = _resolve_frozen(modname, destdir)
+        frozenfile = resolve_frozen_file(modname, destdir)
         _freeze_module(modname, pyfile, frozenfile)
 
 
@@ -459,7 +572,7 @@ def _freeze_module(frozenid, pyfile, frozenfile):
     tmpfile = frozenfile + '.new'
 
     argv = [TOOL, frozenid, pyfile, tmpfile]
-    print('#', '  '.join(os.path.relpath(a) for a in argv))
+    print('#', '  '.join(os.path.relpath(a) for a in argv), flush=True)
     try:
         subprocess.run(argv, check=True)
     except subprocess.CalledProcessError:
@@ -475,18 +588,17 @@ def _freeze_module(frozenid, pyfile, frozenfile):
 
 def main():
     # Expand the raw specs, preserving order.
-    specs = list(parse_frozen_specs())
+    modules = list(parse_frozen_specs(destdir=MODULES_DIR))
-    frozen, frozenids = resolve_frozen_files(specs, MODULES_DIR)
-
-    # Regen build-related files.
-    regen_frozen(specs, (frozenids, frozen))
-    regen_makefile(frozenids, frozen)
-    regen_pcbuild(frozenids, frozen)
 
     # Freeze the target modules.
-    for frozenid in frozenids:
+    for src in _iter_sources(modules):
-        pyfile, frozenfile = frozen[frozenid]
+        _freeze_module(src.frozenid, src.pyfile, src.frozenfile)
-        _freeze_module(frozenid, pyfile, frozenfile)
+
+    # Regen build-related files.
+    regen_manifest(modules)
+    regen_frozen(modules)
+    regen_makefile(modules)
+    regen_pcbuild(modules)
 
 
 if __name__ == '__main__':

Tools/scripts/generate_stdlib_module_names.py

@@ -117,9 +117,19 @@ def list_frozen(names):
         cmd = ' '.join(args)
         print(f"{cmd} failed with exitcode {exitcode}")
         sys.exit(exitcode)
+    submodules = set()
     for line in proc.stdout.splitlines():
         name = line.strip()
-        names.add(name)
+        if '.' in name:
+            submodules.add(name)
+        else:
+            names.add(name)
+    # Make sure all frozen submodules have a known parent.
+    for name in list(submodules):
+        if name.partition('.')[0] in names:
+            submodules.remove(name)
+    if submodules:
+        raise Exception(f'unexpected frozen submodules: {sorted(submodules)}')
 
 
 def list_modules():