gh-90868: Adjust the Generated Objects (gh-99223)

We do the following:

* move the generated _PyUnicode_InitStaticStrings() to its own file
* move the generated _PyStaticObjects_CheckRefcnt() to its own file
* include pycore_global_objects.h in extension modules instead of pycore_runtime_init.h

These changes help us avoid including things that aren't needed.

https://github.com/python/cpython/issues/90868

.gitattributes

@@ -69,7 +69,7 @@ Doc/library/token-list.inc                          generated
 Include/internal/pycore_ast.h                       generated
 Include/internal/pycore_ast_state.h                 generated
 Include/internal/pycore_opcode.h                    generated
-Include/internal/pycore_runtime_init_generated.h    generated
+Include/internal/pycore_*_generated.h               generated
 Include/opcode.h                                    generated
 Include/token.h                                     generated
 Lib/keyword.py                                      generated

Makefile.pre.in

@@ -1638,6 +1638,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_getopt.h \
 		$(srcdir)/Include/internal/pycore_gil.h \
 		$(srcdir)/Include/internal/pycore_global_objects.h \
+		$(srcdir)/Include/internal/pycore_global_objects_fini_generated.h \
 		$(srcdir)/Include/internal/pycore_hamt.h \
 		$(srcdir)/Include/internal/pycore_hashtable.h \
 		$(srcdir)/Include/internal/pycore_import.h \
@@ -1673,6 +1674,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_ucnhash.h \
 		$(srcdir)/Include/internal/pycore_unionobject.h \
 		$(srcdir)/Include/internal/pycore_unicodeobject.h \
+		$(srcdir)/Include/internal/pycore_unicodeobject_generated.h \
 		$(srcdir)/Include/internal/pycore_warnings.h \
 		$(DTRACE_HEADERS) \
 		@PLATFORM_HEADERS@ \

Modules/_asynciomodule.c

@@ -3,10 +3,11 @@
 #endif
 
 #include "Python.h"
-#include "pycore_pyerrors.h"      // _PyErr_ClearExcState()
+#include "pycore_pyerrors.h"        // _PyErr_ClearExcState()
-#include "pycore_pystate.h"       // _PyThreadState_GET()
+#include "pycore_pystate.h"         // _PyThreadState_GET()
-#include "pycore_runtime_init.h"  // _Py_ID()
+#include "pycore_runtime.h"         // _PyRuntime
-#include <stddef.h>               // offsetof()
+#include "pycore_global_objects.h"  // _Py_ID()
+#include <stddef.h>                 // offsetof()
 
 
 /*[clinic input]

Modules/_ctypes/callbacks.c

@@ -8,8 +8,9 @@
 #  include <windows.h>
 #endif
 
-#include "pycore_call.h"          // _PyObject_CallNoArgs()
+#include "pycore_call.h"            // _PyObject_CallNoArgs()
-#include "pycore_runtime_init.h"  // _Py_ID()
+#include "pycore_runtime.h"         // _PyRuntime
+#include "pycore_global_objects.h"  // _Py_ID()
 
 #include <stdbool.h>
 

Modules/_ctypes/callproc.c

@@ -96,7 +96,8 @@
 #define DONT_USE_SEH
 #endif
 
-#include "pycore_runtime_init.h"
+#include "pycore_runtime.h"         // _PyRuntime
+#include "pycore_global_objects.h"  // _Py_ID()
 
 #define CTYPES_CAPSULE_NAME_PYMEM "_ctypes pymem"
 

Modules/_json.c

@@ -9,10 +9,11 @@
 #endif
 
 #include "Python.h"
-#include "pycore_ceval.h"         // _Py_EnterRecursiveCall()
+#include "pycore_ceval.h"           // _Py_EnterRecursiveCall()
-#include "structmember.h"         // PyMemberDef
+#include "pycore_runtime.h"         // _PyRuntime
-#include "pycore_runtime_init.h"  // _Py_ID()
+#include "structmember.h"           // PyMemberDef
-#include <stdbool.h>              // bool
+#include "pycore_global_objects.h"  // _Py_ID()
+#include <stdbool.h>                // bool
 
 
 typedef struct _PyScannerObject {

Objects/unicodeobject.c

@@ -52,9 +52,9 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 #include "pycore_pathconfig.h"    // _Py_DumpPathConfig()
 #include "pycore_pylifecycle.h"   // _Py_SetFileSystemEncoding()
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
-#include "pycore_runtime_init.h"  // _PyUnicode_InitStaticStrings()
 #include "pycore_ucnhash.h"       // _PyUnicode_Name_CAPI
 #include "pycore_unicodeobject.h" // struct _Py_unicode_state
+#include "pycore_unicodeobject_generated.h"  // _PyUnicode_InitStaticStrings()
 #include "stringlib/eq.h"         // unicode_eq()
 
 #ifdef MS_WINDOWS

PCbuild/pythoncore.vcxproj

@@ -219,6 +219,7 @@
     <ClInclude Include="..\Include\internal\pycore_getopt.h" />
     <ClInclude Include="..\Include\internal\pycore_gil.h" />
     <ClInclude Include="..\Include\internal\pycore_global_objects.h" />
+    <ClInclude Include="..\Include\internal\pycore_global_objects_fini_generated.h" />
     <ClInclude Include="..\Include\internal\pycore_hamt.h" />
     <ClInclude Include="..\Include\internal\pycore_hashtable.h" />
     <ClInclude Include="..\Include\internal\pycore_import.h" />
@@ -254,6 +255,7 @@
     <ClInclude Include="..\Include\internal\pycore_ucnhash.h" />
     <ClInclude Include="..\Include\internal\pycore_unionobject.h" />
     <ClInclude Include="..\Include\internal\pycore_unicodeobject.h" />
+    <ClInclude Include="..\Include\internal\pycore_unicodeobject_generated.h" />
     <ClInclude Include="..\Include\internal\pycore_warnings.h" />
     <ClInclude Include="..\Include\intrcheck.h" />
     <ClInclude Include="..\Include\iterobject.h" />

PCbuild/pythoncore.vcxproj.filters

@@ -477,6 +477,9 @@
     <ClInclude Include="..\Include\internal\pycore_unicodeobject.h">
       <Filter>Include\internal</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\internal\pycore_unicodeobject_generated.h">
+      <Filter>Include\internal</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\internal\pycore_warnings.h">
       <Filter>Include\internal</Filter>
     </ClInclude>
@@ -558,6 +561,9 @@
     <ClInclude Include="..\Include\internal\pycore_global_objects.h">
       <Filter>Include\internal</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\internal\pycore_global_objects_fini_generated.h">
+      <Filter>Include\internal</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\internal\pycore_hamt.h">
       <Filter>Include\internal</Filter>
     </ClInclude>

Python/pylifecycle.c

@@ -10,6 +10,7 @@
 #include "pycore_fileutils.h"     // _Py_ResetForceASCII()
 #include "pycore_floatobject.h"   // _PyFloat_InitTypes()
 #include "pycore_genobject.h"     // _PyAsyncGen_Fini()
+#include "pycore_global_objects_fini_generated.h"  // "_PyStaticObjects_CheckRefcnt()
 #include "pycore_import.h"        // _PyImport_BootstrapImp()
 #include "pycore_initconfig.h"    // _PyStatus_OK()
 #include "pycore_list.h"          // _PyList_Fini()

Python/suggestions.c

@@ -1,6 +1,7 @@
 #include "Python.h"
 #include "pycore_frame.h"
-#include "pycore_runtime_init.h"  // _Py_ID()
+#include "pycore_runtime.h"         // _PyRuntime
+#include "pycore_global_objects.h"  // _Py_ID()
 
 #include "pycore_pyerrors.h"
 #include "pycore_code.h"        // _PyCode_GetVarnames()

Tools/build/generate_global_objects.py

@@ -330,7 +330,29 @@ def generate_runtime_init(identifiers, strings):
                 with printer.block('.tuple_empty =', ','):
                     printer.write('.ob_base = _PyVarObject_IMMORTAL_INIT(&PyTuple_Type, 0)')
                     immortal_objects.append(f'(PyObject *)&_Py_SINGLETON(tuple_empty)')
-        printer.write('')
+        printer.write(END)
+        printer.write(after)
+        return immortal_objects
+
+
+def generate_static_strings_initializer(identifiers, strings):
+    # Target the runtime initializer.
+    filename = os.path.join(INTERNAL, 'pycore_unicodeobject_generated.h')
+
+    # Read the non-generated part of the file.
+    with open(filename) as infile:
+        orig = infile.read()
+    lines = iter(orig.rstrip().splitlines())
+    before = '\n'.join(iter_to_marker(lines, START))
+    for _ in iter_to_marker(lines, END):
+        pass
+    after = '\n'.join(lines)
+
+    # Generate the file.
+    with open_for_changes(filename, orig) as outfile:
+        printer = Printer(outfile)
+        printer.write(before)
+        printer.write(START)
         printer.write("static inline void")
         with printer.block("_PyUnicode_InitStaticStrings(void)"):
             printer.write(f'PyObject *string;')
@@ -339,7 +361,29 @@ def generate_runtime_init(identifiers, strings):
                 # since iter_files() ignores .h files.
                 printer.write(f'string = &_Py_ID({i});')
                 printer.write(f'PyUnicode_InternInPlace(&string);')
-        printer.write('')
+            # XXX What about "strings"?
+        printer.write(END)
+        printer.write(after)
+
+
+def generate_global_object_finalizers(immortal_objects):
+    # Target the runtime initializer.
+    filename = os.path.join(INTERNAL, 'pycore_global_objects_fini_generated.h')
+
+    # Read the non-generated part of the file.
+    with open(filename) as infile:
+        orig = infile.read()
+    lines = iter(orig.rstrip().splitlines())
+    before = '\n'.join(iter_to_marker(lines, START))
+    for _ in iter_to_marker(lines, END):
+        pass
+    after = '\n'.join(lines)
+
+    # Generate the file.
+    with open_for_changes(filename, orig) as outfile:
+        printer = Printer(outfile)
+        printer.write(before)
+        printer.write(START)
         printer.write('#ifdef Py_DEBUG')
         printer.write("static inline void")
         with printer.block("_PyStaticObjects_CheckRefcnt(void)"):
@@ -375,7 +419,9 @@ def main() -> None:
     identifiers, strings = get_identifiers_and_strings()
 
     generate_global_strings(identifiers, strings)
-    generate_runtime_init(identifiers, strings)
+    immortal_objects = generate_runtime_init(identifiers, strings)
+    generate_static_strings_initializer(identifiers, strings)
+    generate_global_object_finalizers(immortal_objects)
 
 
 if __name__ == '__main__':