gh-108216: Cleanup #include in internal header files (#108228)

* Add missing includes.
* Remove unused includes.
* Update old include/symbol names to newer names.
* Mention at least one included symbol.
* Sort includes.
* Update Tools/cases_generator/generate_cases.py used to generated
  pycore_opcode_metadata.h.
* Update Parser/asdl_c.py used to generate pycore_ast.h.
* Cleanup also includes in _testcapimodule.c and _testinternalcapi.c.

Include/internal/pycore_ast.h

@@ -10,7 +10,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_asdl.h"
+#include "pycore_asdl.h"          // _ASDL_SEQ_HEAD
 
 typedef struct _mod *mod_ty;
 

Include/internal/pycore_atomic.h

@@ -8,18 +8,18 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "dynamic_annotations.h"   /* _Py_ANNOTATE_MEMORY_ORDER */
+#include "pyconfig.h"             // HAVE_STD_ATOMIC
-#include "pyconfig.h"
+#include "dynamic_annotations.h"  // _Py_ANNOTATE_MEMORY_ORDER
 
 #ifdef HAVE_STD_ATOMIC
-#  include <stdatomic.h>
+#  include <stdatomic.h>          // atomic_store_explicit()
 #endif
 
 
 #if defined(_MSC_VER)
-#include <intrin.h>
+#  include <intrin.h>             // _InterlockedExchange64()
 #  if defined(_M_IX86) || defined(_M_X64)
-#  include <immintrin.h>
+#    include <immintrin.h>        // _InterlockedExchange_HLEAcquire()
 #  endif
 #endif
 

Include/internal/pycore_bitutils.h

@@ -26,10 +26,10 @@ extern "C" {
 #endif
 
 #ifdef _MSC_VER
-   /* Get _byteswap_ushort(), _byteswap_ulong(), _byteswap_uint64() */
+#  include <intrin.h>             // _byteswap_uint64()
-#  include <intrin.h>
 #endif
 
+
 static inline uint16_t
 _Py_bswap16(uint16_t word)
 {

Include/internal/pycore_ceval.h

@@ -8,6 +8,9 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_interp.h"        // PyInterpreterState.eval_frame
+#include "pycore_pystate.h"       // _PyThreadState_GET()
+
 /* Forward declarations */
 struct pyruntimestate;
 struct _ceval_runtime_state;
@@ -16,10 +19,6 @@ struct _ceval_runtime_state;
 #  define Py_DEFAULT_RECURSION_LIMIT 1000
 #endif
 
-#include "pycore_interp.h"        // PyInterpreterState.eval_frame
-#include "pycore_pystate.h"       // _PyThreadState_GET()
-
-
 extern void _Py_FinishPendingCalls(PyThreadState *tstate);
 extern void _PyEval_InitState(PyInterpreterState *, PyThread_type_lock);
 extern void _PyEval_FiniState(struct _ceval_state *ceval);

Include/internal/pycore_ceval_state.h

@@ -8,8 +8,6 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-
-#include "pycore_atomic.h"          /* _Py_atomic_address */
 #include "pycore_gil.h"             // struct _gil_runtime_state
 
 

Include/internal/pycore_condvar.h

@@ -10,7 +10,7 @@
    not present in unistd.h. But they still can be implemented as an external
    library (e.g. gnu pth in pthread emulation) */
 # ifdef HAVE_PTHREAD_H
-#  include <pthread.h> /* _POSIX_THREADS */
+#  include <pthread.h>            // _POSIX_THREADS
 # endif
 #endif
 
@@ -21,7 +21,7 @@
 #define Py_HAVE_CONDVAR
 
 #ifdef HAVE_PTHREAD_H
-#  include <pthread.h>
+#  include <pthread.h>            // pthread_mutex_t
 #endif
 
 #define PyMUTEX_T pthread_mutex_t
@@ -38,7 +38,7 @@
 
 /* include windows if it hasn't been done before */
 #define WIN32_LEAN_AND_MEAN
-#include <windows.h>
+#include <windows.h>              // CRITICAL_SECTION
 
 /* options */
 /* non-emulated condition variables are provided for those that want

Include/internal/pycore_context.h

@@ -5,7 +5,7 @@
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_hamt.h"   /* PyHamtObject */
+#include "pycore_hamt.h"          // PyHamtObject
 
 
 extern PyTypeObject _PyContextTokenMissing_Type;

Include/internal/pycore_dict.h

@@ -9,9 +9,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_dict_state.h"
+#include "pycore_object.h"        // PyDictOrValues
-#include "pycore_object.h"
-#include "pycore_runtime.h"         // _PyRuntime
 
 // Unsafe flavor of PyDict_GetItemWithError(): no error checking
 extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key);

Include/internal/pycore_faulthandler.h

@@ -9,7 +9,7 @@ extern "C" {
 #endif
 
 #ifdef HAVE_SIGACTION
-#  include <signal.h>
+#  include <signal.h>             // sigaction
 #endif
 
 

Include/internal/pycore_fileutils.h

@@ -8,7 +8,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include <locale.h>   /* struct lconv */
+#include <locale.h>               // struct lconv
+
 
 /* A routine to check if a file descriptor can be select()-ed. */
 #ifdef _MSC_VER
@@ -268,7 +269,7 @@ extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t
 // so provide our own implementations. Remove them in case they get added
 // to the Games API family
 #if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP)
-#include <winerror.h>
+#include <winerror.h>             // HRESULT
 
 extern HRESULT PathCchSkipRoot(const wchar_t *pszPath, const wchar_t **ppszRootEnd);
 #endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */

Include/internal/pycore_gil.h

@@ -8,8 +8,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_atomic.h"    /* _Py_atomic_address */
+#include "pycore_atomic.h"        // _Py_atomic_address
-#include "pycore_condvar.h"   /* PyCOND_T */
+#include "pycore_condvar.h"       // PyCOND_T
 
 #ifndef Py_HAVE_CONDVAR
 #  error You need either a POSIX-compatible or a Windows system!

Include/internal/pycore_global_objects.h

@@ -8,11 +8,11 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_hashtable.h"       // _Py_hashtable_t
+#include "pycore_context.h"         // _PyContextTokenMissing
-#include "pycore_gc.h"              // PyGC_Head
+#include "pycore_gc.h"              // _PyGC_Head_UNUSED
 #include "pycore_global_strings.h"  // struct _Py_global_strings
 #include "pycore_hamt.h"            // PyHamtNode_Bitmap
-#include "pycore_context.h"         // _PyContextTokenMissing
+#include "pycore_hashtable.h"       // _Py_hashtable_t
 #include "pycore_typeobject.h"      // pytype_slotdef
 
 

Include/internal/pycore_import.h

@@ -103,7 +103,7 @@ struct _import_state {
 };
 
 #ifdef HAVE_DLOPEN
-#  include <dlfcn.h>
+#  include <dlfcn.h>              // RTLD_NOW, RTLD_LAZY
 #  if HAVE_DECL_RTLD_NOW
 #    define _Py_DLOPEN_FLAGS RTLD_NOW
 #  else

Include/internal/pycore_instruments.h

@@ -6,9 +6,7 @@
 #endif
 
 #include "pycore_bitutils.h"      // _Py_popcount32
-#include "pycore_frame.h"
+#include "pycore_frame.h"         // _PyInterpreterFrame
-
-#include "cpython/code.h"
 
 #ifdef __cplusplus
 extern "C" {

Include/internal/pycore_interp.h

@@ -8,7 +8,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include <stdbool.h>
+#include <stdbool.h>              // bool
 
 #include "pycore_ast_state.h"     // struct ast_state
 #include "pycore_atexit.h"        // struct atexit_state
@@ -21,16 +21,16 @@ extern "C" {
 #include "pycore_exceptions.h"    // struct _Py_exc_state
 #include "pycore_floatobject.h"   // struct _Py_float_state
 #include "pycore_function.h"      // FUNC_MAX_WATCHERS
-#include "pycore_genobject.h"     // struct _Py_async_gen_state
 #include "pycore_gc.h"            // struct _gc_runtime_state
-#include "pycore_global_objects.h"  // struct _Py_interp_static_objects
+#include "pycore_genobject.h"     // struct _Py_async_gen_state
+#include "pycore_global_objects.h"// struct _Py_interp_cached_objects
 #include "pycore_import.h"        // struct _import_state
 #include "pycore_instruments.h"   // _PY_MONITORING_EVENTS
 #include "pycore_list.h"          // struct _Py_list_state
 #include "pycore_object_state.h"  // struct _py_object_state
-#include "pycore_obmalloc.h"      // struct obmalloc_state
+#include "pycore_obmalloc.h"      // struct _obmalloc_state
 #include "pycore_tuple.h"         // struct _Py_tuple_state
-#include "pycore_typeobject.h"    // struct type_cache
+#include "pycore_typeobject.h"    // struct types_state
 #include "pycore_unicodeobject.h" // struct _Py_unicode_state
 #include "pycore_warnings.h"      // struct _warnings_runtime_state
 

Include/internal/pycore_object.h

@@ -12,7 +12,6 @@ extern "C" {
 #include "pycore_gc.h"            // _PyObject_GC_IS_TRACKED()
 #include "pycore_interp.h"        // PyInterpreterState.gc
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
-#include "pycore_runtime.h"       // _PyRuntime
 
 /* Check if an object is consistent. For example, ensure that the reference
    counter is greater than or equal to 1, and ensure that ob_type is not NULL.

Include/internal/pycore_opcode_metadata.h

@@ -7,7 +7,7 @@
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include <stdbool.h>
+#include <stdbool.h>              // bool
 
 
 #define IS_PSEUDO_INSTR(OP)  ( \

Include/internal/pycore_opcode_utils.h

@@ -8,7 +8,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_opcode.h"        // _PyOpcode_Jump
+#include "pycore_opcode.h"        // JUMP_FORWARD
 
 
 #define MAX_REAL_OPCODE 254

Include/internal/pycore_optimizer.h

@@ -8,7 +8,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_uops.h"
+#include "pycore_uops.h"          // _PyUOpInstruction
 
 int _Py_uop_analyze_and_optimize(PyCodeObject *code,
     _PyUOpInstruction *trace, int trace_len, int curr_stackentries);

Include/internal/pycore_pymem_init.h

@@ -8,8 +8,6 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_pymem.h"
-
 
 /********************************/
 /* the allocators' initializers */

Include/internal/pycore_pystate.h

@@ -8,7 +8,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_runtime.h"   /* PyRuntimeState */
+#include "pycore_runtime.h"       // _PyRuntime
 
 
 /* Check if the current thread is the main thread.

Include/internal/pycore_pythread.h

@@ -14,7 +14,7 @@ extern "C" {
    not present in unistd.h. But they still can be implemented as an external
    library (e.g. gnu pth in pthread emulation) */
 #  ifdef HAVE_PTHREAD_H
-#  include <pthread.h> /* _POSIX_THREADS */
+#    include <pthread.h>            // _POSIX_THREADS
 #  endif
 # ifndef _POSIX_THREADS
 /* Check if we're running on HP-UX and _SC_THREADS is defined. If so, then

Include/internal/pycore_runtime.h

@@ -8,24 +8,23 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_atexit.h"          // struct atexit_runtime_state
+#include "pycore_atexit.h"          // struct _atexit_runtime_state
-#include "pycore_atomic.h"          /* _Py_atomic_address */
+#include "pycore_atomic.h"          // _Py_atomic_address
 #include "pycore_ceval_state.h"     // struct _ceval_runtime_state
-#include "pycore_floatobject.h"     // struct _Py_float_runtime_state
 #include "pycore_faulthandler.h"    // struct _faulthandler_runtime_state
-#include "pycore_global_objects.h"  // struct _Py_global_objects
+#include "pycore_floatobject.h"     // struct _Py_float_runtime_state
 #include "pycore_import.h"          // struct _import_runtime_state
 #include "pycore_interp.h"          // PyInterpreterState
 #include "pycore_object_state.h"    // struct _py_object_runtime_state
 #include "pycore_parser.h"          // struct _parser_runtime_state
-#include "pycore_pymem.h"           // struct _pymem_allocators
 #include "pycore_pyhash.h"          // struct pyhash_runtime_state
+#include "pycore_pymem.h"           // struct _pymem_allocators
 #include "pycore_pythread.h"        // struct _pythread_runtime_state
 #include "pycore_signal.h"          // struct _signals_runtime_state
 #include "pycore_time.h"            // struct _time_runtime_state
 #include "pycore_tracemalloc.h"     // struct _tracemalloc_runtime_state
-#include "pycore_typeobject.h"      // struct types_runtime_state
+#include "pycore_typeobject.h"      // struct _types_runtime_state
-#include "pycore_unicodeobject.h"   // struct _Py_unicode_runtime_ids
+#include "pycore_unicodeobject.h"   // struct _Py_unicode_runtime_state
 
 struct _getargs_runtime_state {
     PyThread_type_lock mutex;

Include/internal/pycore_runtime_init.h

@@ -8,11 +8,17 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_long.h"
+#include "pycore_ceval_state.h"   // _PyEval_RUNTIME_PERF_INIT
-#include "pycore_object.h"
+#include "pycore_faulthandler.h"  // _faulthandler_runtime_state_INIT
-#include "pycore_parser.h"
+#include "pycore_floatobject.h"   // _py_float_format_unknown
-#include "pycore_pymem_init.h"
+#include "pycore_object.h"        // _PyObject_HEAD_INIT
-#include "pycore_obmalloc_init.h"
+#include "pycore_obmalloc_init.h" // _obmalloc_global_state_INIT
+#include "pycore_parser.h"        // _parser_runtime_state_INIT
+#include "pycore_pyhash.h"        // pyhash_state_INIT
+#include "pycore_pymem_init.h"    // _pymem_allocators_standard_INIT
+#include "pycore_runtime_init_generated.h"  // _Py_bytes_characters_INIT
+#include "pycore_signal.h"        // _signals_RUNTIME_INIT
+#include "pycore_tracemalloc.h"   // _tracemalloc_runtime_state_INIT
 
 
 extern PyTypeObject _PyExc_MemoryError;

Include/internal/pycore_runtime_init_generated.h

@@ -8,6 +8,9 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_long.h"          // _PyLong_DIGIT_INIT()
+
+
 /* The following is auto-generated by Tools/build/generate_global_objects.py. */
 #define _Py_small_ints_INIT { \
     _PyLong_DIGIT_INIT(-5), \

Include/internal/pycore_typeobject.h

@@ -4,12 +4,12 @@
 extern "C" {
 #endif
 
-#include "pycore_moduleobject.h"
-
 #ifndef Py_BUILD_CORE
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_moduleobject.h"  // PyModuleObject
+
 
 /* state */
 

Modules/_json.c

@@ -12,7 +12,7 @@
 #include "pycore_ceval.h"           // _Py_EnterRecursiveCall()
 #include "pycore_runtime.h"         // _PyRuntime
 
-#include "pycore_global_objects.h"  // _Py_ID()
+#include "pycore_global_strings.h"  // _Py_ID()
 #include <stdbool.h>                // bool
 
 

Modules/_testcapimodule.c

@@ -18,8 +18,8 @@
 #undef NDEBUG
 
 #include "Python.h"
-#include "frameobject.h"          // PyFrame_New
+#include "frameobject.h"          // PyFrame_New()
-#include "marshal.h"              // PyMarshal_WriteLongToFile
+#include "marshal.h"              // PyMarshal_WriteLongToFile()
 
 #include <float.h>                // FLT_MAX
 #include <signal.h>
@@ -37,7 +37,7 @@
 #endif
 
 #ifdef bool
-#  error "The public headers should not include <stdbool.h>, see bpo-46748"
+#  error "The public headers should not include <stdbool.h>, see gh-48924"
 #endif
 
 // Several parts of this module are broken out into files in _testcapi/.

Modules/_testinternalcapi.c

@@ -15,7 +15,7 @@
 #include "pycore_bytesobject.h"   // _PyBytes_Find()
 #include "pycore_compile.h"       // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble, _PyCompile_CleanDoc
 #include "pycore_ceval.h"         // _PyEval_AddPendingCall
-#include "pycore_dict.h"        // _PyDictOrValues_GetValues
+#include "pycore_dict.h"          // _PyDictOrValues_GetValues()
 #include "pycore_fileutils.h"     // _Py_normpath
 #include "pycore_frame.h"         // _PyInterpreterFrame
 #include "pycore_gc.h"            // PyGC_Head
@@ -27,9 +27,7 @@
 #include "pycore_pyerrors.h"      // _Py_UTF8_Edit_Cost()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 
-#include "frameobject.h"
 #include "interpreteridobject.h"  // PyInterpreterID_LookUp()
-#include "osdefs.h"              // MAXPATHLEN
 
 #include "clinic/_testinternalcapi.c.h"
 

Parser/asdl_c.py

@@ -1585,7 +1585,7 @@ def write_header(mod, metadata, f):
         #  error "this header requires Py_BUILD_CORE define"
         #endif
 
-        #include "pycore_asdl.h"
+        #include "pycore_asdl.h"          // _ASDL_SEQ_HEAD
 
     """).lstrip())
 

Tools/cases_generator/generate_cases.py

@@ -410,7 +410,7 @@ class Generator(Analyzer):
                 #endif
             """).strip())
 
-            self.out.emit("\n#include <stdbool.h>")
+            self.out.emit("\n#include <stdbool.h>              // bool")
 
             self.write_pseudo_instrs()