gh-108216: Cleanup #include in internal header files (#108228)

* Add missing includes.
* Remove unused includes.
* Update old include/symbol names to newer names.
* Mention at least one included symbol.
* Sort includes.
* Update Tools/cases_generator/generate_cases.py used to generated
  pycore_opcode_metadata.h.
* Update Parser/asdl_c.py used to generate pycore_ast.h.
* Cleanup also includes in _testcapimodule.c and _testinternalcapi.c.

Include/internal/pycore_ast.h

@@ -10,7 +10,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_asdl.h"
+#include "pycore_asdl.h"          // _ASDL_SEQ_HEAD
 
 typedef struct _mod *mod_ty;
 

Include/internal/pycore_atomic.h

@@ -8,19 +8,19 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "dynamic_annotations.h"   /* _Py_ANNOTATE_MEMORY_ORDER */
+#include "pyconfig.h"             // HAVE_STD_ATOMIC
-#include "pyconfig.h"
+#include "dynamic_annotations.h"  // _Py_ANNOTATE_MEMORY_ORDER
 
 #ifdef HAVE_STD_ATOMIC
-#  include <stdatomic.h>
+#  include <stdatomic.h>          // atomic_store_explicit()
 #endif
 
 
 #if defined(_MSC_VER)
-#include <intrin.h>
+#  include <intrin.h>             // _InterlockedExchange64()
-#if defined(_M_IX86) || defined(_M_X64)
+#  if defined(_M_IX86) || defined(_M_X64)
-#  include <immintrin.h>
+#    include <immintrin.h>        // _InterlockedExchange_HLEAcquire()
-#endif
+#  endif
 #endif
 
 /* This is modeled after the atomics interface from C1x, according to

Include/internal/pycore_bitutils.h

@@ -26,10 +26,10 @@ extern "C" {
 #endif
 
 #ifdef _MSC_VER
-   /* Get _byteswap_ushort(), _byteswap_ulong(), _byteswap_uint64() */
+#  include <intrin.h>             // _byteswap_uint64()
-#  include <intrin.h>
 #endif
 
+
 static inline uint16_t
 _Py_bswap16(uint16_t word)
 {

Include/internal/pycore_ceval.h

@@ -8,6 +8,9 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_interp.h"        // PyInterpreterState.eval_frame
+#include "pycore_pystate.h"       // _PyThreadState_GET()
+
 /* Forward declarations */
 struct pyruntimestate;
 struct _ceval_runtime_state;
@@ -16,10 +19,6 @@ struct _ceval_runtime_state;
 #  define Py_DEFAULT_RECURSION_LIMIT 1000
 #endif
 
-#include "pycore_interp.h"        // PyInterpreterState.eval_frame
-#include "pycore_pystate.h"       // _PyThreadState_GET()
-
-
 extern void _Py_FinishPendingCalls(PyThreadState *tstate);
 extern void _PyEval_InitState(PyInterpreterState *, PyThread_type_lock);
 extern void _PyEval_FiniState(struct _ceval_state *ceval);

Include/internal/pycore_ceval_state.h

@@ -8,8 +8,6 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-
-#include "pycore_atomic.h"          /* _Py_atomic_address */
 #include "pycore_gil.h"             // struct _gil_runtime_state
 
 

Include/internal/pycore_condvar.h

@@ -10,7 +10,7 @@
    not present in unistd.h. But they still can be implemented as an external
    library (e.g. gnu pth in pthread emulation) */
 # ifdef HAVE_PTHREAD_H
-#  include <pthread.h> /* _POSIX_THREADS */
+#  include <pthread.h>            // _POSIX_THREADS
 # endif
 #endif
 
@@ -21,7 +21,7 @@
 #define Py_HAVE_CONDVAR
 
 #ifdef HAVE_PTHREAD_H
-#  include <pthread.h>
+#  include <pthread.h>            // pthread_mutex_t
 #endif
 
 #define PyMUTEX_T pthread_mutex_t
@@ -38,7 +38,7 @@
 
 /* include windows if it hasn't been done before */
 #define WIN32_LEAN_AND_MEAN
-#include <windows.h>
+#include <windows.h>              // CRITICAL_SECTION
 
 /* options */
 /* non-emulated condition variables are provided for those that want

Include/internal/pycore_context.h

@@ -5,7 +5,7 @@
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_hamt.h"   /* PyHamtObject */
+#include "pycore_hamt.h"          // PyHamtObject
 
 
 extern PyTypeObject _PyContextTokenMissing_Type;

Include/internal/pycore_dict.h

@@ -9,9 +9,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_dict_state.h"
+#include "pycore_object.h"        // PyDictOrValues
-#include "pycore_object.h"
-#include "pycore_runtime.h"         // _PyRuntime
 
 // Unsafe flavor of PyDict_GetItemWithError(): no error checking
 extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key);

Include/internal/pycore_faulthandler.h

@@ -9,7 +9,7 @@ extern "C" {
 #endif
 
 #ifdef HAVE_SIGACTION
-#  include <signal.h>
+#  include <signal.h>             // sigaction
 #endif
 
 

Include/internal/pycore_fileutils.h

@@ -8,7 +8,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include <locale.h>   /* struct lconv */
+#include <locale.h>               // struct lconv
+
 
 /* A routine to check if a file descriptor can be select()-ed. */
 #ifdef _MSC_VER
@@ -268,7 +269,7 @@ extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t
 // so provide our own implementations. Remove them in case they get added
 // to the Games API family
 #if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP)
-#include <winerror.h>
+#include <winerror.h>             // HRESULT
 
 extern HRESULT PathCchSkipRoot(const wchar_t *pszPath, const wchar_t **ppszRootEnd);
 #endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */

Include/internal/pycore_gil.h

@@ -8,8 +8,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_atomic.h"    /* _Py_atomic_address */
+#include "pycore_atomic.h"        // _Py_atomic_address
-#include "pycore_condvar.h"   /* PyCOND_T */
+#include "pycore_condvar.h"       // PyCOND_T
 
 #ifndef Py_HAVE_CONDVAR
 #  error You need either a POSIX-compatible or a Windows system!

Include/internal/pycore_global_objects.h

@@ -8,11 +8,11 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_hashtable.h"       // _Py_hashtable_t
+#include "pycore_context.h"         // _PyContextTokenMissing
-#include "pycore_gc.h"              // PyGC_Head
+#include "pycore_gc.h"              // _PyGC_Head_UNUSED
 #include "pycore_global_strings.h"  // struct _Py_global_strings
 #include "pycore_hamt.h"            // PyHamtNode_Bitmap
-#include "pycore_context.h"         // _PyContextTokenMissing
+#include "pycore_hashtable.h"       // _Py_hashtable_t
 #include "pycore_typeobject.h"      // pytype_slotdef
 
 

Include/internal/pycore_import.h

@@ -103,7 +103,7 @@ struct _import_state {
 };
 
 #ifdef HAVE_DLOPEN
-#  include <dlfcn.h>
+#  include <dlfcn.h>              // RTLD_NOW, RTLD_LAZY
 #  if HAVE_DECL_RTLD_NOW
 #    define _Py_DLOPEN_FLAGS RTLD_NOW
 #  else

Include/internal/pycore_instruments.h

@@ -6,9 +6,7 @@
 #endif
 
 #include "pycore_bitutils.h"      // _Py_popcount32
-#include "pycore_frame.h"
+#include "pycore_frame.h"         // _PyInterpreterFrame
-
-#include "cpython/code.h"
 
 #ifdef __cplusplus
 extern "C" {

Include/internal/pycore_interp.h

@@ -8,7 +8,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include <stdbool.h>
+#include <stdbool.h>              // bool
 
 #include "pycore_ast_state.h"     // struct ast_state
 #include "pycore_atexit.h"        // struct atexit_state
@@ -21,16 +21,16 @@ extern "C" {
 #include "pycore_exceptions.h"    // struct _Py_exc_state
 #include "pycore_floatobject.h"   // struct _Py_float_state
 #include "pycore_function.h"      // FUNC_MAX_WATCHERS
-#include "pycore_genobject.h"     // struct _Py_async_gen_state
 #include "pycore_gc.h"            // struct _gc_runtime_state
-#include "pycore_global_objects.h"  // struct _Py_interp_static_objects
+#include "pycore_genobject.h"     // struct _Py_async_gen_state
+#include "pycore_global_objects.h"// struct _Py_interp_cached_objects
 #include "pycore_import.h"        // struct _import_state
 #include "pycore_instruments.h"   // _PY_MONITORING_EVENTS
 #include "pycore_list.h"          // struct _Py_list_state
-#include "pycore_object_state.h"   // struct _py_object_state
+#include "pycore_object_state.h"  // struct _py_object_state
-#include "pycore_obmalloc.h"      // struct obmalloc_state
+#include "pycore_obmalloc.h"      // struct _obmalloc_state
 #include "pycore_tuple.h"         // struct _Py_tuple_state
-#include "pycore_typeobject.h"    // struct type_cache
+#include "pycore_typeobject.h"    // struct types_state
 #include "pycore_unicodeobject.h" // struct _Py_unicode_state
 #include "pycore_warnings.h"      // struct _warnings_runtime_state
 

Include/internal/pycore_object.h

@@ -12,7 +12,6 @@ extern "C" {
 #include "pycore_gc.h"            // _PyObject_GC_IS_TRACKED()
 #include "pycore_interp.h"        // PyInterpreterState.gc
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
-#include "pycore_runtime.h"       // _PyRuntime
 
 /* Check if an object is consistent. For example, ensure that the reference
    counter is greater than or equal to 1, and ensure that ob_type is not NULL.

Include/internal/pycore_opcode_metadata.h

@@ -7,7 +7,7 @@
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include <stdbool.h>
+#include <stdbool.h>              // bool
 
 
 #define IS_PSEUDO_INSTR(OP)  ( \

Include/internal/pycore_opcode_utils.h

@@ -8,7 +8,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_opcode.h"        // _PyOpcode_Jump
+#include "pycore_opcode.h"        // JUMP_FORWARD
 
 
 #define MAX_REAL_OPCODE 254

Include/internal/pycore_optimizer.h

@@ -8,7 +8,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_uops.h"
+#include "pycore_uops.h"          // _PyUOpInstruction
 
 int _Py_uop_analyze_and_optimize(PyCodeObject *code,
     _PyUOpInstruction *trace, int trace_len, int curr_stackentries);

Include/internal/pycore_pymem_init.h

@@ -8,8 +8,6 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_pymem.h"
-
 
 /********************************/
 /* the allocators' initializers */

Include/internal/pycore_pystate.h

@@ -8,7 +8,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_runtime.h"   /* PyRuntimeState */
+#include "pycore_runtime.h"       // _PyRuntime
 
 
 /* Check if the current thread is the main thread.

Include/internal/pycore_pythread.h

@@ -13,9 +13,9 @@ extern "C" {
 /* This means pthreads are not implemented in libc headers, hence the macro
    not present in unistd.h. But they still can be implemented as an external
    library (e.g. gnu pth in pthread emulation) */
-# ifdef HAVE_PTHREAD_H
+#  ifdef HAVE_PTHREAD_H
-#  include <pthread.h> /* _POSIX_THREADS */
+#    include <pthread.h>            // _POSIX_THREADS
-# endif
+#  endif
 # ifndef _POSIX_THREADS
 /* Check if we're running on HP-UX and _SC_THREADS is defined. If so, then
    enough of the Posix threads package is implemented to support python
@@ -34,12 +34,12 @@ extern "C" {
 #endif /* _POSIX_THREADS */
 
 #if defined(_POSIX_THREADS) || defined(HAVE_PTHREAD_STUBS)
-# define _USE_PTHREADS
+#  define _USE_PTHREADS
 #endif
 
 #if defined(_USE_PTHREADS) && defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC)
 // monotonic is supported statically.  It doesn't mean it works on runtime.
-# define CONDATTR_MONOTONIC
+#  define CONDATTR_MONOTONIC
 #endif
 
 

Include/internal/pycore_runtime.h

@@ -8,24 +8,23 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_atexit.h"          // struct atexit_runtime_state
+#include "pycore_atexit.h"          // struct _atexit_runtime_state
-#include "pycore_atomic.h"          /* _Py_atomic_address */
+#include "pycore_atomic.h"          // _Py_atomic_address
 #include "pycore_ceval_state.h"     // struct _ceval_runtime_state
-#include "pycore_floatobject.h"     // struct _Py_float_runtime_state
 #include "pycore_faulthandler.h"    // struct _faulthandler_runtime_state
-#include "pycore_global_objects.h"  // struct _Py_global_objects
+#include "pycore_floatobject.h"     // struct _Py_float_runtime_state
 #include "pycore_import.h"          // struct _import_runtime_state
 #include "pycore_interp.h"          // PyInterpreterState
 #include "pycore_object_state.h"    // struct _py_object_runtime_state
 #include "pycore_parser.h"          // struct _parser_runtime_state
-#include "pycore_pymem.h"           // struct _pymem_allocators
 #include "pycore_pyhash.h"          // struct pyhash_runtime_state
+#include "pycore_pymem.h"           // struct _pymem_allocators
 #include "pycore_pythread.h"        // struct _pythread_runtime_state
 #include "pycore_signal.h"          // struct _signals_runtime_state
 #include "pycore_time.h"            // struct _time_runtime_state
 #include "pycore_tracemalloc.h"     // struct _tracemalloc_runtime_state
-#include "pycore_typeobject.h"      // struct types_runtime_state
+#include "pycore_typeobject.h"      // struct _types_runtime_state
-#include "pycore_unicodeobject.h"   // struct _Py_unicode_runtime_ids
+#include "pycore_unicodeobject.h"   // struct _Py_unicode_runtime_state
 
 struct _getargs_runtime_state {
     PyThread_type_lock mutex;

Include/internal/pycore_runtime_init.h

@@ -8,11 +8,17 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_long.h"
+#include "pycore_ceval_state.h"   // _PyEval_RUNTIME_PERF_INIT
-#include "pycore_object.h"
+#include "pycore_faulthandler.h"  // _faulthandler_runtime_state_INIT
-#include "pycore_parser.h"
+#include "pycore_floatobject.h"   // _py_float_format_unknown
-#include "pycore_pymem_init.h"
+#include "pycore_object.h"        // _PyObject_HEAD_INIT
-#include "pycore_obmalloc_init.h"
+#include "pycore_obmalloc_init.h" // _obmalloc_global_state_INIT
+#include "pycore_parser.h"        // _parser_runtime_state_INIT
+#include "pycore_pyhash.h"        // pyhash_state_INIT
+#include "pycore_pymem_init.h"    // _pymem_allocators_standard_INIT
+#include "pycore_runtime_init_generated.h"  // _Py_bytes_characters_INIT
+#include "pycore_signal.h"        // _signals_RUNTIME_INIT
+#include "pycore_tracemalloc.h"   // _tracemalloc_runtime_state_INIT
 
 
 extern PyTypeObject _PyExc_MemoryError;

Include/internal/pycore_runtime_init_generated.h

@@ -8,6 +8,9 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_long.h"          // _PyLong_DIGIT_INIT()
+
+
 /* The following is auto-generated by Tools/build/generate_global_objects.py. */
 #define _Py_small_ints_INIT { \
     _PyLong_DIGIT_INIT(-5), \

Include/internal/pycore_signal.h

@@ -10,8 +10,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_atomic.h"         // _Py_atomic_address
+#include "pycore_atomic.h"        // _Py_atomic_address
-#include <signal.h>                // NSIG
+#include <signal.h>               // NSIG
 
 
 // Restore signals that the interpreter has called SIG_IGN on to SIG_DFL.

Include/internal/pycore_typeobject.h

@@ -4,12 +4,12 @@
 extern "C" {
 #endif
 
-#include "pycore_moduleobject.h"
-
 #ifndef Py_BUILD_CORE
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_moduleobject.h"  // PyModuleObject
+
 
 /* state */
 

Modules/_json.c

@@ -12,7 +12,7 @@
 #include "pycore_ceval.h"           // _Py_EnterRecursiveCall()
 #include "pycore_runtime.h"         // _PyRuntime
 
-#include "pycore_global_objects.h"  // _Py_ID()
+#include "pycore_global_strings.h"  // _Py_ID()
 #include <stdbool.h>                // bool
 
 

Modules/_testcapimodule.c

@@ -18,8 +18,8 @@
 #undef NDEBUG
 
 #include "Python.h"
-#include "frameobject.h"          // PyFrame_New
+#include "frameobject.h"          // PyFrame_New()
-#include "marshal.h"              // PyMarshal_WriteLongToFile
+#include "marshal.h"              // PyMarshal_WriteLongToFile()
 
 #include <float.h>                // FLT_MAX
 #include <signal.h>
@@ -37,7 +37,7 @@
 #endif
 
 #ifdef bool
-#  error "The public headers should not include <stdbool.h>, see bpo-46748"
+#  error "The public headers should not include <stdbool.h>, see gh-48924"
 #endif
 
 // Several parts of this module are broken out into files in _testcapi/.

Modules/_testinternalcapi.c

@@ -10,26 +10,24 @@
 #undef NDEBUG
 
 #include "Python.h"
-#include "pycore_atomic_funcs.h" // _Py_atomic_int_get()
+#include "pycore_atomic_funcs.h"  // _Py_atomic_int_get()
-#include "pycore_bitutils.h"     // _Py_bswap32()
+#include "pycore_bitutils.h"      // _Py_bswap32()
-#include "pycore_bytesobject.h"  // _PyBytes_Find()
+#include "pycore_bytesobject.h"   // _PyBytes_Find()
-#include "pycore_compile.h"      // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble, _PyCompile_CleanDoc
+#include "pycore_compile.h"       // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble, _PyCompile_CleanDoc
-#include "pycore_ceval.h"        // _PyEval_AddPendingCall
+#include "pycore_ceval.h"         // _PyEval_AddPendingCall
-#include "pycore_dict.h"        // _PyDictOrValues_GetValues
+#include "pycore_dict.h"          // _PyDictOrValues_GetValues()
-#include "pycore_fileutils.h"    // _Py_normpath
+#include "pycore_fileutils.h"     // _Py_normpath
-#include "pycore_frame.h"        // _PyInterpreterFrame
+#include "pycore_frame.h"         // _PyInterpreterFrame
-#include "pycore_gc.h"           // PyGC_Head
+#include "pycore_gc.h"            // PyGC_Head
-#include "pycore_hashtable.h"    // _Py_hashtable_new()
+#include "pycore_hashtable.h"     // _Py_hashtable_new()
-#include "pycore_initconfig.h"   // _Py_GetConfigsAsDict()
+#include "pycore_initconfig.h"    // _Py_GetConfigsAsDict()
-#include "pycore_interp.h"       // _PyInterpreterState_GetConfigCopy()
+#include "pycore_interp.h"        // _PyInterpreterState_GetConfigCopy()
-#include "pycore_object.h"       // _PyObject_IsFreed()
+#include "pycore_object.h"        // _PyObject_IsFreed()
-#include "pycore_pathconfig.h"   // _PyPathConfig_ClearGlobal()
+#include "pycore_pathconfig.h"    // _PyPathConfig_ClearGlobal()
-#include "pycore_pyerrors.h"     // _Py_UTF8_Edit_Cost()
+#include "pycore_pyerrors.h"      // _Py_UTF8_Edit_Cost()
-#include "pycore_pystate.h"      // _PyThreadState_GET()
+#include "pycore_pystate.h"       // _PyThreadState_GET()
 
-#include "frameobject.h"
+#include "interpreteridobject.h"  // PyInterpreterID_LookUp()
-#include "interpreteridobject.h" // PyInterpreterID_LookUp()
-#include "osdefs.h"              // MAXPATHLEN
 
 #include "clinic/_testinternalcapi.c.h"
 

Parser/asdl_c.py

@@ -1585,7 +1585,7 @@ def write_header(mod, metadata, f):
         #  error "this header requires Py_BUILD_CORE define"
         #endif
 
-        #include "pycore_asdl.h"
+        #include "pycore_asdl.h"          // _ASDL_SEQ_HEAD
 
     """).lstrip())
 

Tools/cases_generator/generate_cases.py

@@ -410,7 +410,7 @@ class Generator(Analyzer):
                 #endif
             """).strip())
 
-            self.out.emit("\n#include <stdbool.h>")
+            self.out.emit("\n#include <stdbool.h>              // bool")
 
             self.write_pseudo_instrs()