gh-121404: split compile.c into compile.c and codegen.c (#123651)

Include/internal/pycore_compile.h

@@ -8,6 +8,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_ast.h"       // mod_ty
 #include "pycore_symtable.h"  // _Py_SourceLocation
 #include "pycore_instruction_sequence.h"
 
@@ -63,6 +64,120 @@ typedef struct {
     int u_firstlineno; /* the first lineno of the block */
 } _PyCompile_CodeUnitMetadata;
 
+struct _PyCompiler;
+
+typedef enum {
+    COMPILE_OP_FAST,
+    COMPILE_OP_GLOBAL,
+    COMPILE_OP_DEREF,
+    COMPILE_OP_NAME,
+} _PyCompile_optype;
+
+/* _PyCompile_FBlockInfo tracks the current frame block.
+ *
+ * A frame block is used to handle loops, try/except, and try/finally.
+ * It's called a frame block to distinguish it from a basic block in the
+ * compiler IR.
+ */
+
+enum _PyCompile_FBlockType {
+     COMPILE_FBLOCK_WHILE_LOOP,
+     COMPILE_FBLOCK_FOR_LOOP,
+     COMPILE_FBLOCK_TRY_EXCEPT,
+     COMPILE_FBLOCK_FINALLY_TRY,
+     COMPILE_FBLOCK_FINALLY_END,
+     COMPILE_FBLOCK_WITH,
+     COMPILE_FBLOCK_ASYNC_WITH,
+     COMPILE_FBLOCK_HANDLER_CLEANUP,
+     COMPILE_FBLOCK_POP_VALUE,
+     COMPILE_FBLOCK_EXCEPTION_HANDLER,
+     COMPILE_FBLOCK_EXCEPTION_GROUP_HANDLER,
+     COMPILE_FBLOCK_ASYNC_COMPREHENSION_GENERATOR,
+     COMPILE_FBLOCK_STOP_ITERATION,
+};
+
+typedef struct {
+    enum _PyCompile_FBlockType fb_type;
+    _PyJumpTargetLabel fb_block;
+    _Py_SourceLocation fb_loc;
+    /* (optional) type-specific exit or cleanup block */
+    _PyJumpTargetLabel fb_exit;
+    /* (optional) additional information required for unwinding */
+    void *fb_datum;
+} _PyCompile_FBlockInfo;
+
+
+int _PyCompile_PushFBlock(struct _PyCompiler *c, _Py_SourceLocation loc,
+                          enum _PyCompile_FBlockType t,
+                          _PyJumpTargetLabel block_label,
+                          _PyJumpTargetLabel exit, void *datum);
+void _PyCompile_PopFBlock(struct _PyCompiler *c, enum _PyCompile_FBlockType t,
+                          _PyJumpTargetLabel block_label);
+_PyCompile_FBlockInfo *_PyCompile_TopFBlock(struct _PyCompiler *c);
+
+int _PyCompile_EnterScope(struct _PyCompiler *c, identifier name, int scope_type,
+                          void *key, int lineno, PyObject *private,
+                          _PyCompile_CodeUnitMetadata *umd);
+void _PyCompile_ExitScope(struct _PyCompiler *c);
+Py_ssize_t _PyCompile_AddConst(struct _PyCompiler *c, PyObject *o);
+_PyInstructionSequence *_PyCompile_InstrSequence(struct _PyCompiler *c);
+int _PyCompile_FutureFeatures(struct _PyCompiler *c);
+PyObject *_PyCompile_DeferredAnnotations(struct _PyCompiler *c);
+PyObject *_PyCompile_Mangle(struct _PyCompiler *c, PyObject *name);
+PyObject *_PyCompile_MaybeMangle(struct _PyCompiler *c, PyObject *name);
+int _PyCompile_MaybeAddStaticAttributeToClass(struct _PyCompiler *c, expr_ty e);
+int _PyCompile_GetRefType(struct _PyCompiler *c, PyObject *name);
+int _PyCompile_LookupCellvar(struct _PyCompiler *c, PyObject *name);
+int _PyCompile_ResolveNameop(struct _PyCompiler *c, PyObject *mangled, int scope,
+                             _PyCompile_optype *optype, Py_ssize_t *arg);
+
+int _PyCompile_IsInteractive(struct _PyCompiler *c);
+int _PyCompile_IsNestedScope(struct _PyCompiler *c);
+int _PyCompile_IsInInlinedComp(struct _PyCompiler *c);
+int _PyCompile_ScopeType(struct _PyCompiler *c);
+int _PyCompile_OptimizationLevel(struct _PyCompiler *c);
+PyArena *_PyCompile_Arena(struct _PyCompiler *c);
+int _PyCompile_LookupArg(struct _PyCompiler *c, PyCodeObject *co, PyObject *name);
+PyObject *_PyCompile_Qualname(struct _PyCompiler *c);
+_PyCompile_CodeUnitMetadata *_PyCompile_Metadata(struct _PyCompiler *c);
+PyObject *_PyCompile_StaticAttributesAsTuple(struct _PyCompiler *c);
+
+#ifndef NDEBUG
+int _PyCompile_IsTopLevelAwait(struct _PyCompiler *c);
+#endif
+
+struct symtable *_PyCompile_Symtable(struct _PyCompiler *c);
+PySTEntryObject *_PyCompile_SymtableEntry(struct _PyCompiler *c);
+
+enum {
+    COMPILE_SCOPE_MODULE,
+    COMPILE_SCOPE_CLASS,
+    COMPILE_SCOPE_FUNCTION,
+    COMPILE_SCOPE_ASYNC_FUNCTION,
+    COMPILE_SCOPE_LAMBDA,
+    COMPILE_SCOPE_COMPREHENSION,
+    COMPILE_SCOPE_ANNOTATIONS,
+};
+
+
+typedef struct {
+    PyObject *pushed_locals;
+    PyObject *temp_symbols;
+    PyObject *fast_hidden;
+    _PyJumpTargetLabel cleanup;
+} _PyCompile_InlinedComprehensionState;
+
+int _PyCompile_TweakInlinedComprehensionScopes(struct _PyCompiler *c, _Py_SourceLocation loc,
+                                               PySTEntryObject *entry,
+                                               _PyCompile_InlinedComprehensionState *state);
+int _PyCompile_RevertInlinedComprehensionScopes(struct _PyCompiler *c, _Py_SourceLocation loc,
+                                                _PyCompile_InlinedComprehensionState *state);
+int _PyCompile_AddDeferredAnnotaion(struct _PyCompiler *c, stmt_ty s);
+
+int _PyCodegen_AddReturnAtEnd(struct _PyCompiler *c, int addNone);
+int _PyCodegen_EnterAnonymousScope(struct _PyCompiler* c, mod_ty mod);
+int _PyCodegen_Expression(struct _PyCompiler *c, expr_ty e);
+int _PyCodegen_Body(struct _PyCompiler *c, _Py_SourceLocation loc, asdl_stmt_seq *stmts);
 
 /* Utility for a number of growing arrays used in the compiler */
 int _PyCompile_EnsureArrayLargeEnough(
@@ -74,6 +189,11 @@ int _PyCompile_EnsureArrayLargeEnough(
 
 int _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj);
 
+PyCodeObject *_PyCompile_OptimizeAndAssemble(struct _PyCompiler *c, int addNone);
+
+Py_ssize_t _PyCompile_DictAddObj(PyObject *dict, PyObject *o);
+int _PyCompile_Error(struct _PyCompiler *c, _Py_SourceLocation loc, const char *format, ...);
+int _PyCompile_Warn(struct _PyCompiler *c, _Py_SourceLocation loc, const char *format, ...);
 
 // Export for '_opcode' extension module
 PyAPI_FUNC(PyObject*) _PyCompile_GetUnaryIntrinsicName(int index);

Include/internal/pycore_instruction_sequence.h

@@ -51,6 +51,11 @@ typedef struct {
     int id;
 } _PyJumpTargetLabel;
 
+#define NO_LABEL ((const _PyJumpTargetLabel){-1})
+
+#define SAME_JUMP_TARGET_LABEL(L1, L2) ((L1).id == (L2).id)
+#define IS_JUMP_TARGET_LABEL(L) (!SAME_JUMP_TARGET_LABEL((L), (NO_LABEL)))
+
 PyAPI_FUNC(PyObject*)_PyInstructionSequence_New(void);
 
 int _PyInstructionSequence_UseLabel(_PyInstructionSequence *seq, int lbl);

Makefile.pre.in

@@ -429,6 +429,7 @@ PYTHON_OBJS=	\
 		Python/brc.o \
 		Python/ceval.o \
 		Python/codecs.o \
+		Python/codegen.o \
 		Python/compile.o \
 		Python/context.o \
 		Python/critical_section.o \
@@ -1873,7 +1874,7 @@ regen-sre:
 		$(srcdir)/Modules/_sre/sre_constants.h \
 		$(srcdir)/Modules/_sre/sre_targets.h
 
-Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o: $(srcdir)/Include/internal/pycore_ast.h $(srcdir)/Include/internal/pycore_ast.h
+Python/compile.o Python/codegen.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o: $(srcdir)/Include/internal/pycore_ast.h $(srcdir)/Include/internal/pycore_ast.h
 
 Python/getplatform.o: $(srcdir)/Python/getplatform.c
 		$(CC) -c $(PY_CORE_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c
@@ -2009,7 +2010,7 @@ regen-uop-metadata:
 	    $(srcdir)/Include/internal/pycore_uop_metadata.h.new $(srcdir)/Python/bytecodes.c
 	$(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_metadata.h $(srcdir)/Include/internal/pycore_uop_metadata.h.new
 
-Python/compile.o Python/assemble.o Python/flowgraph.o Python/instruction_sequence.o: \
+Python/compile.o Python/codegen.o Python/assemble.o Python/flowgraph.o Python/instruction_sequence.o: \
                 $(srcdir)/Include/internal/pycore_compile.h \
                 $(srcdir)/Include/internal/pycore_flowgraph.h \
                 $(srcdir)/Include/internal/pycore_instruction_sequence.h \

PCbuild/_freeze_module.vcxproj

@@ -194,6 +194,7 @@
     <ClCompile Include="..\Python\bootstrap_hash.c" />
     <ClCompile Include="..\Python\ceval.c" />
     <ClCompile Include="..\Python\codecs.c" />
+    <ClCompile Include="..\Python\codegen.c" />
     <ClCompile Include="..\Python\compile.c" />
     <ClCompile Include="..\Python\context.c" />
     <ClCompile Include="..\Python\critical_section.c" />

PCbuild/_freeze_module.vcxproj.filters

@@ -97,6 +97,9 @@
     <ClCompile Include="..\Python\perf_jit_trampoline.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\Python\codegen.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\Python\compile.c">
       <Filter>Source Files</Filter>
     </ClCompile>

PCbuild/pythoncore.vcxproj

@@ -581,6 +581,7 @@
     <ClCompile Include="..\Python\brc.c" />
     <ClCompile Include="..\Python\ceval.c" />
     <ClCompile Include="..\Python\codecs.c" />
+    <ClCompile Include="..\Python\codegen.c" />
     <ClCompile Include="..\Python\compile.c" />
     <ClCompile Include="..\Python\context.c" />
     <ClCompile Include="..\Python\critical_section.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -1298,6 +1298,9 @@
     <ClCompile Include="..\Python\codecs.c">
       <Filter>Python</Filter>
     </ClCompile>
+    <ClCompile Include="..\Python\codegen.c">
+      <Filter>Python</Filter>
+    </ClCompile>
     <ClCompile Include="..\Python\compile.c">
       <Filter>Python</Filter>
     </ClCompile>

Python/flowgraph.c

@@ -86,8 +86,6 @@ struct _PyCfgBuilder {
 
 typedef struct _PyCfgBuilder cfg_builder;
 
-static const jump_target_label NO_LABEL = {-1};
-
 #define SAME_LABEL(L1, L2) ((L1).id == (L2).id)
 #define IS_LABEL(L) (!SAME_LABEL((L), (NO_LABEL)))
 

Tools/c-analyzer/cpython/ignored.tsv

@@ -347,7 +347,6 @@ Python/ceval.c	-	_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS	-
 Python/codecs.c	-	Py_hexdigits	-
 Python/codecs.c	-	ucnhash_capi	-
 Python/codecs.c	_PyCodec_InitRegistry	methods	-
-Python/compile.c	-	NO_LABEL	-
 Python/compile.c	-	NO_LOCATION	-
 Python/dynload_shlib.c	-	_PyImport_DynLoadFiletab	-
 Python/dynload_stub.c	-	_PyImport_DynLoadFiletab	-