bpo-40528: Improve AST generation script to do builds simultaneously (GH-19968)

- Switch from getopt to argparse.
- Removed the limitation of not being able to produce both C and H simultaneously.

This will make it run faster since it parses the asdl definition once and uses the generated tree to generate both the header and the C source.

Makefile.pre.in

@@ -845,17 +845,15 @@ regen-pegen:
 
 .PHONY=regen-ast
 regen-ast:
-	# Regenerate Include/Python-ast.h using Parser/asdl_c.py -h
+	# Regenerate Include/Python-ast.h and Python/Python-ast.c using Parser/asdl_c.py
 	$(MKDIR_P) $(srcdir)/Include
-	$(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \
-		-h $(srcdir)/Include/Python-ast.h.new \
-		$(srcdir)/Parser/Python.asdl
-	$(UPDATE_FILE) $(srcdir)/Include/Python-ast.h $(srcdir)/Include/Python-ast.h.new
-	# Regenerate Python/Python-ast.c using Parser/asdl_c.py -c
 	$(MKDIR_P) $(srcdir)/Python
 	$(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \
-		-c $(srcdir)/Python/Python-ast.c.new \
+		$(srcdir)/Parser/Python.asdl \
-		$(srcdir)/Parser/Python.asdl
+		-H $(srcdir)/Include/Python-ast.h.new \
+		-C $(srcdir)/Python/Python-ast.c.new
+
+	$(UPDATE_FILE) $(srcdir)/Include/Python-ast.h $(srcdir)/Include/Python-ast.h.new
 	$(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new
 
 .PHONY: regen-opcode

PCbuild/regen.vcxproj

@@ -176,20 +176,15 @@
     <Warning Text="Pegen updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedParse)' != ''" />
   </Target>
   <Target Name="_RegenAST_H" AfterTargets="_RegenGrammar">
-    <!-- Regenerate Include/Python-ast.h using Parser/asdl_c.py -h -->
+    <!-- Regenerate Include/Python-ast.h and Python/Python-ast.c using Parser/asdl_c.py -h -->
-    <Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; -h &quot;$(IntDir)Python-ast.h&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot;" />
+    <Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot; -H &quot;$(IntDir)Python-ast.h&quot; -C &quot;$(IntDir)Python-ast.c&quot;" />
     <Copy SourceFiles="$(IntDir)Python-ast.h" DestinationFiles="$(PySourcePath)Include\Python-ast.h">
       <Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
     </Copy>
-    <Warning Text="Python-ast.h updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
-  </Target>
-  <Target Name="_RegenAST_C" AfterTargets="_RegenGrammar">
-    <!-- Regenerate Python/Python-ast.c using Parser/asdl_c.py -c -->
-    <Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; -c &quot;$(IntDir)Python-ast.c&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot;" />
     <Copy SourceFiles="$(IntDir)Python-ast.c" DestinationFiles="$(PySourcePath)Python\Python-ast.c">
-      <Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
+      <Output TaskParameter="CopiedFiles" ItemName="_UpdatedC" />
     </Copy>
-    <Warning Text="Python-ast.c updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
+    <Warning Text="ASDL is updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
   </Target>
   <Target Name="_RegenOpcodes" AfterTargets="_RegenAST_C">
     <!-- Regenerate Include/opcode.h from Lib/opcode.py using Tools/scripts/generate_opcode_h.py-->

Parser/asdl_c.py

@@ -1,12 +1,17 @@
 #! /usr/bin/env python
 """Generate C code from an ASDL description."""
 
-import os, sys
+import os
+import sys
+
+from argparse import ArgumentParser
+from pathlib import Path
 
 import asdl
 
 TABSIZE = 4
 MAX_COL = 80
+AUTOGEN_MESSAGE = "/* File automatically generated by {}. */\n\n"
 
 def get_c_type(name):
     """Return a string for the C name of the type.
@@ -1369,24 +1374,7 @@ static struct PyModuleDef _astmodule = {
     f.write('    return 1;\n')
     f.write('};\n\n')
 
-
+def write_header(f, mod):
-common_msg = "/* File automatically generated by %s. */\n\n"
-
-def main(srcfile, dump_module=False):
-    argv0 = sys.argv[0]
-    components = argv0.split(os.sep)
-    # Always join with '/' so different OS does not keep changing the file
-    argv0 = '/'.join(components[-2:])
-    auto_gen_msg = common_msg % argv0
-    mod = asdl.parse(srcfile)
-    if dump_module:
-        print('Parsed Module:')
-        print(mod)
-    if not asdl.check(mod):
-        sys.exit(1)
-    if H_FILE:
-        with open(H_FILE, "w") as f:
-            f.write(auto_gen_msg)
     f.write('#ifndef Py_PYTHON_AST_H\n')
     f.write('#define Py_PYTHON_AST_H\n')
     f.write('#ifdef __cplusplus\n')
@@ -1400,7 +1388,6 @@ def main(srcfile, dump_module=False):
     f.write('\n')
     c = ChainOfVisitors(TypeDefVisitor(f),
                         StructVisitor(f))
-
     c.visit(mod)
     f.write("// Note: these macros affect function definitions, not only call sites.\n")
     PrototypeVisitor(f).visit(mod)
@@ -1415,9 +1402,7 @@ def main(srcfile, dump_module=False):
     f.write('#endif\n')
     f.write('#endif /* !Py_PYTHON_AST_H */\n')
 
-    if C_FILE:
+def write_source(f, mod):
-        with open(C_FILE, "w") as f:
-            f.write(auto_gen_msg)
     f.write('#include <stddef.h>\n')
     f.write('\n')
     f.write('#include "Python.h"\n')
@@ -1439,24 +1424,27 @@ def main(srcfile, dump_module=False):
     )
     v.visit(mod)
 
-if __name__ == "__main__":
+def main(input_file, c_file, h_file, dump_module=False):
-    import getopt
+    auto_gen_msg = AUTOGEN_MESSAGE.format("/".join(Path(__file__).parts[-2:]))
+    mod = asdl.parse(input_file)
+    if dump_module:
+        print('Parsed Module:')
+        print(mod)
+    if not asdl.check(mod):
+        sys.exit(1)
+    for file, writer in (c_file, write_source), (h_file, write_header):
+        if file is not None:
+            with file.open("w") as f:
+                f.write(auto_gen_msg)
+                writer(f, mod)
+            print(file, "regenerated.")
 
-    H_FILE = ''
+if __name__ == "__main__":
-    C_FILE = ''
+    parser = ArgumentParser()
-    dump_module = False
+    parser.add_argument("input_file", type=Path)
-    opts, args = getopt.getopt(sys.argv[1:], "dh:c:")
+    parser.add_argument("-C", "--c-file", type=Path, default=None)
-    for o, v in opts:
+    parser.add_argument("-H", "--h-file", type=Path, default=None)
-        if o == '-h':
+    parser.add_argument("-d", "--dump-module", action="store_true")
-            H_FILE = v
+
-        elif o == '-c':
+    options = parser.parse_args()
-            C_FILE = v
+    main(**vars(options))
-        elif o == '-d':
-            dump_module = True
-    if H_FILE and C_FILE:
-        print('Must specify exactly one output file')
-        sys.exit(1)
-    elif len(args) != 1:
-        print('Must specify single input file')
-        sys.exit(1)
-    main(args[0], dump_module)