bpo-40528: Improve AST generation script to do builds simultaneously (GH-19968)
- Switch from getopt to argparse. - Removed the limitation of not being able to produce both C and H simultaneously. This will make it run faster since it parses the asdl definition once and uses the generated tree to generate both the header and the C source.
This commit is contained in:
Batuhan Taskaya
GitHub
parent
7b7a21bc4f
commit
63b8e0cba3
|
|
@ -845,17 +845,15 @@ regen-pegen:
|
|||
|
||||
.PHONY=regen-ast
|
||||
regen-ast:
|
||||
# Regenerate Include/Python-ast.h using Parser/asdl_c.py -h
|
||||
# Regenerate Include/Python-ast.h and Python/Python-ast.c using Parser/asdl_c.py
|
||||
$(MKDIR_P) $(srcdir)/Include
|
||||
$(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \
|
||||
-h $(srcdir)/Include/Python-ast.h.new \
|
||||
$(srcdir)/Parser/Python.asdl
|
||||
$(UPDATE_FILE) $(srcdir)/Include/Python-ast.h $(srcdir)/Include/Python-ast.h.new
|
||||
# Regenerate Python/Python-ast.c using Parser/asdl_c.py -c
|
||||
$(MKDIR_P) $(srcdir)/Python
|
||||
$(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \
|
||||
-c $(srcdir)/Python/Python-ast.c.new \
|
||||
$(srcdir)/Parser/Python.asdl
|
||||
$(srcdir)/Parser/Python.asdl \
|
||||
-H $(srcdir)/Include/Python-ast.h.new \
|
||||
-C $(srcdir)/Python/Python-ast.c.new
|
||||
|
||||
$(UPDATE_FILE) $(srcdir)/Include/Python-ast.h $(srcdir)/Include/Python-ast.h.new
|
||||
$(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new
|
||||
|
||||
.PHONY: regen-opcode
|
||||
|
|
|
|||
|
|
@ -176,20 +176,15 @@
|
|||
<Warning Text="Pegen updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedParse)' != ''" />
|
||||
</Target>
|
||||
<Target Name="_RegenAST_H" AfterTargets="_RegenGrammar">
|
||||
<!-- Regenerate Include/Python-ast.h using Parser/asdl_c.py -h -->
|
||||
<Exec Command=""$(PythonExe)" "$(PySourcePath)Parser\asdl_c.py" -h "$(IntDir)Python-ast.h" "$(PySourcePath)Parser\Python.asdl"" />
|
||||
<!-- Regenerate Include/Python-ast.h and Python/Python-ast.c using Parser/asdl_c.py -h -->
|
||||
<Exec Command=""$(PythonExe)" "$(PySourcePath)Parser\asdl_c.py" "$(PySourcePath)Parser\Python.asdl" -H "$(IntDir)Python-ast.h" -C "$(IntDir)Python-ast.c"" />
|
||||
<Copy SourceFiles="$(IntDir)Python-ast.h" DestinationFiles="$(PySourcePath)Include\Python-ast.h">
|
||||
<Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
|
||||
</Copy>
|
||||
<Warning Text="Python-ast.h updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
|
||||
</Target>
|
||||
<Target Name="_RegenAST_C" AfterTargets="_RegenGrammar">
|
||||
<!-- Regenerate Python/Python-ast.c using Parser/asdl_c.py -c -->
|
||||
<Exec Command=""$(PythonExe)" "$(PySourcePath)Parser\asdl_c.py" -c "$(IntDir)Python-ast.c" "$(PySourcePath)Parser\Python.asdl"" />
|
||||
<Copy SourceFiles="$(IntDir)Python-ast.c" DestinationFiles="$(PySourcePath)Python\Python-ast.c">
|
||||
<Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
|
||||
<Output TaskParameter="CopiedFiles" ItemName="_UpdatedC" />
|
||||
</Copy>
|
||||
<Warning Text="Python-ast.c updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
|
||||
<Warning Text="ASDL is updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
|
||||
</Target>
|
||||
<Target Name="_RegenOpcodes" AfterTargets="_RegenAST_C">
|
||||
<!-- Regenerate Include/opcode.h from Lib/opcode.py using Tools/scripts/generate_opcode_h.py-->
|
||||
|
|
|
|||
152
Parser/asdl_c.py
152
Parser/asdl_c.py
|
|
@ -1,12 +1,17 @@
|
|||
#! /usr/bin/env python
|
||||
"""Generate C code from an ASDL description."""
|
||||
|
||||
import os, sys
|
||||
import os
|
||||
import sys
|
||||
|
||||
from argparse import ArgumentParser
|
||||
from pathlib import Path
|
||||
|
||||
import asdl
|
||||
|
||||
TABSIZE = 4
|
||||
MAX_COL = 80
|
||||
AUTOGEN_MESSAGE = "/* File automatically generated by {}. */\n\n"
|
||||
|
||||
def get_c_type(name):
|
||||
"""Return a string for the C name of the type.
|
||||
|
|
@ -1369,94 +1374,77 @@ static struct PyModuleDef _astmodule = {
|
|||
f.write(' return 1;\n')
|
||||
f.write('};\n\n')
|
||||
|
||||
def write_header(f, mod):
|
||||
f.write('#ifndef Py_PYTHON_AST_H\n')
|
||||
f.write('#define Py_PYTHON_AST_H\n')
|
||||
f.write('#ifdef __cplusplus\n')
|
||||
f.write('extern "C" {\n')
|
||||
f.write('#endif\n')
|
||||
f.write('\n')
|
||||
f.write('#ifndef Py_LIMITED_API\n')
|
||||
f.write('#include "asdl.h"\n')
|
||||
f.write('\n')
|
||||
f.write('#undef Yield /* undefine macro conflicting with <winbase.h> */\n')
|
||||
f.write('\n')
|
||||
c = ChainOfVisitors(TypeDefVisitor(f),
|
||||
StructVisitor(f))
|
||||
c.visit(mod)
|
||||
f.write("// Note: these macros affect function definitions, not only call sites.\n")
|
||||
PrototypeVisitor(f).visit(mod)
|
||||
f.write("\n")
|
||||
f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
|
||||
f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
|
||||
f.write("int PyAST_Check(PyObject* obj);\n")
|
||||
f.write("#endif /* !Py_LIMITED_API */\n")
|
||||
f.write('\n')
|
||||
f.write('#ifdef __cplusplus\n')
|
||||
f.write('}\n')
|
||||
f.write('#endif\n')
|
||||
f.write('#endif /* !Py_PYTHON_AST_H */\n')
|
||||
|
||||
common_msg = "/* File automatically generated by %s. */\n\n"
|
||||
def write_source(f, mod):
|
||||
f.write('#include <stddef.h>\n')
|
||||
f.write('\n')
|
||||
f.write('#include "Python.h"\n')
|
||||
f.write('#include "%s-ast.h"\n' % mod.name)
|
||||
f.write('#include "structmember.h" // PyMemberDef\n')
|
||||
f.write('\n')
|
||||
|
||||
def main(srcfile, dump_module=False):
|
||||
argv0 = sys.argv[0]
|
||||
components = argv0.split(os.sep)
|
||||
# Always join with '/' so different OS does not keep changing the file
|
||||
argv0 = '/'.join(components[-2:])
|
||||
auto_gen_msg = common_msg % argv0
|
||||
mod = asdl.parse(srcfile)
|
||||
generate_module_def(f, mod)
|
||||
|
||||
v = ChainOfVisitors(
|
||||
PyTypesDeclareVisitor(f),
|
||||
PyTypesVisitor(f),
|
||||
Obj2ModPrototypeVisitor(f),
|
||||
FunctionVisitor(f),
|
||||
ObjVisitor(f),
|
||||
Obj2ModVisitor(f),
|
||||
ASTModuleVisitor(f),
|
||||
PartingShots(f),
|
||||
)
|
||||
v.visit(mod)
|
||||
|
||||
def main(input_file, c_file, h_file, dump_module=False):
|
||||
auto_gen_msg = AUTOGEN_MESSAGE.format("/".join(Path(__file__).parts[-2:]))
|
||||
mod = asdl.parse(input_file)
|
||||
if dump_module:
|
||||
print('Parsed Module:')
|
||||
print(mod)
|
||||
if not asdl.check(mod):
|
||||
sys.exit(1)
|
||||
if H_FILE:
|
||||
with open(H_FILE, "w") as f:
|
||||
f.write(auto_gen_msg)
|
||||
f.write('#ifndef Py_PYTHON_AST_H\n')
|
||||
f.write('#define Py_PYTHON_AST_H\n')
|
||||
f.write('#ifdef __cplusplus\n')
|
||||
f.write('extern "C" {\n')
|
||||
f.write('#endif\n')
|
||||
f.write('\n')
|
||||
f.write('#ifndef Py_LIMITED_API\n')
|
||||
f.write('#include "asdl.h"\n')
|
||||
f.write('\n')
|
||||
f.write('#undef Yield /* undefine macro conflicting with <winbase.h> */\n')
|
||||
f.write('\n')
|
||||
c = ChainOfVisitors(TypeDefVisitor(f),
|
||||
StructVisitor(f))
|
||||
|
||||
c.visit(mod)
|
||||
f.write("// Note: these macros affect function definitions, not only call sites.\n")
|
||||
PrototypeVisitor(f).visit(mod)
|
||||
f.write("\n")
|
||||
f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
|
||||
f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
|
||||
f.write("int PyAST_Check(PyObject* obj);\n")
|
||||
f.write("#endif /* !Py_LIMITED_API */\n")
|
||||
f.write('\n')
|
||||
f.write('#ifdef __cplusplus\n')
|
||||
f.write('}\n')
|
||||
f.write('#endif\n')
|
||||
f.write('#endif /* !Py_PYTHON_AST_H */\n')
|
||||
|
||||
if C_FILE:
|
||||
with open(C_FILE, "w") as f:
|
||||
f.write(auto_gen_msg)
|
||||
f.write('#include <stddef.h>\n')
|
||||
f.write('\n')
|
||||
f.write('#include "Python.h"\n')
|
||||
f.write('#include "%s-ast.h"\n' % mod.name)
|
||||
f.write('#include "structmember.h" // PyMemberDef\n')
|
||||
f.write('\n')
|
||||
|
||||
generate_module_def(f, mod)
|
||||
|
||||
v = ChainOfVisitors(
|
||||
PyTypesDeclareVisitor(f),
|
||||
PyTypesVisitor(f),
|
||||
Obj2ModPrototypeVisitor(f),
|
||||
FunctionVisitor(f),
|
||||
ObjVisitor(f),
|
||||
Obj2ModVisitor(f),
|
||||
ASTModuleVisitor(f),
|
||||
PartingShots(f),
|
||||
)
|
||||
v.visit(mod)
|
||||
for file, writer in (c_file, write_source), (h_file, write_header):
|
||||
if file is not None:
|
||||
with file.open("w") as f:
|
||||
f.write(auto_gen_msg)
|
||||
writer(f, mod)
|
||||
print(file, "regenerated.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
import getopt
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("input_file", type=Path)
|
||||
parser.add_argument("-C", "--c-file", type=Path, default=None)
|
||||
parser.add_argument("-H", "--h-file", type=Path, default=None)
|
||||
parser.add_argument("-d", "--dump-module", action="store_true")
|
||||
|
||||
H_FILE = ''
|
||||
C_FILE = ''
|
||||
dump_module = False
|
||||
opts, args = getopt.getopt(sys.argv[1:], "dh:c:")
|
||||
for o, v in opts:
|
||||
if o == '-h':
|
||||
H_FILE = v
|
||||
elif o == '-c':
|
||||
C_FILE = v
|
||||
elif o == '-d':
|
||||
dump_module = True
|
||||
if H_FILE and C_FILE:
|
||||
print('Must specify exactly one output file')
|
||||
sys.exit(1)
|
||||
elif len(args) != 1:
|
||||
print('Must specify single input file')
|
||||
sys.exit(1)
|
||||
main(args[0], dump_module)
|
||||
options = parser.parse_args()
|
||||
main(**vars(options))
|
||||
|
|
|
|||
Loading…
Reference in New Issue