traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-40528: Improve AST generation script to do builds simultaneously (GH-19968) 

- Switch from getopt to argparse.
- Removed the limitation of not being able to produce both C and H simultaneously.

This will make it run faster since it parses the asdl definition once and uses the generated tree to generate both the header and the C source.
This commit is contained in:
Batuhan Taskaya 2020-05-18 20:42:10 +03:00 committed by GitHub
parent 7b7a21bc4f
commit 63b8e0cba3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 80 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Makefile.pre.in
View File

@ -845,17 +845,15 @@ regen-pegen:
.PHONY=regen-ast .PHONY=regen-ast
regen-ast: regen-ast:
# Regenerate Include/Python-ast.h using Parser/asdl_c.py -h # Regenerate Include/Python-ast.h and Python/Python-ast.c using Parser/asdl_c.py
$(MKDIR_P) $(srcdir)/Include $(MKDIR_P) $(srcdir)/Include
$(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \
-h $(srcdir)/Include/Python-ast.h.new \
$(srcdir)/Parser/Python.asdl
$(UPDATE_FILE) $(srcdir)/Include/Python-ast.h $(srcdir)/Include/Python-ast.h.new
# Regenerate Python/Python-ast.c using Parser/asdl_c.py -c
$(MKDIR_P) $(srcdir)/Python $(MKDIR_P) $(srcdir)/Python
$(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \ $(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \
-c $(srcdir)/Python/Python-ast.c.new \ $(srcdir)/Parser/Python.asdl \
$(srcdir)/Parser/Python.asdl -H $(srcdir)/Include/Python-ast.h.new \
-C $(srcdir)/Python/Python-ast.c.new
$(UPDATE_FILE) $(srcdir)/Include/Python-ast.h $(srcdir)/Include/Python-ast.h.new
$(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new $(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new
.PHONY: regen-opcode .PHONY: regen-opcode

13
PCbuild/regen.vcxproj
View File

@ -176,20 +176,15 @@
<Warning Text="Pegen updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedParse)' != ''" /> <Warning Text="Pegen updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedParse)' != ''" />
</Target> </Target>
<Target Name="_RegenAST_H" AfterTargets="_RegenGrammar"> <Target Name="_RegenAST_H" AfterTargets="_RegenGrammar">
<!-- Regenerate Include/Python-ast.h using Parser/asdl_c.py -h --> <!-- Regenerate Include/Python-ast.h and Python/Python-ast.c using Parser/asdl_c.py -h -->
<Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; -h &quot;$(IntDir)Python-ast.h&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot;" /> <Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot; -H &quot;$(IntDir)Python-ast.h&quot; -C &quot;$(IntDir)Python-ast.c&quot;" />
<Copy SourceFiles="$(IntDir)Python-ast.h" DestinationFiles="$(PySourcePath)Include\Python-ast.h"> <Copy SourceFiles="$(IntDir)Python-ast.h" DestinationFiles="$(PySourcePath)Include\Python-ast.h">
<Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" /> <Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
</Copy> </Copy>
<Warning Text="Python-ast.h updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
</Target>
<Target Name="_RegenAST_C" AfterTargets="_RegenGrammar">
<!-- Regenerate Python/Python-ast.c using Parser/asdl_c.py -c -->
<Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; -c &quot;$(IntDir)Python-ast.c&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot;" />
<Copy SourceFiles="$(IntDir)Python-ast.c" DestinationFiles="$(PySourcePath)Python\Python-ast.c"> <Copy SourceFiles="$(IntDir)Python-ast.c" DestinationFiles="$(PySourcePath)Python\Python-ast.c">
<Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" /> <Output TaskParameter="CopiedFiles" ItemName="_UpdatedC" />
</Copy> </Copy>
<Warning Text="Python-ast.c updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" /> <Warning Text="ASDL is updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
</Target> </Target>
<Target Name="_RegenOpcodes" AfterTargets="_RegenAST_C"> <Target Name="_RegenOpcodes" AfterTargets="_RegenAST_C">
<!-- Regenerate Include/opcode.h from Lib/opcode.py using Tools/scripts/generate_opcode_h.py--> <!-- Regenerate Include/opcode.h from Lib/opcode.py using Tools/scripts/generate_opcode_h.py-->

152
Parser/asdl_c.py
View File

@ -1,12 +1,17 @@
#! /usr/bin/env python #! /usr/bin/env python
"""Generate C code from an ASDL description.""" """Generate C code from an ASDL description."""
import os, sys import os
import sys
from argparse import ArgumentParser
from pathlib import Path
import asdl import asdl
TABSIZE = 4 TABSIZE = 4
MAX_COL = 80 MAX_COL = 80
AUTOGEN_MESSAGE = "/* File automatically generated by {}. */\n\n"
def get_c_type(name): def get_c_type(name):
"""Return a string for the C name of the type. """Return a string for the C name of the type.
@ -1369,94 +1374,77 @@ static struct PyModuleDef _astmodule = {
f.write(' return 1;\n') f.write(' return 1;\n')
f.write('};\n\n') f.write('};\n\n')
def write_header(f, mod):
f.write('#ifndef Py_PYTHON_AST_H\n')
f.write('#define Py_PYTHON_AST_H\n')
f.write('#ifdef __cplusplus\n')
f.write('extern "C" {\n')
f.write('#endif\n')
f.write('\n')
f.write('#ifndef Py_LIMITED_API\n')
f.write('#include "asdl.h"\n')
f.write('\n')
f.write('#undef Yield /* undefine macro conflicting with <winbase.h> */\n')
f.write('\n')
c = ChainOfVisitors(TypeDefVisitor(f),
StructVisitor(f))
c.visit(mod)
f.write("// Note: these macros affect function definitions, not only call sites.\n")
PrototypeVisitor(f).visit(mod)
f.write("\n")
f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
f.write("int PyAST_Check(PyObject* obj);\n")
f.write("#endif /* !Py_LIMITED_API */\n")
f.write('\n')
f.write('#ifdef __cplusplus\n')
f.write('}\n')
f.write('#endif\n')
f.write('#endif /* !Py_PYTHON_AST_H */\n')
common_msg = "/* File automatically generated by %s. */\n\n" def write_source(f, mod):
f.write('#include <stddef.h>\n')
f.write('\n')
f.write('#include "Python.h"\n')
f.write('#include "%s-ast.h"\n' % mod.name)
f.write('#include "structmember.h" // PyMemberDef\n')
f.write('\n')
def main(srcfile, dump_module=False): generate_module_def(f, mod)
argv0 = sys.argv[0]
components = argv0.split(os.sep) v = ChainOfVisitors(
# Always join with '/' so different OS does not keep changing the file PyTypesDeclareVisitor(f),
argv0 = '/'.join(components[-2:]) PyTypesVisitor(f),
auto_gen_msg = common_msg % argv0 Obj2ModPrototypeVisitor(f),
mod = asdl.parse(srcfile) FunctionVisitor(f),
ObjVisitor(f),
Obj2ModVisitor(f),
ASTModuleVisitor(f),
PartingShots(f),
)
v.visit(mod)
def main(input_file, c_file, h_file, dump_module=False):
auto_gen_msg = AUTOGEN_MESSAGE.format("/".join(Path(__file__).parts[-2:]))
mod = asdl.parse(input_file)
if dump_module: if dump_module:
print('Parsed Module:') print('Parsed Module:')
print(mod) print(mod)
if not asdl.check(mod): if not asdl.check(mod):
sys.exit(1) sys.exit(1)
if H_FILE: for file, writer in (c_file, write_source), (h_file, write_header):
with open(H_FILE, "w") as f: if file is not None:
f.write(auto_gen_msg) with file.open("w") as f:
f.write('#ifndef Py_PYTHON_AST_H\n') f.write(auto_gen_msg)
f.write('#define Py_PYTHON_AST_H\n') writer(f, mod)
f.write('#ifdef __cplusplus\n') print(file, "regenerated.")
f.write('extern "C" {\n')
f.write('#endif\n')
f.write('\n')
f.write('#ifndef Py_LIMITED_API\n')
f.write('#include "asdl.h"\n')
f.write('\n')
f.write('#undef Yield /* undefine macro conflicting with <winbase.h> */\n')
f.write('\n')
c = ChainOfVisitors(TypeDefVisitor(f),
StructVisitor(f))
c.visit(mod)
f.write("// Note: these macros affect function definitions, not only call sites.\n")
PrototypeVisitor(f).visit(mod)
f.write("\n")
f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
f.write("int PyAST_Check(PyObject* obj);\n")
f.write("#endif /* !Py_LIMITED_API */\n")
f.write('\n')
f.write('#ifdef __cplusplus\n')
f.write('}\n')
f.write('#endif\n')
f.write('#endif /* !Py_PYTHON_AST_H */\n')
if C_FILE:
with open(C_FILE, "w") as f:
f.write(auto_gen_msg)
f.write('#include <stddef.h>\n')
f.write('\n')
f.write('#include "Python.h"\n')
f.write('#include "%s-ast.h"\n' % mod.name)
f.write('#include "structmember.h" // PyMemberDef\n')
f.write('\n')
generate_module_def(f, mod)
v = ChainOfVisitors(
PyTypesDeclareVisitor(f),
PyTypesVisitor(f),
Obj2ModPrototypeVisitor(f),
FunctionVisitor(f),
ObjVisitor(f),
Obj2ModVisitor(f),
ASTModuleVisitor(f),
PartingShots(f),
)
v.visit(mod)
if __name__ == "__main__": if __name__ == "__main__":
import getopt parser = ArgumentParser()
parser.add_argument("input_file", type=Path)
parser.add_argument("-C", "--c-file", type=Path, default=None)
parser.add_argument("-H", "--h-file", type=Path, default=None)
parser.add_argument("-d", "--dump-module", action="store_true")
H_FILE = '' options = parser.parse_args()
C_FILE = '' main(**vars(options))
dump_module = False
opts, args = getopt.getopt(sys.argv[1:], "dh:c:")
for o, v in opts:
if o == '-h':
H_FILE = v
elif o == '-c':
C_FILE = v
elif o == '-d':
dump_module = True
if H_FILE and C_FILE:
print('Must specify exactly one output file')
sys.exit(1)
elif len(args) != 1:
print('Must specify single input file')
sys.exit(1)
main(args[0], dump_module)