Fix some scripts in the peg generator folder (GH-19853)

Lib/test/test_peg_generator/test_c_parser.py

@@ -15,6 +15,7 @@ with test_tools.imports_under_tool("peg_generator"):
         generate_parser_c_extension,
         generate_c_parser_source,
     )
+    from pegen.ast_dump import ast_dump
 
 
 TEST_TEMPLATE = """
@@ -24,7 +25,10 @@ import ast
 import traceback
 import sys
 import unittest
-from test.test_peg_generator.ast_dump import ast_dump
+
+from test import test_tools
+with test_tools.imports_under_tool("peg_generator"):
+    from pegen.ast_dump import ast_dump
 
 sys.path.insert(0, tmp_dir)
 import parse

Tools/peg_generator/data/cprog.py

@@ -7,4 +7,5 @@ if 1:
         pass
     elif 1:
         pass
-    else: print("else-clause")
+    else:
+        print("else-clause")

Tools/peg_generator/pegen/ast_dump.py

@@ -6,16 +6,17 @@ always fail. We rely on string comparison of the base classes instead.
 TODO: Remove the above-described hack.
 """
 
+
 def ast_dump(node, annotate_fields=True, include_attributes=False, *, indent=None):
     def _format(node, level=0):
         if indent is not None:
             level += 1
-            prefix = '\n' + indent * level
+            prefix = "\n" + indent * level
-            sep = ',\n' + indent * level
+            sep = ",\n" + indent * level
         else:
-            prefix = ''
+            prefix = ""
-            sep = ', '
+            sep = ", "
-        if any(cls.__name__ == 'AST' for cls in node.__class__.__mro__):
+        if any(cls.__name__ == "AST" for cls in node.__class__.__mro__):
             cls = type(node)
             args = []
             allsimple = True
@@ -32,7 +33,7 @@ def ast_dump(node, annotate_fields=True, include_attributes=False, *, indent=Non
                 value, simple = _format(value, level)
                 allsimple = allsimple and simple
                 if keywords:
-                    args.append('%s=%s' % (name, value))
+                    args.append("%s=%s" % (name, value))
                 else:
                     args.append(value)
             if include_attributes and node._attributes:
@@ -45,18 +46,18 @@ def ast_dump(node, annotate_fields=True, include_attributes=False, *, indent=Non
                         continue
                     value, simple = _format(value, level)
                     allsimple = allsimple and simple
-                    args.append('%s=%s' % (name, value))
+                    args.append("%s=%s" % (name, value))
             if allsimple and len(args) <= 3:
-                return '%s(%s)' % (node.__class__.__name__, ', '.join(args)), not args
+                return "%s(%s)" % (node.__class__.__name__, ", ".join(args)), not args
-            return '%s(%s%s)' % (node.__class__.__name__, prefix, sep.join(args)), False
+            return "%s(%s%s)" % (node.__class__.__name__, prefix, sep.join(args)), False
         elif isinstance(node, list):
             if not node:
-                return '[]', True
+                return "[]", True
-            return '[%s%s]' % (prefix, sep.join(_format(x, level)[0] for x in node)), False
+            return "[%s%s]" % (prefix, sep.join(_format(x, level)[0] for x in node)), False
         return repr(node), True
 
-    if all(cls.__name__ != 'AST' for cls in node.__class__.__mro__):
+    if all(cls.__name__ != "AST" for cls in node.__class__.__mro__):
-        raise TypeError('expected AST, got %r' % node.__class__.__name__)
+        raise TypeError("expected AST, got %r" % node.__class__.__name__)
     if indent is not None and not isinstance(indent, str):
-        indent = ' ' * indent
+        indent = " " * indent
     return _format(node)[0]

Tools/peg_generator/pegen/build.py

@@ -3,6 +3,7 @@ import shutil
 import tokenize
 import sys
 import sysconfig
+import tempfile
 import itertools
 
 from typing import Optional, Tuple, List, IO, Iterator, Set, Dict
@@ -162,8 +163,12 @@ def build_c_generator(
         gen.generate(grammar_file)
 
     if compile_extension:
+        with tempfile.TemporaryDirectory() as build_dir:
             compile_c_extension(
-            output_file, verbose=verbose_c_extension, keep_asserts=keep_asserts_in_extension
+                output_file,
+                build_dir=build_dir,
+                verbose=verbose_c_extension,
+                keep_asserts=keep_asserts_in_extension,
             )
     return gen
 

Tools/peg_generator/scripts/benchmark.py

@@ -105,10 +105,7 @@ def run_benchmark_stdlib(subcommand, parser):
             "../../Lib",
             "../../Grammar/python.gram",
             verbose=False,
-            excluded_files=[
+            excluded_files=["*/bad*", "*/lib2to3/tests/data/*",],
-                "*/bad*",
-                "*/lib2to3/tests/data/*",
-            ],
             skip_actions=False,
             tree_arg=0,
             short=True,

Tools/peg_generator/scripts/show_parse.py

@@ -32,6 +32,9 @@ import tempfile
 
 from typing import List
 
+sys.path.insert(0, os.getcwd())
+from pegen.ast_dump import ast_dump
+
 parser = argparse.ArgumentParser()
 parser.add_argument(
     "-d", "--diff", action="store_true", help="show diff between grammar and ast (requires -g)"
@@ -49,7 +52,7 @@ parser.add_argument("program", nargs="+", help="program to parse (will be concat
 
 def format_tree(tree: ast.AST, verbose: bool = False) -> str:
     with tempfile.NamedTemporaryFile("w+") as tf:
-        tf.write(ast.dump(tree, include_attributes=verbose))
+        tf.write(ast_dump(tree, include_attributes=verbose))
         tf.write("\n")
         tf.flush()
         cmd = f"black -q {tf.name}"

Tools/peg_generator/scripts/test_parse_directory.py

@@ -14,6 +14,7 @@ from typing import List, Optional, Any
 
 sys.path.insert(0, os.getcwd())
 from pegen.build import build_c_parser_and_generator
+from pegen.ast_dump import ast_dump
 from pegen.testutil import print_memstats
 from scripts import show_parse
 
@@ -85,8 +86,8 @@ def compare_trees(
     with open(file) as f:
         expected_tree = ast.parse(f.read())
 
-    expected_text = ast.dump(expected_tree, include_attributes=include_attributes)
+    expected_text = ast_dump(expected_tree, include_attributes=include_attributes)
-    actual_text = ast.dump(actual_tree, include_attributes=include_attributes)
+    actual_text = ast_dump(actual_tree, include_attributes=include_attributes)
     if actual_text == expected_text:
         if verbose:
             print("Tree for {file}:")
@@ -164,7 +165,7 @@ def parse_directory(
     if parser == "pegen":
         try:
             from peg_extension import parse  # type: ignore
-        except:
+        except Exception as e:
             print(
                 "An existing parser was not found. Please run `make` or specify a grammar file with the `-g` flag.",
                 file=sys.stderr,

Tools/peg_generator/scripts/test_pypi_packages.py

@@ -6,14 +6,18 @@ import glob
 import tarfile
 import zipfile
 import shutil
+import pathlib
 import sys
 
 from typing import Generator, Any
 
 sys.path.insert(0, ".")
+
 from pegen import build
 from scripts import test_parse_directory
 
+HERE = pathlib.Path(__file__).resolve().parent
+
 argparser = argparse.ArgumentParser(
     prog="test_pypi_packages", description="Helper program to test parsing PyPI packages",
 )
@@ -53,7 +57,8 @@ def find_dirname(package_name: str) -> str:
 def run_tests(dirname: str, tree: int, extension: Any) -> int:
     return test_parse_directory.parse_directory(
         dirname,
-        "data/python.gram",
+        HERE / ".." / ".." / ".." / "Grammar" / "python.gram",
+        HERE / ".." / ".." / ".." / "Grammar" / "Tokens",
         verbose=False,
         excluded_files=[
             "*/failset/*",
@@ -68,6 +73,8 @@ def run_tests(dirname: str, tree: int, extension: Any) -> int:
         tree_arg=tree,
         short=True,
         extension=extension,
+        mode=1,
+        parser="pegen",
     )
 
 
@@ -75,9 +82,13 @@ def main() -> None:
     args = argparser.parse_args()
     tree = args.tree
 
-    extension = build.build_parser_and_generator(
+    extension = build.build_c_parser_and_generator(
-        "data/python.gram", "peg_parser/parse.c", compile_extension=True
+        HERE / ".." / ".." / ".." / "Grammar" / "python.gram",
+        HERE / ".." / ".." / ".." / "Grammar" / "Tokens",
+        "peg_extension/parse.c",
+        compile_extension=True,
     )
+
     for package in get_packages():
         print(f"Extracting files from {package}... ", end="")
         try:
@@ -91,7 +102,6 @@ def main() -> None:
         dirname = find_dirname(package)
         status = run_tests(dirname, tree, extension)
         if status == 0:
-            print("Done")
             shutil.rmtree(dirname)
         else:
             print(f"Failed to parse {dirname}")