gh-113317: Finish splitting Argument Clinic into sub-files (#117513)

Add libclinic.parser module and move the following classes and
functions there:

* Parser
* PythonParser
* create_parser_namespace()

Add libclinic.dsl_parser module and move the following classes,
functions and variables there:

* ConverterArgs
* DSLParser
* FunctionNames
* IndentStack
* ParamState
* StateKeeper
* eval_ast_expr()
* unsupported_special_methods

Add libclinic.app module and move the Clinic class there.

Add libclinic.cli module and move the following functions there:

* create_cli()
* main()
* parse_file()
* run_clinic()

Lib/test/test_clinic.py

@@ -17,18 +17,26 @@ import unittest
 test_tools.skip_if_missing('clinic')
 with test_tools.imports_under_tool('clinic'):
     import libclinic
-    from libclinic.converters import int_converter, str_converter
+    from libclinic import ClinicError, unspecified, NULL, fail
+    from libclinic.converters import int_converter, str_converter, self_converter
     from libclinic.function import (
+        Module, Class, Function, FunctionKind, Parameter,
         permute_optional_groups, permute_right_option_groups,
         permute_left_option_groups)
     import clinic
-    from clinic import DSLParser
+    from libclinic.clanguage import CLanguage
+    from libclinic.converter import converters, legacy_converters
+    from libclinic.return_converters import return_converters, int_return_converter
+    from libclinic.block_parser import Block, BlockParser
+    from libclinic.codegen import BlockPrinter, Destination
+    from libclinic.dsl_parser import DSLParser
+    from libclinic.cli import parse_file, Clinic
 
 
 def _make_clinic(*, filename='clinic_tests', limited_capi=False):
-    clang = clinic.CLanguage(filename)
+    clang = CLanguage(filename)
-    c = clinic.Clinic(clang, filename=filename, limited_capi=limited_capi)
+    c = Clinic(clang, filename=filename, limited_capi=limited_capi)
-    c.block_parser = clinic.BlockParser('', clang)
+    c.block_parser = BlockParser('', clang)
     return c
 
 
@@ -47,7 +55,7 @@ def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None,
     if strip:
         code = code.strip()
     errmsg = re.escape(errmsg)
-    with tc.assertRaisesRegex(clinic.ClinicError, errmsg) as cm:
+    with tc.assertRaisesRegex(ClinicError, errmsg) as cm:
         parser(code)
     if filename is not None:
         tc.assertEqual(cm.exception.filename, filename)
@@ -62,12 +70,12 @@ def restore_dict(converters, old_converters):
 
 
 def save_restore_converters(testcase):
-    testcase.addCleanup(restore_dict, clinic.converters,
+    testcase.addCleanup(restore_dict, converters,
-                        clinic.converters.copy())
+                        converters.copy())
-    testcase.addCleanup(restore_dict, clinic.legacy_converters,
+    testcase.addCleanup(restore_dict, legacy_converters,
-                        clinic.legacy_converters.copy())
+                        legacy_converters.copy())
-    testcase.addCleanup(restore_dict, clinic.return_converters,
+    testcase.addCleanup(restore_dict, return_converters,
-                        clinic.return_converters.copy())
+                        return_converters.copy())
 
 
 class ClinicWholeFileTest(TestCase):
@@ -140,11 +148,11 @@ class ClinicWholeFileTest(TestCase):
         self.expect_failure(raw, err, filename="test.c", lineno=2)
 
     def test_parse_with_body_prefix(self):
-        clang = clinic.CLanguage(None)
+        clang = CLanguage(None)
         clang.body_prefix = "//"
         clang.start_line = "//[{dsl_name} start]"
         clang.stop_line = "//[{dsl_name} stop]"
-        cl = clinic.Clinic(clang, filename="test.c", limited_capi=False)
+        cl = Clinic(clang, filename="test.c", limited_capi=False)
         raw = dedent("""
             //[clinic start]
             //module test
@@ -660,8 +668,8 @@ class ParseFileUnitTest(TestCase):
         self, *, filename, expected_error, verify=True, output=None
     ):
         errmsg = re.escape(dedent(expected_error).strip())
-        with self.assertRaisesRegex(clinic.ClinicError, errmsg):
+        with self.assertRaisesRegex(ClinicError, errmsg):
-            clinic.parse_file(filename, limited_capi=False)
+            parse_file(filename, limited_capi=False)
 
     def test_parse_file_no_extension(self) -> None:
         self.expect_parsing_failure(
@@ -782,13 +790,13 @@ class ClinicLinearFormatTest(TestCase):
 
     def test_text_before_block_marker(self):
         regex = re.escape("found before '{marker}'")
-        with self.assertRaisesRegex(clinic.ClinicError, regex):
+        with self.assertRaisesRegex(ClinicError, regex):
             libclinic.linear_format("no text before marker for you! {marker}",
                                     marker="not allowed!")
 
     def test_text_after_block_marker(self):
         regex = re.escape("found after '{marker}'")
-        with self.assertRaisesRegex(clinic.ClinicError, regex):
+        with self.assertRaisesRegex(ClinicError, regex):
             libclinic.linear_format("{marker} no text after marker for you!",
                                     marker="not allowed!")
 
@@ -810,10 +818,10 @@ class CopyParser:
 
 class ClinicBlockParserTest(TestCase):
     def _test(self, input, output):
-        language = clinic.CLanguage(None)
+        language = CLanguage(None)
 
-        blocks = list(clinic.BlockParser(input, language))
+        blocks = list(BlockParser(input, language))
-        writer = clinic.BlockPrinter(language)
+        writer = BlockPrinter(language)
         c = _make_clinic()
         for block in blocks:
             writer.print_block(block, limited_capi=c.limited_capi, header_includes=c.includes)
@@ -841,8 +849,8 @@ xyz
 """)
 
     def _test_clinic(self, input, output):
-        language = clinic.CLanguage(None)
+        language = CLanguage(None)
-        c = clinic.Clinic(language, filename="file", limited_capi=False)
+        c = Clinic(language, filename="file", limited_capi=False)
         c.parsers['inert'] = InertParser(c)
         c.parsers['copy'] = CopyParser(c)
         computed = c.parse(input)
@@ -875,7 +883,7 @@ class ClinicParserTest(TestCase):
     def parse(self, text):
         c = _make_clinic()
         parser = DSLParser(c)
-        block = clinic.Block(text)
+        block = Block(text)
         parser.parse(block)
         return block
 
@@ -883,8 +891,8 @@ class ClinicParserTest(TestCase):
         block = self.parse(text)
         s = block.signatures
         self.assertEqual(len(s), signatures_in_block)
-        assert isinstance(s[0], clinic.Module)
+        assert isinstance(s[0], Module)
-        assert isinstance(s[function_index], clinic.Function)
+        assert isinstance(s[function_index], Function)
         return s[function_index]
 
     def expect_failure(self, block, err, *,
@@ -899,7 +907,7 @@ class ClinicParserTest(TestCase):
 
     def test_trivial(self):
         parser = DSLParser(_make_clinic())
-        block = clinic.Block("""
+        block = Block("""
             module os
             os.access
         """)
@@ -1188,7 +1196,7 @@ class ClinicParserTest(TestCase):
             Function 'stat' has an invalid parameter declaration:
             \s+'invalid syntax: int = 42'
         """).strip()
-        with self.assertRaisesRegex(clinic.ClinicError, err):
+        with self.assertRaisesRegex(ClinicError, err):
             self.parse_function(block)
 
     def test_param_default_invalid_syntax(self):
@@ -1220,7 +1228,7 @@ class ClinicParserTest(TestCase):
             module os
             os.stat -> int
         """)
-        self.assertIsInstance(function.return_converter, clinic.int_return_converter)
+        self.assertIsInstance(function.return_converter, int_return_converter)
 
     def test_return_converter_invalid_syntax(self):
         block = """
@@ -2036,7 +2044,7 @@ class ClinicParserTest(TestCase):
         parser = DSLParser(_make_clinic())
         parser.flag = False
         parser.directives['setflag'] = lambda : setattr(parser, 'flag', True)
-        block = clinic.Block("setflag")
+        block = Block("setflag")
         parser.parse(block)
         self.assertTrue(parser.flag)
 
@@ -2301,14 +2309,14 @@ class ClinicParserTest(TestCase):
 
     def test_scaffolding(self):
         # test repr on special values
-        self.assertEqual(repr(clinic.unspecified), '<Unspecified>')
+        self.assertEqual(repr(unspecified), '<Unspecified>')
-        self.assertEqual(repr(clinic.NULL), '<Null>')
+        self.assertEqual(repr(NULL), '<Null>')
 
         # test that fail fails
         with support.captured_stdout() as stdout:
             errmsg = 'The igloos are melting'
-            with self.assertRaisesRegex(clinic.ClinicError, errmsg) as cm:
+            with self.assertRaisesRegex(ClinicError, errmsg) as cm:
-                clinic.fail(errmsg, filename='clown.txt', line_number=69)
+                fail(errmsg, filename='clown.txt', line_number=69)
             exc = cm.exception
             self.assertEqual(exc.filename, 'clown.txt')
             self.assertEqual(exc.lineno, 69)
@@ -3998,15 +4006,15 @@ class FormatHelperTests(unittest.TestCase):
 
 class ClinicReprTests(unittest.TestCase):
     def test_Block_repr(self):
-        block = clinic.Block("foo")
+        block = Block("foo")
         expected_repr = "<clinic.Block 'text' input='foo' output=None>"
         self.assertEqual(repr(block), expected_repr)
 
-        block2 = clinic.Block("bar", "baz", [], "eggs", "spam")
+        block2 = Block("bar", "baz", [], "eggs", "spam")
         expected_repr_2 = "<clinic.Block 'baz' input='bar' output='eggs'>"
         self.assertEqual(repr(block2), expected_repr_2)
 
-        block3 = clinic.Block(
+        block3 = Block(
             input="longboi_" * 100,
             dsl_name="wow_so_long",
             signatures=[],
@@ -4021,47 +4029,47 @@ class ClinicReprTests(unittest.TestCase):
     def test_Destination_repr(self):
         c = _make_clinic()
 
-        destination = clinic.Destination(
+        destination = Destination(
             "foo", type="file", clinic=c, args=("eggs",)
         )
         self.assertEqual(
             repr(destination), "<clinic.Destination 'foo' type='file' file='eggs'>"
         )
 
-        destination2 = clinic.Destination("bar", type="buffer", clinic=c)
+        destination2 = Destination("bar", type="buffer", clinic=c)
         self.assertEqual(repr(destination2), "<clinic.Destination 'bar' type='buffer'>")
 
     def test_Module_repr(self):
-        module = clinic.Module("foo", _make_clinic())
+        module = Module("foo", _make_clinic())
         self.assertRegex(repr(module), r"<clinic.Module 'foo' at \d+>")
 
     def test_Class_repr(self):
-        cls = clinic.Class("foo", _make_clinic(), None, 'some_typedef', 'some_type_object')
+        cls = Class("foo", _make_clinic(), None, 'some_typedef', 'some_type_object')
         self.assertRegex(repr(cls), r"<clinic.Class 'foo' at \d+>")
 
     def test_FunctionKind_repr(self):
         self.assertEqual(
-            repr(clinic.FunctionKind.INVALID), "<clinic.FunctionKind.INVALID>"
+            repr(FunctionKind.INVALID), "<clinic.FunctionKind.INVALID>"
         )
         self.assertEqual(
-            repr(clinic.FunctionKind.CLASS_METHOD), "<clinic.FunctionKind.CLASS_METHOD>"
+            repr(FunctionKind.CLASS_METHOD), "<clinic.FunctionKind.CLASS_METHOD>"
         )
 
     def test_Function_and_Parameter_reprs(self):
-        function = clinic.Function(
+        function = Function(
             name='foo',
             module=_make_clinic(),
             cls=None,
             c_basename=None,
             full_name='foofoo',
-            return_converter=clinic.int_return_converter(),
+            return_converter=int_return_converter(),
-            kind=clinic.FunctionKind.METHOD_INIT,
+            kind=FunctionKind.METHOD_INIT,
             coexist=False
         )
         self.assertEqual(repr(function), "<clinic.Function 'foo'>")
 
-        converter = clinic.self_converter('bar', 'bar', function)
+        converter = self_converter('bar', 'bar', function)
-        parameter = clinic.Parameter(
+        parameter = Parameter(
             "bar",
             kind=inspect.Parameter.POSITIONAL_OR_KEYWORD,
             function=function,

Tools/clinic/libclinic/app.py

@@ -0,0 +1,297 @@
+from __future__ import annotations
+import os
+
+from collections.abc import Callable, Sequence
+from typing import Any, TYPE_CHECKING
+
+
+import libclinic
+from libclinic import fail, warn
+from libclinic.function import Class
+from libclinic.block_parser import Block, BlockParser
+from libclinic.crenderdata import Include
+from libclinic.codegen import BlockPrinter, Destination
+from libclinic.parser import Parser, PythonParser
+from libclinic.dsl_parser import DSLParser
+if TYPE_CHECKING:
+    from libclinic.clanguage import CLanguage
+    from libclinic.function import (
+        Module, Function, ClassDict, ModuleDict)
+    from libclinic.codegen import DestinationDict
+
+
+# maps strings to callables.
+# the callable should return an object
+# that implements the clinic parser
+# interface (__init__ and parse).
+#
+# example parsers:
+#   "clinic", handles the Clinic DSL
+#   "python", handles running Python code
+#
+parsers: dict[str, Callable[[Clinic], Parser]] = {
+    'clinic': DSLParser,
+    'python': PythonParser,
+}
+
+
+class Clinic:
+
+    presets_text = """
+preset block
+everything block
+methoddef_ifndef buffer 1
+docstring_prototype suppress
+parser_prototype suppress
+cpp_if suppress
+cpp_endif suppress
+
+preset original
+everything block
+methoddef_ifndef buffer 1
+docstring_prototype suppress
+parser_prototype suppress
+cpp_if suppress
+cpp_endif suppress
+
+preset file
+everything file
+methoddef_ifndef file 1
+docstring_prototype suppress
+parser_prototype suppress
+impl_definition block
+
+preset buffer
+everything buffer
+methoddef_ifndef buffer 1
+impl_definition block
+docstring_prototype suppress
+impl_prototype suppress
+parser_prototype suppress
+
+preset partial-buffer
+everything buffer
+methoddef_ifndef buffer 1
+docstring_prototype block
+impl_prototype suppress
+methoddef_define block
+parser_prototype block
+impl_definition block
+
+"""
+
+    def __init__(
+        self,
+        language: CLanguage,
+        printer: BlockPrinter | None = None,
+        *,
+        filename: str,
+        limited_capi: bool,
+        verify: bool = True,
+    ) -> None:
+        # maps strings to Parser objects.
+        # (instantiated from the "parsers" global.)
+        self.parsers: dict[str, Parser] = {}
+        self.language: CLanguage = language
+        if printer:
+            fail("Custom printers are broken right now")
+        self.printer = printer or BlockPrinter(language)
+        self.verify = verify
+        self.limited_capi = limited_capi
+        self.filename = filename
+        self.modules: ModuleDict = {}
+        self.classes: ClassDict = {}
+        self.functions: list[Function] = []
+        # dict: include name => Include instance
+        self.includes: dict[str, Include] = {}
+
+        self.line_prefix = self.line_suffix = ''
+
+        self.destinations: DestinationDict = {}
+        self.add_destination("block", "buffer")
+        self.add_destination("suppress", "suppress")
+        self.add_destination("buffer", "buffer")
+        if filename:
+            self.add_destination("file", "file", "{dirname}/clinic/{basename}.h")
+
+        d = self.get_destination_buffer
+        self.destination_buffers = {
+            'cpp_if': d('file'),
+            'docstring_prototype': d('suppress'),
+            'docstring_definition': d('file'),
+            'methoddef_define': d('file'),
+            'impl_prototype': d('file'),
+            'parser_prototype': d('suppress'),
+            'parser_definition': d('file'),
+            'cpp_endif': d('file'),
+            'methoddef_ifndef': d('file', 1),
+            'impl_definition': d('block'),
+        }
+
+        DestBufferType = dict[str, list[str]]
+        DestBufferList = list[DestBufferType]
+
+        self.destination_buffers_stack: DestBufferList = []
+        self.ifndef_symbols: set[str] = set()
+
+        self.presets: dict[str, dict[Any, Any]] = {}
+        preset = None
+        for line in self.presets_text.strip().split('\n'):
+            line = line.strip()
+            if not line:
+                continue
+            name, value, *options = line.split()
+            if name == 'preset':
+                self.presets[value] = preset = {}
+                continue
+
+            if len(options):
+                index = int(options[0])
+            else:
+                index = 0
+            buffer = self.get_destination_buffer(value, index)
+
+            if name == 'everything':
+                for name in self.destination_buffers:
+                    preset[name] = buffer
+                continue
+
+            assert name in self.destination_buffers
+            preset[name] = buffer
+
+    def add_include(self, name: str, reason: str,
+                    *, condition: str | None = None) -> None:
+        try:
+            existing = self.includes[name]
+        except KeyError:
+            pass
+        else:
+            if existing.condition and not condition:
+                # If the previous include has a condition and the new one is
+                # unconditional, override the include.
+                pass
+            else:
+                # Already included, do nothing. Only mention a single reason,
+                # no need to list all of them.
+                return
+
+        self.includes[name] = Include(name, reason, condition)
+
+    def add_destination(
+        self,
+        name: str,
+        type: str,
+        *args: str
+    ) -> None:
+        if name in self.destinations:
+            fail(f"Destination already exists: {name!r}")
+        self.destinations[name] = Destination(name, type, self, args)
+
+    def get_destination(self, name: str) -> Destination:
+        d = self.destinations.get(name)
+        if not d:
+            fail(f"Destination does not exist: {name!r}")
+        return d
+
+    def get_destination_buffer(
+        self,
+        name: str,
+        item: int = 0
+    ) -> list[str]:
+        d = self.get_destination(name)
+        return d.buffers[item]
+
+    def parse(self, input: str) -> str:
+        printer = self.printer
+        self.block_parser = BlockParser(input, self.language, verify=self.verify)
+        for block in self.block_parser:
+            dsl_name = block.dsl_name
+            if dsl_name:
+                if dsl_name not in self.parsers:
+                    assert dsl_name in parsers, f"No parser to handle {dsl_name!r} block."
+                    self.parsers[dsl_name] = parsers[dsl_name](self)
+                parser = self.parsers[dsl_name]
+                parser.parse(block)
+            printer.print_block(block,
+                                limited_capi=self.limited_capi,
+                                header_includes=self.includes)
+
+        # these are destinations not buffers
+        for name, destination in self.destinations.items():
+            if destination.type == 'suppress':
+                continue
+            output = destination.dump()
+
+            if output:
+                block = Block("", dsl_name="clinic", output=output)
+
+                if destination.type == 'buffer':
+                    block.input = "dump " + name + "\n"
+                    warn("Destination buffer " + repr(name) + " not empty at end of file, emptying.")
+                    printer.write("\n")
+                    printer.print_block(block,
+                                        limited_capi=self.limited_capi,
+                                        header_includes=self.includes)
+                    continue
+
+                if destination.type == 'file':
+                    try:
+                        dirname = os.path.dirname(destination.filename)
+                        try:
+                            os.makedirs(dirname)
+                        except FileExistsError:
+                            if not os.path.isdir(dirname):
+                                fail(f"Can't write to destination "
+                                     f"{destination.filename!r}; "
+                                     f"can't make directory {dirname!r}!")
+                        if self.verify:
+                            with open(destination.filename) as f:
+                                parser_2 = BlockParser(f.read(), language=self.language)
+                                blocks = list(parser_2)
+                                if (len(blocks) != 1) or (blocks[0].input != 'preserve\n'):
+                                    fail(f"Modified destination file "
+                                         f"{destination.filename!r}; not overwriting!")
+                    except FileNotFoundError:
+                        pass
+
+                    block.input = 'preserve\n'
+                    printer_2 = BlockPrinter(self.language)
+                    printer_2.print_block(block,
+                                          core_includes=True,
+                                          limited_capi=self.limited_capi,
+                                          header_includes=self.includes)
+                    libclinic.write_file(destination.filename,
+                                         printer_2.f.getvalue())
+                    continue
+
+        return printer.f.getvalue()
+
+    def _module_and_class(
+        self, fields: Sequence[str]
+    ) -> tuple[Module | Clinic, Class | None]:
+        """
+        fields should be an iterable of field names.
+        returns a tuple of (module, class).
+        the module object could actually be self (a clinic object).
+        this function is only ever used to find the parent of where
+        a new class/module should go.
+        """
+        parent: Clinic | Module | Class = self
+        module: Clinic | Module = self
+        cls: Class | None = None
+
+        for idx, field in enumerate(fields):
+            if not isinstance(parent, Class):
+                if field in parent.modules:
+                    parent = module = parent.modules[field]
+                    continue
+            if field in parent.classes:
+                parent = cls = parent.classes[field]
+            else:
+                fullname = ".".join(fields[idx:])
+                fail(f"Parent class or module {fullname!r} does not exist.")
+
+        return module, cls
+
+    def __repr__(self) -> str:
+        return "<clinic.Clinic object>"

Tools/clinic/libclinic/clanguage.py

@@ -19,7 +19,7 @@ from libclinic.function import (
 from libclinic.converters import (
     defining_class_converter, object_converter, self_converter)
 if TYPE_CHECKING:
-    from clinic import Clinic
+    from libclinic.app import Clinic
 
 
 def declare_parser(

Tools/clinic/libclinic/cli.py

@@ -0,0 +1,231 @@
+from __future__ import annotations
+
+import argparse
+import inspect
+import os
+import re
+import sys
+from collections.abc import Callable
+from typing import NoReturn
+
+
+# Local imports.
+import libclinic
+import libclinic.cpp
+from libclinic import ClinicError
+from libclinic.language import Language, PythonLanguage
+from libclinic.block_parser import BlockParser
+from libclinic.converter import (
+    ConverterType, converters, legacy_converters)
+from libclinic.return_converters import (
+    return_converters, ReturnConverterType)
+from libclinic.clanguage import CLanguage
+from libclinic.app import Clinic
+
+
+# TODO:
+#
+# soon:
+#
+# * allow mixing any two of {positional-only, positional-or-keyword,
+#   keyword-only}
+#       * dict constructor uses positional-only and keyword-only
+#       * max and min use positional only with an optional group
+#         and keyword-only
+#
+
+
+# Match '#define Py_LIMITED_API'.
+# Match '#  define Py_LIMITED_API 0x030d0000' (without the version).
+LIMITED_CAPI_REGEX = re.compile(r'# *define +Py_LIMITED_API')
+
+
+# "extensions" maps the file extension ("c", "py") to Language classes.
+LangDict = dict[str, Callable[[str], Language]]
+extensions: LangDict = { name: CLanguage for name in "c cc cpp cxx h hh hpp hxx".split() }
+extensions['py'] = PythonLanguage
+
+
+def parse_file(
+        filename: str,
+        *,
+        limited_capi: bool,
+        output: str | None = None,
+        verify: bool = True,
+) -> None:
+    if not output:
+        output = filename
+
+    extension = os.path.splitext(filename)[1][1:]
+    if not extension:
+        raise ClinicError(f"Can't extract file type for file {filename!r}")
+
+    try:
+        language = extensions[extension](filename)
+    except KeyError:
+        raise ClinicError(f"Can't identify file type for file {filename!r}")
+
+    with open(filename, encoding="utf-8") as f:
+        raw = f.read()
+
+    # exit quickly if there are no clinic markers in the file
+    find_start_re = BlockParser("", language).find_start_re
+    if not find_start_re.search(raw):
+        return
+
+    if LIMITED_CAPI_REGEX.search(raw):
+        limited_capi = True
+
+    assert isinstance(language, CLanguage)
+    clinic = Clinic(language,
+                    verify=verify,
+                    filename=filename,
+                    limited_capi=limited_capi)
+    cooked = clinic.parse(raw)
+
+    libclinic.write_file(output, cooked)
+
+
+def create_cli() -> argparse.ArgumentParser:
+    cmdline = argparse.ArgumentParser(
+        prog="clinic.py",
+        description="""Preprocessor for CPython C files.
+
+The purpose of the Argument Clinic is automating all the boilerplate involved
+with writing argument parsing code for builtins and providing introspection
+signatures ("docstrings") for CPython builtins.
+
+For more information see https://devguide.python.org/development-tools/clinic/""")
+    cmdline.add_argument("-f", "--force", action='store_true',
+                         help="force output regeneration")
+    cmdline.add_argument("-o", "--output", type=str,
+                         help="redirect file output to OUTPUT")
+    cmdline.add_argument("-v", "--verbose", action='store_true',
+                         help="enable verbose mode")
+    cmdline.add_argument("--converters", action='store_true',
+                         help=("print a list of all supported converters "
+                               "and return converters"))
+    cmdline.add_argument("--make", action='store_true',
+                         help="walk --srcdir to run over all relevant files")
+    cmdline.add_argument("--srcdir", type=str, default=os.curdir,
+                         help="the directory tree to walk in --make mode")
+    cmdline.add_argument("--exclude", type=str, action="append",
+                         help=("a file to exclude in --make mode; "
+                               "can be given multiple times"))
+    cmdline.add_argument("--limited", dest="limited_capi", action='store_true',
+                         help="use the Limited C API")
+    cmdline.add_argument("filename", metavar="FILE", type=str, nargs="*",
+                         help="the list of files to process")
+    return cmdline
+
+
+def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None:
+    if ns.converters:
+        if ns.filename:
+            parser.error(
+                "can't specify --converters and a filename at the same time"
+            )
+        AnyConverterType = ConverterType | ReturnConverterType
+        converter_list: list[tuple[str, AnyConverterType]] = []
+        return_converter_list: list[tuple[str, AnyConverterType]] = []
+
+        for name, converter in converters.items():
+            converter_list.append((
+                name,
+                converter,
+            ))
+        for name, return_converter in return_converters.items():
+            return_converter_list.append((
+                name,
+                return_converter
+            ))
+
+        print()
+
+        print("Legacy converters:")
+        legacy = sorted(legacy_converters)
+        print('    ' + ' '.join(c for c in legacy if c[0].isupper()))
+        print('    ' + ' '.join(c for c in legacy if c[0].islower()))
+        print()
+
+        for title, attribute, ids in (
+            ("Converters", 'converter_init', converter_list),
+            ("Return converters", 'return_converter_init', return_converter_list),
+        ):
+            print(title + ":")
+
+            ids.sort(key=lambda item: item[0].lower())
+            longest = -1
+            for name, _ in ids:
+                longest = max(longest, len(name))
+
+            for name, cls in ids:
+                callable = getattr(cls, attribute, None)
+                if not callable:
+                    continue
+                signature = inspect.signature(callable)
+                parameters = []
+                for parameter_name, parameter in signature.parameters.items():
+                    if parameter.kind == inspect.Parameter.KEYWORD_ONLY:
+                        if parameter.default != inspect.Parameter.empty:
+                            s = f'{parameter_name}={parameter.default!r}'
+                        else:
+                            s = parameter_name
+                        parameters.append(s)
+                print('    {}({})'.format(name, ', '.join(parameters)))
+            print()
+        print("All converters also accept (c_default=None, py_default=None, annotation=None).")
+        print("All return converters also accept (py_default=None).")
+        return
+
+    if ns.make:
+        if ns.output or ns.filename:
+            parser.error("can't use -o or filenames with --make")
+        if not ns.srcdir:
+            parser.error("--srcdir must not be empty with --make")
+        if ns.exclude:
+            excludes = [os.path.join(ns.srcdir, f) for f in ns.exclude]
+            excludes = [os.path.normpath(f) for f in excludes]
+        else:
+            excludes = []
+        for root, dirs, files in os.walk(ns.srcdir):
+            for rcs_dir in ('.svn', '.git', '.hg', 'build', 'externals'):
+                if rcs_dir in dirs:
+                    dirs.remove(rcs_dir)
+            for filename in files:
+                # handle .c, .cpp and .h files
+                if not filename.endswith(('.c', '.cpp', '.h')):
+                    continue
+                path = os.path.join(root, filename)
+                path = os.path.normpath(path)
+                if path in excludes:
+                    continue
+                if ns.verbose:
+                    print(path)
+                parse_file(path,
+                           verify=not ns.force, limited_capi=ns.limited_capi)
+        return
+
+    if not ns.filename:
+        parser.error("no input files")
+
+    if ns.output and len(ns.filename) > 1:
+        parser.error("can't use -o with multiple filenames")
+
+    for filename in ns.filename:
+        if ns.verbose:
+            print(filename)
+        parse_file(filename, output=ns.output,
+                   verify=not ns.force, limited_capi=ns.limited_capi)
+
+
+def main(argv: list[str] | None = None) -> NoReturn:
+    parser = create_cli()
+    args = parser.parse_args(argv)
+    try:
+        run_clinic(parser, args)
+    except ClinicError as exc:
+        sys.stderr.write(exc.report())
+        sys.exit(1)
+    else:
+        sys.exit(0)

Tools/clinic/libclinic/codegen.py

@@ -3,14 +3,14 @@ import dataclasses as dc
 import io
 import os
 from typing import Final, TYPE_CHECKING
-if TYPE_CHECKING:
-    from clinic import Clinic
 
 import libclinic
 from libclinic import fail
 from libclinic.crenderdata import Include
 from libclinic.language import Language
 from libclinic.block_parser import Block
+if TYPE_CHECKING:
+    from libclinic.app import Clinic
 
 
 @dc.dataclass(slots=True)
@@ -185,3 +185,6 @@ class Destination:
 
     def dump(self) -> str:
         return self.buffers.dump()
+
+
+DestinationDict = dict[str, Destination]

Tools/clinic/libclinic/function.py

@@ -7,10 +7,10 @@ import inspect
 from collections.abc import Iterable, Iterator, Sequence
 from typing import Final, Any, TYPE_CHECKING
 if TYPE_CHECKING:
-    from clinic import Clinic
     from libclinic.converter import CConverter
     from libclinic.converters import self_converter
     from libclinic.return_converters import CReturnConverter
+    from libclinic.app import Clinic
 
 from libclinic import VersionTuple, unspecified
 

Tools/clinic/libclinic/language.py

@@ -11,7 +11,7 @@ from libclinic.function import (
     Module, Class, Function)
 
 if typing.TYPE_CHECKING:
-    from clinic import Clinic
+    from libclinic.app import Clinic
 
 
 class Language(metaclass=abc.ABCMeta):

Tools/clinic/libclinic/parser.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+import contextlib
+import functools
+import io
+from types import NoneType
+from typing import Any, Protocol, TYPE_CHECKING
+
+from libclinic import unspecified
+from libclinic.block_parser import Block
+from libclinic.converter import CConverter, converters
+from libclinic.converters import buffer, robuffer, rwbuffer
+from libclinic.return_converters import CReturnConverter, return_converters
+if TYPE_CHECKING:
+    from libclinic.app import Clinic
+
+
+class Parser(Protocol):
+    def __init__(self, clinic: Clinic) -> None: ...
+    def parse(self, block: Block) -> None: ...
+
+
+@functools.cache
+def _create_parser_base_namespace() -> dict[str, Any]:
+    ns = dict(
+        CConverter=CConverter,
+        CReturnConverter=CReturnConverter,
+        buffer=buffer,
+        robuffer=robuffer,
+        rwbuffer=rwbuffer,
+        unspecified=unspecified,
+        NoneType=NoneType,
+    )
+    for name, converter in converters.items():
+        ns[f'{name}_converter'] = converter
+    for name, return_converter in return_converters.items():
+        ns[f'{name}_return_converter'] = return_converter
+    return ns
+
+
+def create_parser_namespace() -> dict[str, Any]:
+    base_namespace = _create_parser_base_namespace()
+    return base_namespace.copy()
+
+
+class PythonParser:
+    def __init__(self, clinic: Clinic) -> None:
+        pass
+
+    def parse(self, block: Block) -> None:
+        namespace = create_parser_namespace()
+        with contextlib.redirect_stdout(io.StringIO()) as s:
+            exec(block.input, namespace)
+            block.output = s.getvalue()