GH-121970: Rewrite the C-API annotations extension (#121985)

Co-authored-by: Petr Viktorin <encukou@gmail.com>

Doc/.ruff.toml

@@ -5,7 +5,6 @@ line-length = 79
 extend-exclude = [
     "includes/*",
     # Temporary exclusions:
-    "tools/extensions/c_annotations.py",
     "tools/extensions/escape4chm.py",
     "tools/extensions/patchlevel.py",
     "tools/extensions/pyspecific.py",

Doc/c-api/module.rst

@@ -342,7 +342,8 @@ The available slot types are:
    The *value* pointer of this slot must point to a function of the signature:
 
    .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def)
-      :noindex:
+      :no-index-entry:
+      :no-contents-entry:
 
    The function receives a :py:class:`~importlib.machinery.ModuleSpec`
    instance, as defined in :PEP:`451`, and the module definition.
@@ -377,7 +378,8 @@ The available slot types are:
    The signature of the function is:
 
    .. c:function:: int exec_module(PyObject* module)
-      :noindex:
+      :no-index-entry:
+      :no-contents-entry:
 
    If multiple ``Py_mod_exec`` slots are specified, they are processed in the
    order they appear in the *m_slots* array.

Doc/conf.py

@@ -589,14 +589,16 @@ extlinks = {
 }
 extlinks_detect_hardcoded_links = True
 
-# Options for extensions
+# Options for c_annotations
-# ----------------------
+# -------------------------
 
 # Relative filename of the data files
 refcount_file = 'data/refcounts.dat'
 stable_abi_file = 'data/stable_abi.dat'
 
-# sphinxext-opengraph config
+# Options for sphinxext-opengraph
+# -------------------------------
+
 ogp_site_url = 'https://docs.python.org/3/'
 ogp_site_name = 'Python documentation'
 ogp_image = '_static/og-image.png'

Doc/tools/extensions/c_annotations.py

@@ -1,226 +1,305 @@
-"""
+"""Support annotations for C API elements.
-    c_annotations.py
-    ~~~~~~~~~~~~~~~~
 
-    Supports annotations for C API elements:
+* Reference count annotations for C API functions.
+* Stable ABI annotations
+* Limited API annotations
 
-    * reference count annotations for C API functions.  Based on
+Configuration:
-      refcount.py and anno-api.py in the old Python documentation tools.
+* Set ``refcount_file`` to the path to the reference count data file.
-
+* Set ``stable_abi_file`` to the path to stable ABI list.
-    * stable API annotations
-
-    Usage:
-    * Set the `refcount_file` config value to the path to the reference
-    count data file.
-    * Set the `stable_abi_file` config value to the path to stable ABI list.
-
-    :copyright: Copyright 2007-2014 by Georg Brandl.
-    :license: Python license.
 """
 
-from os import path
+from __future__ import annotations
-from docutils import nodes
+
-from docutils.parsers.rst import directives
-from docutils.parsers.rst import Directive
-from docutils.statemachine import StringList
-from sphinx.locale import _ as sphinx_gettext
 import csv
+import dataclasses
+from pathlib import Path
+from typing import TYPE_CHECKING
 
+import sphinx
+from docutils import nodes
+from docutils.statemachine import StringList
 from sphinx import addnodes
-from sphinx.domains.c import CObject
+from sphinx.locale import _ as sphinx_gettext
+from sphinx.util.docutils import SphinxDirective
 
+if TYPE_CHECKING:
+    from sphinx.application import Sphinx
+    from sphinx.util.typing import ExtensionMetadata
 
-REST_ROLE_MAP = {
+ROLE_TO_OBJECT_TYPE = {
-    'function': 'func',
+    "func": "function",
-    'macro': 'macro',
+    "macro": "macro",
-    'member': 'member',
+    "member": "member",
-    'type': 'type',
+    "type": "type",
-    'var': 'data',
+    "data": "var",
 }
 
 
-class RCEntry:
+@dataclasses.dataclass(slots=True)
-    def __init__(self, name):
+class RefCountEntry:
-        self.name = name
+    # Name of the function.
-        self.args = []
+    name: str
-        self.result_type = ''
+    # List of (argument name, type, refcount effect) tuples.
-        self.result_refs = None
+    # (Currently not used. If it was, a dataclass might work better.)
+    args: list = dataclasses.field(default_factory=list)
+    # Return type of the function.
+    result_type: str = ""
+    # Reference count effect for the return value.
+    result_refs: int | None = None
 
 
-class Annotations:
+@dataclasses.dataclass(frozen=True, slots=True)
-    def __init__(self, refcount_filename, stable_abi_file):
+class StableABIEntry:
-        self.refcount_data = {}
+    # Role of the object.
-        with open(refcount_filename, encoding='utf8') as fp:
+    # Source: Each [item_kind] in stable_abi.toml is mapped to a C Domain role.
-            for line in fp:
+    role: str
+    # Name of the object.
+    # Source: [<item_kind>.*] in stable_abi.toml.
+    name: str
+    # Version when the object was added to the stable ABI.
+    # (Source: [<item_kind>.*.added] in stable_abi.toml.
+    added: str
+    # An explananatory blurb for the ifdef.
+    # Source: ``feature_macro.*.doc`` in stable_abi.toml.
+    ifdef_note: str
+    # Defines how much of the struct is exposed. Only relevant for structs.
+    # Source: [<item_kind>.*.struct_abi_kind] in stable_abi.toml.
+    struct_abi_kind: str
+
+
+def read_refcount_data(refcount_filename: Path) -> dict[str, RefCountEntry]:
+    refcount_data = {}
+    refcounts = refcount_filename.read_text(encoding="utf8")
+    for line in refcounts.splitlines():
         line = line.strip()
-                if line[:1] in ("", "#"):
+        if not line or line.startswith("#"):
             # blank lines and comments
             continue
+
+        # Each line is of the form
+        # function ':' type ':' [param name] ':' [refcount effect] ':' [comment]
         parts = line.split(":", 4)
         if len(parts) != 5:
             raise ValueError(f"Wrong field count in {line!r}")
-                function, type, arg, refcount, comment = parts
+        function, type, arg, refcount, _comment = parts
+
         # Get the entry, creating it if needed:
         try:
-                    entry = self.refcount_data[function]
+            entry = refcount_data[function]
         except KeyError:
-                    entry = self.refcount_data[function] = RCEntry(function)
+            entry = refcount_data[function] = RefCountEntry(function)
         if not refcount or refcount == "null":
             refcount = None
         else:
             refcount = int(refcount)
-                # Update the entry with the new parameter or the result
+        # Update the entry with the new parameter
-                # information.
+        # or the result information.
         if arg:
             entry.args.append((arg, type, refcount))
         else:
             entry.result_type = type
             entry.result_refs = refcount
 
-        self.stable_abi_data = {}
+    return refcount_data
-        with open(stable_abi_file, encoding='utf8') as fp:
-            for record in csv.DictReader(fp):
-                name = record['name']
-                self.stable_abi_data[name] = record
 
-    def add_annotations(self, app, doctree):
+
+def read_stable_abi_data(stable_abi_file: Path) -> dict[str, StableABIEntry]:
+    stable_abi_data = {}
+    with open(stable_abi_file, encoding="utf8") as fp:
+        for record in csv.DictReader(fp):
+            name = record["name"]
+            stable_abi_data[name] = StableABIEntry(**record)
+
+    return stable_abi_data
+
+
+def add_annotations(app: Sphinx, doctree: nodes.document) -> None:
+    state = app.env.domaindata["c_annotations"]
+    refcount_data = state["refcount_data"]
+    stable_abi_data = state["stable_abi_data"]
     for node in doctree.findall(addnodes.desc_content):
         par = node.parent
-            if par['domain'] != 'c':
+        if par["domain"] != "c":
             continue
-            if not par[0].has_key('ids') or not par[0]['ids']:
+        if not par[0].get("ids", None):
             continue
-            name = par[0]['ids'][0]
+        name = par[0]["ids"][0]
         if name.startswith("c."):
             name = name[2:]
 
-            objtype = par['objtype']
+        objtype = par["objtype"]
 
-            # Stable ABI annotation. These have two forms:
+        # Stable ABI annotation.
-            #   Part of the [Stable ABI](link).
+        if record := stable_abi_data.get(name):
-            #   Part of the [Stable ABI](link) since version X.Y.
+            if ROLE_TO_OBJECT_TYPE[record.role] != objtype:
-            # For structs, there's some more info in the message:
+                msg = (
-            #   Part of the [Limited API](link) (as an opaque struct).
+                    f"Object type mismatch in limited API annotation for {name}: "
-            #   Part of the [Stable ABI](link) (including all members).
+                    f"{ROLE_TO_OBJECT_TYPE[record.role]!r} != {objtype!r}"
-            #   Part of the [Limited API](link) (Only some members are part
+                )
-            #       of the stable ABI.).
+                raise ValueError(msg)
-            # ... all of which can have "since version X.Y" appended.
+            annotation = _stable_abi_annotation(record)
-            record = self.stable_abi_data.get(name)
+            node.insert(0, annotation)
-            if record:
+
-                if record['role'] != objtype:
+        # Unstable API annotation.
-                    raise ValueError(
+        if name.startswith("PyUnstable"):
-                        f"Object type mismatch in limited API annotation "
+            annotation = _unstable_api_annotation()
-                        f"for {name}: {record['role']!r} != {objtype!r}")
+            node.insert(0, annotation)
-                stable_added = record['added']
+
-                message = sphinx_gettext('Part of the')
+        # Return value annotation
+        if objtype != "function":
+            continue
+        if name not in refcount_data:
+            continue
+        entry = refcount_data[name]
+        if not entry.result_type.endswith("Object*"):
+            continue
+        annotation = _return_value_annotation(entry.result_refs)
+        node.insert(0, annotation)
+
+
+def _stable_abi_annotation(record: StableABIEntry) -> nodes.emphasis:
+    """Create the Stable ABI annotation.
+
+    These have two forms:
+      Part of the `Stable ABI <link>`_.
+      Part of the `Stable ABI <link>`_ since version X.Y.
+    For structs, there's some more info in the message:
+      Part of the `Limited API <link>`_ (as an opaque struct).
+      Part of the `Stable ABI <link>`_ (including all members).
+      Part of the `Limited API <link>`_ (Only some members are part
+          of the stable ABI.).
+    ... all of which can have "since version X.Y" appended.
+    """
+    stable_added = record.added
+    message = sphinx_gettext("Part of the")
     message = message.center(len(message) + 2)
-                emph_node = nodes.emphasis(message, message,
+    emph_node = nodes.emphasis(message, message, classes=["stableabi"])
-                                           classes=['stableabi'])
     ref_node = addnodes.pending_xref(
-                    'Stable ABI', refdomain="std", reftarget='stable',
+        "Stable ABI",
-                    reftype='ref', refexplicit="False")
+        refdomain="std",
-                struct_abi_kind = record['struct_abi_kind']
+        reftarget="stable",
-                if struct_abi_kind in {'opaque', 'members'}:
+        reftype="ref",
-                    ref_node += nodes.Text(sphinx_gettext('Limited API'))
+        refexplicit="False",
+    )
+    struct_abi_kind = record.struct_abi_kind
+    if struct_abi_kind in {"opaque", "members"}:
+        ref_node += nodes.Text(sphinx_gettext("Limited API"))
     else:
-                    ref_node += nodes.Text(sphinx_gettext('Stable ABI'))
+        ref_node += nodes.Text(sphinx_gettext("Stable ABI"))
     emph_node += ref_node
-                if struct_abi_kind == 'opaque':
+    if struct_abi_kind == "opaque":
-                    emph_node += nodes.Text(' ' + sphinx_gettext('(as an opaque struct)'))
+        emph_node += nodes.Text(" " + sphinx_gettext("(as an opaque struct)"))
-                elif struct_abi_kind == 'full-abi':
+    elif struct_abi_kind == "full-abi":
-                    emph_node += nodes.Text(' ' + sphinx_gettext('(including all members)'))
+        emph_node += nodes.Text(
-                if record['ifdef_note']:
+            " " + sphinx_gettext("(including all members)")
-                    emph_node += nodes.Text(' ' + record['ifdef_note'])
+        )
-                if stable_added == '3.2':
+    if record.ifdef_note:
+        emph_node += nodes.Text(f" {record.ifdef_note}")
+    if stable_added == "3.2":
         # Stable ABI was introduced in 3.2.
         pass
     else:
-                    emph_node += nodes.Text(' ' + sphinx_gettext('since version %s') % stable_added)
-                emph_node += nodes.Text('.')
-                if struct_abi_kind == 'members':
         emph_node += nodes.Text(
-                        ' ' + sphinx_gettext('(Only some members are part of the stable ABI.)'))
+            " " + sphinx_gettext("since version %s") % stable_added
-                node.insert(0, emph_node)
-
-            # Unstable API annotation.
-            if name.startswith('PyUnstable'):
-                warn_node = nodes.admonition(
-                    classes=['unstable-c-api', 'warning'])
-                message = sphinx_gettext('This is') + ' '
-                emph_node = nodes.emphasis(message, message)
-                ref_node = addnodes.pending_xref(
-                    'Unstable API', refdomain="std",
-                    reftarget='unstable-c-api',
-                    reftype='ref', refexplicit="False")
-                ref_node += nodes.Text(sphinx_gettext('Unstable API'))
-                emph_node += ref_node
-                emph_node += nodes.Text(sphinx_gettext('. It may change without warning in minor releases.'))
-                warn_node += emph_node
-                node.insert(0, warn_node)
-
-            # Return value annotation
-            if objtype != 'function':
-                continue
-            entry = self.refcount_data.get(name)
-            if not entry:
-                continue
-            elif not entry.result_type.endswith("Object*"):
-                continue
-            classes = ['refcount']
-            if entry.result_refs is None:
-                rc = sphinx_gettext('Return value: Always NULL.')
-                classes.append('return_null')
-            elif entry.result_refs:
-                rc = sphinx_gettext('Return value: New reference.')
-                classes.append('return_new_ref')
-            else:
-                rc = sphinx_gettext('Return value: Borrowed reference.')
-                classes.append('return_borrowed_ref')
-            node.insert(0, nodes.emphasis(rc, rc, classes=classes))
-
-
-def init_annotations(app):
-    annotations = Annotations(
-        path.join(app.srcdir, app.config.refcount_file),
-        path.join(app.srcdir, app.config.stable_abi_file),
         )
-    app.connect('doctree-read', annotations.add_annotations)
+    emph_node += nodes.Text(".")
+    if struct_abi_kind == "members":
+        msg = " " + sphinx_gettext(
+            "(Only some members are part of the stable ABI.)"
+        )
+        emph_node += nodes.Text(msg)
+    return emph_node
 
-    class LimitedAPIList(Directive):
 
+def _unstable_api_annotation() -> nodes.admonition:
+    ref_node = addnodes.pending_xref(
+        "Unstable API",
+        nodes.Text(sphinx_gettext("Unstable API")),
+        refdomain="std",
+        reftarget="unstable-c-api",
+        reftype="ref",
+        refexplicit="False",
+    )
+    emph_node = nodes.emphasis(
+        "This is ",
+        sphinx_gettext("This is") + " ",
+        ref_node,
+        nodes.Text(
+            sphinx_gettext(
+                ". It may change without warning in minor releases."
+            )
+        ),
+    )
+    return nodes.admonition(
+        "",
+        emph_node,
+        classes=["unstable-c-api", "warning"],
+    )
+
+
+def _return_value_annotation(result_refs: int | None) -> nodes.emphasis:
+    classes = ["refcount"]
+    if result_refs is None:
+        rc = sphinx_gettext("Return value: Always NULL.")
+        classes.append("return_null")
+    elif result_refs:
+        rc = sphinx_gettext("Return value: New reference.")
+        classes.append("return_new_ref")
+    else:
+        rc = sphinx_gettext("Return value: Borrowed reference.")
+        classes.append("return_borrowed_ref")
+    return nodes.emphasis(rc, rc, classes=classes)
+
+
+class LimitedAPIList(SphinxDirective):
     has_content = False
     required_arguments = 0
     optional_arguments = 0
     final_argument_whitespace = True
 
-        def run(self):
+    def run(self) -> list[nodes.Node]:
-            content = []
+        state = self.env.domaindata["c_annotations"]
-            for record in annotations.stable_abi_data.values():
+        content = [
-                role = REST_ROLE_MAP[record['role']]
+            f"* :c:{record.role}:`{record.name}`"
-                name = record['name']
+            for record in state["stable_abi_data"].values()
-                content.append(f'* :c:{role}:`{name}`')
+        ]
-
+        node = nodes.paragraph()
-            pnode = nodes.paragraph()
+        self.state.nested_parse(StringList(content), 0, node)
-            self.state.nested_parse(StringList(content), 0, pnode)
+        return [node]
-            return [pnode]
-
-    app.add_directive('limited-api-list', LimitedAPIList)
 
 
-def setup(app):
+def init_annotations(app: Sphinx) -> None:
-    app.add_config_value('refcount_file', '', True)
+    # Using domaindata is a bit hack-ish,
-    app.add_config_value('stable_abi_file', '', True)
+    # but allows storing state without a global variable or closure.
-    app.connect('builder-inited', init_annotations)
+    app.env.domaindata["c_annotations"] = state = {}
+    state["refcount_data"] = read_refcount_data(
+        Path(app.srcdir, app.config.refcount_file)
+    )
+    state["stable_abi_data"] = read_stable_abi_data(
+        Path(app.srcdir, app.config.stable_abi_file)
+    )
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+    app.add_config_value("refcount_file", "", "env", types={str})
+    app.add_config_value("stable_abi_file", "", "env", types={str})
+    app.add_directive("limited-api-list", LimitedAPIList)
+    app.connect("builder-inited", init_annotations)
+    app.connect("doctree-read", add_annotations)
+
+    if sphinx.version_info[:2] < (7, 2):
+        from docutils.parsers.rst import directives
+        from sphinx.domains.c import CObject
 
         # monkey-patch C object...
-    CObject.option_spec = {
+        CObject.option_spec |= {
-        'noindex': directives.flag,
+            "no-index-entry": directives.flag,
-        'stableabi': directives.flag,
+            "no-contents-entry": directives.flag,
         }
-    old_handle_signature = CObject.handle_signature
 
-    def new_handle_signature(self, sig, signode):
+    return {
-        signode.parent['stableabi'] = 'stableabi' in self.options
+        "version": "1.0",
-        return old_handle_signature(self, sig, signode)
+        "parallel_read_safe": True,
-    CObject.handle_signature = new_handle_signature
+        "parallel_write_safe": True,
-    return {'version': '1.0', 'parallel_read_safe': True}
+    }

Tools/build/stable_abi.py

@@ -225,9 +225,9 @@ def gen_python3dll(manifest, args, outfile):
             key=sort_key):
         write(f'EXPORT_DATA({item.name})')
 
-REST_ROLES = {
+ITEM_KIND_TO_DOC_ROLE = {
-    'function': 'function',
+    'function': 'func',
-    'data': 'var',
+    'data': 'data',
     'struct': 'type',
     'macro': 'macro',
     # 'const': 'const',  # all undocumented
@@ -236,22 +236,28 @@ REST_ROLES = {
 
 @generator("doc_list", 'Doc/data/stable_abi.dat')
 def gen_doc_annotations(manifest, args, outfile):
-    """Generate/check the stable ABI list for documentation annotations"""
+    """Generate/check the stable ABI list for documentation annotations
+
+    See ``StableABIEntry`` in ``Doc/tools/extensions/c_annotations.py``
+    for a description of each field.
+    """
     writer = csv.DictWriter(
         outfile,
         ['role', 'name', 'added', 'ifdef_note', 'struct_abi_kind'],
         lineterminator='\n')
     writer.writeheader()
-    for item in manifest.select(REST_ROLES.keys(), include_abi_only=False):
+    kinds = set(ITEM_KIND_TO_DOC_ROLE)
+    for item in manifest.select(kinds, include_abi_only=False):
         if item.ifdef:
             ifdef_note = manifest.contents[item.ifdef].doc
         else:
             ifdef_note = None
         row = {
-            'role': REST_ROLES[item.kind],
+            'role': ITEM_KIND_TO_DOC_ROLE[item.kind],
             'name': item.name,
             'added': item.added,
-            'ifdef_note': ifdef_note}
+            'ifdef_note': ifdef_note,
+        }
         rows = [row]
         if item.kind == 'struct':
             row['struct_abi_kind'] = item.struct_abi_kind
@@ -259,7 +265,8 @@ def gen_doc_annotations(manifest, args, outfile):
                 rows.append({
                     'role': 'member',
                     'name': f'{item.name}.{member_name}',
-                    'added': item.added})
+                    'added': item.added,
+                })
         writer.writerows(rows)
 
 @generator("ctypes_test", 'Lib/test/test_stable_abi_ctypes.py')