Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

GH-121970: Rewrite the C-API annotations extension (#121985) 

Co-authored-by: Petr Viktorin <encukou@gmail.com>
This commit is contained in:
Adam Turner 2024-07-19 13:21:56 +01:00 committed by GitHub
parent 40855f3ab8
commit 22c9d9c1fc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 1158 additions and 1069 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Doc/.ruff.toml
View File

@ -5,7 +5,6 @@ line-length = 79
extend-exclude = [
"includes/*",
# Temporary exclusions:
"tools/extensions/c_annotations.py",
"tools/extensions/escape4chm.py",
"tools/extensions/patchlevel.py",
"tools/extensions/pyspecific.py",

6
Doc/c-api/module.rst
View File

@ -342,7 +342,8 @@ The available slot types are:
The *value* pointer of this slot must point to a function of the signature:
.. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def)
:noindex:
:no-index-entry:
:no-contents-entry:
The function receives a :py:class:`~importlib.machinery.ModuleSpec`
instance, as defined in :PEP:`451`, and the module definition.
@ -377,7 +378,8 @@ The available slot types are:
The signature of the function is:
.. c:function:: int exec_module(PyObject* module)
:noindex:
:no-index-entry:
:no-contents-entry:
If multiple ``Py_mod_exec`` slots are specified, they are processed in the
order they appear in the *m_slots* array.

8
Doc/conf.py
View File

@ -589,14 +589,16 @@ extlinks = {
}
extlinks_detect_hardcoded_links = True
# Options for extensions
# ----------------------
# Options for c_annotations
# -------------------------
# Relative filename of the data files
refcount_file = 'data/refcounts.dat'
stable_abi_file = 'data/stable_abi.dat'
# sphinxext-opengraph config
# Options for sphinxext-opengraph
# -------------------------------
ogp_site_url = 'https://docs.python.org/3/'
ogp_site_name = 'Python documentation'
ogp_image = '_static/og-image.png'

1700
Doc/data/stable_abi.dat generated
View File

File diff suppressed because it is too large Load Diff

489
Doc/tools/extensions/c_annotations.py
View File

@ -1,226 +1,305 @@
"""
c_annotations.py
~~~~~~~~~~~~~~~~
"""Support annotations for C API elements.
Supports annotations for C API elements:
* Reference count annotations for C API functions.
* Stable ABI annotations
* Limited API annotations
* reference count annotations for C API functions. Based on
refcount.py and anno-api.py in the old Python documentation tools.
* stable API annotations
Usage:
* Set the `refcount_file` config value to the path to the reference
count data file.
* Set the `stable_abi_file` config value to the path to stable ABI list.
:copyright: Copyright 2007-2014 by Georg Brandl.
:license: Python license.
Configuration:
* Set ``refcount_file`` to the path to the reference count data file.
* Set ``stable_abi_file`` to the path to stable ABI list.
"""
from os import path
from docutils import nodes
from docutils.parsers.rst import directives
from docutils.parsers.rst import Directive
from docutils.statemachine import StringList
from sphinx.locale import _ as sphinx_gettext
from __future__ import annotations
import csv
import dataclasses
from pathlib import Path
from typing import TYPE_CHECKING
import sphinx
from docutils import nodes
from docutils.statemachine import StringList
from sphinx import addnodes
from sphinx.domains.c import CObject
from sphinx.locale import _ as sphinx_gettext
from sphinx.util.docutils import SphinxDirective
if TYPE_CHECKING:
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
REST_ROLE_MAP = {
'function': 'func',
'macro': 'macro',
'member': 'member',
'type': 'type',
'var': 'data',
ROLE_TO_OBJECT_TYPE = {
"func": "function",
"macro": "macro",
"member": "member",
"type": "type",
"data": "var",
}
class RCEntry:
def __init__(self, name):
self.name = name
self.args = []
self.result_type = ''
self.result_refs = None
@dataclasses.dataclass(slots=True)
class RefCountEntry:
# Name of the function.
name: str
# List of (argument name, type, refcount effect) tuples.
# (Currently not used. If it was, a dataclass might work better.)
args: list = dataclasses.field(default_factory=list)
# Return type of the function.
result_type: str = ""
# Reference count effect for the return value.
result_refs: int | None = None
class Annotations:
def __init__(self, refcount_filename, stable_abi_file):
self.refcount_data = {}
with open(refcount_filename, encoding='utf8') as fp:
for line in fp:
line = line.strip()
if line[:1] in ("", "#"):
# blank lines and comments
continue
parts = line.split(":", 4)
if len(parts) != 5:
raise ValueError(f"Wrong field count in {line!r}")
function, type, arg, refcount, comment = parts
# Get the entry, creating it if needed:
try:
entry = self.refcount_data[function]
except KeyError:
entry = self.refcount_data[function] = RCEntry(function)
if not refcount or refcount == "null":
refcount = None
else:
refcount = int(refcount)
# Update the entry with the new parameter or the result
# information.
if arg:
entry.args.append((arg, type, refcount))
else:
entry.result_type = type
entry.result_refs = refcount
self.stable_abi_data = {}
with open(stable_abi_file, encoding='utf8') as fp:
for record in csv.DictReader(fp):
name = record['name']
self.stable_abi_data[name] = record
def add_annotations(self, app, doctree):
for node in doctree.findall(addnodes.desc_content):
par = node.parent
if par['domain'] != 'c':
continue
if not par[0].has_key('ids') or not par[0]['ids']:
continue
name = par[0]['ids'][0]
if name.startswith("c."):
name = name[2:]
objtype = par['objtype']
# Stable ABI annotation. These have two forms:
# Part of the [Stable ABI](link).
# Part of the [Stable ABI](link) since version X.Y.
# For structs, there's some more info in the message:
# Part of the [Limited API](link) (as an opaque struct).
# Part of the [Stable ABI](link) (including all members).
# Part of the [Limited API](link) (Only some members are part
# of the stable ABI.).
# ... all of which can have "since version X.Y" appended.
record = self.stable_abi_data.get(name)
if record:
if record['role'] != objtype:
raise ValueError(
f"Object type mismatch in limited API annotation "
f"for {name}: {record['role']!r} != {objtype!r}")
stable_added = record['added']
message = sphinx_gettext('Part of the')
message = message.center(len(message) + 2)
emph_node = nodes.emphasis(message, message,
classes=['stableabi'])
ref_node = addnodes.pending_xref(
'Stable ABI', refdomain="std", reftarget='stable',
reftype='ref', refexplicit="False")
struct_abi_kind = record['struct_abi_kind']
if struct_abi_kind in {'opaque', 'members'}:
ref_node += nodes.Text(sphinx_gettext('Limited API'))
else:
ref_node += nodes.Text(sphinx_gettext('Stable ABI'))
emph_node += ref_node
if struct_abi_kind == 'opaque':
emph_node += nodes.Text(' ' + sphinx_gettext('(as an opaque struct)'))
elif struct_abi_kind == 'full-abi':
emph_node += nodes.Text(' ' + sphinx_gettext('(including all members)'))
if record['ifdef_note']:
emph_node += nodes.Text(' ' + record['ifdef_note'])
if stable_added == '3.2':
# Stable ABI was introduced in 3.2.
pass
else:
emph_node += nodes.Text(' ' + sphinx_gettext('since version %s') % stable_added)
emph_node += nodes.Text('.')
if struct_abi_kind == 'members':
emph_node += nodes.Text(
' ' + sphinx_gettext('(Only some members are part of the stable ABI.)'))
node.insert(0, emph_node)
# Unstable API annotation.
if name.startswith('PyUnstable'):
warn_node = nodes.admonition(
classes=['unstable-c-api', 'warning'])
message = sphinx_gettext('This is') + ' '
emph_node = nodes.emphasis(message, message)
ref_node = addnodes.pending_xref(
'Unstable API', refdomain="std",
reftarget='unstable-c-api',
reftype='ref', refexplicit="False")
ref_node += nodes.Text(sphinx_gettext('Unstable API'))
emph_node += ref_node
emph_node += nodes.Text(sphinx_gettext('. It may change without warning in minor releases.'))
warn_node += emph_node
node.insert(0, warn_node)
# Return value annotation
if objtype != 'function':
continue
entry = self.refcount_data.get(name)
if not entry:
continue
elif not entry.result_type.endswith("Object*"):
continue
classes = ['refcount']
if entry.result_refs is None:
rc = sphinx_gettext('Return value: Always NULL.')
classes.append('return_null')
elif entry.result_refs:
rc = sphinx_gettext('Return value: New reference.')
classes.append('return_new_ref')
else:
rc = sphinx_gettext('Return value: Borrowed reference.')
classes.append('return_borrowed_ref')
node.insert(0, nodes.emphasis(rc, rc, classes=classes))
@dataclasses.dataclass(frozen=True, slots=True)
class StableABIEntry:
# Role of the object.
# Source: Each [item_kind] in stable_abi.toml is mapped to a C Domain role.
role: str
# Name of the object.
# Source: [<item_kind>.*] in stable_abi.toml.
name: str
# Version when the object was added to the stable ABI.
# (Source: [<item_kind>.*.added] in stable_abi.toml.
added: str
# An explananatory blurb for the ifdef.
# Source: ``feature_macro.*.doc`` in stable_abi.toml.
ifdef_note: str
# Defines how much of the struct is exposed. Only relevant for structs.
# Source: [<item_kind>.*.struct_abi_kind] in stable_abi.toml.
struct_abi_kind: str
def init_annotations(app):
annotations = Annotations(
path.join(app.srcdir, app.config.refcount_file),
path.join(app.srcdir, app.config.stable_abi_file),
def read_refcount_data(refcount_filename: Path) -> dict[str, RefCountEntry]:
refcount_data = {}
refcounts = refcount_filename.read_text(encoding="utf8")
for line in refcounts.splitlines():
line = line.strip()
if not line or line.startswith("#"):
# blank lines and comments
continue
# Each line is of the form
# function ':' type ':' [param name] ':' [refcount effect] ':' [comment]
parts = line.split(":", 4)
if len(parts) != 5:
raise ValueError(f"Wrong field count in {line!r}")
function, type, arg, refcount, _comment = parts
# Get the entry, creating it if needed:
try:
entry = refcount_data[function]
except KeyError:
entry = refcount_data[function] = RefCountEntry(function)
if not refcount or refcount == "null":
refcount = None
else:
refcount = int(refcount)
# Update the entry with the new parameter
# or the result information.
if arg:
entry.args.append((arg, type, refcount))
else:
entry.result_type = type
entry.result_refs = refcount
return refcount_data
def read_stable_abi_data(stable_abi_file: Path) -> dict[str, StableABIEntry]:
stable_abi_data = {}
with open(stable_abi_file, encoding="utf8") as fp:
for record in csv.DictReader(fp):
name = record["name"]
stable_abi_data[name] = StableABIEntry(**record)
return stable_abi_data
def add_annotations(app: Sphinx, doctree: nodes.document) -> None:
state = app.env.domaindata["c_annotations"]
refcount_data = state["refcount_data"]
stable_abi_data = state["stable_abi_data"]
for node in doctree.findall(addnodes.desc_content):
par = node.parent
if par["domain"] != "c":
continue
if not par[0].get("ids", None):
continue
name = par[0]["ids"][0]
if name.startswith("c."):
name = name[2:]
objtype = par["objtype"]
# Stable ABI annotation.
if record := stable_abi_data.get(name):
if ROLE_TO_OBJECT_TYPE[record.role] != objtype:
msg = (
f"Object type mismatch in limited API annotation for {name}: "
f"{ROLE_TO_OBJECT_TYPE[record.role]!r} != {objtype!r}"
)
raise ValueError(msg)
annotation = _stable_abi_annotation(record)
node.insert(0, annotation)
# Unstable API annotation.
if name.startswith("PyUnstable"):
annotation = _unstable_api_annotation()
node.insert(0, annotation)
# Return value annotation
if objtype != "function":
continue
if name not in refcount_data:
continue
entry = refcount_data[name]
if not entry.result_type.endswith("Object*"):
continue
annotation = _return_value_annotation(entry.result_refs)
node.insert(0, annotation)
def _stable_abi_annotation(record: StableABIEntry) -> nodes.emphasis:
"""Create the Stable ABI annotation.
These have two forms:
Part of the `Stable ABI <link>`_.
Part of the `Stable ABI <link>`_ since version X.Y.
For structs, there's some more info in the message:
Part of the `Limited API <link>`_ (as an opaque struct).
Part of the `Stable ABI <link>`_ (including all members).
Part of the `Limited API <link>`_ (Only some members are part
of the stable ABI.).
... all of which can have "since version X.Y" appended.
"""
stable_added = record.added
message = sphinx_gettext("Part of the")
message = message.center(len(message) + 2)
emph_node = nodes.emphasis(message, message, classes=["stableabi"])
ref_node = addnodes.pending_xref(
"Stable ABI",
refdomain="std",
reftarget="stable",
reftype="ref",
refexplicit="False",
)
app.connect('doctree-read', annotations.add_annotations)
class LimitedAPIList(Directive):
has_content = False
required_arguments = 0
optional_arguments = 0
final_argument_whitespace = True
def run(self):
content = []
for record in annotations.stable_abi_data.values():
role = REST_ROLE_MAP[record['role']]
name = record['name']
content.append(f'* :c:{role}:`{name}`')
pnode = nodes.paragraph()
self.state.nested_parse(StringList(content), 0, pnode)
return [pnode]
app.add_directive('limited-api-list', LimitedAPIList)
struct_abi_kind = record.struct_abi_kind
if struct_abi_kind in {"opaque", "members"}:
ref_node += nodes.Text(sphinx_gettext("Limited API"))
else:
ref_node += nodes.Text(sphinx_gettext("Stable ABI"))
emph_node += ref_node
if struct_abi_kind == "opaque":
emph_node += nodes.Text(" " + sphinx_gettext("(as an opaque struct)"))
elif struct_abi_kind == "full-abi":
emph_node += nodes.Text(
" " + sphinx_gettext("(including all members)")
)
if record.ifdef_note:
emph_node += nodes.Text(f" {record.ifdef_note}")
if stable_added == "3.2":
# Stable ABI was introduced in 3.2.
pass
else:
emph_node += nodes.Text(
" " + sphinx_gettext("since version %s") % stable_added
)
emph_node += nodes.Text(".")
if struct_abi_kind == "members":
msg = " " + sphinx_gettext(
"(Only some members are part of the stable ABI.)"
)
emph_node += nodes.Text(msg)
return emph_node
def setup(app):
app.add_config_value('refcount_file', '', True)
app.add_config_value('stable_abi_file', '', True)
app.connect('builder-inited', init_annotations)
def _unstable_api_annotation() -> nodes.admonition:
ref_node = addnodes.pending_xref(
"Unstable API",
nodes.Text(sphinx_gettext("Unstable API")),
refdomain="std",
reftarget="unstable-c-api",
reftype="ref",
refexplicit="False",
)
emph_node = nodes.emphasis(
"This is ",
sphinx_gettext("This is") + " ",
ref_node,
nodes.Text(
sphinx_gettext(
". It may change without warning in minor releases."
)
),
)
return nodes.admonition(
"",
emph_node,
classes=["unstable-c-api", "warning"],
)
# monkey-patch C object...
CObject.option_spec = {
'noindex': directives.flag,
'stableabi': directives.flag,
def _return_value_annotation(result_refs: int | None) -> nodes.emphasis:
classes = ["refcount"]
if result_refs is None:
rc = sphinx_gettext("Return value: Always NULL.")
classes.append("return_null")
elif result_refs:
rc = sphinx_gettext("Return value: New reference.")
classes.append("return_new_ref")
else:
rc = sphinx_gettext("Return value: Borrowed reference.")
classes.append("return_borrowed_ref")
return nodes.emphasis(rc, rc, classes=classes)
class LimitedAPIList(SphinxDirective):
has_content = False
required_arguments = 0
optional_arguments = 0
final_argument_whitespace = True
def run(self) -> list[nodes.Node]:
state = self.env.domaindata["c_annotations"]
content = [
f"* :c:{record.role}:`{record.name}`"
for record in state["stable_abi_data"].values()
]
node = nodes.paragraph()
self.state.nested_parse(StringList(content), 0, node)
return [node]
def init_annotations(app: Sphinx) -> None:
# Using domaindata is a bit hack-ish,
# but allows storing state without a global variable or closure.
app.env.domaindata["c_annotations"] = state = {}
state["refcount_data"] = read_refcount_data(
Path(app.srcdir, app.config.refcount_file)
)
state["stable_abi_data"] = read_stable_abi_data(
Path(app.srcdir, app.config.stable_abi_file)
)
def setup(app: Sphinx) -> ExtensionMetadata:
app.add_config_value("refcount_file", "", "env", types={str})
app.add_config_value("stable_abi_file", "", "env", types={str})
app.add_directive("limited-api-list", LimitedAPIList)
app.connect("builder-inited", init_annotations)
app.connect("doctree-read", add_annotations)
if sphinx.version_info[:2] < (7, 2):
from docutils.parsers.rst import directives
from sphinx.domains.c import CObject
# monkey-patch C object...
CObject.option_spec |= {
"no-index-entry": directives.flag,
"no-contents-entry": directives.flag,
}
return {
"version": "1.0",
"parallel_read_safe": True,
"parallel_write_safe": True,
}
old_handle_signature = CObject.handle_signature
def new_handle_signature(self, sig, signode):
signode.parent['stableabi'] = 'stableabi' in self.options
return old_handle_signature(self, sig, signode)
CObject.handle_signature = new_handle_signature
return {'version': '1.0', 'parallel_read_safe': True}

23
Tools/build/stable_abi.py
View File

@ -225,9 +225,9 @@ def gen_python3dll(manifest, args, outfile):
key=sort_key):
write(f'EXPORT_DATA({item.name})')
REST_ROLES = {
'function': 'function',
'data': 'var',
ITEM_KIND_TO_DOC_ROLE = {
'function': 'func',
'data': 'data',
'struct': 'type',
'macro': 'macro',
# 'const': 'const', # all undocumented
@ -236,22 +236,28 @@ REST_ROLES = {
@generator("doc_list", 'Doc/data/stable_abi.dat')
def gen_doc_annotations(manifest, args, outfile):
"""Generate/check the stable ABI list for documentation annotations"""
"""Generate/check the stable ABI list for documentation annotations
See ``StableABIEntry`` in ``Doc/tools/extensions/c_annotations.py``
for a description of each field.
"""
writer = csv.DictWriter(
outfile,
['role', 'name', 'added', 'ifdef_note', 'struct_abi_kind'],
lineterminator='\n')
writer.writeheader()
for item in manifest.select(REST_ROLES.keys(), include_abi_only=False):
kinds = set(ITEM_KIND_TO_DOC_ROLE)
for item in manifest.select(kinds, include_abi_only=False):
if item.ifdef:
ifdef_note = manifest.contents[item.ifdef].doc
else:
ifdef_note = None
row = {
'role': REST_ROLES[item.kind],
'role': ITEM_KIND_TO_DOC_ROLE[item.kind],
'name': item.name,
'added': item.added,
'ifdef_note': ifdef_note}
'ifdef_note': ifdef_note,
}
rows = [row]
if item.kind == 'struct':
row['struct_abi_kind'] = item.struct_abi_kind
@ -259,7 +265,8 @@ def gen_doc_annotations(manifest, args, outfile):
rows.append({
'role': 'member',
'name': f'{item.name}.{member_name}',
'added': item.added})
'added': item.added,
})
writer.writerows(rows)
@generator("ctypes_test", 'Lib/test/test_stable_abi_ctypes.py')