mirror of https://github.com/python/cpython
gh-91324: Convert the stable ABI manifest to TOML (GH-92026)
This commit is contained in:
parent
89c6b2b8f6
commit
83bce8ef14
|
@ -20,7 +20,8 @@ class TestStableABIAvailability(unittest.TestCase):
|
|||
ctypes_test.pythonapi[symbol_name]
|
||||
|
||||
def test_feature_macros(self):
|
||||
self.assertEqual(set(get_feature_macros()), EXPECTED_IFDEFS)
|
||||
self.assertEqual(
|
||||
set(get_feature_macros()), EXPECTED_FEATURE_MACROS)
|
||||
|
||||
# The feature macros for Windows are used in creating the DLL
|
||||
# definition, so they must be known on all platforms.
|
||||
|
@ -28,7 +29,7 @@ class TestStableABIAvailability(unittest.TestCase):
|
|||
# the reality.
|
||||
@unittest.skipIf(sys.platform != "win32", "Windows specific test")
|
||||
def test_windows_feature_macros(self):
|
||||
for name, value in WINDOWS_IFDEFS.items():
|
||||
for name, value in WINDOWS_FEATURE_MACROS.items():
|
||||
if value != 'maybe':
|
||||
with self.subTest(name):
|
||||
self.assertEqual(feature_macros[name], value)
|
||||
|
@ -909,5 +910,13 @@ if feature_macros['Py_REF_DEBUG']:
|
|||
'_Py_RefTotal',
|
||||
)
|
||||
|
||||
EXPECTED_IFDEFS = set(['HAVE_FORK', 'MS_WINDOWS', 'PY_HAVE_THREAD_NATIVE_ID', 'Py_REF_DEBUG', 'USE_STACKCHECK'])
|
||||
WINDOWS_IFDEFS = {'MS_WINDOWS': True, 'HAVE_FORK': False, 'USE_STACKCHECK': 'maybe', 'PY_HAVE_THREAD_NATIVE_ID': True, 'Py_REF_DEBUG': 'maybe'}
|
||||
EXPECTED_FEATURE_MACROS = set(['HAVE_FORK',
|
||||
'MS_WINDOWS',
|
||||
'PY_HAVE_THREAD_NATIVE_ID',
|
||||
'Py_REF_DEBUG',
|
||||
'USE_STACKCHECK'])
|
||||
WINDOWS_FEATURE_MACROS = {'HAVE_FORK': False,
|
||||
'MS_WINDOWS': True,
|
||||
'PY_HAVE_THREAD_NATIVE_ID': True,
|
||||
'Py_REF_DEBUG': 'maybe',
|
||||
'USE_STACKCHECK': 'maybe'}
|
||||
|
|
|
@ -1199,7 +1199,7 @@ regen-global-objects: $(srcdir)/Tools/scripts/generate_global_objects.py
|
|||
# ABI
|
||||
|
||||
regen-limited-abi: all
|
||||
$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.txt
|
||||
$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.toml
|
||||
|
||||
############################################################################
|
||||
# Regenerate all generated files
|
||||
|
@ -2476,7 +2476,7 @@ patchcheck: @DEF_MAKE_RULE@
|
|||
$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py
|
||||
|
||||
check-limited-abi: all
|
||||
$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --all $(srcdir)/Misc/stable_abi.txt
|
||||
$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --all $(srcdir)/Misc/stable_abi.toml
|
||||
|
||||
.PHONY: update-config
|
||||
update-config:
|
||||
|
|
File diff suppressed because it is too large
Load Diff
2284
Misc/stable_abi.txt
2284
Misc/stable_abi.txt
File diff suppressed because it is too large
Load Diff
|
@ -14,8 +14,10 @@ import subprocess
|
|||
import sysconfig
|
||||
import argparse
|
||||
import textwrap
|
||||
import tomllib
|
||||
import difflib
|
||||
import shutil
|
||||
import pprint
|
||||
import sys
|
||||
import os
|
||||
import os.path
|
||||
|
@ -46,17 +48,15 @@ MACOS = (sys.platform == "darwin")
|
|||
UNIXY = MACOS or (sys.platform == "linux") # XXX should this be "not Windows"?
|
||||
|
||||
|
||||
# The stable ABI manifest (Misc/stable_abi.txt) exists only to fill the
|
||||
# The stable ABI manifest (Misc/stable_abi.toml) exists only to fill the
|
||||
# following dataclasses.
|
||||
# Feel free to change its syntax (and the `parse_manifest` function)
|
||||
# to better serve that purpose (while keeping it human-readable).
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Manifest:
|
||||
"""Collection of `ABIItem`s forming the stable ABI/limited API."""
|
||||
|
||||
kind = 'manifest'
|
||||
contents: dict = dataclasses.field(default_factory=dict)
|
||||
def __init__(self):
|
||||
self.contents = dict()
|
||||
|
||||
def add(self, item):
|
||||
if item.name in self.contents:
|
||||
|
@ -65,14 +65,6 @@ class Manifest:
|
|||
raise ValueError(f'duplicate ABI item {item.name}')
|
||||
self.contents[item.name] = item
|
||||
|
||||
@property
|
||||
def feature_defines(self):
|
||||
"""Return all feature defines which affect what's available
|
||||
|
||||
These are e.g. HAVE_FORK and MS_WINDOWS.
|
||||
"""
|
||||
return set(item.ifdef for item in self.contents.values()) - {None}
|
||||
|
||||
def select(self, kinds, *, include_abi_only=True, ifdef=None):
|
||||
"""Yield selected items of the manifest
|
||||
|
||||
|
@ -81,7 +73,7 @@ class Manifest:
|
|||
stable ABI.
|
||||
If False, include only items from the limited API
|
||||
(i.e. items people should use today)
|
||||
ifdef: set of feature defines (e.g. {'HAVE_FORK', 'MS_WINDOWS'}).
|
||||
ifdef: set of feature macros (e.g. {'HAVE_FORK', 'MS_WINDOWS'}).
|
||||
If None (default), items are not filtered by this. (This is
|
||||
different from the empty set, which filters out all such
|
||||
conditional items.)
|
||||
|
@ -99,109 +91,74 @@ class Manifest:
|
|||
|
||||
def dump(self):
|
||||
"""Yield lines to recreate the manifest file (sans comments/newlines)"""
|
||||
# Recursive in preparation for struct member & function argument nodes
|
||||
for item in self.contents.values():
|
||||
yield from item.dump(indent=0)
|
||||
fields = dataclasses.fields(item)
|
||||
yield f"[{item.kind}.{item.name}]"
|
||||
for field in fields:
|
||||
if field.name in {'name', 'value', 'kind'}:
|
||||
continue
|
||||
value = getattr(item, field.name)
|
||||
if value == field.default:
|
||||
pass
|
||||
elif value is True:
|
||||
yield f" {field.name} = true"
|
||||
elif value:
|
||||
yield f" {field.name} = {value!r}"
|
||||
|
||||
|
||||
itemclasses = {}
|
||||
def itemclass(kind):
|
||||
"""Register the decorated class in `itemclasses`"""
|
||||
def decorator(cls):
|
||||
itemclasses[kind] = cls
|
||||
return cls
|
||||
return decorator
|
||||
|
||||
@itemclass('function')
|
||||
@itemclass('macro')
|
||||
@itemclass('data')
|
||||
@itemclass('const')
|
||||
@itemclass('typedef')
|
||||
@dataclasses.dataclass
|
||||
class ABIItem:
|
||||
"""Information on one item (function, macro, struct, etc.)"""
|
||||
|
||||
kind: str
|
||||
name: str
|
||||
kind: str
|
||||
added: str = None
|
||||
contents: list = dataclasses.field(default_factory=list)
|
||||
abi_only: bool = False
|
||||
ifdef: str = None
|
||||
struct_abi_kind: str = None
|
||||
members: list = None
|
||||
doc: str = None
|
||||
|
||||
@itemclass('feature_macro')
|
||||
@dataclasses.dataclass(kw_only=True)
|
||||
class FeatureMacro(ABIItem):
|
||||
name: str
|
||||
doc: str
|
||||
windows: bool = False
|
||||
abi_only: bool = True
|
||||
|
||||
KINDS = frozenset({
|
||||
'struct', 'function', 'macro', 'data', 'const', 'typedef', 'ifdef',
|
||||
})
|
||||
@itemclass('struct')
|
||||
@dataclasses.dataclass(kw_only=True)
|
||||
class Struct(ABIItem):
|
||||
struct_abi_kind: str
|
||||
members: list = None
|
||||
|
||||
def dump(self, indent=0):
|
||||
yield f"{' ' * indent}{self.kind} {self.name}"
|
||||
if self.added:
|
||||
yield f"{' ' * (indent+1)}added {self.added}"
|
||||
if self.ifdef:
|
||||
yield f"{' ' * (indent+1)}ifdef {self.ifdef}"
|
||||
if self.abi_only:
|
||||
yield f"{' ' * (indent+1)}abi_only"
|
||||
|
||||
def parse_manifest(file):
|
||||
"""Parse the given file (iterable of lines) to a Manifest"""
|
||||
|
||||
LINE_RE = re.compile('(?P<indent>[ ]*)(?P<kind>[^ ]+)[ ]*(?P<content>.*)')
|
||||
manifest = Manifest()
|
||||
|
||||
# parents of currently processed line, each with its indentation level
|
||||
levels = [(manifest, -1)]
|
||||
data = tomllib.load(file)
|
||||
|
||||
def raise_error(msg):
|
||||
raise SyntaxError(f'line {lineno}: {msg}')
|
||||
|
||||
for lineno, line in enumerate(file, start=1):
|
||||
line, sep, comment = line.partition('#')
|
||||
line = line.rstrip()
|
||||
if not line:
|
||||
continue
|
||||
match = LINE_RE.fullmatch(line)
|
||||
if not match:
|
||||
raise_error(f'invalid syntax: {line}')
|
||||
level = len(match['indent'])
|
||||
kind = match['kind']
|
||||
content = match['content']
|
||||
while level <= levels[-1][1]:
|
||||
levels.pop()
|
||||
parent = levels[-1][0]
|
||||
entry = None
|
||||
if parent.kind == 'manifest':
|
||||
if kind not in kind in ABIItem.KINDS:
|
||||
raise_error(f'{kind} cannot go in {parent.kind}')
|
||||
entry = ABIItem(kind, content)
|
||||
parent.add(entry)
|
||||
elif kind in {'added', 'ifdef'}:
|
||||
if parent.kind not in ABIItem.KINDS:
|
||||
raise_error(f'{kind} cannot go in {parent.kind}')
|
||||
setattr(parent, kind, content)
|
||||
elif kind in {'abi_only'}:
|
||||
if parent.kind not in {'function', 'data'}:
|
||||
raise_error(f'{kind} cannot go in {parent.kind}')
|
||||
parent.abi_only = True
|
||||
elif kind in {'members', 'full-abi', 'opaque'}:
|
||||
if parent.kind not in {'struct'}:
|
||||
raise_error(f'{kind} cannot go in {parent.kind}')
|
||||
if prev := getattr(parent, 'struct_abi_kind', None):
|
||||
raise_error(
|
||||
f'{parent.name} already has {prev}, cannot add {kind}')
|
||||
parent.struct_abi_kind = kind
|
||||
if kind == 'members':
|
||||
parent.members = content.split()
|
||||
elif kind in {'doc'}:
|
||||
if parent.kind not in {'ifdef'}:
|
||||
raise_error(f'{kind} cannot go in {parent.kind}')
|
||||
parent.doc = content
|
||||
elif kind in {'windows'}:
|
||||
if parent.kind not in {'ifdef'}:
|
||||
raise_error(f'{kind} cannot go in {parent.kind}')
|
||||
if not content:
|
||||
parent.windows = True
|
||||
elif content == 'maybe':
|
||||
parent.windows = content
|
||||
else:
|
||||
raise_error(f'Unexpected: {content}')
|
||||
else:
|
||||
raise_error(f"unknown kind {kind!r}")
|
||||
# When adding more, update the comment in stable_abi.txt.
|
||||
levels.append((entry, level))
|
||||
|
||||
ifdef_names = {i.name for i in manifest.select({'ifdef'})}
|
||||
for item in manifest.contents.values():
|
||||
if item.ifdef and item.ifdef not in ifdef_names:
|
||||
raise ValueError(f'{item.name} uses undeclared ifdef {item.ifdef}')
|
||||
for kind, itemclass in itemclasses.items():
|
||||
for name, item_data in data[kind].items():
|
||||
try:
|
||||
item = itemclass(name=name, kind=kind, **item_data)
|
||||
manifest.add(item)
|
||||
except BaseException as exc:
|
||||
exc.add_note(f'in {kind} {name}')
|
||||
raise
|
||||
|
||||
return manifest
|
||||
|
||||
|
@ -246,12 +203,14 @@ def gen_python3dll(manifest, args, outfile):
|
|||
def sort_key(item):
|
||||
return item.name.lower()
|
||||
|
||||
windows_ifdefs = {
|
||||
item.name for item in manifest.select({'ifdef'}) if item.windows
|
||||
windows_feature_macros = {
|
||||
item.name for item in manifest.select({'feature_macro'}) if item.windows
|
||||
}
|
||||
for item in sorted(
|
||||
manifest.select(
|
||||
{'function'}, include_abi_only=True, ifdef=windows_ifdefs),
|
||||
{'function'},
|
||||
include_abi_only=True,
|
||||
ifdef=windows_feature_macros),
|
||||
key=sort_key):
|
||||
write(f'EXPORT_FUNC({item.name})')
|
||||
|
||||
|
@ -259,7 +218,9 @@ def gen_python3dll(manifest, args, outfile):
|
|||
|
||||
for item in sorted(
|
||||
manifest.select(
|
||||
{'data'}, include_abi_only=True, ifdef=windows_ifdefs),
|
||||
{'data'},
|
||||
include_abi_only=True,
|
||||
ifdef=windows_feature_macros),
|
||||
key=sort_key):
|
||||
write(f'EXPORT_DATA({item.name})')
|
||||
|
||||
|
@ -285,17 +246,20 @@ def gen_doc_annotations(manifest, args, outfile):
|
|||
ifdef_note = manifest.contents[item.ifdef].doc
|
||||
else:
|
||||
ifdef_note = None
|
||||
writer.writerow({
|
||||
row = {
|
||||
'role': REST_ROLES[item.kind],
|
||||
'name': item.name,
|
||||
'added': item.added,
|
||||
'ifdef_note': ifdef_note,
|
||||
'struct_abi_kind': item.struct_abi_kind})
|
||||
for member_name in item.members or ():
|
||||
writer.writerow({
|
||||
'role': 'member',
|
||||
'name': f'{item.name}.{member_name}',
|
||||
'added': item.added})
|
||||
'ifdef_note': ifdef_note}
|
||||
rows = [row]
|
||||
if item.kind == 'struct':
|
||||
row['struct_abi_kind'] = item.struct_abi_kind
|
||||
for member_name in item.members or ():
|
||||
rows.append({
|
||||
'role': 'member',
|
||||
'name': f'{item.name}.{member_name}',
|
||||
'added': item.added})
|
||||
writer.writerows(rows)
|
||||
|
||||
@generator("ctypes_test", 'Lib/test/test_stable_abi_ctypes.py')
|
||||
def gen_ctypes_test(manifest, args, outfile):
|
||||
|
@ -323,7 +287,8 @@ def gen_ctypes_test(manifest, args, outfile):
|
|||
ctypes_test.pythonapi[symbol_name]
|
||||
|
||||
def test_feature_macros(self):
|
||||
self.assertEqual(set(get_feature_macros()), EXPECTED_IFDEFS)
|
||||
self.assertEqual(
|
||||
set(get_feature_macros()), EXPECTED_FEATURE_MACROS)
|
||||
|
||||
# The feature macros for Windows are used in creating the DLL
|
||||
# definition, so they must be known on all platforms.
|
||||
|
@ -331,7 +296,7 @@ def gen_ctypes_test(manifest, args, outfile):
|
|||
# the reality.
|
||||
@unittest.skipIf(sys.platform != "win32", "Windows specific test")
|
||||
def test_windows_feature_macros(self):
|
||||
for name, value in WINDOWS_IFDEFS.items():
|
||||
for name, value in WINDOWS_FEATURE_MACROS.items():
|
||||
if value != 'maybe':
|
||||
with self.subTest(name):
|
||||
self.assertEqual(feature_macros[name], value)
|
||||
|
@ -342,7 +307,7 @@ def gen_ctypes_test(manifest, args, outfile):
|
|||
{'function', 'data'},
|
||||
include_abi_only=True,
|
||||
)
|
||||
ifdef_items = {}
|
||||
optional_items = {}
|
||||
for item in items:
|
||||
if item.name in (
|
||||
# Some symbols aren't exported on all platforms.
|
||||
|
@ -351,23 +316,23 @@ def gen_ctypes_test(manifest, args, outfile):
|
|||
):
|
||||
continue
|
||||
if item.ifdef:
|
||||
ifdef_items.setdefault(item.ifdef, []).append(item.name)
|
||||
optional_items.setdefault(item.ifdef, []).append(item.name)
|
||||
else:
|
||||
write(f' "{item.name}",')
|
||||
write(")")
|
||||
for ifdef, names in ifdef_items.items():
|
||||
for ifdef, names in optional_items.items():
|
||||
write(f"if feature_macros[{ifdef!r}]:")
|
||||
write(f" SYMBOL_NAMES += (")
|
||||
for name in names:
|
||||
write(f" {name!r},")
|
||||
write(" )")
|
||||
write("")
|
||||
write(f"EXPECTED_IFDEFS = set({sorted(ifdef_items)})")
|
||||
feature_macros = list(manifest.select({'feature_macro'}))
|
||||
feature_names = sorted(m.name for m in feature_macros)
|
||||
write(f"EXPECTED_FEATURE_MACROS = set({pprint.pformat(feature_names)})")
|
||||
|
||||
windows_ifdef_values = {
|
||||
name: manifest.contents[name].windows for name in ifdef_items
|
||||
}
|
||||
write(f"WINDOWS_IFDEFS = {windows_ifdef_values}")
|
||||
windows_feature_macros = {m.name: m.windows for m in feature_macros}
|
||||
write(f"WINDOWS_FEATURE_MACROS = {pprint.pformat(windows_feature_macros)}")
|
||||
|
||||
|
||||
@generator("testcapi_feature_macros", 'Modules/_testcapi_feature_macros.inc')
|
||||
|
@ -378,7 +343,7 @@ def gen_testcapi_feature_macros(manifest, args, outfile):
|
|||
write()
|
||||
write('// Add an entry in dict `result` for each Stable ABI feature macro.')
|
||||
write()
|
||||
for macro in manifest.select({'ifdef'}):
|
||||
for macro in manifest.select({'feature_macro'}):
|
||||
name = macro.name
|
||||
write(f'#ifdef {name}')
|
||||
write(f' res = PyDict_SetItemString(result, "{name}", Py_True);')
|
||||
|
@ -425,7 +390,8 @@ def do_unixy_check(manifest, args):
|
|||
# Get all macros first: we'll need feature macros like HAVE_FORK and
|
||||
# MS_WINDOWS for everything else
|
||||
present_macros = gcc_get_limited_api_macros(['Include/Python.h'])
|
||||
feature_defines = manifest.feature_defines & present_macros
|
||||
feature_macros = set(m.name for m in manifest.select({'feature_macro'}))
|
||||
feature_macros &= present_macros
|
||||
|
||||
# Check that we have all needed macros
|
||||
expected_macros = set(
|
||||
|
@ -438,7 +404,7 @@ def do_unixy_check(manifest, args):
|
|||
+ 'with Py_LIMITED_API:')
|
||||
|
||||
expected_symbols = set(item.name for item in manifest.select(
|
||||
{'function', 'data'}, include_abi_only=True, ifdef=feature_defines,
|
||||
{'function', 'data'}, include_abi_only=True, ifdef=feature_macros,
|
||||
))
|
||||
|
||||
# Check the static library (*.a)
|
||||
|
@ -458,7 +424,7 @@ def do_unixy_check(manifest, args):
|
|||
|
||||
# Check definitions in the header files
|
||||
expected_defs = set(item.name for item in manifest.select(
|
||||
{'function', 'data'}, include_abi_only=False, ifdef=feature_defines,
|
||||
{'function', 'data'}, include_abi_only=False, ifdef=feature_macros,
|
||||
))
|
||||
found_defs = gcc_get_limited_api_definitions(['Include/Python.h'])
|
||||
missing_defs = expected_defs - found_defs
|
||||
|
@ -635,6 +601,28 @@ def check_private_names(manifest):
|
|||
f'`{name}` is private (underscore-prefixed) and should be '
|
||||
+ 'removed from the stable ABI list or or marked `abi_only`')
|
||||
|
||||
def check_dump(manifest, filename):
|
||||
"""Check that manifest.dump() corresponds to the data.
|
||||
|
||||
Mainly useful when debugging this script.
|
||||
"""
|
||||
dumped = tomllib.loads('\n'.join(manifest.dump()))
|
||||
with filename.open('rb') as file:
|
||||
from_file = tomllib.load(file)
|
||||
if dumped != from_file:
|
||||
print(f'Dump differs from loaded data!', file=sys.stderr)
|
||||
diff = difflib.unified_diff(
|
||||
pprint.pformat(dumped).splitlines(),
|
||||
pprint.pformat(from_file).splitlines(),
|
||||
'<dumped>', str(filename),
|
||||
lineterm='',
|
||||
)
|
||||
for line in diff:
|
||||
print(line, file=sys.stderr)
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description=__doc__,
|
||||
|
@ -696,7 +684,16 @@ def main():
|
|||
run_all_generators = True
|
||||
args.unixy_check = True
|
||||
|
||||
with args.file.open() as file:
|
||||
try:
|
||||
file = args.file.open('rb')
|
||||
except FileNotFoundError as err:
|
||||
if args.file.suffix == '.txt':
|
||||
# Provide a better error message
|
||||
suggestion = args.file.with_suffix('.toml')
|
||||
raise FileNotFoundError(
|
||||
f'{args.file} not found. Did you mean {suggestion} ?') from err
|
||||
raise
|
||||
with file:
|
||||
manifest = parse_manifest(file)
|
||||
|
||||
check_private_names(manifest)
|
||||
|
@ -709,7 +706,7 @@ def main():
|
|||
if args.dump:
|
||||
for line in manifest.dump():
|
||||
print(line)
|
||||
results['dump'] = True
|
||||
results['dump'] = check_dump(manifest, args.file)
|
||||
|
||||
for gen in generators:
|
||||
filename = getattr(args, gen.var_name)
|
||||
|
|
Loading…
Reference in New Issue