gh-91324: List feature macros in the stable ABI manifest, improve tests (GH-32415)

Lib/test/test_stable_abi_ctypes.py

@@ -4,17 +4,35 @@
 """Test that all symbols of the Stable ABI are accessible using ctypes
 """
 
+import sys
 import unittest
 from test.support.import_helper import import_module
+from _testcapi import get_feature_macros
 
+feature_macros = get_feature_macros()
 ctypes_test = import_module('ctypes')
 
 class TestStableABIAvailability(unittest.TestCase):
     def test_available_symbols(self):
+
         for symbol_name in SYMBOL_NAMES:
             with self.subTest(symbol_name):
                 ctypes_test.pythonapi[symbol_name]
 
+    def test_feature_macros(self):
+        self.assertEqual(set(get_feature_macros()), EXPECTED_IFDEFS)
+
+    # The feature macros for Windows are used in creating the DLL
+    # definition, so they must be known on all platforms.
+    # If we are on Windows, we check that the hardcoded data matches
+    # the reality.
+    @unittest.skipIf(sys.platform != "win32", "Windows specific test")
+    def test_windows_feature_macros(self):
+        for name, value in WINDOWS_IFDEFS.items():
+            if value != 'maybe':
+                with self.subTest(name):
+                    self.assertEqual(feature_macros[name], value)
+
 SYMBOL_NAMES = (
 
     "PyAIter_Check",
@@ -855,3 +873,41 @@ SYMBOL_NAMES = (
     "_Py_TrueStruct",
     "_Py_VaBuildValue_SizeT",
 )
+if feature_macros['MS_WINDOWS']:
+    SYMBOL_NAMES += (
+        'PyErr_SetExcFromWindowsErr',
+        'PyErr_SetExcFromWindowsErrWithFilename',
+        'PyErr_SetExcFromWindowsErrWithFilenameObject',
+        'PyErr_SetExcFromWindowsErrWithFilenameObjects',
+        'PyErr_SetFromWindowsErr',
+        'PyErr_SetFromWindowsErrWithFilename',
+        'PyExc_WindowsError',
+        'PyUnicode_AsMBCSString',
+        'PyUnicode_DecodeCodePageStateful',
+        'PyUnicode_DecodeMBCS',
+        'PyUnicode_DecodeMBCSStateful',
+        'PyUnicode_EncodeCodePage',
+    )
+if feature_macros['HAVE_FORK']:
+    SYMBOL_NAMES += (
+        'PyOS_AfterFork',
+        'PyOS_AfterFork_Child',
+        'PyOS_AfterFork_Parent',
+        'PyOS_BeforeFork',
+    )
+if feature_macros['USE_STACKCHECK']:
+    SYMBOL_NAMES += (
+        'PyOS_CheckStack',
+    )
+if feature_macros['PY_HAVE_THREAD_NATIVE_ID']:
+    SYMBOL_NAMES += (
+        'PyThread_get_thread_native_id',
+    )
+if feature_macros['Py_REF_DEBUG']:
+    SYMBOL_NAMES += (
+        '_Py_NegativeRefcount',
+        '_Py_RefTotal',
+    )
+
+EXPECTED_IFDEFS = set(['HAVE_FORK', 'MS_WINDOWS', 'PY_HAVE_THREAD_NATIVE_ID', 'Py_REF_DEBUG', 'USE_STACKCHECK'])
+WINDOWS_IFDEFS = {'MS_WINDOWS': True, 'HAVE_FORK': False, 'USE_STACKCHECK': 'maybe', 'PY_HAVE_THREAD_NATIVE_ID': True, 'Py_REF_DEBUG': 'maybe'}

Misc/stable_abi.txt

@@ -29,6 +29,8 @@
 #   value may change.
 # - typedef: A C typedef which is used in other definitions in the limited API.
 #   Its size/layout/signature must not change.
+# - ifdef: A feature macro: other items may be conditional on whether the macro
+#   is defined or not.
 
 # Each top-level item can have details defined below it:
 # - added: The version in which the item was added to the stable ABI.
@@ -41,6 +43,10 @@
 #     of the stable ABI.
 #   - a combination of the above (functions that were called by macros that
 #     were public in the past)
+# - doc: for `ifdef`, the blurb added in documentation
+# - windows: for `ifdef`, this macro is defined on Windows. (This info is used
+#   to generate the DLL manifest and needs to be available on all platforms.)
+#   `maybe` marks macros defined on some but not all Windows builds.
 
 # For structs, one of the following must be set:
 # - opaque: The struct name is available in the Limited API, but its members
@@ -59,6 +65,24 @@
 #    https://docs.python.org/3/c-api/stable.html#stable
 
 
+# Feature macros for optional functionality:
+
+ifdef MS_WINDOWS
+    doc on Windows
+    windows
+ifdef HAVE_FORK
+    doc on platforms with fork()
+ifdef USE_STACKCHECK
+    doc on platforms with USE_STACKCHECK
+    windows maybe
+ifdef PY_HAVE_THREAD_NATIVE_ID
+    doc on platforms with native thread IDs
+    windows
+ifdef Py_REF_DEBUG
+    doc when Python is compiled in debug mode (with Py_REF_DEBUG)
+    windows maybe
+
+
 # Mentioned in PEP 384:
 
 struct PyObject

Modules/_testcapi_feature_macros.inc

@@ -0,0 +1,49 @@
+// Generated by Tools/scripts/stable_abi.py
+
+// Add an entry in dict `result` for each Stable ABI feature macro.
+
+#ifdef HAVE_FORK
+    res = PyDict_SetItemString(result, "HAVE_FORK", Py_True);
+#else
+    res = PyDict_SetItemString(result, "HAVE_FORK", Py_False);
+#endif
+if (res) {
+    Py_DECREF(result); return NULL;
+}
+
+#ifdef MS_WINDOWS
+    res = PyDict_SetItemString(result, "MS_WINDOWS", Py_True);
+#else
+    res = PyDict_SetItemString(result, "MS_WINDOWS", Py_False);
+#endif
+if (res) {
+    Py_DECREF(result); return NULL;
+}
+
+#ifdef PY_HAVE_THREAD_NATIVE_ID
+    res = PyDict_SetItemString(result, "PY_HAVE_THREAD_NATIVE_ID", Py_True);
+#else
+    res = PyDict_SetItemString(result, "PY_HAVE_THREAD_NATIVE_ID", Py_False);
+#endif
+if (res) {
+    Py_DECREF(result); return NULL;
+}
+
+#ifdef Py_REF_DEBUG
+    res = PyDict_SetItemString(result, "Py_REF_DEBUG", Py_True);
+#else
+    res = PyDict_SetItemString(result, "Py_REF_DEBUG", Py_False);
+#endif
+if (res) {
+    Py_DECREF(result); return NULL;
+}
+
+#ifdef USE_STACKCHECK
+    res = PyDict_SetItemString(result, "USE_STACKCHECK", Py_True);
+#else
+    res = PyDict_SetItemString(result, "USE_STACKCHECK", Py_False);
+#endif
+if (res) {
+    Py_DECREF(result); return NULL;
+}
+

Modules/_testcapimodule.c

@@ -5919,6 +5919,18 @@ frame_getlasti(PyObject *self, PyObject *frame)
     return PyLong_FromLong(lasti);
 }
 
+static PyObject *
+get_feature_macros(PyObject *self, PyObject *Py_UNUSED(args))
+{
+    PyObject *result = PyDict_New();
+    if (!result) {
+        return NULL;
+    }
+    int res;
+#include "_testcapi_feature_macros.inc"
+    return result;
+}
+
 
 static PyObject *negative_dictoffset(PyObject *, PyObject *);
 static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *);
@@ -6214,6 +6226,7 @@ static PyMethodDef TestMethods[] = {
     {"frame_getgenerator", frame_getgenerator, METH_O, NULL},
     {"frame_getbuiltins", frame_getbuiltins, METH_O, NULL},
     {"frame_getlasti", frame_getlasti, METH_O, NULL},
+    {"get_feature_macros", get_feature_macros, METH_NOARGS, NULL},
     {NULL, NULL} /* sentinel */
 };
 

PC/python3dll.c

@@ -19,6 +19,7 @@ EXPORT_FUNC(_Py_CheckRecursiveCall)
 EXPORT_FUNC(_Py_Dealloc)
 EXPORT_FUNC(_Py_DecRef)
 EXPORT_FUNC(_Py_IncRef)
+EXPORT_FUNC(_Py_NegativeRefcount)
 EXPORT_FUNC(_Py_VaBuildValue_SizeT)
 EXPORT_FUNC(_PyArg_Parse_SizeT)
 EXPORT_FUNC(_PyArg_ParseTuple_SizeT)
@@ -730,6 +731,7 @@ EXPORT_DATA(_Py_EllipsisObject)
 EXPORT_DATA(_Py_FalseStruct)
 EXPORT_DATA(_Py_NoneStruct)
 EXPORT_DATA(_Py_NotImplementedStruct)
+EXPORT_DATA(_Py_RefTotal)
 EXPORT_DATA(_Py_SwappedOp)
 EXPORT_DATA(_Py_TrueStruct)
 EXPORT_DATA(_PyWeakref_CallableProxyType)

Tools/scripts/stable_abi.py

@@ -45,21 +45,6 @@ EXCLUDED_HEADERS = {
 MACOS = (sys.platform == "darwin")
 UNIXY = MACOS or (sys.platform == "linux")  # XXX should this be "not Windows"?
 
-IFDEF_DOC_NOTES = {
-    'MS_WINDOWS': 'on Windows',
-    'HAVE_FORK': 'on platforms with fork()',
-    'USE_STACKCHECK': 'on platforms with USE_STACKCHECK',
-    'PY_HAVE_THREAD_NATIVE_ID': 'on platforms with native thread IDs',
-}
-
-# To generate the DLL definition, we need to know which feature macros are
-# defined on Windows. On all platforms.
-# Best way to do that is to hardcode the list (and later test in on Windows).
-WINDOWS_IFDEFS = frozenset({
-    'MS_WINDOWS',
-    'PY_HAVE_THREAD_NATIVE_ID',
-    'USE_STACKCHECK',
-})
 
 # The stable ABI manifest (Misc/stable_abi.txt) exists only to fill the
 # following dataclasses.
@@ -130,9 +115,11 @@ class ABIItem:
     ifdef: str = None
     struct_abi_kind: str = None
     members: list = None
+    doc: str = None
+    windows: bool = False
 
     KINDS = frozenset({
-        'struct', 'function', 'macro', 'data', 'const', 'typedef',
+        'struct', 'function', 'macro', 'data', 'const', 'typedef', 'ifdef',
     })
 
     def dump(self, indent=0):
@@ -171,8 +158,8 @@ def parse_manifest(file):
             levels.pop()
         parent = levels[-1][0]
         entry = None
-        if kind in ABIItem.KINDS:
+        if parent.kind == 'manifest':
-            if parent.kind not in {'manifest'}:
+            if kind not in kind in ABIItem.KINDS:
                 raise_error(f'{kind} cannot go in {parent.kind}')
             entry = ABIItem(kind, content)
             parent.add(entry)
@@ -193,10 +180,29 @@ def parse_manifest(file):
             parent.struct_abi_kind = kind
             if kind == 'members':
                 parent.members = content.split()
+        elif kind in {'doc'}:
+            if parent.kind not in {'ifdef'}:
+                raise_error(f'{kind} cannot go in {parent.kind}')
+            parent.doc = content
+        elif kind in {'windows'}:
+            if parent.kind not in {'ifdef'}:
+                raise_error(f'{kind} cannot go in {parent.kind}')
+            if not content:
+                parent.windows = True
+            elif content == 'maybe':
+                parent.windows = content
+            else:
+                raise_error(f'Unexpected: {content}')
         else:
             raise_error(f"unknown kind {kind!r}")
             # When adding more, update the comment in stable_abi.txt.
         levels.append((entry, level))
+
+    ifdef_names = {i.name for i in manifest.select({'ifdef'})}
+    for item in manifest.contents.values():
+        if item.ifdef and item.ifdef not in ifdef_names:
+            raise ValueError(f'{item.name} uses undeclared ifdef {item.ifdef}')
+
     return manifest
 
 # The tool can run individual "actions".
@@ -240,9 +246,12 @@ def gen_python3dll(manifest, args, outfile):
     def sort_key(item):
         return item.name.lower()
 
+    windows_ifdefs = {
+        item.name for item in manifest.select({'ifdef'}) if item.windows
+    }
     for item in sorted(
             manifest.select(
-                {'function'}, include_abi_only=True, ifdef=WINDOWS_IFDEFS),
+                {'function'}, include_abi_only=True, ifdef=windows_ifdefs),
             key=sort_key):
         write(f'EXPORT_FUNC({item.name})')
 
@@ -250,7 +259,7 @@ def gen_python3dll(manifest, args, outfile):
 
     for item in sorted(
             manifest.select(
-                {'data'}, include_abi_only=True, ifdef=WINDOWS_IFDEFS),
+                {'data'}, include_abi_only=True, ifdef=windows_ifdefs),
             key=sort_key):
         write(f'EXPORT_DATA({item.name})')
 
@@ -273,7 +282,7 @@ def gen_doc_annotations(manifest, args, outfile):
     writer.writeheader()
     for item in manifest.select(REST_ROLES.keys(), include_abi_only=False):
         if item.ifdef:
-            ifdef_note = IFDEF_DOC_NOTES[item.ifdef]
+            ifdef_note = manifest.contents[item.ifdef].doc
         else:
             ifdef_note = None
         writer.writerow({
@@ -298,23 +307,42 @@ def gen_ctypes_test(manifest, args, outfile):
         """Test that all symbols of the Stable ABI are accessible using ctypes
         """
 
+        import sys
         import unittest
         from test.support.import_helper import import_module
+        from _testcapi import get_feature_macros
 
+        feature_macros = get_feature_macros()
         ctypes_test = import_module('ctypes')
 
         class TestStableABIAvailability(unittest.TestCase):
             def test_available_symbols(self):
+
                 for symbol_name in SYMBOL_NAMES:
                     with self.subTest(symbol_name):
                         ctypes_test.pythonapi[symbol_name]
 
+            def test_feature_macros(self):
+                self.assertEqual(set(get_feature_macros()), EXPECTED_IFDEFS)
+
+            # The feature macros for Windows are used in creating the DLL
+            # definition, so they must be known on all platforms.
+            # If we are on Windows, we check that the hardcoded data matches
+            # the reality.
+            @unittest.skipIf(sys.platform != "win32", "Windows specific test")
+            def test_windows_feature_macros(self):
+                for name, value in WINDOWS_IFDEFS.items():
+                    if value != 'maybe':
+                        with self.subTest(name):
+                            self.assertEqual(feature_macros[name], value)
+
         SYMBOL_NAMES = (
     '''))
     items = manifest.select(
         {'function', 'data'},
         include_abi_only=True,
-        ifdef=set())
+    )
+    ifdef_items = {}
     for item in items:
         if item.name in (
                 # Some symbols aren't exported on all platforms.
@@ -322,8 +350,45 @@ def gen_ctypes_test(manifest, args, outfile):
                 'PyModule_Create2', 'PyModule_FromDefAndSpec2',
             ):
             continue
-        write(f'    "{item.name}",')
+        if item.ifdef:
+            ifdef_items.setdefault(item.ifdef, []).append(item.name)
+        else:
+            write(f'    "{item.name}",')
     write(")")
+    for ifdef, names in ifdef_items.items():
+        write(f"if feature_macros[{ifdef!r}]:")
+        write(f"    SYMBOL_NAMES += (")
+        for name in names:
+            write(f"        {name!r},")
+        write("    )")
+    write("")
+    write(f"EXPECTED_IFDEFS = set({sorted(ifdef_items)})")
+
+    windows_ifdef_values = {
+        name: manifest.contents[name].windows for name in ifdef_items
+    }
+    write(f"WINDOWS_IFDEFS = {windows_ifdef_values}")
+
+
+@generator("testcapi_feature_macros", 'Modules/_testcapi_feature_macros.inc')
+def gen_testcapi_feature_macros(manifest, args, outfile):
+    """Generate/check the stable ABI list for documentation annotations"""
+    write = partial(print, file=outfile)
+    write('// Generated by Tools/scripts/stable_abi.py')
+    write()
+    write('// Add an entry in dict `result` for each Stable ABI feature macro.')
+    write()
+    for macro in manifest.select({'ifdef'}):
+        name = macro.name
+        write(f'#ifdef {name}')
+        write(f'    res = PyDict_SetItemString(result, "{name}", Py_True);')
+        write('#else')
+        write(f'    res = PyDict_SetItemString(result, "{name}", Py_False);')
+        write('#endif')
+        write('if (res) {')
+        write('    Py_DECREF(result); return NULL;')
+        write('}')
+        write()
 
 
 def generate_or_check(manifest, args, path, func):