diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 0656ff5581b..311e216e506 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -4,17 +4,35 @@ """Test that all symbols of the Stable ABI are accessible using ctypes """ +import sys import unittest from test.support.import_helper import import_module +from _testcapi import get_feature_macros +feature_macros = get_feature_macros() ctypes_test = import_module('ctypes') class TestStableABIAvailability(unittest.TestCase): def test_available_symbols(self): + for symbol_name in SYMBOL_NAMES: with self.subTest(symbol_name): ctypes_test.pythonapi[symbol_name] + def test_feature_macros(self): + self.assertEqual(set(get_feature_macros()), EXPECTED_IFDEFS) + + # The feature macros for Windows are used in creating the DLL + # definition, so they must be known on all platforms. + # If we are on Windows, we check that the hardcoded data matches + # the reality. + @unittest.skipIf(sys.platform != "win32", "Windows specific test") + def test_windows_feature_macros(self): + for name, value in WINDOWS_IFDEFS.items(): + if value != 'maybe': + with self.subTest(name): + self.assertEqual(feature_macros[name], value) + SYMBOL_NAMES = ( "PyAIter_Check", @@ -855,3 +873,41 @@ SYMBOL_NAMES = ( "_Py_TrueStruct", "_Py_VaBuildValue_SizeT", ) +if feature_macros['MS_WINDOWS']: + SYMBOL_NAMES += ( + 'PyErr_SetExcFromWindowsErr', + 'PyErr_SetExcFromWindowsErrWithFilename', + 'PyErr_SetExcFromWindowsErrWithFilenameObject', + 'PyErr_SetExcFromWindowsErrWithFilenameObjects', + 'PyErr_SetFromWindowsErr', + 'PyErr_SetFromWindowsErrWithFilename', + 'PyExc_WindowsError', + 'PyUnicode_AsMBCSString', + 'PyUnicode_DecodeCodePageStateful', + 'PyUnicode_DecodeMBCS', + 'PyUnicode_DecodeMBCSStateful', + 'PyUnicode_EncodeCodePage', + ) +if feature_macros['HAVE_FORK']: + SYMBOL_NAMES += ( + 'PyOS_AfterFork', + 'PyOS_AfterFork_Child', + 'PyOS_AfterFork_Parent', + 'PyOS_BeforeFork', + ) +if feature_macros['USE_STACKCHECK']: + SYMBOL_NAMES += ( + 'PyOS_CheckStack', + ) +if feature_macros['PY_HAVE_THREAD_NATIVE_ID']: + SYMBOL_NAMES += ( + 'PyThread_get_thread_native_id', + ) +if feature_macros['Py_REF_DEBUG']: + SYMBOL_NAMES += ( + '_Py_NegativeRefcount', + '_Py_RefTotal', + ) + +EXPECTED_IFDEFS = set(['HAVE_FORK', 'MS_WINDOWS', 'PY_HAVE_THREAD_NATIVE_ID', 'Py_REF_DEBUG', 'USE_STACKCHECK']) +WINDOWS_IFDEFS = {'MS_WINDOWS': True, 'HAVE_FORK': False, 'USE_STACKCHECK': 'maybe', 'PY_HAVE_THREAD_NATIVE_ID': True, 'Py_REF_DEBUG': 'maybe'} diff --git a/Misc/stable_abi.txt b/Misc/stable_abi.txt index 66777a62c43..9b1c87ea8f4 100644 --- a/Misc/stable_abi.txt +++ b/Misc/stable_abi.txt @@ -29,6 +29,8 @@ # value may change. # - typedef: A C typedef which is used in other definitions in the limited API. # Its size/layout/signature must not change. +# - ifdef: A feature macro: other items may be conditional on whether the macro +# is defined or not. # Each top-level item can have details defined below it: # - added: The version in which the item was added to the stable ABI. @@ -41,6 +43,10 @@ # of the stable ABI. # - a combination of the above (functions that were called by macros that # were public in the past) +# - doc: for `ifdef`, the blurb added in documentation +# - windows: for `ifdef`, this macro is defined on Windows. (This info is used +# to generate the DLL manifest and needs to be available on all platforms.) +# `maybe` marks macros defined on some but not all Windows builds. # For structs, one of the following must be set: # - opaque: The struct name is available in the Limited API, but its members @@ -59,6 +65,24 @@ # https://docs.python.org/3/c-api/stable.html#stable +# Feature macros for optional functionality: + +ifdef MS_WINDOWS + doc on Windows + windows +ifdef HAVE_FORK + doc on platforms with fork() +ifdef USE_STACKCHECK + doc on platforms with USE_STACKCHECK + windows maybe +ifdef PY_HAVE_THREAD_NATIVE_ID + doc on platforms with native thread IDs + windows +ifdef Py_REF_DEBUG + doc when Python is compiled in debug mode (with Py_REF_DEBUG) + windows maybe + + # Mentioned in PEP 384: struct PyObject diff --git a/Modules/_testcapi_feature_macros.inc b/Modules/_testcapi_feature_macros.inc new file mode 100644 index 00000000000..b1763b57d91 --- /dev/null +++ b/Modules/_testcapi_feature_macros.inc @@ -0,0 +1,49 @@ +// Generated by Tools/scripts/stable_abi.py + +// Add an entry in dict `result` for each Stable ABI feature macro. + +#ifdef HAVE_FORK + res = PyDict_SetItemString(result, "HAVE_FORK", Py_True); +#else + res = PyDict_SetItemString(result, "HAVE_FORK", Py_False); +#endif +if (res) { + Py_DECREF(result); return NULL; +} + +#ifdef MS_WINDOWS + res = PyDict_SetItemString(result, "MS_WINDOWS", Py_True); +#else + res = PyDict_SetItemString(result, "MS_WINDOWS", Py_False); +#endif +if (res) { + Py_DECREF(result); return NULL; +} + +#ifdef PY_HAVE_THREAD_NATIVE_ID + res = PyDict_SetItemString(result, "PY_HAVE_THREAD_NATIVE_ID", Py_True); +#else + res = PyDict_SetItemString(result, "PY_HAVE_THREAD_NATIVE_ID", Py_False); +#endif +if (res) { + Py_DECREF(result); return NULL; +} + +#ifdef Py_REF_DEBUG + res = PyDict_SetItemString(result, "Py_REF_DEBUG", Py_True); +#else + res = PyDict_SetItemString(result, "Py_REF_DEBUG", Py_False); +#endif +if (res) { + Py_DECREF(result); return NULL; +} + +#ifdef USE_STACKCHECK + res = PyDict_SetItemString(result, "USE_STACKCHECK", Py_True); +#else + res = PyDict_SetItemString(result, "USE_STACKCHECK", Py_False); +#endif +if (res) { + Py_DECREF(result); return NULL; +} + diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 6bd73e8f453..9073f33e226 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5919,6 +5919,18 @@ frame_getlasti(PyObject *self, PyObject *frame) return PyLong_FromLong(lasti); } +static PyObject * +get_feature_macros(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *result = PyDict_New(); + if (!result) { + return NULL; + } + int res; +#include "_testcapi_feature_macros.inc" + return result; +} + static PyObject *negative_dictoffset(PyObject *, PyObject *); static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); @@ -6214,6 +6226,7 @@ static PyMethodDef TestMethods[] = { {"frame_getgenerator", frame_getgenerator, METH_O, NULL}, {"frame_getbuiltins", frame_getbuiltins, METH_O, NULL}, {"frame_getlasti", frame_getlasti, METH_O, NULL}, + {"get_feature_macros", get_feature_macros, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/PC/python3dll.c b/PC/python3dll.c index aabc1e83868..50e7a9607be 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -19,6 +19,7 @@ EXPORT_FUNC(_Py_CheckRecursiveCall) EXPORT_FUNC(_Py_Dealloc) EXPORT_FUNC(_Py_DecRef) EXPORT_FUNC(_Py_IncRef) +EXPORT_FUNC(_Py_NegativeRefcount) EXPORT_FUNC(_Py_VaBuildValue_SizeT) EXPORT_FUNC(_PyArg_Parse_SizeT) EXPORT_FUNC(_PyArg_ParseTuple_SizeT) @@ -730,6 +731,7 @@ EXPORT_DATA(_Py_EllipsisObject) EXPORT_DATA(_Py_FalseStruct) EXPORT_DATA(_Py_NoneStruct) EXPORT_DATA(_Py_NotImplementedStruct) +EXPORT_DATA(_Py_RefTotal) EXPORT_DATA(_Py_SwappedOp) EXPORT_DATA(_Py_TrueStruct) EXPORT_DATA(_PyWeakref_CallableProxyType) diff --git a/Tools/scripts/stable_abi.py b/Tools/scripts/stable_abi.py index 7376a4649ca..54075248c7e 100755 --- a/Tools/scripts/stable_abi.py +++ b/Tools/scripts/stable_abi.py @@ -45,21 +45,6 @@ EXCLUDED_HEADERS = { MACOS = (sys.platform == "darwin") UNIXY = MACOS or (sys.platform == "linux") # XXX should this be "not Windows"? -IFDEF_DOC_NOTES = { - 'MS_WINDOWS': 'on Windows', - 'HAVE_FORK': 'on platforms with fork()', - 'USE_STACKCHECK': 'on platforms with USE_STACKCHECK', - 'PY_HAVE_THREAD_NATIVE_ID': 'on platforms with native thread IDs', -} - -# To generate the DLL definition, we need to know which feature macros are -# defined on Windows. On all platforms. -# Best way to do that is to hardcode the list (and later test in on Windows). -WINDOWS_IFDEFS = frozenset({ - 'MS_WINDOWS', - 'PY_HAVE_THREAD_NATIVE_ID', - 'USE_STACKCHECK', -}) # The stable ABI manifest (Misc/stable_abi.txt) exists only to fill the # following dataclasses. @@ -130,9 +115,11 @@ class ABIItem: ifdef: str = None struct_abi_kind: str = None members: list = None + doc: str = None + windows: bool = False KINDS = frozenset({ - 'struct', 'function', 'macro', 'data', 'const', 'typedef', + 'struct', 'function', 'macro', 'data', 'const', 'typedef', 'ifdef', }) def dump(self, indent=0): @@ -171,8 +158,8 @@ def parse_manifest(file): levels.pop() parent = levels[-1][0] entry = None - if kind in ABIItem.KINDS: - if parent.kind not in {'manifest'}: + if parent.kind == 'manifest': + if kind not in kind in ABIItem.KINDS: raise_error(f'{kind} cannot go in {parent.kind}') entry = ABIItem(kind, content) parent.add(entry) @@ -193,10 +180,29 @@ def parse_manifest(file): parent.struct_abi_kind = kind if kind == 'members': parent.members = content.split() + elif kind in {'doc'}: + if parent.kind not in {'ifdef'}: + raise_error(f'{kind} cannot go in {parent.kind}') + parent.doc = content + elif kind in {'windows'}: + if parent.kind not in {'ifdef'}: + raise_error(f'{kind} cannot go in {parent.kind}') + if not content: + parent.windows = True + elif content == 'maybe': + parent.windows = content + else: + raise_error(f'Unexpected: {content}') else: raise_error(f"unknown kind {kind!r}") # When adding more, update the comment in stable_abi.txt. levels.append((entry, level)) + + ifdef_names = {i.name for i in manifest.select({'ifdef'})} + for item in manifest.contents.values(): + if item.ifdef and item.ifdef not in ifdef_names: + raise ValueError(f'{item.name} uses undeclared ifdef {item.ifdef}') + return manifest # The tool can run individual "actions". @@ -240,9 +246,12 @@ def gen_python3dll(manifest, args, outfile): def sort_key(item): return item.name.lower() + windows_ifdefs = { + item.name for item in manifest.select({'ifdef'}) if item.windows + } for item in sorted( manifest.select( - {'function'}, include_abi_only=True, ifdef=WINDOWS_IFDEFS), + {'function'}, include_abi_only=True, ifdef=windows_ifdefs), key=sort_key): write(f'EXPORT_FUNC({item.name})') @@ -250,7 +259,7 @@ def gen_python3dll(manifest, args, outfile): for item in sorted( manifest.select( - {'data'}, include_abi_only=True, ifdef=WINDOWS_IFDEFS), + {'data'}, include_abi_only=True, ifdef=windows_ifdefs), key=sort_key): write(f'EXPORT_DATA({item.name})') @@ -273,7 +282,7 @@ def gen_doc_annotations(manifest, args, outfile): writer.writeheader() for item in manifest.select(REST_ROLES.keys(), include_abi_only=False): if item.ifdef: - ifdef_note = IFDEF_DOC_NOTES[item.ifdef] + ifdef_note = manifest.contents[item.ifdef].doc else: ifdef_note = None writer.writerow({ @@ -298,23 +307,42 @@ def gen_ctypes_test(manifest, args, outfile): """Test that all symbols of the Stable ABI are accessible using ctypes """ + import sys import unittest from test.support.import_helper import import_module + from _testcapi import get_feature_macros + feature_macros = get_feature_macros() ctypes_test = import_module('ctypes') class TestStableABIAvailability(unittest.TestCase): def test_available_symbols(self): + for symbol_name in SYMBOL_NAMES: with self.subTest(symbol_name): ctypes_test.pythonapi[symbol_name] + def test_feature_macros(self): + self.assertEqual(set(get_feature_macros()), EXPECTED_IFDEFS) + + # The feature macros for Windows are used in creating the DLL + # definition, so they must be known on all platforms. + # If we are on Windows, we check that the hardcoded data matches + # the reality. + @unittest.skipIf(sys.platform != "win32", "Windows specific test") + def test_windows_feature_macros(self): + for name, value in WINDOWS_IFDEFS.items(): + if value != 'maybe': + with self.subTest(name): + self.assertEqual(feature_macros[name], value) + SYMBOL_NAMES = ( ''')) items = manifest.select( {'function', 'data'}, include_abi_only=True, - ifdef=set()) + ) + ifdef_items = {} for item in items: if item.name in ( # Some symbols aren't exported on all platforms. @@ -322,8 +350,45 @@ def gen_ctypes_test(manifest, args, outfile): 'PyModule_Create2', 'PyModule_FromDefAndSpec2', ): continue - write(f' "{item.name}",') + if item.ifdef: + ifdef_items.setdefault(item.ifdef, []).append(item.name) + else: + write(f' "{item.name}",') write(")") + for ifdef, names in ifdef_items.items(): + write(f"if feature_macros[{ifdef!r}]:") + write(f" SYMBOL_NAMES += (") + for name in names: + write(f" {name!r},") + write(" )") + write("") + write(f"EXPECTED_IFDEFS = set({sorted(ifdef_items)})") + + windows_ifdef_values = { + name: manifest.contents[name].windows for name in ifdef_items + } + write(f"WINDOWS_IFDEFS = {windows_ifdef_values}") + + +@generator("testcapi_feature_macros", 'Modules/_testcapi_feature_macros.inc') +def gen_testcapi_feature_macros(manifest, args, outfile): + """Generate/check the stable ABI list for documentation annotations""" + write = partial(print, file=outfile) + write('// Generated by Tools/scripts/stable_abi.py') + write() + write('// Add an entry in dict `result` for each Stable ABI feature macro.') + write() + for macro in manifest.select({'ifdef'}): + name = macro.name + write(f'#ifdef {name}') + write(f' res = PyDict_SetItemString(result, "{name}", Py_True);') + write('#else') + write(f' res = PyDict_SetItemString(result, "{name}", Py_False);') + write('#endif') + write('if (res) {') + write(' Py_DECREF(result); return NULL;') + write('}') + write() def generate_or_check(manifest, args, path, func):