Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-91324: List feature macros in the stable ABI manifest, improve tests (GH-32415)

This commit is contained in:
Petr Viktorin 2022-04-28 16:30:28 +02:00 committed by GitHub
parent d1de10784d
commit 6dcbc08c95
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 232 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
Lib/test/test_stable_abi_ctypes.py generated
View File

@ -4,17 +4,35 @@
"""Test that all symbols of the Stable ABI are accessible using ctypes
"""
import sys
import unittest
from test.support.import_helper import import_module
from _testcapi import get_feature_macros
feature_macros = get_feature_macros()
ctypes_test = import_module('ctypes')
class TestStableABIAvailability(unittest.TestCase):
def test_available_symbols(self):
for symbol_name in SYMBOL_NAMES:
with self.subTest(symbol_name):
ctypes_test.pythonapi[symbol_name]
def test_feature_macros(self):
self.assertEqual(set(get_feature_macros()), EXPECTED_IFDEFS)
# The feature macros for Windows are used in creating the DLL
# definition, so they must be known on all platforms.
# If we are on Windows, we check that the hardcoded data matches
# the reality.
@unittest.skipIf(sys.platform != "win32", "Windows specific test")
def test_windows_feature_macros(self):
for name, value in WINDOWS_IFDEFS.items():
if value != 'maybe':
with self.subTest(name):
self.assertEqual(feature_macros[name], value)
SYMBOL_NAMES = (
"PyAIter_Check",
@ -855,3 +873,41 @@ SYMBOL_NAMES = (
"_Py_TrueStruct",
"_Py_VaBuildValue_SizeT",
)
if feature_macros['MS_WINDOWS']:
SYMBOL_NAMES += (
'PyErr_SetExcFromWindowsErr',
'PyErr_SetExcFromWindowsErrWithFilename',
'PyErr_SetExcFromWindowsErrWithFilenameObject',
'PyErr_SetExcFromWindowsErrWithFilenameObjects',
'PyErr_SetFromWindowsErr',
'PyErr_SetFromWindowsErrWithFilename',
'PyExc_WindowsError',
'PyUnicode_AsMBCSString',
'PyUnicode_DecodeCodePageStateful',
'PyUnicode_DecodeMBCS',
'PyUnicode_DecodeMBCSStateful',
'PyUnicode_EncodeCodePage',
)
if feature_macros['HAVE_FORK']:
SYMBOL_NAMES += (
'PyOS_AfterFork',
'PyOS_AfterFork_Child',
'PyOS_AfterFork_Parent',
'PyOS_BeforeFork',
)
if feature_macros['USE_STACKCHECK']:
SYMBOL_NAMES += (
'PyOS_CheckStack',
)
if feature_macros['PY_HAVE_THREAD_NATIVE_ID']:
SYMBOL_NAMES += (
'PyThread_get_thread_native_id',
)
if feature_macros['Py_REF_DEBUG']:
SYMBOL_NAMES += (
'_Py_NegativeRefcount',
'_Py_RefTotal',
)
EXPECTED_IFDEFS = set(['HAVE_FORK', 'MS_WINDOWS', 'PY_HAVE_THREAD_NATIVE_ID', 'Py_REF_DEBUG', 'USE_STACKCHECK'])
WINDOWS_IFDEFS = {'MS_WINDOWS': True, 'HAVE_FORK': False, 'USE_STACKCHECK': 'maybe', 'PY_HAVE_THREAD_NATIVE_ID': True, 'Py_REF_DEBUG': 'maybe'}

24
Misc/stable_abi.txt
View File

@ -29,6 +29,8 @@
# value may change.
# - typedef: A C typedef which is used in other definitions in the limited API.
# Its size/layout/signature must not change.
# - ifdef: A feature macro: other items may be conditional on whether the macro
# is defined or not.
# Each top-level item can have details defined below it:
# - added: The version in which the item was added to the stable ABI.
@ -41,6 +43,10 @@
# of the stable ABI.
# - a combination of the above (functions that were called by macros that
# were public in the past)
# - doc: for `ifdef`, the blurb added in documentation
# - windows: for `ifdef`, this macro is defined on Windows. (This info is used
# to generate the DLL manifest and needs to be available on all platforms.)
# `maybe` marks macros defined on some but not all Windows builds.
# For structs, one of the following must be set:
# - opaque: The struct name is available in the Limited API, but its members
@ -59,6 +65,24 @@
# https://docs.python.org/3/c-api/stable.html#stable
# Feature macros for optional functionality:
ifdef MS_WINDOWS
doc on Windows
windows
ifdef HAVE_FORK
doc on platforms with fork()
ifdef USE_STACKCHECK
doc on platforms with USE_STACKCHECK
windows maybe
ifdef PY_HAVE_THREAD_NATIVE_ID
doc on platforms with native thread IDs
windows
ifdef Py_REF_DEBUG
doc when Python is compiled in debug mode (with Py_REF_DEBUG)
windows maybe
# Mentioned in PEP 384:
struct PyObject

49
Modules/_testcapi_feature_macros.inc Normal file
View File

@ -0,0 +1,49 @@
// Generated by Tools/scripts/stable_abi.py
// Add an entry in dict `result` for each Stable ABI feature macro.
#ifdef HAVE_FORK
res = PyDict_SetItemString(result, "HAVE_FORK", Py_True);
#else
res = PyDict_SetItemString(result, "HAVE_FORK", Py_False);
#endif
if (res) {
Py_DECREF(result); return NULL;
}
#ifdef MS_WINDOWS
res = PyDict_SetItemString(result, "MS_WINDOWS", Py_True);
#else
res = PyDict_SetItemString(result, "MS_WINDOWS", Py_False);
#endif
if (res) {
Py_DECREF(result); return NULL;
}
#ifdef PY_HAVE_THREAD_NATIVE_ID
res = PyDict_SetItemString(result, "PY_HAVE_THREAD_NATIVE_ID", Py_True);
#else
res = PyDict_SetItemString(result, "PY_HAVE_THREAD_NATIVE_ID", Py_False);
#endif
if (res) {
Py_DECREF(result); return NULL;
}
#ifdef Py_REF_DEBUG
res = PyDict_SetItemString(result, "Py_REF_DEBUG", Py_True);
#else
res = PyDict_SetItemString(result, "Py_REF_DEBUG", Py_False);
#endif
if (res) {
Py_DECREF(result); return NULL;
}
#ifdef USE_STACKCHECK
res = PyDict_SetItemString(result, "USE_STACKCHECK", Py_True);
#else
res = PyDict_SetItemString(result, "USE_STACKCHECK", Py_False);
#endif
if (res) {
Py_DECREF(result); return NULL;
}

13
Modules/_testcapimodule.c
View File

@ -5919,6 +5919,18 @@ frame_getlasti(PyObject *self, PyObject *frame)
return PyLong_FromLong(lasti);
}
static PyObject *
get_feature_macros(PyObject *self, PyObject *Py_UNUSED(args))
{
PyObject *result = PyDict_New();
if (!result) {
return NULL;
}
int res;
#include "_testcapi_feature_macros.inc"
return result;
}
static PyObject *negative_dictoffset(PyObject *, PyObject *);
static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *);
@ -6214,6 +6226,7 @@ static PyMethodDef TestMethods[] = {
{"frame_getgenerator", frame_getgenerator, METH_O, NULL},
{"frame_getbuiltins", frame_getbuiltins, METH_O, NULL},
{"frame_getlasti", frame_getlasti, METH_O, NULL},
{"get_feature_macros", get_feature_macros, METH_NOARGS, NULL},
{NULL, NULL} /* sentinel */
};

2
PC/python3dll.c generated
View File

@ -19,6 +19,7 @@ EXPORT_FUNC(_Py_CheckRecursiveCall)
EXPORT_FUNC(_Py_Dealloc)
EXPORT_FUNC(_Py_DecRef)
EXPORT_FUNC(_Py_IncRef)
EXPORT_FUNC(_Py_NegativeRefcount)
EXPORT_FUNC(_Py_VaBuildValue_SizeT)
EXPORT_FUNC(_PyArg_Parse_SizeT)
EXPORT_FUNC(_PyArg_ParseTuple_SizeT)
@ -730,6 +731,7 @@ EXPORT_DATA(_Py_EllipsisObject)
EXPORT_DATA(_Py_FalseStruct)
EXPORT_DATA(_Py_NoneStruct)
EXPORT_DATA(_Py_NotImplementedStruct)
EXPORT_DATA(_Py_RefTotal)
EXPORT_DATA(_Py_SwappedOp)
EXPORT_DATA(_Py_TrueStruct)
EXPORT_DATA(_PyWeakref_CallableProxyType)

111
Tools/scripts/stable_abi.py
View File

@ -45,21 +45,6 @@ EXCLUDED_HEADERS = {
MACOS = (sys.platform == "darwin")
UNIXY = MACOS or (sys.platform == "linux") # XXX should this be "not Windows"?
IFDEF_DOC_NOTES = {
'MS_WINDOWS': 'on Windows',
'HAVE_FORK': 'on platforms with fork()',
'USE_STACKCHECK': 'on platforms with USE_STACKCHECK',
'PY_HAVE_THREAD_NATIVE_ID': 'on platforms with native thread IDs',
}
# To generate the DLL definition, we need to know which feature macros are
# defined on Windows. On all platforms.
# Best way to do that is to hardcode the list (and later test in on Windows).
WINDOWS_IFDEFS = frozenset({
'MS_WINDOWS',
'PY_HAVE_THREAD_NATIVE_ID',
'USE_STACKCHECK',
})
# The stable ABI manifest (Misc/stable_abi.txt) exists only to fill the
# following dataclasses.
@ -130,9 +115,11 @@ class ABIItem:
ifdef: str = None
struct_abi_kind: str = None
members: list = None
doc: str = None
windows: bool = False
KINDS = frozenset({
'struct', 'function', 'macro', 'data', 'const', 'typedef',
'struct', 'function', 'macro', 'data', 'const', 'typedef', 'ifdef',
})
def dump(self, indent=0):
@ -171,8 +158,8 @@ def parse_manifest(file):
levels.pop()
parent = levels[-1][0]
entry = None
if kind in ABIItem.KINDS:
if parent.kind not in {'manifest'}:
if parent.kind == 'manifest':
if kind not in kind in ABIItem.KINDS:
raise_error(f'{kind} cannot go in {parent.kind}')
entry = ABIItem(kind, content)
parent.add(entry)
@ -193,10 +180,29 @@ def parse_manifest(file):
parent.struct_abi_kind = kind
if kind == 'members':
parent.members = content.split()
elif kind in {'doc'}:
if parent.kind not in {'ifdef'}:
raise_error(f'{kind} cannot go in {parent.kind}')
parent.doc = content
elif kind in {'windows'}:
if parent.kind not in {'ifdef'}:
raise_error(f'{kind} cannot go in {parent.kind}')
if not content:
parent.windows = True
elif content == 'maybe':
parent.windows = content
else:
raise_error(f'Unexpected: {content}')
else:
raise_error(f"unknown kind {kind!r}")
# When adding more, update the comment in stable_abi.txt.
levels.append((entry, level))
ifdef_names = {i.name for i in manifest.select({'ifdef'})}
for item in manifest.contents.values():
if item.ifdef and item.ifdef not in ifdef_names:
raise ValueError(f'{item.name} uses undeclared ifdef {item.ifdef}')
return manifest
# The tool can run individual "actions".
@ -240,9 +246,12 @@ def gen_python3dll(manifest, args, outfile):
def sort_key(item):
return item.name.lower()
windows_ifdefs = {
item.name for item in manifest.select({'ifdef'}) if item.windows
}
for item in sorted(
manifest.select(
{'function'}, include_abi_only=True, ifdef=WINDOWS_IFDEFS),
{'function'}, include_abi_only=True, ifdef=windows_ifdefs),
key=sort_key):
write(f'EXPORT_FUNC({item.name})')
@ -250,7 +259,7 @@ def gen_python3dll(manifest, args, outfile):
for item in sorted(
manifest.select(
{'data'}, include_abi_only=True, ifdef=WINDOWS_IFDEFS),
{'data'}, include_abi_only=True, ifdef=windows_ifdefs),
key=sort_key):
write(f'EXPORT_DATA({item.name})')
@ -273,7 +282,7 @@ def gen_doc_annotations(manifest, args, outfile):
writer.writeheader()
for item in manifest.select(REST_ROLES.keys(), include_abi_only=False):
if item.ifdef:
ifdef_note = IFDEF_DOC_NOTES[item.ifdef]
ifdef_note = manifest.contents[item.ifdef].doc
else:
ifdef_note = None
writer.writerow({
@ -298,23 +307,42 @@ def gen_ctypes_test(manifest, args, outfile):
"""Test that all symbols of the Stable ABI are accessible using ctypes
"""
import sys
import unittest
from test.support.import_helper import import_module
from _testcapi import get_feature_macros
feature_macros = get_feature_macros()
ctypes_test = import_module('ctypes')
class TestStableABIAvailability(unittest.TestCase):
def test_available_symbols(self):
for symbol_name in SYMBOL_NAMES:
with self.subTest(symbol_name):
ctypes_test.pythonapi[symbol_name]
def test_feature_macros(self):
self.assertEqual(set(get_feature_macros()), EXPECTED_IFDEFS)
# The feature macros for Windows are used in creating the DLL
# definition, so they must be known on all platforms.
# If we are on Windows, we check that the hardcoded data matches
# the reality.
@unittest.skipIf(sys.platform != "win32", "Windows specific test")
def test_windows_feature_macros(self):
for name, value in WINDOWS_IFDEFS.items():
if value != 'maybe':
with self.subTest(name):
self.assertEqual(feature_macros[name], value)
SYMBOL_NAMES = (
'''))
items = manifest.select(
{'function', 'data'},
include_abi_only=True,
ifdef=set())
)
ifdef_items = {}
for item in items:
if item.name in (
# Some symbols aren't exported on all platforms.
@ -322,8 +350,45 @@ def gen_ctypes_test(manifest, args, outfile):
'PyModule_Create2', 'PyModule_FromDefAndSpec2',
):
continue
write(f' "{item.name}",')
if item.ifdef:
ifdef_items.setdefault(item.ifdef, []).append(item.name)
else:
write(f' "{item.name}",')
write(")")
for ifdef, names in ifdef_items.items():
write(f"if feature_macros[{ifdef!r}]:")
write(f" SYMBOL_NAMES += (")
for name in names:
write(f" {name!r},")
write(" )")
write("")
write(f"EXPECTED_IFDEFS = set({sorted(ifdef_items)})")
windows_ifdef_values = {
name: manifest.contents[name].windows for name in ifdef_items
}
write(f"WINDOWS_IFDEFS = {windows_ifdef_values}")
@generator("testcapi_feature_macros", 'Modules/_testcapi_feature_macros.inc')
def gen_testcapi_feature_macros(manifest, args, outfile):
"""Generate/check the stable ABI list for documentation annotations"""
write = partial(print, file=outfile)
write('// Generated by Tools/scripts/stable_abi.py')
write()
write('// Add an entry in dict `result` for each Stable ABI feature macro.')
write()
for macro in manifest.select({'ifdef'}):
name = macro.name
write(f'#ifdef {name}')
write(f' res = PyDict_SetItemString(result, "{name}", Py_True);')
write('#else')
write(f' res = PyDict_SetItemString(result, "{name}", Py_False);')
write('#endif')
write('if (res) {')
write(' Py_DECREF(result); return NULL;')
write('}')
write()
def generate_or_check(manifest, args, path, func):