Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-123465: Allow Py_RELATIVE_OFFSET for __*offset__ members (GH-123474)

This commit is contained in:
Petr Viktorin 2024-09-05 14:14:05 +02:00 committed by GitHub
parent ce9f84a47b
commit 16be8db6be
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 422 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Doc/c-api/structures.rst
View File

@ -485,7 +485,8 @@ Accessing attributes of extension types
``PyMemberDef`` may contain a definition for the special member
``"__vectorcalloffset__"``, corresponding to
:c:member:`~PyTypeObject.tp_vectorcall_offset` in type objects.
These must be defined with ``Py_T_PYSSIZET`` and ``Py_READONLY``, for example::
This member must be defined with ``Py_T_PYSSIZET``, and either
``Py_READONLY`` or ``Py_READONLY | Py_RELATIVE_OFFSET``. For example::
static PyMemberDef spam_type_members[] = {
{"__vectorcalloffset__", Py_T_PYSSIZET,
@ -506,6 +507,12 @@ Accessing attributes of extension types
``PyMemberDef`` is always available.
Previously, it required including ``"structmember.h"``.
.. versionchanged:: 3.14
:c:macro:`Py_RELATIVE_OFFSET` is now allowed for
``"__vectorcalloffset__"``, ``"__dictoffset__"`` and
``"__weaklistoffset__"``.
.. c:function:: PyObject* PyMember_GetOne(const char *obj_addr, struct PyMemberDef *m)
Get an attribute belonging to the object at address *obj_addr*. The

9
Lib/test/test_call.py
View File

@ -851,8 +851,13 @@ class TestPEP590(unittest.TestCase):
@requires_limited_api
def test_vectorcall_limited_incoming(self):
from _testcapi import pyobject_vectorcall
obj = _testlimitedcapi.LimitedVectorCallClass()
self.assertEqual(pyobject_vectorcall(obj, (), ()), "vectorcall called")
for cls in (_testlimitedcapi.LimitedVectorCallClass,
_testlimitedcapi.LimitedRelativeVectorCallClass):
with self.subTest(cls=cls):
obj = cls()
self.assertEqual(
pyobject_vectorcall(obj, (), ()),
"vectorcall called")
@requires_limited_api
def test_vectorcall_limited_outgoing(self):

86
Lib/test/test_capi/test_misc.py
View File

@ -541,13 +541,18 @@ class CAPITest(unittest.TestCase):
self.assertEqual(new_type_refcnt, sys.getrefcount(A))
def test_heaptype_with_dict(self):
inst = _testcapi.HeapCTypeWithDict()
for cls in (
_testcapi.HeapCTypeWithDict,
_testlimitedcapi.HeapCTypeWithRelativeDict,
):
with self.subTest(cls=cls):
inst = cls()
inst.foo = 42
self.assertEqual(inst.foo, 42)
self.assertEqual(inst.dictobj, inst.__dict__)
self.assertEqual(inst.dictobj, {"foo": 42})
inst = _testcapi.HeapCTypeWithDict()
inst = cls()
self.assertEqual({}, inst.__dict__)
def test_heaptype_with_managed_dict(self):
@ -585,7 +590,12 @@ class CAPITest(unittest.TestCase):
self.assertEqual({}, inst.__dict__)
def test_heaptype_with_weakref(self):
inst = _testcapi.HeapCTypeWithWeakref()
for cls in (
_testcapi.HeapCTypeWithWeakref,
_testlimitedcapi.HeapCTypeWithRelativeWeakref,
):
with self.subTest(cls=cls):
inst = cls()
ref = weakref.ref(inst)
self.assertEqual(ref(), inst)
self.assertEqual(inst.weakreflist, ref)
@ -730,22 +740,32 @@ class CAPITest(unittest.TestCase):
self.assertIsInstance(sub, metaclass)
def test_multiple_inheritance_ctypes_with_weakref_or_dict(self):
for weakref_cls in (_testcapi.HeapCTypeWithWeakref,
_testlimitedcapi.HeapCTypeWithRelativeWeakref):
for dict_cls in (_testcapi.HeapCTypeWithDict,
_testlimitedcapi.HeapCTypeWithRelativeDict):
with self.subTest(weakref_cls=weakref_cls, dict_cls=dict_cls):
with self.assertRaises(TypeError):
class Both1(_testcapi.HeapCTypeWithWeakref, _testcapi.HeapCTypeWithDict):
class Both1(weakref_cls, dict_cls):
pass
with self.assertRaises(TypeError):
class Both2(_testcapi.HeapCTypeWithDict, _testcapi.HeapCTypeWithWeakref):
class Both2(dict_cls, weakref_cls):
pass
def test_multiple_inheritance_ctypes_with_weakref_or_dict_and_other_builtin(self):
for dict_cls in (_testcapi.HeapCTypeWithDict,
_testlimitedcapi.HeapCTypeWithRelativeDict):
for weakref_cls in (_testcapi.HeapCTypeWithWeakref,
_testlimitedcapi.HeapCTypeWithRelativeWeakref):
with self.subTest(dict_cls=dict_cls, weakref_cls=weakref_cls):
with self.assertRaises(TypeError):
class C1(_testcapi.HeapCTypeWithDict, list):
class C1(dict_cls, list):
pass
with self.assertRaises(TypeError):
class C2(_testcapi.HeapCTypeWithWeakref, list):
class C2(weakref_cls, list):
pass
class C3(_testcapi.HeapCTypeWithManagedDict, list):
@ -761,8 +781,9 @@ class CAPITest(unittest.TestCase):
inst.append(0)
str(inst.__weakref__)
for cls in (_testcapi.HeapCTypeWithManagedDict, _testcapi.HeapCTypeWithManagedWeakref):
for cls2 in (_testcapi.HeapCTypeWithDict, _testcapi.HeapCTypeWithWeakref):
for cls in (_testcapi.HeapCTypeWithManagedDict,
_testcapi.HeapCTypeWithManagedWeakref):
for cls2 in (dict_cls, weakref_cls):
class S(cls, cls2):
pass
class B1(C3, cls):
@ -1272,6 +1293,53 @@ class TestHeapTypeRelative(unittest.TestCase):
SystemError, r"PyMember_SetOne used with Py_RELATIVE_OFFSET"):
instance.set_memb_relative(0)
def test_heaptype_relative_special_members_errors(self):
for member_name in "__vectorcalloffset__", "__dictoffset__", "__weaklistoffset__":
with self.subTest(member_name=member_name):
with self.assertRaisesRegex(
SystemError,
r"With Py_RELATIVE_OFFSET, basicsize must be negative."):
_testlimitedcapi.make_heaptype_with_member(
basicsize=sys.getsizeof(object()) + 100,
add_relative_flag=True,
member_name=member_name,
member_offset=0,
member_type=_testlimitedcapi.Py_T_PYSSIZET,
member_flags=_testlimitedcapi.Py_READONLY,
)
with self.assertRaisesRegex(
SystemError,
r"Member offset out of range \(0\.\.-basicsize\)"):
_testlimitedcapi.make_heaptype_with_member(
basicsize=-8,
add_relative_flag=True,
member_name=member_name,
member_offset=-1,
member_type=_testlimitedcapi.Py_T_PYSSIZET,
member_flags=_testlimitedcapi.Py_READONLY,
)
with self.assertRaisesRegex(
SystemError,
r"type of %s must be Py_T_PYSSIZET" % member_name):
_testlimitedcapi.make_heaptype_with_member(
basicsize=-100,
add_relative_flag=True,
member_name=member_name,
member_offset=0,
member_flags=_testlimitedcapi.Py_READONLY,
)
with self.assertRaisesRegex(
SystemError,
r"flags for %s must be " % member_name):
_testlimitedcapi.make_heaptype_with_member(
basicsize=-100,
add_relative_flag=True,
member_name=member_name,
member_offset=0,
member_type=_testlimitedcapi.Py_T_PYSSIZET,
member_flags=0,
)
def test_pyobject_getitemdata_error(self):
"""Test PyObject_GetItemData fails on unsupported types"""
with self.assertRaises(TypeError):

4
Misc/NEWS.d/next/C_API/2024-08-29-15-05-19.gh-issue-123465.eqwNWq.rst Normal file
View File

@ -0,0 +1,4 @@
:c:macro:`Py_RELATIVE_OFFSET` is now allowed in :c:type:`PyMemberDef` for
the special offset member ``"__vectorcalloffset__"``, as well as the
discouraged special offset members ``"__dictoffset__"`` and
``"__weaklistoffset__"``

44
Modules/_testlimitedcapi/clinic/heaptype_relative.c.h generated Normal file
View File

@ -0,0 +1,44 @@
/*[clinic input]
preserve
[clinic start generated code]*/
PyDoc_STRVAR(make_heaptype_with_member__doc__,
"make_heaptype_with_member($module, /, extra_base_size=0, basicsize=0,\n"
" member_offset=0, add_relative_flag=False, *,\n"
" member_name=\'memb\', member_flags=0,\n"
" member_type=-1)\n"
"--\n"
"\n");
#define MAKE_HEAPTYPE_WITH_MEMBER_METHODDEF \
{"make_heaptype_with_member", (PyCFunction)(void(*)(void))make_heaptype_with_member, METH_VARARGS|METH_KEYWORDS, make_heaptype_with_member__doc__},
static PyObject *
make_heaptype_with_member_impl(PyObject *module, int extra_base_size,
int basicsize, int member_offset,
int add_relative_flag,
const char *member_name, int member_flags,
int member_type);
static PyObject *
make_heaptype_with_member(PyObject *module, PyObject *args, PyObject *kwargs)
{
PyObject *return_value = NULL;
static char *_keywords[] = {"extra_base_size", "basicsize", "member_offset", "add_relative_flag", "member_name", "member_flags", "member_type", NULL};
int extra_base_size = 0;
int basicsize = 0;
int member_offset = 0;
int add_relative_flag = 0;
const char *member_name = "memb";
int member_flags = 0;
int member_type = Py_T_BYTE;
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iiip$sii:make_heaptype_with_member", _keywords,
&extra_base_size, &basicsize, &member_offset, &add_relative_flag, &member_name, &member_flags, &member_type))
goto exit;
return_value = make_heaptype_with_member_impl(module, extra_base_size, basicsize, member_offset, add_relative_flag, member_name, member_flags, member_type);
exit:
return return_value;
}
/*[clinic end generated code: output=01933185947faecc input=a9049054013a1b77]*/

147
Modules/_testlimitedcapi/heaptype_relative.c
View File

@ -8,6 +8,8 @@
#include <stddef.h> // max_align_t
#include <string.h> // memset
#include "clinic/heaptype_relative.c.h"
static PyType_Slot empty_slots[] = {
{0, NULL},
};
@ -247,6 +249,81 @@ heaptype_with_member_set_memb_relative(PyObject *self, PyObject *value)
Py_RETURN_NONE;
}
typedef struct {
int padding; // just so the offset isn't 0
PyObject *dict;
} HeapCTypeWithDictStruct;
static void
heapctypewithrelativedict_dealloc(PyObject* self)
{
PyTypeObject *tp = Py_TYPE(self);
HeapCTypeWithDictStruct *data = PyObject_GetTypeData(self, tp);
Py_XDECREF(data->dict);
PyObject_Free(self);
Py_DECREF(tp);
}
static PyType_Spec HeapCTypeWithRelativeDict_spec = {
.name = "_testcapi.HeapCTypeWithRelativeDict",
.basicsize = -(int)sizeof(HeapCTypeWithDictStruct),
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
.slots = (PyType_Slot[]) {
{Py_tp_dealloc, heapctypewithrelativedict_dealloc},
{Py_tp_getset, (PyGetSetDef[]) {
{"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict},
{NULL} /* Sentinel */
}},
{Py_tp_members, (PyMemberDef[]) {
{"dictobj", _Py_T_OBJECT,
offsetof(HeapCTypeWithDictStruct, dict),
Py_RELATIVE_OFFSET},
{"__dictoffset__", Py_T_PYSSIZET,
offsetof(HeapCTypeWithDictStruct, dict),
Py_READONLY | Py_RELATIVE_OFFSET},
{NULL} /* Sentinel */
}},
{0, 0},
}
};
typedef struct {
char padding; // just so the offset isn't 0
PyObject *weakreflist;
} HeapCTypeWithWeakrefStruct;
static void
heapctypewithrelativeweakref_dealloc(PyObject* self)
{
PyTypeObject *tp = Py_TYPE(self);
HeapCTypeWithWeakrefStruct *data = PyObject_GetTypeData(self, tp);
if (data->weakreflist != NULL) {
PyObject_ClearWeakRefs(self);
}
Py_XDECREF(data->weakreflist);
PyObject_Free(self);
Py_DECREF(tp);
}
static PyType_Spec HeapCTypeWithRelativeWeakref_spec = {
.name = "_testcapi.HeapCTypeWithRelativeWeakref",
.basicsize = -(int)sizeof(HeapCTypeWithWeakrefStruct),
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
.slots = (PyType_Slot[]) {
{Py_tp_dealloc, heapctypewithrelativeweakref_dealloc},
{Py_tp_members, (PyMemberDef[]) {
{"weakreflist", _Py_T_OBJECT,
offsetof(HeapCTypeWithWeakrefStruct, weakreflist),
Py_RELATIVE_OFFSET},
{"__weaklistoffset__", Py_T_PYSSIZET,
offsetof(HeapCTypeWithWeakrefStruct, weakreflist),
Py_READONLY | Py_RELATIVE_OFFSET},
{NULL} /* Sentinel */
}},
{0, 0},
}
};
static PyMethodDef heaptype_with_member_methods[] = {
{"get_memb", heaptype_with_member_get_memb, METH_NOARGS},
{"set_memb", heaptype_with_member_set_memb, METH_O},
@ -256,19 +333,31 @@ static PyMethodDef heaptype_with_member_methods[] = {
{NULL},
};
/*[clinic input]
make_heaptype_with_member
extra_base_size: int = 0
basicsize: int = 0
member_offset: int = 0
add_relative_flag: bool = False
*
member_name: str = "memb"
member_flags: int = 0
member_type: int(c_default="Py_T_BYTE") = -1
[clinic start generated code]*/
static PyObject *
make_heaptype_with_member(PyObject *module, PyObject *args)
make_heaptype_with_member_impl(PyObject *module, int extra_base_size,
int basicsize, int member_offset,
int add_relative_flag,
const char *member_name, int member_flags,
int member_type)
/*[clinic end generated code: output=7005db9a07396997 input=007e29cdbe1d3390]*/
{
PyObject *base = NULL;
PyObject *result = NULL;
int extra_base_size, basicsize, offset, add_flag;
int r = PyArg_ParseTuple(args, "iiip", &extra_base_size, &basicsize, &offset, &add_flag);
if (!r) {
goto finally;
}
PyType_Spec base_spec = {
.name = "_testcapi.Base",
.basicsize = sizeof(PyObject) + extra_base_size,
@ -281,7 +370,8 @@ make_heaptype_with_member(PyObject *module, PyObject *args)
}
PyMemberDef members[] = {
{"memb", Py_T_BYTE, offset, add_flag ? Py_RELATIVE_OFFSET : 0},
{member_name, member_type, member_offset,
member_flags | (add_relative_flag ? Py_RELATIVE_OFFSET : 0)},
{0},
};
PyType_Slot slots[] = {
@ -325,7 +415,7 @@ static PyMethodDef TestMethods[] = {
{"make_sized_heaptypes", make_sized_heaptypes, METH_VARARGS},
{"subclass_var_heaptype", subclass_var_heaptype, METH_VARARGS},
{"subclass_heaptype", subclass_heaptype, METH_VARARGS},
{"make_heaptype_with_member", make_heaptype_with_member, METH_VARARGS},
MAKE_HEAPTYPE_WITH_MEMBER_METHODDEF
{"test_alignof_max_align_t", test_alignof_max_align_t, METH_NOARGS},
{NULL},
};
@ -341,5 +431,42 @@ _PyTestLimitedCAPI_Init_HeaptypeRelative(PyObject *m)
return -1;
}
#define ADD_FROM_SPEC(SPEC) do { \
PyObject *tp = PyType_FromSpec(SPEC); \
if (!tp) { \
return -1; \
} \
if (PyModule_AddType(m, (PyTypeObject *)tp) < 0) { \
return -1; \
} \
} while (0)
PyObject *tp;
tp = PyType_FromSpec(&HeapCTypeWithRelativeDict_spec);
if (!tp) {
return -1;
}
if (PyModule_AddType(m, (PyTypeObject *)tp) < 0) {
return -1;
}
Py_DECREF(tp);
tp = PyType_FromSpec(&HeapCTypeWithRelativeWeakref_spec);
if (!tp) {
return -1;
}
if (PyModule_AddType(m, (PyTypeObject *)tp) < 0) {
return -1;
}
Py_DECREF(tp);
if (PyModule_AddIntMacro(m, Py_T_PYSSIZET) < 0) {
return -1;
}
if (PyModule_AddIntMacro(m, Py_READONLY) < 0) {
return -1;
}
return 0;
}

48
Modules/_testlimitedcapi/vectorcall_limited.c
View File

@ -6,6 +6,8 @@
# define Py_LIMITED_API 0x030c0000
#endif
#include <stddef.h> // offsetof
#include "parts.h"
#include "clinic/vectorcall_limited.c.h"
@ -175,6 +177,41 @@ static PyType_Spec LimitedVectorCallClass_spec = {
.slots = LimitedVectorallClass_slots,
};
typedef struct {
vectorcallfunc vfunc;
} LimitedRelativeVectorCallStruct;
static PyObject *
LimitedRelativeVectorCallClass_new(PyTypeObject *tp, PyTypeObject *a, PyTypeObject *kw)
{
PyObject *self = ((allocfunc)PyType_GetSlot(tp, Py_tp_alloc))(tp, 0);
if (!self) {
return NULL;
}
LimitedRelativeVectorCallStruct *data = PyObject_GetTypeData(self, tp);
data->vfunc = LimitedVectorCallClass_vectorcall;
return self;
}
static PyType_Spec LimitedRelativeVectorCallClass_spec = {
.name = "_testlimitedcapi.LimitedRelativeVectorCallClass",
.basicsize = -(int)sizeof(LimitedRelativeVectorCallStruct),
.flags = Py_TPFLAGS_DEFAULT
| Py_TPFLAGS_HAVE_VECTORCALL,
.slots = (PyType_Slot[]) {
{Py_tp_new, LimitedRelativeVectorCallClass_new},
{Py_tp_call, LimitedVectorCallClass_tpcall},
{Py_tp_members, (PyMemberDef[]){
{"__vectorcalloffset__", Py_T_PYSSIZET,
offsetof(LimitedRelativeVectorCallStruct, vfunc),
Py_READONLY | Py_RELATIVE_OFFSET},
{NULL}
}},
{0}
},
};
static PyMethodDef TestMethods[] = {
_TESTLIMITEDCAPI_CALL_VECTORCALL_METHODDEF
_TESTLIMITEDCAPI_CALL_VECTORCALL_METHOD_METHODDEF
@ -197,5 +234,16 @@ _PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *m)
return -1;
}
Py_DECREF(LimitedVectorCallClass);
PyObject *LimitedRelativeVectorCallClass = PyType_FromModuleAndSpec(
m, &LimitedRelativeVectorCallClass_spec, NULL);
if (!LimitedRelativeVectorCallClass) {
return -1;
}
if (PyModule_AddType(m, (PyTypeObject *)LimitedRelativeVectorCallClass) < 0) {
return -1;
}
Py_DECREF(LimitedRelativeVectorCallClass);
return 0;
}

85
Objects/typeobject.c
View File

@ -4642,6 +4642,41 @@ check_basicsize_includes_size_and_offsets(PyTypeObject* type)
return 1;
}
/* Set *dest to the offset specified by a special "__*offset__" member.
* Return 0 on success, -1 on failure.
*/
static inline int
special_offset_from_member(
const PyMemberDef *memb /* may be NULL */,
Py_ssize_t type_data_offset,
Py_ssize_t *dest /* not NULL */)
{
if (memb == NULL) {
*dest = 0;
return 0;
}
if (memb->type != Py_T_PYSSIZET) {
PyErr_Format(
PyExc_SystemError,
"type of %s must be Py_T_PYSSIZET",
memb->name);
return -1;
}
if (memb->flags == Py_READONLY) {
*dest = memb->offset;
return 0;
}
else if (memb->flags == (Py_READONLY | Py_RELATIVE_OFFSET)) {
*dest = memb->offset + type_data_offset;
return 0;
}
PyErr_Format(
PyExc_SystemError,
"flags for %s must be Py_READONLY or (Py_READONLY | Py_RELATIVE_OFFSET)",
memb->name);
return -1;
}
static PyObject *
_PyType_FromMetaclass_impl(
PyTypeObject *metaclass, PyObject *module,
@ -4667,10 +4702,11 @@ _PyType_FromMetaclass_impl(
const PyType_Slot *slot;
Py_ssize_t nmembers = 0;
Py_ssize_t weaklistoffset, dictoffset, vectorcalloffset;
const PyMemberDef *weaklistoffset_member = NULL;
const PyMemberDef *dictoffset_member = NULL;
const PyMemberDef *vectorcalloffset_member = NULL;
char *res_start;
nmembers = weaklistoffset = dictoffset = vectorcalloffset = 0;
for (slot = spec->slots; slot->slot; slot++) {
if (slot->slot < 0
|| (size_t)slot->slot >= Py_ARRAY_LENGTH(pyslot_offsets)) {
@ -4687,24 +4723,6 @@ _PyType_FromMetaclass_impl(
}
for (const PyMemberDef *memb = slot->pfunc; memb->name != NULL; memb++) {
nmembers++;
if (strcmp(memb->name, "__weaklistoffset__") == 0) {
// The PyMemberDef must be a Py_ssize_t and readonly
assert(memb->type == Py_T_PYSSIZET);
assert(memb->flags == Py_READONLY);
weaklistoffset = memb->offset;
}
if (strcmp(memb->name, "__dictoffset__") == 0) {
// The PyMemberDef must be a Py_ssize_t and readonly
assert(memb->type == Py_T_PYSSIZET);
assert(memb->flags == Py_READONLY);
dictoffset = memb->offset;
}
if (strcmp(memb->name, "__vectorcalloffset__") == 0) {
// The PyMemberDef must be a Py_ssize_t and readonly
assert(memb->type == Py_T_PYSSIZET);
assert(memb->flags == Py_READONLY);
vectorcalloffset = memb->offset;
}
if (memb->flags & Py_RELATIVE_OFFSET) {
if (spec->basicsize > 0) {
PyErr_SetString(
@ -4719,6 +4737,15 @@ _PyType_FromMetaclass_impl(
goto finally;
}
}
if (strcmp(memb->name, "__weaklistoffset__") == 0) {
weaklistoffset_member = memb;
}
if (strcmp(memb->name, "__dictoffset__") == 0) {
dictoffset_member = memb;
}
if (strcmp(memb->name, "__vectorcalloffset__") == 0) {
vectorcalloffset_member = memb;
}
}
break;
case Py_tp_doc:
@ -4882,6 +4909,24 @@ _PyType_FromMetaclass_impl(
Py_ssize_t itemsize = spec->itemsize;
/* Compute special offsets */
Py_ssize_t weaklistoffset = 0;
if (special_offset_from_member(weaklistoffset_member, type_data_offset,
&weaklistoffset) < 0) {
goto finally;
}
Py_ssize_t dictoffset = 0;
if (special_offset_from_member(dictoffset_member, type_data_offset,
&dictoffset) < 0) {
goto finally;
}
Py_ssize_t vectorcalloffset = 0;
if (special_offset_from_member(vectorcalloffset_member, type_data_offset,
&vectorcalloffset) < 0) {
goto finally;
}
/* Allocate the new type
*
* Between here and PyType_Ready, we should limit: