From 3368f3c6ae4140a0883e19350e672fd09c9db616 Mon Sep 17 00:00:00 2001 From: Eddie Elizondo Date: Thu, 19 Sep 2019 09:29:05 -0700 Subject: [PATCH] bpo-38140: Make dict and weakref offsets opaque for C heap types (#16076) * Make dict and weakref offsets opaque for C heap types * Add news --- Doc/c-api/type.rst | 3 +- Lib/test/test_capi.py | 27 ++++ .../2019-09-13-01-24-47.bpo-38140.y59qaO.rst | 2 + Modules/_struct.c | 2 +- Modules/_testcapimodule.c | 118 ++++++++++++++++++ Objects/typeobject.c | 27 +++- Parser/asdl_c.py | 8 +- Python/Python-ast.c | 9 +- 8 files changed, 188 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2019-09-13-01-24-47.bpo-38140.y59qaO.rst diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 6416951ac5a..ad3e022d42c 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -187,9 +187,8 @@ The following functions and structs are used to create * :c:member:`~PyTypeObject.tp_cache` * :c:member:`~PyTypeObject.tp_subclasses` * :c:member:`~PyTypeObject.tp_weaklist` + * :c:member:`~PyTypeObject.tp_vectorcall` * :c:member:`~PyTypeObject.tp_print` - * :c:member:`~PyTypeObject.tp_weaklistoffset` - * :c:member:`~PyTypeObject.tp_dictoffset` * :c:member:`~PyBufferProcs.bf_getbuffer` * :c:member:`~PyBufferProcs.bf_releasebuffer` diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index b1d045c5308..79059bc7c96 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -12,6 +12,7 @@ import textwrap import threading import time import unittest +import weakref from test import support from test.support import MISSING_C_DOCSTRINGS from test.support.script_helper import assert_python_failure, assert_python_ok @@ -437,6 +438,32 @@ class CAPITest(unittest.TestCase): # Test that subtype_dealloc decref the newly assigned __class__ only once self.assertEqual(new_type_refcnt, sys.getrefcount(A)) + def test_heaptype_with_dict(self): + inst = _testcapi.HeapCTypeWithDict() + inst.foo = 42 + self.assertEqual(inst.foo, 42) + self.assertEqual(inst.dictobj, inst.__dict__) + self.assertEqual(inst.dictobj, {"foo": 42}) + + inst = _testcapi.HeapCTypeWithDict() + self.assertEqual({}, inst.__dict__) + + def test_heaptype_with_negative_dict(self): + inst = _testcapi.HeapCTypeWithNegativeDict() + inst.foo = 42 + self.assertEqual(inst.foo, 42) + self.assertEqual(inst.dictobj, inst.__dict__) + self.assertEqual(inst.dictobj, {"foo": 42}) + + inst = _testcapi.HeapCTypeWithNegativeDict() + self.assertEqual({}, inst.__dict__) + + def test_heaptype_with_weakref(self): + inst = _testcapi.HeapCTypeWithWeakref() + ref = weakref.ref(inst) + self.assertEqual(ref(), inst) + self.assertEqual(inst.weakreflist, ref) + def test_c_subclass_of_heap_ctype_with_tpdealloc_decrefs_once(self): subclass_instance = _testcapi.HeapCTypeSubclass() type_refcnt = sys.getrefcount(_testcapi.HeapCTypeSubclass) diff --git a/Misc/NEWS.d/next/C API/2019-09-13-01-24-47.bpo-38140.y59qaO.rst b/Misc/NEWS.d/next/C API/2019-09-13-01-24-47.bpo-38140.y59qaO.rst new file mode 100644 index 00000000000..7df1d25632b --- /dev/null +++ b/Misc/NEWS.d/next/C API/2019-09-13-01-24-47.bpo-38140.y59qaO.rst @@ -0,0 +1,2 @@ +Make dict and weakref offsets opaque for C heap types by passing the offsets +through PyMemberDef diff --git a/Modules/_struct.c b/Modules/_struct.c index c387ae2ffc1..cc536b46a62 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -2024,7 +2024,7 @@ static struct PyMethodDef s_methods[] = { }; static PyMemberDef s_members[] = { - {"__weaklistoffset__", T_NONE, offsetof(PyStructObject, weakreflist), READONLY}, + {"__weaklistoffset__", T_PYSSIZET, offsetof(PyStructObject, weakreflist), READONLY}, {NULL} /* sentinel */ }; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 4cca784d014..30a786cd345 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -6363,6 +6363,106 @@ static PyType_Spec HeapCTypeSubclassWithFinalizer_spec = { HeapCTypeSubclassWithFinalizer_slots }; +typedef struct { + PyObject_HEAD + PyObject *dict; +} HeapCTypeWithDictObject; + +static void +heapctypewithdict_dealloc(HeapCTypeWithDictObject* self) +{ + + PyTypeObject *tp = Py_TYPE(self); + Py_XDECREF(self->dict); + PyObject_DEL(self); + Py_DECREF(tp); +} + +static PyGetSetDef heapctypewithdict_getsetlist[] = { + {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, + {NULL} /* Sentinel */ +}; + +static struct PyMemberDef heapctypewithdict_members[] = { + {"dictobj", T_OBJECT, offsetof(HeapCTypeWithDictObject, dict)}, + {"__dictoffset__", T_PYSSIZET, offsetof(HeapCTypeWithDictObject, dict), READONLY}, + {NULL} /* Sentinel */ +}; + +static PyType_Slot HeapCTypeWithDict_slots[] = { + {Py_tp_members, heapctypewithdict_members}, + {Py_tp_getset, heapctypewithdict_getsetlist}, + {Py_tp_dealloc, heapctypewithdict_dealloc}, + {0, 0}, +}; + +static PyType_Spec HeapCTypeWithDict_spec = { + "_testcapi.HeapCTypeWithDict", + sizeof(HeapCTypeWithDictObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + HeapCTypeWithDict_slots +}; + +static struct PyMemberDef heapctypewithnegativedict_members[] = { + {"dictobj", T_OBJECT, offsetof(HeapCTypeWithDictObject, dict)}, + {"__dictoffset__", T_PYSSIZET, -sizeof(void*), READONLY}, + {NULL} /* Sentinel */ +}; + +static PyType_Slot HeapCTypeWithNegativeDict_slots[] = { + {Py_tp_members, heapctypewithnegativedict_members}, + {Py_tp_getset, heapctypewithdict_getsetlist}, + {Py_tp_dealloc, heapctypewithdict_dealloc}, + {0, 0}, +}; + +static PyType_Spec HeapCTypeWithNegativeDict_spec = { + "_testcapi.HeapCTypeWithNegativeDict", + sizeof(HeapCTypeWithDictObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + HeapCTypeWithNegativeDict_slots +}; + +typedef struct { + PyObject_HEAD + PyObject *weakreflist; +} HeapCTypeWithWeakrefObject; + +static struct PyMemberDef heapctypewithweakref_members[] = { + {"weakreflist", T_OBJECT, offsetof(HeapCTypeWithWeakrefObject, weakreflist)}, + {"__weaklistoffset__", T_PYSSIZET, + offsetof(HeapCTypeWithWeakrefObject, weakreflist), READONLY}, + {NULL} /* Sentinel */ +}; + +static void +heapctypewithweakref_dealloc(HeapCTypeWithWeakrefObject* self) +{ + + PyTypeObject *tp = Py_TYPE(self); + if (self->weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *) self); + Py_XDECREF(self->weakreflist); + PyObject_DEL(self); + Py_DECREF(tp); +} + +static PyType_Slot HeapCTypeWithWeakref_slots[] = { + {Py_tp_members, heapctypewithweakref_members}, + {Py_tp_dealloc, heapctypewithweakref_dealloc}, + {0, 0}, +}; + +static PyType_Spec HeapCTypeWithWeakref_spec = { + "_testcapi.HeapCTypeWithWeakref", + sizeof(HeapCTypeWithWeakrefObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + HeapCTypeWithWeakref_slots +}; + static PyMethodDef meth_instance_methods[] = { {"meth_varargs", meth_varargs, METH_VARARGS}, {"meth_varargs_keywords", (PyCFunction)(void(*)(void))meth_varargs_keywords, METH_VARARGS|METH_KEYWORDS}, @@ -6596,6 +6696,24 @@ PyInit__testcapi(void) Py_DECREF(subclass_bases); PyModule_AddObject(m, "HeapCTypeSubclass", HeapCTypeSubclass); + PyObject *HeapCTypeWithDict = PyType_FromSpec(&HeapCTypeWithDict_spec); + if (HeapCTypeWithDict == NULL) { + return NULL; + } + PyModule_AddObject(m, "HeapCTypeWithDict", HeapCTypeWithDict); + + PyObject *HeapCTypeWithNegativeDict = PyType_FromSpec(&HeapCTypeWithNegativeDict_spec); + if (HeapCTypeWithNegativeDict == NULL) { + return NULL; + } + PyModule_AddObject(m, "HeapCTypeWithNegativeDict", HeapCTypeWithNegativeDict); + + PyObject *HeapCTypeWithWeakref = PyType_FromSpec(&HeapCTypeWithWeakref_spec); + if (HeapCTypeWithWeakref == NULL) { + return NULL; + } + PyModule_AddObject(m, "HeapCTypeWithWeakref", HeapCTypeWithWeakref); + PyObject *subclass_with_finalizer_bases = PyTuple_Pack(1, HeapCTypeSubclass); if (subclass_with_finalizer_bases == NULL) { return NULL; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index dfdac9e2e4f..94a4da22d67 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2853,15 +2853,27 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) PyTypeObject *type, *base; PyType_Slot *slot; - Py_ssize_t nmembers; + Py_ssize_t nmembers, weaklistoffset, dictoffset; char *s, *res_start; - nmembers = 0; + nmembers = weaklistoffset = dictoffset = 0; for (slot = spec->slots; slot->slot; slot++) { if (slot->slot == Py_tp_members) { nmembers = 0; for (memb = slot->pfunc; memb->name != NULL; memb++) { nmembers++; + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + // The PyMemberDef must be a Py_ssize_t and readonly + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + weaklistoffset = memb->offset; + } + if (strcmp(memb->name, "__dictoffset__") == 0) { + // The PyMemberDef must be a Py_ssize_t and readonly + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + dictoffset = memb->offset; + } } } } @@ -2990,6 +3002,17 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) res->ht_cached_keys = _PyDict_NewKeysForClass(); } + if (weaklistoffset) { + type->tp_weaklistoffset = weaklistoffset; + if (PyDict_DelItemString((PyObject *)type->tp_dict, "__weaklistoffset__") < 0) + goto fail; + } + if (dictoffset) { + type->tp_dictoffset = dictoffset; + if (PyDict_DelItemString((PyObject *)type->tp_dict, "__dictoffset__") < 0) + goto fail; + } + /* Set type.__module__ */ s = strrchr(spec->name, '.'); if (s != NULL) { diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 0b72f9ee236..3a9d4e6b4b9 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -724,6 +724,11 @@ ast_type_reduce(PyObject *self, PyObject *unused) return Py_BuildValue("O()", Py_TYPE(self)); } +static PyMemberDef ast_type_members[] = { + {"__dictoffset__", T_PYSSIZET, offsetof(AST_object, dict), READONLY}, + {NULL} /* Sentinel */ +}; + static PyMethodDef ast_type_methods[] = { {"__reduce__", ast_type_reduce, METH_NOARGS, NULL}, {NULL} @@ -740,6 +745,7 @@ static PyType_Slot AST_type_slots[] = { {Py_tp_setattro, PyObject_GenericSetAttr}, {Py_tp_traverse, ast_traverse}, {Py_tp_clear, ast_clear}, + {Py_tp_members, ast_type_members}, {Py_tp_methods, ast_type_methods}, {Py_tp_getset, ast_type_getsets}, {Py_tp_init, ast_type_init}, @@ -930,7 +936,6 @@ static int add_ast_fields(void) self.emit("if (init_identifiers() < 0) return 0;", 1) self.emit("state->AST_type = PyType_FromSpec(&AST_type_spec);", 1) self.emit("if (!state->AST_type) return 0;", 1) - self.emit("((PyTypeObject*)state->AST_type)->tp_dictoffset = offsetof(AST_object, dict);", 1) self.emit("if (add_ast_fields() < 0) return 0;", 1) for dfn in mod.dfns: self.visit(dfn) @@ -1372,6 +1377,7 @@ def main(srcfile, dump_module=False): f.write('\n') f.write('#include "Python.h"\n') f.write('#include "%s-ast.h"\n' % mod.name) + f.write('#include "structmember.h"\n') f.write('\n') generate_module_def(f, mod) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 7a38e98c435..558d9ebea5c 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -4,6 +4,7 @@ #include "Python.h" #include "Python-ast.h" +#include "structmember.h" typedef struct { int initialized; @@ -1216,6 +1217,11 @@ ast_type_reduce(PyObject *self, PyObject *unused) return Py_BuildValue("O()", Py_TYPE(self)); } +static PyMemberDef ast_type_members[] = { + {"__dictoffset__", T_PYSSIZET, offsetof(AST_object, dict), READONLY}, + {NULL} /* Sentinel */ +}; + static PyMethodDef ast_type_methods[] = { {"__reduce__", ast_type_reduce, METH_NOARGS, NULL}, {NULL} @@ -1232,6 +1238,7 @@ static PyType_Slot AST_type_slots[] = { {Py_tp_setattro, PyObject_GenericSetAttr}, {Py_tp_traverse, ast_traverse}, {Py_tp_clear, ast_clear}, + {Py_tp_members, ast_type_members}, {Py_tp_methods, ast_type_methods}, {Py_tp_getset, ast_type_getsets}, {Py_tp_init, ast_type_init}, @@ -1421,8 +1428,6 @@ static int init_types(void) if (init_identifiers() < 0) return 0; state->AST_type = PyType_FromSpec(&AST_type_spec); if (!state->AST_type) return 0; - ((PyTypeObject*)state->AST_type)->tp_dictoffset = offsetof(AST_object, - dict); if (add_ast_fields() < 0) return 0; state->mod_type = make_type("mod", state->AST_type, NULL, 0); if (!state->mod_type) return 0;