traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-38140: Make dict and weakref offsets opaque for C heap types (#16076) 

* Make dict and weakref offsets opaque for C heap types

* Add news
This commit is contained in:
Eddie Elizondo 2019-09-19 09:29:05 -07:00 committed by Dino Viehland
parent 079931d122
commit 3368f3c6ae
8 changed files with 188 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Doc/c-api/type.rst
View File

@ -187,9 +187,8 @@ The following functions and structs are used to create
* :c:member:`~PyTypeObject.tp_cache`
* :c:member:`~PyTypeObject.tp_subclasses`
* :c:member:`~PyTypeObject.tp_weaklist`
* :c:member:`~PyTypeObject.tp_vectorcall`
* :c:member:`~PyTypeObject.tp_print`
* :c:member:`~PyTypeObject.tp_weaklistoffset`
* :c:member:`~PyTypeObject.tp_dictoffset`
* :c:member:`~PyBufferProcs.bf_getbuffer`
* :c:member:`~PyBufferProcs.bf_releasebuffer`

27
Lib/test/test_capi.py
View File

@ -12,6 +12,7 @@ import textwrap
import threading
import time
import unittest
import weakref
from test import support
from test.support import MISSING_C_DOCSTRINGS
from test.support.script_helper import assert_python_failure, assert_python_ok
@ -437,6 +438,32 @@ class CAPITest(unittest.TestCase):
# Test that subtype_dealloc decref the newly assigned __class__ only once
self.assertEqual(new_type_refcnt, sys.getrefcount(A))
def test_heaptype_with_dict(self):
inst = _testcapi.HeapCTypeWithDict()
inst.foo = 42
self.assertEqual(inst.foo, 42)
self.assertEqual(inst.dictobj, inst.__dict__)
self.assertEqual(inst.dictobj, {"foo": 42})
inst = _testcapi.HeapCTypeWithDict()
self.assertEqual({}, inst.__dict__)
def test_heaptype_with_negative_dict(self):
inst = _testcapi.HeapCTypeWithNegativeDict()
inst.foo = 42
self.assertEqual(inst.foo, 42)
self.assertEqual(inst.dictobj, inst.__dict__)
self.assertEqual(inst.dictobj, {"foo": 42})
inst = _testcapi.HeapCTypeWithNegativeDict()
self.assertEqual({}, inst.__dict__)
def test_heaptype_with_weakref(self):
inst = _testcapi.HeapCTypeWithWeakref()
ref = weakref.ref(inst)
self.assertEqual(ref(), inst)
self.assertEqual(inst.weakreflist, ref)
def test_c_subclass_of_heap_ctype_with_tpdealloc_decrefs_once(self):
subclass_instance = _testcapi.HeapCTypeSubclass()
type_refcnt = sys.getrefcount(_testcapi.HeapCTypeSubclass)

2
Misc/NEWS.d/next/C API/2019-09-13-01-24-47.bpo-38140.y59qaO.rst Normal file
View File

@ -0,0 +1,2 @@
Make dict and weakref offsets opaque for C heap types by passing the offsets
through PyMemberDef

2
Modules/_struct.c
View File

@ -2024,7 +2024,7 @@ static struct PyMethodDef s_methods[] = {
};
static PyMemberDef s_members[] = {
{"__weaklistoffset__", T_NONE, offsetof(PyStructObject, weakreflist), READONLY},
{"__weaklistoffset__", T_PYSSIZET, offsetof(PyStructObject, weakreflist), READONLY},
{NULL} /* sentinel */
};

118
Modules/_testcapimodule.c
View File

@ -6363,6 +6363,106 @@ static PyType_Spec HeapCTypeSubclassWithFinalizer_spec = {
HeapCTypeSubclassWithFinalizer_slots
};
typedef struct {
PyObject_HEAD
PyObject *dict;
} HeapCTypeWithDictObject;
static void
heapctypewithdict_dealloc(HeapCTypeWithDictObject* self)
{
PyTypeObject *tp = Py_TYPE(self);
Py_XDECREF(self->dict);
PyObject_DEL(self);
Py_DECREF(tp);
}
static PyGetSetDef heapctypewithdict_getsetlist[] = {
{"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict},
{NULL} /* Sentinel */
};
static struct PyMemberDef heapctypewithdict_members[] = {
{"dictobj", T_OBJECT, offsetof(HeapCTypeWithDictObject, dict)},
{"__dictoffset__", T_PYSSIZET, offsetof(HeapCTypeWithDictObject, dict), READONLY},
{NULL} /* Sentinel */
};
static PyType_Slot HeapCTypeWithDict_slots[] = {
{Py_tp_members, heapctypewithdict_members},
{Py_tp_getset, heapctypewithdict_getsetlist},
{Py_tp_dealloc, heapctypewithdict_dealloc},
{0, 0},
};
static PyType_Spec HeapCTypeWithDict_spec = {
"_testcapi.HeapCTypeWithDict",
sizeof(HeapCTypeWithDictObject),
0,
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
HeapCTypeWithDict_slots
};
static struct PyMemberDef heapctypewithnegativedict_members[] = {
{"dictobj", T_OBJECT, offsetof(HeapCTypeWithDictObject, dict)},
{"__dictoffset__", T_PYSSIZET, -sizeof(void*), READONLY},
{NULL} /* Sentinel */
};
static PyType_Slot HeapCTypeWithNegativeDict_slots[] = {
{Py_tp_members, heapctypewithnegativedict_members},
{Py_tp_getset, heapctypewithdict_getsetlist},
{Py_tp_dealloc, heapctypewithdict_dealloc},
{0, 0},
};
static PyType_Spec HeapCTypeWithNegativeDict_spec = {
"_testcapi.HeapCTypeWithNegativeDict",
sizeof(HeapCTypeWithDictObject),
0,
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
HeapCTypeWithNegativeDict_slots
};
typedef struct {
PyObject_HEAD
PyObject *weakreflist;
} HeapCTypeWithWeakrefObject;
static struct PyMemberDef heapctypewithweakref_members[] = {
{"weakreflist", T_OBJECT, offsetof(HeapCTypeWithWeakrefObject, weakreflist)},
{"__weaklistoffset__", T_PYSSIZET,
offsetof(HeapCTypeWithWeakrefObject, weakreflist), READONLY},
{NULL} /* Sentinel */
};
static void
heapctypewithweakref_dealloc(HeapCTypeWithWeakrefObject* self)
{
PyTypeObject *tp = Py_TYPE(self);
if (self->weakreflist != NULL)
PyObject_ClearWeakRefs((PyObject *) self);
Py_XDECREF(self->weakreflist);
PyObject_DEL(self);
Py_DECREF(tp);
}
static PyType_Slot HeapCTypeWithWeakref_slots[] = {
{Py_tp_members, heapctypewithweakref_members},
{Py_tp_dealloc, heapctypewithweakref_dealloc},
{0, 0},
};
static PyType_Spec HeapCTypeWithWeakref_spec = {
"_testcapi.HeapCTypeWithWeakref",
sizeof(HeapCTypeWithWeakrefObject),
0,
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
HeapCTypeWithWeakref_slots
};
static PyMethodDef meth_instance_methods[] = {
{"meth_varargs", meth_varargs, METH_VARARGS},
{"meth_varargs_keywords", (PyCFunction)(void(*)(void))meth_varargs_keywords, METH_VARARGS|METH_KEYWORDS},
@ -6596,6 +6696,24 @@ PyInit__testcapi(void)
Py_DECREF(subclass_bases);
PyModule_AddObject(m, "HeapCTypeSubclass", HeapCTypeSubclass);
PyObject *HeapCTypeWithDict = PyType_FromSpec(&HeapCTypeWithDict_spec);
if (HeapCTypeWithDict == NULL) {
return NULL;
}
PyModule_AddObject(m, "HeapCTypeWithDict", HeapCTypeWithDict);
PyObject *HeapCTypeWithNegativeDict = PyType_FromSpec(&HeapCTypeWithNegativeDict_spec);
if (HeapCTypeWithNegativeDict == NULL) {
return NULL;
}
PyModule_AddObject(m, "HeapCTypeWithNegativeDict", HeapCTypeWithNegativeDict);
PyObject *HeapCTypeWithWeakref = PyType_FromSpec(&HeapCTypeWithWeakref_spec);
if (HeapCTypeWithWeakref == NULL) {
return NULL;
}
PyModule_AddObject(m, "HeapCTypeWithWeakref", HeapCTypeWithWeakref);
PyObject *subclass_with_finalizer_bases = PyTuple_Pack(1, HeapCTypeSubclass);
if (subclass_with_finalizer_bases == NULL) {
return NULL;

27
Objects/typeobject.c
View File

@ -2853,15 +2853,27 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
PyTypeObject *type, *base;
PyType_Slot *slot;
Py_ssize_t nmembers;
Py_ssize_t nmembers, weaklistoffset, dictoffset;
char *s, *res_start;
nmembers = 0;
nmembers = weaklistoffset = dictoffset = 0;
for (slot = spec->slots; slot->slot; slot++) {
if (slot->slot == Py_tp_members) {
nmembers = 0;
for (memb = slot->pfunc; memb->name != NULL; memb++) {
nmembers++;
if (strcmp(memb->name, "__weaklistoffset__") == 0) {
// The PyMemberDef must be a Py_ssize_t and readonly
assert(memb->type == T_PYSSIZET);
assert(memb->flags == READONLY);
weaklistoffset = memb->offset;
}
if (strcmp(memb->name, "__dictoffset__") == 0) {
// The PyMemberDef must be a Py_ssize_t and readonly
assert(memb->type == T_PYSSIZET);
assert(memb->flags == READONLY);
dictoffset = memb->offset;
}
}
}
}
@ -2990,6 +3002,17 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
res->ht_cached_keys = _PyDict_NewKeysForClass();
}
if (weaklistoffset) {
type->tp_weaklistoffset = weaklistoffset;
if (PyDict_DelItemString((PyObject *)type->tp_dict, "__weaklistoffset__") < 0)
goto fail;
}
if (dictoffset) {
type->tp_dictoffset = dictoffset;
if (PyDict_DelItemString((PyObject *)type->tp_dict, "__dictoffset__") < 0)
goto fail;
}
/* Set type.__module__ */
s = strrchr(spec->name, '.');
if (s != NULL) {

8
Parser/asdl_c.py
View File

@ -724,6 +724,11 @@ ast_type_reduce(PyObject *self, PyObject *unused)
return Py_BuildValue("O()", Py_TYPE(self));
}
static PyMemberDef ast_type_members[] = {
{"__dictoffset__", T_PYSSIZET, offsetof(AST_object, dict), READONLY},
{NULL} /* Sentinel */
};
static PyMethodDef ast_type_methods[] = {
{"__reduce__", ast_type_reduce, METH_NOARGS, NULL},
{NULL}
@ -740,6 +745,7 @@ static PyType_Slot AST_type_slots[] = {
{Py_tp_setattro, PyObject_GenericSetAttr},
{Py_tp_traverse, ast_traverse},
{Py_tp_clear, ast_clear},
{Py_tp_members, ast_type_members},
{Py_tp_methods, ast_type_methods},
{Py_tp_getset, ast_type_getsets},
{Py_tp_init, ast_type_init},
@ -930,7 +936,6 @@ static int add_ast_fields(void)
self.emit("if (init_identifiers() < 0) return 0;", 1)
self.emit("state->AST_type = PyType_FromSpec(&AST_type_spec);", 1)
self.emit("if (!state->AST_type) return 0;", 1)
self.emit("((PyTypeObject*)state->AST_type)->tp_dictoffset = offsetof(AST_object, dict);", 1)
self.emit("if (add_ast_fields() < 0) return 0;", 1)
for dfn in mod.dfns:
self.visit(dfn)
@ -1372,6 +1377,7 @@ def main(srcfile, dump_module=False):
f.write('\n')
f.write('#include "Python.h"\n')
f.write('#include "%s-ast.h"\n' % mod.name)
f.write('#include "structmember.h"\n')
f.write('\n')
generate_module_def(f, mod)

9
Python/Python-ast.c generated
View File

@ -4,6 +4,7 @@
#include "Python.h"
#include "Python-ast.h"
#include "structmember.h"
typedef struct {
int initialized;
@ -1216,6 +1217,11 @@ ast_type_reduce(PyObject *self, PyObject *unused)
return Py_BuildValue("O()", Py_TYPE(self));
}
static PyMemberDef ast_type_members[] = {
{"__dictoffset__", T_PYSSIZET, offsetof(AST_object, dict), READONLY},
{NULL} /* Sentinel */
};
static PyMethodDef ast_type_methods[] = {
{"__reduce__", ast_type_reduce, METH_NOARGS, NULL},
{NULL}
@ -1232,6 +1238,7 @@ static PyType_Slot AST_type_slots[] = {
{Py_tp_setattro, PyObject_GenericSetAttr},
{Py_tp_traverse, ast_traverse},
{Py_tp_clear, ast_clear},
{Py_tp_members, ast_type_members},
{Py_tp_methods, ast_type_methods},
{Py_tp_getset, ast_type_getsets},
{Py_tp_init, ast_type_init},
@ -1421,8 +1428,6 @@ static int init_types(void)
if (init_identifiers() < 0) return 0;
state->AST_type = PyType_FromSpec(&AST_type_spec);
if (!state->AST_type) return 0;
((PyTypeObject*)state->AST_type)->tp_dictoffset = offsetof(AST_object,
dict);
if (add_ast_fields() < 0) return 0;
state->mod_type = make_type("mod", state->AST_type, NULL, 0);
if (!state->mod_type) return 0;