traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-40137: Convert _functools module to use PyType_FromModuleAndSpec. (GH-23405)

This commit is contained in:
Hai Shi 2020-12-29 20:45:07 +08:00 committed by GitHub
parent 84402eb110
commit dd39123970
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 255 additions and 224 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Lib/test/test_functools.py
View File

@ -27,8 +27,7 @@ import functools
py_functools = import_helper.import_fresh_module('functools',
blocked=['_functools'])
c_functools = import_helper.import_fresh_module('functools',
fresh=['_functools'])
c_functools = import_helper.import_fresh_module('functools')
decimal = import_helper.import_fresh_module('decimal', fresh=['_decimal'])

1
Misc/NEWS.d/next/Core and Builtins/2020-11-19-23-12-57.bpo-40137.bihl9O.rst Normal file
View File

@ -0,0 +1 @@
Convert functools module to use :c:func:`PyType_FromModuleAndSpec`.

469
Modules/_functoolsmodule.c
View File

@ -24,9 +24,37 @@ typedef struct {
vectorcallfunc vectorcall;
} partialobject;
static PyTypeObject partial_type;
typedef struct _functools_state {
/* this object is used delimit args and keywords in the cache keys */
PyObject *kwd_mark;
PyTypeObject *partial_type;
PyTypeObject *keyobject_type;
PyTypeObject *lru_list_elem_type;
} _functools_state;
static inline _functools_state *
get_functools_state(PyObject *module)
{
void *state = PyModule_GetState(module);
assert(state != NULL);
return (_functools_state *)state;
}
static void partial_setvectorcall(partialobject *pto);
static struct PyModuleDef _functools_module;
static PyObject *
partial_call(partialobject *pto, PyObject *args, PyObject *kwargs);
static inline _functools_state *
get_functools_state_by_type(PyTypeObject *type)
{
PyObject *module = _PyType_GetModuleByDef(type, &_functools_module);
if (module == NULL) {
return NULL;
}
_functools_state *state = get_functools_state(module);
return state;
}
static PyObject *
partial_new(PyTypeObject *type, PyObject *args, PyObject *kw)
@ -42,7 +70,11 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw)
pargs = pkw = NULL;
func = PyTuple_GET_ITEM(args, 0);
if (Py_IS_TYPE(func, &partial_type) && type == &partial_type) {
if (Py_TYPE(func)->tp_call == (ternaryfunc)partial_call) {
// The type of "func" might not be exactly the same type object
// as "type", but if it is called using partial_call, it must have the
// same memory layout (fn, args and kw members).
// We can use its underlying function directly and merge the arguments.
partialobject *part = (partialobject *)func;
if (part->dict == NULL) {
pargs = part->args;
@ -117,6 +149,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw)
static void
partial_dealloc(partialobject *pto)
{
PyTypeObject *tp = Py_TYPE(pto);
/* bpo-31095: UnTrack is needed before calling any callbacks */
PyObject_GC_UnTrack(pto);
if (pto->weakreflist != NULL)
@ -125,7 +158,8 @@ partial_dealloc(partialobject *pto)
Py_XDECREF(pto->args);
Py_XDECREF(pto->kw);
Py_XDECREF(pto->dict);
Py_TYPE(pto)->tp_free(pto);
tp->tp_free(pto);
Py_DECREF(tp);
}
@ -294,6 +328,12 @@ static PyMemberDef partial_memberlist[] = {
"tuple of arguments to future partial calls"},
{"keywords", T_OBJECT, OFF(kw), READONLY,
"dictionary of keyword arguments to future partial calls"},
{"__weaklistoffset__", T_PYSSIZET,
offsetof(partialobject, weakreflist), READONLY},
{"__dictoffset__", T_PYSSIZET,
offsetof(partialobject, dict), READONLY},
{"__vectorcalloffset__", T_PYSSIZET,
offsetof(partialobject, vectorcall), READONLY},
{NULL} /* Sentinel */
};
@ -420,49 +460,28 @@ static PyMethodDef partial_methods[] = {
{NULL, NULL} /* sentinel */
};
static PyTypeObject partial_type = {
PyVarObject_HEAD_INIT(NULL, 0)
"functools.partial", /* tp_name */
sizeof(partialobject), /* tp_basicsize */
0, /* tp_itemsize */
/* methods */
(destructor)partial_dealloc, /* tp_dealloc */
offsetof(partialobject, vectorcall),/* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_as_async */
(reprfunc)partial_repr, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
(ternaryfunc)partial_call, /* tp_call */
0, /* tp_str */
PyObject_GenericGetAttr, /* tp_getattro */
PyObject_GenericSetAttr, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
Py_TPFLAGS_BASETYPE |
Py_TPFLAGS_HAVE_VECTORCALL, /* tp_flags */
partial_doc, /* tp_doc */
(traverseproc)partial_traverse, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
offsetof(partialobject, weakreflist), /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
partial_methods, /* tp_methods */
partial_memberlist, /* tp_members */
partial_getsetlist, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
offsetof(partialobject, dict), /* tp_dictoffset */
0, /* tp_init */
0, /* tp_alloc */
partial_new, /* tp_new */
PyObject_GC_Del, /* tp_free */
static PyType_Slot partial_type_slots[] = {
{Py_tp_dealloc, partial_dealloc},
{Py_tp_repr, partial_repr},
{Py_tp_call, partial_call},
{Py_tp_getattro, PyObject_GenericGetAttr},
{Py_tp_setattro, PyObject_GenericSetAttr},
{Py_tp_doc, (void *)partial_doc},
{Py_tp_traverse, partial_traverse},
{Py_tp_methods, partial_methods},
{Py_tp_members, partial_memberlist},
{Py_tp_getset, partial_getsetlist},
{Py_tp_new, partial_new},
{Py_tp_free, PyObject_GC_Del},
{0, 0}
};
static PyType_Spec partial_type_spec = {
.name = "functools.partial",
.basicsize = sizeof(partialobject),
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_VECTORCALL,
.slots = partial_type_slots
};
@ -474,12 +493,21 @@ typedef struct {
PyObject *object;
} keyobject;
static int
keyobject_clear(keyobject *ko)
{
Py_CLEAR(ko->cmp);
Py_CLEAR(ko->object);
return 0;
}
static void
keyobject_dealloc(keyobject *ko)
{
Py_DECREF(ko->cmp);
Py_XDECREF(ko->object);
PyTypeObject *tp = Py_TYPE(ko);
keyobject_clear(ko);
PyObject_Free(ko);
Py_DECREF(tp);
}
static int
@ -490,15 +518,6 @@ keyobject_traverse(keyobject *ko, visitproc visit, void *arg)
return 0;
}
static int
keyobject_clear(keyobject *ko)
{
Py_CLEAR(ko->cmp);
if (ko->object)
Py_CLEAR(ko->object);
return 0;
}
static PyMemberDef keyobject_members[] = {
{"obj", T_OBJECT,
offsetof(keyobject, object), 0,
@ -512,38 +531,22 @@ keyobject_call(keyobject *ko, PyObject *args, PyObject *kwds);
static PyObject *
keyobject_richcompare(PyObject *ko, PyObject *other, int op);
static PyTypeObject keyobject_type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"functools.KeyWrapper", /* tp_name */
sizeof(keyobject), /* tp_basicsize */
0, /* tp_itemsize */
/* methods */
(destructor)keyobject_dealloc, /* tp_dealloc */
0, /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_as_async */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
(ternaryfunc)keyobject_call, /* tp_call */
0, /* tp_str */
PyObject_GenericGetAttr, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT, /* tp_flags */
0, /* tp_doc */
(traverseproc)keyobject_traverse, /* tp_traverse */
(inquiry)keyobject_clear, /* tp_clear */
keyobject_richcompare, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
0, /* tp_methods */
keyobject_members, /* tp_members */
0, /* tp_getset */
static PyType_Slot keyobject_type_slots[] = {
{Py_tp_dealloc, keyobject_dealloc},
{Py_tp_call, keyobject_call},
{Py_tp_getattro, PyObject_GenericGetAttr},
{Py_tp_traverse, keyobject_traverse},
{Py_tp_clear, keyobject_clear},
{Py_tp_richcompare, keyobject_richcompare},
{Py_tp_members, keyobject_members},
{0, 0}
};
static PyType_Spec keyobject_type_spec = {
.name = "functools.KeyWrapper",
.basicsize = sizeof(keyobject),
.flags = Py_TPFLAGS_DEFAULT,
.slots = keyobject_type_slots
};
static PyObject *
@ -555,9 +558,11 @@ keyobject_call(keyobject *ko, PyObject *args, PyObject *kwds)
if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:K", kwargs, &object))
return NULL;
result = PyObject_New(keyobject, &keyobject_type);
if (!result)
result = PyObject_New(keyobject, Py_TYPE(ko));
if (result == NULL) {
return NULL;
}
Py_INCREF(ko->cmp);
result->cmp = ko->cmp;
Py_INCREF(object);
@ -575,7 +580,7 @@ keyobject_richcompare(PyObject *ko, PyObject *other, int op)
PyObject *answer;
PyObject* stack[2];
if (!Py_IS_TYPE(other, &keyobject_type)) {
if (!Py_IS_TYPE(other, Py_TYPE(ko))) {
PyErr_Format(PyExc_TypeError, "other argument must be K instance");
return NULL;
}
@ -609,10 +614,13 @@ functools_cmp_to_key(PyObject *self, PyObject *args, PyObject *kwds)
PyObject *cmp;
static char *kwargs[] = {"mycmp", NULL};
keyobject *object;
_functools_state *state;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:cmp_to_key", kwargs, &cmp))
return NULL;
object = PyObject_New(keyobject, &keyobject_type);
state = get_functools_state(self);
object = PyObject_New(keyobject, state->keyobject_type);
if (!object)
return NULL;
Py_INCREF(cmp);
@ -729,10 +737,6 @@ iterable is empty.");
*/
/* this object is used delimit args and keywords in the cache keys */
static PyObject *kwd_mark = NULL;
struct lru_list_elem;
struct lru_cache_object;
@ -746,33 +750,23 @@ typedef struct lru_list_elem {
static void
lru_list_elem_dealloc(lru_list_elem *link)
{
PyTypeObject *tp = Py_TYPE(link);
Py_XDECREF(link->key);
Py_XDECREF(link->result);
PyObject_Free(link);
Py_DECREF(tp);
}
static PyTypeObject lru_list_elem_type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"functools._lru_list_elem", /* tp_name */
sizeof(lru_list_elem), /* tp_basicsize */
0, /* tp_itemsize */
/* methods */
(destructor)lru_list_elem_dealloc, /* tp_dealloc */
0, /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_as_async */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT, /* tp_flags */
static PyType_Slot lru_list_elem_type_slots[] = {
{Py_tp_dealloc, lru_list_elem_dealloc},
{0, 0}
};
static PyType_Spec lru_list_elem_type_spec = {
.name = "functools._lru_list_elem",
.basicsize = sizeof(lru_list_elem),
.flags = Py_TPFLAGS_DEFAULT,
.slots = lru_list_elem_type_slots
};
@ -792,10 +786,9 @@ typedef struct lru_cache_object {
PyObject *weakreflist;
} lru_cache_object;
static PyTypeObject lru_cache_type;
static PyObject *
lru_cache_make_key(PyObject *args, PyObject *kwds, int typed)
lru_cache_make_key(_functools_state *state, PyObject *args,
PyObject *kwds, int typed)
{
PyObject *key, *keyword, *value;
Py_ssize_t key_size, pos, key_pos, kwds_size;
@ -834,8 +827,8 @@ lru_cache_make_key(PyObject *args, PyObject *kwds, int typed)
PyTuple_SET_ITEM(key, key_pos++, item);
}
if (kwds_size) {
Py_INCREF(kwd_mark);
PyTuple_SET_ITEM(key, key_pos++, kwd_mark);
Py_INCREF(state->kwd_mark);
PyTuple_SET_ITEM(key, key_pos++, state->kwd_mark);
for (pos = 0; PyDict_Next(kwds, &pos, &keyword, &value);) {
Py_INCREF(keyword);
PyTuple_SET_ITEM(key, key_pos++, keyword);
@ -879,7 +872,12 @@ infinite_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwd
{
PyObject *result;
Py_hash_t hash;
PyObject *key = lru_cache_make_key(args, kwds, self->typed);
_functools_state *state;
state = get_functools_state_by_type(Py_TYPE(self));
if (state == NULL) {
return NULL;
}
PyObject *key = lru_cache_make_key(state, args, kwds, self->typed);
if (!key)
return NULL;
hash = PyObject_Hash(key);
@ -979,8 +977,13 @@ bounded_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds
lru_list_elem *link;
PyObject *key, *result, *testresult;
Py_hash_t hash;
_functools_state *state;
key = lru_cache_make_key(args, kwds, self->typed);
state = get_functools_state_by_type(Py_TYPE(self));
if (state == NULL) {
return NULL;
}
key = lru_cache_make_key(state, args, kwds, self->typed);
if (!key)
return NULL;
hash = PyObject_Hash(key);
@ -1035,7 +1038,7 @@ bounded_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds
{
/* Cache is not full, so put the result in a new link */
link = (lru_list_elem *)PyObject_New(lru_list_elem,
&lru_list_elem_type);
state->lru_list_elem_type);
if (link == NULL) {
Py_DECREF(key);
Py_DECREF(result);
@ -1229,22 +1232,31 @@ lru_cache_clear_list(lru_list_elem *link)
}
}
static int
lru_cache_tp_clear(lru_cache_object *self)
{
lru_list_elem *list = lru_cache_unlink_list(self);
Py_CLEAR(self->func);
Py_CLEAR(self->cache);
Py_CLEAR(self->cache_info_type);
Py_CLEAR(self->dict);
lru_cache_clear_list(list);
return 0;
}
static void
lru_cache_dealloc(lru_cache_object *obj)
{
lru_list_elem *list;
PyTypeObject *tp = Py_TYPE(obj);
/* bpo-31095: UnTrack is needed before calling any callbacks */
PyObject_GC_UnTrack(obj);
if (obj->weakreflist != NULL)
if (obj->weakreflist != NULL) {
PyObject_ClearWeakRefs((PyObject*)obj);
}
list = lru_cache_unlink_list(obj);
Py_XDECREF(obj->cache);
Py_XDECREF(obj->func);
Py_XDECREF(obj->cache_info_type);
Py_XDECREF(obj->dict);
lru_cache_clear_list(list);
Py_TYPE(obj)->tp_free(obj);
lru_cache_tp_clear(obj);
tp->tp_free(obj);
Py_DECREF(tp);
}
static PyObject *
@ -1323,18 +1335,6 @@ lru_cache_tp_traverse(lru_cache_object *self, visitproc visit, void *arg)
return 0;
}
static int
lru_cache_tp_clear(lru_cache_object *self)
{
lru_list_elem *list = lru_cache_unlink_list(self);
Py_CLEAR(self->func);
Py_CLEAR(self->cache);
Py_CLEAR(self->cache_info_type);
Py_CLEAR(self->dict);
lru_cache_clear_list(list);
return 0;
}
PyDoc_STRVAR(lru_cache_doc,
"Create a cached callable that wraps another function.\n\
@ -1366,51 +1366,37 @@ static PyGetSetDef lru_cache_getsetlist[] = {
{NULL}
};
static PyTypeObject lru_cache_type = {
PyVarObject_HEAD_INIT(NULL, 0)
"functools._lru_cache_wrapper", /* tp_name */
sizeof(lru_cache_object), /* tp_basicsize */
0, /* tp_itemsize */
/* methods */
(destructor)lru_cache_dealloc, /* tp_dealloc */
0, /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_as_async */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
(ternaryfunc)lru_cache_call, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_METHOD_DESCRIPTOR,
/* tp_flags */
lru_cache_doc, /* tp_doc */
(traverseproc)lru_cache_tp_traverse,/* tp_traverse */
(inquiry)lru_cache_tp_clear, /* tp_clear */
0, /* tp_richcompare */
offsetof(lru_cache_object, weakreflist),
/* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
lru_cache_methods, /* tp_methods */
0, /* tp_members */
lru_cache_getsetlist, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
lru_cache_descr_get, /* tp_descr_get */
0, /* tp_descr_set */
offsetof(lru_cache_object, dict), /* tp_dictoffset */
0, /* tp_init */
0, /* tp_alloc */
lru_cache_new, /* tp_new */
static PyMemberDef lru_cache_memberlist[] = {
{"__dictoffset__", T_PYSSIZET,
offsetof(lru_cache_object, dict), READONLY},
{"__weaklistoffset__", T_PYSSIZET,
offsetof(lru_cache_object, weakreflist), READONLY},
{NULL} /* Sentinel */
};
static PyType_Slot lru_cache_type_slots[] = {
{Py_tp_dealloc, lru_cache_dealloc},
{Py_tp_call, lru_cache_call},
{Py_tp_doc, (void *)lru_cache_doc},
{Py_tp_traverse, lru_cache_tp_traverse},
{Py_tp_clear, lru_cache_tp_clear},
{Py_tp_methods, lru_cache_methods},
{Py_tp_members, lru_cache_memberlist},
{Py_tp_getset, lru_cache_getsetlist},
{Py_tp_descr_get, lru_cache_descr_get},
{Py_tp_new, lru_cache_new},
{0, 0}
};
static PyType_Spec lru_cache_type_spec = {
.name = "functools._lru_cache_wrapper",
.basicsize = sizeof(lru_cache_object),
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
Py_TPFLAGS_METHOD_DESCRIPTOR,
.slots = lru_cache_type_slots
};
/* module level code ********************************************************/
PyDoc_STRVAR(_functools_doc,
@ -1423,38 +1409,83 @@ static PyMethodDef _functools_methods[] = {
{NULL, NULL} /* sentinel */
};
static void
_functools_free(void *m)
{
// FIXME: Do not clear kwd_mark to avoid NULL pointer dereferencing if we have
// other modules instances that could use it. Will fix when PEP-573 land
// and we could move kwd_mark to a per-module state.
// Py_CLEAR(kwd_mark);
}
static int
_functools_exec(PyObject *module)
{
PyTypeObject *typelist[] = {
&partial_type,
&lru_cache_type
};
if (!kwd_mark) {
kwd_mark = _PyObject_CallNoArg((PyObject *)&PyBaseObject_Type);
if (!kwd_mark) {
_functools_state *state = get_functools_state(module);
state->kwd_mark = _PyObject_CallNoArg((PyObject *)&PyBaseObject_Type);
if (state->kwd_mark == NULL) {
return -1;
}
}
for (size_t i = 0; i < Py_ARRAY_LENGTH(typelist); i++) {
if (PyModule_AddType(module, typelist[i]) < 0) {
state->partial_type = (PyTypeObject *)PyType_FromModuleAndSpec(module,
&partial_type_spec, NULL);
if (state->partial_type == NULL) {
return -1;
}
if (PyModule_AddType(module, state->partial_type) < 0) {
return -1;
}
PyObject *lru_cache_type = PyType_FromModuleAndSpec(module,
&lru_cache_type_spec, NULL);
if (lru_cache_type == NULL) {
return -1;
}
if (PyModule_AddType(module, (PyTypeObject *)lru_cache_type) < 0) {
Py_DECREF(lru_cache_type);
return -1;
}
state->keyobject_type = (PyTypeObject *)PyType_FromModuleAndSpec(module,
&keyobject_type_spec, NULL);
if (state->keyobject_type == NULL) {
return -1;
}
if (PyModule_AddType(module, state->keyobject_type) < 0) {
return -1;
}
state->lru_list_elem_type = (PyTypeObject *)PyType_FromModuleAndSpec(
module, &lru_list_elem_type_spec, NULL);
if (state->lru_list_elem_type == NULL) {
return -1;
}
if (PyModule_AddType(module, state->lru_list_elem_type) < 0) {
return -1;
}
return 0;
}
static int
_functools_traverse(PyObject *module, visitproc visit, void *arg)
{
_functools_state *state = get_functools_state(module);
Py_VISIT(state->kwd_mark);
Py_VISIT(state->partial_type);
Py_VISIT(state->keyobject_type);
Py_VISIT(state->lru_list_elem_type);
return 0;
}
static int
_functools_clear(PyObject *module)
{
_functools_state *state = get_functools_state(module);
Py_CLEAR(state->kwd_mark);
Py_CLEAR(state->partial_type);
Py_CLEAR(state->keyobject_type);
Py_CLEAR(state->lru_list_elem_type);
return 0;
}
static void
_functools_free(void *module)
{
_functools_clear((PyObject *)module);
}
static struct PyModuleDef_Slot _functools_slots[] = {
{Py_mod_exec, _functools_exec},
{0, NULL}
@ -1462,14 +1493,14 @@ static struct PyModuleDef_Slot _functools_slots[] = {
static struct PyModuleDef _functools_module = {
PyModuleDef_HEAD_INIT,
"_functools",
_functools_doc,
0,
_functools_methods,
_functools_slots,
NULL,
NULL,
_functools_free,
.m_name = "_functools",
.m_doc = _functools_doc,
.m_size = sizeof(_functools_state),
.m_methods = _functools_methods,
.m_slots = _functools_slots,
.m_traverse = _functools_traverse,
.m_clear = _functools_clear,
.m_free = _functools_free,
};
PyMODINIT_FUNC