From 86ea58149c3e83f402cecd17e6a536865fb06ce1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 10 May 2019 13:29:55 -0400 Subject: [PATCH] bpo-36737: Use the module state C-API for warnings. (gh-13159) --- Include/internal/pycore_pylifecycle.h | 2 +- Include/internal/pycore_pystate.h | 3 +- .../2019-05-07-12-18-11.bpo-36737.XAo6LY.rst | 2 + Python/_warnings.c | 321 +++++++++++------- Python/pylifecycle.c | 2 +- Python/pystate.c | 4 + 6 files changed, 210 insertions(+), 124 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-05-07-12-18-11.bpo-36737.XAo6LY.rst diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index adb1f5d90a5..7144bbcda7c 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -81,7 +81,7 @@ extern void PyLong_Fini(void); extern void _PyFaulthandler_Fini(void); extern void _PyHash_Fini(void); extern int _PyTraceMalloc_Fini(void); -extern void _PyWarnings_Fini(_PyRuntimeState *runtime); +extern void _PyWarnings_Fini(PyInterpreterState *interp); extern void _PyGILState_Init( _PyRuntimeState *runtime, diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 67bcd147e28..69ceecba40d 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -90,6 +90,8 @@ struct _is { PyObject *pyexitmodule; uint64_t tstate_next_unique_id; + + struct _warnings_runtime_state warnings; }; PyAPI_FUNC(struct _is*) _PyInterpreterState_LookUpID(PY_INT64_T); @@ -179,7 +181,6 @@ typedef struct pyruntimestate { int nexitfuncs; struct _gc_runtime_state gc; - struct _warnings_runtime_state warnings; struct _ceval_runtime_state ceval; struct _gilstate_runtime_state gilstate; diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-07-12-18-11.bpo-36737.XAo6LY.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-07-12-18-11.bpo-36737.XAo6LY.rst new file mode 100644 index 00000000000..7a2c647dbff --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-07-12-18-11.bpo-36737.XAo6LY.rst @@ -0,0 +1,2 @@ +Move PyRuntimeState.warnings into per-interpreter state (via "module +state"). diff --git a/Python/_warnings.c b/Python/_warnings.c index 388b2995408..0b192580e10 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -15,6 +15,134 @@ _Py_IDENTIFIER(default); _Py_IDENTIFIER(ignore); #endif + +/*************************************************************************/ + +typedef struct _warnings_runtime_state WarningsState; + +/* Forward declaration of the _warnings module definition. */ +static struct PyModuleDef warningsmodule; + +/* Given a module object, get its per-module state. */ +static WarningsState * +_Warnings_GetState() +{ + PyThreadState *tstate = PyThreadState_GET(); + if (tstate == NULL) { + PyErr_SetString(PyExc_RuntimeError, + "_Warnings_GetState: could not identify current interpreter"); + return NULL; + } + return &tstate->interp->warnings; +} + +/* Clear the given warnings module state. */ +static void +_Warnings_ClearState(WarningsState *st) +{ + Py_CLEAR(st->filters); + Py_CLEAR(st->once_registry); + Py_CLEAR(st->default_action); +} + +#ifndef Py_DEBUG +static PyObject * +create_filter(PyObject *category, _Py_Identifier *id, const char *modname) +{ + PyObject *modname_obj = NULL; + PyObject *action_str = _PyUnicode_FromId(id); + if (action_str == NULL) { + return NULL; + } + + /* Default to "no module name" for initial filter set */ + if (modname != NULL) { + modname_obj = PyUnicode_InternFromString(modname); + if (modname_obj == NULL) { + return NULL; + } + } else { + modname_obj = Py_None; + } + + /* This assumes the line number is zero for now. */ + return PyTuple_Pack(5, action_str, Py_None, + category, modname_obj, _PyLong_Zero); +} +#endif + +static PyObject * +init_filters(void) +{ +#ifdef Py_DEBUG + /* Py_DEBUG builds show all warnings by default */ + return PyList_New(0); +#else + /* Other builds ignore a number of warning categories by default */ + PyObject *filters = PyList_New(5); + if (filters == NULL) { + return NULL; + } + + size_t pos = 0; /* Post-incremented in each use. */ + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_DeprecationWarning, &PyId_default, "__main__")); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_DeprecationWarning, &PyId_ignore, NULL)); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_PendingDeprecationWarning, &PyId_ignore, NULL)); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_ImportWarning, &PyId_ignore, NULL)); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_ResourceWarning, &PyId_ignore, NULL)); + + for (size_t x = 0; x < pos; x++) { + if (PyList_GET_ITEM(filters, x) == NULL) { + Py_DECREF(filters); + return NULL; + } + } + return filters; +#endif +} + +/* Initialize the given warnings module state. */ +static int +_Warnings_InitState(WarningsState *st) +{ + if (st->filters == NULL) { + st->filters = init_filters(); + if (st->filters == NULL) { + goto error; + } + } + + if (st->once_registry == NULL) { + st->once_registry = PyDict_New(); + if (st->once_registry == NULL) { + goto error; + } + } + + if (st->default_action == NULL) { + st->default_action = PyUnicode_FromString("default"); + if (st->default_action == NULL) { + goto error; + } + } + + st->filters_version = 0; + + return 0; + +error: + _Warnings_ClearState(st); + return -1; +} + + +/*************************************************************************/ + static int check_matched(PyObject *obj, PyObject *arg) { @@ -93,7 +221,7 @@ get_warnings_attr(_Py_Identifier *attr_id, int try_import) static PyObject * -get_once_registry(void) +get_once_registry(WarningsState *st) { PyObject *registry; _Py_IDENTIFIER(onceregistry); @@ -102,8 +230,8 @@ get_once_registry(void) if (registry == NULL) { if (PyErr_Occurred()) return NULL; - assert(_PyRuntime.warnings.once_registry); - return _PyRuntime.warnings.once_registry; + assert(st->once_registry); + return st->once_registry; } if (!PyDict_Check(registry)) { PyErr_Format(PyExc_TypeError, @@ -113,13 +241,13 @@ get_once_registry(void) Py_DECREF(registry); return NULL; } - Py_SETREF(_PyRuntime.warnings.once_registry, registry); + Py_SETREF(st->once_registry, registry); return registry; } static PyObject * -get_default_action(void) +get_default_action(WarningsState *st) { PyObject *default_action; _Py_IDENTIFIER(defaultaction); @@ -129,8 +257,8 @@ get_default_action(void) if (PyErr_Occurred()) { return NULL; } - assert(_PyRuntime.warnings.default_action); - return _PyRuntime.warnings.default_action; + assert(st->default_action); + return st->default_action; } if (!PyUnicode_Check(default_action)) { PyErr_Format(PyExc_TypeError, @@ -140,7 +268,7 @@ get_default_action(void) Py_DECREF(default_action); return NULL; } - Py_SETREF(_PyRuntime.warnings.default_action, default_action); + Py_SETREF(st->default_action, default_action); return default_action; } @@ -154,6 +282,10 @@ get_filter(PyObject *category, PyObject *text, Py_ssize_t lineno, Py_ssize_t i; PyObject *warnings_filters; _Py_IDENTIFIER(filters); + WarningsState *st = _Warnings_GetState(); + if (st == NULL) { + return NULL; + } warnings_filters = get_warnings_attr(&PyId_filters, 0); if (warnings_filters == NULL) { @@ -161,17 +293,17 @@ get_filter(PyObject *category, PyObject *text, Py_ssize_t lineno, return NULL; } else { - Py_SETREF(_PyRuntime.warnings.filters, warnings_filters); + Py_SETREF(st->filters, warnings_filters); } - PyObject *filters = _PyRuntime.warnings.filters; + PyObject *filters = st->filters; if (filters == NULL || !PyList_Check(filters)) { PyErr_SetString(PyExc_ValueError, MODULE_NAME ".filters must be a list"); return NULL; } - /* _PyRuntime.warnings.filters could change while we are iterating over it. */ + /* WarningsState.filters could change while we are iterating over it. */ for (i = 0; i < PyList_GET_SIZE(filters); i++) { PyObject *tmp_item, *action, *msg, *cat, *mod, *ln_obj; Py_ssize_t ln; @@ -232,7 +364,7 @@ get_filter(PyObject *category, PyObject *text, Py_ssize_t lineno, Py_DECREF(tmp_item); } - action = get_default_action(); + action = get_default_action(st); if (action != NULL) { Py_INCREF(Py_None); *item = Py_None; @@ -252,16 +384,20 @@ already_warned(PyObject *registry, PyObject *key, int should_set) if (key == NULL) return -1; + WarningsState *st = _Warnings_GetState(); + if (st == NULL) { + return -1; + } version_obj = _PyDict_GetItemIdWithError(registry, &PyId_version); if (version_obj == NULL || !PyLong_CheckExact(version_obj) - || PyLong_AsLong(version_obj) != _PyRuntime.warnings.filters_version) + || PyLong_AsLong(version_obj) != st->filters_version) { if (PyErr_Occurred()) { return -1; } PyDict_Clear(registry); - version_obj = PyLong_FromLong(_PyRuntime.warnings.filters_version); + version_obj = PyLong_FromLong(st->filters_version); if (version_obj == NULL) return -1; if (_PyDict_SetItemId(registry, &PyId_version, version_obj) < 0) { @@ -567,11 +703,15 @@ warn_explicit(PyObject *category, PyObject *message, if (_PyUnicode_EqualToASCIIString(action, "once")) { if (registry == NULL || registry == Py_None) { - registry = get_once_registry(); + WarningsState *st = _Warnings_GetState(); + if (st == NULL) { + goto cleanup; + } + registry = get_once_registry(st); if (registry == NULL) goto cleanup; } - /* _PyRuntime.warnings.once_registry[(text, category)] = 1 */ + /* WarningsState.once_registry[(text, category)] = 1 */ rc = update_registry(registry, text, category, 0); } else if (_PyUnicode_EqualToASCIIString(action, "module")) { @@ -925,7 +1065,11 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * warnings_filters_mutated(PyObject *self, PyObject *args) { - _PyRuntime.warnings.filters_version++; + WarningsState *st = _Warnings_GetState(); + if (st == NULL) { + return NULL; + } + st->filters_version++; Py_RETURN_NONE; } @@ -1175,78 +1319,16 @@ static PyMethodDef warnings_functions[] = { }; -#ifndef Py_DEBUG -static PyObject * -create_filter(PyObject *category, _Py_Identifier *id, const char *modname) -{ - PyObject *modname_obj = NULL; - PyObject *action_str = _PyUnicode_FromId(id); - if (action_str == NULL) { - return NULL; - } - - /* Default to "no module name" for initial filter set */ - if (modname != NULL) { - modname_obj = PyUnicode_InternFromString(modname); - if (modname_obj == NULL) { - return NULL; - } - } else { - modname_obj = Py_None; - } - - /* This assumes the line number is zero for now. */ - return PyTuple_Pack(5, action_str, Py_None, - category, modname_obj, _PyLong_Zero); -} -#endif - - -static PyObject * -init_filters(void) -{ -#ifdef Py_DEBUG - /* Py_DEBUG builds show all warnings by default */ - return PyList_New(0); -#else - /* Other builds ignore a number of warning categories by default */ - PyObject *filters = PyList_New(5); - if (filters == NULL) { - return NULL; - } - - size_t pos = 0; /* Post-incremented in each use. */ - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_DeprecationWarning, &PyId_default, "__main__")); - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_DeprecationWarning, &PyId_ignore, NULL)); - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_PendingDeprecationWarning, &PyId_ignore, NULL)); - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_ImportWarning, &PyId_ignore, NULL)); - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_ResourceWarning, &PyId_ignore, NULL)); - - for (size_t x = 0; x < pos; x++) { - if (PyList_GET_ITEM(filters, x) == NULL) { - Py_DECREF(filters); - return NULL; - } - } - return filters; -#endif -} - static struct PyModuleDef warningsmodule = { PyModuleDef_HEAD_INIT, - MODULE_NAME, - warnings__doc__, - 0, - warnings_functions, - NULL, - NULL, - NULL, - NULL + MODULE_NAME, /* m_name */ + warnings__doc__, /* m_doc */ + 0, /* m_size */ + warnings_functions, /* m_methods */ + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ }; @@ -1256,49 +1338,46 @@ _PyWarnings_Init(void) PyObject *m; m = PyModule_Create(&warningsmodule); - if (m == NULL) + if (m == NULL) { return NULL; - - struct _warnings_runtime_state *state = &_PyRuntime.warnings; - if (state->filters == NULL) { - state->filters = init_filters(); - if (state->filters == NULL) - return NULL; } - Py_INCREF(state->filters); - if (PyModule_AddObject(m, "filters", state->filters) < 0) - return NULL; - if (state->once_registry == NULL) { - state->once_registry = PyDict_New(); - if (state->once_registry == NULL) - return NULL; + WarningsState *st = _Warnings_GetState(); + if (st == NULL) { + goto error; } - Py_INCREF(state->once_registry); - if (PyModule_AddObject(m, "_onceregistry", - state->once_registry) < 0) - return NULL; - - if (state->default_action == NULL) { - state->default_action = PyUnicode_FromString("default"); - if (state->default_action == NULL) - return NULL; + if (_Warnings_InitState(st) < 0) { + goto error; + } + + Py_INCREF(st->filters); + if (PyModule_AddObject(m, "filters", st->filters) < 0) { + goto error; + } + + Py_INCREF(st->once_registry); + if (PyModule_AddObject(m, "_onceregistry", st->once_registry) < 0) { + goto error; + } + + Py_INCREF(st->default_action); + if (PyModule_AddObject(m, "_defaultaction", st->default_action) < 0) { + goto error; } - Py_INCREF(state->default_action); - if (PyModule_AddObject(m, "_defaultaction", - state->default_action) < 0) - return NULL; - state->filters_version = 0; return m; + +error: + if (st != NULL) { + _Warnings_ClearState(st); + } + Py_DECREF(m); + return NULL; } - +// We need this to ensure that warnings still work until late in finalization. void -_PyWarnings_Fini(_PyRuntimeState *runtime) +_PyWarnings_Fini(PyInterpreterState *interp) { - struct _warnings_runtime_state *state = &runtime->warnings; - Py_CLEAR(state->filters); - Py_CLEAR(state->once_registry); - Py_CLEAR(state->default_action); + _Warnings_ClearState(&interp->warnings); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index bd4d1d92662..32902aa0d59 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1288,7 +1288,7 @@ Py_FinalizeEx(void) PyDict_Fini(); PySlice_Fini(); _PyGC_Fini(runtime); - _PyWarnings_Fini(runtime); + _PyWarnings_Fini(interp); _Py_HashRandomization_Fini(); _PyArg_Fini(); PyAsyncGen_Fini(); diff --git a/Python/pystate.c b/Python/pystate.c index e9c4c7d8376..44acfed6b98 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -5,6 +5,7 @@ #include "pycore_coreconfig.h" #include "pycore_pymem.h" #include "pycore_pystate.h" +#include "pycore_pylifecycle.h" /* -------------------------------------------------------------------------- CAUTION @@ -257,6 +258,9 @@ _PyInterpreterState_Clear(_PyRuntimeState *runtime, PyInterpreterState *interp) Py_CLEAR(interp->after_forkers_parent); Py_CLEAR(interp->after_forkers_child); #endif + if (runtime->finalizing == NULL) { + _PyWarnings_Fini(interp); + } // XXX Once we have one allocator per interpreter (i.e. // per-interpreter GC) we must ensure that all of the interpreter's // objects have been cleaned up at the point.