From e9e3eab0b868c7d0b48e472705024240d5c39d5c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 21 Jan 2022 01:42:25 +0100 Subject: [PATCH] bpo-46417: Finalize structseq types at exit (GH-30645) Add _PyStructSequence_FiniType() and _PyStaticType_Dealloc() functions to finalize a structseq static type in Py_Finalize(). Currrently, these functions do nothing if Python is built in release mode. Clear static types: * AsyncGenHooksType: sys.set_asyncgen_hooks() * FlagsType: sys.flags * FloatInfoType: sys.float_info * Hash_InfoType: sys.hash_info * Int_InfoType: sys.int_info * ThreadInfoType: sys.thread_info * UnraisableHookArgsType: sys.unraisablehook * VersionInfoType: sys.version * WindowsVersionType: sys.getwindowsversion() --- Include/internal/pycore_floatobject.h | 1 + Include/internal/pycore_long.h | 1 + Include/internal/pycore_pyerrors.h | 1 + Include/internal/pycore_pylifecycle.h | 2 + Include/internal/pycore_typeobject.h | 2 + Include/structseq.h | 3 ++ Lib/test/_test_embed_structseq.py | 55 +++++++++++++++++++++++++++ Lib/test/test_embed.py | 12 ++++++ Objects/floatobject.c | 8 ++++ Objects/longobject.c | 11 ++++++ Objects/structseq.c | 30 +++++++++++++++ Objects/typeobject.c | 23 ++++++++++- Programs/_testembed.c | 40 +++++++++++++++++++ Python/errors.c | 11 ++++++ Python/pylifecycle.c | 6 +++ Python/sysmodule.c | 15 ++++++++ Python/thread.c | 11 ++++++ 17 files changed, 230 insertions(+), 2 deletions(-) create mode 100644 Lib/test/_test_embed_structseq.py diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index be6045587de..891e422f594 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -14,6 +14,7 @@ extern "C" { extern void _PyFloat_InitState(PyInterpreterState *); extern PyStatus _PyFloat_InitTypes(PyInterpreterState *); extern void _PyFloat_Fini(PyInterpreterState *); +extern void _PyFloat_FiniType(PyInterpreterState *); /* other API */ diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 4d1a0d04249..436bf084685 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -15,6 +15,7 @@ extern "C" { /* runtime lifecycle */ extern PyStatus _PyLong_InitTypes(PyInterpreterState *); +extern void _PyLong_FiniTypes(PyInterpreterState *interp); /* other API */ diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index f375337a405..e3c445ba5d9 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -12,6 +12,7 @@ extern "C" { /* runtime lifecycle */ extern PyStatus _PyErr_InitTypes(PyInterpreterState *); +extern void _PyErr_FiniTypes(PyInterpreterState *); /* other API */ diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 766e889f237..dfa8fd6bd0d 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -58,6 +58,7 @@ extern PyStatus _PySys_Create( extern PyStatus _PySys_ReadPreinitWarnOptions(PyWideStringList *options); extern PyStatus _PySys_ReadPreinitXOptions(PyConfig *config); extern int _PySys_UpdateConfig(PyThreadState *tstate); +extern void _PySys_Fini(PyInterpreterState *interp); extern PyStatus _PyBuiltins_AddExceptions(PyObject * bltinmod); extern PyStatus _Py_HashRandomization_Init(const PyConfig *); @@ -81,6 +82,7 @@ extern void _PyTraceMalloc_Fini(void); extern void _PyWarnings_Fini(PyInterpreterState *interp); extern void _PyAST_Fini(PyInterpreterState *interp); extern void _PyAtExit_Fini(PyInterpreterState *interp); +extern void _PyThread_FiniType(PyInterpreterState *interp); extern PyStatus _PyGILState_Init(_PyRuntimeState *runtime); extern PyStatus _PyGILState_SetTstate(PyThreadState *tstate); diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 7fd8a1f3509..ba95bbc1c48 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -40,6 +40,8 @@ struct type_cache { extern PyStatus _PyTypes_InitSlotDefs(void); +extern void _PyStaticType_Dealloc(PyTypeObject *type); + #ifdef __cplusplus } diff --git a/Include/structseq.h b/Include/structseq.h index e89265a67c3..8abd2443468 100644 --- a/Include/structseq.h +++ b/Include/structseq.h @@ -27,6 +27,9 @@ PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type, PyAPI_FUNC(int) PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc); #endif +#ifdef Py_BUILD_CORE +PyAPI_FUNC(void) _PyStructSequence_FiniType(PyTypeObject *type); +#endif PyAPI_FUNC(PyTypeObject*) PyStructSequence_NewType(PyStructSequence_Desc *desc); PyAPI_FUNC(PyObject *) PyStructSequence_New(PyTypeObject* type); diff --git a/Lib/test/_test_embed_structseq.py b/Lib/test/_test_embed_structseq.py new file mode 100644 index 00000000000..868f9f83e8b --- /dev/null +++ b/Lib/test/_test_embed_structseq.py @@ -0,0 +1,55 @@ +import sys +import types +import unittest + + +# bpo-46417: Test that structseq types used by the sys module are still +# valid when Py_Finalize()/Py_Initialize() are called multiple times. +class TestStructSeq(unittest.TestCase): + # test PyTypeObject members + def check_structseq(self, obj_type): + # ob_refcnt + self.assertGreaterEqual(sys.getrefcount(obj_type), 1) + # tp_base + self.assertTrue(issubclass(obj_type, tuple)) + # tp_bases + self.assertEqual(obj_type.__bases__, (tuple,)) + # tp_dict + self.assertIsInstance(obj_type.__dict__, types.MappingProxyType) + # tp_mro + self.assertEqual(obj_type.__mro__, (obj_type, tuple, object)) + # tp_name + self.assertIsInstance(type.__name__, str) + # tp_subclasses + self.assertEqual(obj_type.__subclasses__(), []) + + def test_sys_attrs(self): + for attr_name in ( + 'flags', # FlagsType + 'float_info', # FloatInfoType + 'hash_info', # Hash_InfoType + 'int_info', # Int_InfoType + 'thread_info', # ThreadInfoType + 'version_info', # VersionInfoType + ): + with self.subTest(attr=attr_name): + attr = getattr(sys, attr_name) + self.check_structseq(type(attr)) + + def test_sys_funcs(self): + func_names = ['get_asyncgen_hooks'] # AsyncGenHooksType + if hasattr(sys, 'getwindowsversion'): + func_names.append('getwindowsversion') # WindowsVersionType + for func_name in func_names: + with self.subTest(func=func_name): + func = getattr(sys, func_name) + obj = func() + self.check_structseq(type(obj)) + + +try: + unittest.main() +except SystemExit as exc: + if exc.args[0] != 0: + raise +print("Tests passed") diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 9fed0a5f14e..204b194ed3b 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -329,6 +329,18 @@ class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase): self.assertEqual(out, "Py_RunMain(): sys.argv=['-c', 'arg2']\n" * nloop) self.assertEqual(err, '') + def test_finalize_structseq(self): + # bpo-46417: Py_Finalize() clears structseq static types. Check that + # sys attributes using struct types still work when + # Py_Finalize()/Py_Initialize() is called multiple times. + # print() calls type->tp_repr(instance) and so checks that the types + # are still working properly. + script = support.findfile('_test_embed_structseq.py') + with open(script, encoding="utf-8") as fp: + code = fp.read() + out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) + self.assertEqual(out, 'Tests passed\n' * INIT_LOOPS) + class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): maxDiff = 4096 diff --git a/Objects/floatobject.c b/Objects/floatobject.c index f8620d6f8ef..88f25d6b8c8 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -2082,6 +2082,14 @@ _PyFloat_Fini(PyInterpreterState *interp) #endif } +void +_PyFloat_FiniType(PyInterpreterState *interp) +{ + if (_Py_IsMainInterpreter(interp)) { + _PyStructSequence_FiniType(&FloatInfoType); + } +} + /* Print summary info about the state of the optimized allocator */ void _PyFloat_DebugMallocStats(FILE *out) diff --git a/Objects/longobject.c b/Objects/longobject.c index 1b2d1266c6b..5aa53dd91c2 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -5949,3 +5949,14 @@ _PyLong_InitTypes(PyInterpreterState *interp) return _PyStatus_OK(); } + + +void +_PyLong_FiniTypes(PyInterpreterState *interp) +{ + if (!_Py_IsMainInterpreter(interp)) { + return; + } + + _PyStructSequence_FiniType(&Int_InfoType); +} diff --git a/Objects/structseq.c b/Objects/structseq.c index a2eefb0455a..f8bf9477f28 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -532,6 +532,36 @@ PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc) (void)PyStructSequence_InitType2(type, desc); } + +void +_PyStructSequence_FiniType(PyTypeObject *type) +{ + // Ensure that the type is initialized + assert(type->tp_name != NULL); + assert(type->tp_base == &PyTuple_Type); + + // Cannot delete a type if it still has subclasses + if (type->tp_subclasses != NULL) { + return; + } + + // Undo PyStructSequence_NewType() + type->tp_name = NULL; + PyMem_Free(type->tp_members); + + _PyStaticType_Dealloc(type); + assert(Py_REFCNT(type) == 1); + // Undo Py_INCREF(type) of _PyStructSequence_InitType(). + // Don't use Py_DECREF(): static type must not be deallocated + Py_SET_REFCNT(type, 0); + + // Make sure that _PyStructSequence_InitType() will initialize + // the type again + assert(Py_REFCNT(type) == 0); + assert(type->tp_name == NULL); +} + + PyTypeObject * PyStructSequence_NewType(PyStructSequence_Desc *desc) { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index cbf806b074b..66a10a5bc57 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4070,10 +4070,27 @@ type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) extern void _PyDictKeys_DecRef(PyDictKeysObject *keys); + +void +_PyStaticType_Dealloc(PyTypeObject *type) +{ + // _PyStaticType_Dealloc() must not be called if a type has subtypes. + // A subtype can inherit attributes and methods of its parent type, + // and a type must no longer be used once it's deallocated. + assert(type->tp_subclasses == NULL); + + Py_CLEAR(type->tp_dict); + Py_CLEAR(type->tp_bases); + Py_CLEAR(type->tp_mro); + Py_CLEAR(type->tp_cache); + Py_CLEAR(type->tp_subclasses); + type->tp_flags &= ~Py_TPFLAGS_READY; +} + + static void type_dealloc(PyTypeObject *type) { - PyHeapTypeObject *et; PyObject *tp, *val, *tb; /* Assert this is a heap-allocated type object */ @@ -4082,8 +4099,8 @@ type_dealloc(PyTypeObject *type) PyErr_Fetch(&tp, &val, &tb); remove_all_subclasses(type, type->tp_bases); PyErr_Restore(tp, val, tb); + PyObject_ClearWeakRefs((PyObject *)type); - et = (PyHeapTypeObject *)type; Py_XDECREF(type->tp_base); Py_XDECREF(type->tp_dict); Py_XDECREF(type->tp_bases); @@ -4094,6 +4111,8 @@ type_dealloc(PyTypeObject *type) * of most other objects. It's okay to cast it to char *. */ PyObject_Free((char *)type->tp_doc); + + PyHeapTypeObject *et = (PyHeapTypeObject *)type; Py_XDECREF(et->ht_name); Py_XDECREF(et->ht_qualname); Py_XDECREF(et->ht_slots); diff --git a/Programs/_testembed.c b/Programs/_testembed.c index b31781938eb..5bc0a127a87 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -15,12 +15,18 @@ #include // putenv() #include +int main_argc; +char **main_argv; + /********************************************************* * Embedded interpreter tests that need a custom exe * * Executed via 'EmbeddingTests' in Lib/test/test_capi.py *********************************************************/ +// Use to display the usage +#define PROGRAM "test_embed" + /* Use path starting with "./" avoids a search along the PATH */ #define PROGRAM_NAME L"./_testembed" @@ -113,6 +119,36 @@ PyInit_embedded_ext(void) return PyModule_Create(&embedded_ext); } +/**************************************************************************** + * Call Py_Initialize()/Py_Finalize() multiple times and execute Python code + ***************************************************************************/ + +// Used by bpo-46417 to test that structseq types used by the sys module are +// cleared properly and initialized again properly when Python is finalized +// multiple times. +static int test_repeated_init_exec(void) +{ + if (main_argc < 3) { + fprintf(stderr, "usage: %s test_repeated_init_exec CODE\n", PROGRAM); + exit(1); + } + const char *code = main_argv[2]; + + for (int i=1; i <= INIT_LOOPS; i++) { + fprintf(stderr, "--- Loop #%d ---\n", i); + fflush(stderr); + + _testembed_Py_Initialize(); + int err = PyRun_SimpleString(code); + Py_Finalize(); + if (err) { + return 1; + } + } + return 0; +} + + /***************************************************** * Test forcing a particular IO encoding *****************************************************/ @@ -1880,6 +1916,7 @@ struct TestCase static struct TestCase TestCases[] = { // Python initialization + {"test_repeated_init_exec", test_repeated_init_exec}, {"test_forced_io_encoding", test_forced_io_encoding}, {"test_repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters}, {"test_repeated_init_and_inittab", test_repeated_init_and_inittab}, @@ -1946,6 +1983,9 @@ static struct TestCase TestCases[] = { int main(int argc, char *argv[]) { + main_argc = argc; + main_argv = argv; + if (argc > 1) { for (struct TestCase *tc = TestCases; tc && tc->name; tc++) { if (strcmp(argv[1], tc->name) == 0) diff --git a/Python/errors.c b/Python/errors.c index 6c5fe411423..211881ca5eb 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1241,6 +1241,17 @@ _PyErr_InitTypes(PyInterpreterState *interp) } +void +_PyErr_FiniTypes(PyInterpreterState *interp) +{ + if (!_Py_IsMainInterpreter(interp)) { + return; + } + + _PyStructSequence_FiniType(&UnraisableHookArgsType); +} + + static PyObject * make_unraisable_hook_args(PyThreadState *tstate, PyObject *exc_type, PyObject *exc_value, PyObject *exc_tb, diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 8bcad67e80a..0b1f4714769 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1666,11 +1666,17 @@ flush_std_files(void) static void finalize_interp_types(PyInterpreterState *interp) { + _PySys_Fini(interp); _PyExc_Fini(interp); _PyFrame_Fini(interp); _PyAsyncGen_Fini(interp); _PyContext_Fini(interp); + _PyFloat_FiniType(interp); + _PyLong_FiniTypes(interp); + _PyThread_FiniType(interp); + _PyErr_FiniTypes(interp); _PyTypes_Fini(interp); + // Call _PyUnicode_ClearInterned() before _PyDict_Fini() since it uses // a dict internally. _PyUnicode_ClearInterned(interp); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 0b7b61d8b1e..515994f0490 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3102,6 +3102,21 @@ error: } +void +_PySys_Fini(PyInterpreterState *interp) +{ + if (_Py_IsMainInterpreter(interp)) { + _PyStructSequence_FiniType(&VersionInfoType); + _PyStructSequence_FiniType(&FlagsType); +#if defined(MS_WINDOWS) + _PyStructSequence_FiniType(&WindowsVersionType); +#endif + _PyStructSequence_FiniType(&Hash_InfoType); + _PyStructSequence_FiniType(&AsyncGenHooksType); + } +} + + static PyObject * makepathobject(const wchar_t *path, wchar_t delim) { diff --git a/Python/thread.c b/Python/thread.c index b1c0cfe84f2..c2457c4f8fe 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -243,3 +243,14 @@ PyThread_GetInfo(void) PyStructSequence_SET_ITEM(threadinfo, pos++, value); return threadinfo; } + + +void +_PyThread_FiniType(PyInterpreterState *interp) +{ + if (!_Py_IsMainInterpreter(interp)) { + return; + } + + _PyStructSequence_FiniType(&ThreadInfoType); +}