From 92a3c6f493ad411e4cf0acdf305ef4876aa90669 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 6 Dec 2017 18:12:59 +0100 Subject: [PATCH] bpo-32030: Add _PyImport_Fini2() (#4737) PyImport_ExtendInittab() now uses PyMem_RawRealloc() rather than PyMem_Realloc(). PyImport_ExtendInittab() can be called before Py_Initialize() whereas only the PyMem_Raw allocator is supposed to be used before Py_Initialize(). Add _PyImport_Fini2() to release the memory allocated by PyImport_ExtendInittab() at exit. PyImport_ExtendInittab() now forces the usage of the default raw allocator, to be able to release memory in _PyImport_Fini2(). Don't export these functions anymore to be C API, only to Py_BUILD_CORE: * _PyExc_Fini() * _PyImport_Fini() * _PyGC_DumpShutdownStats() * _PyGC_Fini() * _PyType_Fini() * _Py_HashRandomization_Fini() --- Include/pylifecycle.h | 17 +++++++++----- Modules/main.c | 44 +++++++++++++++++++++--------------- Python/import.c | 52 ++++++++++++++++++++++++++++++++++--------- 3 files changed, 78 insertions(+), 35 deletions(-) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 4c87428a23a..bda51f84c3c 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -137,9 +137,18 @@ PyAPI_FUNC(_PyInitError) _Py_HashRandomization_Init(_PyCoreConfig *core_config); #endif /* Various internal finalizers */ -#ifndef Py_LIMITED_API + +#ifdef Py_BUILD_CORE PyAPI_FUNC(void) _PyExc_Fini(void); PyAPI_FUNC(void) _PyImport_Fini(void); +PyAPI_FUNC(void) _PyImport_Fini2(void); +PyAPI_FUNC(void) _PyGC_DumpShutdownStats(void); +PyAPI_FUNC(void) _PyGC_Fini(void); +PyAPI_FUNC(void) _PyType_Fini(void); +PyAPI_FUNC(void) _Py_HashRandomization_Fini(void); +#endif /* Py_BUILD_CORE */ + +#ifndef Py_LIMITED_API PyAPI_FUNC(void) PyMethod_Fini(void); PyAPI_FUNC(void) PyFrame_Fini(void); PyAPI_FUNC(void) PyCFunction_Fini(void); @@ -151,15 +160,11 @@ PyAPI_FUNC(void) PyBytes_Fini(void); PyAPI_FUNC(void) PyByteArray_Fini(void); PyAPI_FUNC(void) PyFloat_Fini(void); PyAPI_FUNC(void) PyOS_FiniInterrupts(void); -PyAPI_FUNC(void) _PyGC_DumpShutdownStats(void); -PyAPI_FUNC(void) _PyGC_Fini(void); PyAPI_FUNC(void) PySlice_Fini(void); -PyAPI_FUNC(void) _PyType_Fini(void); -PyAPI_FUNC(void) _Py_HashRandomization_Fini(void); PyAPI_FUNC(void) PyAsyncGen_Fini(void); PyAPI_FUNC(int) _Py_IsFinalizing(void); -#endif +#endif /* !Py_LIMITED_API */ /* Signals */ typedef void (*PyOS_sighandler_t)(int); diff --git a/Modules/main.c b/Modules/main.c index 6d53c56e356..4ab1ff29fac 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -446,17 +446,18 @@ pymain_optlist_clear(_Py_OptList *list) list->options = NULL; } + +/* Free global variables which cannot be freed in Py_Finalize(): + configuration options set before Py_Initialize() which should + remain valid after Py_Finalize(), since Py_Initialize()/Py_Finalize() can + be called multiple times. + + Called with the current memory allocators. */ static void -pymain_free_impl(_PyMain *pymain) +pymain_free_globals(_PyMain *pymain) { - _Py_CommandLineDetails *cmdline = &pymain->cmdline; - pymain_optlist_clear(&cmdline->warning_options); - pymain_optlist_clear(&cmdline->xoptions); - PyMem_RawFree(cmdline->command); - - pymain_optlist_clear(&pymain->env_warning_options); - Py_CLEAR(pymain->main_importer_path); - + _PyPathConfig_Clear(&_Py_path_config); + _PyImport_Fini2(); _PyMainInterpreterConfig_Clear(&pymain->config); #ifdef __INSURE__ @@ -473,6 +474,20 @@ pymain_free_impl(_PyMain *pymain) #endif /* __INSURE__ */ } + +static void +pymain_free_pymain(_PyMain *pymain) +{ + _Py_CommandLineDetails *cmdline = &pymain->cmdline; + pymain_optlist_clear(&cmdline->warning_options); + pymain_optlist_clear(&cmdline->xoptions); + PyMem_RawFree(cmdline->command); + + pymain_optlist_clear(&pymain->env_warning_options); + Py_CLEAR(pymain->main_importer_path); + +} + static void pymain_free(_PyMain *pymain) { @@ -480,12 +495,12 @@ pymain_free(_PyMain *pymain) PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - pymain_free_impl(pymain); + pymain_free_pymain(pymain); + pymain_free_globals(pymain); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } - static int pymain_run_main_from_importer(_PyMain *pymain) { @@ -1719,13 +1734,6 @@ pymain_impl(_PyMain *pymain) pymain->status = 120; } - /* _PyPathConfig_Clear() cannot be called in Py_FinalizeEx(). - Py_Initialize() and Py_Finalize() can be called multiple times, but it - must not "forget" parameters set by Py_SetProgramName(), Py_SetPath() or - Py_SetPythonHome(), whereas _PyPathConfig_Clear() clear all these - parameters. */ - _PyPathConfig_Clear(&_Py_path_config); - return 0; } diff --git a/Python/import.c b/Python/import.c index 9a9857385cb..cc7417ba84e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -30,6 +30,7 @@ static PyObject *extensions = NULL; extern struct _inittab _PyImport_Inittab[]; struct _inittab *PyImport_Inittab = _PyImport_Inittab; +static struct _inittab *inittab_copy = NULL; /*[clinic input] module _imp @@ -285,6 +286,19 @@ _PyImport_Fini(void) } } +void +_PyImport_Fini2(void) +{ + /* Use the same memory allocator than PyImport_ExtendInittab(). */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + /* Free memory allocated by PyImport_ExtendInittab() */ + PyMem_RawFree(inittab_copy); + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); +} + /* Helper for sys */ PyObject * @@ -2233,9 +2247,9 @@ PyInit_imp(void) int PyImport_ExtendInittab(struct _inittab *newtab) { - static struct _inittab *our_copy = NULL; struct _inittab *p; - int i, n; + Py_ssize_t i, n; + int res = 0; /* Count the number of entries in both tables */ for (n = 0; newtab[n].name != NULL; n++) @@ -2245,19 +2259,35 @@ PyImport_ExtendInittab(struct _inittab *newtab) for (i = 0; PyImport_Inittab[i].name != NULL; i++) ; + /* Force default raw memory allocator to get a known allocator to be able + to release the memory in _PyImport_Fini2() */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + /* Allocate new memory for the combined table */ - p = our_copy; - PyMem_RESIZE(p, struct _inittab, i+n+1); - if (p == NULL) - return -1; + if ((i + n + 1) <= PY_SSIZE_T_MAX / sizeof(struct _inittab)) { + size_t size = sizeof(struct _inittab) * (i + n + 1); + p = PyMem_RawRealloc(inittab_copy, size); + } + else { + p = NULL; + } + if (p == NULL) { + res = -1; + goto done; + } - /* Copy the tables into the new memory */ - if (our_copy != PyImport_Inittab) + /* Copy the tables into the new memory at the first call + to PyImport_ExtendInittab(). */ + if (inittab_copy != PyImport_Inittab) { memcpy(p, PyImport_Inittab, (i+1) * sizeof(struct _inittab)); - PyImport_Inittab = our_copy = p; - memcpy(p+i, newtab, (n+1) * sizeof(struct _inittab)); + } + memcpy(p + i, newtab, (n + 1) * sizeof(struct _inittab)); + PyImport_Inittab = inittab_copy = p; - return 0; +done: + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + return res; } /* Shorthand to add a single entry given a name and a function */