From 31e99080f6f8cf7faaba9fe3a4e0996e49163317 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 20 Dec 2017 23:41:38 +0100 Subject: [PATCH] bpo-32030: Fix usage of memory allocators (#4953) * _Py_InitializeCore() doesn't call _PyMem_SetupAllocators() anymore if the PYTHONMALLOC environment variable is not set. * pymain_cmdline() now sets the allocator to the default, instead of setting the allocator in subfunctions. * Py_SetStandardStreamEncoding() now calls _PyMem_SetDefaultAllocator() to get a known allocator, to be able to release the memory with the same allocator. --- .../2017-12-20-23-22-32.bpo-32030.d1dcwh.rst | 2 ++ Modules/main.c | 26 ++++++++++---- Python/pylifecycle.c | 34 ++++++++++++++++--- 3 files changed, 50 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2017-12-20-23-22-32.bpo-32030.d1dcwh.rst diff --git a/Misc/NEWS.d/next/C API/2017-12-20-23-22-32.bpo-32030.d1dcwh.rst b/Misc/NEWS.d/next/C API/2017-12-20-23-22-32.bpo-32030.d1dcwh.rst new file mode 100644 index 00000000000..8e8b430e247 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2017-12-20-23-22-32.bpo-32030.d1dcwh.rst @@ -0,0 +1,2 @@ +Py_Initialize() doesn't reset the memory allocators to default if the +``PYTHONMALLOC`` environment variable is not set. diff --git a/Modules/main.c b/Modules/main.c index 0a36f9db9d8..1bf706b162c 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -649,11 +649,12 @@ pymain_free_raw(_PyMain *pymain) Py_Initialize()-Py_Finalize() can be called multiple times. */ _PyPathConfig_Clear(&_Py_path_config); + pymain_clear_config(pymain); + /* Force the allocator used by pymain_read_conf() */ PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - _PyCoreConfig_Clear(&pymain->config); pymain_clear_pymain(pymain); clear_wstrlist(orig_argc, orig_argv); @@ -1963,11 +1964,6 @@ pymain_read_conf(_PyMain *pymain, _Py_CommandLineDetails *cmdline) { int res = -1; - /* Force default allocator, since pymain_free() must use the same allocator - than this function. */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - char *oldloc = _PyMem_RawStrdup(setlocale(LC_ALL, NULL)); if (oldloc == NULL) { pymain->err = _Py_INIT_NO_MEMORY(); @@ -2055,7 +2051,6 @@ done: PyMem_RawFree(oldloc); } - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); return res; } @@ -2578,6 +2573,15 @@ pymain_cmdline_impl(_PyMain *pymain, _Py_CommandLineDetails *cmdline) static int pymain_cmdline(_PyMain *pymain) { + /* Force default allocator, since pymain_free() and pymain_clear_config() + must use the same allocator than this function. */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); +#ifdef Py_DEBUG + PyMemAllocatorEx default_alloc; + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &default_alloc); +#endif + _Py_CommandLineDetails cmdline; memset(&cmdline, 0, sizeof(cmdline)); @@ -2588,6 +2592,14 @@ pymain_cmdline(_PyMain *pymain) pymain_set_global_config(pymain, &cmdline); pymain_clear_cmdline(pymain, &cmdline); + +#ifdef Py_DEBUG + /* Make sure that PYMEM_DOMAIN_RAW has not been modified */ + PyMemAllocatorEx cur_alloc; + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &cur_alloc); + assert(memcmp(&cur_alloc, &default_alloc, sizeof(cur_alloc)) == 0); +#endif + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); return res; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 2af76d76d2b..560d0e36d49 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -172,6 +172,15 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors) /* This is too late to have any effect */ return -1; } + + int res = 0; + + /* Py_SetStandardStreamEncoding() can be called before Py_Initialize(), + but Py_Initialize() can change the allocator. Use a known allocator + to be able to release the memory later. */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + /* Can't call PyErr_NoMemory() on errors, as Python hasn't been * initialised yet. * @@ -182,7 +191,8 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors) if (encoding) { _Py_StandardStreamEncoding = _PyMem_RawStrdup(encoding); if (!_Py_StandardStreamEncoding) { - return -2; + res = -2; + goto done; } } if (errors) { @@ -191,7 +201,8 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors) if (_Py_StandardStreamEncoding) { PyMem_RawFree(_Py_StandardStreamEncoding); } - return -3; + res = -3; + goto done; } } #ifdef MS_WINDOWS @@ -200,7 +211,11 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors) Py_LegacyWindowsStdioFlag = 1; } #endif - return 0; + +done: + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + return res; } @@ -597,8 +612,10 @@ _Py_InitializeCore(const _PyCoreConfig *core_config) return err; } - if (_PyMem_SetupAllocators(core_config->allocator) < 0) { - return _Py_INIT_USER_ERR("Unknown PYTHONMALLOC allocator"); + if (core_config->allocator != NULL) { + if (_PyMem_SetupAllocators(core_config->allocator) < 0) { + return _Py_INIT_USER_ERR("Unknown PYTHONMALLOC allocator"); + } } if (_PyRuntime.initialized) { @@ -1818,7 +1835,11 @@ init_sys_streams(PyInterpreterState *interp) error: res = _Py_INIT_ERR("can't initialize sys standard streams"); + /* Use the same allocator than Py_SetStandardStreamEncoding() */ + PyMemAllocatorEx old_alloc; done: + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + /* We won't need them anymore. */ if (_Py_StandardStreamEncoding) { PyMem_RawFree(_Py_StandardStreamEncoding); @@ -1828,6 +1849,9 @@ done: PyMem_RawFree(_Py_StandardStreamErrors); _Py_StandardStreamErrors = NULL; } + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + PyMem_Free(pythonioencoding); Py_XDECREF(bimod); Py_XDECREF(iomod);