From c7f810b34d91a5c2fbe0a8385562015d2dd961f2 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Thu, 27 Jan 2022 18:33:47 +0530 Subject: [PATCH] bpo-46476: Fix memory leak in code objects generated by deepfreeze (GH-30853) Add _Py_Deepfreeze_Fini() and _PyStaticCode_Dealloc() functions. --- Include/internal/pycore_code.h | 2 ++ Include/internal/pycore_pylifecycle.h | 1 + .../2022-01-24-15-39-34.bpo-46476.cvP1Mr.rst | 1 + Objects/codeobject.c | 15 +++++++++++++++ Programs/_bootstrap_python.c | 7 +++++++ Programs/_freeze_module.c | 7 +++++++ Python/pylifecycle.c | 1 + Tools/scripts/deepfreeze.py | 5 +++++ 8 files changed, 39 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-01-24-15-39-34.bpo-46476.cvP1Mr.rst diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index dfc75300315..884a91f3fb7 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -276,6 +276,8 @@ void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, SpecializedCacheEntry *cache); void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, SpecializedCacheEntry *cache); +/* Deallocator function for static codeobjects used in deepfreeze.py */ +void _PyStaticCode_Dealloc(PyCodeObject *co, _Py_CODEUNIT *firstinstr); #ifdef Py_STATS diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 35e560b42ee..439bc5b470b 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -83,6 +83,7 @@ extern void _PyWarnings_Fini(PyInterpreterState *interp); extern void _PyAST_Fini(PyInterpreterState *interp); extern void _PyAtExit_Fini(PyInterpreterState *interp); extern void _PyThread_FiniType(PyInterpreterState *interp); +extern void _Py_Deepfreeze_Fini(void); extern PyStatus _PyGILState_Init(_PyRuntimeState *runtime); extern PyStatus _PyGILState_SetTstate(PyThreadState *tstate); diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-01-24-15-39-34.bpo-46476.cvP1Mr.rst b/Misc/NEWS.d/next/Core and Builtins/2022-01-24-15-39-34.bpo-46476.cvP1Mr.rst new file mode 100644 index 00000000000..26079839a5f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-01-24-15-39-34.bpo-46476.cvP1Mr.rst @@ -0,0 +1 @@ +Fix memory leak in code objects generated by deepfreeze. Patch by Kumar Aditya. \ No newline at end of file diff --git a/Objects/codeobject.c b/Objects/codeobject.c index a413b183be8..f983d66db05 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1906,3 +1906,18 @@ _PyCode_ConstantKey(PyObject *op) } return key; } + +void +_PyStaticCode_Dealloc(PyCodeObject *co, _Py_CODEUNIT *firstinstr) +{ + PyMem_Free(co->co_quickened); + co->co_quickened = NULL; + PyMem_Free(co->co_extra); + co->co_extra = NULL; + co->co_firstinstr = firstinstr; + if (co->co_weakreflist != NULL) { + PyObject_ClearWeakRefs((PyObject *)co); + co->co_weakreflist = NULL; + } + co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE; +} diff --git a/Programs/_bootstrap_python.c b/Programs/_bootstrap_python.c index f2103fd5aaa..f4d0167b62e 100644 --- a/Programs/_bootstrap_python.c +++ b/Programs/_bootstrap_python.c @@ -14,6 +14,12 @@ #include "Python/frozen_modules/importlib._bootstrap_external.h" /* End includes */ +/* Empty finalizer for deepfrozen modules*/ +void +_Py_Deepfreeze_Fini(void) +{ +} + /* Note that a negative size indicates a package. */ static const struct _frozen bootstrap_modules[] = { @@ -103,3 +109,4 @@ error: } Py_ExitStatusException(status); } + diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index b2f1a24016f..99a1d4dfc26 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -22,6 +22,12 @@ #include #endif +/* Empty finalizer for deepfrozen modules */ +void +_Py_Deepfreeze_Fini(void) +{ +} + /* To avoid a circular dependency on frozen.o, we create our own structure of frozen modules instead, left deliberately blank so as to avoid unintentional import of a stale version of _frozen_importlib. */ @@ -235,3 +241,4 @@ error: Py_Finalize(); return 1; } + diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 9d10f94efa7..106e11ea12f 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1723,6 +1723,7 @@ finalize_interp_clear(PyThreadState *tstate) _Py_HashRandomization_Fini(); _PyArg_Fini(); _Py_ClearFileSystemEncoding(); + _Py_Deepfreeze_Fini(); } finalize_interp_types(tstate->interp); diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index a1ef85ea891..78849390f12 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -109,6 +109,7 @@ class Printer: self.cache: Dict[tuple[type, object, str], str] = {} self.hits, self.misses = 0, 0 self.patchups: list[str] = [] + self.deallocs: list[str] = [] self.write('#include "Python.h"') self.write('#include "internal/pycore_gc.h"') self.write('#include "internal/pycore_code.h"') @@ -277,6 +278,7 @@ class Printer: self.write(f".co_varnames = {co_varnames},") self.write(f".co_cellvars = {co_cellvars},") self.write(f".co_freevars = {co_freevars},") + self.deallocs.append(f"_PyStaticCode_Dealloc(&{name}, (_Py_CODEUNIT *) {removesuffix(co_code, '.ob_base.ob_base')}.ob_sval);") return f"& {name}.ob_base" def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: @@ -440,6 +442,9 @@ def generate(args: list[str], output: TextIO) -> None: else: code = compile(fd.read(), f"", "exec") printer.generate_file(modname, code) + with printer.block(f"void\n_Py_Deepfreeze_Fini(void)"): + for p in printer.deallocs: + printer.write(p) if verbose: print(f"Cache hits: {printer.hits}, misses: {printer.misses}")