From 17b73ab99ef12f89d41acec7500a244e68b1aaa4 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 16 Jan 2024 09:32:01 +0000 Subject: [PATCH] GH-113655: Lower the C recursion limit on various platforms (GH-113944) --- Include/cpython/pystate.h | 8 ++++++-- Lib/test/support/__init__.py | 5 ++++- Lib/test/test_ast.py | 2 +- Lib/test/test_compile.py | 8 +++----- Lib/test/test_functools.py | 8 +++++++- Lib/test/test_sys_settrace.py | 6 ++---- .../2024-01-11-01-28-25.gh-issue-113655.Mfioxp.rst | 3 +++ Parser/asdl_c.py | 5 ++--- Python/Python-ast.c | 5 ++--- Python/ast.c | 8 ++------ Python/ast_opt.c | 7 ++----- Python/symtable.c | 9 ++------- Python/traceback.c | 6 +++++- 13 files changed, 41 insertions(+), 39 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-01-11-01-28-25.gh-issue-113655.Mfioxp.rst diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index ed7dd829d4b..10913943c11 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -224,10 +224,14 @@ struct _ts { // recursions, sometimes less. 500 is a more conservative limit. # define Py_C_RECURSION_LIMIT 500 #elif defined(__s390x__) -# define Py_C_RECURSION_LIMIT 1200 +# define Py_C_RECURSION_LIMIT 800 +#elif defined(_WIN32) +# define Py_C_RECURSION_LIMIT 4000 +#elif defined(_Py_ADDRESS_SANITIZER) +# define Py_C_RECURSION_LIMIT 4000 #else // This value is duplicated in Lib/test/support/__init__.py -# define Py_C_RECURSION_LIMIT 8000 +# define Py_C_RECURSION_LIMIT 10000 #endif diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index e5fb725a30b..8344dd1849c 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2377,7 +2377,10 @@ def _get_c_recursion_limit(): return _testcapi.Py_C_RECURSION_LIMIT except (ImportError, AttributeError): # Originally taken from Include/cpython/pystate.h . - return 8000 + if sys.platform == 'win32': + return 4000 + else: + return 10000 # The default C recursion limit. Py_C_RECURSION_LIMIT = _get_c_recursion_limit() diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 64fcb02309d..3789ac22e38 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1126,7 +1126,7 @@ class AST_Tests(unittest.TestCase): def test_ast_recursion_limit(self): fail_depth = support.EXCEEDS_RECURSION_LIMIT crash_depth = 100_000 - success_depth = 1200 + success_depth = int(support.Py_C_RECURSION_LIMIT * 0.8) if _testinternalcapi is not None: remaining = _testinternalcapi.get_c_recursion_remaining() success_depth = min(success_depth, remaining) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 50629b22822..9c36f053314 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -623,12 +623,10 @@ class TestSpecifics(unittest.TestCase): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): - # Expected limit is Py_C_RECURSION_LIMIT * 2 - # Duplicating the limit here is a little ugly. - # Perhaps it should be exposed somewhere... - fail_depth = Py_C_RECURSION_LIMIT * 2 + 1 + # Expected limit is Py_C_RECURSION_LIMIT + fail_depth = Py_C_RECURSION_LIMIT + 1 crash_depth = Py_C_RECURSION_LIMIT * 100 - success_depth = int(Py_C_RECURSION_LIMIT * 1.8) + success_depth = int(Py_C_RECURSION_LIMIT * 0.8) def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 0ef45d3c670..7c66b906d30 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1875,8 +1875,14 @@ class TestLRU: return fib(n-1) + fib(n-2) if not support.Py_DEBUG: + depth = support.Py_C_RECURSION_LIMIT*2//7 with support.infinite_recursion(): - fib(2500) + fib(depth) + if self.module == c_functools: + fib.cache_clear() + with support.infinite_recursion(): + with self.assertRaises(RecursionError): + fib(10000) @py_functools.lru_cache() diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index fc5ca72236b..ae6e192a7ab 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -3037,10 +3037,8 @@ class TestExtendedArgs(unittest.TestCase): self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1}) def test_trace_lots_of_globals(self): - count = 1000 - if _testinternalcapi is not None: - remaining = _testinternalcapi.get_c_recursion_remaining() - count = min(count, remaining) + + count = min(1000, int(support.Py_C_RECURSION_LIMIT * 0.8)) code = """if 1: def f(): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-11-01-28-25.gh-issue-113655.Mfioxp.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-11-01-28-25.gh-issue-113655.Mfioxp.rst new file mode 100644 index 00000000000..30f1b0a7661 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-11-01-28-25.gh-issue-113655.Mfioxp.rst @@ -0,0 +1,3 @@ +Set the C recursion limit to 4000 on Windows, and 10000 on Linux/OSX. This +seems to be near the sweet spot to maintain safety, but not compromise +backwards compatibility. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 4bb33734974..ce92672bf00 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1388,15 +1388,14 @@ PyObject* PyAST_mod2obj(mod_ty t) int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return NULL; } struct validator vstate; - vstate.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + vstate.recursion_limit = Py_C_RECURSION_LIMIT; int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; vstate.recursion_depth = starting_recursion_depth; PyObject *result = ast2obj_mod(state, &vstate, t); diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 699e1c157c5..d77e986ba06 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -13149,15 +13149,14 @@ PyObject* PyAST_mod2obj(mod_ty t) int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return NULL; } struct validator vstate; - vstate.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + vstate.recursion_limit = Py_C_RECURSION_LIMIT; int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; vstate.recursion_depth = starting_recursion_depth; PyObject *result = ast2obj_mod(state, &vstate, t); diff --git a/Python/ast.c b/Python/ast.c index 5f46d4149c2..71b09d889f1 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1037,10 +1037,6 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps) return 1; } - -/* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 2 - int _PyAST_Validate(mod_ty mod) { @@ -1057,9 +1053,9 @@ _PyAST_Validate(mod_ty mod) } /* Be careful here to prevent overflow. */ int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; state.recursion_depth = starting_recursion_depth; - state.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + state.recursion_limit = Py_C_RECURSION_LIMIT; switch (mod->kind) { case Module_kind: diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 04d7ae6eaaf..41e906c66e8 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -1100,9 +1100,6 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat #undef CALL_OPT #undef CALL_SEQ -/* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 2 - int _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features) { @@ -1120,9 +1117,9 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features) } /* Be careful here to prevent overflow. */ int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; state.recursion_depth = starting_recursion_depth; - state.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + state.recursion_limit = Py_C_RECURSION_LIMIT; int ret = astfold_mod(mod, arena, &state); assert(ret || PyErr_Occurred()); diff --git a/Python/symtable.c b/Python/symtable.c index 83137b491f2..743029956e3 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -386,11 +386,6 @@ symtable_new(void) return NULL; } -/* Using a scaling factor means this should automatically adjust when - the recursion limit is adjusted for small or large C stack allocations. -*/ -#define COMPILER_STACK_FRAME_SCALE 2 - struct symtable * _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) { @@ -417,9 +412,9 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) } /* Be careful here to prevent overflow. */ int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; st->recursion_depth = starting_recursion_depth; - st->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + st->recursion_limit = Py_C_RECURSION_LIMIT; /* Make the initial symbol information gathering pass */ if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) { diff --git a/Python/traceback.c b/Python/traceback.c index abd429ac6c1..7a188e56c93 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -965,7 +965,11 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) unsigned int depth = 0; while (1) { if (MAX_FRAME_DEPTH <= depth) { - PUTS(fd, " ...\n"); + if (MAX_FRAME_DEPTH < depth) { + PUTS(fd, "plus "); + _Py_DumpDecimal(fd, depth); + PUTS(fd, " frames\n"); + } break; } dump_frame(fd, frame);