Issue #3697: "Fatal Python error: Cannot recover from stack overflow"

could be easily encountered under Windows in debug mode when exercising
the recursion limit checking code, due to bogus handling of recursion
limit when USE_STACKCHEK was enabled.

Reviewed by Amaury Forgeot d'Arc on IRC.

Include/ceval.h

@@ -42,26 +42,62 @@ PyAPI_FUNC(int) PyEval_MergeCompilerFlags(PyCompilerFlags *cf);
 PyAPI_FUNC(int) Py_AddPendingCall(int (*func)(void *), void *arg);
 PyAPI_FUNC(int) Py_MakePendingCalls(void);
 
-/* Protection against deeply nested recursive calls */
+/* Protection against deeply nested recursive calls
+
+   In Python 3.0, this protection has two levels:
+   * normal anti-recursion protection is triggered when the recursion level
+     exceeds the current recursion limit. It raises a RuntimeError, and sets
+     the "overflowed" flag in the thread state structure. This flag
+     temporarily *disables* the normal protection; this allows cleanup code
+     to potentially outgrow the recursion limit while processing the 
+     RuntimeError.
+   * "last chance" anti-recursion protection is triggered when the recursion
+     level exceeds "current recursion limit + 50". By construction, this
+     protection can only be triggered when the "overflowed" flag is set. It
+     means the cleanup code has itself gone into an infinite loop, or the
+     RuntimeError has been mistakingly ignored. When this protection is
+     triggered, the interpreter aborts with a Fatal Error.
+
+   In addition, the "overflowed" flag is automatically reset when the
+   recursion level drops below "current recursion limit - 50". This heuristic
+   is meant to ensure that the normal anti-recursion protection doesn't get
+   disabled too long.
+
+   Please note: this scheme has its own limitations. See:
+   http://mail.python.org/pipermail/python-dev/2008-August/082106.html
+   for some observations.
+*/
 PyAPI_FUNC(void) Py_SetRecursionLimit(int);
 PyAPI_FUNC(int) Py_GetRecursionLimit(void);
 
-#define Py_EnterRecursiveCall(where)                                    \
+#define Py_EnterRecursiveCall(where)  \
 	    (_Py_MakeRecCheck(PyThreadState_GET()->recursion_depth) &&  \
 	     _Py_CheckRecursiveCall(where))
 #define Py_LeaveRecursiveCall()				\
-    do{ if((--PyThreadState_GET()->recursion_depth) <   \
-	   _Py_CheckRecursionLimit - 50)                \
-	  PyThreadState_GET()->overflowed = 0;          \
-    } while(0)
+    do{ if(_Py_MakeEndRecCheck(PyThreadState_GET()->recursion_depth))  \
+	  PyThreadState_GET()->overflowed = 0;  \
+	} while(0)
 PyAPI_FUNC(int) _Py_CheckRecursiveCall(char *where);
 PyAPI_DATA(int) _Py_CheckRecursionLimit;
+
 #ifdef USE_STACKCHECK
-#  define _Py_MakeRecCheck(x)  (++(x) > --_Py_CheckRecursionLimit)
+/* With USE_STACKCHECK, we artificially decrement the recursion limit in order
+   to trigger regular stack checks in _Py_CheckRecursiveCall(), except if
+   the "overflowed" flag is set, in which case we need the true value
+   of _Py_CheckRecursionLimit for _Py_MakeEndRecCheck() to function properly.
+*/
+#  define _Py_MakeRecCheck(x)  \
+	(++(x) > (_Py_CheckRecursionLimit += PyThreadState_GET()->overflowed - 1))
 #else
 #  define _Py_MakeRecCheck(x)  (++(x) > _Py_CheckRecursionLimit)
 #endif
 
+#ifdef USE_STACKCHECK
+#  define _Py_MakeEndRecCheck(x)  (--(x) < _Py_CheckRecursionLimit - 50)
+#else
+#  define _Py_MakeEndRecCheck(x)  (--(x) < _Py_CheckRecursionLimit - 50)
+#endif
+
 #define Py_ALLOW_RECURSION \
   do { unsigned char _old = PyThreadState_GET()->recursion_critical;\
     PyThreadState_GET()->recursion_critical = 1;

Misc/NEWS

@@ -12,6 +12,11 @@ What's New in Python 3.0 release candidate 1
 Core and Builtins
 -----------------
 
+- Issue #3697: "Fatal Python error: Cannot recover from stack overflow"
+  could be easily encountered under Windows in debug mode when exercising
+  the recursion limit checking code, due to bogus handling of recursion
+  limit when USE_STACKCHEK was enabled.
+
 - Issue 3639: The _warnings module could segfault the interpreter when
   unexpected types were passed in as arguments.
 

Python/ceval.c

@@ -471,6 +471,7 @@ _Py_CheckRecursiveCall(char *where)
 		return -1;
 	}
 #endif
+	_Py_CheckRecursionLimit = recursion_limit;
 	if (tstate->recursion_critical)
 		/* Somebody asked that we don't check for recursion. */
 		return 0;
@@ -489,7 +490,6 @@ _Py_CheckRecursiveCall(char *where)
 			     where);
 		return -1;
 	}
-	_Py_CheckRecursionLimit = recursion_limit;
 	return 0;
 }