bpo-33720: Reduces maximum marshal recursion depth on release builds. (GH-7401)

(cherry picked from commit 2a4a62ba4a)

Co-authored-by: Steve Dower <steve.dower@microsoft.com>

Lib/test/test_marshal.py

@@ -222,7 +222,10 @@ class BugsTestCase(unittest.TestCase):
         # Create a deeply nested structure.
         head = last = []
         # The max stack depth should match the value in Python/marshal.c.
-        if os.name == 'nt' and hasattr(sys, 'gettotalrefcount'):
+        # BUG: https://bugs.python.org/issue33720
+        # Windows always limits the maximum depth on release and debug builds
+        #if os.name == 'nt' and hasattr(sys, 'gettotalrefcount'):
+        if os.name == 'nt':
             MAX_MARSHAL_STACK_DEPTH = 1000
         else:
             MAX_MARSHAL_STACK_DEPTH = 2000

Misc/NEWS.d/next/Windows/2018-06-04-09-20-53.bpo-33720.VKDXHK.rst

@@ -0,0 +1 @@
+Reduces maximum marshal recursion depth on release builds.

Python/marshal.c

@@ -25,8 +25,14 @@ module marshal
  * and risks coring the interpreter.  When the object stack gets this deep,
  * raise an exception instead of continuing.
  * On Windows debug builds, reduce this value.
+ *
+ * BUG: https://bugs.python.org/issue33720
+ * On Windows PGO builds, the r_object function overallocates its stack and
+ * can cause a stack overflow. We reduce the maximum depth for all Windows
+ * releases to protect against this.
+ * #if defined(MS_WINDOWS) && defined(_DEBUG)
  */
-#if defined(MS_WINDOWS) && defined(_DEBUG)
+#if defined(MS_WINDOWS)
 #define MAX_MARSHAL_STACK_DEPTH 1000
 #else
 #define MAX_MARSHAL_STACK_DEPTH 2000