bpo-11105: Do not crash when compiling recursive ASTs (GH-20594)

When compiling an AST object with a direct / indirect reference
cycles, on the conversion phase because of exceeding amount of
calls, a segfault was raised. This patch adds recursion guards to
places for preventing user inputs to not to crash AST but instead
raise a RecursionError.

Lib/test/test_ast.py

@@ -1098,6 +1098,20 @@ Module(
         exec(code, ns)
         self.assertIn('sleep', ns)
 
+    def test_recursion_direct(self):
+        e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0)
+        e.operand = e
+        with self.assertRaises(RecursionError):
+            compile(ast.Expression(e), "<test>", "eval")
+
+    def test_recursion_indirect(self):
+        e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0)
+        f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0)
+        e.operand = f
+        f.operand = e
+        with self.assertRaises(RecursionError):
+            compile(ast.Expression(e), "<test>", "eval")
+
 
 class ASTValidatorTests(unittest.TestCase):
 

Misc/NEWS.d/next/Core and Builtins/2020-06-02-13-21-14.bpo-11105.wceryW.rst

@@ -0,0 +1,3 @@
+When compiling :class:`ast.AST` objects with recursive references
+through :func:`compile`, the interpreter doesn't crash anymore instead
+it raises a :exc:`RecursionError`.

Parser/asdl_c.py

@@ -6,6 +6,7 @@ import sys
 import textwrap
 
 from argparse import ArgumentParser
+from contextlib import contextmanager
 from pathlib import Path
 
 import asdl
@@ -421,6 +422,14 @@ class Obj2ModPrototypeVisitor(PickleVisitor):
 
 
 class Obj2ModVisitor(PickleVisitor):
+    @contextmanager
+    def recursive_call(self, node, level):
+        self.emit('if (Py_EnterRecursiveCall(" while traversing \'%s\' node")) {' % node, level, reflow=False)
+        self.emit('goto failed;', level + 1)
+        self.emit('}', level)
+        yield
+        self.emit('Py_LeaveRecursiveCall();', level)
+
     def funcHeader(self, name):
         ctype = get_c_type(name)
         self.emit("int", 0)
@@ -596,6 +605,7 @@ class Obj2ModVisitor(PickleVisitor):
             self.emit("%s val;" % ctype, depth+2)
             self.emit("PyObject *tmp2 = PyList_GET_ITEM(tmp, i);", depth+2)
             self.emit("Py_INCREF(tmp2);", depth+2)
+            with self.recursive_call(name, depth+2):
                 self.emit("res = obj2ast_%s(state, tmp2, &val, arena);" %
                           field.type, depth+2, reflow=False)
             self.emit("Py_DECREF(tmp2);", depth+2)
@@ -610,6 +620,7 @@ class Obj2ModVisitor(PickleVisitor):
             self.emit("asdl_seq_SET(%s, i, val);" % field.name, depth+2)
             self.emit("}", depth+1)
         else:
+            with self.recursive_call(name, depth+1):
                 self.emit("res = obj2ast_%s(state, tmp, &%s, arena);" %
                           (field.type, field.name), depth+1)
             self.emit("if (res != 0) goto failed;", depth+1)