[3.9] bpo-11105: Do not crash when compiling recursive ASTs (GH-20594) (GH-26522)

When compiling an AST object with a direct / indirect reference
cycles, on the conversion phase because of exceeding amount of
calls, a segfault was raised. This patch adds recursion guards to
places for preventing user inputs to not to crash AST but instead
raise a RecursionError..
(cherry picked from commit f3491242e4)

Co-authored-by: Batuhan Taskaya <batuhan@python.org>

Lib/test/test_ast.py

@@ -1027,6 +1027,20 @@ Module(
         exec(code, ns)
         self.assertIn('sleep', ns)
 
+    def test_recursion_direct(self):
+        e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0)
+        e.operand = e
+        with self.assertRaises(RecursionError):
+            compile(ast.Expression(e), "<test>", "eval")
+
+    def test_recursion_indirect(self):
+        e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0)
+        f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0)
+        e.operand = f
+        f.operand = e
+        with self.assertRaises(RecursionError):
+            compile(ast.Expression(e), "<test>", "eval")
+
 
 class ASTValidatorTests(unittest.TestCase):
 

Misc/NEWS.d/next/Core and Builtins/2020-06-02-13-21-14.bpo-11105.wceryW.rst

@@ -0,0 +1,3 @@
+When compiling :class:`ast.AST` objects with recursive references
+through :func:`compile`, the interpreter doesn't crash anymore instead
+it raises a :exc:`RecursionError`.

Parser/asdl_c.py

@@ -5,6 +5,7 @@ import os
 import sys
 
 from argparse import ArgumentParser
+from contextlib import contextmanager
 from pathlib import Path
 
 import asdl
@@ -394,6 +395,14 @@ class Obj2ModPrototypeVisitor(PickleVisitor):
 
 
 class Obj2ModVisitor(PickleVisitor):
+    @contextmanager
+    def recursive_call(self, node, level):
+        self.emit('if (Py_EnterRecursiveCall(" while traversing \'%s\' node")) {' % node, level, reflow=False)
+        self.emit('goto failed;', level + 1)
+        self.emit('}', level)
+        yield
+        self.emit('Py_LeaveRecursiveCall();', level)
+
     def funcHeader(self, name):
         ctype = get_c_type(name)
         self.emit("int", 0)
@@ -568,8 +577,9 @@ class Obj2ModVisitor(PickleVisitor):
             self.emit("%s val;" % ctype, depth+2)
             self.emit("PyObject *tmp2 = PyList_GET_ITEM(tmp, i);", depth+2)
             self.emit("Py_INCREF(tmp2);", depth+2)
-            self.emit("res = obj2ast_%s(state, tmp2, &val, arena);" %
+            with self.recursive_call(name, depth+2):
-                      field.type, depth+2, reflow=False)
+                self.emit("res = obj2ast_%s(state, tmp2, &val, arena);" %
+                          field.type, depth+2, reflow=False)
             self.emit("Py_DECREF(tmp2);", depth+2)
             self.emit("if (res != 0) goto failed;", depth+2)
             self.emit("if (len != PyList_GET_SIZE(tmp)) {", depth+2)
@@ -582,8 +592,9 @@ class Obj2ModVisitor(PickleVisitor):
             self.emit("asdl_seq_SET(%s, i, val);" % field.name, depth+2)
             self.emit("}", depth+1)
         else:
-            self.emit("res = obj2ast_%s(state, tmp, &%s, arena);" %
+            with self.recursive_call(name, depth+1):
-                      (field.type, field.name), depth+1)
+                self.emit("res = obj2ast_%s(state, tmp, &%s, arena);" %
+                          (field.type, field.name), depth+1)
             self.emit("if (res != 0) goto failed;", depth+1)
 
         self.emit("Py_CLEAR(tmp);", depth+1)