[3.9] bpo-43710: Rollback the 3.9 bpo-42500 fix, it broke the ABI in 3.9.3 (#25179)

This reverts commit 8b795ab554.

It changed the PyThreadState structure size, breaking the ABI in 3.9.3.

Include/cpython/pystate.h

@@ -58,7 +58,8 @@ struct _ts {
     /* Borrowed reference to the current frame (it can be NULL) */
     PyFrameObject *frame;
     int recursion_depth;
-    int recursion_headroom; /* Allow 50 more calls to handle any errors. */
+    char overflowed; /* The stack has overflowed. Allow 50 more calls
+                        to handle the runtime error. */
     char recursion_critical; /* The current calls must not cause
                                 a stack overflow. */
     int stackcheck_counter;

Include/internal/pycore_ceval.h

@@ -90,8 +90,24 @@ static inline int _Py_EnterRecursiveCall_inline(const char *where) {
 
 #define Py_EnterRecursiveCall(where) _Py_EnterRecursiveCall_inline(where)
 
+/* Compute the "lower-water mark" for a recursion limit. When
+ * Py_LeaveRecursiveCall() is called with a recursion depth below this mark,
+ * the overflowed flag is reset to 0. */
+static inline int _Py_RecursionLimitLowerWaterMark(int limit) {
+    if (limit > 200) {
+        return (limit - 50);
+    }
+    else {
+        return (3 * (limit >> 2));
+    }
+}
+
 static inline void _Py_LeaveRecursiveCall(PyThreadState *tstate)  {
     tstate->recursion_depth--;
+    int limit = tstate->interp->ceval.recursion_limit;
+    if (tstate->recursion_depth < _Py_RecursionLimitLowerWaterMark(limit)) {
+        tstate->overflowed = 0;
+    }
 }
 
 static inline void _Py_LeaveRecursiveCall_inline(void)  {

Lib/test/test_exceptions.py

@@ -1043,7 +1043,7 @@ class ExceptionTests(unittest.TestCase):
                 # tstate->recursion_depth is equal to (recursion_limit - 1)
                 # and is equal to recursion_limit when _gen_throw() calls
                 # PyErr_NormalizeException().
-                recurse(setrecursionlimit(depth + 2) - depth)
+                recurse(setrecursionlimit(depth + 2) - depth - 1)
             finally:
                 sys.setrecursionlimit(recursionlimit)
                 print('Done.')
@@ -1073,54 +1073,6 @@ class ExceptionTests(unittest.TestCase):
                       b'while normalizing an exception', err)
         self.assertIn(b'Done.', out)
 
-
-    def test_recursion_in_except_handler(self):
-
-        def set_relative_recursion_limit(n):
-            depth = 1
-            while True:
-                try:
-                    sys.setrecursionlimit(depth)
-                except RecursionError:
-                    depth += 1
-                else:
-                    break
-            sys.setrecursionlimit(depth+n)
-
-        def recurse_in_except():
-            try:
-                1/0
-            except:
-                recurse_in_except()
-
-        def recurse_after_except():
-            try:
-                1/0
-            except:
-                pass
-            recurse_after_except()
-
-        def recurse_in_body_and_except():
-            try:
-                recurse_in_body_and_except()
-            except:
-                recurse_in_body_and_except()
-
-        recursionlimit = sys.getrecursionlimit()
-        try:
-            set_relative_recursion_limit(10)
-            for func in (recurse_in_except, recurse_after_except, recurse_in_body_and_except):
-                with self.subTest(func=func):
-                    try:
-                        func()
-                    except RecursionError:
-                        pass
-                    else:
-                        self.fail("Should have raised a RecursionError")
-        finally:
-            sys.setrecursionlimit(recursionlimit)
-
-
     @cpython_only
     def test_recursion_normalizing_with_no_memory(self):
         # Issue #30697. Test that in the abort that occurs when there is no
@@ -1157,7 +1109,7 @@ class ExceptionTests(unittest.TestCase):
             except MemoryError as e:
                 tb = e.__traceback__
             else:
-                self.fail("Should have raised a MemoryError")
+                self.fail("Should have raises a MemoryError")
             return traceback.format_tb(tb)
 
         tb1 = raiseMemError()

Lib/test/test_sys.py

@@ -219,7 +219,7 @@ class SysModuleTest(unittest.TestCase):
         def f():
             f()
         try:
-            for depth in (50, 75, 100, 250, 1000):
+            for depth in (10, 25, 50, 75, 100, 250, 1000):
                 try:
                     sys.setrecursionlimit(depth)
                 except RecursionError:
@@ -229,17 +229,17 @@ class SysModuleTest(unittest.TestCase):
 
                 # Issue #5392: test stack overflow after hitting recursion
                 # limit twice
-                with self.assertRaises(RecursionError):
+                self.assertRaises(RecursionError, f)
-                    f()
+                self.assertRaises(RecursionError, f)
-                with self.assertRaises(RecursionError):
-                    f()
         finally:
             sys.setrecursionlimit(oldlimit)
 
     @test.support.cpython_only
     def test_setrecursionlimit_recursion_depth(self):
         # Issue #25274: Setting a low recursion limit must be blocked if the
-        # current recursion depth is already higher than limit.
+        # current recursion depth is already higher than the "lower-water
+        # mark". Otherwise, it may not be possible anymore to
+        # reset the overflowed flag to 0.
 
         from _testinternalcapi import get_recursion_depth
 
@@ -260,10 +260,42 @@ class SysModuleTest(unittest.TestCase):
             sys.setrecursionlimit(1000)
 
             for limit in (10, 25, 50, 75, 100, 150, 200):
-                set_recursion_limit_at_depth(limit, limit)
+                # formula extracted from _Py_RecursionLimitLowerWaterMark()
+                if limit > 200:
+                    depth = limit - 50
+                else:
+                    depth = limit * 3 // 4
+                set_recursion_limit_at_depth(depth, limit)
         finally:
             sys.setrecursionlimit(oldlimit)
 
+    # The error message is specific to CPython
+    @test.support.cpython_only
+    def test_recursionlimit_fatalerror(self):
+        # A fatal error occurs if a second recursion limit is hit when recovering
+        # from a first one.
+        code = textwrap.dedent("""
+            import sys
+
+            def f():
+                try:
+                    f()
+                except RecursionError:
+                    f()
+
+            sys.setrecursionlimit(%d)
+            f()""")
+        with test.support.SuppressCrashReport():
+            for i in (50, 1000):
+                sub = subprocess.Popen([sys.executable, '-c', code % i],
+                    stderr=subprocess.PIPE)
+                err = sub.communicate()[1]
+                self.assertTrue(sub.returncode, sub.returncode)
+                self.assertIn(
+                    b"Fatal Python error: _Py_CheckRecursiveCall: "
+                    b"Cannot recover from stack overflow",
+                    err)
+
     def test_getwindowsversion(self):
         # Raise SkipTest if sys doesn't have getwindowsversion attribute
         test.support.get_attribute(sys, "getwindowsversion")

Misc/NEWS.d/next/Core and Builtins/2021-04-03-21-12-15.bpo-43710._Mj01v.rst

@@ -0,0 +1,4 @@
+Reverted the fix for https://bugs.python.org/issue42500 as it changed the
+PyThreadState struct size and broke the 3.9.x ABI in the 3.9.3 release
+(visible on 32-bit platforms using binaries compiled using an earlier
+version of Python 3.9.x headers).

Python/ceval.c

@@ -793,22 +793,23 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where)
         _Py_CheckRecursionLimit = recursion_limit;
     }
 #endif
-    if (tstate->recursion_headroom) {
+    if (tstate->recursion_critical)
+        /* Somebody asked that we don't check for recursion. */
+        return 0;
+    if (tstate->overflowed) {
         if (tstate->recursion_depth > recursion_limit + 50) {
             /* Overflowing while handling an overflow. Give up. */
             Py_FatalError("Cannot recover from stack overflow.");
         }
+        return 0;
     }
-    else {
+    if (tstate->recursion_depth > recursion_limit) {
-        if (tstate->recursion_depth > recursion_limit) {
+        --tstate->recursion_depth;
-            tstate->recursion_headroom++;
+        tstate->overflowed = 1;
-            _PyErr_Format(tstate, PyExc_RecursionError,
+        _PyErr_Format(tstate, PyExc_RecursionError,
-                        "maximum recursion depth exceeded%s",
+                      "maximum recursion depth exceeded%s",
-                        where);
+                      where);
-            tstate->recursion_headroom--;
+        return -1;
-            --tstate->recursion_depth;
-            return -1;
-        }
     }
     return 0;
 }

Python/errors.c

@@ -290,14 +290,12 @@ _PyErr_NormalizeException(PyThreadState *tstate, PyObject **exc,
                           PyObject **val, PyObject **tb)
 {
     int recursion_depth = 0;
-    tstate->recursion_headroom++;
     PyObject *type, *value, *initial_tb;
 
   restart:
     type = *exc;
     if (type == NULL) {
         /* There was no exception, so nothing to do. */
-        tstate->recursion_headroom--;
         return;
     }
 
@@ -349,7 +347,6 @@ _PyErr_NormalizeException(PyThreadState *tstate, PyObject **exc,
     }
     *exc = type;
     *val = value;
-    tstate->recursion_headroom--;
     return;
 
   error:

Python/pystate.c

@@ -576,7 +576,7 @@ new_threadstate(PyInterpreterState *interp, int init)
 
     tstate->frame = NULL;
     tstate->recursion_depth = 0;
-    tstate->recursion_headroom = 0;
+    tstate->overflowed = 0;
     tstate->recursion_critical = 0;
     tstate->stackcheck_counter = 0;
     tstate->tracing = 0;

Python/sysmodule.c

@@ -1160,6 +1160,7 @@ static PyObject *
 sys_setrecursionlimit_impl(PyObject *module, int new_limit)
 /*[clinic end generated code: output=35e1c64754800ace input=b0f7a23393924af3]*/
 {
+    int mark;
     PyThreadState *tstate = _PyThreadState_GET();
 
     if (new_limit < 1) {
@@ -1177,7 +1178,8 @@ sys_setrecursionlimit_impl(PyObject *module, int new_limit)
        Reject too low new limit if the current recursion depth is higher than
        the new low-water mark. Otherwise it may not be possible anymore to
        reset the overflowed flag to 0. */
-    if (tstate->recursion_depth >= new_limit) {
+    mark = _Py_RecursionLimitLowerWaterMark(new_limit);
+    if (tstate->recursion_depth >= mark) {
         _PyErr_Format(tstate, PyExc_RecursionError,
                       "cannot set the recursion limit to %i at "
                       "the recursion depth %i: the limit is too low",