mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-04 17:08:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

[3.12] GH-112215: Backport C recursion changes (GH-115083)

Browse source
This commit is contained in:
Mark Shannon 2024-02-13 09:45:59 +00:00 • committed by GitHub
parent a30bb080dc
commit 4d87832d87
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 65 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

22
Include/cpython/pystate.h
View file

@ -251,12 +251,24 @@ struct _ts {
/* WASI has limited call stack. Python's recursion limit depends on code
layout, optimization, and WASI runtime. Wasmtime can handle about 700
recursions, sometimes less. 500 is a more conservative limit. */
#ifndef C_RECURSION_LIMIT
# ifdef __wasi__
# define C_RECURSION_LIMIT 500
#ifdef Py_DEBUG
# if defined(__wasi__)
# define C_RECURSION_LIMIT 150
# else
// This value is duplicated in Lib/test/support/__init__.py
# define C_RECURSION_LIMIT 1500
# define C_RECURSION_LIMIT 500
# endif
#else
# if defined(__wasi__)
# define C_RECURSION_LIMIT 500
# elif defined(__s390x__)
# define C_RECURSION_LIMIT 800
# elif defined(_WIN32)
# define C_RECURSION_LIMIT 3000
# elif defined(_Py_ADDRESS_SANITIZER)
# define C_RECURSION_LIMIT 4000
# else
// This value is duplicated in Lib/test/support/__init__.py
# define C_RECURSION_LIMIT 10000
# endif
#endif

31
Lib/test/support/__init__.py
View file

@ -2112,13 +2112,13 @@ def set_recursion_limit(limit):
finally:
sys.setrecursionlimit(original_limit)
def infinite_recursion(max_depth=100):
"""Set a lower limit for tests that interact with infinite recursions
(e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some
debug windows builds, due to not enough functions being inlined the
stack size might not handle the default recursion limit (1000). See
bpo-11105 for details."""
if max_depth < 3:
def infinite_recursion(max_depth=None):
if max_depth is None:
# Pick a number large enough to cause problems
# but not take too long for code that can handle
# very deep recursion.
max_depth = 20_000
elif max_depth < 3:
raise ValueError("max_depth must be at least 3, got {max_depth}")
depth = get_recursion_depth()
depth = max(depth - 1, 1) # Ignore infinite_recursion() frame.
@ -2362,7 +2362,22 @@ def adjust_int_max_str_digits(max_digits):
EXCEEDS_RECURSION_LIMIT = 5000
# The default C recursion limit (from Include/cpython/pystate.h).
C_RECURSION_LIMIT = 1500
if Py_DEBUG:
if is_wasi:
C_RECURSION_LIMIT = 150
else:
C_RECURSION_LIMIT = 500
else:
if is_wasi:
C_RECURSION_LIMIT = 500
elif hasattr(os, 'uname') and os.uname().machine == 's390x':
C_RECURSION_LIMIT = 800
elif sys.platform.startswith('win'):
C_RECURSION_LIMIT = 3000
elif check_sanitizer(address=True):
C_RECURSION_LIMIT = 4000
else:
C_RECURSION_LIMIT = 10000
#Windows doesn't have os.uname() but it doesn't support s390x.
skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x',

4
Lib/test/test_ast.py
View file

@ -1087,9 +1087,9 @@ class AST_Tests(unittest.TestCase):
@unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
@support.cpython_only
def test_ast_recursion_limit(self):
fail_depth = support.EXCEEDS_RECURSION_LIMIT
fail_depth = support.C_RECURSION_LIMIT + 1
crash_depth = 100_000
success_depth = 1200
success_depth = int(support.C_RECURSION_LIMIT * 0.9)
def check_limit(prefix, repeated):
expect_ok = prefix + repeated * success_depth

3
Lib/test/test_call.py
View file

@ -1,5 +1,5 @@
import unittest
from test.support import cpython_only, requires_limited_api, skip_on_s390x
from test.support import cpython_only, requires_limited_api, skip_on_s390x, is_wasi, Py_DEBUG
try:
import _testcapi
except ImportError:
@ -932,6 +932,7 @@ class TestErrorMessagesUseQualifiedName(unittest.TestCase):
class TestRecursion(unittest.TestCase):
@skip_on_s390x
@unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack")
def test_super_deep(self):
def recurse(n):

4
Lib/test/test_compile.py
View file

@ -607,9 +607,9 @@ class TestSpecifics(unittest.TestCase):
# Expected limit is C_RECURSION_LIMIT * 2
# Duplicating the limit here is a little ugly.
# Perhaps it should be exposed somewhere...
fail_depth = C_RECURSION_LIMIT * 2 + 1
fail_depth = C_RECURSION_LIMIT + 1
crash_depth = C_RECURSION_LIMIT * 100
success_depth = int(C_RECURSION_LIMIT * 1.8)
success_depth = int(C_RECURSION_LIMIT * 0.9)
def check_limit(prefix, repeated, mode="single"):
expect_ok = prefix + repeated * success_depth

2
Lib/test/test_isinstance.py
View file

@ -352,7 +352,7 @@ def blowstack(fxn, arg, compare_to):
# Make sure that calling isinstance with a deeply nested tuple for its
# argument will raise RecursionError eventually.
tuple_arg = (compare_to,)
for cnt in range(support.EXCEEDS_RECURSION_LIMIT):
for cnt in range(support.C_RECURSION_LIMIT * 2):
tuple_arg = (tuple_arg,)
fxn(arg, tuple_arg)

2
Lib/test/test_plistlib.py
View file

@ -908,7 +908,7 @@ class TestBinaryPlistlib(unittest.TestCase):
self.assertIs(b['x'], b)
def test_deep_nesting(self):
tests = [50, 100_000] if support.is_wasi else [50, 300, 100_000]
tests = [50, 100_000] if support.is_wasi else [50, 600, 100_000]
for N in tests:
chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)]
try:

8
Lib/test/test_sys_settrace.py
View file

@ -2965,16 +2965,18 @@ class TestExtendedArgs(unittest.TestCase):
self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1})
def test_trace_lots_of_globals(self):
count = min(1000, int(support.C_RECURSION_LIMIT * 0.8))
code = """if 1:
def f():
return (
{}
)
""".format("\n+\n".join(f"var{i}\n" for i in range(1000)))
ns = {f"var{i}": i for i in range(1000)}
""".format("\n+\n".join(f"var{i}\n" for i in range(count)))
ns = {f"var{i}": i for i in range(count)}
exec(code, ns)
counts = self.count_traces(ns["f"])
self.assertEqual(counts, {'call': 1, 'line': 2000, 'return': 1})
self.assertEqual(counts, {'call': 1, 'line': count * 2, 'return': 1})
class TestEdgeCases(unittest.TestCase):

2
Misc/NEWS.d/next/Core and Builtins/2024-02-04-01-26-20.gh-issue-112215.NyRPXM.rst Normal file
View file

@ -0,0 +1,2 @@
Change the C recursion limits to more closely reflect the underlying
platform limits.

5
Parser/asdl_c.py
View file

@ -1393,15 +1393,14 @@ PyObject* PyAST_mod2obj(mod_ty t)
int starting_recursion_depth;
/* Be careful here to prevent overflow. */
int COMPILER_STACK_FRAME_SCALE = 2;
PyThreadState *tstate = _PyThreadState_GET();
if (!tstate) {
return NULL;
}
struct validator vstate;
vstate.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
vstate.recursion_limit = C_RECURSION_LIMIT;
int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
starting_recursion_depth = recursion_depth;
vstate.recursion_depth = starting_recursion_depth;
PyObject *result = ast2obj_mod(state, &vstate, t);

5
Python/Python-ast.c generated
View file

@ -13152,15 +13152,14 @@ PyObject* PyAST_mod2obj(mod_ty t)
int starting_recursion_depth;
/* Be careful here to prevent overflow. */
int COMPILER_STACK_FRAME_SCALE = 2;
PyThreadState *tstate = _PyThreadState_GET();
if (!tstate) {
return NULL;
}
struct validator vstate;
vstate.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
vstate.recursion_limit = C_RECURSION_LIMIT;
int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
starting_recursion_depth = recursion_depth;
vstate.recursion_depth = starting_recursion_depth;
PyObject *result = ast2obj_mod(state, &vstate, t);

7
Python/ast.c
View file

@ -1038,9 +1038,6 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps)
}
/* See comments in symtable.c. */
#define COMPILER_STACK_FRAME_SCALE 2
int
_PyAST_Validate(mod_ty mod)
{
@ -1057,9 +1054,9 @@ _PyAST_Validate(mod_ty mod)
}
/* Be careful here to prevent overflow. */
int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
starting_recursion_depth = recursion_depth;
state.recursion_depth = starting_recursion_depth;
state.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
state.recursion_limit = C_RECURSION_LIMIT;
switch (mod->kind) {
case Module_kind:

7
Python/ast_opt.c
View file

@ -1102,9 +1102,6 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat
#undef CALL_OPT
#undef CALL_SEQ
/* See comments in symtable.c. */
#define COMPILER_STACK_FRAME_SCALE 2
int
_PyAST_Optimize(mod_ty mod, PyArena *arena, _PyASTOptimizeState *state)
{
@ -1118,9 +1115,9 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena, _PyASTOptimizeState *state)
}
/* Be careful here to prevent overflow. */
int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
starting_recursion_depth = recursion_depth;
state->recursion_depth = starting_recursion_depth;
state->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
state->recursion_limit = C_RECURSION_LIMIT;
int ret = astfold_mod(mod, arena, state);
assert(ret || PyErr_Occurred());

9
Python/symtable.c
View file

@ -281,11 +281,6 @@ symtable_new(void)
return NULL;
}
/* Using a scaling factor means this should automatically adjust when
the recursion limit is adjusted for small or large C stack allocations.
*/
#define COMPILER_STACK_FRAME_SCALE 2
struct symtable *
_PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
{
@ -312,9 +307,9 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
}
/* Be careful here to prevent overflow. */
int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
starting_recursion_depth = recursion_depth;
st->recursion_depth = starting_recursion_depth;
st->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
st->recursion_limit = C_RECURSION_LIMIT;
/* Make the initial symbol information gathering pass */
if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) {