gh-137838: Fix JIT trace buffer overrun by increasing possible exit stubs (gh-138177)

Include/internal/pycore_optimizer.h

@@ -119,7 +119,7 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp);
 #define JIT_CLEANUP_THRESHOLD 100000
 
 // This is the length of the trace we project initially.
-#define UOP_MAX_TRACE_LENGTH 800
+#define UOP_MAX_TRACE_LENGTH 1200
 
 #define TRACE_STACK_SIZE 5
 

Lib/test/test_sys_settrace.py

@@ -360,6 +360,8 @@ class TraceTestCase(unittest.TestCase):
     # Disable gc collection when tracing, otherwise the
     # deallocators may be traced as well.
     def setUp(self):
+        if os.environ.get('PYTHON_UOPS_OPTIMIZE') == '0':
+            self.skipTest("Line tracing behavior differs when JIT optimizer is disabled")
         self.using_gc = gc.isenabled()
         gc.disable()
         self.addCleanup(sys.settrace, sys.gettrace())

Lib/test/test_trace.py

@@ -142,6 +142,8 @@ class TestLineCounts(unittest.TestCase):
 
         self.assertEqual(self.tracer.results().counts, expected)
 
+    @unittest.skipIf(os.environ.get('PYTHON_UOPS_OPTIMIZE') == '0',
+                     "Line counts differ when JIT optimizer is disabled")
     def test_traced_func_loop(self):
         self.tracer.runfunc(traced_func_loop, 2, 3)
 
@@ -166,6 +168,8 @@ class TestLineCounts(unittest.TestCase):
 
         self.assertEqual(self.tracer.results().counts, expected)
 
+    @unittest.skipIf(os.environ.get('PYTHON_UOPS_OPTIMIZE') == '0',
+                     "Line counts differ when JIT optimizer is disabled")
     def test_trace_func_generator(self):
         self.tracer.runfunc(traced_func_calling_generator)
 
@@ -236,6 +240,8 @@ class TestRunExecCounts(unittest.TestCase):
         self.my_py_filename = fix_ext_py(__file__)
         self.addCleanup(sys.settrace, sys.gettrace())
 
+    @unittest.skipIf(os.environ.get('PYTHON_UOPS_OPTIMIZE') == '0',
+                     "Line counts differ when JIT optimizer is disabled")
     def test_exec_counts(self):
         self.tracer = Trace(count=1, trace=0, countfuncs=0, countcallers=0)
         code = r'''traced_func_loop(2, 5)'''

Misc/NEWS.d/next/Core_and_Builtins/2025-08-27-17-51-38.gh-issue-137838.lK6T0j.rst

@@ -0,0 +1,2 @@
+Fix JIT trace buffer overrun by increasing possible exit stubs.
+Patch by Donghee Na.

Python/optimizer.c

@@ -591,9 +591,8 @@ translate_bytecode_to_trace(
 
     for (;;) {
         target = INSTR_IP(instr, code);
-        // Need space for _DEOPT
-        max_length--;
-
+        // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT
+        max_length-=2;
         uint32_t opcode = instr->op.code;
         uint32_t oparg = instr->op.arg;
 
@@ -1283,6 +1282,11 @@ uop_optimize(
     _Py_BloomFilter_Init(&dependencies);
     _PyUOpInstruction buffer[UOP_MAX_TRACE_LENGTH];
     OPT_STAT_INC(attempts);
+    char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE");
+    bool is_noopt = true;
+    if (env_var == NULL || *env_var == '\0' || *env_var > '0') {
+        is_noopt = false;
+    }
     int length = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies, progress_needed);
     if (length <= 0) {
         // Error or nothing translated
@@ -1290,8 +1294,7 @@ uop_optimize(
     }
     assert(length < UOP_MAX_TRACE_LENGTH);
     OPT_STAT_INC(traces_created);
-    char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE");
-    if (env_var == NULL || *env_var == '\0' || *env_var > '0') {
+    if (!is_noopt) {
         length = _Py_uop_analyze_and_optimize(frame, buffer,
                                            length,
                                            curr_stackentries, &dependencies);