GH-106897: Add RERAISE event to sys.monitoring. (GH-107291)

* Ensures that exception handling events are balanced. Each [re]raise event has a matching unwind/handled event.

Include/cpython/code.h

@@ -10,9 +10,9 @@ extern "C" {
 
 
 /* Count of all "real" monitoring events (not derived from other events) */
-#define _PY_MONITORING_UNGROUPED_EVENTS 14
+#define _PY_MONITORING_UNGROUPED_EVENTS 15
 /* Count of all  monitoring events */
-#define _PY_MONITORING_EVENTS 16
+#define _PY_MONITORING_EVENTS 17
 
 /* Table of which tools are active for each monitored event. */
 typedef struct _Py_Monitors {

Include/internal/pycore_instruments.h

@@ -36,12 +36,13 @@ extern "C" {
 #define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11
 #define PY_MONITORING_EVENT_PY_UNWIND 12
 #define PY_MONITORING_EVENT_PY_THROW 13
+#define PY_MONITORING_EVENT_RERAISE 14
 
 
 /* Ancilliary events */
 
-#define PY_MONITORING_EVENT_C_RETURN 14
+#define PY_MONITORING_EVENT_C_RETURN 15
-#define PY_MONITORING_EVENT_C_RAISE 15
+#define PY_MONITORING_EVENT_C_RAISE 16
 
 
 typedef uint32_t _PyMonitoringEventSet;

Lib/test/test_monitoring.py

@@ -8,7 +8,7 @@ import sys
 import textwrap
 import types
 import unittest
-
+import asyncio
 
 PAIR = (0,1)
 
@@ -243,7 +243,6 @@ class MonitoringEventsBase(MonitoringTestBase):
             expected = func.events
         self.assertEqual(events, expected)
 
-
 class MonitoringEventsTest(MonitoringEventsBase, unittest.TestCase):
 
     def test_just_pass(self):
@@ -632,7 +631,7 @@ class ExceptionRecorder:
 
 class CheckEvents(MonitoringTestBase, unittest.TestCase):
 
-    def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)):
+    def get_events(self, func, tool, recorders):
         try:
             self.assertEqual(sys.monitoring._all_events(), {})
             event_list = []
@@ -646,19 +645,63 @@ class CheckEvents(MonitoringTestBase, unittest.TestCase):
             sys.monitoring.set_events(tool, 0)
             for recorder in recorders:
                 sys.monitoring.register_callback(tool, recorder.event_type, None)
-            self.assertEqual(event_list, expected)
+            return event_list
         finally:
             sys.monitoring.set_events(tool, 0)
             for recorder in recorders:
                 sys.monitoring.register_callback(tool, recorder.event_type, None)
 
+    def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)):
+        events = self.get_events(func, tool, recorders)
+        if events != expected:
+            print(events, file = sys.stderr)
+        self.assertEqual(events, expected)
+
+    def check_balanced(self, func, recorders):
+        events = self.get_events(func, TEST_TOOL, recorders)
+        self.assertEqual(len(events)%2, 0)
+        for r, h in zip(events[::2],events[1::2]):
+            r0 = r[0]
+            self.assertIn(r0, ("raise", "reraise"))
+            h0 = h[0]
+            self.assertIn(h0, ("handled", "unwind"))
+
+
+
 class StopiterationRecorder(ExceptionRecorder):
 
     event_type = E.STOP_ITERATION
 
-class ExceptionMontoringTest(CheckEvents):
+class ReraiseRecorder(ExceptionRecorder):
 
-    recorder = ExceptionRecorder
+    event_type = E.RERAISE
+
+    def __call__(self, code, offset, exc):
+        self.events.append(("reraise", type(exc)))
+
+class UnwindRecorder(ExceptionRecorder):
+
+    event_type = E.PY_UNWIND
+
+    def __call__(self, *args):
+        self.events.append(("unwind", None))
+
+class ExceptionHandledRecorder(ExceptionRecorder):
+
+    event_type = E.EXCEPTION_HANDLED
+
+    def __call__(self, code, offset, exc):
+        self.events.append(("handled", type(exc)))
+
+class ExceptionMonitoringTest(CheckEvents):
+
+
+    exception_recorders = (
+        ExceptionRecorder,
+        ReraiseRecorder,
+        ExceptionHandledRecorder,
+        UnwindRecorder
+    )
 
     def test_simple_try_except(self):
 
@@ -672,6 +715,8 @@ class ExceptionMontoringTest(CheckEvents):
 
         self.check_events(func1, [("raise", KeyError)])
 
+    def test_implicit_stop_iteration(self):
+
         def gen():
             yield 1
             return 2
@@ -682,6 +727,117 @@ class ExceptionMontoringTest(CheckEvents):
 
         self.check_events(implicit_stop_iteration, [("raise", StopIteration)], recorders=(StopiterationRecorder,))
 
+    initial = [
+        ("raise", ZeroDivisionError),
+        ("handled", ZeroDivisionError)
+    ]
+
+    reraise = [
+        ("reraise", ZeroDivisionError),
+        ("handled", ZeroDivisionError)
+    ]
+
+    def test_explicit_reraise(self):
+
+        def func():
+            try:
+                try:
+                    1/0
+                except:
+                    raise
+            except:
+                pass
+
+        self.check_balanced(
+            func,
+            recorders = self.exception_recorders)
+
+    def test_explicit_reraise_named(self):
+
+        def func():
+            try:
+                try:
+                    1/0
+                except Exception as ex:
+                    raise
+            except:
+                pass
+
+        self.check_balanced(
+            func,
+            recorders = self.exception_recorders)
+
+    def test_implicit_reraise(self):
+
+        def func():
+            try:
+                try:
+                    1/0
+                except ValueError:
+                    pass
+            except:
+                pass
+
+        self.check_balanced(
+            func,
+            recorders = self.exception_recorders)
+
+
+    def test_implicit_reraise_named(self):
+
+        def func():
+            try:
+                try:
+                    1/0
+                except ValueError as ex:
+                    pass
+            except:
+                pass
+
+        self.check_balanced(
+            func,
+            recorders = self.exception_recorders)
+
+    def test_try_finally(self):
+
+        def func():
+            try:
+                try:
+                    1/0
+                finally:
+                    pass
+            except:
+                pass
+
+        self.check_balanced(
+            func,
+            recorders = self.exception_recorders)
+
+    def test_async_for(self):
+
+        def func():
+
+            async def async_generator():
+                for i in range(1):
+                    raise ZeroDivisionError
+                    yield i
+
+            async def async_loop():
+                try:
+                    async for item in async_generator():
+                        pass
+                except Exception:
+                    pass
+
+            try:
+                async_loop().send(None)
+            except StopIteration:
+                pass
+
+        self.check_balanced(
+            func,
+            recorders = self.exception_recorders)
+
 class LineRecorder:
 
     event_type = E.LINE
@@ -733,12 +889,12 @@ class TestManyEvents(CheckEvents):
             line3 = 3
 
         self.check_events(func1, recorders = MANY_RECORDERS, expected = [
-            ('line', 'check_events', 10),
+            ('line', 'get_events', 10),
             ('call', 'func1', sys.monitoring.MISSING),
             ('line', 'func1', 1),
             ('line', 'func1', 2),
             ('line', 'func1', 3),
-            ('line', 'check_events', 11),
+            ('line', 'get_events', 11),
             ('call', 'set_events', 2)])
 
     def test_c_call(self):
@@ -749,14 +905,14 @@ class TestManyEvents(CheckEvents):
             line3 = 3
 
         self.check_events(func2, recorders = MANY_RECORDERS, expected = [
-            ('line', 'check_events', 10),
+            ('line', 'get_events', 10),
             ('call', 'func2', sys.monitoring.MISSING),
             ('line', 'func2', 1),
             ('line', 'func2', 2),
             ('call', 'append', [2]),
             ('C return', 'append', [2]),
             ('line', 'func2', 3),
-            ('line', 'check_events', 11),
+            ('line', 'get_events', 11),
             ('call', 'set_events', 2)])
 
     def test_try_except(self):
@@ -770,7 +926,7 @@ class TestManyEvents(CheckEvents):
             line = 6
 
         self.check_events(func3, recorders = MANY_RECORDERS, expected = [
-            ('line', 'check_events', 10),
+            ('line', 'get_events', 10),
             ('call', 'func3', sys.monitoring.MISSING),
             ('line', 'func3', 1),
             ('line', 'func3', 2),
@@ -779,7 +935,7 @@ class TestManyEvents(CheckEvents):
             ('line', 'func3', 4),
             ('line', 'func3', 5),
             ('line', 'func3', 6),
-            ('line', 'check_events', 11),
+            ('line', 'get_events', 11),
             ('call', 'set_events', 2)])
 
 class InstructionRecorder:
@@ -791,7 +947,7 @@ class InstructionRecorder:
 
     def __call__(self, code, offset):
         # Filter out instructions in check_events to lower noise
-        if code.co_name != "check_events":
+        if code.co_name != "get_events":
             self.events.append(("instruction", code.co_name, offset))
 
 
@@ -808,7 +964,7 @@ class TestLineAndInstructionEvents(CheckEvents):
             line3 = 3
 
         self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [
-            ('line', 'check_events', 10),
+            ('line', 'get_events', 10),
             ('line', 'func1', 1),
             ('instruction', 'func1', 2),
             ('instruction', 'func1', 4),
@@ -819,7 +975,7 @@ class TestLineAndInstructionEvents(CheckEvents):
             ('instruction', 'func1', 10),
             ('instruction', 'func1', 12),
             ('instruction', 'func1', 14),
-            ('line', 'check_events', 11)])
+            ('line', 'get_events', 11)])
 
     def test_c_call(self):
 
@@ -829,7 +985,7 @@ class TestLineAndInstructionEvents(CheckEvents):
             line3 = 3
 
         self.check_events(func2, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [
-            ('line', 'check_events', 10),
+            ('line', 'get_events', 10),
             ('line', 'func2', 1),
             ('instruction', 'func2', 2),
             ('instruction', 'func2', 4),
@@ -843,7 +999,7 @@ class TestLineAndInstructionEvents(CheckEvents):
             ('instruction', 'func2', 40),
             ('instruction', 'func2', 42),
             ('instruction', 'func2', 44),
-            ('line', 'check_events', 11)])
+            ('line', 'get_events', 11)])
 
     def test_try_except(self):
 
@@ -856,7 +1012,7 @@ class TestLineAndInstructionEvents(CheckEvents):
             line = 6
 
         self.check_events(func3, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [
-            ('line', 'check_events', 10),
+            ('line', 'get_events', 10),
             ('line', 'func3', 1),
             ('instruction', 'func3', 2),
             ('line', 'func3', 2),
@@ -876,7 +1032,7 @@ class TestLineAndInstructionEvents(CheckEvents):
             ('instruction', 'func3', 30),
             ('instruction', 'func3', 32),
             ('instruction', 'func3', 34),
-            ('line', 'check_events', 11)])
+            ('line', 'get_events', 11)])
 
     def test_with_restart(self):
         def func1():
@@ -885,7 +1041,7 @@ class TestLineAndInstructionEvents(CheckEvents):
             line3 = 3
 
         self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [
-            ('line', 'check_events', 10),
+            ('line', 'get_events', 10),
             ('line', 'func1', 1),
             ('instruction', 'func1', 2),
             ('instruction', 'func1', 4),
@@ -896,12 +1052,12 @@ class TestLineAndInstructionEvents(CheckEvents):
             ('instruction', 'func1', 10),
             ('instruction', 'func1', 12),
             ('instruction', 'func1', 14),
-            ('line', 'check_events', 11)])
+            ('line', 'get_events', 11)])
 
         sys.monitoring.restart_events()
 
         self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [
-            ('line', 'check_events', 10),
+            ('line', 'get_events', 10),
             ('line', 'func1', 1),
             ('instruction', 'func1', 2),
             ('instruction', 'func1', 4),
@@ -912,7 +1068,7 @@ class TestLineAndInstructionEvents(CheckEvents):
             ('instruction', 'func1', 10),
             ('instruction', 'func1', 12),
             ('instruction', 'func1', 14),
-            ('line', 'check_events', 11)])
+            ('line', 'get_events', 11)])
 
 class TestInstallIncrementallly(MonitoringTestBase, unittest.TestCase):
 
@@ -1114,7 +1270,7 @@ class TestBranchAndJumpEvents(CheckEvents):
             ('branch', 'func', 2, 2)])
 
         self.check_events(func, recorders = JUMP_BRANCH_AND_LINE_RECORDERS, expected = [
-            ('line', 'check_events', 10),
+            ('line', 'get_events', 10),
             ('line', 'func', 1),
             ('line', 'func', 2),
             ('branch', 'func', 2, 2),
@@ -1130,7 +1286,7 @@ class TestBranchAndJumpEvents(CheckEvents):
             ('jump', 'func', 4, 2),
             ('line', 'func', 2),
             ('branch', 'func', 2, 2),
-            ('line', 'check_events', 11)])
+            ('line', 'get_events', 11)])
 
     def test_except_star(self):
 
@@ -1149,7 +1305,7 @@ class TestBranchAndJumpEvents(CheckEvents):
 
 
         self.check_events(func, recorders = JUMP_BRANCH_AND_LINE_RECORDERS, expected = [
-            ('line', 'check_events', 10),
+            ('line', 'get_events', 10),
             ('line', 'func', 1),
             ('line', 'func', 2),
             ('line', 'func', 3),
@@ -1160,10 +1316,10 @@ class TestBranchAndJumpEvents(CheckEvents):
             ('jump', 'func', 5, 5),
             ('jump', 'func', 5, '[offset=112]'),
             ('branch', 'func', '[offset=118]', '[offset=120]'),
-            ('line', 'check_events', 11)])
+            ('line', 'get_events', 11)])
 
         self.check_events(func, recorders = FLOW_AND_LINE_RECORDERS, expected = [
-            ('line', 'check_events', 10),
+            ('line', 'get_events', 10),
             ('line', 'func', 1),
             ('line', 'func', 2),
             ('line', 'func', 3),
@@ -1177,7 +1333,7 @@ class TestBranchAndJumpEvents(CheckEvents):
             ('jump', 'func', 5, '[offset=112]'),
             ('branch', 'func', '[offset=118]', '[offset=120]'),
             ('return', None),
-            ('line', 'check_events', 11)])
+            ('line', 'get_events', 11)])
 
 class TestLoadSuperAttr(CheckEvents):
     RECORDERS = CallRecorder, LineRecorder, CRaiseRecorder, CReturnRecorder
@@ -1229,7 +1385,7 @@ class TestLoadSuperAttr(CheckEvents):
         """
         d = self._exec_super(codestr, optimized)
         expected = [
-            ('line', 'check_events', 10),
+            ('line', 'get_events', 10),
             ('call', 'f', sys.monitoring.MISSING),
             ('line', 'f', 1),
             ('call', 'method', d["b"]),
@@ -1242,7 +1398,7 @@ class TestLoadSuperAttr(CheckEvents):
             ('call', 'method', 1),
             ('line', 'method', 1),
             ('line', 'method', 1),
-            ('line', 'check_events', 11),
+            ('line', 'get_events', 11),
             ('call', 'set_events', 2),
         ]
         return d["f"], expected
@@ -1280,7 +1436,7 @@ class TestLoadSuperAttr(CheckEvents):
         """
         d = self._exec_super(codestr, optimized)
         expected = [
-            ('line', 'check_events', 10),
+            ('line', 'get_events', 10),
             ('call', 'f', sys.monitoring.MISSING),
             ('line', 'f', 1),
             ('line', 'f', 2),
@@ -1293,7 +1449,7 @@ class TestLoadSuperAttr(CheckEvents):
             ('C raise', 'super', 1),
             ('line', 'f', 3),
             ('line', 'f', 4),
-            ('line', 'check_events', 11),
+            ('line', 'get_events', 11),
             ('call', 'set_events', 2),
         ]
         return d["f"], expected
@@ -1321,7 +1477,7 @@ class TestLoadSuperAttr(CheckEvents):
         """
         d = self._exec_super(codestr, optimized)
         expected = [
-            ('line', 'check_events', 10),
+            ('line', 'get_events', 10),
             ('call', 'f', sys.monitoring.MISSING),
             ('line', 'f', 1),
             ('call', 'method', d["b"]),
@@ -1330,7 +1486,7 @@ class TestLoadSuperAttr(CheckEvents):
             ('C return', 'super', sys.monitoring.MISSING),
             ('line', 'method', 2),
             ('line', 'method', 1),
-            ('line', 'check_events', 11),
+            ('line', 'get_events', 11),
             ('call', 'set_events', 2)
         ]
         return d["f"], expected
@@ -1355,7 +1511,7 @@ class TestLoadSuperAttr(CheckEvents):
         def get_expected(name, call_method, ns):
             repr_arg = 0 if name == "int" else sys.monitoring.MISSING
             return [
-                ('line', 'check_events', 10),
+                ('line', 'get_events', 10),
                 ('call', 'f', sys.monitoring.MISSING),
                 ('line', 'f', 1),
                 ('call', 'method', ns["c"]),
@@ -1368,7 +1524,7 @@ class TestLoadSuperAttr(CheckEvents):
                         ('C return', '__repr__', repr_arg),
                     ] if call_method else []
                 ),
-                ('line', 'check_events', 11),
+                ('line', 'get_events', 11),
                 ('call', 'set_events', 2),
             ]
 

Misc/NEWS.d/next/Core and Builtins/2023-07-26-12-18-10.gh-issue-106897.EsGurc.rst

@@ -0,0 +1,3 @@
+Add a ``RERAISE`` event to ``sys.monitoring``, which occurs when an
+exception is reraise, either explicitly by a plain ``raise`` statement, or
+implicitly in an ``except`` or ``finally`` block.

Python/bytecodes.c

@@ -720,7 +720,11 @@ dummy_func(
                 exc = args[0];
                 /* fall through */
             case 0:
-                ERROR_IF(do_raise(tstate, exc, cause), exception_unwind);
+                if (do_raise(tstate, exc, cause)) {
+                    assert(oparg == 0);
+                    monitor_reraise(tstate, frame, next_instr-1);
+                    goto exception_unwind;
+                }
                 break;
             default:
                 _PyErr_SetString(tstate, PyExc_SystemError,
@@ -1047,6 +1051,7 @@ dummy_func(
             assert(exc && PyExceptionInstance_Check(exc));
             Py_INCREF(exc);
             _PyErr_SetRaisedException(tstate, exc);
+            monitor_reraise(tstate, frame, next_instr-1);
             goto exception_unwind;
         }
 
@@ -1058,6 +1063,7 @@ dummy_func(
             else {
                 Py_INCREF(exc);
                 _PyErr_SetRaisedException(tstate, exc);
+                monitor_reraise(tstate, frame, next_instr-1);
                 goto exception_unwind;
             }
         }
@@ -1072,6 +1078,7 @@ dummy_func(
             }
             else {
                 _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value));
+                monitor_reraise(tstate, frame, next_instr-1);
                 goto exception_unwind;
             }
         }

Python/ceval.c

@@ -184,6 +184,9 @@ lltrace_resume_frame(_PyInterpreterFrame *frame)
 static void monitor_raise(PyThreadState *tstate,
                  _PyInterpreterFrame *frame,
                  _Py_CODEUNIT *instr);
+static void monitor_reraise(PyThreadState *tstate,
+                 _PyInterpreterFrame *frame,
+                 _Py_CODEUNIT *instr);
 static int monitor_stop_iteration(PyThreadState *tstate,
                  _PyInterpreterFrame *frame,
                  _Py_CODEUNIT *instr);
@@ -840,7 +843,6 @@ error:
             }
         }
         monitor_raise(tstate, frame, next_instr-1);
-
 exception_unwind:
         {
             /* We can't use frame->f_lasti here, as RERAISE may have set it */
@@ -1954,6 +1956,16 @@ monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame,
     do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RAISE);
 }
 
+static void
+monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame,
+              _Py_CODEUNIT *instr)
+{
+    if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_RERAISE)) {
+        return;
+    }
+    do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE);
+}
+
 static int
 monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame,
                        _Py_CODEUNIT *instr)

Python/generated_cases.c.h

@@ -881,7 +881,11 @@
                 exc = args[0];
                 /* fall through */
             case 0:
-                if (do_raise(tstate, exc, cause)) { STACK_SHRINK(oparg); goto exception_unwind; }
+                if (do_raise(tstate, exc, cause)) {
+                    assert(oparg == 0);
+                    monitor_reraise(tstate, frame, next_instr-1);
+                    goto exception_unwind;
+                }
                 break;
             default:
                 _PyErr_SetString(tstate, PyExc_SystemError,
@@ -1237,6 +1241,7 @@
             assert(exc && PyExceptionInstance_Check(exc));
             Py_INCREF(exc);
             _PyErr_SetRaisedException(tstate, exc);
+            monitor_reraise(tstate, frame, next_instr-1);
             goto exception_unwind;
         }
 
@@ -1251,6 +1256,7 @@
             else {
                 Py_INCREF(exc);
                 _PyErr_SetRaisedException(tstate, exc);
+                monitor_reraise(tstate, frame, next_instr-1);
                 goto exception_unwind;
             }
             STACK_SHRINK(2);
@@ -1274,6 +1280,7 @@
             }
             else {
                 _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value));
+                monitor_reraise(tstate, frame, next_instr-1);
                 goto exception_unwind;
             }
             STACK_SHRINK(1);

Python/instrumentation.c

@@ -2030,6 +2030,7 @@ static const char *const event_names [] = {
     [PY_MONITORING_EVENT_C_RETURN] = "C_RETURN",
     [PY_MONITORING_EVENT_PY_THROW] = "PY_THROW",
     [PY_MONITORING_EVENT_RAISE] = "RAISE",
+    [PY_MONITORING_EVENT_RERAISE] = "RERAISE",
     [PY_MONITORING_EVENT_EXCEPTION_HANDLED] = "EXCEPTION_HANDLED",
     [PY_MONITORING_EVENT_C_RAISE] = "C_RAISE",
     [PY_MONITORING_EVENT_PY_UNWIND] = "PY_UNWIND",